From 81983ad515e101e10970423f88c1819df6152a17 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 21 Aug 2026 22:28:21 -0400 Subject: [PATCH 1/4] feat: add read-only Beeper messaging source --- CHANGELOG.md | 11 + README.md | 71 +- docs/plugins.md | 7 + package.json | 12 +- .../generate-beeper-message-like-me-golden.ts | 23 + scripts/package-smoke.ts | 47 +- src/args.test.ts | 43 + src/args.ts | 99 + .../adapters/beeper/wrench-web-adapter.json | 129 ++ src/beeper-local-plugin.test.ts | 29 + src/beeper-message-like-me-cli.ts | 40 + src/beeper-message-like-me-export.test.ts | 693 +++++++ src/beeper-message-like-me-export.ts | 1603 +++++++++++++++++ src/beeper-message-like-me-golden-fixture.ts | 169 ++ src/beeper-message-like-me-source.test.ts | 690 +++++++ src/beeper-message-like-me-source.ts | 1567 ++++++++++++++++ .../beeper-message-like-me-v1/accounts.ndjson | 1 + .../conversations.ndjson | 1 + .../beeper-message-like-me-v1/manifest.json | 1 + .../beeper-message-like-me-v1/messages.ndjson | 2 + .../participants.ndjson | 2 + .../reactions.ndjson | 1 + .../tombstones.ndjson | 1 + src/media/manifest.test.ts | 2 +- src/media/manifest.ts | 2 +- src/plugins/beeper-linked-device/plugin.ts | 77 + src/provider-contract-inventory.test.ts | 7 +- src/provider-plugin-contract-identity.ts | 7 + src/provider-plugin-omni.test.ts | 3 + src/provider-plugin-registry.test.ts | 1 + src/provider-plugin-registry.ts | 1 + src/provider-plugins.generated.ts | 28 +- .../beeper-local-runtime.internal.test.ts | 499 +++++ src/providers/beeper-local-runtime.ts | 1498 +++++++++++++++ src/providers/beeper-local.ts | 366 ++++ src/providers/beeper-omni.ts | 560 ++++++ src/scripts/sync-bundled-adapters.test.ts | 21 +- src/usage.ts | 4 + src/web-session-contract-definitions.ts | 14 + src/wrench.test.ts | 116 +- src/wrench.ts | 72 +- website/build.ts | 4 +- website/source/provider-capabilities.html | 19 +- 43 files changed, 8501 insertions(+), 42 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 scripts/generate-beeper-message-like-me-golden.ts create mode 100644 src/assets/adapters/beeper/wrench-web-adapter.json create mode 100644 src/beeper-local-plugin.test.ts create mode 100644 src/beeper-message-like-me-cli.ts create mode 100644 src/beeper-message-like-me-export.test.ts create mode 100644 src/beeper-message-like-me-export.ts create mode 100644 src/beeper-message-like-me-golden-fixture.ts create mode 100644 src/beeper-message-like-me-source.test.ts create mode 100644 src/beeper-message-like-me-source.ts create mode 100644 src/fixtures/beeper-message-like-me-v1/accounts.ndjson create mode 100644 src/fixtures/beeper-message-like-me-v1/conversations.ndjson create mode 100644 src/fixtures/beeper-message-like-me-v1/manifest.json create mode 100644 src/fixtures/beeper-message-like-me-v1/messages.ndjson create mode 100644 src/fixtures/beeper-message-like-me-v1/participants.ndjson create mode 100644 src/fixtures/beeper-message-like-me-v1/reactions.ndjson create mode 100644 src/fixtures/beeper-message-like-me-v1/tombstones.ndjson create mode 100644 src/plugins/beeper-linked-device/plugin.ts create mode 100644 src/providers/beeper-local-runtime.internal.test.ts create mode 100644 src/providers/beeper-local-runtime.ts create mode 100644 src/providers/beeper-local.ts create mode 100644 src/providers/beeper-omni.ts diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..c38e3bc --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog + +## 0.11.0 - 2026-08-21 + +- Added a pinned, read-only Beeper Desktop provider for contacts and messaging + projections across locally connected accounts. +- Added `wrench beeper export-message-like-me` for private, provenance-preserving + Message Like Me bundles with canonical digests, explicit completeness, and no + media downloads. +- Added strict local-runtime validation, bounded full-export conversion, graph + conformance checks, and a canonical cross-repository bundle fixture. diff --git a/README.md b/README.md index 17685a7..91552ef 100644 --- a/README.md +++ b/README.md @@ -61,10 +61,10 @@ owns the narrow capability boundary that can sit beneath them. ## Install -Pin the public repository to the immutable `v0.10.1` tag: +Pin the public repository to the immutable `v0.11.0` tag: ```sh -bun add --global github:hraness/wrench#v0.10.1 +bun add --global github:hraness/wrench#v0.11.0 wrench adapter sync-bundled --json wrench doctor ``` @@ -88,7 +88,7 @@ Install Wrench in an agent or application that owns its own model, planning, tool loop, approvals, and interface: ```sh -bun add github:hraness/wrench#v0.10.1 +bun add github:hraness/wrench#v0.11.0 ``` ```ts @@ -268,6 +268,7 @@ not turn missing message history into zero activity. | Provider | Contact collection | Directional statistics | | --- | --- | --- | | Gmail | Google People connections | Bounded Gmail message scans with explicit truncation | +| Beeper local Desktop | One bounded account-aware page from the already-authorized local Desktop projection | Unavailable; Wrench does not scan message history while listing contacts | | LinkedIn official API | First-degree connections with locale-selection evidence | Unavailable; the Connections API does not expose ordinary inbox history | | Instagram authenticated web | Unique non-viewer participants from the reviewed first Direct inbox summary page, with explicit first-page and pagination incompleteness | Unavailable until acknowledgement-free message-history paging is reviewed | | WhatsApp linked device | One page of the authenticated account owner's private, quiescent Whatsmeow contact store | Unavailable; Wrench does not treat a linked-device message cache as account-owned history | @@ -305,6 +306,68 @@ Wrench will not install or expose this surface until it can bind the TDLib authorization lifecycle, account identity, local database, paging behavior, and message-history completeness without weakening the linked-device boundary. +### Beeper local read-only projection + +The bundled `beeper-linked-device` source plugin reads an existing Beeper +Desktop authorization through the official Beeper CLI 0.6.2. Wrench accepts +only the exact pinned macOS arm64 binary, the fixed selected `desktop` target, +and three JSON operations: `contacts.list`, `messaging.list`, and +`messaging.read`. Every child command uses Beeper's read-only mode. The plugin +does not expose raw API calls, targets, media downloads, sends, presence, +pairing, sync, or other CLI commands. + +Install the official CLI and authorize it to the local Desktop app first: + +```sh +brew install beeper/tap/cli +beeper setup +wrench adapter sync-bundled --json +wrench auth add beeper-main --linked-device beeper \ + --device-store "${HOME}/.beeper" +wrench auth bind beeper-main --site beeper +``` + +Binding hashes the stable local self-account coordinate before storing or +printing it. The first bind or read may take longer while the pinned CLI unpacks +its embedded payload into an operation-private cache. Read the local account +and conversation identifiers, then request one exact conversation page: + +```sh +wrench beeper-local messaging.list --auth beeper-main \ + --input '{"limit":100}' --json +wrench beeper-local messaging.read --auth beeper-main \ + --input '{"account_id":"","conversation_id":"","limit":100}' --json +``` + +Create a private, agent-ready Message Like Me bundle from every connected +account materialized by Beeper Desktop: + +```sh +wrench beeper export-message-like-me --auth beeper-main \ + --output /absolute/path/to/new-message-like-me-bundle --json +``` + +The command uses the pinned official full export with `--no-attachments`, +validates its complete local chat inventory, removes the duplicate Markdown and +HTML renderings inside operation-private staging, and publishes `manifest.json` +last. The output directory is mode 0700; its six NDJSON artifacts and manifest +are mode 0600 and carry canonical SHA-256 digests. The JSON result reports the +manifest path and digest, record counts, completeness, and warnings. Optional +`--limit-chats`, `--limit-messages`, and `--max-participants` values are recorded +as truncation when reached. Wrench also emits a coherent truncated bundle before +the 500,000-record or 512 MiB bundle ceiling. One chat JSON file is limited to +64 MiB so foreign input cannot force a multi-gigabyte allocation; an oversized +chat is omitted with explicit truncated completeness and a warning. + +Contact and chat lists are bounded to 200 records because CLI 0.6.2 exposes no +continuation cursor for those commands. Message pages derive the next +before/after cursor only from the terminal returned message ID and reject +duplicates or a non-advancing cursor at normalization. Output marks remote +history coverage unknown, preserves account/network/reply/edit/delete and +reaction provenance, and includes attachment metadata without media IDs, +paths, URLs, or downloads. This is a local materialized view, not a claim that +every connected network has finished backfilling its remote history. + ### Gmail Gmail uses the official Gmail and People APIs. Download one Google OAuth @@ -626,7 +689,7 @@ does not expose a shell, package manager, ambient environment, unrestricted filesystem, redirect, retry, or arbitrary request primitive. Read [the plugin guide](docs/plugins.md) before replacing an inert reservation -with an observed contract. The packaged [Wrench Agent Skill](https://github.com/hraness/wrench/blob/v0.10.1/skills/wrench/SKILL.md) +with an observed contract. The packaged [Wrench Agent Skill](https://github.com/hraness/wrench/blob/v0.11.0/skills/wrench/SKILL.md) gives coding agents the same workflow and safety boundary. The packaged [cross-post skill](skills/cross-post-with-wrench/SKILL.md) orchestrates exact, previewed posts across X, LinkedIn, Bluesky, Substack Notes, and Threads while diff --git a/docs/plugins.md b/docs/plugins.md index 7ff4305..f48f989 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -14,6 +14,13 @@ recovery, receipts, bounds, redaction, activation, and lifecycle serialization. A plugin owns exact provider identity, route and operation descriptors, request and response contracts, account probes, execution, and reconciliation logic. +A linked-device binding may either own an explicit `inspect`/`pair`/`syncOnce` +lifecycle or attach read-only to an independently managed local source. The +latter must omit the lifecycle declaration and all mutating surfaces. Its auth +locator is established with `wrench auth add ... --linked-device ... +--device-store ...`, then account-bound with `wrench auth bind`; Wrench must not +suggest pairing or syncing a lifecycle the plugin does not declare. + ## Start inert Create a portable package with one `capture-required` reservation: diff --git a/package.json b/package.json index 3032892..5c38990 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hraness/wrench", - "version": "0.10.1", + "version": "0.11.0", "description": "Open-source CLI and TypeScript SDK for precise web capabilities for AI agents: page capture, verified media archives, encrypted reads, and typed provider operations.", "license": "MIT", "type": "module", @@ -64,6 +64,7 @@ "src/article-draft-embeds.ts", "src/article-draft-images.ts", "src/assets/adapter-template/wrench-adapter.json", + "src/assets/adapters/beeper/wrench-web-adapter.json", "src/assets/adapters/bluesky/wrench-web-adapter.json", "src/assets/adapters/bluesky/wrench-web-adapter.v1.0.0.json", "src/assets/adapters/bluesky/wrench-web-adapter.v1.1.0.json", @@ -134,6 +135,10 @@ "src/assets/code-owned-provider-template/runtime.ts.template", "src/assets/code-owned-provider-template/wrench-adapter.json.template", "src/auth.ts", + "src/beeper-message-like-me-cli.ts", + "src/beeper-message-like-me-export.ts", + "src/beeper-message-like-me-source.ts", + "src/fixtures/beeper-message-like-me-v1", "src/browser-admission.ts", "src/browser-snapshots.ts", "src/browser.ts", @@ -190,6 +195,7 @@ "src/pinned-https.ts", "src/plan-assets.ts", "src/platform-catalog.ts", + "src/plugins/beeper-linked-device/plugin.ts", "src/plugins/bluesky-web/plugin.ts", "src/plugins/gmail-official/plugin.ts", "src/plugins/hacker-news-web/plugin.ts", @@ -243,6 +249,9 @@ "src/read-projection-admission.ts", "src/read-client.ts", "src/read-projections.ts", + "src/providers/beeper-local-runtime.ts", + "src/providers/beeper-local.ts", + "src/providers/beeper-omni.ts", "src/providers/bluesky-web-runtime.ts", "src/providers/bluesky-web.ts", "src/providers/contact-projection.ts", @@ -317,6 +326,7 @@ "src/wrench.ts", "skills", "README.md", + "CHANGELOG.md", "LICENSE" ], "scripts": { diff --git a/scripts/generate-beeper-message-like-me-golden.ts b/scripts/generate-beeper-message-like-me-golden.ts new file mode 100644 index 0000000..986643c --- /dev/null +++ b/scripts/generate-beeper-message-like-me-golden.ts @@ -0,0 +1,23 @@ +import { mkdir } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { exportBeeperMessageLikeMeBundle } from "../src/beeper-message-like-me-export"; +import { + BEEPER_MESSAGE_LIKE_ME_GOLDEN_FINISHED_AT, + BEEPER_MESSAGE_LIKE_ME_GOLDEN_STARTED_AT, + createBeeperMessageLikeMeGoldenSource, +} from "../src/beeper-message-like-me-golden-fixture"; + +const parent = resolve(import.meta.dir, "..", "src", "fixtures"); +const outputRoot = resolve(parent, "beeper-message-like-me-v1"); +await mkdir(parent, { recursive: true, mode: 0o755 }); +const instants = [ + BEEPER_MESSAGE_LIKE_ME_GOLDEN_STARTED_AT, + BEEPER_MESSAGE_LIKE_ME_GOLDEN_FINISHED_AT, +]; +const result = await exportBeeperMessageLikeMeBundle({ + outputRoot, + source: createBeeperMessageLikeMeGoldenSource(), + clock: () => new Date(instants.shift() ?? "invalid"), +}); +process.stdout.write(`${result.manifestSha256}\n`); diff --git a/scripts/package-smoke.ts b/scripts/package-smoke.ts index cf48d03..376ff87 100644 --- a/scripts/package-smoke.ts +++ b/scripts/package-smoke.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { access, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -24,6 +24,27 @@ async function run(command: string[], cwd: string): Promise { if (exitCode !== 0) throw new Error(`Command failed (${String(exitCode)}): ${command.join(" ")}`); } +async function runExpectingFailure( + command: string[], + cwd: string, + expectedExitCode: number, + expectedDiagnostic: string, +): Promise { + const child = Bun.spawn(command, { cwd, stdout: "pipe", stderr: "pipe" }); + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); + if ( + exitCode !== expectedExitCode + || stdout.length !== 0 + || !stderr.includes(expectedDiagnostic) + ) { + throw new Error(`Installed CLI failure contract drifted for: ${command.join(" ")}`); + } +} + const repository = process.cwd(); const work = await mkdtemp(join(tmpdir(), "hraness-package-smoke-")); try { @@ -46,6 +67,30 @@ try { for (const binName of binNames) { await run([join(consumer, "node_modules", ".bin", binName), "--help"], consumer); } + await access(join( + consumer, + "node_modules", + "@hraness", + "wrench", + "src", + "fixtures", + "beeper-message-like-me-v1", + "manifest.json", + )); + await run([ + process.execPath, + "-e", + "await import('./node_modules/@hraness/wrench/src/beeper-message-like-me-cli.ts')", + ], consumer); + await runExpectingFailure([ + join(consumer, "node_modules", ".bin", "wrench"), + "beeper", + "export-message-like-me", + "--auth", + "beeper-main", + "--output", + "relative", + ], consumer, 2, "normalized-absolute-directory"); if (verificationPackages.length > 0) { await run([process.execPath, "add", ...verificationPackages, "--ignore-scripts"], consumer); } diff --git a/src/args.test.ts b/src/args.test.ts index ba0ce51..366212c 100644 --- a/src/args.test.ts +++ b/src/args.test.ts @@ -81,6 +81,49 @@ describe("wrench CLI grammar", () => { }); }); + test("parses the bounded Beeper Message Like Me export command", () => { + expect(parseWrenchArguments([ + "beeper", + "export-message-like-me", + "--auth", + "beeper-main", + "--output", + "/tmp/message-like-me", + "--limit-chats", + "100", + "--limit-messages", + "5000", + "--max-participants", + "250", + "--json", + ])).toEqual({ + ok: true, + value: { + command: "beeper-export-message-like-me", + authId: "beeper-main", + output: "/tmp/message-like-me", + limitChats: 100, + limitMessages: 5000, + maxParticipants: 250, + json: true, + }, + }); + for (const raw of [ + ["beeper", "export-message-like-me", "--auth", "beeper-main"], + [ + "beeper", "export-message-like-me", "--auth", "beeper-main", + "--output", "relative", + ], + [ + "beeper", "export-message-like-me", "--auth", "beeper-main", + "--output", "/tmp/export", "--limit-chats", "0", + ], + ["beeper", "messages"], + ]) { + expect(parseWrenchArguments(raw).ok).toBeFalse(); + } + }); + test("parses adapter and capability management", () => { expect(parseWrenchArguments(["capabilities", "linkedin", "--json"])).toEqual({ ok: true, diff --git a/src/args.ts b/src/args.ts index 8d89142..d383c87 100644 --- a/src/args.ts +++ b/src/args.ts @@ -1,4 +1,5 @@ import { cookieSources, type CookieSource } from "@hraness/kb/clip/args"; +import { isAbsolute, resolve } from "node:path"; import { normalizeAuthSubject, normalizeOAuthScopes, @@ -19,6 +20,15 @@ export type WrenchArguments = | { readonly command: "clip"; readonly arguments: readonly string[] } | { readonly command: "read"; readonly arguments: readonly string[] } | { readonly command: "media"; readonly arguments: readonly string[] } + | { + readonly command: "beeper-export-message-like-me"; + readonly authId: string; + readonly output: string; + readonly limitChats?: number; + readonly limitMessages?: number; + readonly maxParticipants?: number; + readonly json: boolean; + } | { readonly command: "doctor"; readonly json: boolean } | { readonly command: "capabilities"; readonly adapterId?: string; readonly json: boolean } | { readonly command: "plugin-list"; readonly json: boolean } @@ -350,6 +360,25 @@ function simpleJsonOptions(raw: readonly string[], label: string): ParseWrenchRe return raw.includes("--json"); } +function optionalPositiveInteger( + value: string | undefined, + label: string, + maximum: number, +): ParseWrenchFailure | number | undefined { + if (value === undefined) return undefined; + if (!/^[1-9][0-9]*$/u.test(value)) { + return { ok: false, message: `${label} must be a positive integer` }; + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed > maximum) { + return { + ok: false, + message: `${label} must not exceed ${String(maximum)}`, + }; + } + return parsed; +} + function parsePluginScaffoldArguments( raw: readonly string[], label: "plugin scaffold" | "adapter scaffold", @@ -611,6 +640,76 @@ export function parseWrenchArguments(raw: readonly string[]): ParseWrenchResult ) { return { ok: true, value: { command: "media", arguments: raw } }; } + if (first === "beeper") { + if (raw[1] !== "export-message-like-me") { + return { + ok: false, + message: "beeper requires export-message-like-me", + }; + } + const parsed = optionValues( + raw.slice(2), + [ + "--auth", + "--output", + "--limit-chats", + "--limit-messages", + "--max-participants", + ], + ["--json"], + ); + if (isFailure(parsed)) return parsed; + const authId = parsed.values["--auth"]; + const output = parsed.values["--output"]; + if (authId === undefined || validId(authId, "auth ID") !== null) { + return { + ok: false, + message: "beeper export-message-like-me requires --auth ", + }; + } + if ( + output === undefined + || !isAbsolute(output) + || resolve(output) !== output + || Buffer.byteLength(output, "utf8") > 4_096 + || /[\0\r\n]/u.test(output) + ) { + return { + ok: false, + message: "beeper export-message-like-me requires --output ", + }; + } + const limitChats = optionalPositiveInteger( + parsed.values["--limit-chats"], + "--limit-chats", + 100_000, + ); + if (typeof limitChats === "object") return limitChats; + const limitMessages = optionalPositiveInteger( + parsed.values["--limit-messages"], + "--limit-messages", + 1_000_000, + ); + if (typeof limitMessages === "object") return limitMessages; + const maxParticipants = optionalPositiveInteger( + parsed.values["--max-participants"], + "--max-participants", + 2_000, + ); + if (typeof maxParticipants === "object") return maxParticipants; + return { + ok: true, + value: { + command: "beeper-export-message-like-me", + authId, + output, + ...(limitChats === undefined ? {} : { limitChats }), + ...(limitMessages === undefined ? {} : { limitMessages }), + ...(maxParticipants === undefined ? {} : { maxParticipants }), + json: parsed.booleans.has("--json"), + }, + }; + } if (first === "run") return parseWrenchArguments(["invoke", ...raw.slice(1)]); if (first === "doctor") { const json = simpleJsonOptions(raw.slice(1), "doctor"); diff --git a/src/assets/adapters/beeper/wrench-web-adapter.json b/src/assets/adapters/beeper/wrench-web-adapter.json new file mode 100644 index 0000000..f5a91f4 --- /dev/null +++ b/src/assets/adapters/beeper/wrench-web-adapter.json @@ -0,0 +1,129 @@ +{ + "schemaVersion": 4, + "id": "beeper-local", + "version": "1.0.0", + "displayName": "Beeper (Local Read-Only Projection)", + "surfaceId": "beeper", + "origins": [ + "https://www.beeper.com" + ], + "browserDomains": [ + "www.beeper.com" + ], + "operations": { + "contacts.list": { + "description": "List one bounded local Beeper contact projection, preserving account and network provenance without downloading media or emitting presence, read receipts, or provider writes.", + "risk": "R1", + "sideEffect": "none", + "idempotency": "none", + "dedupeWindowMs": 0, + "input": { + "properties": { + "account_id": { + "type": "string", + "description": "Optional exact Beeper account ID returned by this adapter", + "minLength": 1, + "maxLength": 512 + }, + "limit": { + "type": "number", + "description": "Maximum projected contacts", + "minimum": 1, + "maximum": 200 + } + }, + "required": [] + }, + "webSession": { + "site": "beeper", + "action": "contacts.list", + "contractVersion": 1, + "timeoutMs": 60000, + "maxOutputBytes": 10485760 + } + }, + "messaging.list": { + "description": "List one bounded local Beeper conversation projection with account, network, participant, and completeness evidence and no remote mutation.", + "risk": "R1", + "sideEffect": "none", + "idempotency": "none", + "dedupeWindowMs": 0, + "input": { + "properties": { + "account_id": { + "type": "string", + "description": "Optional exact Beeper account ID returned by this adapter", + "minLength": 1, + "maxLength": 512 + }, + "limit": { + "type": "number", + "description": "Maximum projected conversations", + "minimum": 1, + "maximum": 200 + } + }, + "required": [] + }, + "webSession": { + "site": "beeper", + "action": "messaging.list", + "contractVersion": 1, + "timeoutMs": 60000, + "maxOutputBytes": 10485760 + } + }, + "messaging.read": { + "description": "Read one bounded page of an exact Beeper conversation with reply, edit, deletion, reaction, and attachment-shape evidence while omitting media locations.", + "risk": "R1", + "sideEffect": "none", + "idempotency": "none", + "dedupeWindowMs": 0, + "input": { + "properties": { + "account_id": { + "type": "string", + "description": "Exact Beeper account ID bound to the conversation", + "minLength": 1, + "maxLength": 512 + }, + "conversation_id": { + "type": "string", + "description": "Exact canonical Beeper chat ID returned by messaging.list", + "minLength": 1, + "maxLength": 2048 + }, + "before_cursor": { + "type": "string", + "description": "Exact opaque message ID cursor for older messages", + "minLength": 1, + "maxLength": 2048 + }, + "after_cursor": { + "type": "string", + "description": "Exact opaque message ID cursor for newer messages", + "minLength": 1, + "maxLength": 2048 + }, + "limit": { + "type": "number", + "description": "Maximum projected messages", + "minimum": 1, + "maximum": 200 + } + }, + "required": [ + "account_id", + "conversation_id" + ] + }, + "webSession": { + "site": "beeper", + "action": "messaging.read", + "contractVersion": 1, + "timeoutMs": 60000, + "maxOutputBytes": 10485760 + } + } + } +} diff --git a/src/beeper-local-plugin.test.ts b/src/beeper-local-plugin.test.ts new file mode 100644 index 0000000..f7993a1 --- /dev/null +++ b/src/beeper-local-plugin.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, test } from "bun:test"; + +import { providerPluginRegistry } from "./provider-plugins"; + +describe("Beeper linked-device provider plugin", () => { + test("registers exactly the bounded read surface without lifecycle or mutations", () => { + const plugin = providerPluginRegistry.get("beeper-linked-device"); + const binding = providerPluginRegistry.requireSessionRoute("beeper"); + expect(plugin?.displayName).toBe("Beeper Local Read-Only"); + expect(binding.transport).toBe("linked-device"); + if (binding.transport !== "linked-device") { + throw new Error("Beeper installed the wrong transport"); + } + expect(binding.authKinds).toEqual(["linked-device-store"]); + expect(binding.linkedDeviceLifecycle).toBeUndefined(); + expect(binding.operations.map((operation) => operation.name)).toEqual([ + "contacts.list", + "messaging.list", + "messaging.read", + ]); + expect(binding.operations.every((operation) => + operation.risk === "R1" && operation.state === "observed" + )).toBeTrue(); + expect(binding.subject.matches( + `beeper:local:${"a".repeat(64)}`, + )).toBeTrue(); + expect(binding.subject.matches("beeper:mxid:reversible")).toBeFalse(); + }); +}); diff --git a/src/beeper-message-like-me-cli.ts b/src/beeper-message-like-me-cli.ts new file mode 100644 index 0000000..c77ad68 --- /dev/null +++ b/src/beeper-message-like-me-cli.ts @@ -0,0 +1,40 @@ +import type { WrenchAuth } from "./auth"; +import { + exportBeeperMessageLikeMeBundle, + type BeeperMessageLikeMeExportResult, +} from "./beeper-message-like-me-export"; +import { + createBeeperMessageLikeMeSource, + type BeeperMessageLikeMeSourceLimits, +} from "./beeper-message-like-me-source"; + +export type BeeperMessageLikeMeCliRequest = Readonly<{ + auth: WrenchAuth; + outputRoot: string; + limits?: BeeperMessageLikeMeSourceLimits; + environment?: Readonly>; + signal?: AbortSignal; +}>; + +/** + * Narrow CLI composition for the trusted Beeper source and private bundle + * sink. AI runtimes consume the resulting local bundle; this boundary never + * sends its contents to a model or network service. + */ +export async function exportBeeperMessageLikeMeFromAuth( + request: BeeperMessageLikeMeCliRequest, +): Promise { + const source = createBeeperMessageLikeMeSource({ + auth: request.auth, + ...(request.limits === undefined ? {} : { limits: request.limits }), + ...(request.environment === undefined + ? {} + : { environment: request.environment }), + ...(request.signal === undefined ? {} : { signal: request.signal }), + }); + return exportBeeperMessageLikeMeBundle({ + outputRoot: request.outputRoot, + source, + ...(request.signal === undefined ? {} : { signal: request.signal }), + }); +} diff --git a/src/beeper-message-like-me-export.test.ts b/src/beeper-message-like-me-export.test.ts new file mode 100644 index 0000000..ca83076 --- /dev/null +++ b/src/beeper-message-like-me-export.test.ts @@ -0,0 +1,693 @@ +import { createHash } from "node:crypto"; +import { + chmod, + lstat, + mkdir, + mkdtemp, + readFile, + readdir, + realpath, + rm, + symlink, +} from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { afterEach, describe, expect, test } from "bun:test"; + +import { canonicalJson } from "./canonical-json"; +import { + exportBeeperMessageLikeMeBundle, + type BeeperMessageLikeMeExportSource, +} from "./beeper-message-like-me-export"; +import { + BEEPER_MESSAGE_LIKE_ME_GOLDEN_FINISHED_AT, + BEEPER_MESSAGE_LIKE_ME_GOLDEN_STARTED_AT, + createBeeperMessageLikeMeGoldenSource, +} from "./beeper-message-like-me-golden-fixture"; + +const temporaryRoots: string[] = []; + +afterEach(async () => { + await Promise.all(temporaryRoots.splice(0).map((path) => rm(path, { force: true, recursive: true }))); +}); + +async function privateTemporaryRoot(): Promise { + const root = await mkdtemp(join(tmpdir(), "beeper-message-like-me-export-test-")); + temporaryRoots.push(root); + await chmod(root, 0o700); + return realpath(root); +} + +function provenance(providerId: string, revision: string | null = "r1") { + return { + providerId, + providerRevision: revision, + observedAt: "2026-08-21T12:00:00.000Z", + connectedAccountProviderId: "beeper-account-whatsapp", + }; +} + +function records(): readonly unknown[] { + return [ + { + schemaVersion: 1, + kind: "account", + id: "account:whatsapp:primary", + accountId: "account:whatsapp:primary", + network: "whatsapp", + provenance: provenance("beeper-account-whatsapp"), + displayName: "Primary WhatsApp", + handle: "+15555550100", + selfParticipantId: "participant:self", + }, + { + schemaVersion: 1, + kind: "participant", + id: "participant:self", + accountId: "account:whatsapp:primary", + network: "whatsapp", + provenance: provenance("beeper-user-self"), + displayName: "Me", + handle: "+15555550100", + isSelf: true, + }, + { + schemaVersion: 1, + kind: "conversation", + id: "conversation:friend", + accountId: "account:whatsapp:primary", + network: "whatsapp", + provenance: provenance("beeper-chat-1"), + type: "direct", + title: "Friend", + participantIds: ["participant:self", "participant:peer"], + participantsComplete: true, + startedAt: "2026-08-20T12:00:00.000Z", + lastMessageAt: "2026-08-21T11:59:00.000Z", + }, + { + schemaVersion: 1, + kind: "message", + id: "message:edited", + accountId: "account:whatsapp:primary", + network: "whatsapp", + provenance: provenance("beeper-message-2", "message-r3"), + conversationId: "conversation:friend", + senderParticipantId: "participant:self", + direction: "outgoing", + sentAt: "2026-08-21T11:58:00.000Z", + sortKey: "00000000000000000042", + body: null, + bodyTruncated: false, + replyTo: { + messageId: null, + providerId: "beeper-message-1", + }, + edit: { + kind: "in-place", + editedAt: "2026-08-21T11:59:00.000Z", + providerRevision: "message-r3", + }, + deletion: { + state: "deleted-for-me", + observedAt: "2026-08-21T12:00:00.000Z", + providerRevision: "message-r4", + }, + attachments: [{ + kind: "image", + mimeType: "image/jpeg", + name: "photo.jpg", + sizeBytes: 1234, + }], + }, + { + schemaVersion: 1, + kind: "reaction", + id: "reaction:1", + accountId: "account:whatsapp:primary", + network: "whatsapp", + provenance: provenance("beeper-reaction-1"), + messageId: "message:edited", + messageProviderId: "beeper-message-2", + participantId: "participant:self", + body: "👍", + reactedAt: null, + state: "active", + }, + { + schemaVersion: 1, + kind: "tombstone", + id: "tombstone:message:old", + accountId: "account:whatsapp:primary", + network: "whatsapp", + provenance: provenance("beeper-message-old", "deleted-r2"), + entityKind: "message", + entityId: null, + entityProviderId: "beeper-message-old", + deletedAt: "2026-08-21T10:00:00.000Z", + scope: "remote", + providerRevision: "deleted-r2", + }, + { + schemaVersion: 1, + kind: "participant", + id: "participant:peer", + accountId: "account:whatsapp:primary", + network: "whatsapp", + provenance: provenance("beeper-user-peer"), + displayName: "Friend", + handle: "+15555550101", + isSelf: false, + }, + ]; +} + +function source(values: readonly unknown[] = records()): BeeperMessageLikeMeExportSource { + return { + descriptor: { + source: { id: "beeper-local", version: "1.0.0" }, + provider: { id: "beeper", version: "4.1.0" }, + }, + records: (async function* () { + for (const value of values) yield value; + })(), + completion: async () => ({ + completeness: { + kind: "bounded-local", + reason: "desktop-local-cache", + observedFrom: "2026-08-20T12:00:00.000Z", + observedThrough: "2026-08-21T12:00:00.000Z", + }, + warnings: ["attachments-metadata-only", "remote-history-not-claimed"], + }), + }; +} + +function clock(...values: readonly string[]): () => Date { + let index = 0; + return () => new Date(values[index++] ?? "invalid"); +} + +describe("exportBeeperMessageLikeMeBundle", () => { + test("reproduces the checked v1 cross-repository golden bundle byte for byte", async () => { + const parent = await privateTemporaryRoot(); + const outputRoot = join(parent, "golden"); + const result = await exportBeeperMessageLikeMeBundle({ + outputRoot, + source: createBeeperMessageLikeMeGoldenSource(), + clock: clock( + BEEPER_MESSAGE_LIKE_ME_GOLDEN_STARTED_AT, + BEEPER_MESSAGE_LIKE_ME_GOLDEN_FINISHED_AT, + ), + }); + const fixtureRoot = join( + import.meta.dir, + "fixtures", + "beeper-message-like-me-v1", + ); + const expectedFiles = (await readdir(fixtureRoot)).sort(); + expect((await readdir(outputRoot)).sort()).toEqual(expectedFiles); + for (const fileName of expectedFiles) { + expect(await readFile(join(outputRoot, fileName))).toEqual( + await readFile(join(fixtureRoot, fileName)), + ); + } + expect(result.manifestSha256).toBe( + "e46f4a524d53f849cfac594fb5bc8cf28e7a9743c138039b81a0aad4ff4830ef", + ); + expect(result.manifest.completeness).toMatchObject({ + kind: "truncated", + reason: "explicit-source-limit", + }); + const goldenMessages = (await readFile(join(outputRoot, "messages.ndjson"), "utf8")) + .trimEnd() + .split("\n") + .map((line) => JSON.parse(line) as Record); + expect(goldenMessages).toHaveLength(2); + expect(goldenMessages[0]).toMatchObject({ + body: "edited synthetic reply", + replyTo: { messageId: null }, + edit: { kind: "in-place" }, + deletion: null, + }); + expect(goldenMessages[1]).toMatchObject({ + body: null, + edit: null, + deletion: { state: "revoked" }, + }); + const goldenReaction = JSON.parse( + await readFile(join(outputRoot, "reactions.ndjson"), "utf8"), + ) as Record; + expect(goldenReaction.reactedAt).toBeNull(); + }); + + test("streams a private provenance-preserving bundle and publishes its manifest last", async () => { + const parent = await privateTemporaryRoot(); + const outputRoot = join(parent, "message-like-me"); + const result = await exportBeeperMessageLikeMeBundle({ + outputRoot, + source: source(), + clock: clock("2026-08-21T12:01:00.000Z", "2026-08-21T12:02:00.000Z"), + }); + + expect(result.outputRoot).toBe(outputRoot); + expect(result.manifest.counts).toEqual({ + account: 1, + participant: 2, + conversation: 1, + message: 1, + reaction: 1, + tombstone: 1, + }); + expect(result.manifest.timestamps).toEqual({ + startedAt: "2026-08-21T12:01:00.000Z", + finishedAt: "2026-08-21T12:02:00.000Z", + createdAt: "2026-08-21T12:02:00.000Z", + }); + expect(result.manifest.completeness.kind).toBe("bounded-local"); + expect(result.manifest.privacy).toEqual({ + classification: "private-local", + attachments: "metadata-only", + providerUrls: "excluded", + credentials: "excluded", + }); + + const expectedFiles = [ + "accounts.ndjson", + "conversations.ndjson", + "manifest.json", + "messages.ndjson", + "participants.ndjson", + "reactions.ndjson", + "tombstones.ndjson", + ]; + expect((await readdir(outputRoot)).sort()).toEqual(expectedFiles); + expect((await lstat(outputRoot)).mode & 0o777).toBe(0o700); + + for (const artifact of result.manifest.artifacts) { + const path = join(outputRoot, artifact.path); + const contents = await readFile(path); + expect((await lstat(path)).mode & 0o777).toBe(0o600); + expect(contents.byteLength).toBe(artifact.bytes); + expect(createHash("sha256").update(contents).digest("hex")).toBe(artifact.sha256); + } + expect((await lstat(result.manifestPath)).mode & 0o777).toBe(0o600); + const manifestSource = await readFile(result.manifestPath, "utf8"); + expect(createHash("sha256").update(manifestSource).digest("hex")).toBe(result.manifestSha256); + expect(JSON.parse(manifestSource)).toEqual(result.manifest); + + const integrityInput = { + schemaVersion: result.manifest.schemaVersion, + format: result.manifest.format, + source: result.manifest.source, + provider: result.manifest.provider, + timestamps: result.manifest.timestamps, + completeness: result.manifest.completeness, + warnings: result.manifest.warnings, + privacy: result.manifest.privacy, + counts: result.manifest.counts, + artifacts: result.manifest.artifacts, + }; + expect(result.manifest.integrity.bundleSha256).toBe( + createHash("sha256").update(canonicalJson(integrityInput)).digest("hex"), + ); + + const message = JSON.parse(await readFile(join(outputRoot, "messages.ndjson"), "utf8")); + expect(message).toMatchObject({ + accountId: "account:whatsapp:primary", + network: "whatsapp", + sortKey: "00000000000000000042", + body: null, + replyTo: { providerId: "beeper-message-1" }, + edit: { providerRevision: "message-r3" }, + deletion: { providerRevision: "message-r4" }, + provenance: { + connectedAccountProviderId: "beeper-account-whatsapp", + providerId: "beeper-message-2", + }, + }); + expect(manifestSource).not.toContain(outputRoot); + expect(manifestSource).not.toContain("token"); + }); + + test("rejects foreign fields before they can introduce provider URLs", async () => { + const parent = await privateTemporaryRoot(); + const outputRoot = join(parent, "strict"); + const account = { ...records()[0] as Record, mediaUrl: "https://provider.invalid/private" }; + + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot, + source: source([account]), + })).rejects.toThrow("must contain exactly"); + + expect((await lstat(outputRoot)).mode & 0o777).toBe(0o700); + await expect(lstat(join(outputRoot, "manifest.json"))).rejects.toMatchObject({ code: "ENOENT" }); + }); + + test("enforces the async stream record bound", async () => { + const parent = await privateTemporaryRoot(); + const outputRoot = join(parent, "bounded"); + const second = { + ...records()[0] as Record, + id: "account:whatsapp:secondary", + accountId: "account:whatsapp:secondary", + selfParticipantId: "participant:self:secondary", + }; + + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot, + source: source([records()[0], second]), + limits: { maxRecords: 1 }, + })).rejects.toThrow("record stream exceeds the configured record bound"); + await expect(lstat(join(outputRoot, "manifest.json"))).rejects.toMatchObject({ code: "ENOENT" }); + }); + + test("enforces importer-compatible bundle and connected-account bounds", async () => { + const parent = await privateTemporaryRoot(); + const oversizedLimitsOutput = join(parent, "oversized-limits"); + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot: oversizedLimitsOutput, + source: source([]), + limits: { maxRecords: 500_001 }, + })).rejects.toThrow("maxRecords"); + await expect(lstat(oversizedLimitsOutput)).rejects.toMatchObject({ code: "ENOENT" }); + + const accounts = Array.from({ length: 129 }, (_, index) => { + const providerAccountId = `provider-account-${String(index)}`; + return { + schemaVersion: 1, + kind: "account", + id: `account:${String(index)}`, + accountId: `account:${String(index)}`, + network: "beeper", + provenance: { + providerId: providerAccountId, + providerRevision: null, + observedAt: "2026-08-21T12:00:00.000Z", + connectedAccountProviderId: providerAccountId, + }, + displayName: null, + handle: null, + selfParticipantId: `participant:self:${String(index)}`, + }; + }); + const accountBoundOutput = join(parent, "account-bound"); + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot: accountBoundOutput, + source: source(accounts), + })).rejects.toThrow("connected-account bound"); + await expect(lstat(join(accountBoundOutput, "manifest.json"))) + .rejects.toMatchObject({ code: "ENOENT" }); + }); + + test("refuses relative, existing, and symlink-traversing output roots", async () => { + const parent = await privateTemporaryRoot(); + + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot: "relative-bundle", + source: source([]), + })).rejects.toThrow("normalized absolute path"); + + const existing = join(parent, "existing"); + await chmod(parent, 0o700); + await Bun.write(existing, "owned by caller"); + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot: existing, + source: source([]), + })).rejects.toThrow("already exists"); + + const permissive = join(parent, "permissive"); + await mkdir(permissive, { mode: 0o700 }); + await chmod(permissive, 0o777); + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot: join(permissive, "bundle"), + source: source([]), + })).rejects.toThrow("must not be writable by the group or other users"); + + const physical = join(parent, "physical"); + const linked = join(parent, "linked"); + await mkdir(physical, { mode: 0o700 }); + await symlink(physical, linked); + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot: join(linked, "bundle"), + source: source([]), + })).rejects.toThrow("parent must be a real directory"); + }); + + test("validates provider identities and cross-record references without echoing foreign IDs", async () => { + const cases: readonly { + readonly name: string; + readonly values: readonly unknown[]; + readonly message: string; + readonly secret?: string; + }[] = [ + { + name: "duplicate provider coordinate", + values: [ + ...records(), + { + ...records()[1] as Record, + id: "participant:duplicate", + isSelf: false, + }, + ], + message: "repeats an account-scoped provider identity", + }, + { + name: "missing self participant", + values: records().map((value, index) => index === 0 + ? { + ...value as Record, + selfParticipantId: "private-matrix-self-identifier", + } + : value), + message: "account self participant does not resolve", + secret: "private-matrix-self-identifier", + }, + { + name: "conversation participant realm", + values: records().map((value, index) => index === 2 + ? { + ...value as Record, + participantIds: ["private-matrix-participant-identifier"], + } + : value), + message: "conversation participant does not resolve", + secret: "private-matrix-participant-identifier", + }, + { + name: "sender direction", + values: records().map((value, index) => index === 3 + ? { ...value as Record, direction: "incoming" } + : value), + message: "message direction conflicts", + }, + { + name: "reaction provider coordinate", + values: records().map((value, index) => index === 4 + ? { + ...value as Record, + messageProviderId: "private-provider-target-identifier", + } + : value), + message: "reaction message target provider coordinate does not match", + secret: "private-provider-target-identifier", + }, + { + name: "tombstone provider coordinate", + values: records().map((value, index) => index === 5 + ? { + ...value as Record, + entityId: "message:edited", + entityProviderId: "private-provider-tombstone-identifier", + } + : value), + message: "tombstone entity provider coordinate does not match", + secret: "private-provider-tombstone-identifier", + }, + ]; + + for (const item of cases) { + const parent = await privateTemporaryRoot(); + let failure: unknown; + try { + await exportBeeperMessageLikeMeBundle({ + outputRoot: join(parent, item.name.replaceAll(" ", "-")), + source: source(item.values), + }); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toContain(item.message); + if (item.secret !== undefined) { + expect((failure as Error).message).not.toContain(item.secret); + } + } + }); + + test("enforces reply, replacement, and complete-roster graph semantics", async () => { + const base = records(); + const otherConversation = { + ...base[2] as Record, + id: "conversation:other", + provenance: provenance("beeper-chat-2"), + title: "Other", + }; + const message = ( + id: string, + providerId: string, + conversationId = "conversation:friend", + edit: unknown = null, + ) => ({ + ...base[3] as Record, + id, + provenance: provenance(providerId, `${providerId}-revision`), + conversationId, + sortKey: `${providerId}-sort-key`, + body: "synthetic graph fixture", + replyTo: null, + edit, + deletion: null, + attachments: [], + }); + const otherMessage = message( + "message:other", + "beeper-message-3", + "conversation:other", + ); + const participant = { + ...base[1] as Record, + id: "participant:other", + provenance: provenance("beeper-user-other"), + displayName: "Other", + handle: null, + isSelf: false, + }; + const replace = (providerId: string, localId: string | null = null) => ({ + kind: "replacement", + replacesMessageId: localId, + replacesProviderId: providerId, + editedAt: "2026-08-21T11:59:00.000Z", + providerRevision: "replacement-r1", + }); + const mutateMain = (changes: Record) => + base.map((value, index) => index === 3 + ? { ...value as Record, ...changes } + : value); + + const cases: readonly { + readonly name: string; + readonly values: readonly unknown[]; + readonly message: string; + }[] = [{ + name: "complete direct missing peer", + values: base.map((value, index) => index === 2 + ? { + ...value as Record, + participantIds: ["participant:self"], + } + : value), + message: "must contain exactly one self participant and one peer", + }, { + name: "provider-only cross-conversation reply", + values: [ + ...mutateMain({ + replyTo: { messageId: null, providerId: "beeper-message-3" }, + }), + otherConversation, + otherMessage, + ], + message: "reply target belongs to a different conversation", + }, { + name: "self reply", + values: mutateMain({ + replyTo: { messageId: null, providerId: "beeper-message-2" }, + }), + message: "must not reply to itself", + }, { + name: "provider-only cross-conversation replacement", + values: [ + ...mutateMain({ edit: replace("beeper-message-3") }), + otherConversation, + otherMessage, + ], + message: "replacement edit target belongs to a different conversation", + }, { + name: "multiple replacements", + values: [ + ...mutateMain({ edit: replace("beeper-message-3") }), + message("message:target", "beeper-message-3"), + message( + "message:second-replacer", + "beeper-message-4", + "conversation:friend", + replace("beeper-message-3"), + ), + ], + message: "more than one replacement", + }, { + name: "replacement cycle", + values: [ + ...mutateMain({ edit: replace("beeper-message-3") }), + message( + "message:cycle", + "beeper-message-3", + "conversation:friend", + replace("beeper-message-2"), + ), + ], + message: "replacement edit graph contains a cycle", + }, { + name: "sender absent from complete roster", + values: [ + ...mutateMain({ + senderParticipantId: "participant:other", + direction: "incoming", + }), + participant, + ], + message: "sender is absent from the complete conversation roster", + }, { + name: "reaction actor absent from complete roster", + values: [ + ...base.map((value, index) => index === 4 + ? { + ...value as Record, + messageId: null, + participantId: "participant:other", + } + : value), + participant, + ], + message: "reaction participant is absent from the complete conversation roster", + }, { + name: "stable account realm", + values: [ + ...base, + { + ...base[0] as Record, + id: "account:duplicate-realm", + accountId: "account:duplicate-realm", + network: "renamed-whatsapp", + selfParticipantId: "participant:self:duplicate-realm", + }, + { + ...base[1] as Record, + id: "participant:self:duplicate-realm", + accountId: "account:duplicate-realm", + network: "renamed-whatsapp", + }, + ], + message: "repeat one stable provider realm", + }]; + + for (const item of cases) { + const parent = await privateTemporaryRoot(); + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot: join(parent, item.name.replaceAll(" ", "-")), + source: source(item.values), + })).rejects.toThrow(item.message); + } + }); +}); diff --git a/src/beeper-message-like-me-export.ts b/src/beeper-message-like-me-export.ts new file mode 100644 index 0000000..c54e7e5 --- /dev/null +++ b/src/beeper-message-like-me-export.ts @@ -0,0 +1,1603 @@ +import { constants } from "node:fs"; +import { + chmod, + lstat, + mkdir, + open, + realpath, + rename, + rmdir, + type FileHandle, +} from "node:fs/promises"; +import { basename, dirname, isAbsolute, resolve, sep } from "node:path"; +import { createHash } from "node:crypto"; + +import { canonicalJson, sha256 } from "./canonical-json"; + +export const BEEPER_MESSAGE_LIKE_ME_SCHEMA_VERSION = 1 as const; +export const BEEPER_MESSAGE_LIKE_ME_MAX_RECORDS = 500_000 as const; +export const BEEPER_MESSAGE_LIKE_ME_MAX_TOTAL_BYTES = 512 * 1024 * 1024; + +const PRIVATE_DIRECTORY_MODE = 0o700; +const PRIVATE_FILE_MODE = 0o600; +const MAX_IDENTIFIER_BYTES = 1_024; +const MAX_SHORT_TEXT_BYTES = 8 * 1_024; +const MAX_BODY_BYTES = 1024 * 1024; +const MAX_WARNING_CODES = 128; +const MAX_PARTICIPANTS = 10_000; +const MAX_ATTACHMENTS = 256; +const MAX_CONNECTED_ACCOUNTS = 128; + +const HARD_LIMITS = Object.freeze({ + maxRecords: BEEPER_MESSAGE_LIKE_ME_MAX_RECORDS, + maxRecordBytes: 2 * 1024 * 1024, + maxTotalBytes: BEEPER_MESSAGE_LIKE_ME_MAX_TOTAL_BYTES, +}); + +export type BeeperMessageLikeMeExportLimits = { + readonly maxRecords: number; + readonly maxRecordBytes: number; + readonly maxTotalBytes: number; +}; + +export type BeeperMessageLikeMeExportSource = { + /** Parsed as foreign data before any output directory is created. */ + readonly descriptor: unknown; + /** Each yielded value is parsed strictly and written once. */ + readonly records: AsyncIterable; + /** Called only after the record stream ends successfully. */ + readonly completion: () => Promise; +}; + +export type BeeperMessageLikeMeExportRequest = { + readonly outputRoot: string; + readonly source: BeeperMessageLikeMeExportSource; + readonly limits?: Partial; + readonly signal?: AbortSignal; + /** Test seam. Production callers should omit it. */ + readonly clock?: () => Date; +}; + +export type BeeperMessageLikeMeProvenance = { + /** Stable entity identity in the provider's connected-account realm. */ + readonly providerId: string; + readonly providerRevision: string | null; + readonly observedAt: string; + /** Stable provider account identity used to distinguish account incarnations. */ + readonly connectedAccountProviderId: string; +}; + +type BeeperMessageLikeMeRecordCommon = { + readonly schemaVersion: 1; + readonly kind: Kind; + /** Bundle-local identity used for joins inside this export. */ + readonly id: string; + readonly accountId: string; + readonly network: string; + readonly provenance: BeeperMessageLikeMeProvenance; +}; + +export type BeeperMessageLikeMeAccount = BeeperMessageLikeMeRecordCommon<"account"> & { + /** Equal to id for the account record. */ + readonly accountId: string; + readonly displayName: string | null; + readonly handle: string | null; + readonly selfParticipantId: string; +}; + +export type BeeperMessageLikeMeParticipant = BeeperMessageLikeMeRecordCommon<"participant"> & { + readonly displayName: string | null; + readonly handle: string | null; + readonly isSelf: boolean; +}; + +export type BeeperMessageLikeMeConversation = BeeperMessageLikeMeRecordCommon<"conversation"> & { + readonly type: "direct" | "group" | "channel" | "unknown"; + readonly title: string | null; + /** Known roster. It includes self whenever self is present in the provider roster. */ + readonly participantIds: readonly string[]; + /** Only true is a positive assertion that the known roster is complete. */ + readonly participantsComplete: boolean | null; + readonly startedAt: string | null; + readonly lastMessageAt: string | null; +}; + +export type BeeperMessageLikeMeAttachment = { + readonly kind: "audio" | "document" | "image" | "link" | "sticker" | "video" | "unknown"; + readonly mimeType: string | null; + /** Base name only. Provider URLs and local paths are outside this format. */ + readonly name: string | null; + readonly sizeBytes: number | null; +}; + +export type BeeperMessageLikeMeMessage = BeeperMessageLikeMeRecordCommon<"message"> & { + readonly conversationId: string; + readonly senderParticipantId: string | null; + readonly direction: "incoming" | "outgoing" | "unknown"; + readonly sentAt: string; + /** Provider-normalized key whose lexical order preserves provider message order. */ + readonly sortKey: string; + readonly body: string | null; + /** True bodies are unavailable as prose evidence. */ + readonly bodyTruncated: boolean | null; + readonly replyTo: { + readonly messageId: string | null; + readonly providerId: string; + } | null; + readonly edit: { + readonly kind: "in-place"; + readonly editedAt: string; + readonly providerRevision: string; + } | { + readonly kind: "replacement"; + readonly replacesMessageId: string | null; + readonly replacesProviderId: string; + readonly editedAt: string; + readonly providerRevision: string; + } | null; + readonly deletion: { + readonly state: "revoked" | "deleted-for-me" | "revoked-and-deleted-for-me"; + readonly observedAt: string; + readonly providerRevision: string | null; + } | null; + readonly attachments: readonly BeeperMessageLikeMeAttachment[]; +}; + +export type BeeperMessageLikeMeReaction = BeeperMessageLikeMeRecordCommon<"reaction"> & { + readonly messageId: string | null; + readonly messageProviderId: string; + readonly participantId: string | null; + readonly body: string; + /** Null when the provider does not supply a reaction time. Never synthesize it. */ + readonly reactedAt: string | null; + readonly state: "active" | "removed"; +}; + +export type BeeperMessageLikeMeTombstone = BeeperMessageLikeMeRecordCommon<"tombstone"> & { + readonly entityKind: "conversation" | "message" | "reaction"; + readonly entityId: string | null; + readonly entityProviderId: string; + readonly deletedAt: string; + readonly scope: "remote" | "local" | "unknown"; + readonly providerRevision: string | null; +}; + +export type BeeperMessageLikeMeRecord = + | BeeperMessageLikeMeAccount + | BeeperMessageLikeMeParticipant + | BeeperMessageLikeMeConversation + | BeeperMessageLikeMeMessage + | BeeperMessageLikeMeReaction + | BeeperMessageLikeMeTombstone; + +export type BeeperMessageLikeMeArtifact = { + readonly path: string; + readonly mediaType: "application/x-ndjson"; + readonly recordKind: + | "account" + | "participant" + | "conversation" + | "message" + | "reaction" + | "tombstone"; + readonly records: number; + readonly bytes: number; + readonly sha256: string; +}; + +export type BeeperMessageLikeMeManifest = { + readonly schemaVersion: 1; + readonly format: "message-like-me.local-message-bundle"; + readonly source: { + readonly id: "beeper-local"; + readonly version: string; + }; + readonly provider: { + readonly id: "beeper"; + readonly version: string; + }; + readonly timestamps: { + readonly startedAt: string; + readonly finishedAt: string; + readonly createdAt: string; + }; + readonly completeness: { + readonly kind: "bounded-local" | "truncated" | "unknown"; + readonly reason: string | null; + readonly observedFrom: string | null; + readonly observedThrough: string | null; + }; + readonly warnings: readonly string[]; + readonly privacy: { + readonly classification: "private-local"; + readonly attachments: "metadata-only"; + readonly providerUrls: "excluded"; + readonly credentials: "excluded"; + }; + readonly counts: Readonly>; + readonly artifacts: readonly BeeperMessageLikeMeArtifact[]; + readonly integrity: { + readonly algorithm: "sha256"; + readonly bundleSha256: string; + }; +}; + +export type BeeperMessageLikeMeExportResult = { + readonly outputRoot: string; + readonly manifestPath: string; + readonly manifestSha256: string; + readonly manifest: BeeperMessageLikeMeManifest; +}; + +type JsonRecord = Record; +type RecordKind = BeeperMessageLikeMeArtifact["recordKind"]; + +type ParsedDescriptor = { + readonly source: { readonly id: "beeper-local"; readonly version: string }; + readonly provider: { readonly id: "beeper"; readonly version: string }; +}; + +type ParsedCompletion = Pick; + +type ParsedProvenance = { + readonly providerId: string; + readonly providerRevision: string | null; + readonly observedAt: string; + readonly connectedAccountProviderId: string; +}; + +type ParsedRecord = { + readonly kind: RecordKind; + readonly id: string; + readonly accountId?: string; + readonly network?: string; + readonly connectedAccountProviderId?: string; + readonly value: BeeperMessageLikeMeRecord; +}; + +type BundleGraphFact = + | Readonly<{ + kind: "account"; + id: string; + accountId: string; + providerId: string; + network: string; + selfParticipantId: string; + }> + | Readonly<{ + kind: "participant"; + id: string; + accountId: string; + providerId: string; + isSelf: boolean; + }> + | Readonly<{ + kind: "conversation"; + id: string; + accountId: string; + providerId: string; + type: BeeperMessageLikeMeConversation["type"]; + participantIds: readonly string[]; + participantsComplete: boolean | null; + }> + | Readonly<{ + kind: "message"; + id: string; + accountId: string; + providerId: string; + conversationId: string; + senderParticipantId: string | null; + direction: BeeperMessageLikeMeMessage["direction"]; + replyTo: BeeperMessageLikeMeMessage["replyTo"]; + edit: BeeperMessageLikeMeMessage["edit"]; + }> + | Readonly<{ + kind: "reaction"; + id: string; + accountId: string; + providerId: string; + messageId: string | null; + messageProviderId: string; + participantId: string | null; + }> + | Readonly<{ + kind: "tombstone"; + id: string; + accountId: string; + providerId: string; + entityKind: BeeperMessageLikeMeTombstone["entityKind"]; + entityId: string | null; + entityProviderId: string; + }>; + +type BundleGraphInventory = ReadonlyMap< + RecordKind, + ReadonlyMap +>; + +type ArtifactWriter = { + readonly kind: RecordKind; + readonly fileName: string; + readonly partPath: string; + readonly handle: FileHandle; + readonly hash: ReturnType; + records: number; + bytes: number; + closed: boolean; +}; + +const ARTIFACTS = Object.freeze([ + Object.freeze({ kind: "account" as const, fileName: "accounts.ndjson" }), + Object.freeze({ kind: "participant" as const, fileName: "participants.ndjson" }), + Object.freeze({ kind: "conversation" as const, fileName: "conversations.ndjson" }), + Object.freeze({ kind: "message" as const, fileName: "messages.ndjson" }), + Object.freeze({ kind: "reaction" as const, fileName: "reactions.ndjson" }), + Object.freeze({ kind: "tombstone" as const, fileName: "tombstones.ndjson" }), +]); + +function fail(message: string): never { + throw new Error(`Beeper Message Like Me export: ${message}`); +} + +function isErrno(error: unknown, code: string): boolean { + return typeof error === "object" && error !== null && "code" in error + && (error as { readonly code?: unknown }).code === code; +} + +function foreignRecord(value: unknown, label: string): JsonRecord { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return fail(`${label} must be a plain object`); + } + const prototype = Object.getPrototypeOf(value) as unknown; + if (prototype !== Object.prototype && prototype !== null) { + return fail(`${label} must be a plain object`); + } + const keys = Reflect.ownKeys(value); + if (keys.some((key) => typeof key !== "string")) { + return fail(`${label} must not contain symbol keys`); + } + const descriptors = Object.getOwnPropertyDescriptors(value); + for (const key of keys as string[]) { + const descriptor = descriptors[key]; + if (descriptor === undefined || !descriptor.enumerable || !("value" in descriptor)) { + return fail(`${label}.${key} must be an enumerable data property`); + } + } + return value as JsonRecord; +} + +function exactKeys(value: JsonRecord, expected: readonly string[], label: string): void { + const observed = Object.keys(value).sort(); + const wanted = [...expected].sort(); + if ( + observed.length !== wanted.length + || observed.some((key, index) => key !== wanted[index]) + ) { + fail(`${label} must contain exactly: ${expected.join(", ")}`); + } +} + +function utf8Length(value: string): number { + return Buffer.byteLength(value, "utf8"); +} + +function text(value: unknown, label: string, maximumBytes: number): string { + if (typeof value !== "string" || utf8Length(value) > maximumBytes || value.includes("\0")) { + return fail(`${label} must be a NUL-free string of at most ${String(maximumBytes)} UTF-8 bytes`); + } + return value; +} + +function nullableText(value: unknown, label: string, maximumBytes: number): string | null { + return value === null ? null : text(value, label, maximumBytes); +} + +function identifier(value: unknown, label: string): string { + const parsed = text(value, label, MAX_IDENTIFIER_BYTES); + if (parsed.length === 0 || /[\u0000-\u001f\u007f]/u.test(parsed)) { + return fail(`${label} must be a non-empty identifier without ASCII control characters`); + } + return parsed; +} + +function token(value: unknown, label: string, maximumBytes = 128): string { + const parsed = text(value, label, maximumBytes); + if (!/^[a-z0-9](?:[a-z0-9._+-]*[a-z0-9])?$/u.test(parsed)) { + return fail(`${label} must be a lowercase categorical token`); + } + return parsed; +} + +function version(value: unknown, label: string): string { + const parsed = text(value, label, 128); + if (!/^[A-Za-z0-9](?:[A-Za-z0-9._+-]*[A-Za-z0-9])?$/u.test(parsed)) { + return fail(`${label} must be a bounded version token`); + } + return parsed; +} + +function oneOf( + value: unknown, + values: Values, + label: string, +): Values[number] { + if (typeof value !== "string" || !values.includes(value)) { + return fail(`${label} must be one of: ${values.join(", ")}`); + } + return value as Values[number]; +} + +function boolean(value: unknown, label: string): boolean { + if (typeof value !== "boolean") return fail(`${label} must be a boolean`); + return value; +} + +function nullableBoolean(value: unknown, label: string): boolean | null { + return value === null ? null : boolean(value, label); +} + +function integer(value: unknown, label: string, maximum = Number.MAX_SAFE_INTEGER): number { + if (!Number.isSafeInteger(value) || (value as number) < 0 || (value as number) > maximum) { + return fail(`${label} must be a non-negative safe integer at most ${String(maximum)}`); + } + return value as number; +} + +function nullableInteger(value: unknown, label: string): number | null { + return value === null ? null : integer(value, label); +} + +function timestamp(value: unknown, label: string): string { + const parsed = text(value, label, 64); + const instant = new Date(parsed); + if (!Number.isFinite(instant.getTime()) || instant.toISOString() !== parsed) { + return fail(`${label} must be a canonical UTC timestamp with millisecond precision`); + } + return parsed; +} + +function nullableTimestamp(value: unknown, label: string): string | null { + return value === null ? null : timestamp(value, label); +} + +function array(value: unknown, label: string, maximum: number): readonly unknown[] { + if (!Array.isArray(value) || value.length > maximum) { + return fail(`${label} must be an array of at most ${String(maximum)} items`); + } + return value; +} + +function uniqueIdentifiers(value: unknown, label: string, maximum: number): readonly string[] { + const parsed = array(value, label, maximum).map((item, index) => + identifier(item, `${label}[${String(index)}]`)); + if (new Set(parsed).size !== parsed.length) fail(`${label} must not contain duplicates`); + return Object.freeze(parsed); +} + +function parseProvenance(value: unknown, label: string): ParsedProvenance { + const source = foreignRecord(value, label); + exactKeys(source, [ + "providerId", + "providerRevision", + "observedAt", + "connectedAccountProviderId", + ], label); + return Object.freeze({ + providerId: identifier(source.providerId, `${label}.providerId`), + providerRevision: nullableText(source.providerRevision, `${label}.providerRevision`, MAX_IDENTIFIER_BYTES), + observedAt: timestamp(source.observedAt, `${label}.observedAt`), + connectedAccountProviderId: identifier( + source.connectedAccountProviderId, + `${label}.connectedAccountProviderId`, + ), + }); +} + +function parseCommon( + source: JsonRecord, + kind: RecordKind, + expected: readonly string[], + label: string, +): { + readonly id: string; + readonly accountId: string; + readonly network: string; + readonly provenance: ParsedProvenance; +} { + exactKeys(source, ["schemaVersion", "kind", "id", "accountId", "network", "provenance", ...expected], label); + if (source.schemaVersion !== 1) fail(`${label}.schemaVersion must equal 1`); + if (source.kind !== kind) fail(`${label}.kind must equal ${kind}`); + return { + id: identifier(source.id, `${label}.id`), + accountId: identifier(source.accountId, `${label}.accountId`), + network: token(source.network, `${label}.network`, 64), + provenance: parseProvenance(source.provenance, `${label}.provenance`), + }; +} + +function parseAccount(source: JsonRecord, label: string): ParsedRecord { + const common = parseCommon(source, "account", ["displayName", "handle", "selfParticipantId"], label); + const value = Object.freeze({ + schemaVersion: 1, + kind: "account", + ...common, + displayName: nullableText(source.displayName, `${label}.displayName`, MAX_SHORT_TEXT_BYTES), + handle: nullableText(source.handle, `${label}.handle`, MAX_SHORT_TEXT_BYTES), + selfParticipantId: identifier(source.selfParticipantId, `${label}.selfParticipantId`), + }); + if (common.id !== common.accountId) fail(`${label}.id and ${label}.accountId must match`); + if (common.provenance.providerId !== common.provenance.connectedAccountProviderId) { + fail(`${label}.provenance.providerId must identify the connected account`); + } + return { + kind: "account", + id: common.id, + accountId: common.accountId, + network: common.network, + connectedAccountProviderId: common.provenance.connectedAccountProviderId, + value, + }; +} + +function parseParticipant(source: JsonRecord, label: string): ParsedRecord { + const common = parseCommon(source, "participant", ["displayName", "handle", "isSelf"], label); + return { + kind: "participant", + id: common.id, + accountId: common.accountId, + network: common.network, + connectedAccountProviderId: common.provenance.connectedAccountProviderId, + value: Object.freeze({ + schemaVersion: 1, + kind: "participant", + ...common, + displayName: nullableText(source.displayName, `${label}.displayName`, MAX_SHORT_TEXT_BYTES), + handle: nullableText(source.handle, `${label}.handle`, MAX_SHORT_TEXT_BYTES), + isSelf: boolean(source.isSelf, `${label}.isSelf`), + }), + }; +} + +function parseConversation(source: JsonRecord, label: string): ParsedRecord { + const common = parseCommon(source, "conversation", [ + "type", "title", "participantIds", "participantsComplete", "startedAt", "lastMessageAt", + ], label); + const startedAt = nullableTimestamp(source.startedAt, `${label}.startedAt`); + const lastMessageAt = nullableTimestamp(source.lastMessageAt, `${label}.lastMessageAt`); + if (startedAt !== null && lastMessageAt !== null && startedAt > lastMessageAt) { + fail(`${label}.startedAt must not be after lastMessageAt`); + } + return { + kind: "conversation", + id: common.id, + accountId: common.accountId, + network: common.network, + connectedAccountProviderId: common.provenance.connectedAccountProviderId, + value: Object.freeze({ + schemaVersion: 1, + kind: "conversation", + ...common, + type: oneOf(source.type, ["direct", "group", "channel", "unknown"] as const, `${label}.type`), + title: nullableText(source.title, `${label}.title`, MAX_SHORT_TEXT_BYTES), + participantIds: uniqueIdentifiers(source.participantIds, `${label}.participantIds`, MAX_PARTICIPANTS), + participantsComplete: nullableBoolean( + source.participantsComplete, + `${label}.participantsComplete`, + ), + startedAt, + lastMessageAt, + }), + }; +} + +function parseReply(value: unknown, label: string): BeeperMessageLikeMeMessage["replyTo"] { + if (value === null) return null; + const source = foreignRecord(value, label); + exactKeys(source, ["messageId", "providerId"], label); + return Object.freeze({ + messageId: source.messageId === null ? null : identifier(source.messageId, `${label}.messageId`), + providerId: identifier(source.providerId, `${label}.providerId`), + }); +} + +function parseEdit(value: unknown, label: string): BeeperMessageLikeMeMessage["edit"] { + if (value === null) return null; + const source = foreignRecord(value, label); + if (source.kind === "in-place") { + exactKeys(source, ["kind", "editedAt", "providerRevision"], label); + return Object.freeze({ + kind: "in-place", + editedAt: timestamp(source.editedAt, `${label}.editedAt`), + providerRevision: identifier(source.providerRevision, `${label}.providerRevision`), + }); + } + if (source.kind !== "replacement") fail(`${label}.kind is unsupported`); + exactKeys(source, [ + "kind", + "replacesMessageId", + "replacesProviderId", + "editedAt", + "providerRevision", + ], label); + return Object.freeze({ + kind: "replacement", + replacesMessageId: source.replacesMessageId === null + ? null + : identifier(source.replacesMessageId, `${label}.replacesMessageId`), + replacesProviderId: identifier( + source.replacesProviderId, + `${label}.replacesProviderId`, + ), + editedAt: timestamp(source.editedAt, `${label}.editedAt`), + providerRevision: identifier(source.providerRevision, `${label}.providerRevision`), + }); +} + +function parseDeletion(value: unknown, label: string): BeeperMessageLikeMeMessage["deletion"] { + if (value === null) return null; + const source = foreignRecord(value, label); + exactKeys(source, ["state", "observedAt", "providerRevision"], label); + return Object.freeze({ + state: oneOf(source.state, [ + "revoked", "deleted-for-me", "revoked-and-deleted-for-me", + ] as const, `${label}.state`), + observedAt: timestamp(source.observedAt, `${label}.observedAt`), + providerRevision: nullableText(source.providerRevision, `${label}.providerRevision`, MAX_IDENTIFIER_BYTES), + }); +} + +function parseAttachments(value: unknown, label: string): readonly BeeperMessageLikeMeAttachment[] { + return Object.freeze(array(value, label, MAX_ATTACHMENTS).map((item, index) => { + const itemLabel = `${label}[${String(index)}]`; + const source = foreignRecord(item, itemLabel); + exactKeys(source, ["kind", "mimeType", "name", "sizeBytes"], itemLabel); + const name = nullableText(source.name, `${itemLabel}.name`, MAX_SHORT_TEXT_BYTES); + if (name !== null && (name === "." || name === ".." || name.includes("/") || name.includes("\\"))) { + fail(`${itemLabel}.name must be a base name, not a local path`); + } + return Object.freeze({ + kind: oneOf(source.kind, [ + "audio", "document", "image", "link", "sticker", "video", "unknown", + ] as const, `${itemLabel}.kind`), + mimeType: nullableText(source.mimeType, `${itemLabel}.mimeType`, 256), + name, + sizeBytes: nullableInteger(source.sizeBytes, `${itemLabel}.sizeBytes`), + }); + })); +} + +function parseMessage(source: JsonRecord, label: string): ParsedRecord { + const common = parseCommon(source, "message", [ + "conversationId", + "senderParticipantId", + "direction", + "sentAt", + "sortKey", + "body", + "bodyTruncated", + "replyTo", + "edit", + "deletion", + "attachments", + ], label); + const sentAt = timestamp(source.sentAt, `${label}.sentAt`); + const edit = parseEdit(source.edit, `${label}.edit`); + if (edit !== null && edit.editedAt < sentAt) { + fail(`${label}.edit.editedAt must not be before sentAt`); + } + const deletion = parseDeletion(source.deletion, `${label}.deletion`); + const body = nullableText(source.body, `${label}.body`, MAX_BODY_BYTES); + if (deletion !== null && body !== null) { + fail(`${label}.body must be null when deletion is present`); + } + return { + kind: "message", + id: common.id, + accountId: common.accountId, + network: common.network, + connectedAccountProviderId: common.provenance.connectedAccountProviderId, + value: Object.freeze({ + schemaVersion: 1, + kind: "message", + ...common, + conversationId: identifier(source.conversationId, `${label}.conversationId`), + senderParticipantId: source.senderParticipantId === null + ? null + : identifier(source.senderParticipantId, `${label}.senderParticipantId`), + direction: oneOf(source.direction, ["incoming", "outgoing", "unknown"] as const, `${label}.direction`), + sentAt, + sortKey: identifier(source.sortKey, `${label}.sortKey`), + body, + bodyTruncated: nullableBoolean(source.bodyTruncated, `${label}.bodyTruncated`), + replyTo: parseReply(source.replyTo, `${label}.replyTo`), + edit, + deletion, + attachments: parseAttachments(source.attachments, `${label}.attachments`), + }), + }; +} + +function parseReaction(source: JsonRecord, label: string): ParsedRecord { + const common = parseCommon(source, "reaction", [ + "messageId", "messageProviderId", "participantId", "body", "reactedAt", "state", + ], label); + return { + kind: "reaction", + id: common.id, + accountId: common.accountId, + network: common.network, + connectedAccountProviderId: common.provenance.connectedAccountProviderId, + value: Object.freeze({ + schemaVersion: 1, + kind: "reaction", + ...common, + messageId: source.messageId === null + ? null + : identifier(source.messageId, `${label}.messageId`), + messageProviderId: identifier( + source.messageProviderId, + `${label}.messageProviderId`, + ), + participantId: source.participantId === null + ? null + : identifier(source.participantId, `${label}.participantId`), + body: text(source.body, `${label}.body`, MAX_SHORT_TEXT_BYTES), + reactedAt: nullableTimestamp(source.reactedAt, `${label}.reactedAt`), + state: oneOf(source.state, ["active", "removed"] as const, `${label}.state`), + }), + }; +} + +function parseTombstone(source: JsonRecord, label: string): ParsedRecord { + const common = parseCommon(source, "tombstone", [ + "entityKind", "entityId", "entityProviderId", "deletedAt", "scope", "providerRevision", + ], label); + return { + kind: "tombstone", + id: common.id, + accountId: common.accountId, + network: common.network, + connectedAccountProviderId: common.provenance.connectedAccountProviderId, + value: Object.freeze({ + schemaVersion: 1, + kind: "tombstone", + ...common, + entityKind: oneOf(source.entityKind, [ + "conversation", "message", "reaction", + ] as const, `${label}.entityKind`), + entityId: source.entityId === null ? null : identifier(source.entityId, `${label}.entityId`), + entityProviderId: identifier(source.entityProviderId, `${label}.entityProviderId`), + deletedAt: timestamp(source.deletedAt, `${label}.deletedAt`), + scope: oneOf(source.scope, ["remote", "local", "unknown"] as const, `${label}.scope`), + providerRevision: nullableText(source.providerRevision, `${label}.providerRevision`, MAX_IDENTIFIER_BYTES), + }), + }; +} + +function parseRecord(value: unknown, index: number): ParsedRecord { + const label = `record[${String(index)}]`; + const source = foreignRecord(value, label); + switch (source.kind) { + case "account": return parseAccount(source, label); + case "participant": return parseParticipant(source, label); + case "conversation": return parseConversation(source, label); + case "message": return parseMessage(source, label); + case "reaction": return parseReaction(source, label); + case "tombstone": return parseTombstone(source, label); + default: return fail(`${label}.kind is unsupported`); + } +} + +function bundleGraphFact(record: BeeperMessageLikeMeRecord): BundleGraphFact { + const common = { + id: record.id, + accountId: record.accountId, + providerId: record.provenance.providerId, + } as const; + switch (record.kind) { + case "account": + return Object.freeze({ + kind: record.kind, + ...common, + network: record.network, + selfParticipantId: record.selfParticipantId, + }); + case "participant": + return Object.freeze({ + kind: record.kind, + ...common, + isSelf: record.isSelf, + }); + case "conversation": + return Object.freeze({ + kind: record.kind, + ...common, + type: record.type, + participantIds: record.participantIds, + participantsComplete: record.participantsComplete, + }); + case "message": + return Object.freeze({ + kind: record.kind, + ...common, + conversationId: record.conversationId, + senderParticipantId: record.senderParticipantId, + direction: record.direction, + replyTo: record.replyTo, + edit: record.edit, + }); + case "reaction": + return Object.freeze({ + kind: record.kind, + ...common, + messageId: record.messageId, + messageProviderId: record.messageProviderId, + participantId: record.participantId, + }); + case "tombstone": + return Object.freeze({ + kind: record.kind, + ...common, + entityKind: record.entityKind, + entityId: record.entityId, + entityProviderId: record.entityProviderId, + }); + } +} + +function graphRecords( + inventory: BundleGraphInventory, + kind: Kind, +): ReadonlyMap> { + return (inventory.get(kind) ?? new Map()) as ReadonlyMap< + string, + Extract + >; +} + +function assertSameAccount( + fact: BundleGraphFact | undefined, + accountId: string, + label: string, +): asserts fact is BundleGraphFact { + if (fact === undefined || fact.accountId !== accountId) { + fail(`${label} does not resolve inside its account realm`); + } +} + +function graphProviderCoordinate(accountId: string, providerId: string): string { + return sha256(canonicalJson([accountId, providerId])); +} + +function providerGraphRecords( + records: ReadonlyMap< + string, + Extract + >, +): ReadonlyMap< + string, + Extract +> { + const providers = new Map< + string, + Extract + >(); + for (const fact of records.values()) { + providers.set( + graphProviderCoordinate(fact.accountId, fact.providerId), + fact, + ); + } + return providers; +} + +function validateBundleGraph(inventory: BundleGraphInventory): void { + const accounts = graphRecords(inventory, "account"); + const participants = graphRecords(inventory, "participant"); + const conversations = graphRecords(inventory, "conversation"); + const messages = graphRecords(inventory, "message"); + const reactions = graphRecords(inventory, "reaction"); + const messagesByProvider = providerGraphRecords(messages); + const conversationRosters = new Map( + [...conversations.values()].map((conversation) => [ + conversation.id, + new Set(conversation.participantIds), + ]), + ); + const providerInventories = new Map>( + ARTIFACTS.map(({ kind }) => { + const records = graphRecords(inventory, kind); + return [kind, providerGraphRecords(records)]; + }), + ); + const resolveMessageTarget = ( + accountId: string, + localId: string | null, + providerId: string, + label: string, + ): Extract | undefined => { + const localTarget = localId === null ? undefined : messages.get(localId); + if (localId !== null) { + assertSameAccount(localTarget, accountId, label); + if (localTarget.kind !== "message" || localTarget.providerId !== providerId) { + fail(`${label} provider coordinate does not match`); + } + } + const providerTarget = messagesByProvider.get( + graphProviderCoordinate(accountId, providerId), + ); + if ( + localTarget !== undefined + && providerTarget !== undefined + && localTarget.id !== providerTarget.id + ) fail(`${label} local and provider coordinates disagree`); + return localTarget ?? providerTarget; + }; + + const stableAccountRealms = new Set(); + for (const account of accounts.values()) { + const self = participants.get(account.selfParticipantId); + assertSameAccount(self, account.accountId, "account self participant"); + if (self.kind !== "participant" || !self.isSelf) { + fail("account self participant is not marked as self"); + } + const stableRealm = sha256(canonicalJson([ + "beeper", + account.providerId, + self.providerId, + ])); + if (stableAccountRealms.has(stableRealm)) { + fail("account records repeat one stable provider realm"); + } + stableAccountRealms.add(stableRealm); + } + for (const participant of participants.values()) { + const account = accounts.get(participant.accountId); + assertSameAccount(account, participant.accountId, "participant account"); + if (participant.isSelf && account.kind === "account" + && account.selfParticipantId !== participant.id) { + fail("account realm contains an unreferenced self participant"); + } + } + for (const conversation of conversations.values()) { + const account = accounts.get(conversation.accountId); + assertSameAccount(account, conversation.accountId, "conversation account"); + for (const participantId of conversation.participantIds) { + assertSameAccount( + participants.get(participantId), + conversation.accountId, + "conversation participant", + ); + } + if (conversation.type === "direct" && conversation.participantsComplete === true) { + const roster = conversation.participantIds.map((participantId) => { + const participant = participants.get(participantId); + if (participant?.kind !== "participant") { + return fail("complete direct conversation participant has the wrong record kind"); + } + return participant; + }); + const selfCount = roster.filter((participant) => participant.isSelf).length; + if ( + account.kind !== "account" + || conversation.participantIds.length !== 2 + || selfCount !== 1 + || !conversationRosters.get(conversation.id)?.has(account.selfParticipantId) + ) { + fail("complete direct conversation must contain exactly one self participant and one peer"); + } + } + } + const replacementEdges = new Map(); + const replacedProviderCoordinates = new Set(); + for (const message of messages.values()) { + const conversation = conversations.get(message.conversationId); + assertSameAccount(conversation, message.accountId, "message conversation"); + if (conversation.kind !== "conversation") { + fail("message conversation has the wrong record kind"); + } + if (message.senderParticipantId !== null) { + const sender = participants.get(message.senderParticipantId); + assertSameAccount(sender, message.accountId, "message sender"); + if (sender.kind !== "participant") fail("message sender has the wrong record kind"); + const expectedDirection = sender.isSelf ? "outgoing" : "incoming"; + if (message.direction !== expectedDirection) { + fail("message direction conflicts with its sender participant"); + } + if ( + conversation.participantsComplete === true + && !conversationRosters.get(conversation.id)?.has(sender.id) + ) fail("message sender is absent from the complete conversation roster"); + } + if (message.replyTo !== null) { + const target = resolveMessageTarget( + message.accountId, + message.replyTo.messageId, + message.replyTo.providerId, + "message reply target", + ); + if (target !== undefined) { + if (target.id === message.id) fail("message must not reply to itself"); + if (target.conversationId !== message.conversationId) { + fail("message reply target belongs to a different conversation"); + } + } + } + if (message.edit?.kind === "replacement") { + if (message.edit.replacesProviderId === message.providerId) { + fail("replacement edit must not replace its own provider coordinate"); + } + const targetCoordinate = graphProviderCoordinate( + message.accountId, + message.edit.replacesProviderId, + ); + if (replacedProviderCoordinates.has(targetCoordinate)) { + fail("one provider message has more than one replacement"); + } + replacedProviderCoordinates.add(targetCoordinate); + const target = resolveMessageTarget( + message.accountId, + message.edit.replacesMessageId, + message.edit.replacesProviderId, + "replacement edit target", + ); + if (target !== undefined) { + if (target.conversationId !== message.conversationId) { + fail("replacement edit target belongs to a different conversation"); + } + replacementEdges.set(message.id, target.id); + } + } + } + const completedReplacementNodes = new Set(); + for (const start of replacementEdges.keys()) { + if (completedReplacementNodes.has(start)) continue; + const path: string[] = []; + const positions = new Map(); + let current: string | undefined = start; + while (current !== undefined && !completedReplacementNodes.has(current)) { + if (positions.has(current)) fail("replacement edit graph contains a cycle"); + positions.set(current, path.length); + path.push(current); + current = replacementEdges.get(current); + } + for (const id of path) completedReplacementNodes.add(id); + } + for (const reaction of reactions.values()) { + let participant: Extract + | undefined; + if (reaction.participantId !== null) { + participant = participants.get(reaction.participantId); + assertSameAccount( + participant, + reaction.accountId, + "reaction participant", + ); + } + const target = resolveMessageTarget( + reaction.accountId, + reaction.messageId, + reaction.messageProviderId, + "reaction message target", + ); + if (participant !== undefined && target !== undefined) { + const conversation = conversations.get(target.conversationId); + assertSameAccount( + conversation, + reaction.accountId, + "reaction target conversation", + ); + if ( + conversation.kind === "conversation" + && conversation.participantsComplete === true + && !conversationRosters.get(conversation.id)?.has(participant.id) + ) fail("reaction participant is absent from the complete conversation roster"); + } + } + for (const tombstone of graphRecords(inventory, "tombstone").values()) { + const providerTargets = providerInventories.get(tombstone.entityKind); + if (providerTargets === undefined) fail("internal provider inventory is incomplete"); + const providerTarget = providerTargets.get(graphProviderCoordinate( + tombstone.accountId, + tombstone.entityProviderId, + )); + if (tombstone.entityId !== null) { + const target = graphRecords(inventory, tombstone.entityKind).get( + tombstone.entityId, + ); + assertSameAccount(target, tombstone.accountId, "tombstone entity target"); + if (target.providerId !== tombstone.entityProviderId) { + fail("tombstone entity provider coordinate does not match"); + } + if (providerTarget !== undefined && target.id !== providerTarget.id) { + fail("tombstone local and provider coordinates disagree"); + } + } + } +} + +function parseDescriptor(value: unknown): ParsedDescriptor { + const descriptor = foreignRecord(value, "source descriptor"); + exactKeys(descriptor, ["source", "provider"], "source descriptor"); + const source = foreignRecord(descriptor.source, "source descriptor.source"); + exactKeys(source, ["id", "version"], "source descriptor.source"); + if (source.id !== "beeper-local") fail("source descriptor.source.id must equal beeper-local"); + const provider = foreignRecord(descriptor.provider, "source descriptor.provider"); + exactKeys(provider, ["id", "version"], "source descriptor.provider"); + if (provider.id !== "beeper") fail("source descriptor.provider.id must equal beeper"); + return Object.freeze({ + source: Object.freeze({ id: "beeper-local", version: version(source.version, "source descriptor.source.version") }), + provider: Object.freeze({ id: "beeper", version: version(provider.version, "source descriptor.provider.version") }), + }); +} + +function parseCompletion(value: unknown): ParsedCompletion { + const source = foreignRecord(value, "source completion"); + exactKeys(source, ["completeness", "warnings"], "source completion"); + const completeness = foreignRecord(source.completeness, "source completion.completeness"); + exactKeys( + completeness, + ["kind", "reason", "observedFrom", "observedThrough"], + "source completion.completeness", + ); + const observedFrom = nullableTimestamp( + completeness.observedFrom, + "source completion.completeness.observedFrom", + ); + const observedThrough = nullableTimestamp( + completeness.observedThrough, + "source completion.completeness.observedThrough", + ); + if (observedFrom !== null && observedThrough !== null && observedFrom > observedThrough) { + fail("source completion observedFrom must not be after observedThrough"); + } + const warnings = array(source.warnings, "source completion.warnings", MAX_WARNING_CODES) + .map((warning, index) => token(warning, `source completion.warnings[${String(index)}]`)); + if (new Set(warnings).size !== warnings.length) fail("source completion.warnings must not contain duplicates"); + return Object.freeze({ + completeness: Object.freeze({ + kind: oneOf(completeness.kind, ["bounded-local", "truncated", "unknown"] as const, "source completion.completeness.kind"), + reason: completeness.reason === null + ? null + : token(completeness.reason, "source completion.completeness.reason"), + observedFrom, + observedThrough, + }), + warnings: Object.freeze(warnings), + }); +} + +function parseLimits(value: unknown): BeeperMessageLikeMeExportLimits { + if (value === undefined) return HARD_LIMITS; + const source = foreignRecord(value, "export limits"); + const permitted = ["maxRecords", "maxRecordBytes", "maxTotalBytes"]; + if (Object.keys(source).some((key) => !permitted.includes(key))) { + fail(`export limits may contain only: ${permitted.join(", ")}`); + } + const parse = (key: keyof BeeperMessageLikeMeExportLimits): number => { + const candidate = source[key] ?? HARD_LIMITS[key]; + const parsed = integer(candidate, `export limits.${key}`, HARD_LIMITS[key]); + if (parsed === 0) fail(`export limits.${key} must be greater than zero`); + return parsed; + }; + return Object.freeze({ + maxRecords: parse("maxRecords"), + maxRecordBytes: parse("maxRecordBytes"), + maxTotalBytes: parse("maxTotalBytes"), + }); +} + +function now(clock: (() => Date) | undefined, label: string): string { + const value = (clock ?? (() => new Date()))(); + if (!(value instanceof Date) || !Number.isFinite(value.getTime())) { + return fail(`${label} clock value must be a valid Date`); + } + return value.toISOString(); +} + +async function assertAbsent(path: string, label: string): Promise { + try { + await lstat(path); + } catch (error) { + if (isErrno(error, "ENOENT")) return; + throw error; + } + fail(`${label} already exists`); +} + +async function validateOutputRoot(outputRoot: unknown): Promise<{ + readonly outputRoot: string; + readonly parent: string; + readonly parentDevice: number; + readonly parentInode: number; +}> { + if (typeof outputRoot !== "string" || !isAbsolute(outputRoot) || resolve(outputRoot) !== outputRoot) { + return fail("outputRoot must be a normalized absolute path"); + } + if (outputRoot === sep || utf8Length(outputRoot) > 4_096 || basename(outputRoot).includes("\0")) { + return fail("outputRoot is unsafe"); + } + const parent = dirname(outputRoot); + const parentMetadata = await lstat(parent); + if (!parentMetadata.isDirectory() || parentMetadata.isSymbolicLink()) { + return fail("outputRoot parent must be a real directory"); + } + if (await realpath(parent) !== parent) { + return fail("outputRoot parent path must not traverse a symbolic link"); + } + const uid = process.getuid?.(); + if (uid === undefined) fail("private exports require a POSIX user identity"); + if (parentMetadata.uid !== uid) fail("outputRoot parent must be owned by the current user"); + if ((parentMetadata.mode & 0o022) !== 0) { + fail("outputRoot parent must not be writable by the group or other users"); + } + await assertAbsent(outputRoot, "outputRoot"); + return { + outputRoot, + parent, + parentDevice: parentMetadata.dev, + parentInode: parentMetadata.ino, + }; +} + +async function assertParentUnchanged(snapshot: Awaited>): Promise { + const current = await lstat(snapshot.parent); + if ( + !current.isDirectory() + || current.isSymbolicLink() + || current.dev !== snapshot.parentDevice + || current.ino !== snapshot.parentInode + || await realpath(snapshot.parent) !== snapshot.parent + ) { + fail("outputRoot parent changed during export setup"); + } +} + +async function assertPrivateDirectory(path: string): Promise { + const uid = process.getuid?.(); + const metadata = await lstat(path); + if ( + !metadata.isDirectory() + || metadata.isSymbolicLink() + || uid === undefined + || metadata.uid !== uid + || (metadata.mode & 0o777) !== PRIVATE_DIRECTORY_MODE + || await realpath(path) !== path + ) { + fail(`${path} is not the expected private physical directory`); + } +} + +async function assertPrivateFile(path: string, expectedBytes: number): Promise { + const uid = process.getuid?.(); + const metadata = await lstat(path); + if ( + !metadata.isFile() + || metadata.isSymbolicLink() + || uid === undefined + || metadata.uid !== uid + || (metadata.mode & 0o777) !== PRIVATE_FILE_MODE + || metadata.size !== expectedBytes + ) { + fail(`${path} is not the expected private regular file`); + } +} + +async function createWriter(staging: string, kind: RecordKind, fileName: string): Promise { + const partPath = resolve(staging, `${fileName}.part`); + if (!partPath.startsWith(`${staging}${sep}`)) fail("internal artifact path escaped staging"); + const handle = await open( + partPath, + constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, + PRIVATE_FILE_MODE, + ); + try { + await handle.chmod(PRIVATE_FILE_MODE); + const metadata = await handle.stat(); + if (!metadata.isFile() || (metadata.mode & 0o777) !== PRIVATE_FILE_MODE) { + fail(`could not create private staging file ${fileName}`); + } + } catch (error) { + await handle.close(); + throw error; + } + return { + kind, + fileName, + partPath, + handle, + hash: createHash("sha256"), + records: 0, + bytes: 0, + closed: false, + }; +} + +async function writeAll(handle: FileHandle, bytes: Uint8Array): Promise { + let offset = 0; + while (offset < bytes.byteLength) { + const result = await handle.write(bytes, offset, bytes.byteLength - offset, null); + if (result.bytesWritten === 0) fail("staging file stopped accepting bytes"); + offset += result.bytesWritten; + } +} + +async function writeRecord( + writer: ArtifactWriter, + value: Readonly>, + limits: BeeperMessageLikeMeExportLimits, + totalBytes: number, +): Promise { + const bytes = Buffer.from(`${canonicalJson(value)}\n`, "utf8"); + if (bytes.byteLength > limits.maxRecordBytes) { + fail(`one ${writer.kind} record exceeds the configured byte bound`); + } + if (totalBytes + bytes.byteLength > limits.maxTotalBytes) { + fail("bundle records exceed the configured total byte bound"); + } + await writeAll(writer.handle, bytes); + writer.hash.update(bytes); + writer.records += 1; + writer.bytes += bytes.byteLength; + return totalBytes + bytes.byteLength; +} + +async function closeWriter(writer: ArtifactWriter): Promise { + if (writer.closed) return; + writer.closed = true; + await writer.handle.close(); +} + +async function finalizeWriter(writer: ArtifactWriter, outputRoot: string): Promise { + await writer.handle.sync(); + const opened = await writer.handle.stat(); + if (!opened.isFile() || opened.size !== writer.bytes || (opened.mode & 0o777) !== PRIVATE_FILE_MODE) { + fail(`${writer.fileName} changed before finalization`); + } + await closeWriter(writer); + await assertPrivateFile(writer.partPath, writer.bytes); + const finalPath = resolve(outputRoot, writer.fileName); + await assertAbsent(finalPath, writer.fileName); + await rename(writer.partPath, finalPath); + await assertPrivateFile(finalPath, writer.bytes); + return Object.freeze({ + path: writer.fileName, + mediaType: "application/x-ndjson", + recordKind: writer.kind, + records: writer.records, + bytes: writer.bytes, + sha256: writer.hash.digest("hex"), + }); +} + +async function writeManifest( + staging: string, + outputRoot: string, + manifest: BeeperMessageLikeMeManifest, +): Promise { + const partPath = resolve(staging, "manifest.json.part"); + const finalPath = resolve(outputRoot, "manifest.json"); + const bytes = Buffer.from(`${canonicalJson(manifest)}\n`, "utf8"); + const handle = await open( + partPath, + constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, + PRIVATE_FILE_MODE, + ); + try { + await handle.chmod(PRIVATE_FILE_MODE); + await writeAll(handle, bytes); + await handle.sync(); + const metadata = await handle.stat(); + if (!metadata.isFile() || metadata.size !== bytes.byteLength || (metadata.mode & 0o777) !== PRIVATE_FILE_MODE) { + fail("manifest changed before finalization"); + } + } finally { + await handle.close(); + } + await assertPrivateFile(partPath, bytes.byteLength); + await assertAbsent(finalPath, "manifest.json"); + await rename(partPath, finalPath); + await assertPrivateFile(finalPath, bytes.byteLength); + return sha256(bytes.toString("utf8")); +} + +function assertSource(source: BeeperMessageLikeMeExportSource): void { + if ( + typeof source !== "object" + || source === null + || typeof source.completion !== "function" + || typeof source.records !== "object" + || source.records === null + || typeof source.records[Symbol.asyncIterator] !== "function" + ) { + fail("source must expose an async record stream and completion function"); + } +} + +function throwIfAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted === true) fail("export was aborted"); +} + +/** + * Writes a private, local Message Like Me interchange bundle. The source owns + * provider access; this function accepts only bounded foreign records and + * never invokes Beeper, follows media references, or receives credentials. + * + * A successful bundle has `manifest.json`. A failed export deliberately leaves + * a mode-0700 directory without a manifest so it cannot be mistaken for a + * complete bundle or silently overwrite a later retry. + */ +export async function exportBeeperMessageLikeMeBundle( + request: BeeperMessageLikeMeExportRequest, +): Promise { + assertSource(request.source); + const descriptor = parseDescriptor(request.source.descriptor); + const limits = parseLimits(request.limits); + throwIfAborted(request.signal); + const startedAt = now(request.clock, "startedAt"); + const output = await validateOutputRoot(request.outputRoot); + await assertParentUnchanged(output); + await mkdir(output.outputRoot, { mode: PRIVATE_DIRECTORY_MODE }); + await chmod(output.outputRoot, PRIVATE_DIRECTORY_MODE); + await assertPrivateDirectory(output.outputRoot); + + const staging = resolve(output.outputRoot, ".message-like-me-staging"); + await mkdir(staging, { mode: PRIVATE_DIRECTORY_MODE }); + await chmod(staging, PRIVATE_DIRECTORY_MODE); + await assertPrivateDirectory(staging); + + const writers = new Map(); + try { + for (const artifact of ARTIFACTS) { + writers.set(artifact.kind, await createWriter(staging, artifact.kind, artifact.fileName)); + } + + const graphInventory = new Map>( + ARTIFACTS.map((artifact) => [artifact.kind, new Map()]), + ); + const providerCoordinates = new Map>( + ARTIFACTS.map((artifact) => [artifact.kind, new Set()]), + ); + const accountRealms = new Map(); + const referencedAccountRealms = new Map(); + let totalRecords = 0; + let totalBytes = 0; + + for await (const candidate of request.source.records) { + throwIfAborted(request.signal); + totalRecords += 1; + if (totalRecords > limits.maxRecords) fail("record stream exceeds the configured record bound"); + const parsed = parseRecord(candidate, totalRecords - 1); + const records = graphInventory.get(parsed.kind); + const coordinates = providerCoordinates.get(parsed.kind); + if (records === undefined || coordinates === undefined) { + fail("internal record-kind inventory is incomplete"); + } + if (records.has(parsed.id)) { + fail(`${parsed.kind} record repeats a bundle-local identity`); + } + const providerCoordinate = graphProviderCoordinate( + parsed.value.accountId, + parsed.value.provenance.providerId, + ); + if (coordinates.has(providerCoordinate)) { + fail(`${parsed.kind} record repeats an account-scoped provider identity`); + } + records.set(parsed.id, bundleGraphFact(parsed.value)); + coordinates.add(providerCoordinate); + if (parsed.kind === "account" && records.size > MAX_CONNECTED_ACCOUNTS) { + fail("record stream exceeds the connected-account bound"); + } + + if ( + parsed.accountId !== undefined + && parsed.network !== undefined + && parsed.connectedAccountProviderId !== undefined + ) { + const realm = Object.freeze({ + network: parsed.network, + connectedAccountProviderId: parsed.connectedAccountProviderId, + }); + if (parsed.kind === "account") { + const current = accountRealms.get(parsed.accountId); + if ( + current !== undefined + && ( + current.network !== realm.network + || current.connectedAccountProviderId !== realm.connectedAccountProviderId + ) + ) { + fail("account record has conflicting source identity"); + } + accountRealms.set(parsed.accountId, realm); + } else { + const current = referencedAccountRealms.get(parsed.accountId); + if ( + current !== undefined + && ( + current.network !== realm.network + || current.connectedAccountProviderId !== realm.connectedAccountProviderId + ) + ) { + fail("account is referenced with conflicting source identity"); + } + referencedAccountRealms.set(parsed.accountId, realm); + } + } + + const writer = writers.get(parsed.kind); + if (writer === undefined) fail("internal artifact writer is missing"); + totalBytes = await writeRecord(writer, parsed.value, limits, totalBytes); + } + + for (const [accountId, realm] of referencedAccountRealms) { + const accountRealm = accountRealms.get(accountId); + if (accountRealm === undefined) { + fail("record stream references a missing account"); + } + if ( + accountRealm.network !== realm.network + || accountRealm.connectedAccountProviderId !== realm.connectedAccountProviderId + ) { + fail("account does not match its referenced source identity"); + } + } + + validateBundleGraph(graphInventory); + + throwIfAborted(request.signal); + const completion = parseCompletion(await request.source.completion()); + const finishedAt = now(request.clock, "finishedAt"); + if (finishedAt < startedAt) fail("finishedAt must not be before startedAt"); + + const artifacts: BeeperMessageLikeMeArtifact[] = []; + for (const artifact of ARTIFACTS) { + const writer = writers.get(artifact.kind); + if (writer === undefined) fail("internal artifact writer is missing during finalization"); + artifacts.push(await finalizeWriter(writer, output.outputRoot)); + } + + const counts = Object.freeze(Object.fromEntries( + artifacts.map((artifact) => [artifact.recordKind, artifact.records]), + )) as Readonly>; + const manifestProjection = Object.freeze({ + schemaVersion: BEEPER_MESSAGE_LIKE_ME_SCHEMA_VERSION, + format: "message-like-me.local-message-bundle", + source: descriptor.source, + provider: descriptor.provider, + timestamps: Object.freeze({ + startedAt, + finishedAt, + createdAt: finishedAt, + }), + completeness: completion.completeness, + warnings: completion.warnings, + privacy: Object.freeze({ + classification: "private-local", + attachments: "metadata-only", + providerUrls: "excluded", + credentials: "excluded", + }), + counts, + artifacts: Object.freeze(artifacts), + }); + const manifest: BeeperMessageLikeMeManifest = Object.freeze({ + ...manifestProjection, + integrity: Object.freeze({ + algorithm: "sha256", + bundleSha256: sha256(canonicalJson(manifestProjection)), + }), + }); + const manifestSha256 = await writeManifest(staging, output.outputRoot, manifest); + await rmdir(staging); + await assertPrivateDirectory(output.outputRoot); + return Object.freeze({ + outputRoot: output.outputRoot, + manifestPath: resolve(output.outputRoot, "manifest.json"), + manifestSha256, + manifest, + }); + } finally { + await Promise.all([...writers.values()].map((writer) => closeWriter(writer))); + } +} diff --git a/src/beeper-message-like-me-golden-fixture.ts b/src/beeper-message-like-me-golden-fixture.ts new file mode 100644 index 0000000..a740c5f --- /dev/null +++ b/src/beeper-message-like-me-golden-fixture.ts @@ -0,0 +1,169 @@ +import type { BeeperMessageLikeMeExportSource } from "./beeper-message-like-me-export"; + +export const BEEPER_MESSAGE_LIKE_ME_GOLDEN_STARTED_AT = + "2026-08-21T16:00:00.000Z"; +export const BEEPER_MESSAGE_LIKE_ME_GOLDEN_FINISHED_AT = + "2026-08-21T16:00:01.000Z"; + +const observedAt = "2026-08-21T15:59:00.000Z"; +const accountId = "account:synthetic:primary"; +const connectedAccountProviderId = "beeper-account:synthetic-primary"; +const selfParticipantId = "participant:synthetic:self"; +const peerParticipantId = "participant:synthetic:peer"; +const conversationId = "conversation:synthetic:friend"; +const editedMessageId = "message:synthetic:edited"; +const editedMessageProviderId = "beeper-message:synthetic-edited"; +const deletedMessageId = "message:synthetic:deleted"; +const deletedMessageProviderId = "beeper-message:synthetic-deleted"; + +function provenance(providerId: string, providerRevision: string | null) { + return Object.freeze({ + providerId, + providerRevision, + observedAt, + connectedAccountProviderId, + }); +} + +const records = Object.freeze([{ + schemaVersion: 1, + kind: "account", + id: accountId, + accountId, + network: "synthetic", + provenance: provenance(connectedAccountProviderId, null), + displayName: "Synthetic Primary", + handle: "+15555550100", + selfParticipantId, +}, { + schemaVersion: 1, + kind: "participant", + id: selfParticipantId, + accountId, + network: "synthetic", + provenance: provenance("beeper-participant:synthetic-self", null), + displayName: "Synthetic Self", + handle: "+15555550100", + isSelf: true, +}, { + schemaVersion: 1, + kind: "participant", + id: peerParticipantId, + accountId, + network: "synthetic", + provenance: provenance("beeper-participant:synthetic-peer", null), + displayName: "Synthetic Peer", + handle: "+15555550101", + isSelf: false, +}, { + schemaVersion: 1, + kind: "conversation", + id: conversationId, + accountId, + network: "synthetic", + provenance: provenance("beeper-conversation:synthetic-friend", "chat-r1"), + type: "direct", + title: "Synthetic Friend", + participantIds: [selfParticipantId, peerParticipantId], + participantsComplete: true, + startedAt: "2026-08-21T15:50:00.000Z", + lastMessageAt: "2026-08-21T15:58:45.000Z", +}, { + schemaVersion: 1, + kind: "message", + id: editedMessageId, + accountId, + network: "synthetic", + provenance: provenance(editedMessageProviderId, "edit-r2"), + conversationId, + senderParticipantId: selfParticipantId, + direction: "outgoing", + sentAt: "2026-08-21T15:58:00.000Z", + sortKey: "00000000000000000001", + body: "edited synthetic reply", + bodyTruncated: false, + replyTo: { + messageId: null, + providerId: "beeper-message:synthetic-external-reply-target", + }, + edit: { + kind: "in-place", + editedAt: "2026-08-21T15:58:30.000Z", + providerRevision: "edit-r2", + }, + deletion: null, + attachments: [], +}, { + schemaVersion: 1, + kind: "message", + id: deletedMessageId, + accountId, + network: "synthetic", + provenance: provenance(deletedMessageProviderId, "delete-r3"), + conversationId, + senderParticipantId: peerParticipantId, + direction: "incoming", + sentAt: "2026-08-21T15:58:45.000Z", + sortKey: "00000000000000000002", + body: null, + bodyTruncated: false, + replyTo: null, + edit: null, + deletion: { + state: "revoked", + observedAt, + providerRevision: "delete-r3", + }, + attachments: [], +}, { + schemaVersion: 1, + kind: "reaction", + id: "reaction:synthetic:undated", + accountId, + network: "synthetic", + provenance: provenance("beeper-reaction:synthetic-undated", "reaction-r1"), + messageId: editedMessageId, + messageProviderId: editedMessageProviderId, + participantId: peerParticipantId, + body: "👍", + reactedAt: null, + state: "active", +}, { + schemaVersion: 1, + kind: "tombstone", + id: "tombstone:synthetic:message", + accountId, + network: "synthetic", + provenance: provenance("beeper-tombstone:synthetic-message", "delete-r3"), + entityKind: "message", + entityId: deletedMessageId, + entityProviderId: deletedMessageProviderId, + deletedAt: observedAt, + scope: "remote", + providerRevision: "delete-r3", +}] as const); + +export function createBeeperMessageLikeMeGoldenSource(): BeeperMessageLikeMeExportSource { + return Object.freeze({ + descriptor: Object.freeze({ + source: Object.freeze({ id: "beeper-local", version: "1.0.0" }), + provider: Object.freeze({ id: "beeper", version: "0.6.2" }), + }), + records: (async function* () { + for (const record of records) yield record; + })(), + completion: () => Promise.resolve(Object.freeze({ + completeness: Object.freeze({ + kind: "truncated", + reason: "explicit-source-limit", + observedFrom: "2026-08-21T15:50:00.000Z", + observedThrough: "2026-08-21T15:59:00.000Z", + }), + warnings: Object.freeze([ + "attachments-metadata-only", + "remote-history-not-claimed", + "synthetic-golden-fixture", + ]), + })), + }); +} diff --git a/src/beeper-message-like-me-source.test.ts b/src/beeper-message-like-me-source.test.ts new file mode 100644 index 0000000..f48e12f --- /dev/null +++ b/src/beeper-message-like-me-source.test.ts @@ -0,0 +1,690 @@ +import { createHash } from "node:crypto"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + symlinkSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, test } from "bun:test"; + +import type { WrenchAuth } from "./auth"; +import { exportBeeperMessageLikeMeBundle } from "./beeper-message-like-me-export"; +import { + createBeeperMessageLikeMeSource, + type BeeperExportCliInvocation, +} from "./beeper-message-like-me-source"; + +const ACCOUNT_ID = "account-beeper"; +const NETWORK_ACCOUNT_ID = "account-whatsapp"; +const SELF_ID = "@self:beeper.local"; +const CHAT_ID = "chat-synthetic"; +const SUBJECT = `beeper:local:${createHash("sha256") + .update(ACCOUNT_ID, "utf8") + .update("\0", "utf8") + .update(SELF_ID, "utf8") + .digest("hex")}`; + +function privateDirectory(prefix: string): string { + const path = realpathSync(mkdtempSync(join(tmpdir(), prefix))); + chmodSync(path, 0o700); + return path; +} + +function configStore(parent: string): string { + const path = join(parent, "beeper-config"); + mkdirSync(join(path, "targets"), { recursive: true, mode: 0o755 }); + chmodSync(path, 0o755); + writeFileSync( + join(path, "config.json"), + `${JSON.stringify({ defaultTarget: "desktop" })}\n`, + { mode: 0o600 }, + ); + writeFileSync( + join(path, "targets", "desktop.json"), + `${JSON.stringify({ + auth: { token: "fixture" }, + baseURL: "http://127.0.0.1:23380", + id: "desktop", + managed: true, + name: "Desktop", + runtime: "desktop", + type: "desktop", + })}\n`, + { mode: 0o600 }, + ); + return path; +} + +function auth(path: string): WrenchAuth { + return { + schemaVersion: 1, + id: "beeper-export-fixture", + kind: "linked-device-store", + provider: "beeper", + path, + subject: SUBJECT, + }; +} + +function accounts(): readonly unknown[] { + return [{ + accountID: ACCOUNT_ID, + bridge: { id: "beeper", provider: "cloud", type: "matrix" }, + network: "Beeper", + status: "CONNECTED", + user: { + fullName: "Fixture Self", + id: SELF_ID, + isSelf: true, + }, + }, { + accountID: NETWORK_ACCOUNT_ID, + bridge: { id: "whatsapp", provider: "cloud", type: "whatsapp" }, + network: "WhatsApp Personal", + status: "CONNECTED", + user: { + fullName: "Fixture Self", + id: "whatsapp:self", + isSelf: true, + phoneNumber: "+15550000000", + }, + }]; +} + +function chat() { + return { + accountID: NETWORK_ACCOUNT_ID, + id: CHAT_ID, + lastActivity: "2026-08-21T14:00:03.000Z", + network: "WhatsApp Personal", + participants: { + hasMore: false, + items: [{ + fullName: "Ada Fixture", + id: "whatsapp:ada", + isSelf: false, + phoneNumber: "+15550000001", + }], + total: 1, + }, + title: "Ada Fixture", + type: "single", + unreadCount: 0, + }; +} + +function messages(): readonly unknown[] { + return [{ + accountID: NETWORK_ACCOUNT_ID, + attachments: [{ + fileName: "folder/private-photo.jpg", + fileSize: 123, + id: "private-media-id", + mimeType: "image/jpeg", + srcURL: "file:///private/media/photo.jpg", + type: "img", + }], + chatID: CHAT_ID, + editedTimestamp: "2026-08-21T14:00:02.000Z", + id: "message-outgoing", + isSender: true, + linkedMessageID: "message-outside-window", + reactions: [{ + emoji: true, + id: "reaction-1", + participantID: "whatsapp:ada", + reactionKey: "👍", + }, { + emoji: false, + id: "reaction-2", + participantID: "whatsapp:ada", + reactionKey: "https://provider.invalid/private-custom-reaction", + }], + senderID: "whatsapp:self", + senderName: "Fixture Self", + sortKey: "00000000000000000001", + text: "synthetic outgoing body", + timestamp: "2026-08-21T14:00:01.000Z", + type: "TEXT", + }, { + accountID: NETWORK_ACCOUNT_ID, + chatID: CHAT_ID, + id: "message-deleted", + isDeleted: true, + isHidden: false, + isSender: false, + senderID: "whatsapp:ada", + senderName: "Ada Fixture", + sortKey: "00000000000000000002", + text: "deleted foreign body", + timestamp: "2026-08-21T14:00:03.000Z", + type: "TEXT", + }]; +} + +function writeJson(path: string, value: unknown): void { + writeFileSync(path, `${JSON.stringify(value)}\n`, { mode: 0o644 }); +} + +function fixtureExport(invocation: BeeperExportCliInvocation): Promise<{ + exitCode: number; + stdout: string; + stderr: string; +}> { + expect(invocation.arguments[0]).toBe("export"); + expect(invocation.arguments).toContain("--no-attachments"); + expect(invocation.arguments).not.toContain("--json"); + expect(invocation.arguments.indexOf("--read-only")).toBeGreaterThan(0); + expect(invocation.environment.BEEPER_READONLY).toBe("1"); + const outputIndex = invocation.arguments.indexOf("--out"); + const outputRoot = invocation.arguments[outputIndex + 1]; + if (outputRoot === undefined) throw new Error("fixture export omitted --out"); + const chatsRoot = join(outputRoot, "chats"); + const chatRoot = join(chatsRoot, CHAT_ID); + mkdirSync(join(chatRoot, "attachments"), { recursive: true, mode: 0o755 }); + const accountValues = accounts(); + const chatValues = [chat()]; + const messageValues = messages(); + writeJson(join(outputRoot, "accounts.json"), accountValues); + writeJson(join(outputRoot, "chats.json"), chatValues); + writeJson(join(chatRoot, "chat.json"), chat()); + writeJson(join(chatRoot, "messages.json"), messageValues); + writeFileSync( + join(chatRoot, "messages.markdown"), + "private duplicate markdown\n", + { mode: 0o644 }, + ); + writeFileSync( + join(chatRoot, "messages.html"), + "

private duplicate html

\n", + { mode: 0o644 }, + ); + const createdAt = "2026-08-21T13:59:00.000Z"; + const completedAt = "2026-08-21T14:01:00.000Z"; + writeJson(join(outputRoot, ".beeper-export-state.json"), { + chats: { + [CHAT_ID]: { + attachmentCount: 0, + complete: true, + cursor: null, + messageCount: messageValues.length, + startedAt: createdAt, + updatedAt: completedAt, + }, + }, + completedChatIDs: [CHAT_ID], + createdAt, + exportVersion: 1, + }); + writeJson(join(outputRoot, "manifest.json"), { + accounts: accountValues, + attachmentCount: 0, + chatCount: chatValues.length, + completedAt, + createdAt, + messageCount: messageValues.length, + version: 1, + }); + return Promise.resolve({ + exitCode: 0, + stdout: "Exported 1 chats, 2 messages, 0 attachments\n", + stderr: "", + }); +} + +function invocationOutputRoot(invocation: BeeperExportCliInvocation): string { + const outputIndex = invocation.arguments.indexOf("--out"); + const outputRoot = invocation.arguments[outputIndex + 1]; + if (outputRoot === undefined) throw new Error("fixture export omitted --out"); + return outputRoot; +} + +function ndjson(path: string): readonly Record[] { + return readFileSync(path, "utf8").trim().split("\n").filter(Boolean) + .map((line) => JSON.parse(line) as Record); +} + +describe("Beeper Message Like Me source", () => { + test("converts one private official export without media or duplicate renderings", async () => { + const parent = privateDirectory("wrench-beeper-source-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + let removed = false; + try { + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + removed = true; + }, + runExport: fixtureExport, + }, + }); + const result = await exportBeeperMessageLikeMeBundle({ + outputRoot: output, + source, + clock: (() => { + const values = [ + new Date("2026-08-21T14:02:00.000Z"), + new Date("2026-08-21T14:03:00.000Z"), + ]; + return () => values.shift() ?? new Date("invalid"); + })(), + }); + + expect(removed).toBeTrue(); + expect(existsSync(working)).toBeFalse(); + expect(result.manifest.completeness).toEqual({ + kind: "bounded-local", + reason: "desktop-local-export", + observedFrom: "2026-08-21T14:00:01.000Z", + observedThrough: "2026-08-21T14:00:03.000Z", + }); + expect(result.manifest.counts).toEqual({ + account: 2, + participant: 3, + conversation: 1, + message: 2, + reaction: 2, + tombstone: 1, + }); + expect(result.manifest.warnings).toEqual([ + "attachments-metadata-only", + "connected-account-backfill-coverage-unknown", + "remote-history-not-claimed", + ]); + + const accountRows = ndjson(join(output, "accounts.ndjson")); + expect(accountRows.find((row) => row.network === "whatsapp-personal")) + .toMatchObject({ handle: "+15550000000" }); + const conversationRows = ndjson(join(output, "conversations.ndjson")); + expect(conversationRows[0]).toMatchObject({ + participantsComplete: true, + type: "direct", + }); + expect((conversationRows[0]?.participantIds as readonly unknown[]).length) + .toBeGreaterThanOrEqual(2); + const messageRows = ndjson(join(output, "messages.ndjson")); + expect(messageRows).toMatchObject([{ + attachments: [{ name: "private-photo.jpg" }], + direction: "outgoing", + edit: { kind: "in-place" }, + replyTo: { messageId: null }, + }, { + body: null, + deletion: { state: "revoked" }, + direction: "incoming", + }]); + expect(JSON.stringify(messageRows)).not.toContain("private-media-id"); + expect(JSON.stringify(messageRows)).not.toContain("file:///private"); + const reactionRows = ndjson(join(output, "reactions.ndjson")); + expect(reactionRows[0]).toMatchObject({ + body: "👍", + messageId: messageRows[0]?.id, + messageProviderId: (messageRows[0]?.provenance as Record).providerId, + reactedAt: null, + }); + expect(reactionRows[1]).toMatchObject({ + body: "custom-reaction", + reactedAt: null, + }); + expect(JSON.stringify(reactionRows)).not.toContain("provider.invalid"); + const tombstoneRows = ndjson(join(output, "tombstones.ndjson")); + expect(tombstoneRows[0]).toMatchObject({ + entityId: messageRows[1]?.id, + entityProviderId: (messageRows[1]?.provenance as Record).providerId, + scope: "remote", + }); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("marks explicit chat and message limits as truncation", async () => { + const parent = privateDirectory("wrench-beeper-source-limit-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + try { + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + limits: { limitChats: 1, limitMessages: 2 }, + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + runExport: fixtureExport, + }, + }); + const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); + expect(result.manifest.completeness.kind).toBe("truncated"); + expect(result.manifest.warnings).toContain("chat-limit-reached"); + expect(result.manifest.warnings).toContain("message-limit-reached"); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("projects an irregular complete direct roster conservatively as incomplete", async () => { + const parent = privateDirectory("wrench-beeper-source-direct-roster-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + try { + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + runExport: async (invocation) => { + const result = await fixtureExport(invocation); + const extraPeer = { + fullName: "Extra Fixture", + id: "whatsapp:extra", + isSelf: false, + phoneNumber: "+15550000002", + }; + const irregular = chat(); + irregular.participants.items.push(extraPeer); + irregular.participants.total = 2; + const outputRoot = invocationOutputRoot(invocation); + writeJson(join(outputRoot, "chats.json"), [irregular]); + writeJson(join(outputRoot, "chats", CHAT_ID, "chat.json"), irregular); + return result; + }, + }, + }); + const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); + expect(result.manifest.warnings).toContain("participant-roster-incomplete"); + expect(ndjson(join(output, "conversations.ndjson"))[0]).toMatchObject({ + participantsComplete: false, + type: "direct", + }); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("publishes a coherent truncated bundle at the global record budget", async () => { + const parent = privateDirectory("wrench-beeper-source-record-limit-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + try { + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + maxBundleRecords: 7, + runExport: fixtureExport, + }, + }); + const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); + expect(result.manifest.completeness).toMatchObject({ + kind: "truncated", + reason: "bundle-record-limit", + }); + expect(result.manifest.warnings).toContain("bundle-record-limit-reached"); + expect(result.manifest.counts).toEqual({ + account: 2, + participant: 2, + conversation: 0, + message: 0, + reaction: 0, + tombstone: 0, + }); + expect(existsSync(join(output, "manifest.json"))).toBeTrue(); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("publishes a coherent truncated bundle at the global byte budget", async () => { + const parent = privateDirectory("wrench-beeper-source-byte-limit-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + try { + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + maxBundleBytes: 2_500, + runExport: fixtureExport, + }, + }); + const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); + expect(result.manifest.completeness).toMatchObject({ + kind: "truncated", + reason: "bundle-byte-limit", + }); + expect(result.manifest.warnings).toContain("bundle-byte-limit-reached"); + expect(result.manifest.counts.conversation).toBe(0); + expect(existsSync(join(output, "manifest.json"))).toBeTrue(); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("rejects an official export whose accounts do not match the bound auth subject", async () => { + const parent = privateDirectory("wrench-beeper-source-subject-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + let removed = false; + try { + const locator = auth(configStore(parent)); + const source = createBeeperMessageLikeMeSource({ + auth: { + ...locator, + subject: `beeper:local:${"0".repeat(64)}`, + }, + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + removed = true; + }, + runExport: fixtureExport, + }, + }); + await expect(exportBeeperMessageLikeMeBundle({ outputRoot: output, source })) + .rejects.toThrow("did not match the bound auth realm"); + expect(removed).toBeTrue(); + expect(existsSync(working)).toBeFalse(); + expect(existsSync(join(output, "manifest.json"))).toBeFalse(); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("compares only bounded parsed account projections from the official manifest", async () => { + const parent = privateDirectory("wrench-beeper-source-deep-account-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + try { + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + runExport: async (invocation) => { + const result = await fixtureExport(invocation); + const outputRoot = invocationOutputRoot(invocation); + const deepCapability = `${'{"next":'.repeat(5_000)}null${"}".repeat(5_000)}`; + const accountRows = accounts().map((account, index) => { + const encoded = JSON.stringify(account); + return index === 0 + ? `${encoded.slice(0, -1)},"capabilities":${deepCapability}}` + : encoded; + }); + const accountsJson = `[${accountRows.join(",")}]`; + writeFileSync(join(outputRoot, "accounts.json"), `${accountsJson}\n`, { + mode: 0o644, + }); + const manifest = { + accounts: "__BOUNDED_ACCOUNTS__", + attachmentCount: 0, + chatCount: 1, + completedAt: "2026-08-21T14:01:00.000Z", + createdAt: "2026-08-21T13:59:00.000Z", + messageCount: messages().length, + version: 1, + }; + writeFileSync( + join(outputRoot, "manifest.json"), + `${JSON.stringify(manifest).replace( + '"__BOUNDED_ACCOUNTS__"', + accountsJson, + )}\n`, + { mode: 0o644 }, + ); + return result; + }, + }, + }); + const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); + expect(result.manifest.counts.account).toBe(2); + expect(existsSync(join(output, "manifest.json"))).toBeTrue(); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("rejects malformed and symlinked official export inputs and cleans private staging", async () => { + const cases: readonly { + readonly name: string; + readonly mutate: (outputRoot: string) => void; + readonly error: string; + }[] = [{ + name: "malformed-manifest", + mutate: (outputRoot) => { + writeJson(join(outputRoot, "manifest.json"), { + accounts: accounts(), + attachmentCount: 0, + chatCount: 1, + completedAt: "2026-08-21T14:01:00.000Z", + createdAt: "2026-08-21T13:59:00.000Z", + messageCount: 2, + unreviewed: true, + version: 1, + }); + }, + error: "contains an unreviewed field", + }, { + name: "symlinked-messages", + mutate: (outputRoot) => { + const chatRoot = join(outputRoot, "chats", CHAT_ID); + const messagesPath = join(chatRoot, "messages.json"); + unlinkSync(messagesPath); + symlinkSync("chat.json", messagesPath); + }, + error: "must not be a symbolic link", + }]; + + for (const item of cases) { + const parent = privateDirectory(`wrench-beeper-source-${item.name}-test.`); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + let removed = false; + try { + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + removed = true; + }, + runExport: async (invocation) => { + const result = await fixtureExport(invocation); + item.mutate(invocationOutputRoot(invocation)); + return result; + }, + }, + }); + await expect(exportBeeperMessageLikeMeBundle({ outputRoot: output, source })) + .rejects.toThrow(item.error); + expect(removed).toBeTrue(); + expect(existsSync(working)).toBeFalse(); + expect(existsSync(join(output, "manifest.json"))).toBeFalse(); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + } + }); + + test("skips one oversized chat and publishes truthful truncated completeness", async () => { + const parent = privateDirectory("wrench-beeper-source-size-bound-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + let removed = false; + try { + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + removed = true; + }, + maxMessagesJsonBytes: 32, + runExport: fixtureExport, + }, + }); + const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); + expect(result.manifest.completeness).toMatchObject({ + kind: "truncated", + reason: "oversized-chat", + }); + expect(result.manifest.warnings).toContain("oversized-chat-skipped"); + expect(result.manifest.counts).toEqual({ + account: 2, + participant: 2, + conversation: 0, + message: 0, + reaction: 0, + tombstone: 0, + }); + expect(removed).toBeTrue(); + expect(existsSync(join(output, "manifest.json"))).toBeTrue(); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); +}); diff --git a/src/beeper-message-like-me-source.ts b/src/beeper-message-like-me-source.ts new file mode 100644 index 0000000..d23552e --- /dev/null +++ b/src/beeper-message-like-me-source.ts @@ -0,0 +1,1567 @@ +import { constants } from "node:fs"; +import { + chmod, + lstat, + mkdir, + mkdtemp, + open, + readdir, + realpath, + rm, + unlink, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, isAbsolute, join, resolve, sep } from "node:path"; + +import type { WrenchAuth } from "./auth"; +import { canonicalJson, sha256 } from "./canonical-json"; +import { + BEEPER_MESSAGE_LIKE_ME_MAX_RECORDS, + BEEPER_MESSAGE_LIKE_ME_MAX_TOTAL_BYTES, +} from "./beeper-message-like-me-export"; +import type { + BeeperMessageLikeMeAccount, + BeeperMessageLikeMeAttachment, + BeeperMessageLikeMeConversation, + BeeperMessageLikeMeExportSource, + BeeperMessageLikeMeMessage, + BeeperMessageLikeMeParticipant, + BeeperMessageLikeMeReaction, + BeeperMessageLikeMeRecord, + BeeperMessageLikeMeTombstone, +} from "./beeper-message-like-me-export"; +import { + BEEPER_CLI_PIN, + planBeeperMessageLikeMeExportCommand, +} from "./providers/beeper-local"; +import { + beeperSubjectFromAccounts, + parseBeeperExportAccounts, + parseBeeperExportConversation, + parseBeeperExportMessages, + resolvePinnedBeeperCliBinary, + validateBeeperCliStore, + type BeeperAccountProjection, + type BeeperAttachmentProjection, + type BeeperCliInvocationResult, + type BeeperConversationProjection, + type BeeperMessageProjection, + type BeeperReactionProjection, + type BeeperUserProjection, +} from "./providers/beeper-local-runtime"; + +const PRIVATE_DIRECTORY_MODE = 0o700; +const MAX_STDOUT_BYTES = 64 * 1024; +const MAX_STDERR_BYTES = 64 * 1024; +const MAX_ACCOUNTS_JSON_BYTES = 32 * 1024 * 1024; +const MAX_CHATS_JSON_BYTES = 64 * 1024 * 1024; +const MAX_CHAT_JSON_BYTES = 32 * 1024 * 1024; +const MAX_MESSAGES_JSON_BYTES = 64 * 1024 * 1024; +const MAX_EXPORT_CHATS = 100_000; +const MAX_EXPORT_MESSAGES_PER_CHAT = 1_000_000; +const DEFAULT_MAX_PARTICIPANTS = 500; +const DEFAULT_TIMEOUT_MS = 6 * 60 * 60 * 1_000; + +type JsonRecord = Readonly>; + +export type BeeperMessageLikeMeSourceLimits = Readonly<{ + limitChats?: number; + limitMessages?: number; + maxParticipants?: number; + timeoutMs?: number; +}>; + +export type BeeperExportCliInvocation = Readonly<{ + binary: string; + arguments: readonly string[]; + environment: Readonly>; + timeoutMs: number; + maxOutputBytes: number; + maxStderrBytes: number; + signal?: AbortSignal; +}>; + +export type BeeperMessageLikeMeSourceDependencies = Readonly<{ + /** Test-only seam. Production resolves the exact pinned binary hash. */ + binaryPath?: string; + /** Test-only seam for exercising the fixed production bundle record cap. */ + maxBundleRecords?: number; + /** Test-only seam for exercising the fixed production bundle byte cap. */ + maxBundleBytes?: number; + /** Test-only seam for exercising the per-chat JSON allocation cap. */ + maxMessagesJsonBytes?: number; + runExport?: ( + invocation: BeeperExportCliInvocation, + ) => Promise; + createWorkingDirectory?: () => Promise; + removeWorkingDirectory?: (path: string) => Promise; +}>; + +export type BeeperMessageLikeMeSourceRequest = Readonly<{ + auth: WrenchAuth; + limits?: BeeperMessageLikeMeSourceLimits; + signal?: AbortSignal; + environment?: Readonly>; + dependencies?: BeeperMessageLikeMeSourceDependencies; +}>; + +type ParsedLimits = Readonly<{ + limitChats: number | null; + limitMessages: number | null; + maxParticipants: number; + timeoutMs: number; +}>; + +type ParticipantFact = { + readonly id: string; + readonly accountId: string; + readonly providerId: string; + displayName: string | null; + handle: string | null; + isSelf: boolean | null; +}; + +type ConversationScan = Readonly<{ + chat: BeeperConversationProjection; + directory: string; + messagesPath: string; + participantIds: readonly string[]; + participantsComplete: boolean; + startedAt: string | null; + lastMessageAt: string | null; + messageCount: number; + reactionCount: number; + tombstoneCount: number; + nonParticipantRecordBytes: number; +}>; + +type ExportManifest = Readonly<{ + accounts: unknown; + attachmentCount: number; + chatCount: number; + completedAt: string; + createdAt: string; + messageCount: number; + version: 1; +}>; + +function fail(message: string): never { + throw new Error(`Beeper Message Like Me source: ${message}`); +} + +function isErrno(error: unknown, code: string): boolean { + return typeof error === "object" && error !== null && "code" in error + && (error as { readonly code?: unknown }).code === code; +} + +function record(value: unknown, label: string): JsonRecord { + if ( + typeof value !== "object" + || value === null + || Array.isArray(value) + || ( + Object.getPrototypeOf(value) !== Object.prototype + && Object.getPrototypeOf(value) !== null + ) + ) return fail(`${label} must be a plain object`); + return value as JsonRecord; +} + +function exactKeys( + value: JsonRecord, + required: readonly string[], + optional: readonly string[], + label: string, +): void { + const remaining = new Set(Object.keys(value)); + for (const key of required) { + if (!remaining.delete(key)) fail(`${label} omitted a required field`); + } + for (const key of optional) remaining.delete(key); + if (remaining.size > 0) fail(`${label} contains an unreviewed field`); +} + +function array(value: unknown, label: string, maximum: number): readonly unknown[] { + if (!Array.isArray(value) || value.length > maximum) { + return fail(`${label} must be an array inside its reviewed bound`); + } + for (let index = 0; index < value.length; index += 1) { + if (!Object.hasOwn(value, index)) fail(`${label} must not be sparse`); + } + return value; +} + +function integer(value: unknown, label: string, maximum: number): number { + if ( + typeof value !== "number" + || !Number.isSafeInteger(value) + || value < 0 + || value > maximum + ) return fail(`${label} must be a bounded non-negative integer`); + return value; +} + +function positiveInteger( + value: unknown, + label: string, + maximum: number, +): number { + const parsed = integer(value, label, maximum); + if (parsed < 1) return fail(`${label} must be positive`); + return parsed; +} + +function string(value: unknown, label: string, maximum: number): string { + if ( + typeof value !== "string" + || value.length < 1 + || Buffer.byteLength(value, "utf8") > maximum + || /[\0\r\n]/u.test(value) + ) return fail(`${label} must be bounded text`); + return value; +} + +function timestamp(value: unknown, label: string): string { + const source = string(value, label, 64); + const milliseconds = Date.parse(source); + if (!Number.isFinite(milliseconds)) return fail(`${label} must be a timestamp`); + return new Date(milliseconds).toISOString(); +} + +function parseLimits(value: BeeperMessageLikeMeSourceLimits | undefined): ParsedLimits { + return Object.freeze({ + limitChats: value?.limitChats === undefined + ? null + : positiveInteger(value.limitChats, "limitChats", MAX_EXPORT_CHATS), + limitMessages: value?.limitMessages === undefined + ? null + : positiveInteger( + value.limitMessages, + "limitMessages", + MAX_EXPORT_MESSAGES_PER_CHAT, + ), + maxParticipants: value?.maxParticipants === undefined + ? DEFAULT_MAX_PARTICIPANTS + : positiveInteger(value.maxParticipants, "maxParticipants", 2_000), + timeoutMs: value?.timeoutMs === undefined + ? DEFAULT_TIMEOUT_MS + : positiveInteger(value.timeoutMs, "timeoutMs", DEFAULT_TIMEOUT_MS), + }); +} + +function requireAuth(auth: WrenchAuth): Extract { + if ( + auth.kind !== "linked-device-store" + || auth.provider !== "beeper" + || auth.subject === undefined + || !/^beeper:local:[a-f0-9]{64}$/u.test(auth.subject) + ) return fail("export requires an account-bound Beeper linked-device-store auth locator"); + return auth; +} + +function throwIfAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted === true) fail("export was cancelled"); +} + +function digest(parts: readonly string[]): string { + return sha256(canonicalJson(parts)); +} + +function bundleRecordBytes(record: BeeperMessageLikeMeRecord): number { + return Buffer.byteLength(canonicalJson(record), "utf8") + 1; +} + +function localId(kind: string, ...parts: readonly string[]): string { + return `${kind}:${digest(parts)}`; +} + +function providerId(kind: string, ...parts: readonly string[]): string { + return `beeper-${kind}:${digest(parts)}`; +} + +function normalizeNetwork(value: string | null, fallback: string): string { + const normalized = (value ?? fallback) + .normalize("NFKD") + .toLowerCase() + .replace(/[^a-z0-9._+-]+/gu, "-") + .replace(/^[^a-z0-9]+|[^a-z0-9]+$/gu, "") || "unknown"; + if (Buffer.byteLength(normalized, "utf8") <= 64) return normalized; + return `${normalized.slice(0, 47).replace(/[^a-z0-9]+$/u, "")}-${digest([normalized]).slice(0, 16)}`; +} + +function preferredHandle(user: BeeperUserProjection): string | null { + return user.phoneNumber ?? user.email ?? user.username; +} + +function safeBaseName(value: string | null): string | null { + if (value === null) return null; + const name = basename(value.replaceAll("\\", "/")); + if (name.length < 1 || name === "." || name === "..") return null; + return name; +} + +function attachment( + value: BeeperAttachmentProjection, +): BeeperMessageLikeMeAttachment { + const kind: BeeperMessageLikeMeAttachment["kind"] = value.isSticker === true + ? "sticker" + : value.type === "img" + ? "image" + : value.type === "video" + ? "video" + : value.type === "audio" + ? "audio" + : value.fileName !== null + ? "document" + : "unknown"; + return Object.freeze({ + kind, + mimeType: value.mimeType, + name: safeBaseName(value.fileName), + sizeBytes: value.fileSizeBytes, + }); +} + +function reactionBody(value: string): string { + if ( + /[\\/]/u.test(value) + || /^[a-z][a-z0-9+.-]*:/iu.test(value) + || Buffer.byteLength(value, "utf8") > 128 + ) return "custom-reaction"; + if (/^:[A-Za-z0-9_+-]{1,64}:$/u.test(value)) return value; + if (/^[^\p{L}\p{N}]*\p{Extended_Pictographic}[^\p{L}\p{N}]*$/u.test(value)) { + return value; + } + if (["+1", "-1", "like", "love", "laugh", "sad", "angry"].includes(value)) { + return value; + } + return "custom-reaction"; +} + +function safeSegment(value: string): string { + const normalized = value + .replace(/[^a-zA-Z0-9._-]+/gu, "_") + .replace(/^_+|_+$/gu, ""); + return normalized.slice(0, 120) || "item"; +} + +async function assertOwnedDirectory(path: string, root?: string): Promise { + const canonical = await realpath(path); + if (canonical !== path) return fail("export directory traversed a symbolic link"); + if (root !== undefined && canonical !== root && !canonical.startsWith(`${root}${sep}`)) { + return fail("export directory escaped private staging"); + } + const metadata = await lstat(canonical); + if ( + !metadata.isDirectory() + || metadata.isSymbolicLink() + || metadata.uid !== process.getuid?.() + || (metadata.mode & 0o022) !== 0 + ) return fail("export directory is not an owned physical directory"); + return canonical; +} + +async function assertPrivateOwnedDirectory( + path: string, + root?: string, +): Promise { + const canonical = await assertOwnedDirectory(path, root); + const metadata = await lstat(canonical); + if ((metadata.mode & 0o777) !== PRIVATE_DIRECTORY_MODE) { + return fail("private export root permissions changed"); + } + return canonical; +} + +async function readOwnedJson( + path: string, + root: string, + maximumBytes: number, +): Promise { + if (!path.startsWith(`${root}${sep}`)) return fail("export file escaped private staging"); + let handle; + try { + handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + } catch (error) { + if (isErrno(error, "ENOENT")) return fail("official export omitted a required file"); + if (isErrno(error, "ELOOP")) return fail("official export file must not be a symbolic link"); + return fail("official export file could not be opened safely"); + } + try { + const before = await handle.stat(); + if ( + !before.isFile() + || before.uid !== process.getuid?.() + || before.nlink !== 1 + || (before.mode & 0o022) !== 0 + || before.size < 2 + || before.size > maximumBytes + ) return fail("official export file is outside its ownership or size bound"); + const bytes = Buffer.allocUnsafe(before.size); + let offset = 0; + while (offset < bytes.byteLength) { + const result = await handle.read(bytes, offset, bytes.byteLength - offset, offset); + if (result.bytesRead === 0) break; + offset += result.bytesRead; + } + const extra = Buffer.allocUnsafe(1); + const overflow = await handle.read(extra, 0, 1, offset); + const after = await handle.stat(); + if ( + offset !== bytes.byteLength + || overflow.bytesRead !== 0 + || before.dev !== after.dev + || before.ino !== after.ino + || after.nlink !== 1 + || before.size !== after.size + || before.mtimeMs !== after.mtimeMs + || before.ctimeMs !== after.ctimeMs + ) return fail("official export file changed while being read"); + const pathMetadata = await lstat(path); + if ( + pathMetadata.isSymbolicLink() + || pathMetadata.nlink !== 1 + || pathMetadata.dev !== after.dev + || pathMetadata.ino !== after.ino + ) return fail("official export file changed while being read"); + try { + return JSON.parse( + new TextDecoder("utf-8", { fatal: true }).decode(bytes), + ) as unknown; + } catch { + return fail("official export file is not valid UTF-8 JSON"); + } + } finally { + await handle.close(); + } +} + +async function ownedFileSize(path: string, root: string): Promise { + if (!path.startsWith(`${root}${sep}`)) return fail("export file escaped private staging"); + let handle; + try { + handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + } catch (error) { + if (isErrno(error, "ENOENT")) return fail("official export omitted a required file"); + if (isErrno(error, "ELOOP")) return fail("official export file must not be a symbolic link"); + return fail("official export file could not be opened safely"); + } + try { + const metadata = await handle.stat(); + const pathMetadata = await lstat(path); + if ( + !metadata.isFile() + || metadata.uid !== process.getuid?.() + || metadata.nlink !== 1 + || (metadata.mode & 0o022) !== 0 + || metadata.size < 2 + || pathMetadata.isSymbolicLink() + || pathMetadata.dev !== metadata.dev + || pathMetadata.ino !== metadata.ino + || pathMetadata.nlink !== 1 + ) return fail("official export file is outside its ownership or size bound"); + return metadata.size; + } finally { + await handle.close(); + } +} + +function parseManifest(value: unknown): ExportManifest { + const source = record(value, "official export manifest"); + exactKeys(source, [ + "accounts", + "attachmentCount", + "chatCount", + "completedAt", + "createdAt", + "messageCount", + "version", + ], [], "official export manifest"); + if (source.version !== 1) return fail("official export manifest version is unsupported"); + return Object.freeze({ + accounts: source.accounts, + attachmentCount: integer( + source.attachmentCount, + "official export manifest attachmentCount", + Number.MAX_SAFE_INTEGER, + ), + chatCount: integer( + source.chatCount, + "official export manifest chatCount", + MAX_EXPORT_CHATS, + ), + completedAt: timestamp(source.completedAt, "official export manifest completedAt"), + createdAt: timestamp(source.createdAt, "official export manifest createdAt"), + messageCount: integer( + source.messageCount, + "official export manifest messageCount", + Number.MAX_SAFE_INTEGER, + ), + version: 1, + }); +} + +async function readBoundedStream( + stream: ReadableStream, + maximum: number, + label: string, +): Promise { + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + try { + for (;;) { + const item = await reader.read(); + if (item.done) break; + if (item.value.byteLength > maximum - size) return fail(`${label} exceeded its byte bound`); + chunks.push(item.value.slice()); + size += item.value.byteLength; + } + } finally { + reader.releaseLock(); + } + const output = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + output.set(chunk, offset); + offset += chunk.byteLength; + } + return new TextDecoder("utf-8", { fatal: true }).decode(output); +} + +async function runExportCli( + invocation: BeeperExportCliInvocation, +): Promise { + throwIfAborted(invocation.signal); + const child = Bun.spawn([ + "/bin/sh", + "-c", + "umask 077\nexec \"$@\"", + "wrench-beeper-export", + invocation.binary, + ...invocation.arguments, + ], { + env: { ...invocation.environment }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + detached: true, + }); + let timedOut = false; + let cancelled = false; + let forceKill: ReturnType | null = null; + const signalGroup = (signal: "SIGTERM" | "SIGKILL"): void => { + try { + process.kill(-child.pid, signal); + } catch { + // The complete child process group already exited. + } + }; + const terminate = (): void => { + signalGroup("SIGTERM"); + if (forceKill === null) { + forceKill = setTimeout(() => signalGroup("SIGKILL"), 2_000); + } + }; + const onAbort = (): void => { + cancelled = true; + terminate(); + }; + invocation.signal?.addEventListener("abort", onAbort, { once: true }); + const timeout = setTimeout(() => { + timedOut = true; + terminate(); + }, invocation.timeoutMs); + try { + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + readBoundedStream(child.stdout, invocation.maxOutputBytes, "Beeper export stdout"), + readBoundedStream(child.stderr, invocation.maxStderrBytes, "Beeper export stderr"), + ]); + if (cancelled) return fail("official export was cancelled"); + if (timedOut) return fail("official export timed out"); + return Object.freeze({ exitCode, stdout, stderr }); + } catch (error) { + signalGroup("SIGKILL"); + await child.exited; + throw error; + } finally { + clearTimeout(timeout); + if (forceKill !== null) clearTimeout(forceKill); + invocation.signal?.removeEventListener("abort", onAbort); + } +} + +function environmentForExport( + configDirectory: string, + cacheDirectory: string, +): Readonly> { + return Object.freeze({ + PATH: "/usr/bin:/bin", + LANG: "C.UTF-8", + CI: "1", + BEEPER_CLI_CONFIG_DIR: configDirectory, + BEEPER_CLI_BINARY_CACHE_DIR: cacheDirectory, + BEEPER_READONLY: "1", + BEEPER_QUIET: "1", + BEEPER_SKIP_UPDATE_CHECK: "1", + NO_UPDATE_NOTIFIER: "1", + }); +} + +function upsertParticipant( + facts: Map, + account: BeeperAccountProjection, + user: Pick, + self: boolean | null, + createdIds?: Set, +): ParticipantFact { + const id = localId("participant", account.accountId, user.id); + const current = facts.get(id); + const handle = user.phoneNumber ?? user.email ?? user.username; + if (current !== undefined) { + if (self !== null && current.isSelf !== null && current.isSelf !== self) { + return fail("one Beeper participant has conflicting self-direction evidence"); + } + current.displayName ??= user.fullName; + current.handle ??= handle; + current.isSelf ??= self; + return current; + } + const created: ParticipantFact = { + id, + accountId: localId("account", account.accountId), + providerId: providerId("participant", account.accountId, user.id), + displayName: user.fullName, + handle, + isSelf: self, + }; + facts.set(id, created); + createdIds?.add(id); + return created; +} + +function messageTimestampRange( + messages: readonly BeeperMessageProjection[], +): { readonly first: string | null; readonly last: string | null } { + let first: string | null = null; + let last: string | null = null; + for (const message of messages) { + if (first === null || message.timestamp < first) first = message.timestamp; + if (last === null || message.timestamp > last) last = message.timestamp; + } + return Object.freeze({ first, last }); +} + +function parseOfficialState( + value: unknown, + chatIds: readonly string[], + createdAt: string, +): ReadonlyMap { + const source = record(value, "official export state"); + exactKeys(source, [ + "chats", + "completedChatIDs", + "createdAt", + "exportVersion", + ], [], "official export state"); + if (source.exportVersion !== 1 || timestamp(source.createdAt, "official export state createdAt") !== createdAt) { + fail("official export state does not match the completed manifest"); + } + const completed = array( + source.completedChatIDs, + "official export completedChatIDs", + MAX_EXPORT_CHATS, + ).map((item) => string(item, "official export completed chat ID", 2_048)); + const completedSet = new Set(completed); + if (completedSet.size !== completed.length || completedSet.size !== chatIds.length) { + fail("official export completed chat inventory is inconsistent"); + } + const states = record(source.chats, "official export chat states"); + if (Object.keys(states).length !== chatIds.length) { + fail("official export chat state inventory is inconsistent"); + } + const messageCounts = new Map(); + for (const chatId of chatIds) { + if (!completedSet.has(chatId)) { + fail("official export omitted a completed chat marker"); + } + const state = record(states[chatId], "official export chat state"); + exactKeys(state, [ + "attachmentCount", + "complete", + "cursor", + "messageCount", + "startedAt", + "updatedAt", + ], [], "official export chat state"); + if ( + state.complete !== true + || state.cursor !== null + || integer(state.attachmentCount, "official export chat attachmentCount", Number.MAX_SAFE_INTEGER) !== 0 + ) fail("official export chat did not complete without attachments"); + const messageCount = integer( + state.messageCount, + "official export chat messageCount", + MAX_EXPORT_MESSAGES_PER_CHAT, + ); + timestamp(state.startedAt, "official export chat startedAt"); + timestamp(state.updatedAt, "official export chat updatedAt"); + messageCounts.set(chatId, messageCount); + } + return messageCounts; +} + +function accountRecord( + account: BeeperAccountProjection, + network: string, + observedAt: string, + selfParticipantId: string, +): BeeperMessageLikeMeAccount { + const id = localId("account", account.accountId); + const connectedAccountProviderId = providerId("account", account.accountId); + return Object.freeze({ + schemaVersion: 1, + kind: "account", + id, + accountId: id, + network, + provenance: Object.freeze({ + providerId: connectedAccountProviderId, + providerRevision: null, + observedAt, + connectedAccountProviderId, + }), + displayName: account.user.fullName ?? account.network, + handle: preferredHandle(account.user), + selfParticipantId, + }); +} + +function participantRecord( + fact: ParticipantFact, + account: BeeperAccountProjection, + network: string, + observedAt: string, +): BeeperMessageLikeMeParticipant { + return Object.freeze({ + schemaVersion: 1, + kind: "participant", + id: fact.id, + accountId: fact.accountId, + network, + provenance: Object.freeze({ + providerId: fact.providerId, + providerRevision: null, + observedAt, + connectedAccountProviderId: providerId("account", account.accountId), + }), + displayName: fact.displayName, + handle: fact.handle, + isSelf: fact.isSelf === true, + }); +} + +function conversationRecord( + scan: ConversationScan, + account: BeeperAccountProjection, + network: string, + observedAt: string, +): BeeperMessageLikeMeConversation { + const id = localId("conversation", account.accountId, scan.chat.id); + return Object.freeze({ + schemaVersion: 1, + kind: "conversation", + id, + accountId: localId("account", account.accountId), + network, + provenance: Object.freeze({ + providerId: providerId("conversation", account.accountId, scan.chat.id), + providerRevision: scan.chat.lastActivity, + observedAt, + connectedAccountProviderId: providerId("account", account.accountId), + }), + type: scan.chat.type === "single" ? "direct" : "group", + title: scan.chat.title, + participantIds: scan.participantIds, + participantsComplete: scan.participantsComplete, + startedAt: scan.startedAt, + lastMessageAt: scan.lastMessageAt, + }); +} + +function messageRecord( + message: BeeperMessageProjection, + scan: ConversationScan, + messageIds: ReadonlySet, + account: BeeperAccountProjection, + network: string, + observedAt: string, +): BeeperMessageLikeMeMessage { + const id = localId("message", account.accountId, scan.chat.id, message.id); + const providerMessageId = providerId( + "message", + account.accountId, + scan.chat.id, + message.id, + ); + const deletionState: BeeperMessageLikeMeMessage["deletion"] = + message.isDeleted || message.isHidden + ? Object.freeze({ + state: message.isDeleted && message.isHidden + ? "revoked-and-deleted-for-me" + : message.isDeleted + ? "revoked" + : "deleted-for-me", + observedAt, + providerRevision: message.editedTimestamp ?? message.sortKey, + }) + : null; + const replyProviderId = message.linkedMessageId === null + ? null + : providerId( + "message", + account.accountId, + scan.chat.id, + message.linkedMessageId, + ); + return Object.freeze({ + schemaVersion: 1, + kind: "message", + id, + accountId: localId("account", account.accountId), + network, + provenance: Object.freeze({ + providerId: providerMessageId, + providerRevision: message.editedTimestamp ?? message.sortKey, + observedAt, + connectedAccountProviderId: providerId("account", account.accountId), + }), + conversationId: localId("conversation", account.accountId, scan.chat.id), + senderParticipantId: localId("participant", account.accountId, message.senderId), + direction: message.isSender ? "outgoing" : "incoming", + sentAt: message.timestamp, + sortKey: message.sortKey, + body: deletionState === null ? message.text : null, + bodyTruncated: false, + replyTo: message.linkedMessageId === null || replyProviderId === null + ? null + : Object.freeze({ + messageId: messageIds.has(message.linkedMessageId) + ? localId( + "message", + account.accountId, + scan.chat.id, + message.linkedMessageId, + ) + : null, + providerId: replyProviderId, + }), + edit: message.editedTimestamp === null + ? null + : Object.freeze({ + kind: "in-place" as const, + editedAt: message.editedTimestamp, + providerRevision: message.editedTimestamp, + }), + deletion: deletionState, + attachments: Object.freeze(message.attachments.map(attachment)), + }); +} + +function reactionRecord( + reaction: BeeperReactionProjection, + message: BeeperMessageProjection, + scan: ConversationScan, + account: BeeperAccountProjection, + network: string, + observedAt: string, +): BeeperMessageLikeMeReaction { + const id = localId( + "reaction", + account.accountId, + scan.chat.id, + message.id, + reaction.id, + ); + return Object.freeze({ + schemaVersion: 1, + kind: "reaction", + id, + accountId: localId("account", account.accountId), + network, + provenance: Object.freeze({ + providerId: providerId( + "reaction", + account.accountId, + scan.chat.id, + message.id, + reaction.id, + ), + providerRevision: reaction.id, + observedAt, + connectedAccountProviderId: providerId("account", account.accountId), + }), + messageId: localId("message", account.accountId, scan.chat.id, message.id), + messageProviderId: providerId( + "message", + account.accountId, + scan.chat.id, + message.id, + ), + participantId: localId("participant", account.accountId, reaction.participantId), + body: reactionBody(reaction.reactionKey), + reactedAt: null, + state: "active", + }); +} + +function tombstoneRecord( + message: BeeperMessageProjection, + scan: ConversationScan, + account: BeeperAccountProjection, + network: string, + observedAt: string, +): BeeperMessageLikeMeTombstone | null { + if (!message.isDeleted && !message.isHidden) return null; + const messageId = localId("message", account.accountId, scan.chat.id, message.id); + const providerMessageId = providerId( + "message", + account.accountId, + scan.chat.id, + message.id, + ); + return Object.freeze({ + schemaVersion: 1, + kind: "tombstone", + id: localId("tombstone", account.accountId, scan.chat.id, message.id), + accountId: localId("account", account.accountId), + network, + provenance: Object.freeze({ + providerId: providerId("tombstone", account.accountId, scan.chat.id, message.id), + providerRevision: message.editedTimestamp ?? message.sortKey, + observedAt, + connectedAccountProviderId: providerId("account", account.accountId), + }), + entityKind: "message", + entityId: messageId, + entityProviderId: providerMessageId, + deletedAt: observedAt, + scope: message.isDeleted && message.isHidden + ? "unknown" + : message.isDeleted + ? "remote" + : "local", + providerRevision: message.editedTimestamp ?? message.sortKey, + }); +} + +async function assertExactDirectoryEntries( + path: string, + expected: readonly string[], + label: string, +): Promise { + const actual = (await readdir(path)).sort(); + const wanted = [...expected].sort(); + if ( + actual.length !== wanted.length + || actual.some((entry, index) => entry !== wanted[index]) + ) fail(`${label} contained an unexpected file layout`); +} + +/** + * Creates a single-use source for the private bundle sink. The official CLI + * performs its own complete local pagination. Its duplicate transcript files + * remain inside one private staging root and are removed before completion. + */ +export function createBeeperMessageLikeMeSource( + request: BeeperMessageLikeMeSourceRequest, +): BeeperMessageLikeMeExportSource { + const auth = requireAuth(request.auth); + const limits = parseLimits(request.limits); + let consumed = false; + let completion: unknown; + + const records = (async function* (): AsyncGenerator { + if (consumed) return fail("record stream is single-use"); + consumed = true; + throwIfAborted(request.signal); + const configDirectory = await validateBeeperCliStore(auth.path); + const environment = request.environment ?? process.env; + const binary = request.dependencies?.binaryPath + ?? await resolvePinnedBeeperCliBinary(environment); + const maxBundleRecords = request.dependencies?.maxBundleRecords === undefined + ? BEEPER_MESSAGE_LIKE_ME_MAX_RECORDS + : positiveInteger( + request.dependencies.maxBundleRecords, + "test maxBundleRecords", + BEEPER_MESSAGE_LIKE_ME_MAX_RECORDS, + ); + const maxBundleBytes = request.dependencies?.maxBundleBytes === undefined + ? BEEPER_MESSAGE_LIKE_ME_MAX_TOTAL_BYTES + : positiveInteger( + request.dependencies.maxBundleBytes, + "test maxBundleBytes", + BEEPER_MESSAGE_LIKE_ME_MAX_TOTAL_BYTES, + ); + const maxMessagesJsonBytes = request.dependencies?.maxMessagesJsonBytes === undefined + ? MAX_MESSAGES_JSON_BYTES + : positiveInteger( + request.dependencies.maxMessagesJsonBytes, + "test maxMessagesJsonBytes", + MAX_MESSAGES_JSON_BYTES, + ); + if (!isAbsolute(binary)) return fail("Beeper CLI binary path must be absolute"); + const customCreateWorking = request.dependencies?.createWorkingDirectory; + const customRemoveWorking = request.dependencies?.removeWorkingDirectory; + if ((customCreateWorking === undefined) !== (customRemoveWorking === undefined)) { + return fail("test working-directory create and remove seams must be supplied together"); + } + const createWorking = customCreateWorking + ?? (() => mkdtemp(join(tmpdir(), "wrench-beeper-message-like-me-"))); + const removeWorking = customRemoveWorking + ?? ((path: string) => rm(path, { recursive: true, force: true })); + const working = await createWorking(); + if (!isAbsolute(working)) return fail("working directory must be absolute"); + await chmod(working, PRIVATE_DIRECTORY_MODE); + const canonicalWorking = await assertPrivateOwnedDirectory(await realpath(working)); + const rawRoot = resolve(canonicalWorking, "official-export"); + const cacheDirectory = resolve(canonicalWorking, "cli-payload-cache"); + await mkdir(rawRoot, { mode: PRIVATE_DIRECTORY_MODE }); + await chmod(rawRoot, PRIVATE_DIRECTORY_MODE); + await mkdir(cacheDirectory, { mode: PRIVATE_DIRECTORY_MODE }); + await chmod(cacheDirectory, PRIVATE_DIRECTORY_MODE); + await assertPrivateOwnedDirectory(rawRoot, canonicalWorking); + await assertPrivateOwnedDirectory(cacheDirectory, canonicalWorking); + try { + const arguments_ = planBeeperMessageLikeMeExportCommand({ + outputDirectory: rawRoot, + limitChats: limits.limitChats, + limitMessages: limits.limitMessages, + maxParticipants: limits.maxParticipants, + }, limits.timeoutMs); + const run = request.dependencies?.runExport ?? runExportCli; + const result = await run({ + binary, + arguments: arguments_, + environment: environmentForExport(configDirectory, cacheDirectory), + timeoutMs: limits.timeoutMs, + maxOutputBytes: MAX_STDOUT_BYTES, + maxStderrBytes: MAX_STDERR_BYTES, + ...(request.signal === undefined ? {} : { signal: request.signal }), + }); + throwIfAborted(request.signal); + if (result.exitCode !== 0 || result.stderr.trim().length !== 0) { + return fail("official read-only export failed"); + } + // Official v0.6.2 creates nested 0755/0644 entries. The exact 0700 root + // is the privacy boundary; nested entries must still be owned, physical, + // non-writable by others, and regular files must have one link. + await assertPrivateOwnedDirectory(rawRoot, canonicalWorking); + await assertExactDirectoryEntries(rawRoot, [ + ".beeper-export-state.json", + "accounts.json", + "chats", + "chats.json", + "manifest.json", + ], "official export root"); + + const accountsValue = await readOwnedJson( + join(rawRoot, "accounts.json"), + rawRoot, + MAX_ACCOUNTS_JSON_BYTES, + ); + const accounts = parseBeeperExportAccounts(accountsValue); + if (beeperSubjectFromAccounts(accounts) !== auth.subject) { + return fail("official export account did not match the bound auth realm"); + } + const accountsById = new Map(accounts.map((account) => [account.accountId, account])); + const manifest = parseManifest(await readOwnedJson( + join(rawRoot, "manifest.json"), + rawRoot, + MAX_ACCOUNTS_JSON_BYTES, + )); + const manifestAccounts = parseBeeperExportAccounts(manifest.accounts); + if ( + manifest.attachmentCount !== 0 + || canonicalJson(manifestAccounts) !== canonicalJson(accounts) + ) return fail("official export manifest did not prove a no-attachment account snapshot"); + + const listedValues = array( + await readOwnedJson( + join(rawRoot, "chats.json"), + rawRoot, + MAX_CHATS_JSON_BYTES, + ), + "official export chats", + MAX_EXPORT_CHATS, + ); + const listedChats = listedValues.map((value) => + parseBeeperExportConversation(value, accounts)); + if (manifest.chatCount !== listedChats.length) { + return fail("official export manifest chat count did not match chats.json"); + } + const chatSegments = listedChats.map((chat) => safeSegment(chat.id)); + if (new Set(chatSegments).size !== chatSegments.length) { + return fail("official export chat directory names collided"); + } + const chatsRoot = await assertOwnedDirectory(join(rawRoot, "chats"), rawRoot); + await assertExactDirectoryEntries(chatsRoot, chatSegments, "official export chats"); + const stateMessageCounts = parseOfficialState(await readOwnedJson( + join(rawRoot, ".beeper-export-state.json"), + rawRoot, + MAX_CHATS_JSON_BYTES, + ), listedChats.map((chat) => chat.id), manifest.createdAt); + let officialMessageCount = 0; + for (const count of stateMessageCounts.values()) { + officialMessageCount += count; + if (!Number.isSafeInteger(officialMessageCount)) { + return fail("official export message count overflowed"); + } + } + if (officialMessageCount !== manifest.messageCount) { + return fail("official export manifest message count did not match chat state"); + } + + const participantFacts = new Map(); + const selfParticipantByAccount = new Map(); + for (const account of accounts) { + const self = upsertParticipant(participantFacts, account, account.user, true); + selfParticipantByAccount.set(account.accountId, self.id); + } + + const scans: ConversationScan[] = []; + let observedFrom: string | null = null; + let observedThrough: string | null = null; + let participantRosterIncomplete = false; + let messageLimitReached = false; + let oversizedChatSkipped = false; + let scannedRecordCount = accounts.length + selfParticipantByAccount.size; + let scanRecordBudgetExhausted = false; + const listedChatEntries = listedChats.map((chat, index) => { + const segment = chatSegments[index]; + if (segment === undefined) return fail("official export chat segment disappeared"); + return Object.freeze({ chat, segment }); + }).sort((left, right) => + localId("conversation", left.chat.accountId, left.chat.id).localeCompare( + localId("conversation", right.chat.accountId, right.chat.id), + )); + for (const { chat: listedChat, segment } of listedChatEntries) { + throwIfAborted(request.signal); + const directory = await assertOwnedDirectory(join(chatsRoot, segment), rawRoot); + await assertExactDirectoryEntries(directory, [ + "attachments", + "chat.json", + "messages.html", + "messages.json", + "messages.markdown", + ], "official export chat directory"); + const attachmentsDirectory = await assertOwnedDirectory( + join(directory, "attachments"), + rawRoot, + ); + await assertExactDirectoryEntries( + attachmentsDirectory, + [], + "official export attachments directory", + ); + const chat = parseBeeperExportConversation(await readOwnedJson( + join(directory, "chat.json"), + rawRoot, + MAX_CHAT_JSON_BYTES, + ), accounts); + if (chat.id !== listedChat.id || chat.accountId !== listedChat.accountId) { + return fail("official export chat detail did not match chats.json"); + } + const account = accountsById.get(chat.accountId); + if (account === undefined) return fail("official export chat references an unknown account"); + const expectedMessageCount = stateMessageCounts.get(chat.id); + if (expectedMessageCount === undefined) { + return fail("official export chat state disappeared"); + } + if ( + limits.limitMessages !== null + && expectedMessageCount >= limits.limitMessages + ) messageLimitReached = true; + const messagesPath = join(directory, "messages.json"); + if (await ownedFileSize(messagesPath, rawRoot) > maxMessagesJsonBytes) { + oversizedChatSkipped = true; + await unlink(join(directory, "messages.markdown")); + await unlink(join(directory, "messages.html")); + continue; + } + if ( + scanRecordBudgetExhausted + || expectedMessageCount + 1 > maxBundleRecords - scannedRecordCount + ) { + scanRecordBudgetExhausted = true; + await unlink(join(directory, "messages.markdown")); + await unlink(join(directory, "messages.html")); + continue; + } + const newlyCreatedParticipantIds = new Set(); + const participantIds = new Set(); + for (const participant of chat.participants.items) { + participantIds.add(upsertParticipant( + participantFacts, + account, + participant, + participant.isSelf, + newlyCreatedParticipantIds, + ).id); + } + if (chat.type === "single") { + const self = selfParticipantByAccount.get(account.accountId); + if (self === undefined) return fail("Beeper account self participant disappeared"); + participantIds.add(self); + } + const messages = parseBeeperExportMessages( + await readOwnedJson(messagesPath, rawRoot, maxMessagesJsonBytes), + chat.accountId, + chat.id, + MAX_EXPORT_MESSAGES_PER_CHAT, + ); + if (messages.length !== expectedMessageCount) { + return fail("official export chat messages did not match completed state"); + } + const messageIds = new Set(messages.map((message) => message.id)); + const reactionCount = messages.reduce( + (count, message) => count + message.reactions.length, + 0, + ); + const tombstoneCount = messages.reduce( + (count, message) => count + (message.isDeleted || message.isHidden ? 1 : 0), + 0, + ); + if ( + !Number.isSafeInteger(reactionCount) + || !Number.isSafeInteger(tombstoneCount) + ) return fail("official export derived record count overflowed"); + for (const message of messages) { + participantIds.add(upsertParticipant(participantFacts, account, { + id: message.senderId, + fullName: message.senderName, + phoneNumber: null, + email: null, + username: null, + }, message.isSender, newlyCreatedParticipantIds).id); + for (const reaction of message.reactions) { + participantIds.add(upsertParticipant(participantFacts, account, { + id: reaction.participantId, + fullName: null, + phoneNumber: null, + email: null, + username: null, + }, null, newlyCreatedParticipantIds).id); + } + } + const scanRecordCount = 1 + + messages.length + + reactionCount + + tombstoneCount + + newlyCreatedParticipantIds.size; + if (scanRecordCount > maxBundleRecords - scannedRecordCount) { + for (const participantId of newlyCreatedParticipantIds) { + participantFacts.delete(participantId); + } + scanRecordBudgetExhausted = true; + await unlink(join(directory, "messages.markdown")); + await unlink(join(directory, "messages.html")); + continue; + } + scannedRecordCount += scanRecordCount; + const range = messageTimestampRange(messages); + if (range.first !== null && (observedFrom === null || range.first < observedFrom)) { + observedFrom = range.first; + } + if (range.last !== null && (observedThrough === null || range.last > observedThrough)) { + observedThrough = range.last; + } + const roster = [...participantIds].map((participantId) => { + const participant = participantFacts.get(participantId); + if (participant === undefined) { + return fail("Beeper conversation participant disappeared"); + } + return participant; + }); + const directRosterComplete = chat.type !== "single" + || ( + roster.length === 2 + && roster.filter((participant) => participant.isSelf === true).length === 1 + && roster.filter((participant) => participant.isSelf !== true).length === 1 + ); + const participantsComplete = !chat.participants.hasMore + && chat.participants.items.length === chat.participants.total + && directRosterComplete; + if (!participantsComplete) participantRosterIncomplete = true; + const scanDraft: ConversationScan = Object.freeze({ + chat, + directory, + messagesPath, + participantIds: Object.freeze([...participantIds].sort()), + participantsComplete, + startedAt: range.first, + lastMessageAt: range.last, + messageCount: messages.length, + reactionCount, + tombstoneCount, + nonParticipantRecordBytes: 0, + }); + const network = normalizeNetwork(account.network, account.bridge.type); + let nonParticipantRecordBytes = bundleRecordBytes( + conversationRecord(scanDraft, account, network, manifest.completedAt), + ); + for (const message of messages) { + nonParticipantRecordBytes += bundleRecordBytes( + messageRecord( + message, + scanDraft, + messageIds, + account, + network, + manifest.completedAt, + ), + ); + for (const reaction of message.reactions) { + nonParticipantRecordBytes += bundleRecordBytes(reactionRecord( + reaction, + message, + scanDraft, + account, + network, + manifest.completedAt, + )); + } + const tombstone = tombstoneRecord( + message, + scanDraft, + account, + network, + manifest.completedAt, + ); + if (tombstone !== null) { + nonParticipantRecordBytes += bundleRecordBytes(tombstone); + } + } + if (!Number.isSafeInteger(nonParticipantRecordBytes)) { + return fail("official export derived record bytes overflowed"); + } + scans.push(Object.freeze({ + ...scanDraft, + nonParticipantRecordBytes, + })); + // These plaintext renderings are redundant after strict JSON conversion. + await unlink(join(directory, "messages.markdown")); + await unlink(join(directory, "messages.html")); + } + const orderedScans = [...scans].sort((left, right) => + localId("conversation", left.chat.accountId, left.chat.id).localeCompare( + localId("conversation", right.chat.accountId, right.chat.id), + )); + const selectedParticipantIds = new Set(selfParticipantByAccount.values()); + const selectedScans: ConversationScan[] = []; + let selectedRecordCount = accounts.length + selectedParticipantIds.size; + let selectedRecordBytes = 0; + for (const account of accounts) { + const selfParticipantId = selfParticipantByAccount.get(account.accountId); + if (selfParticipantId === undefined) { + return fail("Beeper account self participant disappeared"); + } + const network = normalizeNetwork(account.network, account.bridge.type); + selectedRecordBytes += bundleRecordBytes(accountRecord( + account, + network, + manifest.completedAt, + selfParticipantId, + )); + const selfFact = participantFacts.get(selfParticipantId); + if (selfFact === undefined) return fail("Beeper account self participant disappeared"); + selectedRecordBytes += bundleRecordBytes(participantRecord( + selfFact, + account, + network, + manifest.completedAt, + )); + } + let bundleRecordLimitReached = scanRecordBudgetExhausted; + let bundleByteLimitReached = false; + for (const scan of orderedScans) { + let addedParticipants = 0; + let addedParticipantBytes = 0; + for (const participantId of scan.participantIds) { + if (selectedParticipantIds.has(participantId)) continue; + addedParticipants += 1; + const fact = participantFacts.get(participantId); + const account = fact === undefined + ? undefined + : accountsById.get(scan.chat.accountId); + if (fact === undefined || account === undefined) { + return fail("selected participant source identity disappeared"); + } + addedParticipantBytes += bundleRecordBytes(participantRecord( + fact, + account, + normalizeNetwork(account.network, account.bridge.type), + manifest.completedAt, + )); + } + const scanRecords = 1 + + scan.messageCount + + scan.reactionCount + + scan.tombstoneCount + + addedParticipants; + const scanBytes = scan.nonParticipantRecordBytes + addedParticipantBytes; + const exceedsRecords = scanRecords > maxBundleRecords - selectedRecordCount; + const exceedsBytes = scanBytes > maxBundleBytes - selectedRecordBytes; + if (exceedsRecords || exceedsBytes) { + bundleRecordLimitReached ||= exceedsRecords; + bundleByteLimitReached ||= exceedsBytes; + break; + } + selectedScans.push(scan); + selectedRecordCount += scanRecords; + selectedRecordBytes += scanBytes; + for (const participantId of scan.participantIds) { + selectedParticipantIds.add(participantId); + } + } + observedFrom = null; + observedThrough = null; + for (const scan of selectedScans) { + if ( + scan.startedAt !== null + && (observedFrom === null || scan.startedAt < observedFrom) + ) observedFrom = scan.startedAt; + if ( + scan.lastMessageAt !== null + && (observedThrough === null || scan.lastMessageAt > observedThrough) + ) observedThrough = scan.lastMessageAt; + } + participantRosterIncomplete = selectedScans.some( + (scan) => !scan.participantsComplete, + ); + + for (const account of [...accounts].sort((left, right) => + left.accountId.localeCompare(right.accountId))) { + const selfParticipantId = selfParticipantByAccount.get(account.accountId); + if (selfParticipantId === undefined) return fail("Beeper account has no self participant"); + yield accountRecord( + account, + normalizeNetwork(account.network, account.bridge.type), + manifest.completedAt, + selfParticipantId, + ); + } + for (const fact of [...participantFacts.values()] + .filter((candidate) => selectedParticipantIds.has(candidate.id)) + .sort((left, right) => left.id.localeCompare(right.id))) { + const account = accounts.find((candidate) => + localId("account", candidate.accountId) === fact.accountId); + if (account === undefined) return fail("participant account disappeared"); + yield participantRecord( + fact, + account, + normalizeNetwork(account.network, account.bridge.type), + manifest.completedAt, + ); + } + for (const scan of selectedScans) { + const account = accountsById.get(scan.chat.accountId); + if (account === undefined) return fail("conversation account disappeared"); + const network = normalizeNetwork(account.network, account.bridge.type); + yield conversationRecord(scan, account, network, manifest.completedAt); + const messages = parseBeeperExportMessages( + await readOwnedJson( + scan.messagesPath, + rawRoot, + maxMessagesJsonBytes, + ), + scan.chat.accountId, + scan.chat.id, + MAX_EXPORT_MESSAGES_PER_CHAT, + ); + const messageIds = new Set(messages.map((message) => message.id)); + for (const message of messages) { + yield messageRecord( + message, + scan, + messageIds, + account, + network, + manifest.completedAt, + ); + for (const reaction of message.reactions) { + yield reactionRecord( + reaction, + message, + scan, + account, + network, + manifest.completedAt, + ); + } + const tombstone = tombstoneRecord( + message, + scan, + account, + network, + manifest.completedAt, + ); + if (tombstone !== null) yield tombstone; + } + } + + const warnings = new Set([ + "attachments-metadata-only", + "remote-history-not-claimed", + "connected-account-backfill-coverage-unknown", + ]); + const chatLimitReached = limits.limitChats !== null + && listedChats.length >= limits.limitChats; + if (chatLimitReached) warnings.add("chat-limit-reached"); + if (messageLimitReached) warnings.add("message-limit-reached"); + if (participantRosterIncomplete) warnings.add("participant-roster-incomplete"); + if (oversizedChatSkipped) warnings.add("oversized-chat-skipped"); + if (bundleRecordLimitReached) warnings.add("bundle-record-limit-reached"); + if (bundleByteLimitReached) warnings.add("bundle-byte-limit-reached"); + const truncated = chatLimitReached + || messageLimitReached + || oversizedChatSkipped + || bundleRecordLimitReached + || bundleByteLimitReached; + completion = Object.freeze({ + completeness: Object.freeze({ + kind: truncated ? "truncated" : "bounded-local", + reason: bundleRecordLimitReached + ? "bundle-record-limit" + : bundleByteLimitReached + ? "bundle-byte-limit" + : oversizedChatSkipped + ? "oversized-chat" + : truncated + ? "explicit-source-limit" + : "desktop-local-export", + observedFrom, + observedThrough, + }), + warnings: Object.freeze([...warnings].sort()), + }); + } finally { + await removeWorking(canonicalWorking); + } + })(); + + return Object.freeze({ + descriptor: Object.freeze({ + source: Object.freeze({ id: "beeper-local", version: "1.0.0" }), + provider: Object.freeze({ id: "beeper", version: BEEPER_CLI_PIN.version }), + }), + records, + completion: async () => { + if (completion === undefined) return fail("record stream did not complete"); + return completion; + }, + }); +} diff --git a/src/fixtures/beeper-message-like-me-v1/accounts.ndjson b/src/fixtures/beeper-message-like-me-v1/accounts.ndjson new file mode 100644 index 0000000..d0d5e48 --- /dev/null +++ b/src/fixtures/beeper-message-like-me-v1/accounts.ndjson @@ -0,0 +1 @@ +{"accountId":"account:synthetic:primary","displayName":"Synthetic Primary","handle":"+15555550100","id":"account:synthetic:primary","kind":"account","network":"synthetic","provenance":{"connectedAccountProviderId":"beeper-account:synthetic-primary","observedAt":"2026-08-21T15:59:00.000Z","providerId":"beeper-account:synthetic-primary","providerRevision":null},"schemaVersion":1,"selfParticipantId":"participant:synthetic:self"} diff --git a/src/fixtures/beeper-message-like-me-v1/conversations.ndjson b/src/fixtures/beeper-message-like-me-v1/conversations.ndjson new file mode 100644 index 0000000..7f4f218 --- /dev/null +++ b/src/fixtures/beeper-message-like-me-v1/conversations.ndjson @@ -0,0 +1 @@ +{"accountId":"account:synthetic:primary","id":"conversation:synthetic:friend","kind":"conversation","lastMessageAt":"2026-08-21T15:58:45.000Z","network":"synthetic","participantIds":["participant:synthetic:self","participant:synthetic:peer"],"participantsComplete":true,"provenance":{"connectedAccountProviderId":"beeper-account:synthetic-primary","observedAt":"2026-08-21T15:59:00.000Z","providerId":"beeper-conversation:synthetic-friend","providerRevision":"chat-r1"},"schemaVersion":1,"startedAt":"2026-08-21T15:50:00.000Z","title":"Synthetic Friend","type":"direct"} diff --git a/src/fixtures/beeper-message-like-me-v1/manifest.json b/src/fixtures/beeper-message-like-me-v1/manifest.json new file mode 100644 index 0000000..0aca85e --- /dev/null +++ b/src/fixtures/beeper-message-like-me-v1/manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"bytes":430,"mediaType":"application/x-ndjson","path":"accounts.ndjson","recordKind":"account","records":1,"sha256":"5e29e5ca806fc4a97c22e63d0061c722ce0f62f14e96383e70c15eaff57ee1f8"},{"bytes":797,"mediaType":"application/x-ndjson","path":"participants.ndjson","recordKind":"participant","records":2,"sha256":"a7fb621e298c45f481a3d2f3995cfcbea80693f52c7787c1ff05b69248134186"},{"bytes":571,"mediaType":"application/x-ndjson","path":"conversations.ndjson","recordKind":"conversation","records":1,"sha256":"87957914b3f21ece69815719db37ea6413efdf3201b68bd60a0cf7abbbacb529"},{"bytes":1492,"mediaType":"application/x-ndjson","path":"messages.ndjson","recordKind":"message","records":2,"sha256":"dbe97f5b4a5c46c6c2f3e02294511ceb7868e45a8738f8736c0ec93d6707c394"},{"bytes":521,"mediaType":"application/x-ndjson","path":"reactions.ndjson","recordKind":"reaction","records":1,"sha256":"683c263ce93e82bca0d905bcf6ef4cec6b32d8028aaa52595d0f295f068a6b01"},{"bytes":539,"mediaType":"application/x-ndjson","path":"tombstones.ndjson","recordKind":"tombstone","records":1,"sha256":"34f5a61aa9756c10979372b6147337053a563a6a7c63cd44eab602e2594e05d5"}],"completeness":{"kind":"truncated","observedFrom":"2026-08-21T15:50:00.000Z","observedThrough":"2026-08-21T15:59:00.000Z","reason":"explicit-source-limit"},"counts":{"account":1,"conversation":1,"message":2,"participant":2,"reaction":1,"tombstone":1},"format":"message-like-me.local-message-bundle","integrity":{"algorithm":"sha256","bundleSha256":"56c6ff3bbe60acfd103f234592087a9753d4b90447d0a508575e0e7c5d4cf514"},"privacy":{"attachments":"metadata-only","classification":"private-local","credentials":"excluded","providerUrls":"excluded"},"provider":{"id":"beeper","version":"0.6.2"},"schemaVersion":1,"source":{"id":"beeper-local","version":"1.0.0"},"timestamps":{"createdAt":"2026-08-21T16:00:01.000Z","finishedAt":"2026-08-21T16:00:01.000Z","startedAt":"2026-08-21T16:00:00.000Z"},"warnings":["attachments-metadata-only","remote-history-not-claimed","synthetic-golden-fixture"]} diff --git a/src/fixtures/beeper-message-like-me-v1/messages.ndjson b/src/fixtures/beeper-message-like-me-v1/messages.ndjson new file mode 100644 index 0000000..86c0af4 --- /dev/null +++ b/src/fixtures/beeper-message-like-me-v1/messages.ndjson @@ -0,0 +1,2 @@ +{"accountId":"account:synthetic:primary","attachments":[],"body":"edited synthetic reply","bodyTruncated":false,"conversationId":"conversation:synthetic:friend","deletion":null,"direction":"outgoing","edit":{"editedAt":"2026-08-21T15:58:30.000Z","kind":"in-place","providerRevision":"edit-r2"},"id":"message:synthetic:edited","kind":"message","network":"synthetic","provenance":{"connectedAccountProviderId":"beeper-account:synthetic-primary","observedAt":"2026-08-21T15:59:00.000Z","providerId":"beeper-message:synthetic-edited","providerRevision":"edit-r2"},"replyTo":{"messageId":null,"providerId":"beeper-message:synthetic-external-reply-target"},"schemaVersion":1,"senderParticipantId":"participant:synthetic:self","sentAt":"2026-08-21T15:58:00.000Z","sortKey":"00000000000000000001"} +{"accountId":"account:synthetic:primary","attachments":[],"body":null,"bodyTruncated":false,"conversationId":"conversation:synthetic:friend","deletion":{"observedAt":"2026-08-21T15:59:00.000Z","providerRevision":"delete-r3","state":"revoked"},"direction":"incoming","edit":null,"id":"message:synthetic:deleted","kind":"message","network":"synthetic","provenance":{"connectedAccountProviderId":"beeper-account:synthetic-primary","observedAt":"2026-08-21T15:59:00.000Z","providerId":"beeper-message:synthetic-deleted","providerRevision":"delete-r3"},"replyTo":null,"schemaVersion":1,"senderParticipantId":"participant:synthetic:peer","sentAt":"2026-08-21T15:58:45.000Z","sortKey":"00000000000000000002"} diff --git a/src/fixtures/beeper-message-like-me-v1/participants.ndjson b/src/fixtures/beeper-message-like-me-v1/participants.ndjson new file mode 100644 index 0000000..6a867b6 --- /dev/null +++ b/src/fixtures/beeper-message-like-me-v1/participants.ndjson @@ -0,0 +1,2 @@ +{"accountId":"account:synthetic:primary","displayName":"Synthetic Self","handle":"+15555550100","id":"participant:synthetic:self","isSelf":true,"kind":"participant","network":"synthetic","provenance":{"connectedAccountProviderId":"beeper-account:synthetic-primary","observedAt":"2026-08-21T15:59:00.000Z","providerId":"beeper-participant:synthetic-self","providerRevision":null},"schemaVersion":1} +{"accountId":"account:synthetic:primary","displayName":"Synthetic Peer","handle":"+15555550101","id":"participant:synthetic:peer","isSelf":false,"kind":"participant","network":"synthetic","provenance":{"connectedAccountProviderId":"beeper-account:synthetic-primary","observedAt":"2026-08-21T15:59:00.000Z","providerId":"beeper-participant:synthetic-peer","providerRevision":null},"schemaVersion":1} diff --git a/src/fixtures/beeper-message-like-me-v1/reactions.ndjson b/src/fixtures/beeper-message-like-me-v1/reactions.ndjson new file mode 100644 index 0000000..37e789a --- /dev/null +++ b/src/fixtures/beeper-message-like-me-v1/reactions.ndjson @@ -0,0 +1 @@ +{"accountId":"account:synthetic:primary","body":"👍","id":"reaction:synthetic:undated","kind":"reaction","messageId":"message:synthetic:edited","messageProviderId":"beeper-message:synthetic-edited","network":"synthetic","participantId":"participant:synthetic:peer","provenance":{"connectedAccountProviderId":"beeper-account:synthetic-primary","observedAt":"2026-08-21T15:59:00.000Z","providerId":"beeper-reaction:synthetic-undated","providerRevision":"reaction-r1"},"reactedAt":null,"schemaVersion":1,"state":"active"} diff --git a/src/fixtures/beeper-message-like-me-v1/tombstones.ndjson b/src/fixtures/beeper-message-like-me-v1/tombstones.ndjson new file mode 100644 index 0000000..e3dcebe --- /dev/null +++ b/src/fixtures/beeper-message-like-me-v1/tombstones.ndjson @@ -0,0 +1 @@ +{"accountId":"account:synthetic:primary","deletedAt":"2026-08-21T15:59:00.000Z","entityId":"message:synthetic:deleted","entityKind":"message","entityProviderId":"beeper-message:synthetic-deleted","id":"tombstone:synthetic:message","kind":"tombstone","network":"synthetic","provenance":{"connectedAccountProviderId":"beeper-account:synthetic-primary","observedAt":"2026-08-21T15:59:00.000Z","providerId":"beeper-tombstone:synthetic-message","providerRevision":"delete-r3"},"providerRevision":"delete-r3","schemaVersion":1,"scope":"remote"} diff --git a/src/media/manifest.test.ts b/src/media/manifest.test.ts index c850db8..003af49 100644 --- a/src/media/manifest.test.ts +++ b/src/media/manifest.test.ts @@ -465,7 +465,7 @@ function trackedYtDlpManifest( describe("Wrench media manifest", () => { test("uses one Wrench-owned schema and transcriber identity", () => { expect(WRENCH_MEDIA_SCHEMA_VERSION).toBe(1); - expect(WRENCH_MEDIA_VERSION).toBe("0.10.1"); + expect(WRENCH_MEDIA_VERSION).toBe("0.11.0"); expect(localTranscriptVariantSegments(localIdentity)).toEqual([ "transcript", "local", diff --git a/src/media/manifest.ts b/src/media/manifest.ts index 74aae01..1b643bc 100644 --- a/src/media/manifest.ts +++ b/src/media/manifest.ts @@ -39,7 +39,7 @@ import { import { compareUtf8 } from "./utf8-order"; export const WRENCH_MEDIA_SCHEMA_VERSION = 1 as const; -export const WRENCH_MEDIA_VERSION = "0.10.1" as const; +export const WRENCH_MEDIA_VERSION = "0.11.0" as const; export const WRENCH_MEDIA_MANIFEST_FILE = "wrench-media.json" as const; export const WRENCH_MEDIA_CHECKSUM_FILE = "manifest-sha256.txt" as const; const MAX_ITEM_ENTRIES = 4_096; diff --git a/src/plugins/beeper-linked-device/plugin.ts b/src/plugins/beeper-linked-device/plugin.ts new file mode 100644 index 0000000..ee93762 --- /dev/null +++ b/src/plugins/beeper-linked-device/plugin.ts @@ -0,0 +1,77 @@ +import { + defineProviderPlugin, + lazyWebSessionRuntime, +} from "../../provider-plugin"; +import { + linkedDeviceAuthKinds, + webImplementationSources, + webSessionContractOperations, +} from "../../provider-plugin-builtins"; +import { webSessionContractDefinitions } from "../../web-session-contract-definitions"; +import { + materializeBeeperMessagingList, + materializeBeeperMessagingRead, +} from "../../providers/beeper-omni"; + +const beeperContracts = webSessionContractDefinitions.beeper; +if (beeperContracts === undefined) { + throw new Error("Beeper local read contracts are not installed"); +} + +export const beeperLinkedDevicePlugin = defineProviderPlugin({ + apiVersion: 1, + id: "beeper-linked-device", + version: "1.0.0", + displayName: "Beeper Local Read-Only", + sourceKind: "built-in", + implementationSources: webImplementationSources(import.meta.url, [ + ["kernel/auth.ts", "../../auth.ts"], + ["kernel/storage.ts", "../../storage.ts"], + ["providers/contact-projection.ts", "../../providers/contact-projection.ts"], + ["providers/beeper-local.ts", "../../providers/beeper-local.ts"], + ["providers/beeper-local-runtime.ts", "../../providers/beeper-local-runtime.ts"], + ["providers/beeper-omni.ts", "../../providers/beeper-omni.ts"], + ]), + bindings: [{ + transport: "linked-device", + surfaceId: "beeper", + origin: "https://www.beeper.com", + protectedHostnameFamilies: ["beeper.com"], + authKinds: linkedDeviceAuthKinds, + operations: webSessionContractOperations( + Object.values(beeperContracts), + "b80358d83a7062ea4e901b2b5564d1e72a5198ea5c2bb48f39ff815d15367aee", + {}, + { + "messaging.list": { + state: "supported", + schemaVersion: 1, + materializerId: "beeper-messaging-list", + materializerVersion: 1, + materialize: materializeBeeperMessagingList, + }, + "messaging.read": { + state: "supported", + schemaVersion: 1, + materializerId: "beeper-messaging-read", + materializerVersion: 1, + materialize: materializeBeeperMessagingRead, + }, + }, + ), + subject: { + format: "beeper:local:", + matches: (value) => /^beeper:local:[a-f0-9]{64}$/u.test(value), + }, + runtime: lazyWebSessionRuntime(async () => { + const runtime = await import("../../providers/beeper-local-runtime"); + return { + probe: runtime.probeBeeperLocalSubject, + execute: (_manifest, recipe, input, auth, options) => + runtime.executeBeeperLocalOperation(recipe, input, auth, options), + }; + }), + }], +}); + +export default beeperLinkedDevicePlugin; diff --git a/src/provider-contract-inventory.test.ts b/src/provider-contract-inventory.test.ts index 6a09dcd..359d9d6 100644 --- a/src/provider-contract-inventory.test.ts +++ b/src/provider-contract-inventory.test.ts @@ -65,7 +65,8 @@ function legacyHash(contract, implementationHash, web) { .digest("hex"); } function appendCurrentRow(row) { - if (row[0] === "provider-api" && row[1] === "gmail") { + if ((row[0] === "provider-api" && row[1] === "gmail") + || (row[0] === "linked-device" && row[1] === "beeper")) { currentOnlyRows.push(row); return; } @@ -202,8 +203,8 @@ describe("durable provider contract inventory", () => { expect(inventoryForNodeEnv(nodeEnv)).toEqual({ rows: 282, sha256: predecessorDefaultInventorySha256, - currentOnlyRows: 3, - currentOnlySha256: "4f8870e2e46ea268c6a6062d6fcf06393b9aed4b7670e96bd051d5626adc852a", + currentOnlyRows: 6, + currentOnlySha256: "2791af78d2b6800bb1855e8332137c9e018f9e9d2a7bd6fe505652f73a348ca1", legacyRows: [282, 282, 282, 282, 282, 282, 282], legacySha256: predecessorLegacyInventorySha256, acceptedLegacy: true, diff --git a/src/provider-plugin-contract-identity.ts b/src/provider-plugin-contract-identity.ts index 27d14b6..4715e49 100644 --- a/src/provider-plugin-contract-identity.ts +++ b/src/provider-plugin-contract-identity.ts @@ -19,6 +19,13 @@ export interface ReviewedBuiltInContractIdentityV1 { } const identities = Object.freeze({ + "beeper-linked-device": { + schemaVersion: 1, + pluginVersion: "1.0.0", + implementationSha256: "1110e1a6b99720c912451fa44d764f2f48590cbf7f2568aa199068adedf1c9f0", + legacyReadImplementationSha256: null, + legacyE71ReadImplementationSha256: null, + }, "bluesky-web": { schemaVersion: 1, pluginVersion: "1.0.0", diff --git a/src/provider-plugin-omni.test.ts b/src/provider-plugin-omni.test.ts index 4f3cdbd..d1e93e6 100644 --- a/src/provider-plugin-omni.test.ts +++ b/src/provider-plugin-omni.test.ts @@ -3,6 +3,8 @@ import { describe, expect, test } from "bun:test"; import { providerPluginRegistry } from "./provider-plugins"; const supported = new Set([ + "beeper-linked-device/beeper/messaging.list", + "beeper-linked-device/beeper/messaging.read", "gmail-official/gmail/messaging.list", "gmail-official/gmail/messaging.read", "meta-web/instagram/messaging.list", @@ -39,6 +41,7 @@ describe("provider plugin omni declarations", () => { test("provider-owned materializer files are in each supported closure", () => { const expectedLabels = new Map([ + ["beeper-linked-device", "providers/beeper-omni.ts"], ["gmail-official", "providers/gmail-omni.ts"], ["meta-web", "providers/meta-omni.ts"], ["reddit-web", "providers/reddit-omni.ts"], diff --git a/src/provider-plugin-registry.test.ts b/src/provider-plugin-registry.test.ts index 792b756..b76dc0c 100644 --- a/src/provider-plugin-registry.test.ts +++ b/src/provider-plugin-registry.test.ts @@ -3185,6 +3185,7 @@ describe("provider plugin definition and registry", () => { .sort((left, right) => left.localeCompare(right)); expect(observedContactProviderIds).toEqual([ + "beeper-linked-device", "gmail-official", "linkedin-official", "meta-web", diff --git a/src/provider-plugin-registry.ts b/src/provider-plugin-registry.ts index 2e8b4da..3a3cfab 100644 --- a/src/provider-plugin-registry.ts +++ b/src/provider-plugin-registry.ts @@ -553,6 +553,7 @@ const reviewedDormantDynamicLoaderPolicy = const reviewedDynamicInstalledModuleIdentities = reviewedMetaDynamicInstalledModuleIdentities; const reviewedKbDynamicInstalledPluginIds = new Set([ + "beeper-linked-device", "bluesky-web", "hacker-news-web", "linkedin-web", diff --git a/src/provider-plugins.generated.ts b/src/provider-plugins.generated.ts index 6ec5736..084c22e 100644 --- a/src/provider-plugins.generated.ts +++ b/src/provider-plugins.generated.ts @@ -2,19 +2,20 @@ // Edit src/plugins/*/plugin.ts, then run `bun run src/scripts/generate-provider-plugin-catalog.ts`. import type { ProviderPluginV1 } from "./provider-plugin"; -import sourceProviderPlugin0 from "./plugins/bluesky-web/plugin"; -import sourceProviderPlugin1 from "./plugins/gmail-official/plugin"; -import sourceProviderPlugin2 from "./plugins/hacker-news-web/plugin"; -import sourceProviderPlugin3 from "./plugins/linkedin-official/plugin"; -import sourceProviderPlugin4 from "./plugins/linkedin-web/plugin"; -import sourceProviderPlugin5 from "./plugins/meta-web/plugin"; -import sourceProviderPlugin6 from "./plugins/reddit-web/plugin"; -import sourceProviderPlugin7 from "./plugins/substack-web/plugin"; -import sourceProviderPlugin8 from "./plugins/tiktok-web/plugin"; -import sourceProviderPlugin9 from "./plugins/whatsapp-linked-device/plugin"; -import sourceProviderPlugin10 from "./plugins/x-official/plugin"; -import sourceProviderPlugin11 from "./plugins/x-web/plugin"; -import sourceProviderPlugin12 from "./plugins/youtube-web/plugin"; +import sourceProviderPlugin0 from "./plugins/beeper-linked-device/plugin"; +import sourceProviderPlugin1 from "./plugins/bluesky-web/plugin"; +import sourceProviderPlugin2 from "./plugins/gmail-official/plugin"; +import sourceProviderPlugin3 from "./plugins/hacker-news-web/plugin"; +import sourceProviderPlugin4 from "./plugins/linkedin-official/plugin"; +import sourceProviderPlugin5 from "./plugins/linkedin-web/plugin"; +import sourceProviderPlugin6 from "./plugins/meta-web/plugin"; +import sourceProviderPlugin7 from "./plugins/reddit-web/plugin"; +import sourceProviderPlugin8 from "./plugins/substack-web/plugin"; +import sourceProviderPlugin9 from "./plugins/tiktok-web/plugin"; +import sourceProviderPlugin10 from "./plugins/whatsapp-linked-device/plugin"; +import sourceProviderPlugin11 from "./plugins/x-official/plugin"; +import sourceProviderPlugin12 from "./plugins/x-web/plugin"; +import sourceProviderPlugin13 from "./plugins/youtube-web/plugin"; export const generatedProviderPlugins = Object.freeze([ sourceProviderPlugin0, @@ -30,4 +31,5 @@ export const generatedProviderPlugins = Object.freeze([ sourceProviderPlugin10, sourceProviderPlugin11, sourceProviderPlugin12, + sourceProviderPlugin13, ] as const satisfies readonly ProviderPluginV1[]); diff --git a/src/providers/beeper-local-runtime.internal.test.ts b/src/providers/beeper-local-runtime.internal.test.ts new file mode 100644 index 0000000..5a1c49d --- /dev/null +++ b/src/providers/beeper-local-runtime.internal.test.ts @@ -0,0 +1,499 @@ +import { createHash } from "node:crypto"; +import { + chmodSync, + mkdirSync, + mkdtempSync, + renameSync, + realpathSync, + rmSync, + symlinkSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, test } from "bun:test"; + +import type { WrenchAuth } from "../auth"; +import type { OperationInput, WebSessionRecipe } from "../model"; +import { + materializeBeeperMessagingList, + materializeBeeperMessagingRead, +} from "./beeper-omni"; +import { + executeBeeperLocalOperation, + parseBeeperExportMessages, + probeBeeperLocalSubject, + validateBeeperCliStore, + type BeeperCliInvocation, + type BeeperCliInvocationResult, +} from "./beeper-local-runtime"; +import { + parseBeeperMessagingReadInput, + planBeeperAccountsListCommand, + planBeeperReadCommand, +} from "./beeper-local"; + +const ACCOUNT_ID = "account-beeper"; +const NETWORK_ACCOUNT_ID = "account-signal"; +const SELF_ID = "@self:beeper.local"; +const CHAT_ID = "chat-synthetic"; +const SUBJECT = `beeper:local:${createHash("sha256") + .update(ACCOUNT_ID, "utf8") + .update("\0", "utf8") + .update(SELF_ID, "utf8") + .digest("hex")}`; + +function envelope(data: unknown): BeeperCliInvocationResult { + return Object.freeze({ + exitCode: 0, + stdout: `${JSON.stringify({ success: true, data, error: null })}\n`, + stderr: "", + }); +} + +function accounts(): readonly unknown[] { + return Object.freeze([{ + accountID: ACCOUNT_ID, + bridge: { id: "beeper", provider: "cloud", type: "matrix" }, + loginID: "redacted-login", + network: "Beeper", + status: "CONNECTED", + user: { + displayText: "Fixture Self", + email: "self@example.test", + fullName: "Fixture Self", + id: SELF_ID, + imgURL: "file:///private/avatar-self", + isSelf: true, + phoneNumber: "+15550000000", + username: "fixture-self", + }, + }, { + accountID: NETWORK_ACCOUNT_ID, + bridge: { id: "signal", provider: "cloud", type: "signal" }, + loginID: "+15550000000", + network: "Signal", + status: "CONNECTED", + user: { + fullName: "Fixture Self", + id: "signal:self", + isSelf: true, + }, + }]); +} + +function contacts(): readonly unknown[] { + return Object.freeze([{ + accountID: NETWORK_ACCOUNT_ID, + fullName: "Ada Fixture", + id: "signal:ada", + }]); +} + +function chats(): readonly unknown[] { + return Object.freeze([{ + accountID: NETWORK_ACCOUNT_ID, + id: CHAT_ID, + lastActivity: "2026-08-21T14:00:00.000Z", + network: "Signal", + participants: { + hasMore: false, + items: [{ + displayText: "Ada Fixture", + fullName: "Ada Fixture", + id: "signal:ada", + isSelf: false, + }, { + fullName: "Fixture Self", + id: "signal:self", + isSelf: true, + }], + total: 2, + }, + title: "Ada Fixture", + type: "single", + unreadCount: 0, + }]); +} + +function messages(includeDirection = true): readonly unknown[] { + const outgoing: Record = { + accountID: NETWORK_ACCOUNT_ID, + attachments: [{ + fileName: "photo.jpg", + fileSize: 42, + id: "private-attachment-id", + mimeType: "image/jpeg", + posterImg: "https://media.example.test/poster-token", + size: { height: 20, width: 10 }, + srcURL: "file:///private/beeper/media/photo.jpg", + type: "img", + }], + chatID: CHAT_ID, + editedTimestamp: "2026-08-21T14:00:02.000Z", + id: "message-outgoing", + linkedMessageID: "message-prior", + mentions: [], + reactions: [{ + emoji: true, + id: "reaction-private-id", + imgURL: "https://media.example.test/reaction-token", + participantID: "signal:ada", + reactionKey: "👍", + }], + seen: true, + senderID: "signal:self", + senderName: "Fixture Self", + sortKey: "00000000000000000001", + text: "one synthetic outgoing message", + timestamp: "2026-08-21T14:00:01.000Z", + type: "TEXT", + }; + if (includeDirection) outgoing.isSender = true; + return Object.freeze([outgoing, { + accountID: NETWORK_ACCOUNT_ID, + chatID: CHAT_ID, + id: "message-deleted", + isDeleted: true, + isHidden: false, + isSender: false, + senderID: "signal:ada", + senderName: "Ada Fixture", + sortKey: "00000000000000000002", + text: "must not survive deletion projection", + timestamp: "2026-08-21T14:00:03.000Z", + type: "TEXT", + }]); +} + +function privateStore(): string { + const path = realpathSync(mkdtempSync(join(tmpdir(), "wrench-beeper-store."))); + // The official CLI uses an owned 0755 directory with private 0600 files. + chmodSync(path, 0o755); + mkdirSync(join(path, "targets"), { mode: 0o755 }); + writeFileSync( + join(path, "config.json"), + `${JSON.stringify({ defaultTarget: "desktop" })}\n`, + { mode: 0o600 }, + ); + writeFileSync( + join(path, "targets", "desktop.json"), + `${JSON.stringify({ + auth: { token: "fixture-never-read-by-test-runner" }, + baseURL: "http://127.0.0.1:23384", + id: "desktop", + managed: true, + name: "Desktop", + runtime: "desktop", + type: "desktop", + })}\n`, + { mode: 0o600 }, + ); + return path; +} + +function auth(path: string): WrenchAuth { + return { + schemaVersion: 1, + id: "beeper-fixture", + kind: "linked-device-store", + provider: "beeper", + path, + subject: SUBJECT, + }; +} + +function recipe(action: string): WebSessionRecipe { + return { + site: "beeper", + action, + contractVersion: 1, + timeoutMs: 60_000, + maxOutputBytes: 32 * 1024 * 1024, + }; +} + +function runner( + calls: BeeperCliInvocation[], + options: { readonly includeDirection?: boolean } = {}, +): (invocation: BeeperCliInvocation) => Promise { + return async (invocation) => { + calls.push(invocation); + const command = invocation.arguments.slice(0, 2).join(" "); + if (invocation.arguments[0] === "version") { + return envelope({ name: "@beeper/cli", version: "0.6.2" }); + } + if (command === "accounts list") return envelope(accounts()); + if (command === "contacts list") return envelope(contacts()); + if (command === "chats list") return envelope(chats()); + if (command === "messages list") { + return envelope(messages(options.includeDirection ?? true)); + } + throw new Error(`unexpected fixture command ${command}`); + }; +} + +async function execute( + path: string, + action: string, + input: OperationInput, + calls: BeeperCliInvocation[], + options: { readonly includeDirection?: boolean } = {}, +) { + return executeBeeperLocalOperation(recipe(action), input, auth(path), { + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createCacheDirectory: async () => join(path, "ephemeral-cache"), + removeCacheDirectory: async () => undefined, + run: runner(calls, options), + }, + }); +} + +describe("Beeper local read runtime", () => { + test("plans only fixed read commands with command paths before Oclif global flags", () => { + expect(planBeeperAccountsListCommand(1_500).argv).toEqual([ + "accounts", + "list", + "--read-only", + "--json", + "--full", + "--quiet", + "--target", + "desktop", + "--timeout", + "2s", + ]); + const input = parseBeeperMessagingReadInput({ + account_id: NETWORK_ACCOUNT_ID, + conversation_id: CHAT_ID, + before_cursor: "message-cursor", + limit: 2, + }); + const command = planBeeperReadCommand("messaging.read", input, 60_000); + expect(command.argv.slice(0, 2)).toEqual(["messages", "list"]); + expect(command.argv.indexOf("--read-only")).toBeGreaterThan(1); + expect(command.argv).toContain("--before-cursor"); + expect(command.argv.join(" ")).not.toMatch(/\b(?:api|export|send|download|watch)\b/u); + expect(() => parseBeeperMessagingReadInput({ + account_id: NETWORK_ACCOUNT_ID, + conversation_id: CHAT_ID, + before_cursor: "before", + after_cursor: "after", + })).toThrow("only one cursor direction"); + }); + + test("executes contacts, chats, and messages through strict synthetic JSON", async () => { + const path = privateStore(); + const calls: BeeperCliInvocation[] = []; + try { + const contactsResult = await execute( + path, + "contacts.list", + { account_id: NETWORK_ACCOUNT_ID, limit: 2 }, + calls, + ); + const listResult = await execute( + path, + "messaging.list", + { account_id: NETWORK_ACCOUNT_ID, limit: 2 }, + calls, + ); + const readResult = await execute( + path, + "messaging.read", + { account_id: NETWORK_ACCOUNT_ID, conversation_id: CHAT_ID, limit: 2 }, + calls, + ); + + expect(contactsResult).toMatchObject({ + status: "succeeded", + dispatchStarted: false, + output: { + accountSubject: SUBJECT, + contacts: [{ accountId: NETWORK_ACCOUNT_ID, fullName: "Ada Fixture" }], + operation: "contacts.list", + provider: "beeper", + }, + }); + expect(listResult).toMatchObject({ + status: "succeeded", + output: { + conversations: [{ accountId: NETWORK_ACCOUNT_ID, id: CHAT_ID }], + operation: "messaging.list", + }, + }); + expect(readResult).toMatchObject({ + status: "succeeded", + output: { + continuation: { cursor: "message-deleted", direction: "before" }, + messages: [{ + attachments: [{ fileName: "photo.jpg", mimeType: "image/jpeg" }], + id: "message-outgoing", + isSender: true, + linkedMessageId: "message-prior", + }, { + id: "message-deleted", + isDeleted: true, + text: null, + }], + tombstones: [{ messageId: "message-deleted", state: "deleted" }], + }, + }); + const serialized = JSON.stringify(readResult.output); + expect(serialized).not.toContain("file:///private"); + expect(serialized).not.toContain("media.example.test"); + expect(serialized).not.toContain("private-attachment-id"); + + const listPage = materializeBeeperMessagingList( + { account_id: NETWORK_ACCOUNT_ID, limit: 2 }, + listResult.output, + ); + expect(listPage).toMatchObject({ + completeness: { kind: "bounded-local" }, + cursor: { direction: "none", nextInput: null }, + entities: [{ kind: "conversation", title: "Ada Fixture" }], + }); + const readPage = materializeBeeperMessagingRead( + { account_id: NETWORK_ACCOUNT_ID, conversation_id: CHAT_ID, limit: 2 }, + readResult.output, + ); + expect(readPage).toMatchObject({ + completeness: { kind: "bounded-local" }, + cursor: { + direction: "backward", + nextInput: { before_cursor: "message-deleted" }, + }, + entities: [{ + direction: "outgoing", + replyToProviderId: expect.stringContaining("message:"), + state: "active", + }, { + body: null, + direction: "incoming", + state: "revoked", + }], + }); + + expect(calls).toHaveLength(9); + for (const invocation of calls) { + expect(invocation.environment.BEEPER_READONLY).toBe("1"); + expect(invocation.environment.BEEPER_DESKTOP_BASE_URL).toBeUndefined(); + expect(invocation.arguments.indexOf("--read-only")).toBeGreaterThan(0); + } + expect(calls.filter((call) => call.arguments[0] === "accounts")).toHaveLength(3); + } finally { + rmSync(path, { recursive: true, force: true }); + } + }); + + test("rejects message drift when style-critical isSender is absent", async () => { + const path = privateStore(); + try { + await expect(execute( + path, + "messaging.read", + { account_id: NETWORK_ACCOUNT_ID, conversation_id: CHAT_ID, limit: 2 }, + [], + { includeDirection: false }, + )).rejects.toThrow("isSender is required"); + } finally { + rmSync(path, { recursive: true, force: true }); + } + }); + + test("aligns exported sort keys and attachment metadata with the bundle contract", () => { + const first = messages()[0] as Record; + expect(() => parseBeeperExportMessages([{ + ...first, + sortKey: "s".repeat(1_025), + }], NETWORK_ACCOUNT_ID, CHAT_ID, 1)).toThrow("sortKey must be bounded text"); + expect(() => parseBeeperExportMessages([{ + ...first, + attachments: [{ + type: "img", + mimeType: "m".repeat(257), + }], + }], NETWORK_ACCOUNT_ID, CHAT_ID, 1)).toThrow("mimeType must be bounded text"); + expect(() => parseBeeperExportMessages([{ + ...first, + attachments: Array.from({ length: 257 }, () => ({ type: "img" })), + }], NETWORK_ACCOUNT_ID, CHAT_ID, 1)).toThrow("attachments must be an array of at most 256 items"); + }); + + test("keeps first-run CLI payload extraction inside the overall probe deadline", async () => { + const path = privateStore(); + const calls: BeeperCliInvocation[] = []; + try { + const subject = await probeBeeperLocalSubject(auth(path), { + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createCacheDirectory: async () => join(path, "fresh-payload-cache"), + removeCacheDirectory: async () => undefined, + run: async (invocation) => { + calls.push(invocation); + if (invocation.arguments[0] === "version") { + expect(invocation.timeoutMs).toBeGreaterThan(5_000); + await new Promise((resolve) => setTimeout(resolve, 10)); + return envelope({ name: "@beeper/cli", version: "0.6.2" }); + } + if (invocation.arguments.slice(0, 2).join(" ") === "accounts list") { + return envelope(accounts()); + } + throw new Error("unexpected subject-probe fixture command"); + }, + }, + }); + expect(subject).toBe(SUBJECT); + expect(calls.map((call) => call.arguments[0])).toEqual([ + "version", + "accounts", + ]); + } finally { + rmSync(path, { recursive: true, force: true }); + } + }); + + test("accepts the official 0755/0600 target shape and rejects target drift and symlinks", async () => { + const path = privateStore(); + try { + await expect(validateBeeperCliStore(path)).resolves.toBe(path); + writeFileSync( + join(path, "config.json"), + `${JSON.stringify({ defaultTarget: "other" })}\n`, + { mode: 0o600 }, + ); + chmodSync(join(path, "config.json"), 0o600); + await expect(validateBeeperCliStore(path)).rejects.toThrow( + "must select the fixed desktop target", + ); + writeFileSync( + join(path, "config.json"), + `${JSON.stringify({ defaultTarget: "desktop" })}\n`, + { mode: 0o600 }, + ); + chmodSync(join(path, "config.json"), 0o600); + renameSync(join(path, "targets"), join(path, "targets.real")); + symlinkSync("targets.real", join(path, "targets")); + await expect(validateBeeperCliStore(path)).rejects.toThrow( + "targets directory must be an owned physical", + ); + unlinkSync(join(path, "targets")); + renameSync(join(path, "targets.real"), join(path, "targets")); + + const realConfig = join(path, "config.real.json"); + writeFileSync( + realConfig, + `${JSON.stringify({ defaultTarget: "desktop" })}\n`, + { mode: 0o600 }, + ); + unlinkSync(join(path, "config.json")); + symlinkSync(realConfig, join(path, "config.json")); + await expect(validateBeeperCliStore(path)).rejects.toThrow(); + } finally { + rmSync(path, { recursive: true, force: true }); + } + }); +}); diff --git a/src/providers/beeper-local-runtime.ts b/src/providers/beeper-local-runtime.ts new file mode 100644 index 0000000..2a4b273 --- /dev/null +++ b/src/providers/beeper-local-runtime.ts @@ -0,0 +1,1498 @@ +import { createHash } from "node:crypto"; +import { constants, createReadStream } from "node:fs"; +import { + lstat, + mkdtemp, + open, + realpath, + rm, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { isAbsolute, join } from "node:path"; +import { types as nodeTypes } from "node:util"; + +import type { WrenchAuth } from "../auth"; +import type { OperationInput, WebSessionRecipe } from "../model"; +import { OperationDeadline } from "../operation-deadline"; +import { wrenchStateHome } from "../storage"; +import type { + WebSessionCleanupBarrierRegistrar, + WebSessionExecution, + WebSessionOperationDeadline, +} from "../web-session-execution"; +import { startWebSessionCleanupTrackedOperation } from "../web-session-execution"; +import { + BEEPER_CLI_PIN, + BEEPER_DESKTOP_TARGET, + BEEPER_LOCAL_OPERATIONS, + BEEPER_ORIGIN, + isBeeperLocalOperation, + parseBeeperOperationInput, + planBeeperAccountsListCommand, + planBeeperReadCommand, + type BeeperLocalOperationName, + type BeeperMessagingReadInput, + type BeeperOperationInput, + type BeeperReadCommand, +} from "./beeper-local"; +import { projectContactDirectionStats } from "./contact-projection"; + +const MAX_STDERR_BYTES = 64 * 1024; +const MAX_ACCOUNTS = 128; +const MAX_USERS = 200; +const MAX_CHATS = 200; +const MAX_MESSAGES = 200; +const MAX_TEXT_BYTES = 1_048_576; +const OPERATION_LABEL = "Beeper local read operation"; +const SUBJECT_PROBE_TIMEOUT_MS = 120_000; + +type BeeperAuth = Extract; +type JsonRecord = Readonly>; + +export type BeeperCliInvocation = Readonly<{ + binary: string; + arguments: readonly string[]; + environment: Readonly>; + timeoutMs: number; + maxOutputBytes: number; + maxStderrBytes: number; + signal?: AbortSignal; +}>; + +export type BeeperCliInvocationResult = Readonly<{ + exitCode: number; + stdout: string; + stderr: string; +}>; + +export type BeeperLocalRuntimeDependencies = Readonly<{ + /** Test-only absolute binary seam. Production always resolves the exact pin. */ + binaryPath?: string; + run?: (invocation: BeeperCliInvocation) => Promise; + createCacheDirectory?: () => Promise; + removeCacheDirectory?: (path: string) => Promise; +}>; + +export type BeeperUserProjection = Readonly<{ + id: string; + fullName: string | null; + username: string | null; + phoneNumber: string | null; + email: string | null; + isSelf: boolean | null; + cannotMessage: boolean | null; +}>; + +export type BeeperAccountProjection = Readonly<{ + accountId: string; + bridge: Readonly<{ + id: string; + type: string; + provider: "cloud" | "self-hosted" | "local" | "platform-sdk"; + }>; + network: string | null; + loginId: string | null; + status: string; + statusText: string | null; + user: BeeperUserProjection; +}>; + +export type BeeperParticipantProjection = BeeperUserProjection & Readonly<{ + isAdmin: boolean | null; + isNetworkBot: boolean | null; + isPending: boolean | null; +}>; + +export type BeeperConversationProjection = Readonly<{ + id: string; + localChatId: string | null; + accountId: string; + network: string; + title: string; + type: "single" | "group"; + description: string | null; + lastActivity: string | null; + unreadCount: number; + unreadMentionsCount: number | null; + isMarkedUnread: boolean | null; + isArchived: boolean | null; + isLowPriority: boolean | null; + isMuted: boolean | null; + isPinned: boolean | null; + isReadOnly: boolean | null; + messageExpirySeconds: number | null; + participants: Readonly<{ + items: readonly BeeperParticipantProjection[]; + total: number; + hasMore: boolean; + }>; +}>; + +export type BeeperAttachmentProjection = Readonly<{ + type: "unknown" | "img" | "video" | "audio"; + durationSeconds: number | null; + fileName: string | null; + fileSizeBytes: number | null; + mimeType: string | null; + width: number | null; + height: number | null; + isGif: boolean | null; + isSticker: boolean | null; + isVoiceNote: boolean | null; + transcription: Readonly<{ + engine: string; + text: string; + language: string | null; + }> | null; +}>; + +export type BeeperReactionProjection = Readonly<{ + id: string; + participantId: string; + reactionKey: string; + emoji: boolean | null; +}>; + +export type BeeperMessageProjection = Readonly<{ + id: string; + accountId: string; + conversationId: string; + senderId: string; + senderName: string | null; + isSender: boolean; + sortKey: string; + timestamp: string; + editedTimestamp: string | null; + text: string | null; + type: string | null; + linkedMessageId: string | null; + mentions: readonly string[] | null; + isDeleted: boolean; + isHidden: boolean; + isUnread: boolean | null; + seen: boolean | string | Readonly> | null; + attachments: readonly BeeperAttachmentProjection[]; + reactions: readonly BeeperReactionProjection[]; +}>; + +export type BeeperTombstoneProjection = Readonly<{ + accountId: string; + conversationId: string; + messageId: string; + state: "deleted" | "hidden" | "deleted-and-hidden"; + observedAt: string; +}>; + +function strictRecord(value: unknown, label: string): JsonRecord { + if ( + nodeTypes.isProxy(value) + || typeof value !== "object" + || value === null + || Array.isArray(value) + || ( + Object.getPrototypeOf(value) !== Object.prototype + && Object.getPrototypeOf(value) !== null + ) + ) throw new Error(`${label} must be a plain object`); + const descriptors = Object.getOwnPropertyDescriptors(value); + for (const key of Reflect.ownKeys(descriptors)) { + if (typeof key !== "string") throw new Error(`${label} must not contain symbols`); + const descriptor = descriptors[key]; + if ( + descriptor === undefined + || !("value" in descriptor) + || !descriptor.enumerable + ) throw new Error(`${label}.${key} must be an enumerable data property`); + } + return value as JsonRecord; +} + +function exactKeys( + value: JsonRecord, + required: readonly string[], + optional: readonly string[], + label: string, +): void { + const allowed = new Set([...required, ...optional]); + for (const key of Object.keys(value)) { + if (!allowed.has(key)) throw new Error(`${label} contains unreviewed property ${key}`); + } + for (const key of required) { + if (!Object.hasOwn(value, key)) throw new Error(`${label}.${key} is required`); + } +} + +function strictArray(value: unknown, label: string, maximum: number): readonly unknown[] { + if ( + nodeTypes.isProxy(value) + || !Array.isArray(value) + || Object.getPrototypeOf(value) !== Array.prototype + || value.length > maximum + ) throw new Error(`${label} must be an array of at most ${maximum} items`); + for (let index = 0; index < value.length; index += 1) { + if (!Object.hasOwn(value, index)) throw new Error(`${label} must not be sparse`); + } + return value; +} + +function boundedString( + value: unknown, + label: string, + maximum: number, + allowEmpty = false, +): string { + if ( + typeof value !== "string" + || (!allowEmpty && value.length === 0) + || Buffer.byteLength(value, "utf8") > maximum + || /[\0]/u.test(value) + ) throw new Error(`${label} must be bounded text`); + return value; +} + +function nullableString(value: unknown, label: string, maximum: number): string | null { + return value === undefined || value === null + ? null + : boundedString(value, label, maximum, true); +} + +function optionalBoolean(value: unknown, label: string): boolean | null { + if (value === undefined || value === null) return null; + if (typeof value !== "boolean") throw new Error(`${label} must be boolean`); + return value; +} + +function requiredBoolean(value: unknown, label: string): boolean { + if (typeof value !== "boolean") throw new Error(`${label} must be boolean`); + return value; +} + +function integer( + value: unknown, + label: string, + minimum: number, + maximum: number, +): number { + if ( + typeof value !== "number" + || !Number.isSafeInteger(value) + || value < minimum + || value > maximum + ) throw new Error(`${label} must be an integer from ${minimum} through ${maximum}`); + return value; +} + +function optionalInteger( + value: unknown, + label: string, + minimum: number, + maximum: number, +): number | null { + return value === undefined || value === null + ? null + : integer(value, label, minimum, maximum); +} + +function finiteNumber( + value: unknown, + label: string, + minimum: number, + maximum: number, +): number { + if ( + typeof value !== "number" + || !Number.isFinite(value) + || value < minimum + || value > maximum + ) throw new Error(`${label} must be a bounded number`); + return value; +} + +function optionalFiniteNumber( + value: unknown, + label: string, + minimum: number, + maximum: number, +): number | null { + return value === undefined || value === null + ? null + : finiteNumber(value, label, minimum, maximum); +} + +function timestamp(value: unknown, label: string): string { + const source = boundedString(value, label, 64); + const milliseconds = Date.parse(source); + if (!Number.isFinite(milliseconds)) throw new Error(`${label} must be a timestamp`); + return new Date(milliseconds).toISOString(); +} + +function optionalTimestamp(value: unknown, label: string): string | null { + return value === undefined || value === null ? null : timestamp(value, label); +} + +function parseUser(value: unknown, label: string): BeeperUserProjection { + const source = strictRecord(value, label); + exactKeys(source, ["id"], [ + "cannotMessage", + "displayText", + "email", + "fullName", + "imgURL", + "isSelf", + "phoneNumber", + "username", + ], label); + // imgURL is intentionally validated only as a nullable bounded string and then + // omitted so no local path, media URL, or expiring credential leaves the runtime. + nullableString(source.imgURL, `${label}.imgURL`, 16_384); + nullableString(source.displayText, `${label}.displayText`, 2_048); + return Object.freeze({ + id: boundedString(source.id, `${label}.id`, 2_048), + fullName: nullableString(source.fullName, `${label}.fullName`, 2_048), + username: nullableString(source.username, `${label}.username`, 2_048), + phoneNumber: nullableString(source.phoneNumber, `${label}.phoneNumber`, 128), + email: nullableString(source.email, `${label}.email`, 2_048), + isSelf: optionalBoolean(source.isSelf, `${label}.isSelf`), + cannotMessage: optionalBoolean(source.cannotMessage, `${label}.cannotMessage`), + }); +} + +function parseAccount(value: unknown, label: string): BeeperAccountProjection { + const source = strictRecord(value, label); + exactKeys(source, ["accountID", "bridge", "status", "user"], [ + "capabilities", + "default", + "loginID", + "network", + "statusText", + ], label); + if (source.capabilities !== undefined) strictRecord(source.capabilities, `${label}.capabilities`); + optionalBoolean(source.default, `${label}.default`); + const bridge = strictRecord(source.bridge, `${label}.bridge`); + exactKeys(bridge, ["id", "provider", "type"], [], `${label}.bridge`); + const provider = boundedString(bridge.provider, `${label}.bridge.provider`, 64); + if ( + provider !== "cloud" + && provider !== "self-hosted" + && provider !== "local" + && provider !== "platform-sdk" + ) throw new Error(`${label}.bridge.provider is unsupported`); + return Object.freeze({ + accountId: boundedString(source.accountID, `${label}.accountID`, 512), + bridge: Object.freeze({ + id: boundedString(bridge.id, `${label}.bridge.id`, 512), + type: boundedString(bridge.type, `${label}.bridge.type`, 512), + provider, + }), + network: nullableString(source.network, `${label}.network`, 512), + loginId: nullableString(source.loginID, `${label}.loginID`, 512), + status: boundedString(source.status, `${label}.status`, 128), + statusText: nullableString(source.statusText, `${label}.statusText`, 2_048), + user: parseUser(source.user, `${label}.user`), + }); +} + +function parseAccounts(value: unknown): readonly BeeperAccountProjection[] { + const accounts = strictArray(value, "Beeper accounts", MAX_ACCOUNTS) + .map((item, index) => parseAccount(item, `Beeper accounts[${index}]`)); + const ids = accounts.map((account) => account.accountId); + if (new Set(ids).size !== ids.length) throw new Error("Beeper accounts repeat an account ID"); + return Object.freeze(accounts); +} + +export function parseBeeperExportAccounts( + value: unknown, +): readonly BeeperAccountProjection[] { + return parseAccounts(value); +} + +export function beeperSubjectFromAccounts( + accounts: readonly BeeperAccountProjection[], +): string { + const candidates = accounts.filter((account) => + account.user.isSelf === true + && ( + account.bridge.type.toLowerCase() === "matrix" + || account.network?.toLowerCase() === "beeper" + )); + if (candidates.length !== 1) { + throw new Error("Beeper local projection did not expose one stable self Matrix identity"); + } + const account = candidates[0]!; + const digest = createHash("sha256") + .update(account.accountId, "utf8") + .update("\0", "utf8") + .update(account.user.id, "utf8") + .digest("hex"); + return `beeper:local:${digest}`; +} + +function parseParticipant(value: unknown, label: string): BeeperParticipantProjection { + const source = strictRecord(value, label); + exactKeys(source, ["id"], [ + "cannotMessage", + "displayText", + "email", + "fullName", + "imgURL", + "isSelf", + "phoneNumber", + "username", + "isAdmin", + "isNetworkBot", + "isPending", + ], label); + const user = parseUser( + Object.freeze(Object.fromEntries(Object.entries(source).filter(([key]) => + key !== "isAdmin" && key !== "isNetworkBot" && key !== "isPending"))), + label, + ); + return Object.freeze({ + ...user, + isAdmin: optionalBoolean(source.isAdmin, `${label}.isAdmin`), + isNetworkBot: optionalBoolean(source.isNetworkBot, `${label}.isNetworkBot`), + isPending: optionalBoolean(source.isPending, `${label}.isPending`), + }); +} + +function parseConversation( + value: unknown, + label: string, + accountIds: ReadonlySet, + expectedAccountId: string | null, +): BeeperConversationProjection { + const source = strictRecord(value, label); + exactKeys(source, [ + "id", + "accountID", + "network", + "participants", + "title", + "type", + "unreadCount", + ], [ + "capabilities", + "description", + "draft", + "imgURL", + "isArchived", + "isLowPriority", + "isMarkedUnread", + "isMuted", + "isPinned", + "isReadOnly", + "lastActivity", + "lastReadMessageSortKey", + "localChatID", + "messageExpirySeconds", + "preview", + "reminder", + "snooze", + "unreadMentionsCount", + ], label); + const accountId = boundedString(source.accountID, `${label}.accountID`, 512); + if (!accountIds.has(accountId) || (expectedAccountId !== null && accountId !== expectedAccountId)) { + throw new Error(`${label}.accountID did not bind the requested account realm`); + } + if (source.capabilities !== undefined) strictRecord(source.capabilities, `${label}.capabilities`); + if (source.draft !== undefined && source.draft !== null) strictRecord(source.draft, `${label}.draft`); + nullableString(source.imgURL, `${label}.imgURL`, 16_384); + nullableString(source.lastReadMessageSortKey, `${label}.lastReadMessageSortKey`, 2_048); + if (source.preview !== undefined) strictRecord(source.preview, `${label}.preview`); + if (source.reminder !== undefined && source.reminder !== null) strictRecord(source.reminder, `${label}.reminder`); + if (source.snooze !== undefined && source.snooze !== null) strictRecord(source.snooze, `${label}.snooze`); + const type = boundedString(source.type, `${label}.type`, 32); + if (type !== "single" && type !== "group") throw new Error(`${label}.type is unsupported`); + const participants = strictRecord(source.participants, `${label}.participants`); + exactKeys(participants, ["hasMore", "items", "total"], [], `${label}.participants`); + const participantItems = strictArray( + participants.items, + `${label}.participants.items`, + 2_000, + ).map((item, index) => + parseParticipant(item, `${label}.participants.items[${index}]`)); + return Object.freeze({ + id: boundedString(source.id, `${label}.id`, 2_048), + localChatId: nullableString(source.localChatID, `${label}.localChatID`, 2_048), + accountId, + network: boundedString(source.network, `${label}.network`, 512), + title: boundedString(source.title, `${label}.title`, 4_096, true), + type, + description: nullableString(source.description, `${label}.description`, 65_536), + lastActivity: optionalTimestamp(source.lastActivity, `${label}.lastActivity`), + unreadCount: integer(source.unreadCount, `${label}.unreadCount`, 0, 100_000_000), + unreadMentionsCount: optionalInteger( + source.unreadMentionsCount, + `${label}.unreadMentionsCount`, + 0, + 100_000_000, + ), + isMarkedUnread: optionalBoolean(source.isMarkedUnread, `${label}.isMarkedUnread`), + isArchived: optionalBoolean(source.isArchived, `${label}.isArchived`), + isLowPriority: optionalBoolean(source.isLowPriority, `${label}.isLowPriority`), + isMuted: optionalBoolean(source.isMuted, `${label}.isMuted`), + isPinned: optionalBoolean(source.isPinned, `${label}.isPinned`), + isReadOnly: optionalBoolean(source.isReadOnly, `${label}.isReadOnly`), + messageExpirySeconds: optionalInteger( + source.messageExpirySeconds, + `${label}.messageExpirySeconds`, + 0, + Number.MAX_SAFE_INTEGER, + ), + participants: Object.freeze({ + items: Object.freeze(participantItems), + total: integer(participants.total, `${label}.participants.total`, 0, 100_000_000), + hasMore: requiredBoolean(participants.hasMore, `${label}.participants.hasMore`), + }), + }); +} + +export function parseBeeperExportConversation( + value: unknown, + accounts: readonly BeeperAccountProjection[], +): BeeperConversationProjection { + return parseConversation( + value, + "Beeper export chat", + new Set(accounts.map((account) => account.accountId)), + null, + ); +} + +function parseAttachment(value: unknown, label: string): BeeperAttachmentProjection { + const source = strictRecord(value, label); + exactKeys(source, ["type"], [ + "duration", + "fileName", + "fileSize", + "id", + "isGif", + "isSticker", + "isVoiceNote", + "mimeType", + "posterImg", + "size", + "srcURL", + "transcription", + ], label); + const type = boundedString(source.type, `${label}.type`, 32); + if (type !== "unknown" && type !== "img" && type !== "video" && type !== "audio") { + throw new Error(`${label}.type is unsupported`); + } + // Validate but never project provider IDs, local paths, or media URLs. + nullableString(source.id, `${label}.id`, 16_384); + nullableString(source.posterImg, `${label}.posterImg`, 16_384); + nullableString(source.srcURL, `${label}.srcURL`, 16_384); + let width: number | null = null; + let height: number | null = null; + if (source.size !== undefined && source.size !== null) { + const size = strictRecord(source.size, `${label}.size`); + exactKeys(size, [], ["height", "width"], `${label}.size`); + width = optionalInteger(size.width, `${label}.size.width`, 0, 1_000_000); + height = optionalInteger(size.height, `${label}.size.height`, 0, 1_000_000); + } + let transcription: BeeperAttachmentProjection["transcription"] = null; + if (source.transcription !== undefined && source.transcription !== null) { + const value = strictRecord(source.transcription, `${label}.transcription`); + exactKeys(value, ["engine", "transcription"], ["language"], `${label}.transcription`); + transcription = Object.freeze({ + engine: boundedString(value.engine, `${label}.transcription.engine`, 512), + text: boundedString( + value.transcription, + `${label}.transcription.transcription`, + MAX_TEXT_BYTES, + true, + ), + language: nullableString(value.language, `${label}.transcription.language`, 128), + }); + } + return Object.freeze({ + type, + durationSeconds: optionalFiniteNumber(source.duration, `${label}.duration`, 0, 31_536_000), + fileName: nullableString(source.fileName, `${label}.fileName`, 4_096), + fileSizeBytes: optionalInteger(source.fileSize, `${label}.fileSize`, 0, Number.MAX_SAFE_INTEGER), + mimeType: nullableString(source.mimeType, `${label}.mimeType`, 256), + width, + height, + isGif: optionalBoolean(source.isGif, `${label}.isGif`), + isSticker: optionalBoolean(source.isSticker, `${label}.isSticker`), + isVoiceNote: optionalBoolean(source.isVoiceNote, `${label}.isVoiceNote`), + transcription, + }); +} + +function parseReaction(value: unknown, label: string): BeeperReactionProjection { + const source = strictRecord(value, label); + exactKeys(source, ["id", "participantID", "reactionKey"], ["emoji", "imgURL"], label); + nullableString(source.imgURL, `${label}.imgURL`, 16_384); + return Object.freeze({ + id: boundedString(source.id, `${label}.id`, 2_048), + participantId: boundedString(source.participantID, `${label}.participantID`, 2_048), + reactionKey: boundedString(source.reactionKey, `${label}.reactionKey`, 2_048, true), + emoji: optionalBoolean(source.emoji, `${label}.emoji`), + }); +} + +function parseSeen( + value: unknown, + label: string, +): BeeperMessageProjection["seen"] { + if (value === undefined || value === null) return null; + if (typeof value === "boolean") return value; + if (typeof value === "string") return boundedString(value, label, 2_048, true); + const source = strictRecord(value, label); + if (Object.keys(source).length > 2_000) throw new Error(`${label} contains too many entries`); + const result: Record = Object.create(null) as Record; + for (const [key, item] of Object.entries(source)) { + const safeKey = boundedString(key, `${label} key`, 2_048); + result[safeKey] = typeof item === "boolean" + ? item + : boundedString(item, `${label}.${safeKey}`, 2_048, true); + } + return Object.freeze(result); +} + +function parseMessage( + value: unknown, + label: string, + expected: BeeperMessagingReadInput, +): BeeperMessageProjection { + const source = strictRecord(value, label); + exactKeys(source, [ + "id", + "accountID", + "chatID", + "senderID", + "isSender", + "sortKey", + "timestamp", + ], [ + "attachments", + "editedTimestamp", + "isDeleted", + "isHidden", + "isUnread", + "linkedMessageID", + "links", + "mentions", + "reactions", + "seen", + "senderName", + "sendStatus", + "text", + "type", + ], label); + const accountId = boundedString(source.accountID, `${label}.accountID`, 512); + const conversationId = boundedString(source.chatID, `${label}.chatID`, 2_048); + if (accountId !== expected.accountId || conversationId !== expected.conversationId) { + throw new Error(`${label} did not bind the requested account and conversation`); + } + if (source.links !== undefined) { + for (const [index, item] of strictArray(source.links, `${label}.links`, 1_000).entries()) { + const link = strictRecord(item, `${label}.links[${index}]`); + exactKeys(link, ["title", "url"], [ + "favicon", + "img", + "imgSize", + "originalURL", + "summary", + ], `${label}.links[${index}]`); + boundedString(link.title, `${label}.links[${index}].title`, 8_192, true); + boundedString(link.url, `${label}.links[${index}].url`, 16_384); + nullableString(link.favicon, `${label}.links[${index}].favicon`, 16_384); + nullableString(link.img, `${label}.links[${index}].img`, 16_384); + nullableString(link.originalURL, `${label}.links[${index}].originalURL`, 16_384); + nullableString(link.summary, `${label}.links[${index}].summary`, 65_536); + if (link.imgSize !== undefined && link.imgSize !== null) { + const size = strictRecord(link.imgSize, `${label}.links[${index}].imgSize`); + exactKeys(size, [], ["height", "width"], `${label}.links[${index}].imgSize`); + optionalInteger(size.height, `${label}.links[${index}].imgSize.height`, 0, 1_000_000); + optionalInteger(size.width, `${label}.links[${index}].imgSize.width`, 0, 1_000_000); + } + } + } + if (source.sendStatus !== undefined) { + const status = strictRecord(source.sendStatus, `${label}.sendStatus`); + exactKeys(status, ["status", "timestamp"], [ + "deliveredToUsers", + "internalError", + "message", + "reason", + ], `${label}.sendStatus`); + boundedString(status.status, `${label}.sendStatus.status`, 64); + timestamp(status.timestamp, `${label}.sendStatus.timestamp`); + nullableString(status.internalError, `${label}.sendStatus.internalError`, 65_536); + nullableString(status.message, `${label}.sendStatus.message`, 65_536); + nullableString(status.reason, `${label}.sendStatus.reason`, 2_048); + if (status.deliveredToUsers !== undefined) { + strictArray(status.deliveredToUsers, `${label}.sendStatus.deliveredToUsers`, 2_000) + .forEach((item, index) => + boundedString(item, `${label}.sendStatus.deliveredToUsers[${index}]`, 2_048)); + } + } + const isDeleted = optionalBoolean(source.isDeleted, `${label}.isDeleted`) ?? false; + const isHidden = optionalBoolean(source.isHidden, `${label}.isHidden`) ?? false; + const text = nullableString(source.text, `${label}.text`, MAX_TEXT_BYTES); + const mentions = source.mentions === undefined || source.mentions === null + ? null + : Object.freeze(strictArray(source.mentions, `${label}.mentions`, 2_000) + .map((item, index) => + boundedString(item, `${label}.mentions[${index}]`, 2_048))); + const attachments = source.attachments === undefined + ? [] + : strictArray(source.attachments, `${label}.attachments`, 256) + .map((item, index) => parseAttachment(item, `${label}.attachments[${index}]`)); + const reactions = source.reactions === undefined + ? [] + : strictArray(source.reactions, `${label}.reactions`, 10_000) + .map((item, index) => parseReaction(item, `${label}.reactions[${index}]`)); + return Object.freeze({ + id: boundedString(source.id, `${label}.id`, 2_048), + accountId, + conversationId, + senderId: boundedString(source.senderID, `${label}.senderID`, 2_048), + senderName: nullableString(source.senderName, `${label}.senderName`, 2_048), + isSender: requiredBoolean(source.isSender, `${label}.isSender`), + sortKey: boundedString(source.sortKey, `${label}.sortKey`, 1_024), + timestamp: timestamp(source.timestamp, `${label}.timestamp`), + editedTimestamp: optionalTimestamp(source.editedTimestamp, `${label}.editedTimestamp`), + text: isDeleted || isHidden ? null : text, + type: nullableString(source.type, `${label}.type`, 128), + linkedMessageId: nullableString(source.linkedMessageID, `${label}.linkedMessageID`, 2_048), + mentions, + isDeleted, + isHidden, + isUnread: optionalBoolean(source.isUnread, `${label}.isUnread`), + seen: parseSeen(source.seen, `${label}.seen`), + attachments: Object.freeze(attachments), + reactions: Object.freeze(reactions), + }); +} + +export function parseBeeperExportMessages( + value: unknown, + accountId: string, + conversationId: string, + maximum: number, +): readonly BeeperMessageProjection[] { + if (!Number.isSafeInteger(maximum) || maximum < 1 || maximum > 1_000_000) { + throw new Error("Beeper export message bound is invalid"); + } + const expected: BeeperMessagingReadInput = Object.freeze({ + accountId, + conversationId, + beforeCursor: null, + afterCursor: null, + limit: Math.min(maximum, 200), + }); + const messages = strictArray(value, "Beeper export messages", maximum) + .map((item, index) => parseMessage( + item, + `Beeper export messages[${index}]`, + expected, + )); + const ids = messages.map((message) => message.id); + if (new Set(ids).size !== ids.length) { + throw new Error("Beeper export messages repeat a stable ID"); + } + return Object.freeze(messages); +} + +export function parseBeeperCliEnvelope(value: unknown, label: string): unknown { + const source = strictRecord(value, label); + exactKeys(source, ["success", "data", "error"], [], label); + if (source.success !== true || source.error !== null) { + throw new Error(`${label} did not report success`); + } + return source.data; +} + +function parseJsonOutput(stdout: string, label: string): unknown { + const raw = stdout.trim(); + if (raw.length === 0) throw new Error(`${label} omitted JSON output`); + let value: unknown; + try { + value = JSON.parse(raw) as unknown; + } catch { + throw new Error(`${label} returned malformed JSON`); + } + return parseBeeperCliEnvelope(value, label); +} + +function parseContacts( + value: unknown, + accountIds: ReadonlySet, + expectedAccountId: string | null, +): readonly Readonly<{ accountId: string; user: BeeperUserProjection }>[] { + return Object.freeze(strictArray(value, "Beeper contacts", MAX_USERS).map((item, index) => { + const source = strictRecord(item, `Beeper contacts[${index}]`); + const accountId = boundedString(source.accountID, `Beeper contacts[${index}].accountID`, 512); + if (!accountIds.has(accountId) || (expectedAccountId !== null && accountId !== expectedAccountId)) { + throw new Error(`Beeper contacts[${index}].accountID did not bind the requested realm`); + } + const user = parseUser( + Object.freeze(Object.fromEntries(Object.entries(source).filter(([key]) => key !== "accountID"))), + `Beeper contacts[${index}]`, + ); + return Object.freeze({ accountId, user }); + })); +} + +function requireBeeperAuth(auth: WrenchAuth): BeeperAuth { + if (auth.kind !== "linked-device-store" || auth.provider !== "beeper") { + throw new Error("Beeper local reads require a beeper linked-device-store auth locator"); + } + return auth; +} + +function sha256File(path: string): Promise { + return new Promise((resolve, reject) => { + const hash = createHash("sha256"); + const stream = createReadStream(path); + stream.on("data", (chunk) => hash.update(chunk)); + stream.on("error", reject); + stream.on("end", () => resolve(hash.digest("hex"))); + }); +} + +async function pinnedBinaryCandidate(path: string): Promise { + let canonical: string; + try { + canonical = await realpath(path); + } catch { + return null; + } + const stats = await lstat(canonical); + if ( + !stats.isFile() + || (stats.mode & 0o022) !== 0 + || (stats.mode & 0o111) === 0 + || (stats.uid !== process.getuid?.() && stats.uid !== 0) + || process.platform !== "darwin" + || process.arch !== "arm64" + ) return null; + return await sha256File(canonical) === BEEPER_CLI_PIN.darwinArm64BinarySha256 + ? canonical + : null; +} + +export async function resolvePinnedBeeperCliBinary( + environment: Readonly> = process.env, +): Promise { + const candidates = [ + join(wrenchStateHome(environment), "tools", "beeper", BEEPER_CLI_PIN.version, "beeper"), + "/opt/homebrew/bin/beeper", + "/usr/local/bin/beeper", + ]; + for (const candidate of candidates) { + const found = await pinnedBinaryCandidate(candidate); + if (found !== null) return found; + } + throw new Error( + `pinned Beeper CLI ${BEEPER_CLI_PIN.version} is not installed or failed integrity verification`, + ); +} + +function localDesktopBaseUrl(value: unknown, label: string): void { + if (value === undefined) return; + const source = boundedString(value, label, 256); + let url: URL; + try { + url = new URL(source); + } catch { + throw new Error(`${label} must be a loopback Beeper Desktop URL`); + } + const port = Number(url.port); + if ( + url.protocol !== "http:" + || url.hostname !== "127.0.0.1" + || url.username !== "" + || url.password !== "" + || url.pathname !== "/" + || url.search !== "" + || url.hash !== "" + || !Number.isSafeInteger(port) + || port < 23_373 + || port > 23_392 + ) throw new Error(`${label} must be a reviewed loopback Beeper Desktop URL`); +} + +async function readPrivateJsonFile(path: string, label: string): Promise { + let handle; + try { + handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + } catch (error) { + if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") { + return null; + } + throw error; + } + try { + const before = await handle.stat(); + if ( + !before.isFile() + || before.uid !== process.getuid?.() + || (before.mode & 0o077) !== 0 + || before.size < 2 + || before.size > 4 * 1024 * 1024 + ) throw new Error(`${label} must be one private regular file`); + const bytes = Buffer.allocUnsafe(before.size); + let offset = 0; + while (offset < bytes.byteLength) { + const result = await handle.read( + bytes, + offset, + bytes.byteLength - offset, + offset, + ); + if (result.bytesRead === 0) break; + offset += result.bytesRead; + } + const overflow = Buffer.allocUnsafe(1); + const extra = await handle.read(overflow, 0, 1, offset); + const after = await handle.stat(); + if ( + offset !== bytes.byteLength + || extra.bytesRead !== 0 + || before.dev !== after.dev + || before.ino !== after.ino + || before.size !== after.size + || before.mtimeMs !== after.mtimeMs + || before.ctimeMs !== after.ctimeMs + ) throw new Error(`${label} changed while it was being read`); + const pathStats = await lstat(path); + if ( + pathStats.isSymbolicLink() + || pathStats.dev !== after.dev + || pathStats.ino !== after.ino + ) throw new Error(`${label} changed while it was being read`); + let value: unknown; + try { + const decoded = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + value = JSON.parse(decoded) as unknown; + } catch { + throw new Error(`${label} must contain valid UTF-8 JSON`); + } + return strictRecord(value, label); + } finally { + await handle.close(); + } +} + +export async function validateBeeperCliStore(path: string): Promise { + if (!isAbsolute(path)) throw new Error("Beeper CLI config directory must be absolute"); + const canonical = await realpath(path); + if (canonical !== path) throw new Error("Beeper CLI config directory must be canonical"); + const stats = await lstat(canonical); + if ( + !stats.isDirectory() + || stats.isSymbolicLink() + || stats.uid !== process.getuid?.() + || (stats.mode & 0o022) !== 0 + ) throw new Error("Beeper CLI config directory must be an owned non-writable-by-others directory"); + const config = await readPrivateJsonFile(join(canonical, "config.json"), "Beeper CLI config"); + if (config !== null) { + exactKeys(config, [], ["auth", "baseURL", "defaultAccount", "defaultTarget"], "Beeper CLI config"); + localDesktopBaseUrl(config.baseURL, "Beeper CLI config.baseURL"); + if (config.defaultTarget !== BEEPER_DESKTOP_TARGET) { + throw new Error("Beeper CLI config must select the fixed desktop target"); + } + } + const targetsPath = join(canonical, "targets"); + const canonicalTargets = await realpath(targetsPath); + const targetDirectoryStats = await lstat(targetsPath); + if ( + canonicalTargets !== targetsPath + || !targetDirectoryStats.isDirectory() + || targetDirectoryStats.isSymbolicLink() + || targetDirectoryStats.uid !== process.getuid?.() + || (targetDirectoryStats.mode & 0o022) !== 0 + ) { + throw new Error( + "Beeper CLI targets directory must be an owned physical non-writable-by-others directory", + ); + } + const target = await readPrivateJsonFile( + join(canonicalTargets, "desktop.json"), + "Beeper Desktop target", + ); + if (target !== null) { + exactKeys(target, ["id", "type", "baseURL"], [ + "auth", + "dataDir", + "managed", + "name", + "port", + "profile", + "runtime", + "serverEnv", + ], "Beeper Desktop target"); + if (target.id !== "desktop" || target.type !== "desktop") { + throw new Error("Beeper Desktop target must identify the fixed desktop realm"); + } + localDesktopBaseUrl(target.baseURL, "Beeper Desktop target.baseURL"); + } + if (config === null || target === null) { + throw new Error("Beeper CLI config directory has no authorized selected Desktop target"); + } + return canonical; +} + +async function readBoundedStream( + stream: ReadableStream, + maximum: number, + label: string, +): Promise { + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let byteLength = 0; + try { + for (;;) { + const item = await reader.read(); + if (item.done) break; + if (item.value.byteLength > maximum - byteLength) { + throw new Error(`${label} exceeded its byte bound`); + } + chunks.push(item.value.slice()); + byteLength += item.value.byteLength; + } + } finally { + reader.releaseLock(); + } + const output = new Uint8Array(byteLength); + let offset = 0; + for (const chunk of chunks) { + output.set(chunk, offset); + offset += chunk.byteLength; + } + return new TextDecoder("utf-8", { fatal: true }).decode(output); +} + +async function runBeeperCli( + invocation: BeeperCliInvocation, +): Promise { + if (invocation.signal?.aborted === true) throw new Error("Beeper CLI command was cancelled"); + const ownsProcessGroup = process.platform !== "win32"; + const child = Bun.spawn([invocation.binary, ...invocation.arguments], { + env: { ...invocation.environment }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + detached: ownsProcessGroup, + }); + let timedOut = false; + let cancelled = false; + let forceKill: ReturnType | null = null; + const signalChild = (signal: "SIGTERM" | "SIGKILL"): void => { + try { + if (ownsProcessGroup) process.kill(-child.pid, signal); + else child.kill(signal); + } catch { + // The complete CLI process group already exited. + } + }; + const terminate = (): void => { + signalChild("SIGTERM"); + if (forceKill === null) forceKill = setTimeout(() => signalChild("SIGKILL"), 1_000); + }; + const onAbort = (): void => { + cancelled = true; + terminate(); + }; + invocation.signal?.addEventListener("abort", onAbort, { once: true }); + const timeout = setTimeout(() => { + timedOut = true; + terminate(); + }, invocation.timeoutMs); + try { + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + readBoundedStream(child.stdout, invocation.maxOutputBytes, "Beeper CLI stdout"), + readBoundedStream(child.stderr, invocation.maxStderrBytes, "Beeper CLI stderr"), + ]); + if (cancelled) throw new Error("Beeper CLI command was cancelled"); + if (timedOut) throw new Error("Beeper CLI command timed out"); + return Object.freeze({ exitCode, stdout, stderr }); + } catch (error) { + signalChild("SIGKILL"); + await child.exited; + throw error; + } finally { + clearTimeout(timeout); + if (forceKill !== null) clearTimeout(forceKill); + invocation.signal?.removeEventListener("abort", onAbort); + } +} + +function remainingTimeoutMs( + timeoutMs: number, + deadline: WebSessionOperationDeadline | undefined, +): number { + deadline?.throwIfUnavailable(OPERATION_LABEL); + const remaining = Math.min(timeoutMs, deadline?.remainingTimeMs() ?? timeoutMs); + if (remaining < 1) throw new Error("Beeper local read operation timed out"); + return remaining; +} + +function environmentForBeeper( + configDirectory: string, + cacheDirectory: string, +): Readonly> { + return Object.freeze({ + PATH: "/usr/bin:/bin", + LANG: "C.UTF-8", + CI: "1", + BEEPER_CLI_CONFIG_DIR: configDirectory, + BEEPER_CLI_BINARY_CACHE_DIR: cacheDirectory, + BEEPER_READONLY: "1", + BEEPER_QUIET: "1", + BEEPER_SKIP_UPDATE_CHECK: "1", + NO_UPDATE_NOTIFIER: "1", + }); +} + +async function executeCommand( + binary: string, + command: BeeperReadCommand, + environment: Readonly>, + timeoutMs: number, + maxOutputBytes: number, + dependencies: BeeperLocalRuntimeDependencies | undefined, + deadline: WebSessionOperationDeadline | undefined, +): Promise { + const run = dependencies?.run ?? runBeeperCli; + const invoke = () => run({ + binary, + arguments: command.argv, + environment, + timeoutMs: remainingTimeoutMs(timeoutMs, deadline), + maxOutputBytes, + maxStderrBytes: MAX_STDERR_BYTES, + ...(deadline === undefined ? {} : { signal: deadline.signal }), + }); + const result = deadline === undefined + ? await invoke() + : await deadline.run(invoke, OPERATION_LABEL); + deadline?.throwIfUnavailable(OPERATION_LABEL); + if (result.exitCode !== 0 || result.stderr.trim().length !== 0) { + throw new Error("Beeper CLI read failed before producing reviewed output"); + } + return parseJsonOutput(result.stdout, `Beeper CLI ${command.action}`); +} + +async function withRuntime( + auth: BeeperAuth, + timeoutMs: number, + maxOutputBytes: number, + dependencies: BeeperLocalRuntimeDependencies | undefined, + environment: Readonly>, + deadline: WebSessionOperationDeadline | undefined, + operation: (context: Readonly<{ + binary: string; + environment: Readonly>; + accounts: readonly BeeperAccountProjection[]; + subject: string; + run: (command: BeeperReadCommand, maximum?: number) => Promise; + }>) => Promise, +): Promise { + const configDirectory = await validateBeeperCliStore(auth.path); + const binary = dependencies?.binaryPath ?? await resolvePinnedBeeperCliBinary(environment); + if (dependencies?.binaryPath !== undefined && !isAbsolute(binary)) { + throw new Error("test Beeper CLI binary path must be absolute"); + } + const createCache = dependencies?.createCacheDirectory + ?? (() => mkdtemp(join(tmpdir(), "wrench-beeper-cli-"))); + const removeCache = dependencies?.removeCacheDirectory + ?? ((path: string) => rm(path, { recursive: true, force: true })); + const cacheDirectory = await createCache(); + if (!isAbsolute(cacheDirectory)) throw new Error("Beeper CLI cache directory must be absolute"); + const childEnvironment = environmentForBeeper(configDirectory, cacheDirectory); + const run = (command: BeeperReadCommand, maximum = maxOutputBytes): Promise => + executeCommand( + binary, + command, + childEnvironment, + timeoutMs, + maximum, + dependencies, + deadline, + ); + try { + const version = await executeCommand( + binary, + Object.freeze({ + action: "accounts.list", + argv: Object.freeze(["version", "--read-only", "--json", "--quiet"]), + }), + childEnvironment, + timeoutMs, + 4_096, + dependencies, + deadline, + ); + const versionRecord = strictRecord(version, "Beeper CLI version"); + exactKeys(versionRecord, ["name", "version"], [], "Beeper CLI version"); + if (versionRecord.name !== "@beeper/cli" || versionRecord.version !== BEEPER_CLI_PIN.version) { + throw new Error("Beeper CLI runtime version did not match its pin"); + } + const accounts = parseAccounts(await run(planBeeperAccountsListCommand(timeoutMs), 8 * 1024 * 1024)); + const subject = beeperSubjectFromAccounts(accounts); + if (auth.subject !== undefined && auth.subject !== subject) { + throw new Error("Beeper CLI current account did not match the bound auth realm"); + } + return await operation(Object.freeze({ binary, environment: childEnvironment, accounts, subject, run })); + } finally { + await removeCache(cacheDirectory); + } +} + +export async function probeBeeperLocalSubject( + authValue: WrenchAuth, + options: { + readonly signal?: AbortSignal; + readonly dependencies?: BeeperLocalRuntimeDependencies; + readonly environment?: Readonly>; + } = {}, +): Promise { + const auth = requireBeeperAuth(authValue); + const deadline = new OperationDeadline(SUBJECT_PROBE_TIMEOUT_MS, { + ...(options.signal === undefined ? {} : { signal: options.signal }), + }); + try { + return await withRuntime( + auth, + SUBJECT_PROBE_TIMEOUT_MS, + 8 * 1024 * 1024, + options.dependencies, + options.environment ?? process.env, + deadline, + async ({ subject }) => subject, + ); + } finally { + deadline.dispose(); + } +} + +function unavailableContactStats() { + return projectContactDirectionStats( + Object.freeze({ + count: null, + complete: false, + lowerBound: false, + truncated: false, + lastAt: null, + lastAtComplete: false, + lastAtBasis: "unavailable" as const, + incompleteReasons: Object.freeze(["beeper-message-history-not-scanned"]), + }), + Object.freeze({ + count: null, + complete: false, + lowerBound: false, + truncated: false, + lastAt: null, + lastAtComplete: false, + lastAtBasis: "unavailable" as const, + incompleteReasons: Object.freeze(["beeper-message-history-not-scanned"]), + }), + ); +} + +function contactOutput( + accounts: readonly BeeperAccountProjection[], + subject: string, + input: Extract, + raw: unknown, +) { + const accountId = "accountId" in input ? input.accountId : null; + const accountIds = new Set(accounts.map((account) => account.accountId)); + if (accountId !== null && !accountIds.has(accountId)) { + throw new Error("contacts.list requested an account outside the bound Beeper realm"); + } + const contacts = parseContacts(raw, accountIds, accountId).map((contact) => Object.freeze({ + accountId: contact.accountId, + ...contact.user, + ...unavailableContactStats(), + })); + const limitReached = contacts.length === input.limit; + return Object.freeze({ + provider: "beeper", + operation: "contacts.list", + accountSubject: subject, + projection: "bounded-local-desktop-api", + accounts, + requestedAccountId: accountId, + contacts: Object.freeze(contacts), + completeness: Object.freeze({ + localPageComplete: !limitReached, + remoteContactSetComplete: false, + limitReached, + warnings: Object.freeze([ + "beeper-contact-pagination-cursor-not-exposed-by-cli-v0.6.2", + "provider-history-coverage-varies-by-connected-account", + ]), + }), + }); +} + +function conversationOutput( + accounts: readonly BeeperAccountProjection[], + subject: string, + input: Extract, + raw: unknown, +) { + const accountId = "accountId" in input ? input.accountId : null; + const accountIds = new Set(accounts.map((account) => account.accountId)); + if (accountId !== null && !accountIds.has(accountId)) { + throw new Error("messaging.list requested an account outside the bound Beeper realm"); + } + const conversations = strictArray(raw, "Beeper conversations", MAX_CHATS) + .map((item, index) => parseConversation( + item, + `Beeper conversations[${index}]`, + accountIds, + accountId, + )); + const ids = conversations.map((conversation) => `${conversation.accountId}\0${conversation.id}`); + if (new Set(ids).size !== ids.length) throw new Error("Beeper conversations repeat an account-scoped ID"); + const limitReached = conversations.length === input.limit; + return Object.freeze({ + provider: "beeper", + operation: "messaging.list", + accountSubject: subject, + projection: "bounded-local-desktop-api", + accounts, + requestedAccountId: accountId, + conversations: Object.freeze(conversations), + completeness: Object.freeze({ + localPageComplete: !limitReached, + remoteConversationSetComplete: false, + limitReached, + warnings: Object.freeze([ + "beeper-chat-pagination-cursor-not-exposed-by-cli-v0.6.2", + "newly-connected-accounts-may-have-incomplete-history", + ]), + }), + }); +} + +function messageOutput( + accounts: readonly BeeperAccountProjection[], + subject: string, + input: BeeperMessagingReadInput, + raw: unknown, +) { + if (!accounts.some((account) => account.accountId === input.accountId)) { + throw new Error("messaging.read requested an account outside the bound Beeper realm"); + } + const messages = strictArray(raw, "Beeper messages", MAX_MESSAGES) + .map((item, index) => parseMessage(item, `Beeper messages[${index}]`, input)); + const ids = messages.map((message) => message.id); + if (new Set(ids).size !== ids.length) throw new Error("Beeper messages repeat a stable ID"); + const tombstones = messages.flatMap((message): readonly BeeperTombstoneProjection[] => { + if (!message.isDeleted && !message.isHidden) return []; + return [Object.freeze({ + accountId: message.accountId, + conversationId: message.conversationId, + messageId: message.id, + state: message.isDeleted && message.isHidden + ? "deleted-and-hidden" + : message.isDeleted + ? "deleted" + : "hidden", + observedAt: message.editedTimestamp ?? message.timestamp, + })]; + }); + const limitReached = messages.length === input.limit; + const continuation = limitReached && messages.length > 0 + ? Object.freeze({ + direction: input.afterCursor === null ? "before" : "after", + cursor: messages[messages.length - 1]!.id, + }) + : null; + return Object.freeze({ + provider: "beeper", + operation: "messaging.read", + accountSubject: subject, + projection: "bounded-local-desktop-api", + accountId: input.accountId, + conversationId: input.conversationId, + requestCursor: input.beforeCursor ?? input.afterCursor, + requestDirection: input.afterCursor === null ? "before" : "after", + messages: Object.freeze(messages), + tombstones: Object.freeze(tombstones), + continuation, + completeness: Object.freeze({ + localPageComplete: !limitReached, + remoteConversationHistoryComplete: false, + limitReached, + warnings: Object.freeze([ + "continuation-is-derived-from-terminal-returned-message-id", + "edits-reactions-and-deletions-may-require-overlap-reconciliation", + "newly-connected-accounts-may-have-incomplete-history", + ]), + }), + }); +} + +export async function executeBeeperLocalOperation( + recipe: WebSessionRecipe, + inputValue: OperationInput, + authValue: WrenchAuth, + options: { + readonly dependencies?: BeeperLocalRuntimeDependencies; + readonly environment?: Readonly>; + readonly operationDeadline?: WebSessionOperationDeadline; + readonly registerCleanupBarrier?: WebSessionCleanupBarrierRegistrar; + } = {}, +): Promise { + if ( + recipe.site !== "beeper" + || recipe.contractVersion !== 1 + || !isBeeperLocalOperation(recipe.action) + ) throw new Error("Beeper local read recipe is not installed"); + const action: BeeperLocalOperationName = recipe.action; + const contract = BEEPER_LOCAL_OPERATIONS[action]; + if (contract.state !== "observed" || contract.effect !== "read") { + throw new Error(`Beeper local operation ${action} is not executable`); + } + const input = parseBeeperOperationInput(action, inputValue); + const auth = requireBeeperAuth(authValue); + options.operationDeadline?.throwIfUnavailable(OPERATION_LABEL); + return startWebSessionCleanupTrackedOperation( + options.registerCleanupBarrier, + async () => withRuntime( + auth, + recipe.timeoutMs, + recipe.maxOutputBytes, + options.dependencies, + options.environment ?? process.env, + options.operationDeadline, + async ({ accounts, subject, run }) => { + if (auth.subject === undefined) { + throw new Error("Beeper auth must be account-bound before private reads"); + } + const raw = await run(planBeeperReadCommand(action, input, recipe.timeoutMs)); + const output = action === "contacts.list" + ? contactOutput(accounts, subject, input, raw) + : action === "messaging.list" + ? conversationOutput(accounts, subject, input, raw) + : messageOutput(accounts, subject, input as BeeperMessagingReadInput, raw); + const encoded = Buffer.from(JSON.stringify(output), "utf8"); + if (encoded.byteLength > recipe.maxOutputBytes) { + throw new Error("Beeper local projection exceeded the reviewed output bound"); + } + return Object.freeze({ + status: "succeeded" as const, + output, + finalUrl: BEEPER_ORIGIN, + dispatchStarted: false, + dispatch: Object.freeze({ planned: 0, started: 0, verified: 0 }), + }); + }, + ), + async (operation) => { + await operation.then(() => undefined, () => undefined); + }, + ); +} diff --git a/src/providers/beeper-local.ts b/src/providers/beeper-local.ts new file mode 100644 index 0000000..15039e7 --- /dev/null +++ b/src/providers/beeper-local.ts @@ -0,0 +1,366 @@ +/** + * Beeper Desktop local read policy and fixed official CLI command plans. + * + * Wrench never exposes the CLI's raw API, target, command, URL, or token + * surfaces. The pinned CLI is only a bounded client for the already-authorized + * local Desktop projection. + */ + +import { isAbsolute } from "node:path"; + +import type { OperationInput } from "../model"; + +export const BEEPER_CLI_PIN = Object.freeze({ + implementation: "github.com/beeper/cli", + version: "0.6.2", + commit: "a416af06023449a87312dc11e54643fd9dc94b8c", + darwinArm64ArchiveSha256: + "688ccde7e7d044d33980cd06474bf1ae7215ccf8ca79967262fa3bfb85a2589a", + darwinArm64BinarySha256: + "48aa895449129c793a212ea19f69a534adc34a8adc4037ca1d7da9e648716425", + releaseUrl: + "https://github.com/beeper/cli/releases/tag/v0.6.2", + downloadUrl: + "https://github.com/beeper/cli/releases/download/v0.6.2/beeper-cli-0.6.2-macos-arm64.zip", +} as const); + +export const BEEPER_ORIGIN = "https://www.beeper.com" as const; +export const BEEPER_DESKTOP_TARGET = "desktop" as const; + +export const BEEPER_LOCAL_OPERATION_NAMES = Object.freeze([ + "contacts.list", + "messaging.list", + "messaging.read", +] as const); + +export type BeeperLocalOperationName = + (typeof BEEPER_LOCAL_OPERATION_NAMES)[number]; + +export const BEEPER_LOCAL_OPERATIONS = Object.freeze({ + "contacts.list": Object.freeze({ + effect: "read", + risk: "R1", + state: "observed", + reason: + "the pinned official Beeper CLI reads one bounded account-aware contact projection from local Desktop in read-only mode; it does not download media or expose raw requests", + }), + "messaging.list": Object.freeze({ + effect: "read", + risk: "R1", + state: "observed", + reason: + "the pinned official Beeper CLI reads one bounded local chat projection in read-only mode and preserves account, network, participant, and local-completeness evidence", + }), + "messaging.read": Object.freeze({ + effect: "read", + risk: "R1", + state: "observed", + reason: + "the pinned official Beeper CLI reads one exact account-bound conversation page in read-only mode and preserves reply, edit, deletion, reaction, and attachment-shape evidence", + }), +} as const); + +export type BeeperContactsListInput = Readonly<{ + accountId: string | null; + limit: number; +}>; + +export type BeeperMessagingListInput = Readonly<{ + accountId: string | null; + limit: number; +}>; + +export type BeeperMessagingReadInput = Readonly<{ + accountId: string; + conversationId: string; + beforeCursor: string | null; + afterCursor: string | null; + limit: number; +}>; + +export type BeeperOperationInput = + | BeeperContactsListInput + | BeeperMessagingListInput + | BeeperMessagingReadInput; + +export type BeeperReadCommand = Readonly<{ + action: "accounts.list" | BeeperLocalOperationName; + argv: readonly string[]; +}>; + +export type BeeperMessageLikeMeExportCommandOptions = Readonly<{ + outputDirectory: string; + limitChats: number | null; + limitMessages: number | null; + maxParticipants: number; +}>; + +function record(value: unknown, label: string): Readonly> { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + return value as Readonly>; +} + +function exactKeys( + value: Readonly>, + required: readonly string[], + optional: readonly string[], + label: string, +): void { + const keys = new Set(Object.keys(value)); + for (const key of required) { + if (!keys.delete(key)) throw new Error(`${label} omitted ${key}`); + } + for (const key of optional) keys.delete(key); + if (keys.size > 0) throw new Error(`${label} contained unsupported fields`); +} + +function integer( + value: unknown, + label: string, + minimum: number, + maximum: number, +): number { + if ( + typeof value !== "number" + || !Number.isSafeInteger(value) + || value < minimum + || value > maximum + ) throw new Error(`${label} must be an integer from ${minimum} through ${maximum}`); + return value; +} + +function boundedOpaque(value: unknown, label: string, maximum: number): string { + if ( + typeof value !== "string" + || value.length < 1 + || Buffer.byteLength(value, "utf8") > maximum + || /[\0\r\n]/u.test(value) + ) throw new Error(`${label} must be bounded opaque text`); + return value; +} + +function optionalOpaque( + value: unknown, + label: string, + maximum: number, +): string | null { + return value === undefined ? null : boundedOpaque(value, label, maximum); +} + +export function parseBeeperContactsListInput( + input: OperationInput, +): BeeperContactsListInput { + const source = record(input, "contacts.list input"); + exactKeys(source, [], ["account_id", "limit"], "contacts.list input"); + return Object.freeze({ + accountId: optionalOpaque( + source.account_id, + "contacts.list input.account_id", + 512, + ), + limit: source.limit === undefined + ? 200 + : integer(source.limit, "contacts.list input.limit", 1, 200), + }); +} + +export function parseBeeperMessagingListInput( + input: OperationInput, +): BeeperMessagingListInput { + const source = record(input, "messaging.list input"); + exactKeys(source, [], ["account_id", "limit"], "messaging.list input"); + return Object.freeze({ + accountId: optionalOpaque( + source.account_id, + "messaging.list input.account_id", + 512, + ), + limit: source.limit === undefined + ? 200 + : integer(source.limit, "messaging.list input.limit", 1, 200), + }); +} + +export function parseBeeperMessagingReadInput( + input: OperationInput, +): BeeperMessagingReadInput { + const source = record(input, "messaging.read input"); + exactKeys( + source, + ["account_id", "conversation_id"], + ["before_cursor", "after_cursor", "limit"], + "messaging.read input", + ); + const beforeCursor = optionalOpaque( + source.before_cursor, + "messaging.read input.before_cursor", + 2_048, + ); + const afterCursor = optionalOpaque( + source.after_cursor, + "messaging.read input.after_cursor", + 2_048, + ); + if (beforeCursor !== null && afterCursor !== null) { + throw new Error("messaging.read input accepts only one cursor direction"); + } + return Object.freeze({ + accountId: boundedOpaque( + source.account_id, + "messaging.read input.account_id", + 512, + ), + conversationId: boundedOpaque( + source.conversation_id, + "messaging.read input.conversation_id", + 2_048, + ), + beforeCursor, + afterCursor, + limit: source.limit === undefined + ? 200 + : integer(source.limit, "messaging.read input.limit", 1, 200), + }); +} + +export function parseBeeperOperationInput( + action: BeeperLocalOperationName, + input: OperationInput, +): BeeperOperationInput { + if (action === "contacts.list") return parseBeeperContactsListInput(input); + if (action === "messaging.list") return parseBeeperMessagingListInput(input); + return parseBeeperMessagingReadInput(input); +} + +function globalArguments(timeoutMs: number): readonly string[] { + const seconds = Math.max(1, Math.min(3_600, Math.ceil(timeoutMs / 1_000))); + return Object.freeze([ + "--read-only", + "--json", + "--full", + "--quiet", + "--target", + BEEPER_DESKTOP_TARGET, + "--timeout", + `${seconds}s`, + ]); +} + +export function planBeeperAccountsListCommand(timeoutMs: number): BeeperReadCommand { + return Object.freeze({ + action: "accounts.list", + argv: Object.freeze(["accounts", "list", ...globalArguments(timeoutMs)]), + }); +} + +export function planBeeperMessageLikeMeExportCommand( + options: BeeperMessageLikeMeExportCommandOptions, + timeoutMs: number, +): readonly string[] { + const boundedInteger = ( + value: number, + label: string, + maximum: number, + ): number => { + if (!Number.isSafeInteger(value) || value < 1 || value > maximum) { + throw new Error(`${label} must be an integer from 1 through ${maximum}`); + } + return value; + }; + if (!isAbsolute(options.outputDirectory)) { + throw new Error("Beeper export output directory must be absolute"); + } + const maxParticipants = boundedInteger( + options.maxParticipants, + "Beeper export maxParticipants", + 2_000, + ); + const limitChats = options.limitChats === null + ? null + : boundedInteger(options.limitChats, "Beeper export limitChats", 100_000); + const limitMessages = options.limitMessages === null + ? null + : boundedInteger( + options.limitMessages, + "Beeper export limitMessages", + 1_000_000, + ); + return Object.freeze([ + "export", + "--out", + options.outputDirectory, + "--no-attachments", + "--max-participants", + String(maxParticipants), + ...(limitChats === null ? [] : ["--limit-chats", String(limitChats)]), + ...(limitMessages === null + ? [] + : ["--limit-messages", String(limitMessages)]), + ...globalArguments(timeoutMs).filter((argument) => argument !== "--json" && argument !== "--full"), + ]); +} + +export function planBeeperReadCommand( + action: BeeperLocalOperationName, + input: BeeperOperationInput, + timeoutMs: number, +): BeeperReadCommand { + const common = globalArguments(timeoutMs); + if (action === "contacts.list") { + const value = input as BeeperContactsListInput; + return Object.freeze({ + action, + argv: Object.freeze([ + "contacts", + "list", + "--limit", + String(value.limit), + ...(value.accountId === null ? [] : ["--account", value.accountId]), + ...common, + ]), + }); + } + if (action === "messaging.list") { + const value = input as BeeperMessagingListInput; + return Object.freeze({ + action, + argv: Object.freeze([ + "chats", + "list", + "--limit", + String(value.limit), + ...(value.accountId === null ? [] : ["--account", value.accountId]), + ...common, + ]), + }); + } + const value = input as BeeperMessagingReadInput; + return Object.freeze({ + action, + argv: Object.freeze([ + "messages", + "list", + "--chat", + value.conversationId, + "--limit", + String(value.limit), + ...(value.beforeCursor === null + ? [] + : ["--before-cursor", value.beforeCursor]), + ...(value.afterCursor === null + ? [] + : ["--after-cursor", value.afterCursor]), + ...common, + ]), + }); +} + +export function isBeeperLocalOperation( + value: string, +): value is BeeperLocalOperationName { + return BEEPER_LOCAL_OPERATION_NAMES.includes( + value as BeeperLocalOperationName, + ); +} diff --git a/src/providers/beeper-omni.ts b/src/providers/beeper-omni.ts new file mode 100644 index 0000000..02269b4 --- /dev/null +++ b/src/providers/beeper-omni.ts @@ -0,0 +1,560 @@ +import { types as nodeTypes } from "node:util"; + +import type { OperationInput } from "../model"; +import { OmniMaterializerDriftError } from "../omni-model"; +import type { + OmniAttachmentV1, + OmniParticipantV1, + ProviderConversationV1, + ProviderMaterializedPageV1, + ProviderMessageV1, +} from "../omni-model"; + +type JsonRecord = Readonly>; + +function drift(path: string, message: string): never { + throw new OmniMaterializerDriftError("beeper", path, message); +} + +function record(value: unknown, path: string): JsonRecord { + if ( + nodeTypes.isProxy(value) + || typeof value !== "object" + || value === null + || Array.isArray(value) + || ( + Object.getPrototypeOf(value) !== Object.prototype + && Object.getPrototypeOf(value) !== null + ) + ) return drift(path, "must be a plain object"); + return value as JsonRecord; +} + +function exactKeys( + value: JsonRecord, + required: readonly string[], + optional: readonly string[], + path: string, +): void { + const allowed = new Set([...required, ...optional]); + for (const key of Object.keys(value)) { + if (!allowed.has(key)) drift(path, `contains unreviewed property ${key}`); + } + for (const key of required) { + if (!Object.hasOwn(value, key)) drift(`${path}.${key}`, "is required"); + } +} + +function array(value: unknown, path: string, maximum: number): readonly unknown[] { + if ( + nodeTypes.isProxy(value) + || !Array.isArray(value) + || Object.getPrototypeOf(value) !== Array.prototype + || value.length > maximum + ) return drift(path, `must be an array of at most ${maximum} items`); + for (let index = 0; index < value.length; index += 1) { + if (!Object.hasOwn(value, index)) drift(`${path}[${index}]`, "must not be sparse"); + } + return value; +} + +function string(value: unknown, path: string, maximum: number, allowEmpty = false): string { + if ( + typeof value !== "string" + || (!allowEmpty && value.length === 0) + || Buffer.byteLength(value, "utf8") > maximum + || value.includes("\0") + ) return drift(path, "must be bounded text"); + return value; +} + +function nullableString(value: unknown, path: string, maximum: number): string | null { + return value === null ? null : string(value, path, maximum, true); +} + +function boolean(value: unknown, path: string): boolean { + if (typeof value !== "boolean") return drift(path, "must be boolean"); + return value; +} + +function nullableBoolean(value: unknown, path: string): boolean | null { + return value === null ? null : boolean(value, path); +} + +function integer(value: unknown, path: string, minimum: number, maximum: number): number { + if ( + typeof value !== "number" + || !Number.isSafeInteger(value) + || value < minimum + || value > maximum + ) return drift(path, `must be an integer from ${minimum} through ${maximum}`); + return value; +} + +function nullableInteger( + value: unknown, + path: string, + minimum: number, + maximum: number, +): number | null { + return value === null ? null : integer(value, path, minimum, maximum); +} + +function timestamp(value: unknown, path: string): string { + const source = string(value, path, 64); + const milliseconds = Date.parse(source); + if (!Number.isFinite(milliseconds)) return drift(path, "must be a timestamp"); + return new Date(milliseconds).toISOString(); +} + +function nullableTimestamp(value: unknown, path: string): string | null { + return value === null ? null : timestamp(value, path); +} + +function encoded(value: string): string { + return Buffer.from(value, "utf8").toString("base64url"); +} + +function conversationProviderId(accountId: string, conversationId: string): string { + return `beeper:${encoded(accountId)}:chat:${encoded(conversationId)}`; +} + +function messageProviderId(accountId: string, messageId: string): string { + return `beeper:${encoded(accountId)}:message:${encoded(messageId)}`; +} + +function userProviderId(accountId: string, userId: string): string { + return `beeper:${encoded(accountId)}:user:${encoded(userId)}`; +} + +function subject(value: unknown, path: string): string { + const source = string(value, path, 512); + if (!/^beeper:local:[a-f0-9]{64}$/u.test(source)) { + return drift(path, "must be a hashed bound Beeper local subject"); + } + return source; +} + +function listInput(input: OperationInput): Readonly<{ accountId: string | null; limit: number }> { + const source = record(input, "messaging.list input"); + exactKeys(source, [], ["account_id", "limit"], "messaging.list input"); + return Object.freeze({ + accountId: source.account_id === undefined + ? null + : string(source.account_id, "messaging.list input.account_id", 512), + limit: source.limit === undefined + ? 200 + : integer(source.limit, "messaging.list input.limit", 1, 200), + }); +} + +function readInput(input: OperationInput): Readonly<{ + accountId: string; + conversationId: string; + beforeCursor: string | null; + afterCursor: string | null; + limit: number; +}> { + const source = record(input, "messaging.read input"); + exactKeys( + source, + ["account_id", "conversation_id"], + ["before_cursor", "after_cursor", "limit"], + "messaging.read input", + ); + const beforeCursor = source.before_cursor === undefined + ? null + : string(source.before_cursor, "messaging.read input.before_cursor", 2_048); + const afterCursor = source.after_cursor === undefined + ? null + : string(source.after_cursor, "messaging.read input.after_cursor", 2_048); + if (beforeCursor !== null && afterCursor !== null) { + drift("messaging.read input", "accepts only one cursor direction"); + } + return Object.freeze({ + accountId: string(source.account_id, "messaging.read input.account_id", 512), + conversationId: string( + source.conversation_id, + "messaging.read input.conversation_id", + 2_048, + ), + beforeCursor, + afterCursor, + limit: source.limit === undefined + ? 200 + : integer(source.limit, "messaging.read input.limit", 1, 200), + }); +} + +function participant(value: unknown, path: string, accountId: string): OmniParticipantV1 { + const source = record(value, path); + exactKeys(source, [ + "id", + "fullName", + "username", + "phoneNumber", + "email", + "isSelf", + "cannotMessage", + "isAdmin", + "isNetworkBot", + "isPending", + ], [], path); + const id = string(source.id, `${path}.id`, 2_048); + const displayName = nullableString(source.fullName, `${path}.fullName`, 2_048); + const username = nullableString(source.username, `${path}.username`, 2_048); + nullableString(source.phoneNumber, `${path}.phoneNumber`, 128); + nullableString(source.email, `${path}.email`, 2_048); + nullableBoolean(source.isSelf, `${path}.isSelf`); + nullableBoolean(source.cannotMessage, `${path}.cannotMessage`); + nullableBoolean(source.isAdmin, `${path}.isAdmin`); + nullableBoolean(source.isNetworkBot, `${path}.isNetworkBot`); + nullableBoolean(source.isPending, `${path}.isPending`); + return Object.freeze({ + providerId: userProviderId(accountId, id), + displayName, + handle: username, + }); +} + +function conversation(value: unknown, path: string): ProviderConversationV1 { + const source = record(value, path); + exactKeys(source, [ + "id", + "localChatId", + "accountId", + "network", + "title", + "type", + "description", + "lastActivity", + "unreadCount", + "unreadMentionsCount", + "isMarkedUnread", + "isArchived", + "isLowPriority", + "isMuted", + "isPinned", + "isReadOnly", + "messageExpirySeconds", + "participants", + ], [], path); + const accountId = string(source.accountId, `${path}.accountId`, 512); + const id = string(source.id, `${path}.id`, 2_048); + nullableString(source.localChatId, `${path}.localChatId`, 2_048); + string(source.network, `${path}.network`, 512); + const title = string(source.title, `${path}.title`, 4_096, true); + const type = string(source.type, `${path}.type`, 32); + if (type !== "single" && type !== "group") drift(`${path}.type`, "must be single or group"); + const description = nullableString(source.description, `${path}.description`, 65_536); + const orderedAt = nullableTimestamp(source.lastActivity, `${path}.lastActivity`); + const unreadCount = integer(source.unreadCount, `${path}.unreadCount`, 0, 100_000_000); + nullableInteger(source.unreadMentionsCount, `${path}.unreadMentionsCount`, 0, 100_000_000); + const markedUnread = nullableBoolean(source.isMarkedUnread, `${path}.isMarkedUnread`); + const archived = nullableBoolean(source.isArchived, `${path}.isArchived`); + nullableBoolean(source.isLowPriority, `${path}.isLowPriority`); + nullableBoolean(source.isMuted, `${path}.isMuted`); + nullableBoolean(source.isPinned, `${path}.isPinned`); + nullableBoolean(source.isReadOnly, `${path}.isReadOnly`); + nullableInteger(source.messageExpirySeconds, `${path}.messageExpirySeconds`, 0, Number.MAX_SAFE_INTEGER); + const participants = record(source.participants, `${path}.participants`); + exactKeys(participants, ["items", "total", "hasMore"], [], `${path}.participants`); + const items = array(participants.items, `${path}.participants.items`, 2_000) + .map((item, index) => participant(item, `${path}.participants.items[${index}]`, accountId)); + integer(participants.total, `${path}.participants.total`, 0, 100_000_000); + boolean(participants.hasMore, `${path}.participants.hasMore`); + return Object.freeze({ + kind: "conversation", + providerId: conversationProviderId(accountId, id), + providerRevision: orderedAt, + orderedAt, + detail: "summary", + title, + summary: description, + participants: Object.freeze(items), + unread: markedUnread ?? unreadCount > 0, + unreadCount, + archived, + pending: null, + }); +} + +function attachment(value: unknown, path: string): OmniAttachmentV1 { + const source = record(value, path); + exactKeys(source, [ + "type", + "durationSeconds", + "fileName", + "fileSizeBytes", + "mimeType", + "width", + "height", + "isGif", + "isSticker", + "isVoiceNote", + "transcription", + ], [], path); + const type = string(source.type, `${path}.type`, 32); + nullableInteger(source.width, `${path}.width`, 0, 1_000_000); + nullableInteger(source.height, `${path}.height`, 0, 1_000_000); + nullableBoolean(source.isGif, `${path}.isGif`); + const isSticker = nullableBoolean(source.isSticker, `${path}.isSticker`) === true; + nullableBoolean(source.isVoiceNote, `${path}.isVoiceNote`); + if (source.durationSeconds !== null) { + if (typeof source.durationSeconds !== "number" || !Number.isFinite(source.durationSeconds)) { + drift(`${path}.durationSeconds`, "must be null or a bounded number"); + } + } + if (source.transcription !== null) record(source.transcription, `${path}.transcription`); + const kind: OmniAttachmentV1["kind"] = isSticker + ? "sticker" + : type === "img" + ? "image" + : type === "video" + ? "video" + : type === "audio" + ? "audio" + : "unknown"; + return Object.freeze({ + kind, + mimeType: nullableString(source.mimeType, `${path}.mimeType`, 512), + name: nullableString(source.fileName, `${path}.fileName`, 4_096), + sizeBytes: nullableInteger( + source.fileSizeBytes, + `${path}.fileSizeBytes`, + 0, + Number.MAX_SAFE_INTEGER, + ), + }); +} + +function message( + value: unknown, + path: string, + expectedAccountId: string, + expectedConversationId: string, +): ProviderMessageV1 { + const source = record(value, path); + exactKeys(source, [ + "id", + "accountId", + "conversationId", + "senderId", + "senderName", + "isSender", + "sortKey", + "timestamp", + "editedTimestamp", + "text", + "type", + "linkedMessageId", + "mentions", + "isDeleted", + "isHidden", + "isUnread", + "seen", + "attachments", + "reactions", + ], [], path); + const accountId = string(source.accountId, `${path}.accountId`, 512); + const conversationId = string(source.conversationId, `${path}.conversationId`, 2_048); + if (accountId !== expectedAccountId || conversationId !== expectedConversationId) { + drift(path, "must bind the requested account and conversation"); + } + const id = string(source.id, `${path}.id`, 2_048); + const senderId = string(source.senderId, `${path}.senderId`, 2_048); + const senderName = nullableString(source.senderName, `${path}.senderName`, 2_048); + const isSender = boolean(source.isSender, `${path}.isSender`); + const sortKey = string(source.sortKey, `${path}.sortKey`, 2_048); + const orderedAt = timestamp(source.timestamp, `${path}.timestamp`); + const editedTimestamp = nullableTimestamp(source.editedTimestamp, `${path}.editedTimestamp`); + const body = nullableString(source.text, `${path}.text`, 1_048_576); + nullableString(source.type, `${path}.type`, 128); + const replyId = nullableString(source.linkedMessageId, `${path}.linkedMessageId`, 2_048); + if (source.mentions !== null) array(source.mentions, `${path}.mentions`, 2_000); + const isDeleted = boolean(source.isDeleted, `${path}.isDeleted`); + const isHidden = boolean(source.isHidden, `${path}.isHidden`); + const unread = nullableBoolean(source.isUnread, `${path}.isUnread`); + if (source.seen !== null && typeof source.seen === "object") record(source.seen, `${path}.seen`); + const attachments = array(source.attachments, `${path}.attachments`, 1_000) + .map((item, index) => attachment(item, `${path}.attachments[${index}]`)); + array(source.reactions, `${path}.reactions`, 10_000).forEach((item, index) => + record(item, `${path}.reactions[${index}]`)); + const state: ProviderMessageV1["state"] = isDeleted && isHidden + ? "revoked-and-deleted-for-me" + : isDeleted + ? "revoked" + : isHidden + ? "deleted-for-me" + : "active"; + return Object.freeze({ + kind: "message", + providerId: messageProviderId(accountId, id), + providerRevision: editedTimestamp ?? sortKey, + orderedAt, + conversationProviderId: conversationProviderId(accountId, conversationId), + sender: Object.freeze({ + providerId: userProviderId(accountId, senderId), + displayName: senderName, + handle: null, + }), + recipients: Object.freeze([]), + direction: isSender ? "outgoing" : "incoming", + subject: null, + body, + unread, + replyToProviderId: replyId === null ? null : messageProviderId(accountId, replyId), + state, + attachments: Object.freeze(attachments), + }); +} + +function validateEnvelope( + output: unknown, + operation: "messaging.list" | "messaging.read", +): { readonly source: JsonRecord; readonly subject: string } { + const source = record(output, `${operation} output`); + if (source.provider !== "beeper" || source.operation !== operation) { + drift(`${operation} output`, "must identify the exact Beeper operation"); + } + if (source.projection !== "bounded-local-desktop-api") { + drift(`${operation} output.projection`, "must be bounded-local-desktop-api"); + } + return Object.freeze({ + source, + subject: subject(source.accountSubject, `${operation} output.accountSubject`), + }); +} + +export function materializeBeeperMessagingList( + input: OperationInput, + output: unknown, +): ProviderMaterializedPageV1 { + const parsed = listInput(input); + const { source, subject: accountSubject } = validateEnvelope(output, "messaging.list"); + exactKeys(source, [ + "provider", + "operation", + "accountSubject", + "projection", + "accounts", + "requestedAccountId", + "conversations", + "completeness", + ], [], "messaging.list output"); + array(source.accounts, "messaging.list output.accounts", 128); + if (source.requestedAccountId !== parsed.accountId) { + drift("messaging.list output.requestedAccountId", "must bind input.account_id"); + } + const entities = array( + source.conversations, + "messaging.list output.conversations", + parsed.limit, + ).map((item, index) => conversation(item, `messaging.list output.conversations[${index}]`)); + const ids = entities.map((entity) => entity.providerId); + if (new Set(ids).size !== ids.length) { + drift("messaging.list output.conversations", "contains duplicate account-scoped IDs"); + } + record(source.completeness, "messaging.list output.completeness"); + return Object.freeze({ + schemaVersion: 1, + partition: `${accountSubject}:conversations:${parsed.accountId ?? "all"}`, + completeness: Object.freeze({ + kind: "bounded-local", + reason: "Beeper exposed a bounded local Desktop projection; CLI v0.6.2 does not expose chat-list continuation metadata.", + }), + cursor: Object.freeze({ direction: "none", request: null, nextInput: null }), + entities: Object.freeze(entities), + tombstones: Object.freeze([]), + }); +} + +export function materializeBeeperMessagingRead( + input: OperationInput, + output: unknown, +): ProviderMaterializedPageV1 { + const parsed = readInput(input); + const { source, subject: accountSubject } = validateEnvelope(output, "messaging.read"); + exactKeys(source, [ + "provider", + "operation", + "accountSubject", + "projection", + "accountId", + "conversationId", + "requestCursor", + "requestDirection", + "messages", + "tombstones", + "continuation", + "completeness", + ], [], "messaging.read output"); + if (source.accountId !== parsed.accountId || source.conversationId !== parsed.conversationId) { + drift("messaging.read output", "must bind input account and conversation"); + } + const requestCursor = parsed.beforeCursor ?? parsed.afterCursor; + if (source.requestCursor !== requestCursor) { + drift("messaging.read output.requestCursor", "must bind the requested cursor"); + } + const requestDirection = parsed.afterCursor === null ? "before" : "after"; + if (source.requestDirection !== requestDirection) { + drift("messaging.read output.requestDirection", "must bind the requested direction"); + } + const entities = array(source.messages, "messaging.read output.messages", parsed.limit) + .map((item, index) => message( + item, + `messaging.read output.messages[${index}]`, + parsed.accountId, + parsed.conversationId, + )); + const ids = entities.map((entity) => entity.providerId); + if (new Set(ids).size !== ids.length) { + drift("messaging.read output.messages", "contains duplicate stable message IDs"); + } + array(source.tombstones, "messaging.read output.tombstones", parsed.limit); + record(source.completeness, "messaging.read output.completeness"); + let nextInput: Readonly> | null = null; + if (source.continuation !== null) { + const continuation = record(source.continuation, "messaging.read output.continuation"); + exactKeys( + continuation, + ["direction", "cursor"], + [], + "messaging.read output.continuation", + ); + if (continuation.direction !== requestDirection) { + drift("messaging.read output.continuation.direction", "must preserve direction"); + } + const cursor = string( + continuation.cursor, + "messaging.read output.continuation.cursor", + 2_048, + ); + if (cursor === requestCursor) { + drift("messaging.read output.continuation.cursor", "must advance"); + } + nextInput = Object.freeze({ + account_id: parsed.accountId, + conversation_id: parsed.conversationId, + limit: parsed.limit, + ...(requestDirection === "before" + ? { before_cursor: cursor } + : { after_cursor: cursor }), + }); + } + return Object.freeze({ + schemaVersion: 1, + partition: `${accountSubject}:messages:${encoded(parsed.accountId)}:${encoded(parsed.conversationId)}`, + completeness: Object.freeze({ + kind: "bounded-local", + reason: "Beeper exposed one bounded local message page; connected-account backfill and older edit, reaction, and deletion coverage may be incomplete.", + }), + cursor: Object.freeze({ + direction: requestDirection === "before" ? "backward" : "forward", + request: requestCursor, + nextInput, + }), + entities: Object.freeze(entities), + tombstones: Object.freeze([]), + }); +} diff --git a/src/scripts/sync-bundled-adapters.test.ts b/src/scripts/sync-bundled-adapters.test.ts index 7cecc33..d5fc492 100644 --- a/src/scripts/sync-bundled-adapters.test.ts +++ b/src/scripts/sync-bundled-adapters.test.ts @@ -139,7 +139,7 @@ function capturingInstaller( describe("single-process bundled adapter generation sync", () => { test("derives all current and archived inventory from assets with registry parity", () => { const discovered = discoverBundledAdapters(); - expect(discovered).toHaveLength(18); + expect(discovered).toHaveLength(19); expect(discovered.flatMap((adapter) => adapter.upgradeFrom.map((baseline) => `${adapter.id}@${baseline.manifest.version}` @@ -204,6 +204,7 @@ describe("single-process bundled adapter generation sync", () => { "threads-web@1.3.0": "125936943ac8f0ed00e85367ee01dc953d5115c55f4849a85d8f8b68e4e286f5", }); expect(discovered.map((adapter) => adapter.id)).toEqual([ + "beeper-local", "bluesky-web", "facebook-group-web", "facebook-marketplace-web", @@ -223,7 +224,7 @@ describe("single-process bundled adapter generation sync", () => { "x-web", "youtube-web", ]); - expect(new Set(discovered.map((adapter) => adapter.routeKey)).size).toBe(18); + expect(new Set(discovered.map((adapter) => adapter.routeKey)).size).toBe(19); }); test("validates every immutable source snapshot before one generation commit", async () => { @@ -264,13 +265,13 @@ describe("single-process bundled adapter generation sync", () => { ), }); - expect(validations).toBe(18); + expect(validations).toBe(19); expect(validationRegistries.size).toBe(1); expect([...validationRegistries][0]).not.toBe(providerPluginRegistry); - expect(committed.validationsAtInstall).toBe(18); - expect(committed.selections).toHaveLength(18); + expect(committed.validationsAtInstall).toBe(19); + expect(committed.selections).toHaveLength(19); expect(result).toEqual({ - installed: 18, + installed: 19, preserved: 0, commitId: "00000000-0000-4000-8000-000000000001", }); @@ -313,7 +314,7 @@ describe("single-process bundled adapter generation sync", () => { output, activeRegistry, ); - if (validations === 18) { + if (validations === 19) { installPortableProviderPlugin(packagePath, { trustExecutableCode: true, expectedCurrentBundleSha256: null, @@ -332,7 +333,7 @@ describe("single-process bundled adapter generation sync", () => { } catch (error) { failure = error instanceof Error ? error.message : String(error); } - expect(validations).toBe(18); + expect(validations).toBe(19); expect(publicationCalls).toBe(0); expect(failure).toContain( "portable provider plugin catalog changed during bundled adapter validation", @@ -424,8 +425,8 @@ describe("single-process bundled adapter generation sync", () => { }, }); - expect(result.installed).toBe(18); - expect(committed).toHaveLength(18); + expect(result.installed).toBe(19); + expect(committed).toHaveLength(19); expect(committed.every((selection) => selection.state === "present" && selection.manifest.id === selection.id diff --git a/src/usage.ts b/src/usage.ts index f272682..0c97f45 100644 --- a/src/usage.ts +++ b/src/usage.ts @@ -74,6 +74,10 @@ export const wrenchUsage = `Usage: wrench auth sync --once [--json] Explicitly connect and refresh the local projection wrench auth remove --yes + wrench beeper export-message-like-me --auth --output + [--limit-chats ] [--limit-messages ] + [--max-participants ] [--json] + wrench adapter init (--origin | --platform ) --output [--force] wrench adapter sync-bundled [--json] Install or safely upgrade reviewed bundled manifests diff --git a/src/web-session-contract-definitions.ts b/src/web-session-contract-definitions.ts index 533fc41..88cfd06 100644 --- a/src/web-session-contract-definitions.ts +++ b/src/web-session-contract-definitions.ts @@ -1,4 +1,5 @@ import blueskyWebManifest from "./assets/adapters/bluesky/wrench-web-adapter.json"; +import beeperWebManifest from "./assets/adapters/beeper/wrench-web-adapter.json"; import facebookGroupWebManifest from "./assets/adapters/facebook-group/wrench-web-adapter.json"; import facebookMarketplaceWebManifest from "./assets/adapters/facebook-marketplace/wrench-web-adapter.json"; import facebookPageWebManifest from "./assets/adapters/facebook-page/wrench-web-adapter.json"; @@ -42,6 +43,7 @@ export type WebSessionContract = { }; const bundledManifests: Readonly>> = { + beeper: beeperWebManifest, bluesky: blueskyWebManifest, facebook: facebookWebManifest, "facebook-group": facebookGroupWebManifest, @@ -381,6 +383,11 @@ const REDDIT_WEB_OPERATIONS = operationPolicies("reddit", [ "messaging.read", "posts.read", ]); +const BEEPER_LOCAL_OPERATIONS = operationPolicies("beeper", [ + "contacts.list", + "messaging.list", + "messaging.read", +]); const SUBSTACK_WEB_OPERATIONS = operationPolicies("substack", [ "articles.read", "comments.read", @@ -616,6 +623,12 @@ const whatsapp = { "reactions.set": contract("whatsapp", "reactions.set", WHATSAPP_WEB_OPERATIONS["reactions.set"].risk, WHATSAPP_WEB_OPERATIONS["reactions.set"].state, WHATSAPP_WEB_OPERATIONS["reactions.set"].reason), } as const satisfies Readonly>>; +const beeper = { + "contacts.list": contract("beeper", "contacts.list", BEEPER_LOCAL_OPERATIONS["contacts.list"].risk, BEEPER_LOCAL_OPERATIONS["contacts.list"].state, BEEPER_LOCAL_OPERATIONS["contacts.list"].reason), + "messaging.list": contract("beeper", "messaging.list", BEEPER_LOCAL_OPERATIONS["messaging.list"].risk, BEEPER_LOCAL_OPERATIONS["messaging.list"].state, BEEPER_LOCAL_OPERATIONS["messaging.list"].reason), + "messaging.read": contract("beeper", "messaging.read", BEEPER_LOCAL_OPERATIONS["messaging.read"].risk, BEEPER_LOCAL_OPERATIONS["messaging.read"].state, BEEPER_LOCAL_OPERATIONS["messaging.read"].reason), +} as const satisfies Readonly>>; + const substack = { "articles.publish": contract("substack", "articles.publish", SUBSTACK_WEB_OPERATIONS["articles.publish"].risk, SUBSTACK_WEB_OPERATIONS["articles.publish"].state, SUBSTACK_WEB_OPERATIONS["articles.publish"].reason), "articles.read": contract("substack", "articles.read", SUBSTACK_WEB_OPERATIONS["articles.read"].risk, SUBSTACK_WEB_OPERATIONS["articles.read"].state, SUBSTACK_WEB_OPERATIONS["articles.read"].reason), @@ -676,6 +689,7 @@ const youtube = { } as const satisfies Readonly>>; export const webSessionContractDefinitions = { + beeper, bluesky, facebook, "facebook-group": facebookGroup, diff --git a/src/wrench.test.ts b/src/wrench.test.ts index a12c48b..e3d59cc 100644 --- a/src/wrench.test.ts +++ b/src/wrench.test.ts @@ -4,7 +4,7 @@ import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSyn import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; -import { createAuth, loadAuth, saveAuth } from "./auth"; +import { createAuth, loadAuth, removeAuth, saveAuth } from "./auth"; import { PreservedBrowserArtifactsError } from "./browser"; import type * as MediaRuntimeModule from "./media"; import type { DoctorReport as MediaDoctorReport } from "./media/doctor"; @@ -755,6 +755,33 @@ describe("auth CLI", () => { } }); + test("guides lifecycle-free Beeper stores to subject binding instead of pairing", async () => { + const testState = state(); + try { + const beeperStore = join( + realpathSync(testState.directory), + "beeper-cli-config", + ); + const added = capture(); + expect(await main([ + "auth", "add", "beeper-local", + "--linked-device", "beeper", + "--device-store", beeperStore, + ], testState.environment, added.output)).toBe(0); + expect(loadAuth("beeper-local", testState.environment)).toMatchObject({ + kind: "linked-device-store", + provider: "beeper", + path: beeperStore, + }); + expect(added.stdout()).toContain( + "wrench auth bind beeper-local --site beeper", + ); + expect(added.stdout()).not.toContain("wrench auth pair beeper-local"); + } finally { + rmSync(testState.directory, { recursive: true, force: true }); + } + }); + test("returns structured exit 5 for an indeterminate linked-device sync", async () => { const testState = state(); const deviceStore = mkdtempSync(join(tmpdir(), "wrench-whatsapp-store-")); @@ -843,6 +870,93 @@ describe("auth CLI", () => { } }); + test("runs the bounded Beeper Message Like Me export through the private runtime", async () => { + const testState = state(); + const deviceStore = realpathSync(mkdtempSync(join( + tmpdir(), + "wrench-beeper-cli-device-store-", + ))); + chmodSync(deviceStore, 0o700); + try { + const boundSubject = `beeper:local:${"a".repeat(64)}`; + saveAuth(createAuth("beeper-main", { + linkedDeviceProvider: "beeper", + deviceStore, + subject: boundSubject, + }), testState.environment); + let observed: Record | undefined; + const wrench = capture(); + const code = await main([ + "beeper", + "export-message-like-me", + "--auth", + "beeper-main", + "--output", + "/tmp/message-like-me-fixture", + "--limit-chats", + "10", + "--limit-messages", + "500", + "--json", + ], testState.environment, wrench.output, { + loadBeeperMessageLikeMeCliRuntime: () => Promise.resolve({ + exportBeeperMessageLikeMeFromAuth: (request) => { + observed = request as unknown as Record; + expect(() => removeAuth("beeper-main", testState.environment)) + .toThrow("active read projection transition"); + return Promise.resolve({ + outputRoot: "/tmp/message-like-me-fixture", + manifestPath: "/tmp/message-like-me-fixture/manifest.json", + manifestSha256: "b".repeat(64), + manifest: { + completeness: { + kind: "bounded-local", + reason: "desktop-local-export", + observedFrom: null, + observedThrough: null, + }, + warnings: ["remote-history-not-claimed"], + counts: { + account: 9, + participant: 20, + conversation: 10, + message: 500, + reaction: 4, + tombstone: 1, + }, + }, + } as never); + }, + }), + }); + + expect(code).toBe(0); + expect(wrench.stderr()).toBe(""); + expect(observed).toMatchObject({ + auth: { + id: "beeper-main", + kind: "linked-device-store", + provider: "beeper", + subject: boundSubject, + }, + outputRoot: "/tmp/message-like-me-fixture", + limits: { limitChats: 10, limitMessages: 500 }, + }); + expect(JSON.parse(wrench.stdout())).toMatchObject({ + ok: true, + manifestSha256: "b".repeat(64), + completeness: { kind: "bounded-local" }, + warnings: ["remote-history-not-claimed"], + counts: { account: 9, message: 500 }, + }); + expect(wrench.stdout()).not.toContain(boundSubject); + expect(loadAuth("beeper-main", testState.environment).subject).toBe(boundSubject); + } finally { + rmSync(testState.directory, { recursive: true, force: true }); + rmSync(deviceStore, { recursive: true, force: true }); + } + }); + test("persists subjects supplied with cookie-source and cookies-file locators", async () => { const testState = state(); try { diff --git a/src/wrench.ts b/src/wrench.ts index 04deaa2..f8676fa 100644 --- a/src/wrench.ts +++ b/src/wrench.ts @@ -23,6 +23,7 @@ import { type WrenchAuth, } from "./auth"; import { parseWrenchArguments, wrenchUsage, type WrenchArguments } from "./args"; +import type * as BeeperMessageLikeMeCliRuntimeModule from "./beeper-message-like-me-cli"; import type { GmailCaptureRunner } from "./gmail-capture"; import type * as MediaRuntimeModule from "./media"; import { @@ -85,6 +86,7 @@ import { releasePortableProviderPluginInvocationLease, } from "./provider-plugin-invocation-lease"; import { requireProviderPluginAuth } from "./provider-plugin-auth"; +import { acquireReadProjectionAuthAdmission } from "./read-projection-admission"; import { recoverLinkedDeviceLifecycleAdmissions, } from "./linked-device-lifecycle-admission"; @@ -196,6 +198,7 @@ type Output = { }; type MediaRuntime = typeof MediaRuntimeModule; +type BeeperMessageLikeMeCliRuntime = typeof BeeperMessageLikeMeCliRuntimeModule; /** * Wrench's stable boundary around the independently versioned KB doctor. @@ -216,6 +219,8 @@ const defaultOutput: Output = { }; const loadMediaRuntime = (): Promise => import("./media"); +const loadBeeperMessageLikeMeCliRuntime = (): Promise => + import("./beeper-message-like-me-cli"); const runDefaultGmailCapture: GmailCaptureRunner = async (...arguments_) => { const { runGmailCapture } = await import("./gmail-capture"); @@ -247,6 +252,8 @@ export type WrenchDependencies = { readonly gmailCaptureMain: GmailCaptureRunner; readonly inspectClipEnvironment: () => Promise; readonly loadMediaRuntime: () => Promise; + readonly loadBeeperMessageLikeMeCliRuntime: + () => Promise; readonly providerPluginRegistry: ProviderPluginRegistry; readonly probePluginSubject: ( binding: ProviderPluginBindingV1, @@ -299,6 +306,7 @@ const defaultDependencies: WrenchDependencies = { gmailCaptureMain: runDefaultGmailCapture, inspectClipEnvironment: inspectDefaultClipEnvironment, loadMediaRuntime, + loadBeeperMessageLikeMeCliRuntime, providerPluginRegistry, probePluginSubject: async (binding, auth, signal) => { requireProviderPluginAuth(binding, auth); @@ -358,6 +366,9 @@ function resolveDependencies(overrides: Partial): WrenchDepe overrides.gmailCaptureMain ?? defaultDependencies.gmailCaptureMain, inspectClipEnvironment: overrides.inspectClipEnvironment ?? defaultDependencies.inspectClipEnvironment, loadMediaRuntime: overrides.loadMediaRuntime ?? defaultDependencies.loadMediaRuntime, + loadBeeperMessageLikeMeCliRuntime: + overrides.loadBeeperMessageLikeMeCliRuntime + ?? defaultDependencies.loadBeeperMessageLikeMeCliRuntime, providerPluginRegistry: overrides.providerPluginRegistry ?? defaultDependencies.providerPluginRegistry, probePluginSubject: overrides.probePluginSubject @@ -1545,6 +1556,51 @@ async function runCommand( ...(signal === undefined ? {} : { signal }), }); } + if (arguments_.command === "beeper-export-message-like-me") { + const admission = acquireReadProjectionAuthAdmission( + arguments_.authId, + environment, + ); + try { + const auth = loadAuth(arguments_.authId, environment); + if (auth.kind !== "linked-device-store" || auth.provider !== "beeper") { + throw new Error( + "Message Like Me export requires a Beeper linked-device-store auth locator", + ); + } + const runtime = await dependencies.loadBeeperMessageLikeMeCliRuntime(); + const result = await runtime.exportBeeperMessageLikeMeFromAuth({ + auth, + outputRoot: arguments_.output, + limits: { + ...(arguments_.limitChats === undefined + ? {} + : { limitChats: arguments_.limitChats }), + ...(arguments_.limitMessages === undefined + ? {} + : { limitMessages: arguments_.limitMessages }), + ...(arguments_.maxParticipants === undefined + ? {} + : { maxParticipants: arguments_.maxParticipants }), + }, + environment, + ...(signal === undefined ? {} : { signal }), + }); + const summary = Object.freeze({ + ok: true, + outputRoot: result.outputRoot, + manifestPath: result.manifestPath, + manifestSha256: result.manifestSha256, + completeness: result.manifest.completeness, + warnings: result.manifest.warnings, + counts: result.manifest.counts, + }); + print(output, summary, arguments_.json); + return 0; + } finally { + admission.release(); + } + } if (arguments_.command === "doctor") { return doctor(arguments_, environment, output, dependencies); } @@ -1871,9 +1927,21 @@ async function runCommand( const path = saveAuth(auth, environment, { force: arguments_.force }); output.stdout(`Saved ${safe(auth.id)} auth locator (${auth.kind}) to ${safe(path)}.\n`); if (auth.kind === "linked-device-store") { - output.stdout( - `Next: wrench auth pair ${safe(auth.id)} (optionally add --phone ).\n`, + const binding = dependencies.providerPluginRegistry.resolveSessionRoute( + auth.provider, ); + if ( + binding?.transport === "linked-device" + && binding.linkedDeviceLifecycle !== undefined + ) { + output.stdout( + `Next: wrench auth pair ${safe(auth.id)} (optionally add --phone ).\n`, + ); + } else { + output.stdout( + `Next: wrench auth bind ${safe(auth.id)} --site ${safe(auth.provider)}.\n`, + ); + } } return 0; } diff --git a/website/build.ts b/website/build.ts index f53db2b..78a8adf 100644 --- a/website/build.ts +++ b/website/build.ts @@ -15,7 +15,7 @@ export const SITE_DESCRIPTION = "Open-source CLI and TypeScript SDK for precise web capabilities for AI agents: page capture, verified media archives, encrypted reads, and typed provider operations." as const; export const REPOSITORY_URL = "https://github.com/hraness/wrench" as const; export const PUBLISHER_URL = "https://github.com/hraness" as const; -export const CONTENT_REVIEWED_RELEASE = "v0.10.1" as const; +export const CONTENT_REVIEWED_RELEASE = "v0.11.0" as const; export const DEFAULT_POSTHOG_HOST = "https://us.i.posthog.com" as const; export const PUBLIC_PAGES = [ @@ -45,7 +45,7 @@ export const PUBLIC_PAGES = [ { canonicalPath: "/provider-capabilities/", description: - "See Wrench contact, email, inbox, and message capabilities for Gmail, LinkedIn, Instagram, WhatsApp, Facebook, and Telegram, including current limits.", + "See Wrench contact, email, inbox, and message capabilities for Gmail, Beeper, LinkedIn, Instagram, WhatsApp, Facebook, and Telegram, including current limits.", outputFile: "provider-capabilities/index.html", sourceFile: "provider-capabilities.html", title: "Wrench provider capabilities for contacts, email, and messages", diff --git a/website/source/provider-capabilities.html b/website/source/provider-capabilities.html index a555515..53507d3 100644 --- a/website/source/provider-capabilities.html +++ b/website/source/provider-capabilities.html @@ -6,7 +6,7 @@ Wrench provider capabilities for contacts, email, and messages - + @@ -15,7 +15,7 @@ - + @@ -23,7 +23,7 @@ - + @@ -60,6 +60,7 @@

Current contact and messaging coverage

ProviderContactsInbox and messagesImportant limit Gmail officialGoogle People connections plus bounded directional Gmail statisticsObserved inbox or bounded search listing, exact thread read, and private thread clipping with attachmentsCounts and dates expose completeness; scanning is bounded and reading does not mark mail seen + Beeper Desktop localObserved account-aware page from an already-authorized local Desktop projectionBounded conversation and message reads across connected networks, plus a private Message Like Me bundle exportRemote backfill remains unknown; full export excludes media bytes and truncates coherently at local record or byte ceilings LinkedIn officialObserved first-degree connections with exact account binding and locale-selection evidenceNo ordinary inbox operation through the official API; consumer-web inbox operations remain capture-requiredRequires approved r_1st_connections and r_liteprofile; message statistics are unavailable Instagram webObserved unique non-viewer participants from one reviewed first Direct inbox pageObserved bounded first inbox summary page; individual thread read remains capture-requiredContact set and message statistics are explicitly incomplete; no acknowledgement or presence request is issued WhatsApp linked deviceObserved bounded page from the private, quiescent Whatsmeow contact storeObserved bounded local linked-device conversation list and exact conversation readNo new WhatsApp connection or acknowledgement is emitted; directional contact statistics are unavailable @@ -71,6 +72,18 @@

Current contact and messaging coverage

This matrix covers the contact and messaging providers above. Wrench also ships other provider adapters and operations. Their current installed contract states remain discoverable through the CLI rather than being generalized from this table.

+
+

Beeper projects connected local accounts without a browser or model service

+

The bundled source plugin accepts only the pinned official Beeper CLI 0.6.2, one already-authorized Desktop target, and read-only JSON operations. Wrench hashes the stable self-account coordinate before storing the auth subject. It exposes no send, raw request, media download, pairing, presence, or background-sync surface.

+ 
wrench auth add beeper-main --linked-device beeper \
+  --device-store "${HOME}/.beeper"
+wrench auth bind beeper-main --site beeper
+
+wrench beeper export-message-like-me --auth beeper-main \
+  --output /absolute/path/to/new-message-like-me-bundle --json
+

The export uses Beeper's local full-export pagination with attachments disabled. Wrench validates account, roster, message, reply, edit, reaction, and tombstone provenance; removes duplicate plaintext renderings inside private staging; then writes six mode-0600 NDJSON artifacts and publishes the digested manifest last inside a mode-0700 directory. Completeness describes only the locally materialized export. A 500,000-record or 512 MiB ceiling produces a coherent truncated bundle, and any chat JSON file over the 64 MiB per-chat bound is omitted with an explicit warning.

+
+

Gmail contacts, inbox search, thread reads, and clips

Gmail uses the official Gmail and Google People APIs. Its auth locator must match a current-user-owned mode-0600 token document with the exact provider, subject, and sorted contacts.readonly and gmail.readonly scopes.

From 77446351b2742fc9eace3e741032bb792bc9b7f6 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Sat, 22 Aug 2026 09:31:10 -0400 Subject: [PATCH 2/4] fix: align Wrench media release version --- src/media/manifest.test.ts | 2 +- src/media/manifest.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/media/manifest.test.ts b/src/media/manifest.test.ts index 003af49..560a9a2 100644 --- a/src/media/manifest.test.ts +++ b/src/media/manifest.test.ts @@ -465,7 +465,7 @@ function trackedYtDlpManifest( describe("Wrench media manifest", () => { test("uses one Wrench-owned schema and transcriber identity", () => { expect(WRENCH_MEDIA_SCHEMA_VERSION).toBe(1); - expect(WRENCH_MEDIA_VERSION).toBe("0.11.0"); + expect(WRENCH_MEDIA_VERSION).toBe("0.12.0"); expect(localTranscriptVariantSegments(localIdentity)).toEqual([ "transcript", "local", diff --git a/src/media/manifest.ts b/src/media/manifest.ts index 1b643bc..fc790bc 100644 --- a/src/media/manifest.ts +++ b/src/media/manifest.ts @@ -39,7 +39,7 @@ import { import { compareUtf8 } from "./utf8-order"; export const WRENCH_MEDIA_SCHEMA_VERSION = 1 as const; -export const WRENCH_MEDIA_VERSION = "0.11.0" as const; +export const WRENCH_MEDIA_VERSION = "0.12.0" as const; export const WRENCH_MEDIA_MANIFEST_FILE = "wrench-media.json" as const; export const WRENCH_MEDIA_CHECKSUM_FILE = "manifest-sha256.txt" as const; const MAX_ITEM_ENTRIES = 4_096; From 5d59ca0cf0b7eb09f3a25449a3345abd1b82332d Mon Sep 17 00:00:00 2001 From: 0thernet Date: Sat, 22 Aug 2026 15:30:55 -0400 Subject: [PATCH 3/4] feat: add sequential Beeper message export --- README.md | 91 +- package.json | 1 + src/beeper-message-like-me-cli.test.ts | 113 + src/beeper-message-like-me-cli.ts | 70 +- src/beeper-message-like-me-export.test.ts | 411 +++- src/beeper-message-like-me-export.ts | 743 +++++- src/beeper-message-like-me-golden-fixture.ts | 47 +- src/beeper-message-like-me-recovery.test.ts | 509 ++++ src/beeper-message-like-me-recovery.ts | 729 ++++++ src/beeper-message-like-me-source.test.ts | 1471 ++++++++++- src/beeper-message-like-me-source.ts | 2192 +++++++++++++++-- .../beeper-message-like-me-v1/accounts.ndjson | 1 + .../beeper-message-like-me-v1/manifest.json | 2 +- .../participants.ndjson | 1 + .../beeper-local-runtime.internal.test.ts | 228 +- src/providers/beeper-local-runtime.ts | 176 +- src/providers/beeper-local.ts | 43 +- src/usage.ts | 2 +- src/wrench.test.ts | 140 +- src/wrench.ts | 115 + 20 files changed, 6703 insertions(+), 382 deletions(-) create mode 100644 src/beeper-message-like-me-cli.test.ts create mode 100644 src/beeper-message-like-me-recovery.test.ts create mode 100644 src/beeper-message-like-me-recovery.ts diff --git a/README.md b/README.md index b69ab08..9f6af6e 100644 --- a/README.md +++ b/README.md @@ -349,6 +349,18 @@ wrench auth add beeper-main --linked-device beeper \ wrench auth bind beeper-main --site beeper ``` +The export integrity pin is the official macOS arm64 CLI 0.6.2, not the +moving Homebrew formula name. The command above is sufficient only while +`beeper version` reports 0.6.2. If the tap has advanced, use the 0.6.2 asset +from the [official CLI releases](https://github.com/beeper/cli/releases) +asset and install its `beeper` executable at +`/tools/beeper/0.6.2/beeper` (the default state home is +`~/.local/share/wrench`). Wrench requires archive SHA-256 +`688ccde7e7d044d33980cd06474bf1ae7215ccf8ca79967262fa3bfb85a2589a` +and executable SHA-256 +`48aa895449129c793a212ea19f69a534adc34a8adc4037ca1d7da9e648716425`; +it rejects every other version or byte sequence before reading private data. + Binding hashes the stable local self-account coordinate before storing or printing it. The first bind or read may take longer while the pinned CLI unpacks its embedded payload into an operation-private cache. Read the local account @@ -369,17 +381,74 @@ wrench beeper export-message-like-me --auth beeper-main \ --output /absolute/path/to/new-message-like-me-bundle --json ``` -The command uses the pinned official full export with `--no-attachments`, -validates its complete local chat inventory, removes the duplicate Markdown and -HTML renderings inside operation-private staging, and publishes `manifest.json` -last. The output directory is mode 0700; its six NDJSON artifacts and manifest -are mode 0600 and carry canonical SHA-256 digests. The JSON result reports the -manifest path and digest, record counts, completeness, and warnings. Optional -`--limit-chats`, `--limit-messages`, and `--max-participants` values are recorded -as truncation when reached. Wrench also emits a coherent truncated bundle before -the 500,000-record or 512 MiB bundle ceiling. One chat JSON file is limited to -64 MiB so foreign input cannot force a multi-gigabyte allocation; an oversized -chat is omitted with explicit truncated completeness and a warning. +The command uses the pinned official CLI directly. It enumerates the connected +account realm, then runs the official `export --no-attachments` command once per +account in a deterministic order. Each invocation selects its account through +an operation-private CLI config, so account identifiers never appear in command +arguments, environment paths, or progress output. Stderr reports the account +ordinal and cumulative validated chat and message counts. Long account, +conversion, bundle-validation, and publication phases repeat their elapsed time +every 30 seconds, including final private-shard cleanup. It prints the private +recovery check before that work begins, so stale cleanup is visible too. A final +account enumeration rejects a realm that changed while the sequential snapshot +was running. + +Wrench retains each validated raw account shard until the complete sanitized +bundle passes its graph and digest checks. It builds all six NDJSON artifacts +and `manifest.json` in a private sibling directory, fsyncs them, and exposes the +seven-file bundle with one atomic directory rename. The requested output path +stays absent until that commit. Success removes the raw shards; failure or +cancellation removes owned staging and leaves no partial output. The output +directory is mode 0700, and every file is mode 0600 with a canonical SHA-256 +digest. + +Each connected account has exactly one normalized self participant, anchored by +the account user's stable Beeper ID. Before record allocation, Wrench makes a +bounded hash-only pass over the selected chats. Explicit chat `isSelf` values +and message `isSender` values establish account-local self and peer evidence. +Later evidence applies to earlier chats, message files stay bound to their +validated SHA-256 digests, and contradictory evidence stops the export without +publishing. Reactions inherit a normalized participant reference while their +raw provider tuple remains only inside a composite hash. Nonunique provider +reaction IDs are preserved with the categorical +`reaction-provider-id-non-unique` warning. + +The JSON result reports the manifest path and digest, record counts, +completeness, and warnings. `--limit-chats` is global across the account +sequence. `--limit-messages` and `--max-participants` apply to each chat, which +matches the official CLI flags. Reached limits are recorded as truncation. +Wrench always passes hard ceilings of 100,000 chats and 1,000,000 messages per +chat, and it emits a coherent truncated bundle before the 500,000-record or 512 +MiB bundle ceiling. One chat JSON file is limited to 64 MiB so foreign input +cannot force a multi-gigabyte allocation; an oversized chat is omitted with +explicit truncated completeness and a warning. While the official CLI is +running, Wrench monitors the complete private working tree against a 4 GiB +ceiling every 500 ms and independently checks that at least 2 GiB remains free +on the filesystem. This is a monitored safety ceiling, not an operating-system +quota. After each account validates, Wrench immediately removes the redundant +Markdown and HTML renderings while retaining the hash-bound JSON needed for the +final conversion. Cleanup first moves each owned directory into a private +quarantine and verifies its filesystem identity before recursive removal. + +Before credentials or message bytes enter a raw working directory, Wrench +wins one atomic export-admission claim shared across all Beeper auth IDs. A +second invocation stops before account discovery while a live or +uninspectable owner holds that claim. A later invocation can reclaim it only +after proving that the exact owner is no longer running. + +After admission, Wrench writes a durable private lease containing the directory +and process identities. +The atomic bundle stage receives the same protection. A later invocation +reclaims a stale directory only after proving that its exact owner, and any +recorded Beeper child, is no longer running. Live or indeterminate owners are +left untouched and the command stops with a categorical error. If a crash +lands between the atomic rename and lease release, recovery recognizes the +same directory at the requested output path and preserves the published +bundle. + +The Beeper Desktop API MCP project is intended to expose Beeper tools to an MCP +client. This export path uses the official CLI because Wrench needs a pinned, +bounded, read-only file snapshot that it can validate and publish atomically. Contact and chat lists are bounded to 200 records because CLI 0.6.2 exposes no continuation cursor for those commands. Message pages derive the next diff --git a/package.json b/package.json index 9ecb8d9..8326e87 100644 --- a/package.json +++ b/package.json @@ -148,6 +148,7 @@ "src/auth.ts", "src/beeper-message-like-me-cli.ts", "src/beeper-message-like-me-export.ts", + "src/beeper-message-like-me-recovery.ts", "src/beeper-message-like-me-source.ts", "src/fixtures/beeper-message-like-me-v1", "src/browser-admission.ts", diff --git a/src/beeper-message-like-me-cli.test.ts b/src/beeper-message-like-me-cli.test.ts new file mode 100644 index 0000000..7bb1920 --- /dev/null +++ b/src/beeper-message-like-me-cli.test.ts @@ -0,0 +1,113 @@ +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + realpathSync, + rmSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, test } from "bun:test"; + +import type { WrenchAuth } from "./auth"; +import { exportBeeperMessageLikeMeFromAuth } from "./beeper-message-like-me-cli"; +import { + acquireBeeperMessageLikeMeExportAdmission, + createBeeperMessageLikeMeDirectoryLease, + releaseBeeperMessageLikeMeExportAdmission, + releaseBeeperMessageLikeMeDirectoryLease, +} from "./beeper-message-like-me-recovery"; + +const temporaryRoots: string[] = []; + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) { + rmSync(root, { force: true, recursive: true }); + } +}); + +describe("Beeper Message Like Me CLI recovery preflight", () => { + test("rejects an active export before inspecting the requested auth", async () => { + const root = realpathSync( + mkdtempSync(join(tmpdir(), "wrench-beeper-cli-recovery-test-")), + ); + temporaryRoots.push(root); + chmodSync(root, 0o700); + const parent = join(root, "private-exports"); + mkdirSync(parent, { mode: 0o700 }); + const working = join(parent, "active-working"); + mkdirSync(working, { mode: 0o700 }); + const environment = { WRENCH_STATE_HOME: join(root, "state") }; + const nowMs = Date.now(); + const activeLease = await createBeeperMessageLikeMeDirectoryLease({ + role: "raw-working", + path: working, + recoverAfterMs: nowMs + 60_000, + environment, + nowMs, + }); + const deliberatelyInvalidAuth = { + schemaVersion: 999, + kind: "not-an-auth-kind", + } as unknown as WrenchAuth; + const progress: string[] = []; + + try { + await expect(exportBeeperMessageLikeMeFromAuth({ + auth: deliberatelyInvalidAuth, + outputRoot: join(parent, "must-not-be-created"), + environment, + onProgress: (event) => progress.push(event.phase), + })).rejects.toThrow( + "another export is active or prior private export recovery is indeterminate", + ); + expect(existsSync(working)).toBeTrue(); + expect(existsSync(join(parent, "must-not-be-created"))).toBeFalse(); + expect(existsSync(activeLease.claimPath)).toBeTrue(); + expect(progress).toEqual(["recovery-started"]); + + const admissionAfterFailure = acquireBeeperMessageLikeMeExportAdmission({ + environment, + }); + releaseBeeperMessageLikeMeExportAdmission(admissionAfterFailure); + } finally { + releaseBeeperMessageLikeMeDirectoryLease(activeLease); + } + }); + + test("releases global admission after post-recovery source setup fails", async () => { + const root = realpathSync( + mkdtempSync(join(tmpdir(), "wrench-beeper-cli-source-failure-test-")), + ); + temporaryRoots.push(root); + chmodSync(root, 0o700); + const parent = join(root, "private-exports"); + mkdirSync(parent, { mode: 0o700 }); + const outputRoot = join(parent, "must-not-be-created"); + const environment = { WRENCH_STATE_HOME: join(root, "state") }; + const deliberatelyInvalidAuth = { + schemaVersion: 999, + kind: "not-an-auth-kind", + } as unknown as WrenchAuth; + const progress: string[] = []; + + await expect(exportBeeperMessageLikeMeFromAuth({ + auth: deliberatelyInvalidAuth, + outputRoot, + environment, + onProgress: (event) => progress.push(event.phase), + })).rejects.toThrow(); + + expect(progress).toEqual(["recovery-started", "recovery-completed"]); + expect(existsSync(outputRoot)).toBeFalse(); + expect(readdirSync(parent)).toEqual([]); + + const admissionAfterFailure = acquireBeeperMessageLikeMeExportAdmission({ + environment, + }); + releaseBeeperMessageLikeMeExportAdmission(admissionAfterFailure); + }); +}); diff --git a/src/beeper-message-like-me-cli.ts b/src/beeper-message-like-me-cli.ts index c77ad68..87d647f 100644 --- a/src/beeper-message-like-me-cli.ts +++ b/src/beeper-message-like-me-cli.ts @@ -5,8 +5,14 @@ import { } from "./beeper-message-like-me-export"; import { createBeeperMessageLikeMeSource, + type BeeperMessageLikeMeProgress, type BeeperMessageLikeMeSourceLimits, } from "./beeper-message-like-me-source"; +import { + acquireBeeperMessageLikeMeExportAdmission, + recoverBeeperMessageLikeMeDirectoryLeases, + releaseBeeperMessageLikeMeExportAdmission, +} from "./beeper-message-like-me-recovery"; export type BeeperMessageLikeMeCliRequest = Readonly<{ auth: WrenchAuth; @@ -14,6 +20,7 @@ export type BeeperMessageLikeMeCliRequest = Readonly<{ limits?: BeeperMessageLikeMeSourceLimits; environment?: Readonly>; signal?: AbortSignal; + onProgress?: (progress: BeeperMessageLikeMeProgress) => void; }>; /** @@ -24,17 +31,54 @@ export type BeeperMessageLikeMeCliRequest = Readonly<{ export async function exportBeeperMessageLikeMeFromAuth( request: BeeperMessageLikeMeCliRequest, ): Promise { - const source = createBeeperMessageLikeMeSource({ - auth: request.auth, - ...(request.limits === undefined ? {} : { limits: request.limits }), - ...(request.environment === undefined - ? {} - : { environment: request.environment }), - ...(request.signal === undefined ? {} : { signal: request.signal }), - }); - return exportBeeperMessageLikeMeBundle({ - outputRoot: request.outputRoot, - source, - ...(request.signal === undefined ? {} : { signal: request.signal }), - }); + try { + const environment = request.environment ?? process.env; + request.onProgress?.(Object.freeze({ phase: "recovery-started" })); + const admission = acquireBeeperMessageLikeMeExportAdmission({ environment }); + try { + const recovery = await recoverBeeperMessageLikeMeDirectoryLeases({ + environment, + }); + if (recovery.active > 0 || recovery.indeterminate > 0) { + throw new Error( + "Beeper Message Like Me export: another export is active or prior private export recovery is indeterminate", + ); + } + request.onProgress?.(Object.freeze({ + phase: "recovery-completed", + recovered: recovery.recovered, + published: recovery.published, + })); + const source = createBeeperMessageLikeMeSource({ + auth: request.auth, + ...(request.limits === undefined ? {} : { limits: request.limits }), + ...(request.environment === undefined + ? {} + : { environment: request.environment }), + ...(request.signal === undefined ? {} : { signal: request.signal }), + ...(request.onProgress === undefined + ? {} + : { onProgress: request.onProgress }), + }); + return await exportBeeperMessageLikeMeBundle({ + outputRoot: request.outputRoot, + source, + ...(request.onProgress === undefined + ? {} + : { onProgress: request.onProgress }), + ...(request.signal === undefined ? {} : { signal: request.signal }), + recoveryEnvironment: environment, + }); + } finally { + releaseBeeperMessageLikeMeExportAdmission(admission); + } + } catch (error) { + if ( + error instanceof Error + && /^(?:Beeper |Message Like Me |pinned Beeper CLI )/u.test(error.message) + ) throw error; + throw new Error( + "Beeper Message Like Me export: private local file operation failed", + ); + } } diff --git a/src/beeper-message-like-me-export.test.ts b/src/beeper-message-like-me-export.test.ts index ca83076..0d3e34b 100644 --- a/src/beeper-message-like-me-export.test.ts +++ b/src/beeper-message-like-me-export.test.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import { chmod, + link, lstat, mkdir, mkdtemp, @@ -24,6 +25,7 @@ import { BEEPER_MESSAGE_LIKE_ME_GOLDEN_STARTED_AT, createBeeperMessageLikeMeGoldenSource, } from "./beeper-message-like-me-golden-fixture"; +import { recoverBeeperMessageLikeMeDirectoryLeases } from "./beeper-message-like-me-recovery"; const temporaryRoots: string[] = []; @@ -213,12 +215,39 @@ describe("exportBeeperMessageLikeMeBundle", () => { ); } expect(result.manifestSha256).toBe( - "e46f4a524d53f849cfac594fb5bc8cf28e7a9743c138039b81a0aad4ff4830ef", + "dcef93293af9af0f3b0ff303992517ce2eece6d4bf0b7477e30c0b9d77a2c7f1", ); - expect(result.manifest.completeness).toMatchObject({ - kind: "truncated", - reason: "explicit-source-limit", + expect(result.manifest.source).toEqual({ id: "beeper-local", version: "1.1.0" }); + expect(result.manifest.counts).toEqual({ + account: 2, + participant: 3, + conversation: 1, + message: 2, + reaction: 1, + tombstone: 1, }); + expect(result.manifest.completeness).toEqual({ + kind: "bounded-local", + reason: "desktop-local-sequential-export", + observedFrom: "2026-08-21T15:50:00.000Z", + observedThrough: "2026-08-21T15:59:00.000Z", + }); + expect(result.manifest.warnings).toEqual([ + "attachments-metadata-only", + "connected-account-backfill-coverage-unknown", + "remote-history-not-claimed", + "sequential-account-snapshot", + "synthetic-golden-fixture", + ]); + const goldenAccounts = (await readFile(join(outputRoot, "accounts.ndjson"), "utf8")) + .trimEnd() + .split("\n") + .map((line) => JSON.parse(line) as Record); + expect(goldenAccounts).toHaveLength(2); + expect(goldenAccounts.map(({ network }) => network)).toEqual([ + "synthetic", + "synthetic-secondary", + ]); const goldenMessages = (await readFile(join(outputRoot, "messages.ndjson"), "utf8")) .trimEnd() .split("\n") @@ -241,13 +270,15 @@ describe("exportBeeperMessageLikeMeBundle", () => { expect(goldenReaction.reactedAt).toBeNull(); }); - test("streams a private provenance-preserving bundle and publishes its manifest last", async () => { + test("streams a private provenance-preserving bundle and publishes all seven files atomically", async () => { const parent = await privateTemporaryRoot(); const outputRoot = join(parent, "message-like-me"); + const progress: unknown[] = []; const result = await exportBeeperMessageLikeMeBundle({ outputRoot, source: source(), clock: clock("2026-08-21T12:01:00.000Z", "2026-08-21T12:02:00.000Z"), + onProgress: (item) => progress.push(item), }); expect(result.outputRoot).toBe(outputRoot); @@ -271,6 +302,11 @@ describe("exportBeeperMessageLikeMeBundle", () => { providerUrls: "excluded", credentials: "excluded", }); + expect(progress).toEqual([ + { phase: "bundle-building", elapsedSeconds: 0, records: 1, bytes: 398 }, + { phase: "bundle-validating", elapsedSeconds: 0, records: 7, bytes: 3_356 }, + { phase: "bundle-publishing", elapsedSeconds: 0, records: 7, bytes: 3_356 }, + ]); const expectedFiles = [ "accounts.ndjson", @@ -328,6 +364,244 @@ describe("exportBeeperMessageLikeMeBundle", () => { }); expect(manifestSource).not.toContain(outputRoot); expect(manifestSource).not.toContain("token"); + expect(await readdir(parent)).toEqual(["message-like-me"]); + }); + + test("keeps outputRoot absent until the complete bundle is ready", async () => { + const parent = await privateTemporaryRoot(); + const outputRoot = join(parent, "atomic"); + const fixture = source(); + const observedSource: BeeperMessageLikeMeExportSource = { + descriptor: fixture.descriptor, + records: (async function* () { + for (const value of records()) { + await expect(lstat(outputRoot)).rejects.toMatchObject({ code: "ENOENT" }); + yield value; + } + })(), + completion: async () => { + await expect(lstat(outputRoot)).rejects.toMatchObject({ code: "ENOENT" }); + return fixture.completion(); + }, + }; + + await exportBeeperMessageLikeMeBundle({ outputRoot, source: observedSource }); + + expect((await readdir(outputRoot)).sort()).toEqual([ + "accounts.ndjson", + "conversations.ndjson", + "manifest.json", + "messages.ndjson", + "participants.ndjson", + "reactions.ndjson", + "tombstones.ndjson", + ]); + expect(await readdir(parent)).toEqual(["atomic"]); + }); + + test("atomically preserves an outputRoot claimed before publication", async () => { + const parent = await privateTemporaryRoot(); + const outputRoot = join(parent, "claimed"); + const fixture = source(); + + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot, + source: { + ...fixture, + completion: async () => { + await mkdir(outputRoot, { mode: 0o700 }); + return fixture.completion(); + }, + }, + })).rejects.toThrow("outputRoot appeared before atomic publication"); + + expect(await readdir(outputRoot)).toEqual([]); + expect(await readdir(parent)).toEqual(["claimed"]); + }); + + test("disposes its source exactly once after success publication or failure cleanup", async () => { + const successParent = await privateTemporaryRoot(); + const successOutput = join(successParent, "dispose-success"); + const successCalls: boolean[] = []; + await exportBeeperMessageLikeMeBundle({ + outputRoot: successOutput, + source: { + ...source(), + dispose: async (published) => { + successCalls.push(published); + expect(await readFile(join(successOutput, "manifest.json"), "utf8")) + .toContain("message-like-me.local-message-bundle"); + }, + }, + }); + expect(successCalls).toEqual([true]); + + const failureParent = await privateTemporaryRoot(); + const failureOutput = join(failureParent, "dispose-failure"); + const failureCalls: boolean[] = []; + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot: failureOutput, + source: { + ...source(), + completion: async () => { + throw new Error("synthetic source failure"); + }, + dispose: async (published) => { + failureCalls.push(published); + await expect(lstat(failureOutput)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readdir(failureParent)).toEqual([]); + }, + }, + })).rejects.toThrow("synthetic source failure"); + expect(failureCalls).toEqual([false]); + }); + + test("rolls a verified publication back when source disposal fails", async () => { + const parent = await privateTemporaryRoot(); + const outputRoot = join(parent, "dispose-rollback"); + const calls: boolean[] = []; + + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot, + source: { + ...source(), + dispose: async (published) => { + calls.push(published); + expect(await readdir(outputRoot)).toContain("manifest.json"); + throw new Error("synthetic disposal failure"); + }, + }, + })).rejects.toThrow("synthetic disposal failure"); + + expect(calls).toEqual([true]); + await expect(lstat(outputRoot)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readdir(parent)).toEqual([]); + }); + + test("preserves publication and source-disposal failures together", async () => { + const parent = await privateTemporaryRoot(); + const outputRoot = join(parent, "dual-failure"); + const fixture = source(); + let caught: unknown; + + try { + await exportBeeperMessageLikeMeBundle({ + outputRoot, + source: { + ...fixture, + completion: async () => { + throw new Error("synthetic publication failure"); + }, + dispose: async () => { + throw new Error("synthetic disposal failure"); + }, + }, + }); + } catch (error) { + caught = error; + } + + expect(caught).toBeInstanceOf(AggregateError); + expect((caught as AggregateError).errors.map((error) => + error instanceof Error ? error.message : String(error))).toEqual([ + "synthetic publication failure", + "synthetic disposal failure", + ]); + await expect(lstat(outputRoot)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readdir(parent)).toEqual([]); + }); + + test("removes private staging and leaves outputRoot absent on failure and cancellation", async () => { + const failureParent = await privateTemporaryRoot(); + const failureOutput = join(failureParent, "completion-failure"); + const failureFixture = source(); + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot: failureOutput, + source: { + ...failureFixture, + completion: async () => { + await expect(lstat(failureOutput)).rejects.toMatchObject({ code: "ENOENT" }); + throw new Error("synthetic completion failure"); + }, + }, + })).rejects.toThrow("synthetic completion failure"); + await expect(lstat(failureOutput)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readdir(failureParent)).toEqual([]); + + const cancellationParent = await privateTemporaryRoot(); + const cancellationOutput = join(cancellationParent, "cancelled"); + const cancellation = new AbortController(); + const cancellationFixture = source(); + const cancellationDisposals: boolean[] = []; + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot: cancellationOutput, + signal: cancellation.signal, + source: { + ...cancellationFixture, + records: (async function* () { + for (const value of records()) yield value; + cancellation.abort(); + })(), + dispose: async (published) => { + cancellationDisposals.push(published); + await expect(lstat(cancellationOutput)).rejects.toMatchObject({ code: "ENOENT" }); + }, + }, + })).rejects.toThrow("export was aborted"); + expect(cancellationDisposals).toEqual([false]); + await expect(lstat(cancellationOutput)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readdir(cancellationParent)).toEqual([]); + }); + + test("re-reads staged files before publication and removes a corrupted stage", async () => { + const parent = await privateTemporaryRoot(); + const outputRoot = join(parent, "corrupted"); + const fixture = source(); + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot, + source: { + ...fixture, + completion: async () => { + const siblings = await readdir(parent); + expect(siblings).toHaveLength(1); + const accountsPart = join(parent, siblings[0]!, "accounts.ndjson.part"); + const bytes = await readFile(accountsPart); + bytes[0] = bytes[0] === 0x7b ? 0x5b : 0x7b; + await Bun.write(accountsPart, bytes); + await chmod(accountsPart, 0o600); + return fixture.completion(); + }, + }, + })).rejects.toThrow("accounts.ndjson changed before publication"); + + await expect(lstat(outputRoot)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readdir(parent)).toEqual([]); + }); + + test("rejects a staged artifact hardlinked outside private staging", async () => { + const parent = await privateTemporaryRoot(); + const outputRoot = join(parent, "hardlinked"); + const alias = join(parent, "foreign-alias.ndjson"); + const fixture = source(); + + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot, + source: { + ...fixture, + completion: async () => { + const siblings = await readdir(parent); + expect(siblings).toHaveLength(1); + await link( + join(parent, siblings[0]!, "accounts.ndjson.part"), + alias, + ); + return fixture.completion(); + }, + }, + })).rejects.toThrow("accounts.ndjson changed before finalization"); + + await expect(lstat(outputRoot)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readFile(alias, "utf8")).toContain("account:whatsapp:primary"); }); test("rejects foreign fields before they can introduce provider URLs", async () => { @@ -340,8 +614,25 @@ describe("exportBeeperMessageLikeMeBundle", () => { source: source([account]), })).rejects.toThrow("must contain exactly"); - expect((await lstat(outputRoot)).mode & 0o777).toBe(0o700); - await expect(lstat(join(outputRoot, "manifest.json"))).rejects.toMatchObject({ code: "ENOENT" }); + await expect(lstat(outputRoot)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readdir(parent)).toEqual([]); + }); + + test("rejects a non-function source disposer before creating staging", async () => { + const parent = await privateTemporaryRoot(); + const outputRoot = join(parent, "invalid-dispose"); + const invalid = { + ...source(), + dispose: "not-a-function", + } as unknown as BeeperMessageLikeMeExportSource; + + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot, + source: invalid, + })).rejects.toThrow("optional dispose function"); + + await expect(lstat(outputRoot)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readdir(parent)).toEqual([]); }); test("enforces the async stream record bound", async () => { @@ -359,7 +650,8 @@ describe("exportBeeperMessageLikeMeBundle", () => { source: source([records()[0], second]), limits: { maxRecords: 1 }, })).rejects.toThrow("record stream exceeds the configured record bound"); - await expect(lstat(join(outputRoot, "manifest.json"))).rejects.toMatchObject({ code: "ENOENT" }); + await expect(lstat(outputRoot)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readdir(parent)).toEqual([]); }); test("enforces importer-compatible bundle and connected-account bounds", async () => { @@ -396,8 +688,8 @@ describe("exportBeeperMessageLikeMeBundle", () => { outputRoot: accountBoundOutput, source: source(accounts), })).rejects.toThrow("connected-account bound"); - await expect(lstat(join(accountBoundOutput, "manifest.json"))) - .rejects.toMatchObject({ code: "ENOENT" }); + await expect(lstat(accountBoundOutput)).rejects.toMatchObject({ code: "ENOENT" }); + expect((await readdir(parent)).sort()).toEqual([]); }); test("refuses relative, existing, and symlink-traversing output roots", async () => { @@ -415,6 +707,15 @@ describe("exportBeeperMessageLikeMeBundle", () => { outputRoot: existing, source: source([]), })).rejects.toThrow("already exists"); + expect(await readFile(existing, "utf8")).toBe("owned by caller"); + + const existingDirectory = join(parent, "existing-directory"); + await mkdir(existingDirectory, { mode: 0o700 }); + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot: existingDirectory, + source: source([]), + })).rejects.toThrow("already exists"); + expect(await readdir(existingDirectory)).toEqual([]); const permissive = join(parent, "permissive"); await mkdir(permissive, { mode: 0o700 }); @@ -689,5 +990,95 @@ describe("exportBeeperMessageLikeMeBundle", () => { source: source(item.values), })).rejects.toThrow(item.message); } + }); + }); + +describe("Beeper Message Like Me bundle recovery", () => { + test("releases its durable stage claim after successful publication", async () => { + const parent = await privateTemporaryRoot(); + const outputRoot = join(parent, "recovery-success"); + const stateRoot = join(parent, "state"); + const environment = { WRENCH_STATE_HOME: stateRoot }; + + await exportBeeperMessageLikeMeBundle({ + outputRoot, + source: source(), + recoveryEnvironment: environment, + }); + + expect((await readdir(parent)).sort()).toEqual([ + "recovery-success", + "state", + ]); + expect(await readdir(join( + stateRoot, + "recovery", + "beeper-message-like-me-directory-leases", + ))).toEqual([]); + expect(await recoverBeeperMessageLikeMeDirectoryLeases({ environment })) + .toEqual({ recovered: 0, published: 0, active: 0, indeterminate: 0 }); + }); + + test("removes its failed stage and releases its durable claim", async () => { + const parent = await privateTemporaryRoot(); + const outputRoot = join(parent, "recovery-failure"); + const stateRoot = join(parent, "state"); + const environment = { WRENCH_STATE_HOME: stateRoot }; + const fixture = source(); + + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot, + source: { + ...fixture, + completion: async () => { + throw new Error("synthetic recovery publication failure"); + }, + }, + recoveryEnvironment: environment, + })).rejects.toThrow("synthetic recovery publication failure"); + + await expect(lstat(outputRoot)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readdir(parent)).toEqual(["state"]); + expect(await readdir(join( + stateRoot, + "recovery", + "beeper-message-like-me-directory-leases", + ))).toEqual([]); + expect(await recoverBeeperMessageLikeMeDirectoryLeases({ environment })) + .toEqual({ recovered: 0, published: 0, active: 0, indeterminate: 0 }); + }); + + test("keeps its stage claim through a durable disposal-failure rollback", async () => { + const parent = await privateTemporaryRoot(); + const outputRoot = join(parent, "recovery-disposal-rollback"); + const stateRoot = join(parent, "state"); + const environment = { WRENCH_STATE_HOME: stateRoot }; + + await expect(exportBeeperMessageLikeMeBundle({ + outputRoot, + source: { + ...source(), + dispose: async () => { + expect((await readdir(join( + stateRoot, + "recovery", + "beeper-message-like-me-directory-leases", + ))).length).toBe(1); + expect(await readdir(outputRoot)).toContain("manifest.json"); + throw new Error("synthetic recovery disposal failure"); + }, + }, + recoveryEnvironment: environment, + })).rejects.toThrow("synthetic recovery disposal failure"); + + await expect(lstat(outputRoot)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readdir(parent)).toEqual(["state"]); + expect(await readdir(join( + stateRoot, + "recovery", + "beeper-message-like-me-directory-leases", + ))).toEqual([]); + expect(await recoverBeeperMessageLikeMeDirectoryLeases({ environment })) + .toEqual({ recovered: 0, published: 0, active: 0, indeterminate: 0 }); }); }); diff --git a/src/beeper-message-like-me-export.ts b/src/beeper-message-like-me-export.ts index c54e7e5..ef63a45 100644 --- a/src/beeper-message-like-me-export.ts +++ b/src/beeper-message-like-me-export.ts @@ -2,8 +2,9 @@ import { constants } from "node:fs"; import { chmod, lstat, - mkdir, + mkdtemp, open, + readdir, realpath, rename, rmdir, @@ -11,8 +12,15 @@ import { } from "node:fs/promises"; import { basename, dirname, isAbsolute, resolve, sep } from "node:path"; import { createHash } from "node:crypto"; +import { dlopen, ptr } from "bun:ffi"; import { canonicalJson, sha256 } from "./canonical-json"; +import { + createBeeperMessageLikeMeDirectoryLease, + releaseBeeperMessageLikeMeDirectoryLease, + type BeeperMessageLikeMeDirectoryLease, +} from "./beeper-message-like-me-recovery"; +import { removePrivateDirectoryTree } from "./storage"; export const BEEPER_MESSAGE_LIKE_ME_SCHEMA_VERSION = 1 as const; export const BEEPER_MESSAGE_LIKE_ME_MAX_RECORDS = 500_000 as const; @@ -27,6 +35,17 @@ const MAX_WARNING_CODES = 128; const MAX_PARTICIPANTS = 10_000; const MAX_ATTACHMENTS = 256; const MAX_CONNECTED_ACCOUNTS = 128; +const BUNDLE_HEARTBEAT_INTERVAL_MS = 30_000; +const DARWIN_RENAME_EXCL = 0x0000_0004; +const LINUX_RENAME_NOREPLACE = 0x0000_0001; +const AT_FDCWD = -100; + +type NativeExclusiveRename = ( + source: Uint8Array, + destination: Uint8Array, +) => number; + +let cachedNativeExclusiveRename: NativeExclusiveRename | undefined; const HARD_LIMITS = Object.freeze({ maxRecords: BEEPER_MESSAGE_LIKE_ME_MAX_RECORDS, @@ -40,6 +59,30 @@ export type BeeperMessageLikeMeExportLimits = { readonly maxTotalBytes: number; }; +export type BeeperMessageLikeMeBundleProgress = + | Readonly<{ + phase: "bundle-building"; + elapsedSeconds: number; + records: number; + bytes: number; + }> + | Readonly<{ + phase: "bundle-validating"; + elapsedSeconds: number; + records: number; + bytes: number; + }> + | Readonly<{ + phase: "bundle-publishing"; + elapsedSeconds: number; + records: number; + bytes: number; + }> + | Readonly<{ + phase: "private-cleanup"; + elapsedSeconds: number; + }>; + export type BeeperMessageLikeMeExportSource = { /** Parsed as foreign data before any output directory is created. */ readonly descriptor: unknown; @@ -47,6 +90,8 @@ export type BeeperMessageLikeMeExportSource = { readonly records: AsyncIterable; /** Called only after the record stream ends successfully. */ readonly completion: () => Promise; + /** Called exactly once after publication or failure cleanup. */ + readonly dispose?: (published: boolean) => Promise; }; export type BeeperMessageLikeMeExportRequest = { @@ -54,6 +99,9 @@ export type BeeperMessageLikeMeExportRequest = { readonly source: BeeperMessageLikeMeExportSource; readonly limits?: Partial; readonly signal?: AbortSignal; + readonly onProgress?: (progress: BeeperMessageLikeMeBundleProgress) => void; + /** Internal CLI composition seam for durable private-stage recovery. */ + readonly recoveryEnvironment?: Readonly>; /** Test seam. Production callers should omit it. */ readonly clock?: () => Date; }; @@ -326,6 +374,29 @@ type ArtifactWriter = { closed: boolean; }; +type PrivateDirectoryIdentity = { + readonly device: number; + readonly inode: number; +}; + +type StagedManifest = { + readonly bytes: number; + readonly sha256: string; +}; + +type PublishedDirectory = { + readonly path: string; + readonly parent: string; + readonly identity: PrivateDirectoryIdentity; + readonly parentIdentity: PrivateDirectoryIdentity; +}; + +type PublishedBundle = { + readonly result: BeeperMessageLikeMeExportResult; + readonly directory: PublishedDirectory; + readonly directoryLease?: BeeperMessageLikeMeDirectoryLease; +}; + const ARTIFACTS = Object.freeze([ Object.freeze({ kind: "account" as const, fileName: "accounts.ndjson" }), Object.freeze({ kind: "participant" as const, fileName: "participants.ndjson" }), @@ -1205,6 +1276,71 @@ async function assertAbsent(path: string, label: string): Promise { fail(`${label} already exists`); } +function nativeExclusiveRename(): NativeExclusiveRename { + if (cachedNativeExclusiveRename !== undefined) { + return cachedNativeExclusiveRename; + } + if (process.platform === "darwin") { + const library = dlopen("/usr/lib/libSystem.B.dylib", { + renamex_np: { + args: ["cstring", "cstring", "u32"], + returns: "int", + }, + } as const); + cachedNativeExclusiveRename = (source, destination) => + library.symbols.renamex_np( + ptr(source), + ptr(destination), + DARWIN_RENAME_EXCL, + ); + return cachedNativeExclusiveRename; + } + if (process.platform === "linux") { + const library = dlopen("libc.so.6", { + renameat2: { + args: ["int", "cstring", "int", "cstring", "u32"], + returns: "int", + }, + } as const); + cachedNativeExclusiveRename = (source, destination) => + library.symbols.renameat2( + AT_FDCWD, + ptr(source), + AT_FDCWD, + ptr(destination), + LINUX_RENAME_NOREPLACE, + ); + return cachedNativeExclusiveRename; + } + return fail("atomic no-clobber publication is unsupported on this platform"); +} + +async function renameDirectoryExclusive( + source: string, + destination: string, +): Promise { + const encodePath = (path: string): Buffer => { + if (path.includes("\0")) return fail("atomic publication path was invalid"); + return Buffer.from(`${path}\0`, "utf8"); + }; + const result = nativeExclusiveRename()( + encodePath(source), + encodePath(destination), + ); + if (result === 0) return; + let destinationExists = false; + try { + await lstat(destination); + destinationExists = true; + } catch (error) { + if (!isErrno(error, "ENOENT")) throw error; + } + if (destinationExists) { + return fail("outputRoot appeared before atomic publication"); + } + return fail("atomic no-clobber publication failed"); +} + async function validateOutputRoot(outputRoot: unknown): Promise<{ readonly outputRoot: string; readonly parent: string; @@ -1242,9 +1378,13 @@ async function validateOutputRoot(outputRoot: unknown): Promise<{ async function assertParentUnchanged(snapshot: Awaited>): Promise { const current = await lstat(snapshot.parent); + const uid = process.getuid?.(); if ( !current.isDirectory() || current.isSymbolicLink() + || uid === undefined + || current.uid !== uid + || (current.mode & 0o022) !== 0 || current.dev !== snapshot.parentDevice || current.ino !== snapshot.parentInode || await realpath(snapshot.parent) !== snapshot.parent @@ -1253,7 +1393,10 @@ async function assertParentUnchanged(snapshot: Awaited { +async function assertPrivateDirectory( + path: string, + expected?: PrivateDirectoryIdentity, +): Promise { const uid = process.getuid?.(); const metadata = await lstat(path); if ( @@ -1262,10 +1405,13 @@ async function assertPrivateDirectory(path: string): Promise { || uid === undefined || metadata.uid !== uid || (metadata.mode & 0o777) !== PRIVATE_DIRECTORY_MODE + || (expected !== undefined + && (metadata.dev !== expected.device || metadata.ino !== expected.inode)) || await realpath(path) !== path ) { - fail(`${path} is not the expected private physical directory`); + fail("private staging directory changed"); } + return Object.freeze({ device: metadata.dev, inode: metadata.ino }); } async function assertPrivateFile(path: string, expectedBytes: number): Promise { @@ -1276,10 +1422,143 @@ async function assertPrivateFile(path: string, expectedBytes: number): Promise>, +): Promise<{ readonly path: string; readonly identity: PrivateDirectoryIdentity }> { + for (let attempt = 0; attempt < 8; attempt += 1) { + await assertParentUnchanged(output); + const candidate = await mkdtemp(resolve(output.parent, ".message-like-me-staging-")); + let identity: PrivateDirectoryIdentity | undefined; + try { + await chmod(candidate, PRIVATE_DIRECTORY_MODE); + identity = await assertPrivateDirectory(candidate); + const metadata = await lstat(candidate); + if (dirname(candidate) !== output.parent || metadata.dev !== output.parentDevice) { + fail("private staging directory must share the output parent and filesystem"); + } + if (candidate === output.outputRoot) { + await removeOwnedPrivateDirectory(candidate, identity); + continue; + } + return Object.freeze({ path: candidate, identity }); + } catch (error) { + if (identity === undefined) { + try { + await rmdir(candidate); + } catch { + // Never recursively remove a directory whose identity was not captured. + } + } else { + await removeOwnedPrivateDirectory(candidate, identity); + } + throw error; + } + } + return fail("could not allocate a private staging directory distinct from outputRoot"); +} + +async function removeOwnedPrivateDirectory( + path: string, + identity: PrivateDirectoryIdentity, +): Promise { + try { + removePrivateDirectoryTree(path, Object.freeze({ + device: String(identity.device), + inode: String(identity.inode), + })); + } catch { + return fail("private directory could not be removed from quarantine safely"); + } +} + +async function syncDirectory( + path: string, + expected: PrivateDirectoryIdentity, +): Promise { + const handle = await open( + path, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ); + try { + const metadata = await handle.stat(); + if ( + !metadata.isDirectory() + || metadata.dev !== expected.device + || metadata.ino !== expected.inode + ) { + fail("directory changed before synchronization"); + } + await handle.sync(); + } finally { + await handle.close(); + } +} + +async function privateFileSha256(path: string, expectedBytes: number): Promise { + const handle = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const uid = process.getuid?.(); + const before = await handle.stat(); + const entryBefore = await lstat(path); + if ( + !before.isFile() + || uid === undefined + || before.uid !== uid + || before.nlink !== 1 + || (before.mode & 0o777) !== PRIVATE_FILE_MODE + || before.size !== expectedBytes + || !entryBefore.isFile() + || entryBefore.isSymbolicLink() + || entryBefore.dev !== before.dev + || entryBefore.ino !== before.ino + || entryBefore.nlink !== 1 + ) { + fail("private staged artifact changed before bundle validation"); + } + const hash = createHash("sha256"); + const buffer = Buffer.allocUnsafe(64 * 1024); + let offset = 0; + while (offset < expectedBytes) { + const length = Math.min(buffer.byteLength, expectedBytes - offset); + const { bytesRead } = await handle.read(buffer, 0, length, offset); + if (bytesRead === 0) fail("private staged artifact ended during bundle validation"); + hash.update(buffer.subarray(0, bytesRead)); + offset += bytesRead; + } + const after = await handle.stat(); + const entryAfter = await lstat(path); + if ( + after.dev !== before.dev + || after.ino !== before.ino + || !after.isFile() + || after.uid !== before.uid + || after.nlink !== 1 + || after.size !== before.size + || after.mtimeMs !== before.mtimeMs + || after.ctimeMs !== before.ctimeMs + || (after.mode & 0o777) !== PRIVATE_FILE_MODE + || !entryAfter.isFile() + || entryAfter.isSymbolicLink() + || entryAfter.dev !== after.dev + || entryAfter.ino !== after.ino + || entryAfter.nlink !== 1 + ) { + fail("private staged artifact changed during bundle validation"); + } + return hash.digest("hex"); + } finally { + await handle.close(); } } @@ -1293,8 +1572,15 @@ async function createWriter(staging: string, kind: RecordKind, fileName: string) ); try { await handle.chmod(PRIVATE_FILE_MODE); + const uid = process.getuid?.(); const metadata = await handle.stat(); - if (!metadata.isFile() || (metadata.mode & 0o777) !== PRIVATE_FILE_MODE) { + if ( + !metadata.isFile() + || uid === undefined + || metadata.uid !== uid + || metadata.nlink !== 1 + || (metadata.mode & 0o777) !== PRIVATE_FILE_MODE + ) { fail(`could not create private staging file ${fileName}`); } } catch (error) { @@ -1348,15 +1634,23 @@ async function closeWriter(writer: ArtifactWriter): Promise { await writer.handle.close(); } -async function finalizeWriter(writer: ArtifactWriter, outputRoot: string): Promise { +async function finalizeWriter(writer: ArtifactWriter, staging: string): Promise { await writer.handle.sync(); + const uid = process.getuid?.(); const opened = await writer.handle.stat(); - if (!opened.isFile() || opened.size !== writer.bytes || (opened.mode & 0o777) !== PRIVATE_FILE_MODE) { + if ( + !opened.isFile() + || uid === undefined + || opened.uid !== uid + || opened.nlink !== 1 + || opened.size !== writer.bytes + || (opened.mode & 0o777) !== PRIVATE_FILE_MODE + ) { fail(`${writer.fileName} changed before finalization`); } await closeWriter(writer); await assertPrivateFile(writer.partPath, writer.bytes); - const finalPath = resolve(outputRoot, writer.fileName); + const finalPath = resolve(staging, writer.fileName); await assertAbsent(finalPath, writer.fileName); await rename(writer.partPath, finalPath); await assertPrivateFile(finalPath, writer.bytes); @@ -1372,11 +1666,10 @@ async function finalizeWriter(writer: ArtifactWriter, outputRoot: string): Promi async function writeManifest( staging: string, - outputRoot: string, manifest: BeeperMessageLikeMeManifest, -): Promise { +): Promise { const partPath = resolve(staging, "manifest.json.part"); - const finalPath = resolve(outputRoot, "manifest.json"); + const finalPath = resolve(staging, "manifest.json"); const bytes = Buffer.from(`${canonicalJson(manifest)}\n`, "utf8"); const handle = await open( partPath, @@ -1387,8 +1680,16 @@ async function writeManifest( await handle.chmod(PRIVATE_FILE_MODE); await writeAll(handle, bytes); await handle.sync(); + const uid = process.getuid?.(); const metadata = await handle.stat(); - if (!metadata.isFile() || metadata.size !== bytes.byteLength || (metadata.mode & 0o777) !== PRIVATE_FILE_MODE) { + if ( + !metadata.isFile() + || uid === undefined + || metadata.uid !== uid + || metadata.nlink !== 1 + || metadata.size !== bytes.byteLength + || (metadata.mode & 0o777) !== PRIVATE_FILE_MODE + ) { fail("manifest changed before finalization"); } } finally { @@ -1398,7 +1699,42 @@ async function writeManifest( await assertAbsent(finalPath, "manifest.json"); await rename(partPath, finalPath); await assertPrivateFile(finalPath, bytes.byteLength); - return sha256(bytes.toString("utf8")); + return Object.freeze({ + bytes: bytes.byteLength, + sha256: sha256(bytes.toString("utf8")), + }); +} + +async function validateCompleteBundle( + root: string, + identity: PrivateDirectoryIdentity, + artifacts: readonly BeeperMessageLikeMeArtifact[], + manifest: StagedManifest, +): Promise { + await assertPrivateDirectory(root, identity); + const expectedNames = [ + ...artifacts.map((artifact) => artifact.path), + "manifest.json", + ].sort(); + const observedNames = (await readdir(root)).sort(); + if ( + observedNames.length !== expectedNames.length + || observedNames.some((name, index) => name !== expectedNames[index]) + ) { + fail("private staging directory does not contain the exact complete bundle"); + } + for (const artifact of artifacts) { + const path = resolve(root, artifact.path); + await assertPrivateFile(path, artifact.bytes); + if (await privateFileSha256(path, artifact.bytes) !== artifact.sha256) { + fail(`${artifact.path} changed before publication`); + } + } + const manifestPath = resolve(root, "manifest.json"); + await assertPrivateFile(manifestPath, manifest.bytes); + if (await privateFileSha256(manifestPath, manifest.bytes) !== manifest.sha256) { + fail("manifest.json changed before publication"); + } } function assertSource(source: BeeperMessageLikeMeExportSource): void { @@ -1409,8 +1745,9 @@ function assertSource(source: BeeperMessageLikeMeExportSource): void { || typeof source.records !== "object" || source.records === null || typeof source.records[Symbol.asyncIterator] !== "function" + || (source.dispose !== undefined && typeof source.dispose !== "function") ) { - fail("source must expose an async record stream and completion function"); + fail("source must expose an async record stream, completion function, and optional dispose function"); } } @@ -1418,35 +1755,146 @@ function throwIfAborted(signal: AbortSignal | undefined): void { if (signal?.aborted === true) fail("export was aborted"); } +type BundlePhaseHeartbeat = Readonly<{ + stop: () => void; + assertHealthy: () => void; +}>; + +function startBundlePhaseHeartbeat( + request: BeeperMessageLikeMeExportRequest, + phase: "bundle-validating" | "bundle-publishing", + records: number, + bytes: number, +): BundlePhaseHeartbeat { + const startedAt = Date.now(); + let failed = false; + const report = (elapsedSeconds: number): void => { + request.onProgress?.(Object.freeze({ + phase, + elapsedSeconds, + records, + bytes, + })); + }; + report(0); + const heartbeat = request.onProgress === undefined + ? null + : setInterval(() => { + try { + report(Math.max(1, Math.floor((Date.now() - startedAt) / 1_000))); + } catch { + failed = true; + } + }, BUNDLE_HEARTBEAT_INTERVAL_MS); + return Object.freeze({ + stop: () => { + if (heartbeat !== null) clearInterval(heartbeat); + }, + assertHealthy: () => { + if (failed) fail("export progress reporting failed"); + }, + }); +} + +function startPrivateCleanupHeartbeat( + request: BeeperMessageLikeMeExportRequest, +): BundlePhaseHeartbeat { + const startedAt = Date.now(); + let failed = false; + const report = (elapsedSeconds: number): void => { + request.onProgress?.(Object.freeze({ + phase: "private-cleanup", + elapsedSeconds, + })); + }; + try { + report(0); + } catch { + failed = true; + } + const heartbeat = request.onProgress === undefined + ? null + : setInterval(() => { + try { + report(Math.max(1, Math.floor((Date.now() - startedAt) / 1_000))); + } catch { + failed = true; + } + }, BUNDLE_HEARTBEAT_INTERVAL_MS); + return Object.freeze({ + stop: () => { + if (heartbeat !== null) clearInterval(heartbeat); + }, + assertHealthy: () => { + if (failed) fail("export progress reporting failed"); + }, + }); +} + /** * Writes a private, local Message Like Me interchange bundle. The source owns * provider access; this function accepts only bounded foreign records and * never invokes Beeper, follows media references, or receives credentials. * - * A successful bundle has `manifest.json`. A failed export deliberately leaves - * a mode-0700 directory without a manifest so it cannot be mistaken for a - * complete bundle or silently overwrite a later retry. + * The complete bundle is built and validated in a mode-0700 sibling directory + * on the destination filesystem. `outputRoot` remains absent until one final + * atomic rename publishes all seven files together. Failure and cancellation + * remove the owned staging directory without publishing a partial bundle. */ -export async function exportBeeperMessageLikeMeBundle( +async function publishBeeperMessageLikeMeBundle( request: BeeperMessageLikeMeExportRequest, -): Promise { - assertSource(request.source); +): Promise { + let published = false; const descriptor = parseDescriptor(request.source.descriptor); const limits = parseLimits(request.limits); throwIfAborted(request.signal); const startedAt = now(request.clock, "startedAt"); const output = await validateOutputRoot(request.outputRoot); - await assertParentUnchanged(output); - await mkdir(output.outputRoot, { mode: PRIVATE_DIRECTORY_MODE }); - await chmod(output.outputRoot, PRIVATE_DIRECTORY_MODE); - await assertPrivateDirectory(output.outputRoot); - - const staging = resolve(output.outputRoot, ".message-like-me-staging"); - await mkdir(staging, { mode: PRIVATE_DIRECTORY_MODE }); - await chmod(staging, PRIVATE_DIRECTORY_MODE); - await assertPrivateDirectory(staging); + const stagedDirectory = await createPrivateStagingDirectory(output); + const staging = stagedDirectory.path; + let directoryLease: BeeperMessageLikeMeDirectoryLease | undefined; + + if (request.recoveryEnvironment !== undefined) { + try { + const leaseCreatedAtMs = Date.now(); + directoryLease = await createBeeperMessageLikeMeDirectoryLease({ + role: "bundle-stage", + path: staging, + outputRoot: output.outputRoot, + recoverAfterMs: leaseCreatedAtMs, + nowMs: leaseCreatedAtMs, + environment: request.recoveryEnvironment, + }); + } catch (leaseError) { + const cleanupErrors: unknown[] = []; + try { + await removeOwnedPrivateDirectory(staging, stagedDirectory.identity); + } catch (error) { + cleanupErrors.push(error); + } + try { + await syncDirectory(output.parent, Object.freeze({ + device: output.parentDevice, + inode: output.parentInode, + })); + } catch (error) { + cleanupErrors.push(error); + } + if (cleanupErrors.length > 0) { + throw new AggregateError( + [leaseError, ...cleanupErrors], + "Beeper Message Like Me export: recovery setup and cleanup both failed", + ); + } + throw leaseError; + } + } const writers = new Map(); + let bundlePhaseHeartbeat: BundlePhaseHeartbeat | undefined; + let renamed = false; + let operationFailed = false; + let operationError: unknown; try { for (const artifact of ARTIFACTS) { writers.set(artifact.kind, await createWriter(staging, artifact.kind, artifact.fileName)); @@ -1462,6 +1910,8 @@ export async function exportBeeperMessageLikeMeBundle( const referencedAccountRealms = new Map(); let totalRecords = 0; let totalBytes = 0; + const bundleStartedAt = Date.now(); + let lastBundleProgressAt = bundleStartedAt; for await (const candidate of request.source.records) { throwIfAborted(request.signal); @@ -1528,6 +1978,22 @@ export async function exportBeeperMessageLikeMeBundle( const writer = writers.get(parsed.kind); if (writer === undefined) fail("internal artifact writer is missing"); totalBytes = await writeRecord(writer, parsed.value, limits, totalBytes); + const progressAt = Date.now(); + if ( + totalRecords === 1 + || progressAt - lastBundleProgressAt >= 30_000 + ) { + request.onProgress?.(Object.freeze({ + phase: "bundle-building", + elapsedSeconds: Math.max( + 0, + Math.floor((progressAt - bundleStartedAt) / 1_000), + ), + records: totalRecords, + bytes: totalBytes, + })); + lastBundleProgressAt = progressAt; + } } for (const [accountId, realm] of referencedAccountRealms) { @@ -1543,6 +2009,12 @@ export async function exportBeeperMessageLikeMeBundle( } } + bundlePhaseHeartbeat = startBundlePhaseHeartbeat( + request, + "bundle-validating", + totalRecords, + totalBytes, + ); validateBundleGraph(graphInventory); throwIfAborted(request.signal); @@ -1554,7 +2026,7 @@ export async function exportBeeperMessageLikeMeBundle( for (const artifact of ARTIFACTS) { const writer = writers.get(artifact.kind); if (writer === undefined) fail("internal artifact writer is missing during finalization"); - artifacts.push(await finalizeWriter(writer, output.outputRoot)); + artifacts.push(await finalizeWriter(writer, staging)); } const counts = Object.freeze(Object.fromEntries( @@ -1588,16 +2060,215 @@ export async function exportBeeperMessageLikeMeBundle( bundleSha256: sha256(canonicalJson(manifestProjection)), }), }); - const manifestSha256 = await writeManifest(staging, output.outputRoot, manifest); - await rmdir(staging); - await assertPrivateDirectory(output.outputRoot); - return Object.freeze({ + const stagedManifest = await writeManifest(staging, manifest); + await validateCompleteBundle( + staging, + stagedDirectory.identity, + artifacts, + stagedManifest, + ); + await syncDirectory(staging, stagedDirectory.identity); + throwIfAborted(request.signal); + await assertParentUnchanged(output); + bundlePhaseHeartbeat.stop(); + bundlePhaseHeartbeat.assertHealthy(); + bundlePhaseHeartbeat = startBundlePhaseHeartbeat( + request, + "bundle-publishing", + totalRecords, + totalBytes, + ); + + const result = Object.freeze({ outputRoot: output.outputRoot, manifestPath: resolve(output.outputRoot, "manifest.json"), - manifestSha256, + manifestSha256: stagedManifest.sha256, manifest, }); + + await renameDirectoryExclusive(staging, output.outputRoot); + renamed = true; + await validateCompleteBundle( + output.outputRoot, + stagedDirectory.identity, + artifacts, + stagedManifest, + ); + throwIfAborted(request.signal); + await assertParentUnchanged(output); + await syncDirectory(output.parent, Object.freeze({ + device: output.parentDevice, + inode: output.parentInode, + })); + bundlePhaseHeartbeat.stop(); + bundlePhaseHeartbeat.assertHealthy(); + bundlePhaseHeartbeat = undefined; + const publishedDirectory = Object.freeze({ + path: output.outputRoot, + parent: output.parent, + identity: stagedDirectory.identity, + parentIdentity: Object.freeze({ + device: output.parentDevice, + inode: output.parentInode, + }), + }); + published = true; + return Object.freeze({ + result, + directory: publishedDirectory, + ...(directoryLease === undefined ? {} : { directoryLease }), + }); + } catch (error) { + operationFailed = true; + operationError = error; + throw error; + } finally { + const cleanupErrors: unknown[] = []; + bundlePhaseHeartbeat?.stop(); + const closeResults = await Promise.allSettled( + [...writers.values()].map((writer) => closeWriter(writer)), + ); + for (const result of closeResults) { + if (result.status === "rejected") cleanupErrors.push(result.reason); + } + if (!published) { + let removalDurable = true; + try { + await removeOwnedPrivateDirectory( + renamed ? output.outputRoot : staging, + stagedDirectory.identity, + ); + } catch (error) { + removalDurable = false; + cleanupErrors.push(error); + } + try { + await syncDirectory(output.parent, Object.freeze({ + device: output.parentDevice, + inode: output.parentInode, + })); + } catch (error) { + removalDurable = false; + cleanupErrors.push(error); + } + if (directoryLease !== undefined && removalDurable) { + try { + releaseBeeperMessageLikeMeDirectoryLease(directoryLease); + } catch (error) { + cleanupErrors.push(error); + } + } + } + if (cleanupErrors.length > 0) { + if (operationFailed) { + throw new AggregateError( + [operationError, ...cleanupErrors], + "Beeper Message Like Me export: publication and cleanup both failed", + ); + } + if (cleanupErrors.length === 1) throw cleanupErrors[0]; + throw new AggregateError( + cleanupErrors, + "Beeper Message Like Me export: cleanup failed", + ); + } + } +} + +export async function exportBeeperMessageLikeMeBundle( + request: BeeperMessageLikeMeExportRequest, +): Promise { + assertSource(request.source); + let publication: PublishedBundle | undefined; + let publicationFailed = false; + let publicationError: unknown; + try { + publication = await publishBeeperMessageLikeMeBundle(request); + return publication.result; + } catch (error) { + publicationFailed = true; + publicationError = error; + throw error; } finally { - await Promise.all([...writers.values()].map((writer) => closeWriter(writer))); + const finalizationErrors: unknown[] = []; + const cleanupHeartbeat = request.source.dispose === undefined + ? undefined + : startPrivateCleanupHeartbeat(request); + try { + await request.source.dispose?.(publication !== undefined); + } catch (error) { + finalizationErrors.push(error); + } + let releaseAttempted = false; + if ( + publication !== undefined + && finalizationErrors.length === 0 + && publication.directoryLease !== undefined + ) { + releaseAttempted = true; + try { + releaseBeeperMessageLikeMeDirectoryLease(publication.directoryLease); + } catch (error) { + finalizationErrors.push(error); + } + } + cleanupHeartbeat?.stop(); + try { + cleanupHeartbeat?.assertHealthy(); + } catch (error) { + finalizationErrors.push(error); + } + if (finalizationErrors.length > 0) { + const errors = publicationFailed + ? [publicationError, ...finalizationErrors] + : [...finalizationErrors]; + if (publication !== undefined) { + let rollbackDurable = true; + try { + await removeOwnedPrivateDirectory( + publication.directory.path, + publication.directory.identity, + ); + } catch (error) { + rollbackDurable = false; + errors.push(error); + } + try { + await syncDirectory( + publication.directory.parent, + publication.directory.parentIdentity, + ); + } catch (error) { + rollbackDurable = false; + errors.push(error); + } + if ( + rollbackDurable + && !releaseAttempted + && publication.directoryLease !== undefined + ) { + try { + releaseBeeperMessageLikeMeDirectoryLease( + publication.directoryLease, + ); + } catch (error) { + errors.push(error); + } + } + if (errors.length > finalizationErrors.length) { + throw new AggregateError( + errors, + "Beeper Message Like Me export: finalization and published output rollback failed", + ); + } + } + if (errors.length === 1) throw errors[0]; + throw new AggregateError( + errors, + publicationFailed + ? "Beeper Message Like Me export: publication and source disposal both failed" + : "Beeper Message Like Me export: finalization failed", + ); + } } } diff --git a/src/beeper-message-like-me-golden-fixture.ts b/src/beeper-message-like-me-golden-fixture.ts index a740c5f..7f05266 100644 --- a/src/beeper-message-like-me-golden-fixture.ts +++ b/src/beeper-message-like-me-golden-fixture.ts @@ -10,18 +10,25 @@ const accountId = "account:synthetic:primary"; const connectedAccountProviderId = "beeper-account:synthetic-primary"; const selfParticipantId = "participant:synthetic:self"; const peerParticipantId = "participant:synthetic:peer"; +const secondaryAccountId = "account:synthetic:secondary"; +const secondaryConnectedAccountProviderId = "beeper-account:synthetic-secondary"; +const secondarySelfParticipantId = "participant:synthetic:secondary-self"; const conversationId = "conversation:synthetic:friend"; const editedMessageId = "message:synthetic:edited"; const editedMessageProviderId = "beeper-message:synthetic-edited"; const deletedMessageId = "message:synthetic:deleted"; const deletedMessageProviderId = "beeper-message:synthetic-deleted"; -function provenance(providerId: string, providerRevision: string | null) { +function provenance( + providerId: string, + providerRevision: string | null, + accountProviderId = connectedAccountProviderId, +) { return Object.freeze({ providerId, providerRevision, observedAt, - connectedAccountProviderId, + connectedAccountProviderId: accountProviderId, }); } @@ -35,6 +42,20 @@ const records = Object.freeze([{ displayName: "Synthetic Primary", handle: "+15555550100", selfParticipantId, +}, { + schemaVersion: 1, + kind: "account", + id: secondaryAccountId, + accountId: secondaryAccountId, + network: "synthetic-secondary", + provenance: provenance( + secondaryConnectedAccountProviderId, + null, + secondaryConnectedAccountProviderId, + ), + displayName: "Synthetic Secondary", + handle: "synthetic-secondary@example.invalid", + selfParticipantId: secondarySelfParticipantId, }, { schemaVersion: 1, kind: "participant", @@ -55,6 +76,20 @@ const records = Object.freeze([{ displayName: "Synthetic Peer", handle: "+15555550101", isSelf: false, +}, { + schemaVersion: 1, + kind: "participant", + id: secondarySelfParticipantId, + accountId: secondaryAccountId, + network: "synthetic-secondary", + provenance: provenance( + "beeper-participant:synthetic-secondary-self", + null, + secondaryConnectedAccountProviderId, + ), + displayName: "Synthetic Secondary Self", + handle: "synthetic-secondary@example.invalid", + isSelf: true, }, { schemaVersion: 1, kind: "conversation", @@ -146,7 +181,7 @@ const records = Object.freeze([{ export function createBeeperMessageLikeMeGoldenSource(): BeeperMessageLikeMeExportSource { return Object.freeze({ descriptor: Object.freeze({ - source: Object.freeze({ id: "beeper-local", version: "1.0.0" }), + source: Object.freeze({ id: "beeper-local", version: "1.1.0" }), provider: Object.freeze({ id: "beeper", version: "0.6.2" }), }), records: (async function* () { @@ -154,14 +189,16 @@ export function createBeeperMessageLikeMeGoldenSource(): BeeperMessageLikeMeExpo })(), completion: () => Promise.resolve(Object.freeze({ completeness: Object.freeze({ - kind: "truncated", - reason: "explicit-source-limit", + kind: "bounded-local", + reason: "desktop-local-sequential-export", observedFrom: "2026-08-21T15:50:00.000Z", observedThrough: "2026-08-21T15:59:00.000Z", }), warnings: Object.freeze([ "attachments-metadata-only", + "connected-account-backfill-coverage-unknown", "remote-history-not-claimed", + "sequential-account-snapshot", "synthetic-golden-fixture", ]), })), diff --git a/src/beeper-message-like-me-recovery.test.ts b/src/beeper-message-like-me-recovery.test.ts new file mode 100644 index 0000000..33e7ec5 --- /dev/null +++ b/src/beeper-message-like-me-recovery.test.ts @@ -0,0 +1,509 @@ +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { afterEach, describe, expect, test } from "bun:test"; + +import { + acquireBeeperMessageLikeMeExportAdmission, + createBeeperMessageLikeMeDirectoryLease, + recoverBeeperMessageLikeMeDirectoryLeases, + releaseBeeperMessageLikeMeExportAdmission, + releaseBeeperMessageLikeMeDirectoryLease, + updateBeeperMessageLikeMeDirectoryLease, +} from "./beeper-message-like-me-recovery"; + +const temporaryRoots: string[] = []; + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) { + rmSync(root, { force: true, recursive: true }); + } +}); + +function recoveryFixture(name: string): Readonly<{ + environment: Readonly>; + outputRoot: string; + parent: string; + root: string; + target: string; +}> { + const root = realpathSync( + mkdtempSync(join(tmpdir(), `wrench-beeper-recovery-${name}-`)), + ); + temporaryRoots.push(root); + chmodSync(root, 0o700); + const parent = join(root, "private-exports"); + mkdirSync(parent, { mode: 0o700 }); + const target = join(parent, "working"); + mkdirSync(target, { mode: 0o700 }); + return Object.freeze({ + environment: Object.freeze({ WRENCH_STATE_HOME: join(root, "state") }), + outputRoot: join(parent, "published"), + parent, + root, + target, + }); +} + +async function waitForFiles(paths: readonly string[]): Promise { + const deadline = performance.now() + 10_000; + while (!paths.every((path) => existsSync(path))) { + if (performance.now() >= deadline) { + throw new Error("timed out waiting for synchronized admission children"); + } + await Bun.sleep(10); + } +} + +describe("Beeper Message Like Me export admission", () => { + test("rejects a second live owner and permits acquisition after release", () => { + const fixture = recoveryFixture("export-admission-live"); + const first = acquireBeeperMessageLikeMeExportAdmission({ + environment: fixture.environment, + }); + + expect(() => acquireBeeperMessageLikeMeExportAdmission({ + environment: fixture.environment, + })).toThrow("another export is active"); + expect(existsSync(first.claimPath)).toBeTrue(); + + releaseBeeperMessageLikeMeExportAdmission(first); + releaseBeeperMessageLikeMeExportAdmission(first); + expect(first.released).toBeTrue(); + expect(existsSync(first.claimPath)).toBeFalse(); + + const second = acquireBeeperMessageLikeMeExportAdmission({ + environment: fixture.environment, + }); + expect(second.claimId).not.toBe(first.claimId); + releaseBeeperMessageLikeMeExportAdmission(second); + }); + + test("reclaims a dead owner before admitting the next export", () => { + const fixture = recoveryFixture("export-admission-dead"); + const first = acquireBeeperMessageLikeMeExportAdmission({ + environment: fixture.environment, + }); + const second = acquireBeeperMessageLikeMeExportAdmission({ + environment: fixture.environment, + inspectOwnerForTest: () => "different-or-dead", + }); + + expect(second.claimId).not.toBe(first.claimId); + expect(() => releaseBeeperMessageLikeMeExportAdmission(first)) + .toThrow("export admission changed before release"); + expect(existsSync(second.claimPath)).toBeTrue(); + releaseBeeperMessageLikeMeExportAdmission(second); + }); + + test("retains an owner whose liveness cannot be inspected", () => { + const fixture = recoveryFixture("export-admission-unknown"); + const first = acquireBeeperMessageLikeMeExportAdmission({ + environment: fixture.environment, + }); + const original = readFileSync(first.claimPath, "utf8"); + + expect(() => acquireBeeperMessageLikeMeExportAdmission({ + environment: fixture.environment, + inspectOwnerForTest: () => "unknown", + })).toThrow("prior export owner cannot be inspected safely"); + expect(readFileSync(first.claimPath, "utf8")).toBe(original); + + releaseBeeperMessageLikeMeExportAdmission(first); + }); + + test("fails closed without removing a claim changed before release", () => { + const fixture = recoveryFixture("export-admission-cas"); + const admission = acquireBeeperMessageLikeMeExportAdmission({ + environment: fixture.environment, + }); + const replacement = `${readFileSync(admission.claimPath, "utf8")} `; + writeFileSync(admission.claimPath, replacement, { mode: 0o600 }); + + expect(() => releaseBeeperMessageLikeMeExportAdmission(admission)) + .toThrow("export admission changed before release"); + expect(readFileSync(admission.claimPath, "utf8")).toBe(replacement); + expect(admission.released).toBeFalse(); + }); + + test("admits exactly one of two synchronized processes", async () => { + const fixture = recoveryFixture("export-admission-race"); + const barrier = join(fixture.root, "start"); + const release = join(fixture.root, "release"); + const moduleUrl = pathToFileURL(join( + import.meta.dir, + "beeper-message-like-me-recovery.ts", + )).href; + const children = [0, 1].map((index) => { + const ready = join(fixture.root, `ready-${index}`); + const result = join(fixture.root, `result-${index}`); + const source = ` + import { existsSync, writeFileSync } from "node:fs"; + import { + acquireBeeperMessageLikeMeExportAdmission, + releaseBeeperMessageLikeMeExportAdmission, + } from ${JSON.stringify(moduleUrl)}; + const environment = Object.freeze({ + WRENCH_STATE_HOME: ${JSON.stringify(fixture.environment.WRENCH_STATE_HOME)}, + }); + writeFileSync(${JSON.stringify(ready)}, "ready\\n", { mode: 0o600 }); + while (!existsSync(${JSON.stringify(barrier)})) await Bun.sleep(5); + let admission; + try { + admission = acquireBeeperMessageLikeMeExportAdmission({ environment }); + } catch { + writeFileSync(${JSON.stringify(result)}, "blocked\\n", { mode: 0o600 }); + } + if (admission !== undefined) { + writeFileSync(${JSON.stringify(result)}, "acquired\\n", { mode: 0o600 }); + while (!existsSync(${JSON.stringify(release)})) await Bun.sleep(5); + releaseBeeperMessageLikeMeExportAdmission(admission); + } + `; + return Object.freeze({ + child: Bun.spawn([process.execPath, "-e", source], { + env: { ...process.env, NODE_ENV: "test" }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }), + ready, + result, + }); + }); + + let exitCodes: readonly number[] = []; + try { + await waitForFiles(children.map((child) => child.ready)); + writeFileSync(barrier, "start\n", { mode: 0o600 }); + await waitForFiles(children.map((child) => child.result)); + expect(children.map((child) => readFileSync(child.result, "utf8").trim()).sort()) + .toEqual(["acquired", "blocked"]); + } finally { + writeFileSync(release, "release\n", { mode: 0o600 }); + exitCodes = await Promise.all(children.map((child) => child.child.exited)); + } + expect(exitCodes).toEqual([0, 0]); + + const after = acquireBeeperMessageLikeMeExportAdmission({ + environment: fixture.environment, + }); + releaseBeeperMessageLikeMeExportAdmission(after); + }); +}); + +describe("Beeper Message Like Me directory lease recovery", () => { + test("retains an exact live owner's raw directory and claim", async () => { + const fixture = recoveryFixture("active"); + const lease = await createBeeperMessageLikeMeDirectoryLease({ + role: "raw-working", + path: fixture.target, + recoverAfterMs: 2_000, + environment: fixture.environment, + nowMs: 1_000, + }); + + const report = await recoverBeeperMessageLikeMeDirectoryLeases({ + environment: fixture.environment, + nowMs: 10_000, + inspectOwner: () => "exact-live-owner", + }); + + expect(report).toEqual({ + recovered: 0, + published: 0, + active: 1, + indeterminate: 0, + }); + expect(existsSync(fixture.target)).toBeTrue(); + expect(existsSync(lease.claimPath)).toBeTrue(); + }); + + test("reclaims a settled raw directory owned by a dead process", async () => { + const fixture = recoveryFixture("settled"); + const lease = await createBeeperMessageLikeMeDirectoryLease({ + role: "raw-working", + path: fixture.target, + recoverAfterMs: 50_000, + environment: fixture.environment, + nowMs: 1_000, + }); + updateBeeperMessageLikeMeDirectoryLease(lease, "launching"); + updateBeeperMessageLikeMeDirectoryLease(lease, "settled"); + + const report = await recoverBeeperMessageLikeMeDirectoryLeases({ + environment: fixture.environment, + nowMs: 1_001, + inspectOwner: () => "different-or-dead", + }); + + expect(report).toEqual({ + recovered: 1, + published: 0, + active: 0, + indeterminate: 0, + }); + expect(existsSync(fixture.target)).toBeFalse(); + expect(existsSync(lease.claimPath)).toBeFalse(); + }); + + test("retains a dead launching lease until its deadline, then reclaims it", async () => { + const fixture = recoveryFixture("launching-deadline"); + const lease = await createBeeperMessageLikeMeDirectoryLease({ + role: "raw-working", + path: fixture.target, + recoverAfterMs: 2_000, + environment: fixture.environment, + nowMs: 1_000, + }); + updateBeeperMessageLikeMeDirectoryLease(lease, "launching"); + + const retained = await recoverBeeperMessageLikeMeDirectoryLeases({ + environment: fixture.environment, + nowMs: 1_999, + inspectOwner: () => "different-or-dead", + }); + expect(retained).toEqual({ + recovered: 0, + published: 0, + active: 0, + indeterminate: 1, + }); + expect(existsSync(fixture.target)).toBeTrue(); + expect(existsSync(lease.claimPath)).toBeTrue(); + + const recovered = await recoverBeeperMessageLikeMeDirectoryLeases({ + environment: fixture.environment, + nowMs: 2_000, + inspectOwner: () => "different-or-dead", + }); + expect(recovered).toEqual({ + recovered: 1, + published: 0, + active: 0, + indeterminate: 0, + }); + expect(existsSync(fixture.target)).toBeFalse(); + expect(existsSync(lease.claimPath)).toBeFalse(); + }); + + test("classifies an orphaned running child before recovering raw data", async () => { + const cases = [ + { + name: "live-child", + childStatus: "exact-live-owner" as const, + report: { recovered: 0, published: 0, active: 1, indeterminate: 0 }, + retained: true, + }, + { + name: "unknown-child", + childStatus: "unknown" as const, + report: { recovered: 0, published: 0, active: 0, indeterminate: 1 }, + retained: true, + }, + { + name: "dead-child", + childStatus: "different-or-dead" as const, + report: { recovered: 1, published: 0, active: 0, indeterminate: 0 }, + retained: false, + }, + ]; + + for (const item of cases) { + const fixture = recoveryFixture(item.name); + const lease = await createBeeperMessageLikeMeDirectoryLease({ + role: "raw-working", + path: fixture.target, + recoverAfterMs: 50_000, + environment: fixture.environment, + nowMs: 1_000, + }); + updateBeeperMessageLikeMeDirectoryLease(lease, "launching"); + updateBeeperMessageLikeMeDirectoryLease(lease, "running", process.pid); + let inspections = 0; + + const report = await recoverBeeperMessageLikeMeDirectoryLeases({ + environment: fixture.environment, + nowMs: 1_001, + inspectOwner: () => { + inspections += 1; + return inspections === 1 ? "different-or-dead" : item.childStatus; + }, + }); + + expect(report).toEqual(item.report); + expect(inspections).toBe(2); + expect(existsSync(fixture.target)).toBe(item.retained); + expect(existsSync(lease.claimPath)).toBe(item.retained); + } + }); + + test("fails closed and preserves a replacement at the leased path", async () => { + const fixture = recoveryFixture("replacement"); + const lease = await createBeeperMessageLikeMeDirectoryLease({ + role: "raw-working", + path: fixture.target, + recoverAfterMs: 1_000, + environment: fixture.environment, + nowMs: 1_000, + }); + updateBeeperMessageLikeMeDirectoryLease(lease, "launching"); + updateBeeperMessageLikeMeDirectoryLease(lease, "settled"); + const displaced = join(fixture.parent, "displaced-original"); + renameSync(fixture.target, displaced); + mkdirSync(fixture.target, { mode: 0o700 }); + const replacementMarker = join(fixture.target, "replacement.txt"); + writeFileSync(replacementMarker, "replacement\n", { mode: 0o600 }); + + await expect(recoverBeeperMessageLikeMeDirectoryLeases({ + environment: fixture.environment, + nowMs: 1_001, + inspectOwner: () => "different-or-dead", + })).rejects.toThrow("directory lease target changed before recovery"); + + expect(readFileSync(replacementMarker, "utf8")).toBe("replacement\n"); + expect(existsSync(displaced)).toBeTrue(); + expect(existsSync(lease.claimPath)).toBeTrue(); + }); + + test("reclaims an unpublished bundle stage owned by a dead process", async () => { + const fixture = recoveryFixture("stage-dead"); + const lease = await createBeeperMessageLikeMeDirectoryLease({ + role: "bundle-stage", + path: fixture.target, + outputRoot: fixture.outputRoot, + recoverAfterMs: 1_000, + environment: fixture.environment, + nowMs: 1_000, + }); + + const report = await recoverBeeperMessageLikeMeDirectoryLeases({ + environment: fixture.environment, + nowMs: 1_001, + inspectOwner: () => "different-or-dead", + }); + + expect(report).toEqual({ + recovered: 1, + published: 0, + active: 0, + indeterminate: 0, + }); + expect(existsSync(fixture.target)).toBeFalse(); + expect(existsSync(lease.claimPath)).toBeFalse(); + }); + + test("preserves and reports a stage atomically renamed to its output", async () => { + const fixture = recoveryFixture("stage-published"); + const lease = await createBeeperMessageLikeMeDirectoryLease({ + role: "bundle-stage", + path: fixture.target, + outputRoot: fixture.outputRoot, + recoverAfterMs: 1_000, + environment: fixture.environment, + nowMs: 1_000, + }); + const marker = join(fixture.target, "published.txt"); + writeFileSync(marker, "published\n", { mode: 0o600 }); + renameSync(fixture.target, fixture.outputRoot); + + const report = await recoverBeeperMessageLikeMeDirectoryLeases({ + environment: fixture.environment, + nowMs: 1_001, + inspectOwner: () => "different-or-dead", + }); + + expect(report).toEqual({ + recovered: 0, + published: 1, + active: 0, + indeterminate: 0, + }); + expect(readFileSync(join(fixture.outputRoot, "published.txt"), "utf8")) + .toBe("published\n"); + expect(existsSync(lease.claimPath)).toBeFalse(); + }); + + test("fails closed when an unrelated directory appears at a staged output", async () => { + const fixture = recoveryFixture("stage-output-replacement"); + const lease = await createBeeperMessageLikeMeDirectoryLease({ + role: "bundle-stage", + path: fixture.target, + outputRoot: fixture.outputRoot, + recoverAfterMs: 1_000, + environment: fixture.environment, + nowMs: 1_000, + }); + rmSync(fixture.target, { recursive: true }); + mkdirSync(fixture.outputRoot, { mode: 0o700 }); + const marker = join(fixture.outputRoot, "replacement.txt"); + writeFileSync(marker, "replacement\n", { mode: 0o600 }); + + await expect(recoverBeeperMessageLikeMeDirectoryLeases({ + environment: fixture.environment, + nowMs: 1_001, + inspectOwner: () => "different-or-dead", + })).rejects.toThrow("directory lease output changed before recovery"); + + expect(readFileSync(marker, "utf8")).toBe("replacement\n"); + expect(existsSync(lease.claimPath)).toBeTrue(); + }); + + test("updates and releases by compare-and-swap while rejecting a changed claim", async () => { + const fixture = recoveryFixture("lifecycle-cas"); + const lease = await createBeeperMessageLikeMeDirectoryLease({ + role: "raw-working", + path: fixture.target, + recoverAfterMs: 2_000, + environment: fixture.environment, + nowMs: 1_000, + }); + + updateBeeperMessageLikeMeDirectoryLease(lease, "launching"); + expect(lease.claim.phase).toBe("launching"); + updateBeeperMessageLikeMeDirectoryLease(lease, "running", process.pid); + expect(lease.claim.phase).toBe("running"); + expect(lease.claim.childOwner?.pid).toBe(process.pid); + updateBeeperMessageLikeMeDirectoryLease(lease, "settled"); + expect(lease.claim.phase).toBe("settled"); + expect(lease.claim.childOwner).toBeNull(); + releaseBeeperMessageLikeMeDirectoryLease(lease); + releaseBeeperMessageLikeMeDirectoryLease(lease); + expect(lease.released).toBeTrue(); + expect(existsSync(lease.claimPath)).toBeFalse(); + expect(existsSync(fixture.target)).toBeTrue(); + + const secondTarget = join(fixture.parent, "second-working"); + mkdirSync(secondTarget, { mode: 0o700 }); + const changed = await createBeeperMessageLikeMeDirectoryLease({ + role: "raw-working", + path: secondTarget, + recoverAfterMs: 2_000, + environment: fixture.environment, + nowMs: 1_000, + }); + writeFileSync( + changed.claimPath, + `${readFileSync(changed.claimPath, "utf8")} `, + { mode: 0o600 }, + ); + + expect(() => updateBeeperMessageLikeMeDirectoryLease(changed, "launching")) + .toThrow("directory lease changed before its lifecycle update"); + expect(changed.claim.phase).toBe("preparing"); + expect(existsSync(secondTarget)).toBeTrue(); + expect(existsSync(changed.claimPath)).toBeTrue(); + }); +}); diff --git a/src/beeper-message-like-me-recovery.ts b/src/beeper-message-like-me-recovery.ts new file mode 100644 index 0000000..ec4856b --- /dev/null +++ b/src/beeper-message-like-me-recovery.ts @@ -0,0 +1,729 @@ +import { randomUUID } from "node:crypto"; +import { lstat, realpath } from "node:fs/promises"; +import { dirname, isAbsolute, join, resolve } from "node:path"; + +import { canonicalJson, sha256 } from "./canonical-json"; +import { + captureProcessOwnerIdentity, + currentProcessStartIdentity, + processOwnerStatus, + type ProcessOwnerIdentity, + type ProcessOwnerStatus, +} from "./process-identity"; +import { + createPrivateJsonIfAbsent, + ensurePrivateDirectory, + readPrivateStateFileIfPresent, + removePrivateDirectoryTree, + removePrivateStateFileIfUnchanged, + snapshotPrivateStateDirectory, + readPrivateStateFilesBatch, + wrenchStateHome, + writePrivateJsonIfUnchanged, + type PrivateDirectoryIdentity, +} from "./storage"; + +const CLAIM_KIND = "beeper-message-like-me-directory-lease"; +const CLAIM_SCHEMA_VERSION = 1; +const EXPORT_ADMISSION_KIND = "beeper-message-like-me-export-admission"; +const EXPORT_ADMISSION_SCHEMA_VERSION = 1; +const EXPORT_ADMISSION_FILE = "active.json"; +const MAX_EXPORT_ADMISSION_ACQUIRE_ATTEMPTS = 8; +const MAX_CLAIMS = 64; +const MAX_CLAIM_BYTES = 16 * 1024; +const UUID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; +const DIGEST_PATTERN = /^[a-f0-9]{64}$/u; + +type LeaseRole = "raw-working" | "bundle-stage"; +type RawPhase = "preparing" | "launching" | "running" | "settled"; + +type ExportAdmissionClaim = Readonly<{ + schemaVersion: 1; + kind: typeof EXPORT_ADMISSION_KIND; + id: string; + owner: ProcessOwnerIdentity; +}>; + +type ExportAdmissionSnapshot = Readonly<{ + claim: ExportAdmissionClaim; + contentSha256: string; +}>; + +export type BeeperMessageLikeMeExportAdmission = { + readonly claimPath: string; + readonly environment: Readonly>; + readonly claimId: string; + readonly contentSha256: string; + released: boolean; +}; + +type DirectoryLeaseClaim = Readonly<{ + schemaVersion: 1; + kind: typeof CLAIM_KIND; + id: string; + role: LeaseRole; + path: string; + parentPath: string; + parentIdentity: PrivateDirectoryIdentity; + directoryIdentity: PrivateDirectoryIdentity; + outputRoot: string | null; + owner: ProcessOwnerIdentity; + childOwner: ProcessOwnerIdentity | null; + phase: RawPhase; + createdAtMs: number; + recoverAfterMs: number; +}>; + +export type BeeperMessageLikeMeDirectoryLease = { + readonly claimPath: string; + readonly environment: Readonly>; + claim: DirectoryLeaseClaim; + contentSha256: string; + released: boolean; +}; + +export type BeeperMessageLikeMeRecoveryReport = Readonly<{ + recovered: number; + published: number; + active: number; + indeterminate: number; +}>; + +function fail(message: string): never { + throw new Error(`Beeper Message Like Me recovery: ${message}`); +} + +function isErrno(error: unknown, code: string): boolean { + return typeof error === "object" && error !== null && "code" in error + && (error as { readonly code?: unknown }).code === code; +} + +function record(value: unknown, label: string): Readonly> { + if ( + typeof value !== "object" + || value === null + || Array.isArray(value) + || Object.getPrototypeOf(value) !== Object.prototype + ) return fail(`${label} must be a plain object`); + return value as Readonly>; +} + +function exactKeys( + value: Readonly>, + keys: readonly string[], + label: string, +): void { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if ( + actual.length !== expected.length + || actual.some((key, index) => key !== expected[index]) + ) fail(`${label} has an unsupported shape`); +} + +function boundedPath(value: unknown, label: string): string { + if ( + typeof value !== "string" + || !isAbsolute(value) + || resolve(value) !== value + || value.length > 4_096 + || /[\0\r\n]/u.test(value) + ) return fail(`${label} is invalid`); + return value; +} + +function identity(value: unknown, label: string): PrivateDirectoryIdentity { + const source = record(value, label); + exactKeys(source, ["device", "inode"], label); + if ( + typeof source.device !== "string" + || !/^\d{1,40}$/u.test(source.device) + || typeof source.inode !== "string" + || !/^\d{1,40}$/u.test(source.inode) + ) return fail(`${label} is invalid`); + return Object.freeze({ device: source.device, inode: source.inode }); +} + +function owner(value: unknown, label: string): ProcessOwnerIdentity { + const source = record(value, label); + exactKeys(source, ["bootId", "pid", "processStartId"], label); + if ( + typeof source.pid !== "number" + || !Number.isSafeInteger(source.pid) + || source.pid < 1 + || source.pid > 2_147_483_647 + || typeof source.bootId !== "string" + || !DIGEST_PATTERN.test(source.bootId) + || typeof source.processStartId !== "string" + || !DIGEST_PATTERN.test(source.processStartId) + ) return fail(`${label} is invalid`); + return Object.freeze({ + pid: source.pid, + bootId: source.bootId, + processStartId: source.processStartId, + }); +} + +function parseExportAdmission(value: unknown): ExportAdmissionClaim { + const source = record(value, "export admission"); + exactKeys(source, ["id", "kind", "owner", "schemaVersion"], "export admission"); + if ( + source.schemaVersion !== EXPORT_ADMISSION_SCHEMA_VERSION + || source.kind !== EXPORT_ADMISSION_KIND + || typeof source.id !== "string" + || !UUID_PATTERN.test(source.id) + ) return fail("export admission identity is invalid"); + return Object.freeze({ + schemaVersion: 1, + kind: EXPORT_ADMISSION_KIND, + id: source.id, + owner: owner(source.owner, "export admission owner"), + }); +} + +function milliseconds(value: unknown, label: string): number { + if ( + typeof value !== "number" + || !Number.isSafeInteger(value) + || value < 0 + ) return fail(`${label} is invalid`); + return value; +} + +function parseClaim(value: unknown): DirectoryLeaseClaim { + const source = record(value, "directory lease"); + exactKeys(source, [ + "childOwner", + "createdAtMs", + "directoryIdentity", + "id", + "kind", + "outputRoot", + "owner", + "parentIdentity", + "parentPath", + "path", + "phase", + "recoverAfterMs", + "role", + "schemaVersion", + ], "directory lease"); + if ( + source.schemaVersion !== CLAIM_SCHEMA_VERSION + || source.kind !== CLAIM_KIND + || typeof source.id !== "string" + || !UUID_PATTERN.test(source.id) + || (source.role !== "raw-working" && source.role !== "bundle-stage") + || !["preparing", "launching", "running", "settled"].includes( + typeof source.phase === "string" ? source.phase : "", + ) + ) return fail("directory lease identity is invalid"); + const path = boundedPath(source.path, "directory lease path"); + const parentPath = boundedPath(source.parentPath, "directory lease parent"); + const outputRoot = source.outputRoot === null + ? null + : boundedPath(source.outputRoot, "directory lease output"); + const createdAtMs = milliseconds(source.createdAtMs, "directory lease creation time"); + const recoverAfterMs = milliseconds( + source.recoverAfterMs, + "directory lease recovery time", + ); + if ( + dirname(path) !== parentPath + || recoverAfterMs < createdAtMs + || (source.role === "bundle-stage") !== (outputRoot !== null) + || (outputRoot !== null && dirname(outputRoot) !== parentPath) + || (source.role === "bundle-stage" && source.phase !== "preparing") + || (source.role === "bundle-stage" && source.childOwner !== null) + || (source.phase === "running") !== (source.childOwner !== null) + ) return fail("directory lease coordinates are inconsistent"); + return Object.freeze({ + schemaVersion: 1, + kind: CLAIM_KIND, + id: source.id, + role: source.role, + path, + parentPath, + parentIdentity: identity(source.parentIdentity, "directory lease parent identity"), + directoryIdentity: identity( + source.directoryIdentity, + "directory lease directory identity", + ), + outputRoot, + owner: owner(source.owner, "directory lease owner"), + childOwner: source.childOwner === null + ? null + : owner(source.childOwner, "directory lease child owner"), + phase: source.phase as RawPhase, + createdAtMs, + recoverAfterMs, + }); +} + +function claimBytes(claim: DirectoryLeaseClaim): string { + return `${canonicalJson(claim)}\n`; +} + +function claimSha256(claim: DirectoryLeaseClaim): string { + return sha256(claimBytes(claim)); +} + +function leaseRoot( + environment: Readonly>, +): string { + return join( + wrenchStateHome(environment), + "recovery", + "beeper-message-like-me-directory-leases", + ); +} + +function exportAdmissionRoot( + environment: Readonly>, +): string { + return join( + wrenchStateHome(environment), + "recovery", + "beeper-message-like-me-export-admission", + ); +} + +function exportAdmissionBytes(claim: ExportAdmissionClaim): string { + return `${canonicalJson(claim)}\n`; +} + +function readExportAdmission( + root: string, + environment: Readonly>, +): ExportAdmissionSnapshot | null { + const content = readPrivateStateFileIfPresent( + join(root, EXPORT_ADMISSION_FILE), + MAX_CLAIM_BYTES, + "Beeper Message Like Me export admission", + environment, + ); + if (content === null) return null; + let decoded: unknown; + try { + decoded = JSON.parse(content) as unknown; + } catch { + return fail("export admission is not valid JSON"); + } + const claim = parseExportAdmission(decoded); + if (content !== exportAdmissionBytes(claim)) { + return fail("export admission is not canonical"); + } + return Object.freeze({ claim, contentSha256: sha256(content) }); +} + +function stateMutationCreateContention(error: unknown): boolean { + return error instanceof Error + && error.message.includes("state file mutation is already active"); +} + +function acquiredExportAdmission( + root: string, + environment: Readonly>, + snapshot: ExportAdmissionSnapshot, +): BeeperMessageLikeMeExportAdmission { + return { + claimPath: join(root, EXPORT_ADMISSION_FILE), + environment, + claimId: snapshot.claim.id, + contentSha256: snapshot.contentSha256, + released: false, + }; +} + +export function acquireBeeperMessageLikeMeExportAdmission(options: Readonly<{ + environment?: Readonly>; + /** Test-only process-liveness seam. */ + inspectOwnerForTest?: (owner: ProcessOwnerIdentity) => ProcessOwnerStatus; +}> = {}): BeeperMessageLikeMeExportAdmission { + if ( + options.inspectOwnerForTest !== undefined + && process.env.NODE_ENV !== "test" + ) return fail("export admission liveness injection is available only in tests"); + const environment = options.environment ?? process.env; + const inspectOwner = options.inspectOwnerForTest ?? processOwnerStatus; + const root = exportAdmissionRoot(environment); + try { + ensurePrivateDirectory(root); + for ( + let attempt = 0; + attempt < MAX_EXPORT_ADMISSION_ACQUIRE_ATTEMPTS; + attempt += 1 + ) { + const observed = readExportAdmission(root, environment); + if (observed !== null) { + const status = inspectOwner(observed.claim.owner); + if (status === "exact-live-owner") { + return fail("another export is active"); + } + if (status === "unknown") { + return fail("prior export owner cannot be inspected safely"); + } + if (!removePrivateStateFileIfUnchanged( + join(root, EXPORT_ADMISSION_FILE), + { expectedCurrentContentSha256: observed.contentSha256 }, + environment, + )) continue; + } + + const processIdentity = currentProcessStartIdentity(); + const claim = Object.freeze({ + schemaVersion: 1 as const, + kind: EXPORT_ADMISSION_KIND, + id: randomUUID(), + owner: Object.freeze({ pid: process.pid, ...processIdentity }), + }); + const snapshot = Object.freeze({ + claim, + contentSha256: sha256(exportAdmissionBytes(claim)), + }); + let created = false; + try { + created = createPrivateJsonIfAbsent( + join(root, EXPORT_ADMISSION_FILE), + claim, + { environment }, + ).created; + } catch (error) { + const committed = readExportAdmission(root, environment); + if ( + committed !== null + && committed.contentSha256 === snapshot.contentSha256 + ) return acquiredExportAdmission(root, environment, snapshot); + if (!stateMutationCreateContention(error)) throw error; + } + if (created) return acquiredExportAdmission(root, environment, snapshot); + } + return fail("export admission could not be acquired after bounded contention"); + } catch (error) { + if ( + error instanceof Error + && error.message.startsWith("Beeper Message Like Me recovery:") + ) throw error; + return fail("export admission could not be acquired safely"); + } +} + +export function releaseBeeperMessageLikeMeExportAdmission( + admission: BeeperMessageLikeMeExportAdmission, +): void { + if (admission.released) return; + try { + if (!removePrivateStateFileIfUnchanged(admission.claimPath, { + expectedCurrentContentSha256: admission.contentSha256, + }, admission.environment)) return fail("export admission changed before release"); + admission.released = true; + } catch (error) { + if ( + error instanceof Error + && error.message.startsWith("Beeper Message Like Me recovery:") + ) throw error; + return fail("export admission could not be released safely"); + } +} + +async function physicalDirectoryIdentity( + path: string, + privateMode: boolean, +): Promise { + const canonical = await realpath(path); + const metadata = await lstat(path, { bigint: true }); + const uid = process.getuid?.(); + if ( + canonical !== path + || !metadata.isDirectory() + || metadata.isSymbolicLink() + || uid === undefined + || metadata.uid !== BigInt(uid) + || (privateMode + ? (metadata.mode & 0o777n) !== 0o700n + : (metadata.mode & 0o022n) !== 0n) + ) return fail("directory lease target is not an owned physical directory"); + return Object.freeze({ + device: metadata.dev.toString(), + inode: metadata.ino.toString(), + }); +} + +function sameIdentity( + left: PrivateDirectoryIdentity, + right: PrivateDirectoryIdentity, +): boolean { + return left.device === right.device && left.inode === right.inode; +} + +export async function createBeeperMessageLikeMeDirectoryLease(request: Readonly<{ + role: LeaseRole; + path: string; + outputRoot?: string; + recoverAfterMs: number; + environment?: Readonly>; + nowMs?: number; +}>): Promise { + const environment = request.environment ?? process.env; + const path = boundedPath(request.path, "directory lease path"); + const parentPath = dirname(path); + const outputRoot = request.outputRoot === undefined + ? null + : boundedPath(request.outputRoot, "directory lease output"); + const nowMs = milliseconds(request.nowMs ?? Date.now(), "directory lease creation time"); + if ( + !Number.isSafeInteger(request.recoverAfterMs) + || request.recoverAfterMs < nowMs + || (request.role === "bundle-stage") !== (outputRoot !== null) + || (outputRoot !== null && dirname(outputRoot) !== parentPath) + ) return fail("directory lease request is invalid"); + const processIdentity = currentProcessStartIdentity(); + const claim = Object.freeze({ + schemaVersion: 1 as const, + kind: CLAIM_KIND, + id: randomUUID(), + role: request.role, + path, + parentPath, + parentIdentity: await physicalDirectoryIdentity(parentPath, false), + directoryIdentity: await physicalDirectoryIdentity(path, true), + outputRoot, + owner: Object.freeze({ pid: process.pid, ...processIdentity }), + childOwner: null, + phase: "preparing" as const, + createdAtMs: nowMs, + recoverAfterMs: request.recoverAfterMs, + }); + const root = leaseRoot(environment); + try { + ensurePrivateDirectory(root); + const claimPath = join(root, `${claim.id}.json`); + if (!createPrivateJsonIfAbsent(claimPath, claim, { environment }).created) { + return fail("directory lease ID was already present"); + } + return { + claimPath, + environment, + claim, + contentSha256: claimSha256(claim), + released: false, + }; + } catch (error) { + if ( + error instanceof Error + && error.message.startsWith("Beeper Message Like Me recovery:") + ) throw error; + return fail("directory lease could not be created safely"); + } +} + +export function updateBeeperMessageLikeMeDirectoryLease( + lease: BeeperMessageLikeMeDirectoryLease, + phase: RawPhase, + childPid?: number, +): void { + if (lease.released || lease.claim.role !== "raw-working") { + return fail("only an active raw directory lease can change phase"); + } + const currentPhase = lease.claim.phase; + const transitionIsValid = + (phase === "launching" + && (currentPhase === "preparing" || currentPhase === "settled")) + || (phase === "running" && currentPhase === "launching") + || ( + phase === "settled" + && (currentPhase === "launching" || currentPhase === "running") + ); + if (!transitionIsValid) { + return fail("raw directory lease lifecycle transition is invalid"); + } + let childOwner: ProcessOwnerIdentity | null = null; + if (phase === "running") { + if ( + childPid === undefined + || !Number.isSafeInteger(childPid) + || childPid < 1 + || childPid > 2_147_483_647 + ) return fail("running directory lease requires a child process identity"); + childOwner = captureProcessOwnerIdentity(childPid); + } else if (childPid !== undefined) { + return fail("only a running directory lease accepts a child process identity"); + } + const next = Object.freeze({ ...lease.claim, phase, childOwner }); + try { + if (!writePrivateJsonIfUnchanged(lease.claimPath, next, { + expectedCurrentContentSha256: lease.contentSha256, + })) return fail("directory lease changed before its lifecycle update"); + lease.claim = next; + lease.contentSha256 = claimSha256(next); + } catch (error) { + if ( + error instanceof Error + && error.message.startsWith("Beeper Message Like Me recovery:") + ) throw error; + return fail("directory lease lifecycle could not be updated safely"); + } +} + +export function releaseBeeperMessageLikeMeDirectoryLease( + lease: BeeperMessageLikeMeDirectoryLease, +): void { + if (lease.released) return; + try { + if (!removePrivateStateFileIfUnchanged(lease.claimPath, { + expectedCurrentContentSha256: lease.contentSha256, + }, lease.environment)) return fail("directory lease changed before release"); + lease.released = true; + } catch (error) { + if ( + error instanceof Error + && error.message.startsWith("Beeper Message Like Me recovery:") + ) throw error; + return fail("directory lease could not be released safely"); + } +} + +async function identityIfPresent(path: string): Promise { + try { + return await physicalDirectoryIdentity(path, true); + } catch (error) { + if (isErrno(error, "ENOENT")) return null; + throw error; + } +} + +function rawLeaseRecoveryDisposition( + claim: DirectoryLeaseClaim, + nowMs: number, + inspectOwner: (owner: ProcessOwnerIdentity) => ProcessOwnerStatus, +): "active" | "indeterminate" | "recoverable" { + if (claim.phase === "preparing" || claim.phase === "settled") { + return "recoverable"; + } + if (claim.phase === "running" && claim.childOwner !== null) { + const childStatus = inspectOwner(claim.childOwner); + return childStatus === "exact-live-owner" + ? "active" + : childStatus === "unknown" + ? "indeterminate" + : "recoverable"; + } + return nowMs >= claim.recoverAfterMs ? "recoverable" : "indeterminate"; +} + +export async function recoverBeeperMessageLikeMeDirectoryLeases(options: Readonly<{ + environment?: Readonly>; + nowMs?: number; + inspectOwner?: (owner: ProcessOwnerIdentity) => ProcessOwnerStatus; +}> = {}): Promise { + const environment = options.environment ?? process.env; + const nowMs = milliseconds(options.nowMs ?? Date.now(), "directory recovery time"); + const inspectOwner = options.inspectOwner ?? processOwnerStatus; + const root = leaseRoot(environment); + try { + ensurePrivateDirectory(root); + const snapshot = snapshotPrivateStateDirectory( + root, + environment, + undefined, + { recoverOrphanedMutationClaims: true }, + ); + if (snapshot.identity === null || snapshot.entries.length > MAX_CLAIMS) { + return fail("directory lease collection exceeded its reviewed bound"); + } + const names = snapshot.entries.map((entry) => { + if ( + entry.kind !== "file" + || !UUID_PATTERN.test(entry.name.replace(/\.json$/u, "")) + || !entry.name.endsWith(".json") + ) return fail("directory lease collection contains an unsupported entry"); + return entry.name; + }).sort(); + const files = readPrivateStateFilesBatch(root, names, { + maximumBytesPerFile: MAX_CLAIM_BYTES, + maximumTotalBytes: MAX_CLAIM_BYTES * Math.max(1, names.length), + environment, + expectedDirectoryIdentity: snapshot.identity, + }); + let recovered = 0; + let published = 0; + let active = 0; + let indeterminate = 0; + for (const file of files) { + if (file.status !== "present") { + return fail("directory lease changed during recovery inspection"); + } + let decoded: unknown; + try { + decoded = JSON.parse(file.content) as unknown; + } catch { + return fail("directory lease is not valid JSON"); + } + const claim = parseClaim(decoded); + if ( + file.name !== `${claim.id}.json` + || file.content !== claimBytes(claim) + ) return fail("directory lease is not canonical"); + const claimPath = join(root, file.name); + const ownerStatus = inspectOwner(claim.owner); + if (ownerStatus === "exact-live-owner") { + active += 1; + continue; + } + if (ownerStatus === "unknown") { + indeterminate += 1; + continue; + } + if (claim.role === "raw-working") { + const disposition = rawLeaseRecoveryDisposition( + claim, + nowMs, + inspectOwner, + ); + if (disposition === "active") { + active += 1; + continue; + } + if (disposition === "indeterminate") { + indeterminate += 1; + continue; + } + } + const parentIdentity = await physicalDirectoryIdentity(claim.parentPath, false); + if (!sameIdentity(parentIdentity, claim.parentIdentity)) { + return fail("directory lease parent changed before recovery"); + } + const current = await identityIfPresent(claim.path); + if (current !== null && !sameIdentity(current, claim.directoryIdentity)) { + return fail("directory lease target changed before recovery"); + } + if (current !== null) { + removePrivateDirectoryTree(claim.path, claim.directoryIdentity); + recovered += 1; + } else { + // Completes an interrupted identity-bound helper quarantine when present. + if (removePrivateDirectoryTree(claim.path, claim.directoryIdentity)) { + recovered += 1; + } else if (claim.outputRoot !== null) { + const outputIdentity = await identityIfPresent(claim.outputRoot); + if ( + outputIdentity !== null + && !sameIdentity(outputIdentity, claim.directoryIdentity) + ) return fail("directory lease output changed before recovery"); + if (outputIdentity !== null) published += 1; + } + } + if (!removePrivateStateFileIfUnchanged(claimPath, { + expectedCurrentContentSha256: sha256(file.content), + }, environment)) return fail("directory lease changed before recovery release"); + } + return Object.freeze({ recovered, published, active, indeterminate }); + } catch (error) { + if ( + error instanceof Error + && error.message.startsWith("Beeper Message Like Me recovery:") + ) throw error; + return fail("directory lease recovery failed safely"); + } +} diff --git a/src/beeper-message-like-me-source.test.ts b/src/beeper-message-like-me-source.test.ts index f48e12f..67dbd03 100644 --- a/src/beeper-message-like-me-source.test.ts +++ b/src/beeper-message-like-me-source.test.ts @@ -2,9 +2,12 @@ import { createHash } from "node:crypto"; import { chmodSync, existsSync, + linkSync, + lstatSync, mkdirSync, mkdtempSync, readFileSync, + readdirSync, realpathSync, rmSync, symlinkSync, @@ -19,9 +22,18 @@ import { describe, expect, test } from "bun:test"; import type { WrenchAuth } from "./auth"; import { exportBeeperMessageLikeMeBundle } from "./beeper-message-like-me-export"; import { + assertUniqueOfficialAccountSelector, createBeeperMessageLikeMeSource, + enforceBeeperRawWorkingBudget, + runExportCli, type BeeperExportCliInvocation, + type BeeperMessageLikeMeProgress, } from "./beeper-message-like-me-source"; +import { + createBeeperMessageLikeMeDirectoryLease, + releaseBeeperMessageLikeMeDirectoryLease, +} from "./beeper-message-like-me-recovery"; +import { parseBeeperExportAccounts } from "./providers/beeper-local-runtime"; const ACCOUNT_ID = "account-beeper"; const NETWORK_ACCOUNT_ID = "account-whatsapp"; @@ -51,12 +63,16 @@ function configStore(parent: string): string { writeFileSync( join(path, "targets", "desktop.json"), `${JSON.stringify({ - auth: { token: "fixture" }, + auth: { + accessToken: "fixture-stored-access-token", + source: "manual", + tokenType: "Bearer", + }, baseURL: "http://127.0.0.1:23380", id: "desktop", - managed: true, + managed: false, name: "Desktop", - runtime: "desktop", + runtime: { install: "desktop", port: 23_373 }, type: "desktop", })}\n`, { mode: 0o600 }, @@ -175,56 +191,131 @@ function writeJson(path: string, value: unknown): void { writeFileSync(path, `${JSON.stringify(value)}\n`, { mode: 0o644 }); } -function fixtureExport(invocation: BeeperExportCliInvocation): Promise<{ +function fixtureSelectedAccount(invocation: BeeperExportCliInvocation): string | null { + if (invocation.arguments[0] !== "export") return null; + const configDirectory = invocation.environment.BEEPER_CLI_CONFIG_DIR; + if (configDirectory === undefined) { + throw new Error("fixture export omitted BEEPER_CLI_CONFIG_DIR"); + } + const config = JSON.parse( + readFileSync(join(configDirectory, "config.json"), "utf8"), + ) as Record; + return typeof config.defaultAccount === "string" ? config.defaultAccount : null; +} + +function fixtureCli(invocation: BeeperExportCliInvocation): Promise<{ exitCode: number; stdout: string; stderr: string; }> { + for (const value of [ + ...invocation.arguments, + ...Object.values(invocation.environment), + ...(invocation.workingRoot === undefined ? [] : [invocation.workingRoot]), + ]) { + expect(value).not.toContain(ACCOUNT_ID); + expect(value).not.toContain(NETWORK_ACCOUNT_ID); + } + expect(invocation.environment.BEEPER_READONLY).toBe("1"); + const configDirectory = invocation.environment.BEEPER_CLI_CONFIG_DIR; + if (configDirectory === undefined) { + throw new Error("fixture invocation omitted BEEPER_CLI_CONFIG_DIR"); + } + const privateConfig = JSON.parse( + readFileSync(join(configDirectory, "config.json"), "utf8"), + ) as Record; + const privateTarget = JSON.parse( + readFileSync(join(configDirectory, "targets", "desktop.json"), "utf8"), + ) as Record; + expect(Object.keys(privateTarget).sort()).toEqual([ + "auth", + "baseURL", + "id", + "managed", + "type", + ]); + expect(privateTarget).toMatchObject({ + baseURL: "http://127.0.0.1:23380", + id: "desktop", + managed: false, + type: "desktop", + }); + expect((privateTarget.auth as Record).tokenType).toBe("Bearer"); + expect(typeof (privateTarget.auth as Record).accessToken).toBe("string"); + expect(privateConfig.auth).toBeUndefined(); + + if ( + invocation.arguments[0] === "accounts" + && invocation.arguments[1] === "list" + ) { + expect(invocation.arguments).toContain("--json"); + expect(invocation.arguments.indexOf("--read-only")).toBeGreaterThan(0); + return Promise.resolve({ + exitCode: 0, + stdout: `${JSON.stringify({ success: true, data: accounts(), error: null })}\n`, + stderr: "", + }); + } + expect(invocation.arguments[0]).toBe("export"); + expect(invocation.maxWorkingBytes).toBe(4 * 1024 * 1024 * 1024); + expect(invocation.workingRoot).toBeDefined(); expect(invocation.arguments).toContain("--no-attachments"); expect(invocation.arguments).not.toContain("--json"); + expect(invocation.arguments).not.toContain("--account"); + expect(invocation.arguments).not.toContain("--events"); expect(invocation.arguments.indexOf("--read-only")).toBeGreaterThan(0); - expect(invocation.environment.BEEPER_READONLY).toBe("1"); + const selectedAccount = fixtureSelectedAccount(invocation); + if (selectedAccount === null) throw new Error("fixture export omitted defaultAccount"); + expect([ACCOUNT_ID, NETWORK_ACCOUNT_ID]).toContain(selectedAccount); const outputIndex = invocation.arguments.indexOf("--out"); const outputRoot = invocation.arguments[outputIndex + 1]; if (outputRoot === undefined) throw new Error("fixture export omitted --out"); const chatsRoot = join(outputRoot, "chats"); - const chatRoot = join(chatsRoot, CHAT_ID); - mkdirSync(join(chatRoot, "attachments"), { recursive: true, mode: 0o755 }); + mkdirSync(chatsRoot, { recursive: true, mode: 0o755 }); const accountValues = accounts(); - const chatValues = [chat()]; - const messageValues = messages(); + const includesChat = selectedAccount === NETWORK_ACCOUNT_ID; + const chatValues = includesChat ? [chat()] : []; + const messageValues = includesChat ? messages() : []; writeJson(join(outputRoot, "accounts.json"), accountValues); writeJson(join(outputRoot, "chats.json"), chatValues); - writeJson(join(chatRoot, "chat.json"), chat()); - writeJson(join(chatRoot, "messages.json"), messageValues); - writeFileSync( - join(chatRoot, "messages.markdown"), - "private duplicate markdown\n", - { mode: 0o644 }, - ); - writeFileSync( - join(chatRoot, "messages.html"), - "

private duplicate html

\n", - { mode: 0o644 }, - ); - const createdAt = "2026-08-21T13:59:00.000Z"; - const completedAt = "2026-08-21T14:01:00.000Z"; - writeJson(join(outputRoot, ".beeper-export-state.json"), { - chats: { - [CHAT_ID]: { - attachmentCount: 0, - complete: true, - cursor: null, - messageCount: messageValues.length, - startedAt: createdAt, - updatedAt: completedAt, + const createdAt = includesChat + ? "2026-08-21T13:59:00.000Z" + : "2026-08-21T14:01:00.000Z"; + const completedAt = includesChat + ? "2026-08-21T14:01:00.000Z" + : "2026-08-21T14:01:30.000Z"; + if (includesChat) { + const chatRoot = join(chatsRoot, CHAT_ID); + mkdirSync(join(chatRoot, "attachments"), { recursive: true, mode: 0o755 }); + writeJson(join(chatRoot, "chat.json"), chat()); + writeJson(join(chatRoot, "messages.json"), messageValues); + writeFileSync( + join(chatRoot, "messages.markdown"), + "private duplicate markdown\n", + { mode: 0o644 }, + ); + writeFileSync( + join(chatRoot, "messages.html"), + "

private duplicate html

\n", + { mode: 0o644 }, + ); + writeJson(join(outputRoot, ".beeper-export-state.json"), { + chats: { + [CHAT_ID]: { + attachmentCount: 0, + complete: true, + cursor: null, + messageCount: messageValues.length, + startedAt: createdAt, + updatedAt: completedAt, + }, }, - }, - completedChatIDs: [CHAT_ID], - createdAt, - exportVersion: 1, - }); + completedChatIDs: [CHAT_ID], + createdAt, + exportVersion: 1, + }); + } writeJson(join(outputRoot, "manifest.json"), { accounts: accountValues, attachmentCount: 0, @@ -236,7 +327,9 @@ function fixtureExport(invocation: BeeperExportCliInvocation): Promise<{ }); return Promise.resolve({ exitCode: 0, - stdout: "Exported 1 chats, 2 messages, 0 attachments\n", + stdout: includesChat + ? "Exported 1 chats, 2 messages, 0 attachments\n" + : "Exported 0 chats, 0 messages, 0 attachments\n", stderr: "", }); } @@ -248,29 +341,512 @@ function invocationOutputRoot(invocation: BeeperExportCliInvocation): string { return outputRoot; } +function whatsAppOutputRoot(invocation: BeeperExportCliInvocation): string | null { + return fixtureSelectedAccount(invocation) === NETWORK_ACCOUNT_ID + ? invocationOutputRoot(invocation) + : null; +} + function ndjson(path: string): readonly Record[] { return readFileSync(path, "utf8").trim().split("\n").filter(Boolean) .map((line) => JSON.parse(line) as Record); } +const SELF_ALIAS_ID = "whatsapp:private-late-self-alias"; + +function aliasChatOrderKey(chatId: string): string { + return createHash("sha256") + .update(JSON.stringify([NETWORK_ACCOUNT_ID, chatId])) + .digest("hex"); +} + +function orderedSelfAliasChatIds(): readonly [string, string] { + const ordered = ["alias-chat-alpha", "alias-chat-omega"] + .sort((left, right) => aliasChatOrderKey(left).localeCompare(aliasChatOrderKey(right))); + const early = ordered[0]; + const late = ordered[1]; + if (early === undefined || late === undefined) { + throw new Error("self-alias fixture chat order disappeared"); + } + return [early, late]; +} + +function writeSelfAliasFixture( + outputRoot: string, + conflict: "incoming" | "participant" | null, + extraLateSelfAliases = 0, +): void { + const [earlyChatId, lateChatId] = orderedSelfAliasChatIds(); + const baseChat = chat(); + const peer = baseChat.participants.items[0]!; + const selfAlias = { + fullName: "Private Late Self Alias", + id: SELF_ALIAS_ID, + isSelf: true, + }; + const earlyParticipants = conflict === "participant" + ? [peer, { ...selfAlias, isSelf: false }] + : [peer]; + const earlyChat = { + ...baseChat, + id: earlyChatId, + lastActivity: "2026-08-21T14:00:01.000Z", + participants: { + hasMore: false, + items: earlyParticipants, + total: earlyParticipants.length, + }, + title: "Early Alias Fixture", + }; + const lateChat = { + ...baseChat, + id: lateChatId, + lastActivity: "2026-08-21T14:00:02.000Z", + participants: { + hasMore: false, + items: [ + peer, + selfAlias, + ...Array.from({ length: extraLateSelfAliases }, (_, index) => ({ + fullName: `Private Extra Self Alias ${String(index)}`, + id: `whatsapp:private-extra-self-alias-${String(index)}`, + isSelf: true, + })), + ], + total: 2 + extraLateSelfAliases, + }, + title: "Late Alias Evidence Fixture", + }; + const baseMessage = messages()[0] as Record; + const earlyMessage = { + ...baseMessage, + attachments: [], + chatID: earlyChatId, + editedTimestamp: null, + id: "message-alias-early", + isSender: conflict !== "incoming", + linkedMessageID: null, + reactions: [{ + emoji: true, + id: "reaction-alias-self", + participantID: SELF_ALIAS_ID, + reactionKey: "👍", + }, { + emoji: true, + id: "reaction-alias-self", + participantID: "whatsapp:self", + reactionKey: "👍", + }], + senderID: conflict === "incoming" ? SELF_ALIAS_ID : "whatsapp:self", + senderName: conflict === "incoming" ? "Private Late Self Alias" : "Fixture Self", + sortKey: "00000000000000000001", + text: "synthetic early alias body", + timestamp: "2026-08-21T14:00:01.000Z", + }; + const lateMessage = { + ...baseMessage, + attachments: [], + chatID: lateChatId, + editedTimestamp: null, + id: "message-alias-late", + linkedMessageID: null, + reactions: [], + senderID: SELF_ALIAS_ID, + senderName: "Private Late Self Alias", + sortKey: "00000000000000000002", + text: "synthetic late alias body", + timestamp: "2026-08-21T14:00:02.000Z", + }; + const chatsRoot = join(outputRoot, "chats"); + rmSync(chatsRoot, { recursive: true, force: true }); + mkdirSync(chatsRoot, { mode: 0o755 }); + for (const [chatValue, messageValue] of [ + [earlyChat, earlyMessage], + [lateChat, lateMessage], + ] as const) { + const chatRoot = join(chatsRoot, chatValue.id); + mkdirSync(join(chatRoot, "attachments"), { recursive: true, mode: 0o755 }); + writeJson(join(chatRoot, "chat.json"), chatValue); + writeJson(join(chatRoot, "messages.json"), [messageValue]); + writeFileSync(join(chatRoot, "messages.markdown"), "private duplicate markdown\n"); + writeFileSync(join(chatRoot, "messages.html"), "

private duplicate html

\n"); + } + writeJson(join(outputRoot, "chats.json"), [earlyChat, lateChat]); + writeJson(join(outputRoot, ".beeper-export-state.json"), { + chats: { + [earlyChatId]: { + attachmentCount: 0, + complete: true, + cursor: null, + messageCount: 1, + startedAt: "2026-08-21T13:59:00.000Z", + updatedAt: "2026-08-21T14:00:01.000Z", + }, + [lateChatId]: { + attachmentCount: 0, + complete: true, + cursor: null, + messageCount: 1, + startedAt: "2026-08-21T14:00:01.000Z", + updatedAt: "2026-08-21T14:00:02.000Z", + }, + }, + completedChatIDs: [earlyChatId, lateChatId], + createdAt: "2026-08-21T13:59:00.000Z", + exportVersion: 1, + }); + writeJson(join(outputRoot, "manifest.json"), { + accounts: accounts(), + attachmentCount: 0, + chatCount: 2, + completedAt: "2026-08-21T14:00:02.000Z", + createdAt: "2026-08-21T13:59:00.000Z", + messageCount: 2, + version: 1, + }); +} + describe("Beeper Message Like Me source", () => { + test("settles a durable raw lease after a real immediately exiting child", async () => { + const parent = privateDirectory("wrench-beeper-fast-child-test."); + const working = join(parent, "working"); + mkdirSync(working, { mode: 0o700 }); + const environment = { WRENCH_STATE_HOME: join(parent, "state") }; + const nowMs = Date.now(); + const lease = await createBeeperMessageLikeMeDirectoryLease({ + role: "raw-working", + path: working, + recoverAfterMs: nowMs + 60_000, + environment, + nowMs, + }); + try { + const result = await runExportCli({ + binary: "/usr/bin/true", + arguments: [], + environment: { PATH: "/usr/bin:/bin" }, + timeoutMs: 5_000, + maxOutputBytes: 1_024, + maxStderrBytes: 1_024, + directoryLease: lease, + }); + + expect(result).toEqual({ exitCode: 0, stdout: "", stderr: "" }); + expect(lease.claim.phase).toBe("settled"); + expect(lease.claim.childOwner).toBeNull(); + } finally { + releaseBeeperMessageLikeMeDirectoryLease(lease); + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("matches the exact pinned CLI account selector fields", () => { + const selectedId = "account.selected-private"; + const rows = [{ + accountID: selectedId, + bridge: { id: "selected", provider: "cloud", type: "signal" }, + network: "Signal", + status: "CONNECTED", + user: { fullName: "Selected Person", id: "selected:self", isSelf: true }, + }, { + accountID: "account-other", + bridge: { id: "other", provider: "cloud", type: "whatsapp" }, + network: "WhatsApp", + status: "CONNECTED", + user: { fullName: "Other Person", id: "other:self", isSelf: true }, + }] as const; + for (const field of ["displayName", "name"] as const) { + const colliding = rows.map((row, index) => index === 1 + ? { ...row, user: { ...row.user, [field]: " account selected_private " } } + : row); + const parsed = parseBeeperExportAccounts(colliding); + const selected = parsed.find((account) => account.accountId === selectedId); + expect(selected).toBeDefined(); + try { + assertUniqueOfficialAccountSelector(selected!, parsed); + throw new Error("selector collision was not rejected"); + } catch (error) { + expect(String(error)).toContain("ambiguous under the pinned CLI selector rules"); + expect(String(error)).not.toContain(selectedId); + expect(String(error)).not.toContain("account-other"); + } + } + const fullNameOnly = rows.map((row, index) => index === 1 + ? { ...row, user: { ...row.user, fullName: " account selected_private " } } + : row); + const parsed = parseBeeperExportAccounts(fullNameOnly); + const selected = parsed.find((account) => account.accountId === selectedId); + expect(selected).toBeDefined(); + expect(() => assertUniqueOfficialAccountSelector(selected!, parsed)).not.toThrow(); + }); + + test("enforces the operation-wide raw staging byte and entry policy", async () => { + const parent = privateDirectory("wrench-beeper-source-raw-budget-test."); + try { + writeFileSync(join(parent, "raw.json"), `${"x".repeat(64)}\n`, { mode: 0o600 }); + const measured = await enforceBeeperRawWorkingBudget( + parent, + 1024 * 1024, + 0, + ); + expect(measured.entries).toBe(1); + expect(measured.bytes).toBeGreaterThanOrEqual(65); + await expect(enforceBeeperRawWorkingBudget(parent, 32, 0)) + .rejects.toThrow("raw export staging exceeded its byte budget"); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("accepts and charges the official relative CLI payload-cache link shape", async () => { + const root = privateDirectory("wrench-beeper-source-cache-links-test."); + const cacheRoot = join(root, "cli-payload-cache"); + const filesRoot = join(cacheRoot, "files"); + const linksRoot = join(cacheRoot, "links"); + const entries = [cacheRoot, filesRoot, linksRoot]; + const linkEntries: string[] = []; + try { + mkdirSync(filesRoot, { recursive: true, mode: 0o700 }); + mkdirSync(linksRoot, { mode: 0o700 }); + for (let index = 0; index < 56; index += 1) { + const segment = String(index).padStart(2, "0"); + const target = join(filesRoot, `payload-${segment}.bin`); + const link = join(linksRoot, `payload-${segment}`); + writeFileSync(target, `official-cache-payload-${segment}\n`, { mode: 0o600 }); + symlinkSync(`../files/payload-${segment}.bin`, link); + entries.push(target, link); + linkEntries.push(link); + } + + const measured = await enforceBeeperRawWorkingBudget( + root, + 1024 * 1024, + 0, + ); + const expectedBytes = entries.reduce((total, path) => { + const metadata = lstatSync(path); + return total + Math.max(metadata.size, metadata.blocks * 512); + }, 0); + + expect(measured.entries).toBe(entries.length); + expect(measured.bytes).toBe(expectedBytes); + const linkBytes = linkEntries.reduce((total, path) => { + const metadata = lstatSync(path); + return total + Math.max(metadata.size, metadata.blocks * 512); + }, 0); + const nonLinkBytes = entries + .filter((path) => !linkEntries.includes(path)) + .reduce((total, path) => { + const metadata = lstatSync(path); + return total + Math.max(metadata.size, metadata.blocks * 512); + }, 0); + expect(measured.bytes - nonLinkBytes).toBe(linkBytes); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("rejects every unsafe CLI payload-cache symbolic-link shape", async () => { + const cases: readonly { + readonly name: string; + readonly prepare: (fixture: Readonly<{ + cacheRoot: string; + parent: string; + working: string; + }>) => void; + readonly error: string; + }[] = [{ + name: "outside-cache", + prepare: ({ cacheRoot, working }) => { + writeFileSync(join(cacheRoot, "payload.bin"), "payload\n", { mode: 0o600 }); + symlinkSync("cli-payload-cache/payload.bin", join(working, "outside-link")); + }, + error: "symbolic link outside its CLI payload cache", + }, { + name: "absolute-target", + prepare: ({ cacheRoot }) => { + const target = join(cacheRoot, "zz-payload.bin"); + writeFileSync(target, "payload\n", { mode: 0o600 }); + symlinkSync(target, join(cacheRoot, "aa-link")); + }, + error: "unsafe CLI payload-cache symbolic link", + }, { + name: "escaping-target", + prepare: ({ cacheRoot, parent }) => { + writeFileSync(join(parent, "outside.bin"), "payload\n", { mode: 0o600 }); + symlinkSync("../../outside.bin", join(cacheRoot, "aa-link")); + }, + error: "unsafe CLI payload-cache symbolic link", + }, { + name: "dangling-target", + prepare: ({ cacheRoot }) => { + symlinkSync("missing.bin", join(cacheRoot, "aa-link")); + }, + error: "unsafe CLI payload-cache symbolic link", + }, { + name: "control-target", + prepare: ({ cacheRoot }) => { + symlinkSync("payload\n.bin", join(cacheRoot, "aa-link")); + }, + error: "unsafe CLI payload-cache symbolic link", + }, { + name: "control-name", + prepare: ({ cacheRoot }) => { + writeFileSync(join(cacheRoot, "zz-payload.bin"), "payload\n", { mode: 0o600 }); + symlinkSync("zz-payload.bin", join(cacheRoot, "aa-link\n")); + }, + error: "unsafe CLI payload-cache symbolic link", + }, { + name: "overlong-target", + prepare: ({ cacheRoot }) => { + symlinkSync("a".repeat(513), join(cacheRoot, "aa-link")); + }, + error: "unsafe CLI payload-cache symbolic link", + }, { + name: "chained-target", + prepare: ({ cacheRoot }) => { + writeFileSync(join(cacheRoot, "zz-payload.bin"), "payload\n", { mode: 0o600 }); + symlinkSync("zz-payload.bin", join(cacheRoot, "bb-inner-link")); + symlinkSync("bb-inner-link", join(cacheRoot, "aa-outer-link")); + }, + error: "unsafe CLI payload-cache symbolic link", + }, { + name: "directory-target", + prepare: ({ cacheRoot }) => { + mkdirSync(join(cacheRoot, "zz-directory"), { mode: 0o700 }); + symlinkSync("zz-directory", join(cacheRoot, "aa-link")); + }, + error: "unsafe CLI payload-cache symbolic link", + }, { + name: "writable-target", + prepare: ({ cacheRoot }) => { + const target = join(cacheRoot, "zz-payload.bin"); + writeFileSync(target, "payload\n", { mode: 0o600 }); + chmodSync(target, 0o620); + symlinkSync("zz-payload.bin", join(cacheRoot, "aa-link")); + }, + error: "unsafe CLI payload-cache symbolic link", + }, { + name: "multiply-linked-target", + prepare: ({ cacheRoot, parent }) => { + const target = join(cacheRoot, "zz-payload.bin"); + writeFileSync(target, "payload\n", { mode: 0o600 }); + linkSync(target, join(parent, "second-hard-link.bin")); + symlinkSync("zz-payload.bin", join(cacheRoot, "aa-link")); + }, + error: "unsafe CLI payload-cache symbolic link", + }, { + name: "symlink-directory-escape", + prepare: ({ cacheRoot, parent }) => { + const outside = join(parent, "outside-directory"); + mkdirSync(outside, { mode: 0o700 }); + writeFileSync(join(outside, "payload.bin"), "payload\n", { mode: 0o600 }); + symlinkSync("../../outside-directory", join(cacheRoot, "bb-alias")); + symlinkSync("bb-alias/payload.bin", join(cacheRoot, "aa-outer-link")); + }, + error: "unsafe CLI payload-cache symbolic link", + }]; + + for (const item of cases) { + const parent = privateDirectory( + `wrench-beeper-source-cache-link-${item.name}-test.`, + ); + const working = join(parent, "working"); + const cacheRoot = join(working, "cli-payload-cache"); + mkdirSync(cacheRoot, { recursive: true, mode: 0o700 }); + try { + item.prepare({ cacheRoot, parent, working }); + await expect(enforceBeeperRawWorkingBudget( + working, + 1024 * 1024, + 0, + )).rejects.toThrow(item.error); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + } + }); + + test("preserves a categorical initial raw-staging failure without paths", async () => { + const parent = privateDirectory("wrench-beeper-source-preflight-error-test."); + const working = join(parent, "working"); + const cacheRoot = join(working, "cli-payload-cache"); + mkdirSync(cacheRoot, { recursive: true, mode: 0o700 }); + writeFileSync(join(cacheRoot, "payload.bin"), "private-payload\n", { + mode: 0o600, + }); + symlinkSync( + "cli-payload-cache/payload.bin", + join(working, "outside-link"), + ); + try { + const error = await runExportCli({ + binary: "/usr/bin/true", + arguments: [], + environment: { PATH: "/usr/bin:/bin" }, + timeoutMs: 5_000, + maxOutputBytes: 1_024, + maxStderrBytes: 1_024, + workingRoot: working, + maxWorkingBytes: 1024 * 1024, + }).then( + () => null, + (reason: unknown) => reason, + ); + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain( + "raw export staging contained a symbolic link outside its CLI payload cache", + ); + expect((error as Error).message).not.toContain(parent); + expect((error as Error).message).not.toContain("private-payload"); + expect((error as Error).message).not.toContain( + "official export raw staging safety check failed", + ); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + test("converts one private official export without media or duplicate renderings", async () => { const parent = privateDirectory("wrench-beeper-source-test."); const working = join(parent, "working"); const output = join(parent, "message-like-me"); mkdirSync(working, { mode: 0o700 }); let removed = false; + const invocations: BeeperExportCliInvocation[] = []; + const progress: BeeperMessageLikeMeProgress[] = []; try { const source = createBeeperMessageLikeMeSource({ auth: auth(configStore(parent)), + onProgress: (item) => progress.push(item), dependencies: { binaryPath: "/fixture/beeper-0.6.2", createWorkingDirectory: async () => working, removeWorkingDirectory: async (path) => { + expect(existsSync(join( + working, + "account-shards", + "account-001", + "manifest.json", + ))).toBeTrue(); + expect(readdirSync(output).sort()).toEqual([ + "accounts.ndjson", + "conversations.ndjson", + "manifest.json", + "messages.ndjson", + "participants.ndjson", + "reactions.ndjson", + "tombstones.ndjson", + ]); rmSync(path, { recursive: true, force: true }); removed = true; }, - runExport: fixtureExport, + runCli: async (invocation) => { + invocations.push(invocation); + invocation.onHeartbeat?.(30); + return fixtureCli(invocation); + }, }, }); const result = await exportBeeperMessageLikeMeBundle({ @@ -287,9 +863,101 @@ describe("Beeper Message Like Me source", () => { expect(removed).toBeTrue(); expect(existsSync(working)).toBeFalse(); + expect(invocations.map((invocation) => + invocation.arguments.slice(0, 2).join(" "))).toEqual([ + "accounts list", + "export --out", + "export --out", + "accounts list", + ]); + const exportInvocations = invocations.filter((invocation) => + invocation.arguments[0] === "export"); + expect(exportInvocations.map((invocation) => { + const index = invocation.arguments.indexOf("--limit-chats"); + return invocation.arguments[index + 1]; + })).toEqual(["100000", "99999"]); + expect(exportInvocations.map((invocation) => { + const index = invocation.arguments.indexOf("--limit-messages"); + return invocation.arguments[index + 1]; + })).toEqual(["1000000", "1000000"]); + expect(invocations.filter((invocation) => + invocation.arguments[0] === "export").map((invocation) => + invocation.environment.BEEPER_CLI_CONFIG_DIR)).toEqual([ + join(working, "account-selectors", "account-001"), + join(working, "account-selectors", "account-002"), + ]); + expect(invocations.filter((invocation) => + invocation.arguments[0] === "accounts").map((invocation) => + invocation.environment.BEEPER_CLI_CONFIG_DIR)).toEqual([ + join(working, "account-selectors", "inventory"), + join(working, "account-selectors", "inventory"), + ]); + expect(JSON.stringify(progress)).not.toContain(ACCOUNT_ID); + expect(JSON.stringify(progress)).not.toContain(NETWORK_ACCOUNT_ID); + expect(progress).toEqual([ + { phase: "preparing" }, + { + phase: "accounts-progress", + stage: "discovering", + elapsedSeconds: 30, + }, + { phase: "accounts-discovered", accounts: 2 }, + { phase: "account-started", account: 1, accounts: 2 }, + { + phase: "account-progress", + account: 1, + accounts: 2, + elapsedSeconds: 30, + }, + { + phase: "account-validating", + account: 1, + accounts: 2, + elapsedSeconds: 0, + }, + { + phase: "account-completed", + account: 1, + accounts: 2, + chats: 1, + messages: 2, + }, + { phase: "account-started", account: 2, accounts: 2 }, + { + phase: "account-progress", + account: 2, + accounts: 2, + elapsedSeconds: 30, + }, + { + phase: "account-validating", + account: 2, + accounts: 2, + elapsedSeconds: 0, + }, + { + phase: "account-completed", + account: 2, + accounts: 2, + chats: 1, + messages: 2, + }, + { phase: "accounts-verifying", accounts: 2 }, + { + phase: "accounts-progress", + stage: "verifying", + elapsedSeconds: 30, + }, + { + phase: "conversion-started", + accounts: 2, + chats: 1, + messages: 2, + }, + ]); expect(result.manifest.completeness).toEqual({ kind: "bounded-local", - reason: "desktop-local-export", + reason: "desktop-local-sequential-export", observedFrom: "2026-08-21T14:00:01.000Z", observedThrough: "2026-08-21T14:00:03.000Z", }); @@ -301,10 +969,15 @@ describe("Beeper Message Like Me source", () => { reaction: 2, tombstone: 1, }); + expect(result.manifest.source).toEqual({ + id: "beeper-local", + version: "1.1.0", + }); expect(result.manifest.warnings).toEqual([ "attachments-metadata-only", "connected-account-backfill-coverage-unknown", "remote-history-not-claimed", + "sequential-account-snapshot", ]); const accountRows = ndjson(join(output, "accounts.ndjson")); @@ -353,25 +1026,472 @@ describe("Beeper Message Like Me source", () => { } }); + test("normalizes a late cross-chat self alias before exact record and byte budgets", async () => { + const parents: string[] = []; + const exportAliasFixture = async (maxBundleBytes?: number) => { + const parent = privateDirectory("wrench-beeper-source-self-alias-test."); + parents.push(parent); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + maxBundleRecords: 11, + ...(maxBundleBytes === undefined ? {} : { maxBundleBytes }), + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + const outputRoot = whatsAppOutputRoot(invocation); + if (outputRoot !== null) writeSelfAliasFixture(outputRoot, null); + return result; + }, + }, + }); + const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); + return { output, result }; + }; + + try { + const baseline = await exportAliasFixture(); + const exactRecordBytes = baseline.result.manifest.artifacts.reduce( + (sum, artifact) => sum + artifact.bytes, + 0, + ); + const byteBounded = await exportAliasFixture(exactRecordBytes); + for (const { output, result } of [baseline, byteBounded]) { + expect(result.manifest.completeness.kind).toBe("bounded-local"); + expect(result.manifest.counts).toEqual({ + account: 2, + participant: 3, + conversation: 2, + message: 2, + reaction: 2, + tombstone: 0, + }); + expect(Object.values(result.manifest.counts).reduce((sum, value) => sum + value, 0)) + .toBe(11); + expect(result.manifest.warnings).not.toContain("bundle-record-limit-reached"); + expect(result.manifest.warnings).not.toContain("bundle-byte-limit-reached"); + expect(result.manifest.warnings).not.toContain("participant-roster-incomplete"); + expect(result.manifest.warnings).not.toContain("self-alias-evidence-limit-reached"); + expect(result.manifest.warnings).toContain("reaction-provider-id-non-unique"); + const accountRows = ndjson(join(output, "accounts.ndjson")); + const accountRow = accountRows.find(({ network }) => network === "whatsapp-personal"); + expect(accountRow).toBeDefined(); + const selfParticipantId = accountRow!.selfParticipantId; + const participantRows = ndjson(join(output, "participants.ndjson")); + expect(participantRows.filter((row) => + row.accountId === accountRow!.id && row.isSelf === true)).toEqual([ + expect.objectContaining({ id: selfParticipantId }), + ]); + const conversationRows = ndjson(join(output, "conversations.ndjson")); + expect(conversationRows).toHaveLength(2); + for (const conversation of conversationRows) { + expect(conversation.participantsComplete).toBeTrue(); + expect(conversation.participantIds).toContain(selfParticipantId); + expect(conversation.participantIds).toHaveLength(2); + } + const messageRows = ndjson(join(output, "messages.ndjson")); + expect(messageRows).toHaveLength(2); + expect(messageRows.every((message) => + message.direction === "outgoing" + && message.senderParticipantId === selfParticipantId)).toBeTrue(); + const reactionRows = ndjson(join(output, "reactions.ndjson")); + expect(reactionRows).toHaveLength(2); + expect(reactionRows.every((reaction) => + reaction.participantId === selfParticipantId)).toBeTrue(); + expect(new Set(reactionRows.map(({ id }) => id)).size).toBe(2); + expect(new Set(reactionRows.map((reaction) => + (reaction.provenance as Record).providerId)).size).toBe(2); + expect(JSON.stringify({ + accountRows, + conversationRows, + messageRows, + participantRows, + reactionRows, + })).not.toContain(SELF_ALIAS_ID); + } + expect(byteBounded.result.manifest.artifacts.reduce( + (sum, artifact) => sum + artifact.bytes, + 0, + )).toBe(exactRecordBytes); + } finally { + for (const parent of parents) rmSync(parent, { recursive: true, force: true }); + } + }); + + test("rejects incoming and roster peer evidence for a later self alias", async () => { + for (const conflict of ["incoming", "participant"] as const) { + const parent = privateDirectory( + `wrench-beeper-source-self-alias-${conflict}-conflict-test.`, + ); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + let removed = false; + try { + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + removed = true; + }, + maxBundleRecords: 11, + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + const outputRoot = whatsAppOutputRoot(invocation); + if (outputRoot !== null) writeSelfAliasFixture(outputRoot, conflict); + return result; + }, + }, + }); + await expect(exportBeeperMessageLikeMeBundle({ outputRoot: output, source })) + .rejects.toThrow("official export has peer evidence for an account self alias"); + expect(removed).toBeTrue(); + expect(existsSync(working)).toBeFalse(); + expect(existsSync(output)).toBeFalse(); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + } + }); + + test("publishes a truthful prefix when bounded self-alias evidence is exhausted", async () => { + const parent = privateDirectory("wrench-beeper-source-self-alias-limit-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + try { + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + maxBundleRecords: 11, + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + const outputRoot = whatsAppOutputRoot(invocation); + if (outputRoot !== null) writeSelfAliasFixture(outputRoot, null, 12); + return result; + }, + }, + }); + const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); + expect(result.manifest.completeness).toMatchObject({ + kind: "truncated", + reason: "self-alias-evidence-limit", + }); + expect(result.manifest.warnings).toContain("self-alias-evidence-limit-reached"); + expect(result.manifest.warnings).not.toContain("bundle-record-limit-reached"); + expect(result.manifest.counts.conversation).toBe(1); + expect(existsSync(join(output, "manifest.json"))).toBeTrue(); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("rejects an account user that contradicts its explicit self anchor", async () => { + const parent = privateDirectory("wrench-beeper-source-account-self-conflict-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + const conflictingAccounts = accounts().map((value) => { + const account = value as Record; + if (account.accountID !== NETWORK_ACCOUNT_ID) return account; + return { + ...account, + user: { + ...(account.user as Record), + isSelf: false, + }, + }; + }); + try { + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + if (invocation.arguments[0] === "accounts") { + return { + ...result, + stdout: `${JSON.stringify({ + success: true, + data: conflictingAccounts, + error: null, + })}\n`, + }; + } + const outputRoot = invocationOutputRoot(invocation); + writeJson(join(outputRoot, "accounts.json"), conflictingAccounts); + const manifest = JSON.parse( + readFileSync(join(outputRoot, "manifest.json"), "utf8"), + ) as Record; + writeJson(join(outputRoot, "manifest.json"), { + ...manifest, + accounts: conflictingAccounts, + }); + return result; + }, + }, + }); + await expect(exportBeeperMessageLikeMeBundle({ outputRoot: output, source })) + .rejects.toThrow("Beeper account user contradicts its self identity anchor"); + expect(existsSync(working)).toBeFalse(); + expect(existsSync(output)).toBeFalse(); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("deduplicates exact official reactions before participant and bundle budgets", async () => { + const parents: string[] = []; + const exportDeduplicatedFixture = async (maxBundleBytes?: number) => { + const parent = privateDirectory("wrench-beeper-source-reaction-dedup-test."); + parents.push(parent); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + const reaction = { + emoji: true, + id: "reaction-exact-duplicate", + participantID: "whatsapp:reaction-only-participant", + reactionKey: "🔥", + }; + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + maxBundleRecords: 11, + ...(maxBundleBytes === undefined ? {} : { maxBundleBytes }), + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + const outputRoot = whatsAppOutputRoot(invocation); + if (outputRoot === null) return result; + const [first, ...remaining] = messages(); + writeJson(join(outputRoot, "chats", CHAT_ID, "messages.json"), [{ + ...(first as Record), + reactions: [{ + ...reaction, + imgURL: "https://media.example.test/first-reaction-image", + }, { + ...reaction, + emoji: false, + imgURL: "file:///private/different-ignored-reaction-image", + }], + }, ...remaining]); + return result; + }, + }, + }); + const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); + return { output, result }; + }; + + try { + const baseline = await exportDeduplicatedFixture(); + const exactRecordBytes = baseline.result.manifest.artifacts.reduce( + (sum, artifact) => sum + artifact.bytes, + 0, + ); + const byteBounded = await exportDeduplicatedFixture(exactRecordBytes); + + for (const { output, result } of [baseline, byteBounded]) { + expect(result.manifest.counts).toEqual({ + account: 2, + participant: 4, + conversation: 1, + message: 2, + reaction: 1, + tombstone: 1, + }); + expect(Object.values(result.manifest.counts).reduce((sum, value) => sum + value, 0)) + .toBe(11); + expect(result.manifest.warnings).not.toContain("bundle-record-limit-reached"); + expect(result.manifest.warnings).not.toContain("bundle-byte-limit-reached"); + expect(result.manifest.warnings).not.toContain("reaction-provider-id-non-unique"); + const reactionRows = ndjson(join(output, "reactions.ndjson")); + const participantRows = ndjson(join(output, "participants.ndjson")); + expect(reactionRows).toHaveLength(1); + expect(participantRows.filter(({ id }) => id === reactionRows[0]!.participantId)) + .toHaveLength(1); + expect(reactionRows[0]!.provenance).toMatchObject({ + providerRevision: null, + }); + expect(JSON.stringify(reactionRows)).not.toContain("reaction-exact-duplicate"); + expect(JSON.stringify(reactionRows)).not.toContain("different-ignored"); + } + expect(byteBounded.result.manifest.artifacts.reduce( + (sum, artifact) => sum + artifact.bytes, + 0, + )).toBe(exactRecordBytes); + } finally { + for (const parent of parents) rmSync(parent, { recursive: true, force: true }); + } + }); + + test("preserves nonunique official reaction provider IDs with composite identities", async () => { + const cases = [{ + name: "reaction-key", + second: { reactionKey: "private-provider-reaction-beta" }, + participantCount: 3, + reactionParticipantCount: 1, + maxBundleRecords: 11, + }, { + name: "participant", + second: { participantID: "whatsapp:self" }, + participantCount: 3, + reactionParticipantCount: 2, + maxBundleRecords: 11, + }] as const; + + for (const item of cases) { + const parent = privateDirectory( + `wrench-beeper-source-reaction-nonunique-${item.name}-test.`, + ); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + try { + const reaction = { + emoji: true, + id: "whatsapp:ada/private-provider-reaction-id", + participantID: "whatsapp:ada", + reactionKey: "private-provider-reaction-alpha", + }; + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + maxBundleRecords: item.maxBundleRecords, + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + const outputRoot = whatsAppOutputRoot(invocation); + if (outputRoot === null) return result; + const [first, ...remaining] = messages(); + writeJson(join(outputRoot, "chats", CHAT_ID, "messages.json"), [{ + ...(first as Record), + reactions: [reaction, { ...reaction, ...item.second }], + }, ...remaining]); + return result; + }, + }, + }); + const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); + + expect(result.manifest.completeness.kind).toBe("bounded-local"); + expect(result.manifest.counts).toEqual({ + account: 2, + participant: item.participantCount, + conversation: 1, + message: 2, + reaction: 2, + tombstone: 1, + }); + expect(Object.values(result.manifest.counts).reduce((sum, value) => sum + value, 0)) + .toBe(item.maxBundleRecords); + expect(result.manifest.warnings).toEqual([ + "attachments-metadata-only", + "connected-account-backfill-coverage-unknown", + "reaction-provider-id-non-unique", + "remote-history-not-claimed", + "sequential-account-snapshot", + ]); + const reactionRows = ndjson(join(output, "reactions.ndjson")); + expect(reactionRows).toHaveLength(2); + expect(new Set(reactionRows.map(({ id }) => id)).size).toBe(2); + expect(new Set(reactionRows.map((row) => + (row.provenance as Record).providerId)).size).toBe(2); + expect(new Set(reactionRows.map(({ participantId }) => participantId)).size) + .toBe(item.reactionParticipantCount); + expect(reactionRows.map(({ body }) => body)).toEqual([ + "custom-reaction", + "custom-reaction", + ]); + expect(reactionRows.every((row) => + (row.provenance as Record).providerRevision === null)) + .toBeTrue(); + const publicProjection = JSON.stringify({ + manifest: result.manifest, + reactionRows, + }); + expect(publicProjection).not.toContain("private-provider-reaction-id"); + expect(publicProjection).not.toContain("private-provider-reaction-alpha"); + expect(publicProjection).not.toContain("private-provider-reaction-beta"); + expect(publicProjection).not.toContain("whatsapp:ada"); + expect(publicProjection).not.toContain("whatsapp:self"); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + } + }); + test("marks explicit chat and message limits as truncation", async () => { const parent = privateDirectory("wrench-beeper-source-limit-test."); const working = join(parent, "working"); const output = join(parent, "message-like-me"); mkdirSync(working, { mode: 0o700 }); + const invocations: BeeperExportCliInvocation[] = []; + const progress: BeeperMessageLikeMeProgress[] = []; try { const source = createBeeperMessageLikeMeSource({ auth: auth(configStore(parent)), limits: { limitChats: 1, limitMessages: 2 }, + onProgress: (item) => progress.push(item), dependencies: { binaryPath: "/fixture/beeper-0.6.2", createWorkingDirectory: async () => working, removeWorkingDirectory: async (path) => { rmSync(path, { recursive: true, force: true }); }, - runExport: fixtureExport, + runCli: async (invocation) => { + invocations.push(invocation); + return fixtureCli(invocation); + }, }, }); const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); + const exportInvocations = invocations.filter((invocation) => + invocation.arguments[0] === "export"); + expect(exportInvocations).toHaveLength(1); + const chatLimitIndex = exportInvocations[0]?.arguments.indexOf("--limit-chats") ?? -1; + expect(exportInvocations[0]?.arguments[chatLimitIndex + 1]).toBe("1"); + expect(progress).toContainEqual({ + phase: "account-skipped", + account: 2, + accounts: 2, + reason: "chat-limit-reached", + }); + expect(progress.at(-1)).toEqual({ + phase: "conversion-started", + accounts: 2, + chats: 1, + messages: 2, + }); expect(result.manifest.completeness.kind).toBe("truncated"); expect(result.manifest.warnings).toContain("chat-limit-reached"); expect(result.manifest.warnings).toContain("message-limit-reached"); @@ -380,6 +1500,40 @@ describe("Beeper Message Like Me source", () => { } }); + test("honors cancellation before validating a completed account shard", async () => { + const parent = privateDirectory("wrench-beeper-source-validation-cancel-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + const controller = new AbortController(); + let removed = false; + try { + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + signal: controller.signal, + onProgress: (progress) => { + if (progress.phase === "account-validating") controller.abort(); + }, + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + removed = true; + }, + runCli: fixtureCli, + }, + }); + await expect(exportBeeperMessageLikeMeBundle({ outputRoot: output, source })) + .rejects.toThrow("export was cancelled"); + expect(removed).toBeTrue(); + expect(existsSync(working)).toBeFalse(); + expect(existsSync(output)).toBeFalse(); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + test("projects an irregular complete direct roster conservatively as incomplete", async () => { const parent = privateDirectory("wrench-beeper-source-direct-roster-test."); const working = join(parent, "working"); @@ -394,8 +1548,10 @@ describe("Beeper Message Like Me source", () => { removeWorkingDirectory: async (path) => { rmSync(path, { recursive: true, force: true }); }, - runExport: async (invocation) => { - const result = await fixtureExport(invocation); + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + const outputRoot = whatsAppOutputRoot(invocation); + if (outputRoot === null) return result; const extraPeer = { fullName: "Extra Fixture", id: "whatsapp:extra", @@ -405,7 +1561,6 @@ describe("Beeper Message Like Me source", () => { const irregular = chat(); irregular.participants.items.push(extraPeer); irregular.participants.total = 2; - const outputRoot = invocationOutputRoot(invocation); writeJson(join(outputRoot, "chats.json"), [irregular]); writeJson(join(outputRoot, "chats", CHAT_ID, "chat.json"), irregular); return result; @@ -438,7 +1593,7 @@ describe("Beeper Message Like Me source", () => { rmSync(path, { recursive: true, force: true }); }, maxBundleRecords: 7, - runExport: fixtureExport, + runCli: fixtureCli, }, }); const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); @@ -476,7 +1631,7 @@ describe("Beeper Message Like Me source", () => { rmSync(path, { recursive: true, force: true }); }, maxBundleBytes: 2_500, - runExport: fixtureExport, + runCli: fixtureCli, }, }); const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); @@ -512,7 +1667,7 @@ describe("Beeper Message Like Me source", () => { rmSync(path, { recursive: true, force: true }); removed = true; }, - runExport: fixtureExport, + runCli: fixtureCli, }, }); await expect(exportBeeperMessageLikeMeBundle({ outputRoot: output, source })) @@ -525,6 +1680,212 @@ describe("Beeper Message Like Me source", () => { } }); + test("rejects connected account realm drift across the sequential snapshot", async () => { + const parent = privateDirectory("wrench-beeper-source-realm-drift-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + let accountInventories = 0; + let removed = false; + try { + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + removed = true; + }, + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + if ( + invocation.arguments[0] !== "accounts" + || invocation.arguments[1] !== "list" + || ++accountInventories !== 2 + ) return result; + const changedAccounts = accounts().map((value, index) => { + if (index !== 1) return value; + const account = value as Record; + return { + ...account, + user: { + ...(account.user as Record), + displayName: "Changed selector alias", + }, + }; + }); + return { + ...result, + stdout: `${JSON.stringify({ + success: true, + data: changedAccounts, + error: null, + })}\n`, + }; + }, + }, + }); + await expect(exportBeeperMessageLikeMeBundle({ outputRoot: output, source })) + .rejects.toThrow("connected Beeper account inventory changed during export"); + expect(accountInventories).toBe(2); + expect(removed).toBeTrue(); + expect(existsSync(working)).toBeFalse(); + expect(existsSync(join(output, "manifest.json"))).toBeFalse(); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("rejects self-alias evidence drift between accounting and emission", async () => { + const parent = privateDirectory("wrench-beeper-source-self-alias-drift-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + try { + const baseSource = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + maxBundleRecords: 11, + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + const outputRoot = whatsAppOutputRoot(invocation); + if (outputRoot !== null) writeSelfAliasFixture(outputRoot, null); + return result; + }, + }, + }); + let mutated = false; + const source = { + ...baseSource, + records: (async function* () { + for await (const record of baseSource.records) { + yield record; + if (!mutated) { + const [, lateChatId] = orderedSelfAliasChatIds(); + const messagesPath = join( + working, + "account-shards", + "account-001", + "chats", + lateChatId, + "messages.json", + ); + const original = readFileSync(messagesPath, "utf8"); + const changed = original.replace( + SELF_ALIAS_ID, + "whatsapp:private-late-peer-alias", + ); + expect(changed.length).toBe(original.length); + expect(changed).not.toBe(original); + writeFileSync(messagesPath, changed); + mutated = true; + } + } + })(), + }; + + await expect(exportBeeperMessageLikeMeBundle({ outputRoot: output, source })) + .rejects.toThrow("official export messages changed between validated passes"); + expect(mutated).toBeTrue(); + expect(existsSync(working)).toBeFalse(); + expect(existsSync(output)).toBeFalse(); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("rejects reaction tuple bytes changed between validation and emission", async () => { + const parent = privateDirectory("wrench-beeper-source-message-drift-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + try { + const baseSource = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + runCli: fixtureCli, + }, + }); + let mutated = false; + const source = { + ...baseSource, + records: (async function* () { + for await (const record of baseSource.records) { + yield record; + if (!mutated) { + const messagesPath = join( + working, + "account-shards", + "account-001", + "chats", + CHAT_ID, + "messages.json", + ); + const original = readFileSync(messagesPath, "utf8"); + const changed = original.replace("reaction-1", "reaction-x"); + expect(changed.length).toBe(original.length); + expect(changed).not.toBe(original); + writeFileSync(messagesPath, changed); + mutated = true; + } + } + })(), + }; + + await expect(exportBeeperMessageLikeMeBundle({ outputRoot: output, source })) + .rejects.toThrow("messages changed between validated passes"); + expect(mutated).toBeTrue(); + expect(existsSync(working)).toBeFalse(); + expect(existsSync(output)).toBeFalse(); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("retains raw working data for a retry when source disposal fails", async () => { + const parent = privateDirectory("wrench-beeper-source-disposal-retry-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + let attempts = 0; + try { + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + attempts += 1; + if (attempts === 1) throw new Error("synthetic transient cleanup failure"); + rmSync(path, { recursive: true, force: true }); + }, + runCli: fixtureCli, + }, + }); + + await expect(exportBeeperMessageLikeMeBundle({ outputRoot: output, source })) + .rejects.toThrow("synthetic transient cleanup failure"); + expect(existsSync(output)).toBeFalse(); + expect(existsSync(working)).toBeTrue(); + await source.dispose?.(false); + expect(attempts).toBe(2); + expect(existsSync(working)).toBeFalse(); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + test("compares only bounded parsed account projections from the official manifest", async () => { const parent = privateDirectory("wrench-beeper-source-deep-account-test."); const working = join(parent, "working"); @@ -539,9 +1900,10 @@ describe("Beeper Message Like Me source", () => { removeWorkingDirectory: async (path) => { rmSync(path, { recursive: true, force: true }); }, - runExport: async (invocation) => { - const result = await fixtureExport(invocation); - const outputRoot = invocationOutputRoot(invocation); + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + const outputRoot = whatsAppOutputRoot(invocation); + if (outputRoot === null) return result; const deepCapability = `${'{"next":'.repeat(5_000)}null${"}".repeat(5_000)}`; const accountRows = accounts().map((account, index) => { const encoded = JSON.stringify(account); @@ -629,9 +1991,10 @@ describe("Beeper Message Like Me source", () => { rmSync(path, { recursive: true, force: true }); removed = true; }, - runExport: async (invocation) => { - const result = await fixtureExport(invocation); - item.mutate(invocationOutputRoot(invocation)); + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + const outputRoot = whatsAppOutputRoot(invocation); + if (outputRoot !== null) item.mutate(outputRoot); return result; }, }, @@ -664,7 +2027,7 @@ describe("Beeper Message Like Me source", () => { removed = true; }, maxMessagesJsonBytes: 32, - runExport: fixtureExport, + runCli: fixtureCli, }, }); const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); diff --git a/src/beeper-message-like-me-source.ts b/src/beeper-message-like-me-source.ts index d23552e..67184cc 100644 --- a/src/beeper-message-like-me-source.ts +++ b/src/beeper-message-like-me-source.ts @@ -1,4 +1,5 @@ -import { constants } from "node:fs"; +import { constants, type Stats } from "node:fs"; +import { createHash } from "node:crypto"; import { chmod, lstat, @@ -6,15 +7,24 @@ import { mkdtemp, open, readdir, + readlink, realpath, - rm, + rmdir, + statfs, unlink, } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { basename, isAbsolute, join, resolve, sep } from "node:path"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import type { WrenchAuth } from "./auth"; +import { removePrivateDirectoryTree } from "./storage"; import { canonicalJson, sha256 } from "./canonical-json"; +import { + createBeeperMessageLikeMeDirectoryLease, + releaseBeeperMessageLikeMeDirectoryLease, + updateBeeperMessageLikeMeDirectoryLease, + type BeeperMessageLikeMeDirectoryLease, +} from "./beeper-message-like-me-recovery"; import { BEEPER_MESSAGE_LIKE_ME_MAX_RECORDS, BEEPER_MESSAGE_LIKE_ME_MAX_TOTAL_BYTES, @@ -22,6 +32,7 @@ import { import type { BeeperMessageLikeMeAccount, BeeperMessageLikeMeAttachment, + BeeperMessageLikeMeBundleProgress, BeeperMessageLikeMeConversation, BeeperMessageLikeMeExportSource, BeeperMessageLikeMeMessage, @@ -32,10 +43,12 @@ import type { } from "./beeper-message-like-me-export"; import { BEEPER_CLI_PIN, + planBeeperAccountsListCommand, planBeeperMessageLikeMeExportCommand, } from "./providers/beeper-local"; import { beeperSubjectFromAccounts, + parseBeeperCliEnvelope, parseBeeperExportAccounts, parseBeeperExportConversation, parseBeeperExportMessages, @@ -51,14 +64,24 @@ import { } from "./providers/beeper-local-runtime"; const PRIVATE_DIRECTORY_MODE = 0o700; +const PRIVATE_FILE_MODE = 0o600; const MAX_STDOUT_BYTES = 64 * 1024; const MAX_STDERR_BYTES = 64 * 1024; +const ACCOUNT_HEARTBEAT_INTERVAL_MS = 30_000; +const MAX_ACCOUNTS_LIST_BYTES = 8 * 1024 * 1024; +const MAX_BEEPER_CONFIG_BYTES = 4 * 1024 * 1024; const MAX_ACCOUNTS_JSON_BYTES = 32 * 1024 * 1024; const MAX_CHATS_JSON_BYTES = 64 * 1024 * 1024; const MAX_CHAT_JSON_BYTES = 32 * 1024 * 1024; const MAX_MESSAGES_JSON_BYTES = 64 * 1024 * 1024; const MAX_EXPORT_CHATS = 100_000; const MAX_EXPORT_MESSAGES_PER_CHAT = 1_000_000; +const MAX_RAW_WORKING_BYTES = 4 * 1024 * 1024 * 1024; +const MIN_FREE_FILESYSTEM_BYTES = 2 * 1024 * 1024 * 1024; +const MAX_RAW_WORKING_ENTRIES = 1_000_000; +const MAX_RAW_CACHE_SYMLINK_TARGET_BYTES = 512; +const RAW_WORKING_MONITOR_INTERVAL_MS = 500; +const RAW_WORKING_RECOVERY_GRACE_MS = 5 * 60 * 1_000; const DEFAULT_MAX_PARTICIPANTS = 500; const DEFAULT_TIMEOUT_MS = 6 * 60 * 60 * 1_000; @@ -78,9 +101,80 @@ export type BeeperExportCliInvocation = Readonly<{ timeoutMs: number; maxOutputBytes: number; maxStderrBytes: number; + workingRoot?: string; + maxWorkingBytes?: number; + onHeartbeat?: (elapsedSeconds: number) => void; signal?: AbortSignal; + directoryLease?: BeeperMessageLikeMeDirectoryLease; }>; +export type BeeperMessageLikeMeProgress = + | BeeperMessageLikeMeBundleProgress + | Readonly<{ + phase: "recovery-started"; + }> + | Readonly<{ + phase: "recovery-completed"; + recovered: number; + published: number; + }> + | Readonly<{ + phase: "preparing"; + }> + | Readonly<{ + phase: "accounts-discovered"; + accounts: number; + }> + | Readonly<{ + phase: "accounts-progress"; + stage: "discovering" | "verifying"; + elapsedSeconds: number; + }> + | Readonly<{ + phase: "account-started"; + account: number; + accounts: number; + }> + | Readonly<{ + phase: "account-validating"; + account: number; + accounts: number; + elapsedSeconds: number; + }> + | Readonly<{ + phase: "account-progress"; + account: number; + accounts: number; + elapsedSeconds: number; + }> + | Readonly<{ + phase: "account-skipped"; + account: number; + accounts: number; + reason: "chat-limit-reached"; + }> + | Readonly<{ + phase: "account-completed"; + account: number; + accounts: number; + chats: number; + messages: number; + }> + | Readonly<{ + phase: "accounts-verifying"; + accounts: number; + }> + | Readonly<{ + phase: "conversion-progress"; + elapsedSeconds: number; + }> + | Readonly<{ + phase: "conversion-started"; + accounts: number; + chats: number; + messages: number; + }>; + export type BeeperMessageLikeMeSourceDependencies = Readonly<{ /** Test-only seam. Production resolves the exact pinned binary hash. */ binaryPath?: string; @@ -90,7 +184,7 @@ export type BeeperMessageLikeMeSourceDependencies = Readonly<{ maxBundleBytes?: number; /** Test-only seam for exercising the per-chat JSON allocation cap. */ maxMessagesJsonBytes?: number; - runExport?: ( + runCli?: ( invocation: BeeperExportCliInvocation, ) => Promise; createWorkingDirectory?: () => Promise; @@ -101,6 +195,7 @@ export type BeeperMessageLikeMeSourceRequest = Readonly<{ auth: WrenchAuth; limits?: BeeperMessageLikeMeSourceLimits; signal?: AbortSignal; + onProgress?: (progress: BeeperMessageLikeMeProgress) => void; environment?: Readonly>; dependencies?: BeeperMessageLikeMeSourceDependencies; }>; @@ -121,20 +216,65 @@ type ParticipantFact = { isSelf: boolean | null; }; +type PrivateDirectoryIdentity = Readonly<{ + device: number; + inode: number; +}>; + type ConversationScan = Readonly<{ - chat: BeeperConversationProjection; - directory: string; + chat: Pick< + BeeperConversationProjection, + "id" | "accountId" | "lastActivity" | "title" | "type" + >; + root: string; messagesPath: string; + messagesSha256: string; + observedAt: string; participantIds: readonly string[]; participantsComplete: boolean; startedAt: string | null; lastMessageAt: string | null; messageCount: number; reactionCount: number; + reactionProviderIdNonUniqueGroups: number; tombstoneCount: number; nonParticipantRecordBytes: number; }>; +type ValidatedShardChat = Readonly<{ + chat: Pick; + root: string; + chatPath: string; + chatSha256: string; + messagesPath: string; + expectedMessageCount: number; + observedAt: string; +}>; + +type ValidatedAccountShard = Readonly<{ + accountId: string; + completedAt: string; + chats: readonly ValidatedShardChat[]; + messageCount: number; +}>; + +type SelfAliasPrepass = Readonly<{ + aliasesByAccount: ReadonlyMap>; + coveredChats: readonly ValidatedShardChat[]; + evidenceLimitReached: boolean; + messagesSha256ByPath: ReadonlyMap; +}>; + +type BeeperCliStoreSnapshot = Readonly<{ + config: JsonRecord; + desktopTarget: JsonRecord; +}>; + +type OperationPrivateBeeperStore = Readonly<{ + path: string; + identity: PrivateDirectoryIdentity; +}>; + type ExportManifest = Readonly<{ accounts: unknown; attachmentCount: number; @@ -251,14 +391,16 @@ function parseLimits(value: BeeperMessageLikeMeSourceLimits | undefined): Parsed function requireAuth(auth: WrenchAuth): Extract { +}> & Readonly<{ provider: "beeper"; subject: string }> { if ( auth.kind !== "linked-device-store" || auth.provider !== "beeper" || auth.subject === undefined || !/^beeper:local:[a-f0-9]{64}$/u.test(auth.subject) ) return fail("export requires an account-bound Beeper linked-device-store auth locator"); - return auth; + return auth as Extract & Readonly<{ provider: "beeper"; subject: string }>; } function throwIfAborted(signal: AbortSignal | undefined): void { @@ -375,11 +517,14 @@ async function assertPrivateOwnedDirectory( return canonical; } -async function readOwnedJson( +async function readOwnedJsonDocument( path: string, root: string, maximumBytes: number, -): Promise { + privateFile = false, + signal?: AbortSignal, +): Promise> { + throwIfAborted(signal); if (!path.startsWith(`${root}${sep}`)) return fail("export file escaped private staging"); let handle; try { @@ -396,12 +541,14 @@ async function readOwnedJson( || before.uid !== process.getuid?.() || before.nlink !== 1 || (before.mode & 0o022) !== 0 + || (privateFile && (before.mode & 0o077) !== 0) || before.size < 2 || before.size > maximumBytes ) return fail("official export file is outside its ownership or size bound"); const bytes = Buffer.allocUnsafe(before.size); let offset = 0; while (offset < bytes.byteLength) { + throwIfAborted(signal); const result = await handle.read(bytes, offset, bytes.byteLength - offset, offset); if (result.bytesRead === 0) break; offset += result.bytesRead; @@ -420,24 +567,46 @@ async function readOwnedJson( || before.ctimeMs !== after.ctimeMs ) return fail("official export file changed while being read"); const pathMetadata = await lstat(path); + throwIfAborted(signal); if ( pathMetadata.isSymbolicLink() || pathMetadata.nlink !== 1 || pathMetadata.dev !== after.dev || pathMetadata.ino !== after.ino ) return fail("official export file changed while being read"); + let value: unknown; try { - return JSON.parse( - new TextDecoder("utf-8", { fatal: true }).decode(bytes), - ) as unknown; + const decoded = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + value = JSON.parse(decoded) as unknown; } catch { return fail("official export file is not valid UTF-8 JSON"); } + throwIfAborted(signal); + return Object.freeze({ + value, + sha256: createHash("sha256").update(bytes).digest("hex"), + }); } finally { await handle.close(); } } +async function readOwnedJson( + path: string, + root: string, + maximumBytes: number, + privateFile = false, + signal?: AbortSignal, +): Promise { + return (await readOwnedJsonDocument( + path, + root, + maximumBytes, + privateFile, + signal, + )).value; +} + async function ownedFileSize(path: string, root: string): Promise { if (!path.startsWith(`${root}${sep}`)) return fail("export file escaped private staging"); let handle; @@ -480,6 +649,11 @@ function parseManifest(value: unknown): ExportManifest { "version", ], [], "official export manifest"); if (source.version !== 1) return fail("official export manifest version is unsupported"); + const createdAt = timestamp(source.createdAt, "official export manifest createdAt"); + const completedAt = timestamp(source.completedAt, "official export manifest completedAt"); + if (completedAt < createdAt) { + return fail("official export manifest completed before it started"); + } return Object.freeze({ accounts: source.accounts, attachmentCount: integer( @@ -492,8 +666,8 @@ function parseManifest(value: unknown): ExportManifest { "official export manifest chatCount", MAX_EXPORT_CHATS, ), - completedAt: timestamp(source.completedAt, "official export manifest completedAt"), - createdAt: timestamp(source.createdAt, "official export manifest createdAt"), + completedAt, + createdAt, messageCount: integer( source.messageCount, "official export manifest messageCount", @@ -531,26 +705,385 @@ async function readBoundedStream( return new TextDecoder("utf-8", { fatal: true }).decode(output); } -async function runExportCli( +function pathInside(root: string, path: string): boolean { + return path.startsWith(`${root}${sep}`); +} + +function sameFilesystemObject(left: Stats, right: Stats): boolean { + return left.dev === right.dev + && left.ino === right.ino + && left.uid === right.uid + && left.mode === right.mode + && left.nlink === right.nlink + && left.size === right.size + && left.ctimeMs === right.ctimeMs + && left.mtimeMs === right.mtimeMs; +} + +class BeeperRawStagingInvariantError extends Error {} + +function rawStagingFail(message: string): never { + throw new BeeperRawStagingInvariantError( + `Beeper Message Like Me source: ${message}`, + ); +} + +function unsafeCacheSymlink(): never { + return rawStagingFail( + "raw export staging contained an unsafe CLI payload-cache symbolic link", + ); +} + +async function assertSafeCliPayloadCacheSymlink( + path: string, + directory: string, + cacheRoot: string, + linkMetadata: Stats, + uid: number, +): Promise { + if (!pathInside(cacheRoot, path)) { + return rawStagingFail( + "raw export staging contained a symbolic link outside its CLI payload cache", + ); + } + if ( + linkMetadata.uid !== uid + || linkMetadata.nlink !== 1 + || /[\u0000-\u001f\u007f\ufffd]/u.test(basename(path)) + ) return unsafeCacheSymlink(); + + let target: string; + try { + target = await readlink(path); + } catch { + return unsafeCacheSymlink(); + } + if ( + target.length === 0 + || isAbsolute(target) + || Buffer.byteLength(target, "utf8") > MAX_RAW_CACHE_SYMLINK_TARGET_BYTES + || /[\u0000-\u001f\u007f\ufffd]/u.test(target) + ) return unsafeCacheSymlink(); + const lexicalTarget = resolve(directory, target); + if (!pathInside(cacheRoot, lexicalTarget)) return unsafeCacheSymlink(); + + const targetParent = dirname(lexicalTarget); + const targetParentRelative = relative(cacheRoot, targetParent); + const linkParentRelative = relative(cacheRoot, directory); + if ( + dirname(path) !== directory + || targetParentRelative === ".." + || targetParentRelative.startsWith(`..${sep}`) + || isAbsolute(targetParentRelative) + || linkParentRelative === ".." + || linkParentRelative.startsWith(`..${sep}`) + || isAbsolute(linkParentRelative) + ) return unsafeCacheSymlink(); + const parentSnapshots: { readonly path: string; readonly metadata: Stats }[] = []; + try { + const cacheMetadata = await lstat(cacheRoot); + if ( + !cacheMetadata.isDirectory() + || cacheMetadata.isSymbolicLink() + || cacheMetadata.uid !== uid + || (cacheMetadata.mode & 0o022) !== 0 + ) return unsafeCacheSymlink(); + parentSnapshots.push({ path: cacheRoot, metadata: cacheMetadata }); + for (const parentRelative of [targetParentRelative, linkParentRelative]) { + if (parentRelative === "") continue; + let currentParent = cacheRoot; + for (const segment of parentRelative.split(sep)) { + if (segment === "" || segment === "." || segment === "..") { + return unsafeCacheSymlink(); + } + currentParent = join(currentParent, segment); + if (parentSnapshots.some((snapshot) => snapshot.path === currentParent)) { + continue; + } + const metadata = await lstat(currentParent); + if ( + !metadata.isDirectory() + || metadata.isSymbolicLink() + || metadata.uid !== uid + || (metadata.mode & 0o022) !== 0 + ) return unsafeCacheSymlink(); + parentSnapshots.push({ path: currentParent, metadata }); + } + } + } catch { + return unsafeCacheSymlink(); + } + + let targetHandle; + try { + targetHandle = await open( + lexicalTarget, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + } catch { + return unsafeCacheSymlink(); + } + try { + const targetBefore = await targetHandle.stat(); + if ( + !targetBefore.isFile() + || targetBefore.uid !== uid + || targetBefore.nlink !== 1 + || (targetBefore.mode & 0o022) !== 0 + ) return unsafeCacheSymlink(); + let targetPathMetadata: Stats; + let targetCanonical: string; + let linkCanonical: string; + let linkAfter: Stats; + let targetAfterText: string; + try { + targetPathMetadata = await lstat(lexicalTarget); + if ( + !targetPathMetadata.isFile() + || targetPathMetadata.isSymbolicLink() + || targetPathMetadata.uid !== uid + || targetPathMetadata.nlink !== 1 + || (targetPathMetadata.mode & 0o022) !== 0 + || !sameFilesystemObject(targetBefore, targetPathMetadata) + ) return unsafeCacheSymlink(); + targetCanonical = await realpath(lexicalTarget); + linkCanonical = await realpath(path); + linkAfter = await lstat(path); + targetAfterText = await readlink(path); + } catch { + return unsafeCacheSymlink(); + } + const targetAfter = await targetHandle.stat(); + if ( + !sameFilesystemObject(targetBefore, targetAfter) + || targetCanonical !== lexicalTarget + || linkCanonical !== lexicalTarget + || !sameFilesystemObject(linkMetadata, linkAfter) + || targetAfterText !== target + ) return unsafeCacheSymlink(); + for (const snapshot of parentSnapshots) { + let current: Stats; + try { + current = await lstat(snapshot.path); + } catch { + return unsafeCacheSymlink(); + } + if ( + !current.isDirectory() + || current.isSymbolicLink() + || !sameFilesystemObject(snapshot.metadata, current) + || await realpath(snapshot.path).catch(() => null) !== snapshot.path + ) return unsafeCacheSymlink(); + } + } finally { + await targetHandle.close(); + } +} + +/** @internal Exported only for focused safety tests. */ +export async function enforceBeeperRawWorkingBudget( + root: string, + maximumBytes: number, + minimumFreeBytes = MIN_FREE_FILESYSTEM_BYTES, + signal?: AbortSignal, +): Promise> { + throwIfAborted(signal); + if ( + !isAbsolute(root) + || !Number.isSafeInteger(maximumBytes) + || maximumBytes < 1 + || maximumBytes > MAX_RAW_WORKING_BYTES + || !Number.isSafeInteger(minimumFreeBytes) + || minimumFreeBytes < 0 + ) return rawStagingFail("raw export staging budget was invalid"); + const canonicalRoot = await assertPrivateOwnedDirectory(await realpath(root)); + const cacheRoot = join(canonicalRoot, "cli-payload-cache"); + const uid = process.getuid?.(); + if (uid === undefined) { + return rawStagingFail("raw export staging requires a POSIX user identity"); + } + const stack = [canonicalRoot]; + let bytes = 0; + let entries = 0; + while (stack.length > 0) { + throwIfAborted(signal); + const directory = stack.pop(); + if (directory === undefined) break; + let names: readonly string[]; + try { + names = (await readdir(directory)).sort(); + } catch (error) { + if (isErrno(error, "ENOENT")) continue; + return rawStagingFail("raw export staging could not be inspected safely"); + } + for (const name of names) { + throwIfAborted(signal); + const path = join(directory, name); + let metadata; + try { + metadata = await lstat(path); + } catch (error) { + if (isErrno(error, "ENOENT")) continue; + return rawStagingFail("raw export staging could not be inspected safely"); + } + entries += 1; + if (entries > MAX_RAW_WORKING_ENTRIES) { + return rawStagingFail("raw export staging exceeded its entry budget"); + } + const allocatedBytes = metadata.blocks * 512; + const entryBytes = Math.max(metadata.size, allocatedBytes); + if ( + !Number.isSafeInteger(entryBytes) + || entryBytes < 0 + || entryBytes > maximumBytes - bytes + ) return rawStagingFail("raw export staging exceeded its byte budget"); + bytes += entryBytes; + if (metadata.isSymbolicLink()) { + await assertSafeCliPayloadCacheSymlink( + path, + directory, + cacheRoot, + metadata, + uid, + ); + } else if ( + metadata.uid !== uid + || (metadata.mode & 0o022) !== 0 + ) { + return rawStagingFail( + "raw export staging contained an unsafe filesystem entry", + ); + } else if (metadata.isDirectory()) { + const canonical = await realpath(path).catch(() => null); + if ( + canonical !== path + || !canonical.startsWith(`${canonicalRoot}${sep}`) + ) { + return rawStagingFail( + "raw export staging directory changed during inspection", + ); + } + stack.push(path); + } else if (!metadata.isFile() || metadata.nlink !== 1) { + return rawStagingFail( + "raw export staging contained an unsafe filesystem entry", + ); + } + } + } + let filesystem; + try { + filesystem = await statfs(canonicalRoot); + } catch { + return rawStagingFail( + "raw export filesystem capacity could not be inspected", + ); + } + const availableBytes = filesystem.bavail * filesystem.bsize; + if ( + !Number.isSafeInteger(availableBytes) + || availableBytes < minimumFreeBytes + ) return rawStagingFail("raw export filesystem reserve would be exhausted"); + return Object.freeze({ bytes, entries }); +} + +async function enforceBeeperRawFilesystemReserve( + root: string, + signal: AbortSignal | undefined, +): Promise { + throwIfAborted(signal); + let filesystem; + try { + filesystem = await statfs(root); + } catch { + return fail("raw export filesystem capacity could not be inspected"); + } + throwIfAborted(signal); + const availableBytes = filesystem.bavail * filesystem.bsize; + if ( + !Number.isSafeInteger(availableBytes) + || availableBytes < MIN_FREE_FILESYSTEM_BYTES + ) return fail("raw export filesystem reserve would be exhausted"); +} + +/** @internal Exported only for focused process-lifecycle safety tests. */ +export async function runExportCli( invocation: BeeperExportCliInvocation, ): Promise { throwIfAborted(invocation.signal); - const child = Bun.spawn([ - "/bin/sh", - "-c", - "umask 077\nexec \"$@\"", - "wrench-beeper-export", - invocation.binary, - ...invocation.arguments, - ], { - env: { ...invocation.environment }, - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", - detached: true, - }); + if ( + (invocation.workingRoot === undefined) + !== (invocation.maxWorkingBytes === undefined) + ) return fail("raw export staging monitor was incompletely configured"); + if ( + invocation.workingRoot !== undefined + && invocation.maxWorkingBytes !== undefined + ) { + try { + await enforceBeeperRawWorkingBudget( + invocation.workingRoot, + invocation.maxWorkingBytes, + MIN_FREE_FILESYSTEM_BYTES, + invocation.signal, + ); + } catch (error) { + throwIfAborted(invocation.signal); + if (error instanceof BeeperRawStagingInvariantError) throw error; + return fail("official export raw staging safety check failed"); + } + } + if (invocation.directoryLease !== undefined) { + updateBeeperMessageLikeMeDirectoryLease( + invocation.directoryLease, + "launching", + ); + } + let leaseSettlementAttempted = false; + const settleDirectoryLease = (): void => { + if ( + invocation.directoryLease === undefined + || leaseSettlementAttempted + ) return; + leaseSettlementAttempted = true; + updateBeeperMessageLikeMeDirectoryLease( + invocation.directoryLease, + "settled", + ); + }; + const child = (() => { + try { + return Bun.spawn([ + "/bin/sh", + "-c", + "umask 077\nexec \"$@\"", + "wrench-beeper-export", + invocation.binary, + ...invocation.arguments, + ], { + env: { ...invocation.environment }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + detached: true, + }); + } catch (error) { + try { + settleDirectoryLease(); + } catch (settlementError) { + throw new AggregateError( + [error, settlementError], + "Beeper export launch and recovery lifecycle settlement both failed", + ); + } + throw error; + } + })(); let timedOut = false; let cancelled = false; + let heartbeatFailed = false; + let workingBudgetFailed = false; + let workingBudgetInspection: Promise | undefined; + let filesystemInspection: Promise | undefined; let forceKill: ReturnType | null = null; const signalGroup = (signal: "SIGTERM" | "SIGKILL"): void => { try { @@ -559,12 +1092,83 @@ async function runExportCli( // The complete child process group already exited. } }; + if (invocation.directoryLease !== undefined) { + try { + updateBeeperMessageLikeMeDirectoryLease( + invocation.directoryLease, + "running", + child.pid, + ); + } catch (error) { + signalGroup("SIGKILL"); + let reapError: unknown; + try { + await child.exited; + } catch (childError) { + reapError = childError; + } + let settlementError: unknown; + try { + settleDirectoryLease(); + } catch (leaseError) { + settlementError = leaseError; + } + const failures = [error, reapError, settlementError].filter( + (failure) => failure !== undefined, + ); + if (failures.length > 1) { + throw new AggregateError( + failures, + "Beeper export recovery lifecycle failed after launch", + ); + } + throw error; + } + } const terminate = (): void => { signalGroup("SIGTERM"); if (forceKill === null) { forceKill = setTimeout(() => signalGroup("SIGKILL"), 2_000); } }; + const inspectWorkingBudget = (): void => { + if ( + invocation.workingRoot === undefined + || invocation.maxWorkingBytes === undefined + || workingBudgetInspection !== undefined + ) return; + workingBudgetInspection = (async () => { + try { + await enforceBeeperRawWorkingBudget( + invocation.workingRoot!, + invocation.maxWorkingBytes!, + MIN_FREE_FILESYSTEM_BYTES, + invocation.signal, + ); + } catch { + workingBudgetFailed = true; + terminate(); + } + })().finally(() => { + workingBudgetInspection = undefined; + }); + }; + const inspectFilesystemReserve = (): void => { + if (invocation.workingRoot === undefined || filesystemInspection !== undefined) return; + filesystemInspection = (async () => { + try { + await enforceBeeperRawFilesystemReserve( + invocation.workingRoot!, + invocation.signal, + ); + } catch { + workingBudgetFailed = true; + terminate(); + } + })().finally(() => { + filesystemInspection = undefined; + }); + }; const onAbort = (): void => { cancelled = true; terminate(); @@ -574,21 +1178,99 @@ async function runExportCli( timedOut = true; terminate(); }, invocation.timeoutMs); + const startedAt = Date.now(); + const heartbeat = invocation.onHeartbeat === undefined + ? null + : setInterval(() => { + try { + invocation.onHeartbeat?.( + Math.max(1, Math.floor((Date.now() - startedAt) / 1_000)), + ); + } catch { + heartbeatFailed = true; + terminate(); + } + }, ACCOUNT_HEARTBEAT_INTERVAL_MS); + const workingBudgetMonitor = invocation.workingRoot === undefined + ? null + : setInterval(() => { + inspectWorkingBudget(); + inspectFilesystemReserve(); + }, RAW_WORKING_MONITOR_INTERVAL_MS); + inspectWorkingBudget(); + inspectFilesystemReserve(); try { const [exitCode, stdout, stderr] = await Promise.all([ child.exited, readBoundedStream(child.stdout, invocation.maxOutputBytes, "Beeper export stdout"), - readBoundedStream(child.stderr, invocation.maxStderrBytes, "Beeper export stderr"), + readBoundedStream( + child.stderr, + invocation.maxStderrBytes, + "Beeper export stderr", + ), ]); + settleDirectoryLease(); + if (workingBudgetInspection !== undefined) await workingBudgetInspection; + if (filesystemInspection !== undefined) await filesystemInspection; + if ( + !workingBudgetFailed + && invocation.workingRoot !== undefined + && invocation.maxWorkingBytes !== undefined + ) { + try { + await enforceBeeperRawWorkingBudget( + invocation.workingRoot, + invocation.maxWorkingBytes, + MIN_FREE_FILESYSTEM_BYTES, + invocation.signal, + ); + } catch { + workingBudgetFailed = true; + } + } if (cancelled) return fail("official export was cancelled"); if (timedOut) return fail("official export timed out"); + if (heartbeatFailed) return fail("export progress reporting failed"); + if (workingBudgetFailed) { + return fail("official export exceeded its raw staging safety budget"); + } return Object.freeze({ exitCode, stdout, stderr }); } catch (error) { signalGroup("SIGKILL"); - await child.exited; + let reapError: unknown; + try { + await child.exited; + } catch (childError) { + reapError = childError; + } + let settlementError: unknown; + try { + settleDirectoryLease(); + } catch (leaseError) { + settlementError = leaseError; + } + const failures = [error, reapError, settlementError].filter( + (failure) => failure !== undefined, + ); + if (failures.length > 1) { + throw new AggregateError( + failures, + "Beeper export and recovery lifecycle settlement both failed", + ); + } throw error; } finally { clearTimeout(timeout); + if (heartbeat !== null) clearInterval(heartbeat); + if (workingBudgetMonitor !== null) clearInterval(workingBudgetMonitor); + const pendingWorkingBudgetInspection = workingBudgetInspection; + if (pendingWorkingBudgetInspection !== undefined) { + await pendingWorkingBudgetInspection; + } + const pendingFilesystemInspection = filesystemInspection; + if (pendingFilesystemInspection !== undefined) { + await pendingFilesystemInspection; + } if (forceKill !== null) clearTimeout(forceKill); invocation.signal?.removeEventListener("abort", onAbort); } @@ -611,14 +1293,719 @@ function environmentForExport( }); } +function remainingTimeoutMs(deadlineMs: number): number { + const remaining = deadlineMs - Date.now(); + if (remaining <= 0) return fail("official export exceeded its command-wide timeout"); + return Math.max(1, Math.floor(remaining)); +} + +async function withProgressHeartbeat( + operation: () => Promise, + onHeartbeat: ((elapsedSeconds: number) => void) | undefined, +): Promise { + if (onHeartbeat === undefined) return operation(); + const startedAt = Date.now(); + let heartbeatFailed = false; + const heartbeat = setInterval(() => { + try { + onHeartbeat(Math.max(1, Math.floor((Date.now() - startedAt) / 1_000))); + } catch { + heartbeatFailed = true; + } + }, ACCOUNT_HEARTBEAT_INTERVAL_MS); + try { + const value = await operation(); + if (heartbeatFailed) return fail("export progress reporting failed"); + return value; + } finally { + clearInterval(heartbeat); + } +} + +function parseCliJson(stdout: string, label: string): unknown { + const source = stdout.trim(); + if (source.length === 0) return fail(`${label} omitted JSON output`); + let value: unknown; + try { + value = JSON.parse(source) as unknown; + } catch { + return fail(`${label} returned malformed JSON`); + } + try { + return parseBeeperCliEnvelope(value, label); + } catch { + return fail(`${label} returned an invalid success envelope`); + } +} + +async function enumerateAccounts( + binary: string, + environment: Readonly>, + deadlineMs: number, + run: (invocation: BeeperExportCliInvocation) => Promise, + directoryLease: BeeperMessageLikeMeDirectoryLease | undefined, + signal: AbortSignal | undefined, + onHeartbeat?: (elapsedSeconds: number) => void, +): Promise { + const timeoutMs = remainingTimeoutMs(deadlineMs); + const command = planBeeperAccountsListCommand(timeoutMs); + const result = await run({ + binary, + arguments: command.argv, + environment, + timeoutMs, + maxOutputBytes: MAX_ACCOUNTS_LIST_BYTES, + maxStderrBytes: MAX_STDERR_BYTES, + ...(directoryLease === undefined ? {} : { directoryLease }), + ...(onHeartbeat === undefined ? {} : { onHeartbeat }), + ...(signal === undefined ? {} : { signal }), + }); + throwIfAborted(signal); + if (result.exitCode !== 0 || result.stderr.trim().length !== 0) { + return fail("official account enumeration failed"); + } + try { + return parseBeeperExportAccounts(parseCliJson(result.stdout, "official account enumeration")); + } catch { + return fail("official account enumeration returned an unsupported projection"); + } +} + +function accountOutputRealm( + account: BeeperAccountProjection, +): Readonly> { + return Object.freeze({ + accountId: account.accountId, + bridge: account.bridge, + network: account.network, + user: Object.freeze({ + id: account.user.id, + displayName: account.selectorAliases.displayName, + name: account.selectorAliases.name, + fullName: account.user.fullName, + username: account.user.username, + phoneNumber: account.user.phoneNumber, + email: account.user.email, + isSelf: account.user.isSelf, + }), + }); +} + +function outputRealmDigest(accounts: readonly BeeperAccountProjection[]): string { + return sha256(canonicalJson([...accounts] + .sort((left, right) => left.accountId.localeCompare(right.accountId)) + .map(accountOutputRealm))); +} + +function assertOutputRealm( + accounts: readonly BeeperAccountProjection[], + expectedDigest: string, +): void { + if (outputRealmDigest(accounts) !== expectedDigest) { + fail("connected Beeper account inventory changed during export"); + } +} + +function normalizeOfficialAccountSelector(value: string | null): string { + return (value ?? "").trim().toLowerCase().replace(/[\s._-]+/gu, ""); +} + +/** @internal Exported only for pinned-selector safety tests. */ +export function assertUniqueOfficialAccountSelector( + selected: BeeperAccountProjection, + accounts: readonly BeeperAccountProjection[], +): void { + const wanted = normalizeOfficialAccountSelector(selected.accountId); + const matches = accounts.filter((account) => [ + account.accountId, + account.network, + account.bridge.type, + account.bridge.id, + account.user.id, + account.user.username, + account.selectorAliases.displayName, + account.selectorAliases.name, + account.user.email, + ].some((candidate) => normalizeOfficialAccountSelector(candidate) === wanted)); + if (matches.length !== 1 || matches[0]?.accountId !== selected.accountId) { + fail("one Beeper account ID is ambiguous under the pinned CLI selector rules"); + } +} + +async function readCliStoreSnapshot( + configDirectory: string, +): Promise { + const assertLocalDesktopUrl = (value: unknown, label: string): string | undefined => { + if (value === undefined) return; + const source = string(value, label, 2_048); + let parsed: URL; + try { + parsed = new URL(source); + } catch { + return fail(`${label} is no longer a reviewed loopback Desktop URL`); + } + const port = Number(parsed.port); + if ( + parsed.protocol !== "http:" + || parsed.hostname !== "127.0.0.1" + || parsed.username !== "" + || parsed.password !== "" + || parsed.pathname !== "/" + || parsed.search !== "" + || parsed.hash !== "" + || !Number.isSafeInteger(port) + || port < 23_373 + || port > 23_392 + ) return fail(`${label} is no longer a reviewed loopback Desktop URL`); + return source; + }; + const storedAuth = (value: unknown, label: string): JsonRecord => { + const auth = record(value, label); + exactKeys(auth, ["accessToken", "tokenType"], [ + "clientID", + "expiresAt", + "scope", + "source", + ], label); + const accessToken = string(auth.accessToken, `${label}.accessToken`, 64 * 1024); + if (auth.tokenType !== "Bearer") return fail(`${label}.tokenType is unsupported`); + const optionalText = (key: "clientID" | "scope", maximum: number): string | undefined => + auth[key] === undefined ? undefined : string(auth[key], `${label}.${key}`, maximum); + const clientID = optionalText("clientID", 2_048); + const scope = optionalText("scope", 2_048); + const expiresAt = auth.expiresAt === undefined + ? undefined + : timestamp(auth.expiresAt, `${label}.expiresAt`); + const source = auth.source === undefined + ? undefined + : string(auth.source, `${label}.source`, 64); + if ( + source !== undefined + && !["desktop-db", "desktop-cache", "desktop-oauth", "remote-oauth", "manual"].includes(source) + ) return fail(`${label}.source is unsupported`); + return Object.freeze({ + accessToken, + tokenType: "Bearer", + ...(clientID === undefined ? {} : { clientID }), + ...(expiresAt === undefined ? {} : { expiresAt }), + ...(scope === undefined ? {} : { scope }), + ...(source === undefined ? {} : { source }), + }); + }; + const assertPrivateSourceFile = async (path: string, label: string): Promise => { + const metadata = await lstat(path); + if ( + !metadata.isFile() + || metadata.isSymbolicLink() + || metadata.uid !== process.getuid?.() + || metadata.nlink !== 1 + || (metadata.mode & 0o077) !== 0 + || await realpath(path) !== path + ) return fail(`${label} is no longer one private physical file`); + }; + const configPath = join(configDirectory, "config.json"); + await assertPrivateSourceFile(configPath, "Beeper CLI config"); + const config = record( + await readOwnedJson( + configPath, + configDirectory, + MAX_BEEPER_CONFIG_BYTES, + true, + ), + "Beeper CLI config", + ); + exactKeys( + config, + [], + ["auth", "baseURL", "defaultAccount", "defaultTarget"], + "Beeper CLI config", + ); + if (config.defaultTarget !== "desktop") { + return fail("Beeper CLI config no longer selects the fixed Desktop target"); + } + const configBaseUrl = assertLocalDesktopUrl( + config.baseURL, + "Beeper CLI config.baseURL", + ); + const configAuth = config.auth === undefined + ? undefined + : storedAuth(config.auth, "Beeper CLI config.auth"); + const targetsRoot = await assertOwnedDirectory( + join(configDirectory, "targets"), + configDirectory, + ); + const targetPath = join(targetsRoot, "desktop.json"); + await assertPrivateSourceFile(targetPath, "Beeper Desktop target"); + const desktopTarget = record( + await readOwnedJson( + targetPath, + configDirectory, + MAX_BEEPER_CONFIG_BYTES, + true, + ), + "Beeper Desktop target", + ); + exactKeys(desktopTarget, ["id", "type", "baseURL"], [ + "auth", + "dataDir", + "managed", + "name", + "port", + "profile", + "runtime", + "serverEnv", + ], "Beeper Desktop target"); + if (desktopTarget.id !== "desktop" || desktopTarget.type !== "desktop") { + return fail("Beeper CLI target no longer identifies the fixed Desktop realm"); + } + const targetBaseUrl = assertLocalDesktopUrl( + desktopTarget.baseURL, + "Beeper Desktop target.baseURL", + ); + if (targetBaseUrl === undefined) return fail("Beeper Desktop target omitted baseURL"); + if ( + (desktopTarget.managed !== undefined && desktopTarget.managed !== false) + || desktopTarget.dataDir !== undefined + || desktopTarget.profile !== undefined + || desktopTarget.serverEnv !== undefined + ) return fail("Beeper Desktop target contains an active endpoint override"); + if (desktopTarget.port !== undefined) { + const port = positiveInteger( + desktopTarget.port, + "Beeper Desktop target.port", + 23_392, + ); + if (port < 23_373) return fail("Beeper Desktop target.port is outside the reviewed range"); + } + if (desktopTarget.runtime !== undefined) { + const runtime = record(desktopTarget.runtime, "Beeper Desktop target.runtime"); + exactKeys(runtime, ["install", "port"], [], "Beeper Desktop target.runtime"); + if (runtime.install !== "desktop") { + return fail("Beeper Desktop target.runtime.install is unsupported"); + } + const port = positiveInteger( + runtime.port, + "Beeper Desktop target.runtime.port", + 23_392, + ); + if (port < 23_373) { + return fail("Beeper Desktop target.runtime.port is outside the reviewed range"); + } + } + if (desktopTarget.name !== undefined) { + string(desktopTarget.name, "Beeper Desktop target.name", 2_048); + } + const targetAuth = desktopTarget.auth === undefined + ? undefined + : storedAuth(desktopTarget.auth, "Beeper Desktop target.auth"); + const effectiveAuth = targetAuth + ?? (configAuth !== undefined + && (configBaseUrl === undefined || configBaseUrl === targetBaseUrl) + ? configAuth + : undefined); + if (effectiveAuth === undefined) { + return fail("Beeper Desktop target has no effective stored access token"); + } + return Object.freeze({ + config: Object.freeze({ + baseURL: targetBaseUrl, + defaultTarget: "desktop", + }), + desktopTarget: Object.freeze({ + auth: effectiveAuth, + baseURL: targetBaseUrl, + id: "desktop", + managed: false, + type: "desktop", + }), + }); +} + +async function writePrivateJsonExclusive(path: string, value: unknown): Promise { + const bytes = Buffer.from(`${JSON.stringify(value)}\n`, "utf8"); + if (bytes.byteLength < 2 || bytes.byteLength > MAX_BEEPER_CONFIG_BYTES) { + return fail("operation-private Beeper config exceeded its size bound"); + } + const handle = await open( + path, + constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, + PRIVATE_FILE_MODE, + ); + try { + await handle.chmod(PRIVATE_FILE_MODE); + let offset = 0; + while (offset < bytes.byteLength) { + const result = await handle.write(bytes, offset, bytes.byteLength - offset, offset); + if (result.bytesWritten === 0) return fail("operation-private Beeper config stopped accepting bytes"); + offset += result.bytesWritten; + } + await handle.sync(); + const metadata = await handle.stat(); + if ( + !metadata.isFile() + || metadata.uid !== process.getuid?.() + || metadata.nlink !== 1 + || (metadata.mode & 0o777) !== PRIVATE_FILE_MODE + || metadata.size !== bytes.byteLength + ) return fail("operation-private Beeper config was not written privately"); + } finally { + await handle.close(); + } + const metadata = await lstat(path); + if (metadata.isSymbolicLink() || metadata.nlink !== 1) { + return fail("operation-private Beeper config changed after creation"); + } +} + +async function createOperationPrivateBeeperStore( + root: string, + snapshot: BeeperCliStoreSnapshot, + accountId?: string, +): Promise { + await mkdir(root, { mode: PRIVATE_DIRECTORY_MODE }); + await chmod(root, PRIVATE_DIRECTORY_MODE); + const targets = join(root, "targets"); + await mkdir(targets, { mode: PRIVATE_DIRECTORY_MODE }); + await chmod(targets, PRIVATE_DIRECTORY_MODE); + await assertPrivateOwnedDirectory(root); + await assertPrivateOwnedDirectory(targets, root); + const config = Object.freeze({ + ...snapshot.config, + ...(accountId === undefined ? {} : { defaultAccount: accountId }), + }); + await writePrivateJsonExclusive(join(root, "config.json"), config); + await writePrivateJsonExclusive(join(targets, "desktop.json"), snapshot.desktopTarget); + try { + const path = await validateBeeperCliStore(root); + const metadata = await lstat(path); + return Object.freeze({ + path, + identity: Object.freeze({ device: metadata.dev, inode: metadata.ino }), + }); + } catch { + return fail("operation-private Beeper selector store did not validate"); + } +} + +async function validateAccountShard( + rawRoot: string, + canonicalWorking: string, + selected: BeeperAccountProjection, + baselineRealmDigest: string, + expectedSubject: string, + signal: AbortSignal | undefined, +): Promise { + throwIfAborted(signal); + await chmod(rawRoot, PRIVATE_DIRECTORY_MODE); + await assertPrivateOwnedDirectory(rawRoot, canonicalWorking); + const accounts = parseBeeperExportAccounts(await readOwnedJson( + join(rawRoot, "accounts.json"), + rawRoot, + MAX_ACCOUNTS_JSON_BYTES, + false, + signal, + )); + throwIfAborted(signal); + if (beeperSubjectFromAccounts(accounts) !== expectedSubject) { + return fail("official export account did not match the bound auth realm"); + } + assertOutputRealm(accounts, baselineRealmDigest); + const manifest = parseManifest(await readOwnedJson( + join(rawRoot, "manifest.json"), + rawRoot, + MAX_ACCOUNTS_JSON_BYTES, + false, + signal, + )); + const manifestAccounts = parseBeeperExportAccounts(manifest.accounts); + if ( + manifest.attachmentCount !== 0 + || canonicalJson(manifestAccounts) !== canonicalJson(accounts) + || outputRealmDigest(manifestAccounts) !== outputRealmDigest(accounts) + ) return fail("official export manifest did not prove a no-attachment account snapshot"); + const listedValues = array( + await readOwnedJson( + join(rawRoot, "chats.json"), + rawRoot, + MAX_CHATS_JSON_BYTES, + false, + signal, + ), + "official export chats", + MAX_EXPORT_CHATS, + ); + const listedChats: Array> = []; + for (const value of listedValues) { + throwIfAborted(signal); + const chat = parseBeeperExportConversation(value, accounts); + listedChats.push(Object.freeze({ id: chat.id, accountId: chat.accountId })); + } + if (manifest.chatCount !== listedChats.length) { + return fail("official export manifest chat count did not match chats.json"); + } + if (listedChats.some((chat) => chat.accountId !== selected.accountId)) { + return fail("official per-account export crossed its selected account boundary"); + } + await assertExactDirectoryEntries(rawRoot, [ + ...(listedChats.length === 0 ? [] : [".beeper-export-state.json"]), + "accounts.json", + "chats", + "chats.json", + "manifest.json", + ], "official export root"); + throwIfAborted(signal); + const chatSegments = listedChats.map((chat) => safeSegment(chat.id)); + if (new Set(chatSegments).size !== chatSegments.length) { + return fail("official export chat directory names collided"); + } + const chatsRoot = await assertOwnedDirectory(join(rawRoot, "chats"), rawRoot); + await assertExactDirectoryEntries(chatsRoot, chatSegments, "official export chats"); + const stateMessageCounts = listedChats.length === 0 + ? new Map() + : parseOfficialState(await readOwnedJson( + join(rawRoot, ".beeper-export-state.json"), + rawRoot, + MAX_CHATS_JSON_BYTES, + false, + signal, + ), listedChats.map((chat) => chat.id), manifest.createdAt); + let officialMessageCount = 0; + const chats: ValidatedShardChat[] = []; + for (const [index, listedChat] of listedChats.entries()) { + throwIfAborted(signal); + const segment = chatSegments[index]; + if (segment === undefined) return fail("official export chat segment disappeared"); + const directory = await assertOwnedDirectory(join(chatsRoot, segment), rawRoot); + await assertExactDirectoryEntries(directory, [ + "attachments", + "chat.json", + "messages.html", + "messages.json", + "messages.markdown", + ], "official export chat directory"); + const attachmentsDirectory = await assertOwnedDirectory( + join(directory, "attachments"), + rawRoot, + ); + await assertExactDirectoryEntries( + attachmentsDirectory, + [], + "official export attachments directory", + ); + const chatPath = join(directory, "chat.json"); + const chatDocument = await readOwnedJsonDocument( + chatPath, + rawRoot, + MAX_CHAT_JSON_BYTES, + false, + signal, + ); + const chat = parseBeeperExportConversation(chatDocument.value, accounts); + if ( + chat.id !== listedChat.id + || chat.accountId !== listedChat.accountId + || chat.accountId !== selected.accountId + ) return fail("official export chat detail crossed its selected account boundary"); + const expectedMessageCount = stateMessageCounts.get(chat.id); + if (expectedMessageCount === undefined) { + return fail("official export chat state disappeared"); + } + officialMessageCount += expectedMessageCount; + if (!Number.isSafeInteger(officialMessageCount)) { + return fail("official export message count overflowed"); + } + try { + await unlink(join(directory, "messages.markdown")); + await unlink(join(directory, "messages.html")); + } catch { + return fail("redundant official export renderings could not be removed safely"); + } + await assertExactDirectoryEntries(directory, [ + "attachments", + "chat.json", + "messages.json", + ], "sanitized official export chat directory"); + chats.push(Object.freeze({ + chat: Object.freeze({ id: chat.id, accountId: chat.accountId }), + root: rawRoot, + chatPath, + chatSha256: chatDocument.sha256, + messagesPath: join(directory, "messages.json"), + expectedMessageCount, + observedAt: manifest.completedAt, + })); + } + if (officialMessageCount !== manifest.messageCount) { + return fail("official export manifest message count did not match chat state"); + } + throwIfAborted(signal); + return Object.freeze({ + accountId: selected.accountId, + completedAt: manifest.completedAt, + chats: Object.freeze(chats), + messageCount: officialMessageCount, + }); +} + +async function prepassSelfAliases( + chats: readonly ValidatedShardChat[], + accounts: readonly BeeperAccountProjection[], + accountsById: ReadonlyMap, + maxMessagesJsonBytes: number, + maxEvidence: number, + signal: AbortSignal | undefined, + heartbeat: () => void, +): Promise { + const aliasesByAccount = new Map>(); + const peerEvidenceByAccount = new Map>(); + const evidenceIdsByAccount = new Map>(); + const coveredChats: ValidatedShardChat[] = []; + const messagesSha256ByPath = new Map(); + let evidenceCount = 0; + const evidenceSet = ( + collection: Map>, + accountId: string, + ): Set => { + let values = collection.get(accountId); + if (values === undefined) { + values = new Set(); + collection.set(accountId, values); + } + return values; + }; + for (const account of accounts) { + if (account.user.isSelf === false) { + return fail("Beeper account user contradicts its self identity anchor"); + } + const coordinate = digest([ + "beeper-self-alias-v1", + account.accountId, + account.user.id, + ]); + evidenceSet(aliasesByAccount, account.accountId).add(coordinate); + evidenceSet(evidenceIdsByAccount, account.accountId).add(coordinate); + evidenceCount += 1; + } + + let evidenceLimitReached = evidenceCount > maxEvidence; + for (const validated of chats) { + if (evidenceLimitReached) break; + heartbeat(); + throwIfAborted(signal); + const chatDocument = await readOwnedJsonDocument( + validated.chatPath, + validated.root, + MAX_CHAT_JSON_BYTES, + false, + signal, + ); + if (chatDocument.sha256 !== validated.chatSha256) { + return fail("official export chat changed before self-alias prepass"); + } + const chat = parseBeeperExportConversation(chatDocument.value, accounts); + if ( + chat.id !== validated.chat.id + || chat.accountId !== validated.chat.accountId + ) return fail("official export chat identity changed before self-alias prepass"); + const account = accountsById.get(chat.accountId); + if (account === undefined) return fail("official export chat references an unknown account"); + const positiveEvidence = new Set(); + const peerEvidence = new Set(); + for (const participant of chat.participants.items) { + const coordinate = digest([ + "beeper-self-alias-v1", + account.accountId, + participant.id, + ]); + if (participant.isSelf === true) positiveEvidence.add(coordinate); + if (participant.isSelf === false) peerEvidence.add(coordinate); + } + if (await ownedFileSize(validated.messagesPath, validated.root) > maxMessagesJsonBytes) { + coveredChats.push(validated); + continue; + } + const messagesDocument = await readOwnedJsonDocument( + validated.messagesPath, + validated.root, + maxMessagesJsonBytes, + false, + signal, + ); + const messages = parseBeeperExportMessages( + messagesDocument.value, + chat.accountId, + chat.id, + MAX_EXPORT_MESSAGES_PER_CHAT, + ); + if (messages.length !== validated.expectedMessageCount) { + return fail("official export chat messages did not match completed state"); + } + for (const message of messages) { + (message.isSender ? positiveEvidence : peerEvidence).add(digest([ + "beeper-self-alias-v1", + account.accountId, + message.senderId, + ])); + } + const accountEvidenceIds = evidenceSet(evidenceIdsByAccount, account.accountId); + let newEvidence = 0; + for (const sourceId of new Set([...positiveEvidence, ...peerEvidence])) { + if (!accountEvidenceIds.has(sourceId)) newEvidence += 1; + } + if (newEvidence > maxEvidence - evidenceCount) { + evidenceLimitReached = true; + break; + } + const aliases = evidenceSet(aliasesByAccount, account.accountId); + const peers = evidenceSet(peerEvidenceByAccount, account.accountId); + for (const sourceId of positiveEvidence) aliases.add(sourceId); + for (const sourceId of peerEvidence) peers.add(sourceId); + for (const sourceId of new Set([...positiveEvidence, ...peerEvidence])) { + accountEvidenceIds.add(sourceId); + } + evidenceCount += newEvidence; + messagesSha256ByPath.set(validated.messagesPath, messagesDocument.sha256); + coveredChats.push(validated); + } + + for (const [accountId, peerEvidence] of peerEvidenceByAccount) { + const aliases = aliasesByAccount.get(accountId); + if (aliases === undefined) return fail("Beeper account self-alias set disappeared"); + for (const sourceId of peerEvidence) { + if (aliases.has(sourceId)) { + return fail("official export has peer evidence for an account self alias"); + } + } + } + return Object.freeze({ + aliasesByAccount, + coveredChats: Object.freeze(coveredChats), + evidenceLimitReached, + messagesSha256ByPath, + }); +} + +function canonicalParticipantSourceId( + account: BeeperAccountProjection, + sourceId: string, + aliasesByAccount: ReadonlyMap>, +): string { + const aliases = aliasesByAccount.get(account.accountId); + if (aliases === undefined) return fail("Beeper account self-alias set disappeared"); + const coordinate = digest(["beeper-self-alias-v1", account.accountId, sourceId]); + return aliases.has(coordinate) ? account.user.id : sourceId; +} + function upsertParticipant( facts: Map, account: BeeperAccountProjection, user: Pick, self: boolean | null, + aliasesByAccount: ReadonlyMap>, createdIds?: Set, ): ParticipantFact { - const id = localId("participant", account.accountId, user.id); + const sourceId = canonicalParticipantSourceId(account, user.id, aliasesByAccount); + const id = localId("participant", account.accountId, sourceId); const current = facts.get(id); const handle = user.phoneNumber ?? user.email ?? user.username; if (current !== undefined) { @@ -633,7 +2020,7 @@ function upsertParticipant( const created: ParticipantFact = { id, accountId: localId("account", account.accountId), - providerId: providerId("participant", account.accountId, user.id), + providerId: providerId("participant", account.accountId, sourceId), displayName: user.fullName, handle, isSelf: self, @@ -797,6 +2184,7 @@ function messageRecord( scan: ConversationScan, messageIds: ReadonlySet, account: BeeperAccountProjection, + aliasesByAccount: ReadonlyMap>, network: string, observedAt: string, ): BeeperMessageLikeMeMessage { @@ -840,7 +2228,11 @@ function messageRecord( connectedAccountProviderId: providerId("account", account.accountId), }), conversationId: localId("conversation", account.accountId, scan.chat.id), - senderParticipantId: localId("participant", account.accountId, message.senderId), + senderParticipantId: localId( + "participant", + account.accountId, + canonicalParticipantSourceId(account, message.senderId, aliasesByAccount), + ), direction: message.isSender ? "outgoing" : "incoming", sentAt: message.timestamp, sortKey: message.sortKey, @@ -876,15 +2268,23 @@ function reactionRecord( message: BeeperMessageProjection, scan: ConversationScan, account: BeeperAccountProjection, + aliasesByAccount: ReadonlyMap>, network: string, observedAt: string, ): BeeperMessageLikeMeReaction { + const participantSourceId = canonicalParticipantSourceId( + account, + reaction.participantId, + aliasesByAccount, + ); const id = localId( "reaction", account.accountId, scan.chat.id, message.id, reaction.id, + reaction.participantId, + reaction.reactionKey, ); return Object.freeze({ schemaVersion: 1, @@ -899,8 +2299,10 @@ function reactionRecord( scan.chat.id, message.id, reaction.id, + reaction.participantId, + reaction.reactionKey, ), - providerRevision: reaction.id, + providerRevision: null, observedAt, connectedAccountProviderId: providerId("account", account.accountId), }), @@ -911,7 +2313,7 @@ function reactionRecord( scan.chat.id, message.id, ), - participantId: localId("participant", account.accountId, reaction.participantId), + participantId: localId("participant", account.accountId, participantSourceId), body: reactionBody(reaction.reactionKey), reactedAt: null, state: "active", @@ -971,10 +2373,29 @@ async function assertExactDirectoryEntries( ) fail(`${label} contained an unexpected file layout`); } +async function removePrivateOwnedDirectory( + path: string, + root: string, + expected: PrivateDirectoryIdentity, +): Promise { + if (path !== root && !path.startsWith(`${root}${sep}`)) { + return fail("private export cleanup escaped its owned root"); + } + try { + removePrivateDirectoryTree(path, Object.freeze({ + device: String(expected.device), + inode: String(expected.inode), + })); + } catch { + return fail("private export directory could not be removed from quarantine safely"); + } +} + /** - * Creates a single-use source for the private bundle sink. The official CLI - * performs its own complete local pagination. Its duplicate transcript files - * remain inside one private staging root and are removed before completion. + * Creates a single-use source for the private bundle sink. The pinned official + * CLI paginates one operation-private account shard at a time. Wrench validates + * every completed shard, reports only ordinal progress, and projects all shards + * in one deterministic global conversation order. */ export function createBeeperMessageLikeMeSource( request: BeeperMessageLikeMeSourceRequest, @@ -983,11 +2404,24 @@ export function createBeeperMessageLikeMeSource( const limits = parseLimits(request.limits); let consumed = false; let completion: unknown; + let disposeWorking: (() => Promise) | undefined; + let disposed = false; + let disposalInFlight: Promise | undefined; + let progressHeartbeat: ReturnType | undefined; + let progressHeartbeatFailed = false; + const stopProgressHeartbeat = (): void => { + if (progressHeartbeat !== undefined) clearInterval(progressHeartbeat); + progressHeartbeat = undefined; + }; + const assertProgressHeartbeat = (): void => { + if (progressHeartbeatFailed) return fail("export progress reporting failed"); + }; const records = (async function* (): AsyncGenerator { if (consumed) return fail("record stream is single-use"); consumed = true; throwIfAborted(request.signal); + request.onProgress?.(Object.freeze({ phase: "preparing" })); const configDirectory = await validateBeeperCliStore(auth.path); const environment = request.environment ?? process.env; const binary = request.dependencies?.binaryPath @@ -1021,114 +2455,339 @@ export function createBeeperMessageLikeMeSource( } const createWorking = customCreateWorking ?? (() => mkdtemp(join(tmpdir(), "wrench-beeper-message-like-me-"))); - const removeWorking = customRemoveWorking - ?? ((path: string) => rm(path, { recursive: true, force: true })); const working = await createWorking(); - if (!isAbsolute(working)) return fail("working directory must be absolute"); - await chmod(working, PRIVATE_DIRECTORY_MODE); - const canonicalWorking = await assertPrivateOwnedDirectory(await realpath(working)); - const rawRoot = resolve(canonicalWorking, "official-export"); + let canonicalWorking: string; + try { + if (!isAbsolute(working)) return fail("working directory must be absolute"); + await chmod(working, PRIVATE_DIRECTORY_MODE); + canonicalWorking = await assertPrivateOwnedDirectory(await realpath(working)); + } catch (error) { + if (customCreateWorking === undefined && isAbsolute(working)) { + try { + await rmdir(working); + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + "Beeper export working-directory setup and cleanup both failed", + ); + } + } + throw error; + } + const workingMetadata = await lstat(canonicalWorking); + const workingIdentity = Object.freeze({ + device: workingMetadata.dev, + inode: workingMetadata.ino, + }); + let workingLease: BeeperMessageLikeMeDirectoryLease | undefined; + if (customCreateWorking === undefined) { + try { + workingLease = await createBeeperMessageLikeMeDirectoryLease({ + role: "raw-working", + path: canonicalWorking, + recoverAfterMs: + Date.now() + limits.timeoutMs + RAW_WORKING_RECOVERY_GRACE_MS, + environment, + }); + } catch (error) { + try { + await removePrivateOwnedDirectory( + canonicalWorking, + canonicalWorking, + workingIdentity, + ); + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + "Beeper export recovery setup and empty-directory cleanup both failed", + ); + } + throw error; + } + } + disposeWorking = async () => { + if (customRemoveWorking === undefined) { + await removePrivateOwnedDirectory( + canonicalWorking, + canonicalWorking, + workingIdentity, + ); + if (workingLease !== undefined) { + releaseBeeperMessageLikeMeDirectoryLease(workingLease); + } + return; + } + const current = await lstat(canonicalWorking); + if ( + !current.isDirectory() + || current.isSymbolicLink() + || current.uid !== process.getuid?.() + || (current.mode & 0o777) !== PRIVATE_DIRECTORY_MODE + || current.dev !== workingIdentity.device + || current.ino !== workingIdentity.inode + || await realpath(canonicalWorking) !== canonicalWorking + ) return fail("private export working directory changed before cleanup"); + await customRemoveWorking(canonicalWorking); + try { + await lstat(canonicalWorking); + } catch (error) { + if (isErrno(error, "ENOENT")) return; + throw error; + } + return fail("private export working directory survived cleanup"); + }; const cacheDirectory = resolve(canonicalWorking, "cli-payload-cache"); - await mkdir(rawRoot, { mode: PRIVATE_DIRECTORY_MODE }); - await chmod(rawRoot, PRIVATE_DIRECTORY_MODE); + const shardsDirectory = resolve(canonicalWorking, "account-shards"); + const selectorsDirectory = resolve(canonicalWorking, "account-selectors"); await mkdir(cacheDirectory, { mode: PRIVATE_DIRECTORY_MODE }); await chmod(cacheDirectory, PRIVATE_DIRECTORY_MODE); - await assertPrivateOwnedDirectory(rawRoot, canonicalWorking); + await mkdir(shardsDirectory, { mode: PRIVATE_DIRECTORY_MODE }); + await chmod(shardsDirectory, PRIVATE_DIRECTORY_MODE); + await mkdir(selectorsDirectory, { mode: PRIVATE_DIRECTORY_MODE }); + await chmod(selectorsDirectory, PRIVATE_DIRECTORY_MODE); await assertPrivateOwnedDirectory(cacheDirectory, canonicalWorking); - try { - const arguments_ = planBeeperMessageLikeMeExportCommand({ - outputDirectory: rawRoot, - limitChats: limits.limitChats, - limitMessages: limits.limitMessages, - maxParticipants: limits.maxParticipants, - }, limits.timeoutMs); - const run = request.dependencies?.runExport ?? runExportCli; - const result = await run({ + await assertPrivateOwnedDirectory(shardsDirectory, canonicalWorking); + await assertPrivateOwnedDirectory(selectorsDirectory, canonicalWorking); + { + const run = request.dependencies?.runCli ?? runExportCli; + const deadlineMs = Date.now() + limits.timeoutMs; + const storeSnapshot = await readCliStoreSnapshot(configDirectory); + const inventoryStore = await createOperationPrivateBeeperStore( + resolve(selectorsDirectory, "inventory"), + storeSnapshot, + ); + const baseEnvironment = environmentForExport(inventoryStore.path, cacheDirectory); + const accounts = await enumerateAccounts( binary, - arguments: arguments_, - environment: environmentForExport(configDirectory, cacheDirectory), - timeoutMs: limits.timeoutMs, - maxOutputBytes: MAX_STDOUT_BYTES, - maxStderrBytes: MAX_STDERR_BYTES, - ...(request.signal === undefined ? {} : { signal: request.signal }), - }); - throwIfAborted(request.signal); - if (result.exitCode !== 0 || result.stderr.trim().length !== 0) { - return fail("official read-only export failed"); - } - // Official v0.6.2 creates nested 0755/0644 entries. The exact 0700 root - // is the privacy boundary; nested entries must still be owned, physical, - // non-writable by others, and regular files must have one link. - await assertPrivateOwnedDirectory(rawRoot, canonicalWorking); - await assertExactDirectoryEntries(rawRoot, [ - ".beeper-export-state.json", - "accounts.json", - "chats", - "chats.json", - "manifest.json", - ], "official export root"); - - const accountsValue = await readOwnedJson( - join(rawRoot, "accounts.json"), - rawRoot, - MAX_ACCOUNTS_JSON_BYTES, + baseEnvironment, + deadlineMs, + run, + workingLease, + request.signal, + (elapsedSeconds) => request.onProgress?.(Object.freeze({ + phase: "accounts-progress", + stage: "discovering", + elapsedSeconds, + })), ); - const accounts = parseBeeperExportAccounts(accountsValue); if (beeperSubjectFromAccounts(accounts) !== auth.subject) { return fail("official export account did not match the bound auth realm"); } - const accountsById = new Map(accounts.map((account) => [account.accountId, account])); - const manifest = parseManifest(await readOwnedJson( - join(rawRoot, "manifest.json"), - rawRoot, - MAX_ACCOUNTS_JSON_BYTES, - )); - const manifestAccounts = parseBeeperExportAccounts(manifest.accounts); - if ( - manifest.attachmentCount !== 0 - || canonicalJson(manifestAccounts) !== canonicalJson(accounts) - ) return fail("official export manifest did not prove a no-attachment account snapshot"); - - const listedValues = array( - await readOwnedJson( - join(rawRoot, "chats.json"), - rawRoot, - MAX_CHATS_JSON_BYTES, - ), - "official export chats", - MAX_EXPORT_CHATS, - ); - const listedChats = listedValues.map((value) => - parseBeeperExportConversation(value, accounts)); - if (manifest.chatCount !== listedChats.length) { - return fail("official export manifest chat count did not match chats.json"); - } - const chatSegments = listedChats.map((chat) => safeSegment(chat.id)); - if (new Set(chatSegments).size !== chatSegments.length) { - return fail("official export chat directory names collided"); + const baselineRealmDigest = outputRealmDigest(accounts); + const orderedAccounts = [...accounts].sort((left, right) => { + const leftKey = digest([left.accountId]); + const rightKey = digest([right.accountId]); + return leftKey.localeCompare(rightKey) || left.accountId.localeCompare(right.accountId); + }); + for (const account of orderedAccounts) { + assertUniqueOfficialAccountSelector(account, accounts); } - const chatsRoot = await assertOwnedDirectory(join(rawRoot, "chats"), rawRoot); - await assertExactDirectoryEntries(chatsRoot, chatSegments, "official export chats"); - const stateMessageCounts = parseOfficialState(await readOwnedJson( - join(rawRoot, ".beeper-export-state.json"), - rawRoot, - MAX_CHATS_JSON_BYTES, - ), listedChats.map((chat) => chat.id), manifest.createdAt); - let officialMessageCount = 0; - for (const count of stateMessageCounts.values()) { - officialMessageCount += count; - if (!Number.isSafeInteger(officialMessageCount)) { - return fail("official export message count overflowed"); + request.onProgress?.(Object.freeze({ + phase: "accounts-discovered", + accounts: orderedAccounts.length, + })); + + const shards: ValidatedAccountShard[] = []; + const accountObservedAt = new Map(); + const effectiveChatLimit = limits.limitChats ?? MAX_EXPORT_CHATS; + const effectiveMessageLimit = limits.limitMessages + ?? MAX_EXPORT_MESSAGES_PER_CHAT; + let remainingChats = effectiveChatLimit; + let cumulativeChats = 0; + let cumulativeMessages = 0; + for (const [index, account] of orderedAccounts.entries()) { + throwIfAborted(request.signal); + const ordinal = index + 1; + if (remainingChats === 0) { + request.onProgress?.(Object.freeze({ + phase: "account-skipped", + account: ordinal, + accounts: orderedAccounts.length, + reason: "chat-limit-reached", + })); + continue; + } + const ordinalSegment = `account-${String(ordinal).padStart(3, "0")}`; + const selectorRoot = resolve(selectorsDirectory, ordinalSegment); + const shardRoot = resolve(shardsDirectory, ordinalSegment); + const selectorStore = await createOperationPrivateBeeperStore( + selectorRoot, + storeSnapshot, + account.accountId, + ); + await mkdir(shardRoot, { mode: PRIVATE_DIRECTORY_MODE }); + await chmod(shardRoot, PRIVATE_DIRECTORY_MODE); + await assertPrivateOwnedDirectory(shardRoot, canonicalWorking); + request.onProgress?.(Object.freeze({ + phase: "account-started", + account: ordinal, + accounts: orderedAccounts.length, + })); + const timeoutMs = remainingTimeoutMs(deadlineMs); + const arguments_ = planBeeperMessageLikeMeExportCommand({ + outputDirectory: shardRoot, + limitChats: remainingChats, + limitMessages: effectiveMessageLimit, + maxParticipants: limits.maxParticipants, + }, timeoutMs); + const result = await run({ + binary, + arguments: arguments_, + environment: environmentForExport(selectorStore.path, cacheDirectory), + timeoutMs, + maxOutputBytes: MAX_STDOUT_BYTES, + maxStderrBytes: MAX_STDERR_BYTES, + workingRoot: canonicalWorking, + maxWorkingBytes: MAX_RAW_WORKING_BYTES, + ...(workingLease === undefined ? {} : { directoryLease: workingLease }), + onHeartbeat: (elapsedSeconds) => request.onProgress?.(Object.freeze({ + phase: "account-progress", + account: ordinal, + accounts: orderedAccounts.length, + elapsedSeconds, + })), + ...(request.signal === undefined ? {} : { signal: request.signal }), + }); + throwIfAborted(request.signal); + if (result.exitCode !== 0 || result.stderr.trim().length !== 0) { + return fail("official read-only export failed"); + } + request.onProgress?.(Object.freeze({ + phase: "account-validating", + account: ordinal, + accounts: orderedAccounts.length, + elapsedSeconds: 0, + })); + const shard = await withProgressHeartbeat( + () => validateAccountShard( + shardRoot, + canonicalWorking, + account, + baselineRealmDigest, + auth.subject, + request.signal, + ), + request.onProgress === undefined + ? undefined + : (elapsedSeconds) => request.onProgress?.(Object.freeze({ + phase: "account-validating", + account: ordinal, + accounts: orderedAccounts.length, + elapsedSeconds, + })), + ); + await removePrivateOwnedDirectory( + selectorRoot, + canonicalWorking, + selectorStore.identity, + ); + shards.push(shard); + accountObservedAt.set(account.accountId, shard.completedAt); + if (shard.chats.length > remainingChats) { + return fail("official per-account export exceeded its allocated global chat budget"); } + cumulativeChats += shard.chats.length; + cumulativeMessages += shard.messageCount; + if ( + !Number.isSafeInteger(cumulativeChats) + || !Number.isSafeInteger(cumulativeMessages) + ) return fail("official export aggregate counts overflowed"); + remainingChats -= shard.chats.length; + request.onProgress?.(Object.freeze({ + phase: "account-completed", + account: ordinal, + accounts: orderedAccounts.length, + chats: cumulativeChats, + messages: cumulativeMessages, + })); + } + request.onProgress?.(Object.freeze({ + phase: "accounts-verifying", + accounts: orderedAccounts.length, + })); + const finalAccounts = await enumerateAccounts( + binary, + baseEnvironment, + deadlineMs, + run, + workingLease, + request.signal, + (elapsedSeconds) => request.onProgress?.(Object.freeze({ + phase: "accounts-progress", + stage: "verifying", + elapsedSeconds, + })), + ); + if (beeperSubjectFromAccounts(finalAccounts) !== auth.subject) { + return fail("official export account did not match the bound auth realm"); + } + assertOutputRealm(finalAccounts, baselineRealmDigest); + await removePrivateOwnedDirectory( + resolve(selectorsDirectory, "inventory"), + canonicalWorking, + inventoryStore.identity, + ); + const finalObservedAt = new Date().toISOString(); + for (const account of accounts) { + accountObservedAt.set( + account.accountId, + accountObservedAt.get(account.accountId) ?? finalObservedAt, + ); } - if (officialMessageCount !== manifest.messageCount) { - return fail("official export manifest message count did not match chat state"); + const observedAtForAccount = (accountId: string): string => { + const value = accountObservedAt.get(accountId); + if (value === undefined) return fail("Beeper account observation time disappeared"); + return value; + }; + const validatedChats = shards.flatMap((shard) => shard.chats); + request.onProgress?.(Object.freeze({ + phase: "conversion-started", + accounts: orderedAccounts.length, + chats: cumulativeChats, + messages: cumulativeMessages, + })); + if (request.onProgress !== undefined) { + const conversionStartedAt = Date.now(); + progressHeartbeat = setInterval(() => { + try { + request.onProgress?.(Object.freeze({ + phase: "conversion-progress", + elapsedSeconds: Math.max( + 1, + Math.floor((Date.now() - conversionStartedAt) / 1_000), + ), + })); + } catch { + progressHeartbeatFailed = true; + } + }, ACCOUNT_HEARTBEAT_INTERVAL_MS); } + const accountsById = new Map(accounts.map((account) => [account.accountId, account])); + const allListedChatEntries = [...validatedChats].sort((left, right) => + localId("conversation", left.chat.accountId, left.chat.id).localeCompare( + localId("conversation", right.chat.accountId, right.chat.id), + )); + const selfAliasPrepass = await prepassSelfAliases( + allListedChatEntries, + accounts, + accountsById, + maxMessagesJsonBytes, + maxBundleRecords, + request.signal, + assertProgressHeartbeat, + ); + const aliasesByAccount = selfAliasPrepass.aliasesByAccount; const participantFacts = new Map(); const selfParticipantByAccount = new Map(); for (const account of accounts) { - const self = upsertParticipant(participantFacts, account, account.user, true); + const self = upsertParticipant( + participantFacts, + account, + account.user, + true, + aliasesByAccount, + ); selfParticipantByAccount.set(account.accountId, self.id); } @@ -1140,65 +2799,73 @@ export function createBeeperMessageLikeMeSource( let oversizedChatSkipped = false; let scannedRecordCount = accounts.length + selfParticipantByAccount.size; let scanRecordBudgetExhausted = false; - const listedChatEntries = listedChats.map((chat, index) => { - const segment = chatSegments[index]; - if (segment === undefined) return fail("official export chat segment disappeared"); - return Object.freeze({ chat, segment }); - }).sort((left, right) => - localId("conversation", left.chat.accountId, left.chat.id).localeCompare( - localId("conversation", right.chat.accountId, right.chat.id), - )); - for (const { chat: listedChat, segment } of listedChatEntries) { + const listedChatEntries = selfAliasPrepass.coveredChats; + for (const { + chat: validatedChat, + root: shardRoot, + chatPath, + chatSha256, + messagesPath, + expectedMessageCount, + observedAt, + } of listedChatEntries) { + assertProgressHeartbeat(); throwIfAborted(request.signal); - const directory = await assertOwnedDirectory(join(chatsRoot, segment), rawRoot); - await assertExactDirectoryEntries(directory, [ - "attachments", - "chat.json", - "messages.html", - "messages.json", - "messages.markdown", - ], "official export chat directory"); - const attachmentsDirectory = await assertOwnedDirectory( - join(directory, "attachments"), - rawRoot, - ); - await assertExactDirectoryEntries( - attachmentsDirectory, - [], - "official export attachments directory", - ); - const chat = parseBeeperExportConversation(await readOwnedJson( - join(directory, "chat.json"), - rawRoot, + const chatDocument = await readOwnedJsonDocument( + chatPath, + shardRoot, MAX_CHAT_JSON_BYTES, - ), accounts); - if (chat.id !== listedChat.id || chat.accountId !== listedChat.accountId) { - return fail("official export chat detail did not match chats.json"); + false, + request.signal, + ); + if (chatDocument.sha256 !== chatSha256) { + return fail("official export chat changed between validated passes"); } + const chat = parseBeeperExportConversation(chatDocument.value, accounts); + if ( + chat.id !== validatedChat.id + || chat.accountId !== validatedChat.accountId + ) return fail("official export chat identity changed between validated passes"); const account = accountsById.get(chat.accountId); if (account === undefined) return fail("official export chat references an unknown account"); - const expectedMessageCount = stateMessageCounts.get(chat.id); - if (expectedMessageCount === undefined) { - return fail("official export chat state disappeared"); - } if ( - limits.limitMessages !== null - && expectedMessageCount >= limits.limitMessages + expectedMessageCount >= effectiveMessageLimit ) messageLimitReached = true; - const messagesPath = join(directory, "messages.json"); - if (await ownedFileSize(messagesPath, rawRoot) > maxMessagesJsonBytes) { + const prepassMessagesSha256 = selfAliasPrepass.messagesSha256ByPath.get(messagesPath); + if (await ownedFileSize(messagesPath, shardRoot) > maxMessagesJsonBytes) { + if (prepassMessagesSha256 !== undefined) { + return fail("official export messages changed after self-alias prepass"); + } oversizedChatSkipped = true; - await unlink(join(directory, "messages.markdown")); - await unlink(join(directory, "messages.html")); continue; } + if (prepassMessagesSha256 === undefined) { + return fail("official export self-alias message proof disappeared"); + } + const messagesDocument = await readOwnedJsonDocument( + messagesPath, + shardRoot, + maxMessagesJsonBytes, + false, + request.signal, + ); + if (messagesDocument.sha256 !== prepassMessagesSha256) { + return fail("official export messages changed after self-alias prepass"); + } + const messages = parseBeeperExportMessages( + messagesDocument.value, + chat.accountId, + chat.id, + MAX_EXPORT_MESSAGES_PER_CHAT, + ); + if (messages.length !== expectedMessageCount) { + return fail("official export chat messages did not match completed state"); + } if ( scanRecordBudgetExhausted || expectedMessageCount + 1 > maxBundleRecords - scannedRecordCount ) { scanRecordBudgetExhausted = true; - await unlink(join(directory, "messages.markdown")); - await unlink(join(directory, "messages.html")); continue; } const newlyCreatedParticipantIds = new Set(); @@ -1209,6 +2876,7 @@ export function createBeeperMessageLikeMeSource( account, participant, participant.isSelf, + aliasesByAccount, newlyCreatedParticipantIds, ).id); } @@ -1217,20 +2885,19 @@ export function createBeeperMessageLikeMeSource( if (self === undefined) return fail("Beeper account self participant disappeared"); participantIds.add(self); } - const messages = parseBeeperExportMessages( - await readOwnedJson(messagesPath, rawRoot, maxMessagesJsonBytes), - chat.accountId, - chat.id, - MAX_EXPORT_MESSAGES_PER_CHAT, - ); - if (messages.length !== expectedMessageCount) { - return fail("official export chat messages did not match completed state"); - } const messageIds = new Set(messages.map((message) => message.id)); const reactionCount = messages.reduce( (count, message) => count + message.reactions.length, 0, ); + const reactionProviderIdNonUniqueGroups = messages.reduce( + (count, message) => count + new Set( + message.reactions + .filter((reaction) => reaction.providerIdNonUnique) + .map((reaction) => reaction.id), + ).size, + 0, + ); const tombstoneCount = messages.reduce( (count, message) => count + (message.isDeleted || message.isHidden ? 1 : 0), 0, @@ -1246,7 +2913,7 @@ export function createBeeperMessageLikeMeSource( phoneNumber: null, email: null, username: null, - }, message.isSender, newlyCreatedParticipantIds).id); + }, message.isSender, aliasesByAccount, newlyCreatedParticipantIds).id); for (const reaction of message.reactions) { participantIds.add(upsertParticipant(participantFacts, account, { id: reaction.participantId, @@ -1254,7 +2921,7 @@ export function createBeeperMessageLikeMeSource( phoneNumber: null, email: null, username: null, - }, null, newlyCreatedParticipantIds).id); + }, null, aliasesByAccount, newlyCreatedParticipantIds).id); } } const scanRecordCount = 1 @@ -1267,8 +2934,6 @@ export function createBeeperMessageLikeMeSource( participantFacts.delete(participantId); } scanRecordBudgetExhausted = true; - await unlink(join(directory, "messages.markdown")); - await unlink(join(directory, "messages.html")); continue; } scannedRecordCount += scanRecordCount; @@ -1297,21 +2962,30 @@ export function createBeeperMessageLikeMeSource( && directRosterComplete; if (!participantsComplete) participantRosterIncomplete = true; const scanDraft: ConversationScan = Object.freeze({ - chat, - directory, + chat: Object.freeze({ + id: chat.id, + accountId: chat.accountId, + lastActivity: chat.lastActivity, + title: chat.title, + type: chat.type, + }), + root: shardRoot, messagesPath, + messagesSha256: messagesDocument.sha256, + observedAt, participantIds: Object.freeze([...participantIds].sort()), participantsComplete, startedAt: range.first, lastMessageAt: range.last, messageCount: messages.length, reactionCount, + reactionProviderIdNonUniqueGroups, tombstoneCount, nonParticipantRecordBytes: 0, }); const network = normalizeNetwork(account.network, account.bridge.type); let nonParticipantRecordBytes = bundleRecordBytes( - conversationRecord(scanDraft, account, network, manifest.completedAt), + conversationRecord(scanDraft, account, network, observedAt), ); for (const message of messages) { nonParticipantRecordBytes += bundleRecordBytes( @@ -1320,8 +2994,9 @@ export function createBeeperMessageLikeMeSource( scanDraft, messageIds, account, + aliasesByAccount, network, - manifest.completedAt, + observedAt, ), ); for (const reaction of message.reactions) { @@ -1330,8 +3005,9 @@ export function createBeeperMessageLikeMeSource( message, scanDraft, account, + aliasesByAccount, network, - manifest.completedAt, + observedAt, )); } const tombstone = tombstoneRecord( @@ -1339,7 +3015,7 @@ export function createBeeperMessageLikeMeSource( scanDraft, account, network, - manifest.completedAt, + observedAt, ); if (tombstone !== null) { nonParticipantRecordBytes += bundleRecordBytes(tombstone); @@ -1352,9 +3028,6 @@ export function createBeeperMessageLikeMeSource( ...scanDraft, nonParticipantRecordBytes, })); - // These plaintext renderings are redundant after strict JSON conversion. - await unlink(join(directory, "messages.markdown")); - await unlink(join(directory, "messages.html")); } const orderedScans = [...scans].sort((left, right) => localId("conversation", left.chat.accountId, left.chat.id).localeCompare( @@ -1373,7 +3046,7 @@ export function createBeeperMessageLikeMeSource( selectedRecordBytes += bundleRecordBytes(accountRecord( account, network, - manifest.completedAt, + observedAtForAccount(account.accountId), selfParticipantId, )); const selfFact = participantFacts.get(selfParticipantId); @@ -1382,7 +3055,7 @@ export function createBeeperMessageLikeMeSource( selfFact, account, network, - manifest.completedAt, + observedAtForAccount(account.accountId), )); } let bundleRecordLimitReached = scanRecordBudgetExhausted; @@ -1404,7 +3077,7 @@ export function createBeeperMessageLikeMeSource( fact, account, normalizeNetwork(account.network, account.bridge.type), - manifest.completedAt, + scan.observedAt, )); } const scanRecords = 1 @@ -1442,6 +3115,9 @@ export function createBeeperMessageLikeMeSource( participantRosterIncomplete = selectedScans.some( (scan) => !scan.participantsComplete, ); + const validatedReactionProviderIdNonUnique = selectedScans.some( + (scan) => scan.reactionProviderIdNonUniqueGroups > 0, + ); for (const account of [...accounts].sort((left, right) => left.accountId.localeCompare(right.accountId))) { @@ -1450,7 +3126,7 @@ export function createBeeperMessageLikeMeSource( yield accountRecord( account, normalizeNetwork(account.network, account.bridge.type), - manifest.completedAt, + observedAtForAccount(account.accountId), selfParticipantId, ); } @@ -1464,24 +3140,55 @@ export function createBeeperMessageLikeMeSource( fact, account, normalizeNetwork(account.network, account.bridge.type), - manifest.completedAt, + observedAtForAccount(account.accountId), ); } + let emittedReactionProviderIdNonUnique = false; for (const scan of selectedScans) { + assertProgressHeartbeat(); const account = accountsById.get(scan.chat.accountId); if (account === undefined) return fail("conversation account disappeared"); const network = normalizeNetwork(account.network, account.bridge.type); - yield conversationRecord(scan, account, network, manifest.completedAt); + yield conversationRecord(scan, account, network, scan.observedAt); + const messagesDocument = await readOwnedJsonDocument( + scan.messagesPath, + scan.root, + maxMessagesJsonBytes, + false, + request.signal, + ); + if (messagesDocument.sha256 !== scan.messagesSha256) { + return fail("official export messages changed between validated passes"); + } const messages = parseBeeperExportMessages( - await readOwnedJson( - scan.messagesPath, - rawRoot, - maxMessagesJsonBytes, - ), + messagesDocument.value, scan.chat.accountId, scan.chat.id, MAX_EXPORT_MESSAGES_PER_CHAT, ); + if (messages.length !== scan.messageCount) { + return fail("official export message count changed between validated passes"); + } + const reactionCount = messages.reduce( + (count, message) => count + message.reactions.length, + 0, + ); + const reactionProviderIdNonUniqueGroups = messages.reduce( + (count, message) => count + new Set( + message.reactions + .filter((reaction) => reaction.providerIdNonUnique) + .map((reaction) => reaction.id), + ).size, + 0, + ); + if ( + reactionCount !== scan.reactionCount + || reactionProviderIdNonUniqueGroups + !== scan.reactionProviderIdNonUniqueGroups + ) { + return fail("official export reaction identity changed between validated passes"); + } + emittedReactionProviderIdNonUnique ||= reactionProviderIdNonUniqueGroups > 0; const messageIds = new Set(messages.map((message) => message.id)); for (const message of messages) { yield messageRecord( @@ -1489,8 +3196,9 @@ export function createBeeperMessageLikeMeSource( scan, messageIds, account, + aliasesByAccount, network, - manifest.completedAt, + scan.observedAt, ); for (const reaction of message.reactions) { yield reactionRecord( @@ -1498,8 +3206,9 @@ export function createBeeperMessageLikeMeSource( message, scan, account, + aliasesByAccount, network, - manifest.completedAt, + scan.observedAt, ); } const tombstone = tombstoneRecord( @@ -1507,30 +3216,48 @@ export function createBeeperMessageLikeMeSource( scan, account, network, - manifest.completedAt, + scan.observedAt, ); if (tombstone !== null) yield tombstone; } } + if ( + emittedReactionProviderIdNonUnique + !== validatedReactionProviderIdNonUnique + ) return fail("official export reaction identity changed between validated passes"); const warnings = new Set([ "attachments-metadata-only", "remote-history-not-claimed", "connected-account-backfill-coverage-unknown", + "sequential-account-snapshot", ]); - const chatLimitReached = limits.limitChats !== null - && listedChats.length >= limits.limitChats; + const chatLimitReached = cumulativeChats >= effectiveChatLimit; + const hardChatLimitReached = chatLimitReached && limits.limitChats === null; + const hardMessageLimitReached = messageLimitReached + && limits.limitMessages === null; + const hardSourceLimitReached = hardChatLimitReached + || hardMessageLimitReached; if (chatLimitReached) warnings.add("chat-limit-reached"); if (messageLimitReached) warnings.add("message-limit-reached"); if (participantRosterIncomplete) warnings.add("participant-roster-incomplete"); if (oversizedChatSkipped) warnings.add("oversized-chat-skipped"); if (bundleRecordLimitReached) warnings.add("bundle-record-limit-reached"); if (bundleByteLimitReached) warnings.add("bundle-byte-limit-reached"); + if (selfAliasPrepass.evidenceLimitReached) { + warnings.add("self-alias-evidence-limit-reached"); + } + if (emittedReactionProviderIdNonUnique) { + warnings.add("reaction-provider-id-non-unique"); + } const truncated = chatLimitReached || messageLimitReached || oversizedChatSkipped || bundleRecordLimitReached - || bundleByteLimitReached; + || bundleByteLimitReached + || selfAliasPrepass.evidenceLimitReached; + stopProgressHeartbeat(); + assertProgressHeartbeat(); completion = Object.freeze({ completeness: Object.freeze({ kind: truncated ? "truncated" : "bounded-local", @@ -1538,24 +3265,26 @@ export function createBeeperMessageLikeMeSource( ? "bundle-record-limit" : bundleByteLimitReached ? "bundle-byte-limit" - : oversizedChatSkipped - ? "oversized-chat" - : truncated - ? "explicit-source-limit" - : "desktop-local-export", + : selfAliasPrepass.evidenceLimitReached + ? "self-alias-evidence-limit" + : oversizedChatSkipped + ? "oversized-chat" + : hardSourceLimitReached + ? "source-hard-limit" + : truncated + ? "explicit-source-limit" + : "desktop-local-sequential-export", observedFrom, observedThrough, }), warnings: Object.freeze([...warnings].sort()), }); - } finally { - await removeWorking(canonicalWorking); } })(); return Object.freeze({ descriptor: Object.freeze({ - source: Object.freeze({ id: "beeper-local", version: "1.0.0" }), + source: Object.freeze({ id: "beeper-local", version: "1.1.0" }), provider: Object.freeze({ id: "beeper", version: BEEPER_CLI_PIN.version }), }), records, @@ -1563,5 +3292,26 @@ export function createBeeperMessageLikeMeSource( if (completion === undefined) return fail("record stream did not complete"); return completion; }, + dispose: async (_published: boolean) => { + stopProgressHeartbeat(); + if (disposed) return; + if (disposalInFlight !== undefined) return disposalInFlight; + const dispose = disposeWorking; + if (dispose === undefined) { + disposed = true; + return; + } + const attempt = (async () => { + await dispose(); + disposeWorking = undefined; + disposed = true; + })(); + disposalInFlight = attempt; + try { + await attempt; + } finally { + if (!disposed) disposalInFlight = undefined; + } + }, }); } diff --git a/src/fixtures/beeper-message-like-me-v1/accounts.ndjson b/src/fixtures/beeper-message-like-me-v1/accounts.ndjson index d0d5e48..2c91aff 100644 --- a/src/fixtures/beeper-message-like-me-v1/accounts.ndjson +++ b/src/fixtures/beeper-message-like-me-v1/accounts.ndjson @@ -1 +1,2 @@ {"accountId":"account:synthetic:primary","displayName":"Synthetic Primary","handle":"+15555550100","id":"account:synthetic:primary","kind":"account","network":"synthetic","provenance":{"connectedAccountProviderId":"beeper-account:synthetic-primary","observedAt":"2026-08-21T15:59:00.000Z","providerId":"beeper-account:synthetic-primary","providerRevision":null},"schemaVersion":1,"selfParticipantId":"participant:synthetic:self"} +{"accountId":"account:synthetic:secondary","displayName":"Synthetic Secondary","handle":"synthetic-secondary@example.invalid","id":"account:synthetic:secondary","kind":"account","network":"synthetic-secondary","provenance":{"connectedAccountProviderId":"beeper-account:synthetic-secondary","observedAt":"2026-08-21T15:59:00.000Z","providerId":"beeper-account:synthetic-secondary","providerRevision":null},"schemaVersion":1,"selfParticipantId":"participant:synthetic:secondary-self"} diff --git a/src/fixtures/beeper-message-like-me-v1/manifest.json b/src/fixtures/beeper-message-like-me-v1/manifest.json index 0aca85e..0d9b70e 100644 --- a/src/fixtures/beeper-message-like-me-v1/manifest.json +++ b/src/fixtures/beeper-message-like-me-v1/manifest.json @@ -1 +1 @@ -{"artifacts":[{"bytes":430,"mediaType":"application/x-ndjson","path":"accounts.ndjson","recordKind":"account","records":1,"sha256":"5e29e5ca806fc4a97c22e63d0061c722ce0f62f14e96383e70c15eaff57ee1f8"},{"bytes":797,"mediaType":"application/x-ndjson","path":"participants.ndjson","recordKind":"participant","records":2,"sha256":"a7fb621e298c45f481a3d2f3995cfcbea80693f52c7787c1ff05b69248134186"},{"bytes":571,"mediaType":"application/x-ndjson","path":"conversations.ndjson","recordKind":"conversation","records":1,"sha256":"87957914b3f21ece69815719db37ea6413efdf3201b68bd60a0cf7abbbacb529"},{"bytes":1492,"mediaType":"application/x-ndjson","path":"messages.ndjson","recordKind":"message","records":2,"sha256":"dbe97f5b4a5c46c6c2f3e02294511ceb7868e45a8738f8736c0ec93d6707c394"},{"bytes":521,"mediaType":"application/x-ndjson","path":"reactions.ndjson","recordKind":"reaction","records":1,"sha256":"683c263ce93e82bca0d905bcf6ef4cec6b32d8028aaa52595d0f295f068a6b01"},{"bytes":539,"mediaType":"application/x-ndjson","path":"tombstones.ndjson","recordKind":"tombstone","records":1,"sha256":"34f5a61aa9756c10979372b6147337053a563a6a7c63cd44eab602e2594e05d5"}],"completeness":{"kind":"truncated","observedFrom":"2026-08-21T15:50:00.000Z","observedThrough":"2026-08-21T15:59:00.000Z","reason":"explicit-source-limit"},"counts":{"account":1,"conversation":1,"message":2,"participant":2,"reaction":1,"tombstone":1},"format":"message-like-me.local-message-bundle","integrity":{"algorithm":"sha256","bundleSha256":"56c6ff3bbe60acfd103f234592087a9753d4b90447d0a508575e0e7c5d4cf514"},"privacy":{"attachments":"metadata-only","classification":"private-local","credentials":"excluded","providerUrls":"excluded"},"provider":{"id":"beeper","version":"0.6.2"},"schemaVersion":1,"source":{"id":"beeper-local","version":"1.0.0"},"timestamps":{"createdAt":"2026-08-21T16:00:01.000Z","finishedAt":"2026-08-21T16:00:01.000Z","startedAt":"2026-08-21T16:00:00.000Z"},"warnings":["attachments-metadata-only","remote-history-not-claimed","synthetic-golden-fixture"]} +{"artifacts":[{"bytes":913,"mediaType":"application/x-ndjson","path":"accounts.ndjson","recordKind":"account","records":2,"sha256":"822cb760142762378a27aa32d35aa2f8422c353cf139700bdd754454c9516db6"},{"bytes":1262,"mediaType":"application/x-ndjson","path":"participants.ndjson","recordKind":"participant","records":3,"sha256":"1287cb91bcce7e5e6299115a86e4c18f57bb6801b07424e95652974a1ad9d87f"},{"bytes":571,"mediaType":"application/x-ndjson","path":"conversations.ndjson","recordKind":"conversation","records":1,"sha256":"87957914b3f21ece69815719db37ea6413efdf3201b68bd60a0cf7abbbacb529"},{"bytes":1492,"mediaType":"application/x-ndjson","path":"messages.ndjson","recordKind":"message","records":2,"sha256":"dbe97f5b4a5c46c6c2f3e02294511ceb7868e45a8738f8736c0ec93d6707c394"},{"bytes":521,"mediaType":"application/x-ndjson","path":"reactions.ndjson","recordKind":"reaction","records":1,"sha256":"683c263ce93e82bca0d905bcf6ef4cec6b32d8028aaa52595d0f295f068a6b01"},{"bytes":539,"mediaType":"application/x-ndjson","path":"tombstones.ndjson","recordKind":"tombstone","records":1,"sha256":"34f5a61aa9756c10979372b6147337053a563a6a7c63cd44eab602e2594e05d5"}],"completeness":{"kind":"bounded-local","observedFrom":"2026-08-21T15:50:00.000Z","observedThrough":"2026-08-21T15:59:00.000Z","reason":"desktop-local-sequential-export"},"counts":{"account":2,"conversation":1,"message":2,"participant":3,"reaction":1,"tombstone":1},"format":"message-like-me.local-message-bundle","integrity":{"algorithm":"sha256","bundleSha256":"f4c1c8f99e9ff74a4e8ba300a7e0e417f6e2cadb21a90dc3e712e3f50b518c1d"},"privacy":{"attachments":"metadata-only","classification":"private-local","credentials":"excluded","providerUrls":"excluded"},"provider":{"id":"beeper","version":"0.6.2"},"schemaVersion":1,"source":{"id":"beeper-local","version":"1.1.0"},"timestamps":{"createdAt":"2026-08-21T16:00:01.000Z","finishedAt":"2026-08-21T16:00:01.000Z","startedAt":"2026-08-21T16:00:00.000Z"},"warnings":["attachments-metadata-only","connected-account-backfill-coverage-unknown","remote-history-not-claimed","sequential-account-snapshot","synthetic-golden-fixture"]} diff --git a/src/fixtures/beeper-message-like-me-v1/participants.ndjson b/src/fixtures/beeper-message-like-me-v1/participants.ndjson index 6a867b6..9bf8961 100644 --- a/src/fixtures/beeper-message-like-me-v1/participants.ndjson +++ b/src/fixtures/beeper-message-like-me-v1/participants.ndjson @@ -1,2 +1,3 @@ {"accountId":"account:synthetic:primary","displayName":"Synthetic Self","handle":"+15555550100","id":"participant:synthetic:self","isSelf":true,"kind":"participant","network":"synthetic","provenance":{"connectedAccountProviderId":"beeper-account:synthetic-primary","observedAt":"2026-08-21T15:59:00.000Z","providerId":"beeper-participant:synthetic-self","providerRevision":null},"schemaVersion":1} {"accountId":"account:synthetic:primary","displayName":"Synthetic Peer","handle":"+15555550101","id":"participant:synthetic:peer","isSelf":false,"kind":"participant","network":"synthetic","provenance":{"connectedAccountProviderId":"beeper-account:synthetic-primary","observedAt":"2026-08-21T15:59:00.000Z","providerId":"beeper-participant:synthetic-peer","providerRevision":null},"schemaVersion":1} +{"accountId":"account:synthetic:secondary","displayName":"Synthetic Secondary Self","handle":"synthetic-secondary@example.invalid","id":"participant:synthetic:secondary-self","isSelf":true,"kind":"participant","network":"synthetic-secondary","provenance":{"connectedAccountProviderId":"beeper-account:synthetic-secondary","observedAt":"2026-08-21T15:59:00.000Z","providerId":"beeper-participant:synthetic-secondary-self","providerRevision":null},"schemaVersion":1} diff --git a/src/providers/beeper-local-runtime.internal.test.ts b/src/providers/beeper-local-runtime.internal.test.ts index 5a1c49d..548b892 100644 --- a/src/providers/beeper-local-runtime.internal.test.ts +++ b/src/providers/beeper-local-runtime.internal.test.ts @@ -3,6 +3,7 @@ import { chmodSync, mkdirSync, mkdtempSync, + readFileSync, renameSync, realpathSync, rmSync, @@ -23,6 +24,7 @@ import { } from "./beeper-omni"; import { executeBeeperLocalOperation, + parseBeeperExportAccounts, parseBeeperExportMessages, probeBeeperLocalSubject, validateBeeperCliStore, @@ -32,6 +34,7 @@ import { import { parseBeeperMessagingReadInput, planBeeperAccountsListCommand, + planBeeperMessageLikeMeExportCommand, planBeeperReadCommand, } from "./beeper-local"; @@ -61,12 +64,14 @@ function accounts(): readonly unknown[] { network: "Beeper", status: "CONNECTED", user: { + displayName: "Official Display Alias", displayText: "Fixture Self", email: "self@example.test", fullName: "Fixture Self", id: SELF_ID, imgURL: "file:///private/avatar-self", isSelf: true, + name: "Official Name Alias", phoneNumber: "+15550000000", username: "fixture-self", }, @@ -181,12 +186,16 @@ function privateStore(): string { writeFileSync( join(path, "targets", "desktop.json"), `${JSON.stringify({ - auth: { token: "fixture-never-read-by-test-runner" }, + auth: { + accessToken: "fixture-never-read-by-test-runner", + source: "manual", + tokenType: "Bearer", + }, baseURL: "http://127.0.0.1:23384", id: "desktop", - managed: true, + managed: false, name: "Desktop", - runtime: "desktop", + runtime: { install: "desktop", port: 23_373 }, type: "desktop", })}\n`, { mode: 0o600 }, @@ -253,6 +262,18 @@ async function execute( } describe("Beeper local read runtime", () => { + test("preserves the pinned CLI account selector aliases independently", () => { + const parsed = parseBeeperExportAccounts(accounts()); + expect(parsed[0]?.selectorAliases).toEqual({ + displayName: "Official Display Alias", + name: "Official Name Alias", + }); + expect(parsed[0]?.user).toMatchObject({ + fullName: "Fixture Self", + }); + expect(Object.keys(parsed[0] ?? {})).not.toContain("selectorAliases"); + }); + test("plans only fixed read commands with command paths before Oclif global flags", () => { expect(planBeeperAccountsListCommand(1_500).argv).toEqual([ "accounts", @@ -285,6 +306,62 @@ describe("Beeper local read runtime", () => { })).toThrow("only one cursor direction"); }); + test("plans the official export without an account or diagnostic surface", () => { + expect(planBeeperMessageLikeMeExportCommand({ + outputDirectory: "/private/export/account-1", + limitChats: 12, + limitMessages: 345, + maxParticipants: 67, + }, 61_001)).toEqual([ + "export", + "--out", + "/private/export/account-1", + "--no-attachments", + "--max-participants", + "67", + "--limit-chats", + "12", + "--limit-messages", + "345", + "--read-only", + "--quiet", + "--target", + "desktop", + "--timeout", + "62s", + ]); + const hardBounded = planBeeperMessageLikeMeExportCommand({ + outputDirectory: "/private/export/account-2", + limitChats: 100_000, + limitMessages: 1_000_000, + maxParticipants: 500, + }, 3_600_001); + expect(hardBounded).not.toContain("--account"); + expect(hardBounded).not.toContain("--events"); + expect(hardBounded).not.toContain("--json"); + expect(hardBounded).not.toContain("--full"); + expect(hardBounded).not.toContain("--debug"); + expect(hardBounded).not.toContain("--base-url"); + expect(hardBounded).toEqual([ + "export", + "--out", + "/private/export/account-2", + "--no-attachments", + "--max-participants", + "500", + "--limit-chats", + "100000", + "--limit-messages", + "1000000", + "--read-only", + "--quiet", + "--target", + "desktop", + "--timeout", + "3601s", + ]); + }); + test("executes contacts, chats, and messages through strict synthetic JSON", async () => { const path = privateStore(); const calls: BeeperCliInvocation[] = []; @@ -423,6 +500,114 @@ describe("Beeper local read runtime", () => { }], NETWORK_ACCOUNT_ID, CHAT_ID, 1)).toThrow("attachments must be an array of at most 256 items"); }); + test("collapses retained reaction tuples regardless of dropped fields", () => { + const first = messages()[0] as Record; + const original = (first.reactions as readonly Record[])[0]!; + const distinct = { + emoji: false, + id: "reaction-distinct", + participantID: "signal:self", + reactionKey: "custom-reaction", + }; + const nullableEmoji = { + id: "reaction-nullable-emoji", + participantID: "signal:ada", + reactionKey: "nullable-emoji-reaction", + }; + const parsed = parseBeeperExportMessages([{ + ...first, + reactions: [{ + ...original, + imgURL: "https://media.example.test/first-token", + }, distinct, { + ...original, + emoji: false, + imgURL: "file:///private/different-ignored-reaction-image", + }, nullableEmoji, { + ...nullableEmoji, + emoji: true, + }], + }], NETWORK_ACCOUNT_ID, CHAT_ID, 1); + + expect(parsed[0]!.reactions.map(({ id }) => id)).toEqual([ + "reaction-private-id", + "reaction-distinct", + "reaction-nullable-emoji", + ]); + expect(parsed[0]!.reactions[0]).toEqual({ + emoji: true, + id: "reaction-private-id", + participantId: "signal:ada", + providerIdNonUnique: false, + reactionKey: "👍", + }); + expect(parsed[0]!.reactions[2]!.emoji).toBeNull(); + expect(parsed[0]!.reactions.every((reaction) => !reaction.providerIdNonUnique)) + .toBeTrue(); + expect(JSON.stringify(parsed[0]!.reactions)).not.toContain("first-token"); + expect(JSON.stringify(parsed[0]!.reactions)).not.toContain("different-ignored"); + expect(() => parseBeeperExportMessages([{ + ...first, + reactions: [original, { + ...original, + imgURL: "x".repeat(16_385), + }], + }], NETWORK_ACCOUNT_ID, CHAT_ID, 1)).toThrow("imgURL must be bounded text"); + }); + + test("retains and marks every tuple in a nonunique reaction provider-ID group", () => { + const first = messages()[0] as Record; + const original = (first.reactions as readonly Record[])[0]!; + const parsed = parseBeeperExportMessages([{ + ...first, + reactions: [original, { + ...original, + reactionKey: "second-private-reaction-key", + }, { + ...original, + participantID: "signal:second-private-participant", + reactionKey: "second-private-reaction-key", + }, { + ...original, + emoji: false, + imgURL: "file:///private/ignored-duplicate-image", + }, { + emoji: false, + id: "reaction-unique-provider-id", + participantID: "signal:ada", + reactionKey: "unique-reaction-key", + }], + }], NETWORK_ACCOUNT_ID, CHAT_ID, 1); + + expect(parsed[0]!.reactions.map((reaction) => ({ + id: reaction.id, + participantId: reaction.participantId, + reactionKey: reaction.reactionKey, + providerIdNonUnique: reaction.providerIdNonUnique, + }))).toEqual([{ + id: "reaction-private-id", + participantId: "signal:ada", + reactionKey: "👍", + providerIdNonUnique: true, + }, { + id: "reaction-private-id", + participantId: "signal:ada", + reactionKey: "second-private-reaction-key", + providerIdNonUnique: true, + }, { + id: "reaction-private-id", + participantId: "signal:second-private-participant", + reactionKey: "second-private-reaction-key", + providerIdNonUnique: true, + }, { + id: "reaction-unique-provider-id", + participantId: "signal:ada", + reactionKey: "unique-reaction-key", + providerIdNonUnique: false, + }]); + expect(JSON.stringify(parsed[0]!.reactions)).not.toContain("ignored-duplicate-image"); + }); + test("keeps first-run CLI payload extraction inside the overall probe deadline", async () => { const path = privateStore(); const calls: BeeperCliInvocation[] = []; @@ -460,6 +645,25 @@ describe("Beeper local read runtime", () => { const path = privateStore(); try { await expect(validateBeeperCliStore(path)).resolves.toBe(path); + const targetPath = join(path, "targets", "desktop.json"); + const target = JSON.parse(readFileSync(targetPath, "utf8")) as Record; + writeFileSync( + targetPath, + `${JSON.stringify({ ...target, auth: undefined })}\n`, + { mode: 0o600 }, + ); + await expect(validateBeeperCliStore(path)).rejects.toThrow( + "no effective stored access token", + ); + writeFileSync( + targetPath, + `${JSON.stringify({ ...target, managed: true, port: 23_392 })}\n`, + { mode: 0o600 }, + ); + await expect(validateBeeperCliStore(path)).rejects.toThrow( + "active endpoint override", + ); + writeFileSync(targetPath, `${JSON.stringify(target)}\n`, { mode: 0o600 }); writeFileSync( join(path, "config.json"), `${JSON.stringify({ defaultTarget: "other" })}\n`, @@ -496,4 +700,22 @@ describe("Beeper local read runtime", () => { rmSync(path, { recursive: true, force: true }); } }); + + test("does not disclose a rejected config-store path in diagnostics", async () => { + const parent = realpathSync(mkdtempSync(join(tmpdir(), "wrench-beeper-private-path."))); + const missing = join(parent, "sensitive-account-store-name"); + try { + await expect(validateBeeperCliStore(missing)).rejects.toThrow( + "Beeper CLI config directory could not be validated safely", + ); + try { + await validateBeeperCliStore(missing); + } catch (error) { + expect(String(error)).not.toContain(missing); + expect(String(error)).not.toContain("sensitive-account-store-name"); + } + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); }); diff --git a/src/providers/beeper-local-runtime.ts b/src/providers/beeper-local-runtime.ts index 2a4b273..001139f 100644 --- a/src/providers/beeper-local-runtime.ts +++ b/src/providers/beeper-local-runtime.ts @@ -85,6 +85,11 @@ export type BeeperUserProjection = Readonly<{ export type BeeperAccountProjection = Readonly<{ accountId: string; + /** Pinned CLI resolver fields, intentionally non-enumerable in runtime output. */ + selectorAliases: Readonly<{ + displayName: string | null; + name: string | null; + }>; bridge: Readonly<{ id: string; type: string; @@ -151,6 +156,7 @@ export type BeeperReactionProjection = Readonly<{ participantId: string; reactionKey: string; emoji: boolean | null; + providerIdNonUnique: boolean; }>; export type BeeperMessageProjection = Readonly<{ @@ -334,11 +340,13 @@ function parseUser(value: unknown, label: string): BeeperUserProjection { const source = strictRecord(value, label); exactKeys(source, ["id"], [ "cannotMessage", + "displayName", "displayText", "email", "fullName", "imgURL", "isSelf", + "name", "phoneNumber", "username", ], label); @@ -377,8 +385,17 @@ function parseAccount(value: unknown, label: string): BeeperAccountProjection { && provider !== "local" && provider !== "platform-sdk" ) throw new Error(`${label}.bridge.provider is unsupported`); - return Object.freeze({ + const userSource = strictRecord(source.user, `${label}.user`); + const projection: BeeperAccountProjection = { accountId: boundedString(source.accountID, `${label}.accountID`, 512), + selectorAliases: Object.freeze({ + displayName: nullableString( + userSource.displayName, + `${label}.user.displayName`, + 2_048, + ), + name: nullableString(userSource.name, `${label}.user.name`, 2_048), + }), bridge: Object.freeze({ id: boundedString(bridge.id, `${label}.bridge.id`, 512), type: boundedString(bridge.type, `${label}.bridge.type`, 512), @@ -389,7 +406,9 @@ function parseAccount(value: unknown, label: string): BeeperAccountProjection { status: boundedString(source.status, `${label}.status`, 128), statusText: nullableString(source.statusText, `${label}.statusText`, 2_048), user: parseUser(source.user, `${label}.user`), - }); + }; + Object.defineProperty(projection, "selectorAliases", { enumerable: false }); + return Object.freeze(projection); } function parseAccounts(value: unknown): readonly BeeperAccountProjection[] { @@ -630,9 +649,52 @@ function parseReaction(value: unknown, label: string): BeeperReactionProjection participantId: boundedString(source.participantID, `${label}.participantID`, 2_048), reactionKey: boundedString(source.reactionKey, `${label}.reactionKey`, 2_048, true), emoji: optionalBoolean(source.emoji, `${label}.emoji`), + providerIdNonUnique: false, }); } +function parseReactions(value: unknown, label: string): readonly BeeperReactionProjection[] { + const parsed = strictArray(value, label, 10_000) + .map((item, index) => parseReaction(item, `${label}[${index}]`)); + const byId = new Map>; + }>(); + const result: BeeperReactionProjection[] = []; + for (const reaction of parsed) { + let group = byId.get(reaction.id); + if (group === undefined) { + group = { + indexes: [], + tuplesByParticipant: new Map(), + }; + byId.set(reaction.id, group); + } + let byReactionKey = group.tuplesByParticipant.get(reaction.participantId); + if (byReactionKey === undefined) { + byReactionKey = new Map(); + group.tuplesByParticipant.set(reaction.participantId, byReactionKey); + } + if (byReactionKey.has(reaction.reactionKey)) continue; + const index = result.length; + byReactionKey.set(reaction.reactionKey, index); + group.indexes.push(index); + result.push(reaction); + } + for (const group of byId.values()) { + if (group.indexes.length < 2) continue; + for (const index of group.indexes) { + const reaction = result[index]; + if (reaction === undefined) throw new Error("Beeper reaction projection disappeared"); + result[index] = Object.freeze({ + ...reaction, + providerIdNonUnique: true, + }); + } + } + return Object.freeze(result); +} + function parseSeen( value: unknown, label: string, @@ -744,8 +806,7 @@ function parseMessage( .map((item, index) => parseAttachment(item, `${label}.attachments[${index}]`)); const reactions = source.reactions === undefined ? [] - : strictArray(source.reactions, `${label}.reactions`, 10_000) - .map((item, index) => parseReaction(item, `${label}.reactions[${index}]`)); + : parseReactions(source.reactions, `${label}.reactions`); return Object.freeze({ id: boundedString(source.id, `${label}.id`, 2_048), accountId, @@ -893,8 +954,8 @@ export async function resolvePinnedBeeperCliBinary( ); } -function localDesktopBaseUrl(value: unknown, label: string): void { - if (value === undefined) return; +function localDesktopBaseUrl(value: unknown, label: string): string | undefined { + if (value === undefined) return undefined; const source = boundedString(value, label, 256); let url: URL; try { @@ -915,6 +976,44 @@ function localDesktopBaseUrl(value: unknown, label: string): void { || port < 23_373 || port > 23_392 ) throw new Error(`${label} must be a reviewed loopback Beeper Desktop URL`); + return source; +} + +function storedBeeperAuth(value: unknown, label: string): JsonRecord { + const auth = strictRecord(value, label); + exactKeys(auth, ["accessToken", "tokenType"], [ + "clientID", + "expiresAt", + "scope", + "source", + ], label); + boundedString(auth.accessToken, `${label}.accessToken`, 64 * 1024); + if (auth.tokenType !== "Bearer") { + throw new Error(`${label}.tokenType is unsupported`); + } + if (auth.clientID !== undefined) { + boundedString(auth.clientID, `${label}.clientID`, 2_048); + } + if (auth.scope !== undefined) { + boundedString(auth.scope, `${label}.scope`, 2_048); + } + if (auth.expiresAt !== undefined) { + const expiresAt = boundedString(auth.expiresAt, `${label}.expiresAt`, 64); + if (!Number.isFinite(Date.parse(expiresAt))) { + throw new Error(`${label}.expiresAt must be a timestamp`); + } + } + if (auth.source !== undefined) { + const source = boundedString(auth.source, `${label}.source`, 64); + if (![ + "desktop-db", + "desktop-cache", + "desktop-oauth", + "remote-oauth", + "manual", + ].includes(source)) throw new Error(`${label}.source is unsupported`); + } + return auth; } async function readPrivateJsonFile(path: string, label: string): Promise { @@ -979,7 +1078,7 @@ async function readPrivateJsonFile(path: string, label: string): Promise { +async function validateBeeperCliStoreInternal(path: string): Promise { if (!isAbsolute(path)) throw new Error("Beeper CLI config directory must be absolute"); const canonical = await realpath(path); if (canonical !== path) throw new Error("Beeper CLI config directory must be canonical"); @@ -991,9 +1090,17 @@ export async function validateBeeperCliStore(path: string): Promise { || (stats.mode & 0o022) !== 0 ) throw new Error("Beeper CLI config directory must be an owned non-writable-by-others directory"); const config = await readPrivateJsonFile(join(canonical, "config.json"), "Beeper CLI config"); + let configAuth: JsonRecord | undefined; + let configBaseUrl: string | undefined; if (config !== null) { exactKeys(config, [], ["auth", "baseURL", "defaultAccount", "defaultTarget"], "Beeper CLI config"); - localDesktopBaseUrl(config.baseURL, "Beeper CLI config.baseURL"); + configBaseUrl = localDesktopBaseUrl(config.baseURL, "Beeper CLI config.baseURL"); + configAuth = config.auth === undefined + ? undefined + : storedBeeperAuth(config.auth, "Beeper CLI config.auth"); + if (config.defaultAccount !== undefined) { + boundedString(config.defaultAccount, "Beeper CLI config.defaultAccount", 512); + } if (config.defaultTarget !== BEEPER_DESKTOP_TARGET) { throw new Error("Beeper CLI config must select the fixed desktop target"); } @@ -1016,6 +1123,8 @@ export async function validateBeeperCliStore(path: string): Promise { join(canonicalTargets, "desktop.json"), "Beeper Desktop target", ); + let targetAuth: JsonRecord | undefined; + let targetBaseUrl: string | undefined; if (target !== null) { exactKeys(target, ["id", "type", "baseURL"], [ "auth", @@ -1030,14 +1139,63 @@ export async function validateBeeperCliStore(path: string): Promise { if (target.id !== "desktop" || target.type !== "desktop") { throw new Error("Beeper Desktop target must identify the fixed desktop realm"); } - localDesktopBaseUrl(target.baseURL, "Beeper Desktop target.baseURL"); + targetBaseUrl = localDesktopBaseUrl(target.baseURL, "Beeper Desktop target.baseURL"); + if ( + (target.managed !== undefined && target.managed !== false) + || target.dataDir !== undefined + || target.profile !== undefined + || target.serverEnv !== undefined + ) { + throw new Error("Beeper Desktop target contains an active endpoint override"); + } + if (target.port !== undefined) { + integer(target.port, "Beeper Desktop target.port", 23_373, 23_392); + } + if (target.runtime !== undefined) { + const runtime = strictRecord(target.runtime, "Beeper Desktop target.runtime"); + exactKeys(runtime, ["install", "port"], [], "Beeper Desktop target.runtime"); + if (runtime.install !== "desktop") { + throw new Error("Beeper Desktop target.runtime.install is unsupported"); + } + integer( + runtime.port, + "Beeper Desktop target.runtime.port", + 23_373, + 23_392, + ); + } + if (target.name !== undefined) { + boundedString(target.name, "Beeper Desktop target.name", 2_048); + } + targetAuth = target.auth === undefined + ? undefined + : storedBeeperAuth(target.auth, "Beeper Desktop target.auth"); } if (config === null || target === null) { throw new Error("Beeper CLI config directory has no authorized selected Desktop target"); } + const effectiveAuth = targetAuth + ?? (configAuth !== undefined + && (configBaseUrl === undefined || configBaseUrl === targetBaseUrl) + ? configAuth + : undefined); + if (effectiveAuth === undefined) { + throw new Error("Beeper CLI config directory has no effective stored access token"); + } return canonical; } +export async function validateBeeperCliStore(path: string): Promise { + try { + return await validateBeeperCliStoreInternal(path); + } catch (error) { + if (error instanceof Error && error.message.startsWith("Beeper ")) { + throw error; + } + throw new Error("Beeper CLI config directory could not be validated safely"); + } +} + async function readBoundedStream( stream: ReadableStream, maximum: number, diff --git a/src/providers/beeper-local.ts b/src/providers/beeper-local.ts index 15039e7..e489427 100644 --- a/src/providers/beeper-local.ts +++ b/src/providers/beeper-local.ts @@ -90,8 +90,8 @@ export type BeeperReadCommand = Readonly<{ export type BeeperMessageLikeMeExportCommandOptions = Readonly<{ outputDirectory: string; - limitChats: number | null; - limitMessages: number | null; + limitChats: number; + limitMessages: number; maxParticipants: number; }>; @@ -277,16 +277,20 @@ export function planBeeperMessageLikeMeExportCommand( "Beeper export maxParticipants", 2_000, ); - const limitChats = options.limitChats === null - ? null - : boundedInteger(options.limitChats, "Beeper export limitChats", 100_000); - const limitMessages = options.limitMessages === null - ? null - : boundedInteger( - options.limitMessages, - "Beeper export limitMessages", - 1_000_000, - ); + const limitChats = boundedInteger( + options.limitChats, + "Beeper export limitChats", + 100_000, + ); + const limitMessages = boundedInteger( + options.limitMessages, + "Beeper export limitMessages", + 1_000_000, + ); + const timeoutSeconds = Math.max( + 1, + Math.min(6 * 60 * 60, Math.ceil(timeoutMs / 1_000)), + ); return Object.freeze([ "export", "--out", @@ -294,11 +298,16 @@ export function planBeeperMessageLikeMeExportCommand( "--no-attachments", "--max-participants", String(maxParticipants), - ...(limitChats === null ? [] : ["--limit-chats", String(limitChats)]), - ...(limitMessages === null - ? [] - : ["--limit-messages", String(limitMessages)]), - ...globalArguments(timeoutMs).filter((argument) => argument !== "--json" && argument !== "--full"), + "--limit-chats", + String(limitChats), + "--limit-messages", + String(limitMessages), + "--read-only", + "--quiet", + "--target", + BEEPER_DESKTOP_TARGET, + "--timeout", + `${timeoutSeconds}s`, ]); } diff --git a/src/usage.ts b/src/usage.ts index 0c97f45..8b28936 100644 --- a/src/usage.ts +++ b/src/usage.ts @@ -74,7 +74,7 @@ export const wrenchUsage = `Usage: wrench auth sync --once [--json] Explicitly connect and refresh the local projection wrench auth remove --yes - wrench beeper export-message-like-me --auth --output + wrench beeper export-message-like-me --auth --output [--limit-chats ] [--limit-messages ] [--max-participants ] [--json] diff --git a/src/wrench.test.ts b/src/wrench.test.ts index f7b3149..160d991 100644 --- a/src/wrench.test.ts +++ b/src/wrench.test.ts @@ -904,6 +904,120 @@ describe("auth CLI", () => { observed = request as unknown as Record; expect(() => removeAuth("beeper-main", testState.environment)) .toThrow("active read projection transition"); + const privateMetadata = { + accountId: "private-account-id", + accountName: "Private Account Name", + network: "Private Network", + }; + request.onProgress?.({ + phase: "recovery-started", + ...privateMetadata, + }); + request.onProgress?.({ + phase: "recovery-completed", + recovered: 0, + published: 0, + ...privateMetadata, + }); + request.onProgress?.({ + phase: "preparing", + ...privateMetadata, + }); + request.onProgress?.({ + phase: "accounts-progress", + stage: "discovering", + elapsedSeconds: 30, + ...privateMetadata, + }); + request.onProgress?.({ + phase: "accounts-discovered", + accounts: 3, + ...privateMetadata, + }); + request.onProgress?.({ + phase: "account-started", + account: 1, + accounts: 3, + ...privateMetadata, + }); + request.onProgress?.({ + phase: "account-progress", + account: 1, + accounts: 3, + elapsedSeconds: 30, + ...privateMetadata, + }); + request.onProgress?.({ + phase: "account-validating", + account: 1, + accounts: 3, + elapsedSeconds: 0, + ...privateMetadata, + }); + request.onProgress?.({ + phase: "account-completed", + account: 1, + accounts: 3, + chats: 4, + messages: 50, + ...privateMetadata, + }); + request.onProgress?.({ + phase: "account-skipped", + account: 2, + accounts: 3, + reason: "chat-limit-reached", + ...privateMetadata, + }); + request.onProgress?.({ + phase: "accounts-verifying", + accounts: 3, + ...privateMetadata, + }); + request.onProgress?.({ + phase: "accounts-progress", + stage: "verifying", + elapsedSeconds: 60, + ...privateMetadata, + }); + request.onProgress?.({ + phase: "conversion-started", + accounts: 3, + chats: 10, + messages: 500, + ...privateMetadata, + }); + request.onProgress?.({ + phase: "conversion-progress", + elapsedSeconds: 30, + ...privateMetadata, + }); + request.onProgress?.({ + phase: "bundle-building", + elapsedSeconds: 30, + records: 50, + bytes: 4_096, + ...privateMetadata, + }); + request.onProgress?.({ + phase: "bundle-validating", + elapsedSeconds: 0, + records: 100, + bytes: 8_192, + ...privateMetadata, + }); + request.onProgress?.({ + phase: "bundle-publishing", + elapsedSeconds: 0, + records: 100, + bytes: 8_192, + ...privateMetadata, + }); + request.onProgress?.({ + phase: "private-cleanup", + elapsedSeconds: 0, + ...privateMetadata, + }); return Promise.resolve({ outputRoot: "/tmp/message-like-me-fixture", manifestPath: "/tmp/message-like-me-fixture/manifest.json", @@ -931,7 +1045,30 @@ describe("auth CLI", () => { }); expect(code).toBe(0); - expect(wrench.stderr()).toBe(""); + expect(wrench.stderr()).toBe([ + "wrench: Beeper export: checking prior private export state", + "wrench: Beeper export: private recovery complete; 0 directories reclaimed, 0 published bundles preserved", + "wrench: Beeper export: preparing pinned official CLI", + "wrench: Beeper export: discovering accounts; 30s elapsed", + "wrench: Beeper export: 3 accounts discovered", + "wrench: Beeper export: account 1/3 started", + "wrench: Beeper export: account 1/3 running; 30s elapsed", + "wrench: Beeper export: account 1/3 validating; 0s elapsed", + "wrench: Beeper export: account 1/3 complete; 4 chats, 50 messages total", + "wrench: Beeper export: account 2/3 skipped; chat limit reached", + "wrench: Beeper export: verifying 3 connected accounts", + "wrench: Beeper export: verifying connected accounts; 60s elapsed", + "wrench: Beeper export: converting 3 accounts; 10 chats, 500 messages total", + "wrench: Beeper export: converting local bundle; 30s elapsed", + "wrench: Beeper export: building local bundle; 50 records, 4096 bytes; 30s elapsed", + "wrench: Beeper export: validating local bundle; 100 records, 8192 bytes; 0s elapsed", + "wrench: Beeper export: publishing local bundle atomically; 100 records, 8192 bytes; 0s elapsed", + "wrench: Beeper export: removing private raw shards; 0s elapsed", + "", + ].join("\n")); + expect(wrench.stderr()).not.toContain("private-account-id"); + expect(wrench.stderr()).not.toContain("Private Account Name"); + expect(wrench.stderr()).not.toContain("Private Network"); expect(observed).toMatchObject({ auth: { id: "beeper-main", @@ -941,6 +1078,7 @@ describe("auth CLI", () => { }, outputRoot: "/tmp/message-like-me-fixture", limits: { limitChats: 10, limitMessages: 500 }, + onProgress: expect.any(Function), }); expect(JSON.parse(wrench.stdout())).toMatchObject({ ok: true, diff --git a/src/wrench.ts b/src/wrench.ts index f8676fa..4fd8bf1 100644 --- a/src/wrench.ts +++ b/src/wrench.ts @@ -24,6 +24,7 @@ import { } from "./auth"; import { parseWrenchArguments, wrenchUsage, type WrenchArguments } from "./args"; import type * as BeeperMessageLikeMeCliRuntimeModule from "./beeper-message-like-me-cli"; +import type { BeeperMessageLikeMeProgress } from "./beeper-message-like-me-source"; import type { GmailCaptureRunner } from "./gmail-capture"; import type * as MediaRuntimeModule from "./media"; import { @@ -440,6 +441,117 @@ function print(output: Output, value: unknown, json: boolean): void { output.stdout(json ? safeJson(value) : `${safe(typeof value === "string" ? value : JSON.stringify(value, null, 2))}\n`); } +function beeperProgressInteger(value: number, minimum: number): number { + if (!Number.isSafeInteger(value) || value < minimum) { + throw new Error("Beeper export progress was invalid"); + } + return value; +} + +function beeperProgressPosition( + progress: Readonly<{ account: number; accounts: number }>, +): string { + const account = beeperProgressInteger(progress.account, 1); + const accounts = beeperProgressInteger(progress.accounts, 1); + if (account > accounts) throw new Error("Beeper export progress was invalid"); + return `${account}/${accounts}`; +} + +function plural(count: number, singular: string, plural_: string): string { + return count === 1 ? singular : plural_; +} + +function renderBeeperMessageLikeMeProgress( + progress: BeeperMessageLikeMeProgress, +): string { + if (progress.phase === "recovery-started") { + return "wrench: Beeper export: checking prior private export state\n"; + } + if (progress.phase === "recovery-completed") { + const recovered = beeperProgressInteger(progress.recovered, 0); + const published = beeperProgressInteger(progress.published, 0); + return `wrench: Beeper export: private recovery complete; ${recovered} ${plural(recovered, "directory", "directories")} reclaimed, ${published} published ${plural(published, "bundle", "bundles")} preserved\n`; + } + if (progress.phase === "preparing") { + return "wrench: Beeper export: preparing pinned official CLI\n"; + } + if (progress.phase === "accounts-discovered") { + const accounts = beeperProgressInteger(progress.accounts, 1); + return `wrench: Beeper export: ${accounts} ${plural(accounts, "account", "accounts")} discovered\n`; + } + if (progress.phase === "accounts-progress") { + if (progress.stage !== "discovering" && progress.stage !== "verifying") { + throw new Error("Beeper export progress was invalid"); + } + const elapsedSeconds = beeperProgressInteger(progress.elapsedSeconds, 0); + const action = progress.stage === "discovering" + ? "discovering accounts" + : "verifying connected accounts"; + return `wrench: Beeper export: ${action}; ${elapsedSeconds}s elapsed\n`; + } + if (progress.phase === "account-started") { + return `wrench: Beeper export: account ${beeperProgressPosition(progress)} started\n`; + } + if (progress.phase === "account-validating") { + const elapsedSeconds = beeperProgressInteger(progress.elapsedSeconds, 0); + return `wrench: Beeper export: account ${beeperProgressPosition(progress)} validating; ${elapsedSeconds}s elapsed\n`; + } + if (progress.phase === "account-progress") { + const elapsedSeconds = beeperProgressInteger(progress.elapsedSeconds, 0); + return `wrench: Beeper export: account ${beeperProgressPosition(progress)} running; ${elapsedSeconds}s elapsed\n`; + } + if (progress.phase === "account-skipped") { + if (progress.reason !== "chat-limit-reached") { + throw new Error("Beeper export progress was invalid"); + } + return `wrench: Beeper export: account ${beeperProgressPosition(progress)} skipped; chat limit reached\n`; + } + if (progress.phase === "account-completed") { + const chats = beeperProgressInteger(progress.chats, 0); + const messages = beeperProgressInteger(progress.messages, 0); + return `wrench: Beeper export: account ${beeperProgressPosition(progress)} complete; ${chats} ${plural(chats, "chat", "chats")}, ${messages} ${plural(messages, "message", "messages")} total\n`; + } + if (progress.phase === "accounts-verifying") { + const accounts = beeperProgressInteger(progress.accounts, 1); + return `wrench: Beeper export: verifying ${accounts} connected ${plural(accounts, "account", "accounts")}\n`; + } + if (progress.phase === "conversion-started") { + const accounts = beeperProgressInteger(progress.accounts, 1); + const chats = beeperProgressInteger(progress.chats, 0); + const messages = beeperProgressInteger(progress.messages, 0); + return `wrench: Beeper export: converting ${accounts} ${plural(accounts, "account", "accounts")}; ${chats} ${plural(chats, "chat", "chats")}, ${messages} ${plural(messages, "message", "messages")} total\n`; + } + if (progress.phase === "conversion-progress") { + const elapsedSeconds = beeperProgressInteger(progress.elapsedSeconds, 0); + return `wrench: Beeper export: converting local bundle; ${elapsedSeconds}s elapsed\n`; + } + if (progress.phase === "bundle-building") { + const elapsedSeconds = beeperProgressInteger(progress.elapsedSeconds, 0); + const records = beeperProgressInteger(progress.records, 0); + const bytes = beeperProgressInteger(progress.bytes, 0); + return `wrench: Beeper export: building local bundle; ${records} ${plural(records, "record", "records")}, ${bytes} bytes; ${elapsedSeconds}s elapsed\n`; + } + if (progress.phase === "bundle-validating") { + const elapsedSeconds = beeperProgressInteger(progress.elapsedSeconds, 0); + const records = beeperProgressInteger(progress.records, 0); + const bytes = beeperProgressInteger(progress.bytes, 0); + return `wrench: Beeper export: validating local bundle; ${records} ${plural(records, "record", "records")}, ${bytes} bytes; ${elapsedSeconds}s elapsed\n`; + } + if (progress.phase === "bundle-publishing") { + const elapsedSeconds = beeperProgressInteger(progress.elapsedSeconds, 0); + const records = beeperProgressInteger(progress.records, 0); + const bytes = beeperProgressInteger(progress.bytes, 0); + return `wrench: Beeper export: publishing local bundle atomically; ${records} ${plural(records, "record", "records")}, ${bytes} bytes; ${elapsedSeconds}s elapsed\n`; + } + if (progress.phase === "private-cleanup") { + const elapsedSeconds = beeperProgressInteger(progress.elapsedSeconds, 0); + return `wrench: Beeper export: removing private raw shards; ${elapsedSeconds}s elapsed\n`; + } + const exhaustive: never = progress; + void exhaustive; + throw new Error("Beeper export progress was invalid"); +} + type PreparedCapture = { readonly arguments: readonly string[]; readonly runtimeOptions: ClipRuntimeOptions; @@ -1584,6 +1696,9 @@ async function runCommand( : { maxParticipants: arguments_.maxParticipants }), }, environment, + onProgress: (progress) => { + output.stderr(renderBeeperMessageLikeMeProgress(progress)); + }, ...(signal === undefined ? {} : { signal }), }); const summary = Object.freeze({ From 9e925d8f678342020ff9c2d280ea6a4517c2bd0b Mon Sep 17 00:00:00 2001 From: 0thernet Date: Sat, 22 Aug 2026 16:57:27 -0400 Subject: [PATCH 4/4] fix: bind Beeper truncation to retained prefix --- CHANGELOG.md | 21 +- README.md | 54 +- package.json | 2 +- skills/wrench/references/install.md | 4 +- src/beeper-message-like-me-source.test.ts | 398 ++++++++++ src/beeper-message-like-me-source.ts | 858 ++++++++++++++++++---- src/media/manifest.test.ts | 2 +- src/media/manifest.ts | 2 +- website/build.ts | 2 +- 9 files changed, 1159 insertions(+), 184 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c38e3bc..f16c825 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,18 @@ # Changelog -## 0.11.0 - 2026-08-21 +## 0.13.0 - 2026-08-22 -- Added a pinned, read-only Beeper Desktop provider for contacts and messaging +- Add a pinned, read-only Beeper Desktop provider for contacts and messaging projections across locally connected accounts. -- Added `wrench beeper export-message-like-me` for private, provenance-preserving - Message Like Me bundles with canonical digests, explicit completeness, and no - media downloads. -- Added strict local-runtime validation, bounded full-export conversion, graph - conformance checks, and a canonical cross-repository bundle fixture. +- Add `wrench beeper export-message-like-me` for private, + provenance-preserving Message Like Me bundles with canonical digests, + explicit completeness, graph validation, and no media downloads. +- Export Beeper accounts sequentially through the pinned official CLI with + redacted account-level progress, elapsed-time heartbeats, and cumulative + chat and message counts. +- Add durable process-aware recovery, global export admission, monitored raw + working limits, and atomic validated Message Like Me bundle publication. +- Normalize account-local self aliases before record allocation, preserve + distinct provider reaction facts, and reject contradictory identity or + snapshot evidence without exposing private coordinates. Bound repeated + participant work independently from output record cardinality. diff --git a/README.md b/README.md index 9f6af6e..6788fb2 100644 --- a/README.md +++ b/README.md @@ -75,10 +75,10 @@ The skill teaches Codex, Claude Code, Cursor, and other compatible coding agents when to use Wrench, how to preserve its trust boundaries, and how to install the CLI if it is missing. Start a new agent session after installation. -Install the current immutable CLI release from the `v0.12.0` tag: +Install the current immutable CLI release from the `v0.13.0` tag: ```sh -bun add --global github:hraness/wrench#v0.12.0 +bun add --global github:hraness/wrench#v0.13.0 wrench adapter sync-bundled --json wrench doctor ``` @@ -102,7 +102,7 @@ Install Wrench in an agent or application that owns its own model, planning, tool loop, approvals, and interface: ```sh -bun add github:hraness/wrench#v0.12.0 +bun add github:hraness/wrench#v0.13.0 ``` ```ts @@ -355,11 +355,11 @@ moving Homebrew formula name. The command above is sufficient only while from the [official CLI releases](https://github.com/beeper/cli/releases) asset and install its `beeper` executable at `/tools/beeper/0.6.2/beeper` (the default state home is -`~/.local/share/wrench`). Wrench requires archive SHA-256 -`688ccde7e7d044d33980cd06474bf1ae7215ccf8ca79967262fa3bfb85a2589a` -and executable SHA-256 -`48aa895449129c793a212ea19f69a534adc34a8adc4037ca1d7da9e648716425`; -it rejects every other version or byte sequence before reading private data. +`~/.local/share/wrench`). The reviewed release archive has SHA-256 +`688ccde7e7d044d33980cd06474bf1ae7215ccf8ca79967262fa3bfb85a2589a`. +After installation, Wrench enforces executable SHA-256 +`48aa895449129c793a212ea19f69a534adc34a8adc4037ca1d7da9e648716425` +and rejects every other executable byte sequence before reading private data. Binding hashes the stable local self-account coordinate before storing or printing it. The first bind or read may take longer while the pinned CLI unpacks @@ -403,15 +403,18 @@ directory is mode 0700, and every file is mode 0600 with a canonical SHA-256 digest. Each connected account has exactly one normalized self participant, anchored by -the account user's stable Beeper ID. Before record allocation, Wrench makes a -bounded hash-only pass over the selected chats. Explicit chat `isSelf` values -and message `isSender` values establish account-local self and peer evidence. -Later evidence applies to earlier chats, message files stay bound to their -validated SHA-256 digests, and contradictory evidence stops the export without -publishing. Reactions inherit a normalized participant reference while their -raw provider tuple remains only inside a composite hash. Nonunique provider -reaction IDs are preserved with the categorical -`reaction-provider-id-non-unique` warning. +the account user's stable Beeper ID. Before emitting records, Wrench proves a +deterministic candidate chat prefix against the record, byte, and participant +work bounds, then derives only hashed identity evidence from that prefix. If +normalization changes the admitted prefix, Wrench discards the provisional +state and repeats with the shorter prefix. Explicit chat `isSelf` values and +message `isSender` values establish account-local self and peer evidence. Later +admitted evidence applies to earlier chats, a rejected suffix cannot affect the +retained facts, message files stay bound to their validated SHA-256 digests, and +contradictory retained evidence stops the export without publishing. Reactions +inherit a normalized participant reference while their raw provider tuple +remains only inside a composite hash. Nonunique provider reaction IDs are +preserved with the categorical `reaction-provider-id-non-unique` warning. The JSON result reports the manifest path and digest, record counts, completeness, and warnings. `--limit-chats` is global across the account @@ -419,9 +422,13 @@ sequence. `--limit-messages` and `--max-participants` apply to each chat, which matches the official CLI flags. Reached limits are recorded as truncation. Wrench always passes hard ceilings of 100,000 chats and 1,000,000 messages per chat, and it emits a coherent truncated bundle before the 500,000-record or 512 -MiB bundle ceiling. One chat JSON file is limited to 64 MiB so foreign input -cannot force a multi-gigabyte allocation; an oversized chat is omitted with -explicit truncated completeness and a warning. While the official CLI is +MiB bundle ceiling. Conversion also stops at a deterministic chat boundary +before 250,000 participant occurrences across account anchors, rosters, message +senders, reaction actors, and implied self insertions for direct chats. This +bounds normalization work even when many chats repeat the same participants. +One chat JSON file is limited to 64 MiB so foreign input cannot force a +multi-gigabyte allocation; an oversized chat is omitted with explicit truncated +completeness and a warning. While the official CLI is running, Wrench monitors the complete private working tree against a 4 GiB ceiling every 500 ms and independently checks that at least 2 GiB remains free on the filesystem. This is a monitored safety ceiling, not an operating-system @@ -446,9 +453,10 @@ lands between the atomic rename and lease release, recovery recognizes the same directory at the requested output path and preserves the published bundle. -The Beeper Desktop API MCP project is intended to expose Beeper tools to an MCP -client. This export path uses the official CLI because Wrench needs a pinned, -bounded, read-only file snapshot that it can validate and publish atomically. +The [Beeper Desktop API MCP project](https://github.com/beeper/desktop-api-mcp) +is intended to expose Beeper tools to an MCP client. This export path uses the +official CLI because Wrench needs a pinned, bounded, read-only file snapshot +that it can validate and publish atomically. Contact and chat lists are bounded to 200 records because CLI 0.6.2 exposes no continuation cursor for those commands. Message pages derive the next diff --git a/package.json b/package.json index 8326e87..4009550 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hraness/wrench", - "version": "0.12.0", + "version": "0.13.0", "description": "Open-source CLI and TypeScript SDK for precise web capabilities for AI agents: page capture, verified media archives, encrypted reads, and typed provider operations.", "license": "MIT", "type": "module", diff --git a/skills/wrench/references/install.md b/skills/wrench/references/install.md index 18a8c50..5036ed9 100644 --- a/skills/wrench/references/install.md +++ b/skills/wrench/references/install.md @@ -21,7 +21,7 @@ When the user asked to install or use Wrench, install the current immutable release and its reviewed bundled adapter manifests: ```sh -bun add --global github:hraness/wrench#v0.12.0 +bun add --global github:hraness/wrench#v0.13.0 wrench adapter sync-bundled --json wrench --help wrench doctor --json @@ -31,7 +31,7 @@ Do not clone the repository merely to run the CLI. Importing the SDK is a separate project dependency and does not install a global command: ```sh -bun add github:hraness/wrench#v0.12.0 +bun add github:hraness/wrench#v0.13.0 ``` ## Add only required optional tools diff --git a/src/beeper-message-like-me-source.test.ts b/src/beeper-message-like-me-source.test.ts index 67dbd03..f939c6a 100644 --- a/src/beeper-message-like-me-source.test.ts +++ b/src/beeper-message-like-me-source.test.ts @@ -352,6 +352,22 @@ function ndjson(path: string): readonly Record[] { .map((line) => JSON.parse(line) as Record); } +const MESSAGE_LIKE_ME_DATA_FILES = [ + "accounts.ndjson", + "participants.ndjson", + "conversations.ndjson", + "messages.ndjson", + "reactions.ndjson", + "tombstones.ndjson", +] as const; + +function messageLikeMeDataDigests(root: string): Readonly> { + return Object.freeze(Object.fromEntries(MESSAGE_LIKE_ME_DATA_FILES.map((file) => [ + file, + createHash("sha256").update(readFileSync(join(root, file))).digest("hex"), + ]))); +} + const SELF_ALIAS_ID = "whatsapp:private-late-self-alias"; function aliasChatOrderKey(chatId: string): string { @@ -506,6 +522,143 @@ function writeSelfAliasFixture( }); } +const SHARED_METADATA_PEER_ID = "whatsapp:shared-metadata-peer"; +const EXCLUDED_METADATA_NAME = "Excluded Fixture Metadata"; +const EXCLUDED_METADATA_HANDLE = "+15550000999"; + +function orderedParticipantMetadataChatIds(): readonly [string, string] { + const ordered = ["metadata-chat-alpha", "metadata-chat-omega"] + .sort((left, right) => aliasChatOrderKey(left).localeCompare(aliasChatOrderKey(right))); + const selected = ordered[0]; + const excluded = ordered[1]; + if (selected === undefined || excluded === undefined) { + throw new Error("participant metadata fixture chat order disappeared"); + } + return [selected, excluded]; +} + +function writeParticipantMetadataFixture( + outputRoot: string, + includeExcluded: boolean, +): void { + const [selectedChatId, excludedChatId] = orderedParticipantMetadataChatIds(); + const baseChat = chat(); + const selectedChat = { + ...baseChat, + id: selectedChatId, + lastActivity: "2026-08-21T14:00:01.000Z", + participants: { + hasMore: false, + items: [{ id: SHARED_METADATA_PEER_ID, isSelf: false }], + total: 1, + }, + title: "Selected Metadata Fixture", + }; + const excludedChat = { + ...baseChat, + id: excludedChatId, + lastActivity: "2026-08-21T14:00:02.000Z", + participants: { + hasMore: false, + items: [{ + fullName: EXCLUDED_METADATA_NAME, + id: SHARED_METADATA_PEER_ID, + isSelf: false, + phoneNumber: EXCLUDED_METADATA_HANDLE, + }, { + fullName: "Excluded Self Alias", + id: SELF_ALIAS_ID, + isSelf: true, + }], + total: 2, + }, + title: "Excluded Metadata Fixture", + }; + const baseMessage = messages()[0] as Record; + const selectedMessage = { + ...baseMessage, + attachments: [], + chatID: selectedChatId, + editedTimestamp: null, + id: "message-metadata-selected", + linkedMessageID: null, + reactions: [{ + emoji: true, + id: "reaction-metadata-selected", + participantID: SELF_ALIAS_ID, + reactionKey: "👍", + }], + sortKey: "00000000000000000001", + text: "synthetic selected metadata body", + timestamp: "2026-08-21T14:00:01.000Z", + }; + const excludedMessage = { + ...baseMessage, + attachments: [], + chatID: excludedChatId, + editedTimestamp: null, + id: "message-metadata-excluded", + linkedMessageID: null, + reactions: [{ + emoji: true, + id: "reaction-metadata-excluded", + participantID: SHARED_METADATA_PEER_ID, + reactionKey: "👍", + }], + senderID: SELF_ALIAS_ID, + senderName: "Excluded Self Alias", + sortKey: "00000000000000000002", + text: "synthetic excluded metadata body", + timestamp: "2026-08-21T14:00:02.000Z", + }; + const entries = includeExcluded + ? [[selectedChat, selectedMessage], [excludedChat, excludedMessage]] as const + : [[selectedChat, selectedMessage]] as const; + const chatsRoot = join(outputRoot, "chats"); + rmSync(chatsRoot, { recursive: true, force: true }); + mkdirSync(chatsRoot, { mode: 0o755 }); + for (const [chatValue, messageValue] of entries) { + const chatRoot = join(chatsRoot, chatValue.id); + mkdirSync(join(chatRoot, "attachments"), { recursive: true, mode: 0o755 }); + writeJson(join(chatRoot, "chat.json"), chatValue); + writeJson(join(chatRoot, "messages.json"), [messageValue]); + writeFileSync(join(chatRoot, "messages.markdown"), "private duplicate markdown\n"); + writeFileSync(join(chatRoot, "messages.html"), "

private duplicate html

\n"); + } + const chatValues = entries.map(([chatValue]) => chatValue); + writeJson(join(outputRoot, "chats.json"), chatValues); + const stateChats = Object.fromEntries(entries.map(([chatValue], index) => [ + chatValue.id, + { + attachmentCount: 0, + complete: true, + cursor: null, + messageCount: 1, + startedAt: index === 0 + ? "2026-08-21T13:59:00.000Z" + : "2026-08-21T14:00:01.000Z", + updatedAt: index === 0 + ? "2026-08-21T14:00:01.000Z" + : "2026-08-21T14:00:02.000Z", + }, + ])); + writeJson(join(outputRoot, ".beeper-export-state.json"), { + chats: stateChats, + completedChatIDs: entries.map(([chatValue]) => chatValue.id), + createdAt: "2026-08-21T13:59:00.000Z", + exportVersion: 1, + }); + writeJson(join(outputRoot, "manifest.json"), { + accounts: accounts(), + attachmentCount: 0, + chatCount: entries.length, + completedAt: "2026-08-21T14:00:03.000Z", + createdAt: "2026-08-21T13:59:00.000Z", + messageCount: entries.length, + version: 1, + }); +} + describe("Beeper Message Like Me source", () => { test("settles a durable raw lease after a real immediately exiting child", async () => { const parent = privateDirectory("wrench-beeper-fast-child-test."); @@ -1647,6 +1800,251 @@ describe("Beeper Message Like Me source", () => { } }); + test("does not retain participant metadata from a record-rejected chat", async () => { + const parent = privateDirectory("wrench-beeper-source-record-metadata-test."); + const prefixWorking = join(parent, "prefix-working"); + const prefixOutput = join(parent, "prefix-bundle"); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(prefixWorking, { mode: 0o700 }); + mkdirSync(working, { mode: 0o700 }); + try { + const prefixSource = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => prefixWorking, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + const outputRoot = whatsAppOutputRoot(invocation); + if (outputRoot !== null) writeParticipantMetadataFixture(outputRoot, false); + return result; + }, + }, + }); + await exportBeeperMessageLikeMeBundle({ outputRoot: prefixOutput, source: prefixSource }); + + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + maxBundleRecords: 10, + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + const outputRoot = whatsAppOutputRoot(invocation); + if (outputRoot !== null) writeParticipantMetadataFixture(outputRoot, true); + return result; + }, + }, + }); + const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); + expect(result.manifest.warnings).toContain("bundle-record-limit-reached"); + expect(result.manifest.counts).toMatchObject({ + conversation: 1, + message: 1, + reaction: 1, + }); + const peer = ndjson(join(output, "participants.ndjson")) + .find((record) => record.isSelf === false); + expect(peer).toMatchObject({ displayName: null, handle: null, isSelf: false }); + const participantBytes = readFileSync(join(output, "participants.ndjson"), "utf8"); + expect(participantBytes).not.toContain(EXCLUDED_METADATA_NAME); + expect(participantBytes).not.toContain(EXCLUDED_METADATA_HANDLE); + const selfParticipantIds = new Set(ndjson(join(output, "accounts.ndjson")) + .map((record) => record.selfParticipantId)); + const reaction = ndjson(join(output, "reactions.ndjson"))[0]; + expect(reaction).toBeDefined(); + expect(selfParticipantIds.has(reaction!.participantId)).toBeFalse(); + expect(messageLikeMeDataDigests(output)).toEqual( + messageLikeMeDataDigests(prefixOutput), + ); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("does not charge or retain participant metadata from a byte-rejected chat", async () => { + const parent = privateDirectory("wrench-beeper-source-byte-metadata-test."); + const prefixWorking = join(parent, "prefix-working"); + const prefixOutput = join(parent, "prefix-bundle"); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(prefixWorking, { mode: 0o700 }); + mkdirSync(working, { mode: 0o700 }); + try { + const prefixSource = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => prefixWorking, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + const outputRoot = whatsAppOutputRoot(invocation); + if (outputRoot !== null) writeParticipantMetadataFixture(outputRoot, false); + return result; + }, + }, + }); + await exportBeeperMessageLikeMeBundle({ outputRoot: prefixOutput, source: prefixSource }); + const selectedPrefixBytes = MESSAGE_LIKE_ME_DATA_FILES.reduce( + (total, file) => total + lstatSync(join(prefixOutput, file)).size, + 0, + ); + + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + maxBundleBytes: selectedPrefixBytes, + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + const outputRoot = whatsAppOutputRoot(invocation); + if (outputRoot !== null) writeParticipantMetadataFixture(outputRoot, true); + return result; + }, + }, + }); + const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); + expect(result.manifest.warnings).toContain("bundle-byte-limit-reached"); + expect(result.manifest.counts).toMatchObject({ + conversation: 1, + message: 1, + reaction: 1, + }); + const peer = ndjson(join(output, "participants.ndjson")) + .find((record) => record.isSelf === false); + expect(peer).toMatchObject({ displayName: null, handle: null, isSelf: false }); + const participantBytes = readFileSync(join(output, "participants.ndjson"), "utf8"); + expect(participantBytes).not.toContain(EXCLUDED_METADATA_NAME); + expect(participantBytes).not.toContain(EXCLUDED_METADATA_HANDLE); + const selfParticipantIds = new Set(ndjson(join(output, "accounts.ndjson")) + .map((record) => record.selfParticipantId)); + const reaction = ndjson(join(output, "reactions.ndjson"))[0]; + expect(reaction).toBeDefined(); + expect(selfParticipantIds.has(reaction!.participantId)).toBeFalse(); + expect(messageLikeMeDataDigests(output)).toEqual( + messageLikeMeDataDigests(prefixOutput), + ); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("rejects message size-class drift between alias refinement passes", async () => { + const parent = privateDirectory("wrench-beeper-source-size-class-drift-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + let refinements = 0; + let removed = false; + try { + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + removed = true; + }, + maxBundleRecords: 10, + maxMessagesJsonBytes: 4_096, + onSelfAliasRefinementPass: async (pass) => { + refinements += 1; + expect(pass).toBe(1); + const [selectedChatId] = orderedParticipantMetadataChatIds(); + const messagesPath = join( + working, + "account-shards", + "account-001", + "chats", + selectedChatId, + "messages.json", + ); + const original = readFileSync(messagesPath, "utf8"); + expect(Buffer.byteLength(original)).toBeLessThan(4_096); + writeFileSync(messagesPath, original.padEnd(4_097, " ")); + }, + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + const outputRoot = whatsAppOutputRoot(invocation); + if (outputRoot !== null) writeParticipantMetadataFixture(outputRoot, true); + return result; + }, + }, + }); + await expect(exportBeeperMessageLikeMeBundle({ outputRoot: output, source })) + .rejects.toThrow("official export messages changed between self-alias passes"); + expect(refinements).toBe(1); + expect(removed).toBeTrue(); + expect(existsSync(working)).toBeFalse(); + expect(existsSync(output)).toBeFalse(); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + + test("bounds repeated participant work before excluded alias evidence is admitted", async () => { + const parent = privateDirectory("wrench-beeper-source-occurrence-limit-test."); + const working = join(parent, "working"); + const output = join(parent, "message-like-me"); + mkdirSync(working, { mode: 0o700 }); + try { + const source = createBeeperMessageLikeMeSource({ + auth: auth(configStore(parent)), + dependencies: { + binaryPath: "/fixture/beeper-0.6.2", + createWorkingDirectory: async () => working, + removeWorkingDirectory: async (path) => { + rmSync(path, { recursive: true, force: true }); + }, + maxParticipantOccurrences: 6, + runCli: async (invocation) => { + const result = await fixtureCli(invocation); + const outputRoot = whatsAppOutputRoot(invocation); + if (outputRoot !== null) writeParticipantMetadataFixture(outputRoot, true); + return result; + }, + }, + }); + const result = await exportBeeperMessageLikeMeBundle({ outputRoot: output, source }); + expect(result.manifest.completeness).toMatchObject({ + kind: "truncated", + reason: "participant-occurrence-limit", + }); + expect(result.manifest.warnings).toContain("participant-occurrence-limit-reached"); + expect(result.manifest.counts).toMatchObject({ + conversation: 1, + message: 1, + reaction: 1, + }); + const selfParticipantIds = new Set(ndjson(join(output, "accounts.ndjson")) + .map((record) => record.selfParticipantId)); + const reaction = ndjson(join(output, "reactions.ndjson"))[0]; + expect(reaction).toBeDefined(); + expect(selfParticipantIds.has(reaction!.participantId)).toBeFalse(); + const participantBytes = readFileSync(join(output, "participants.ndjson"), "utf8"); + expect(participantBytes).not.toContain(EXCLUDED_METADATA_NAME); + expect(participantBytes).not.toContain(EXCLUDED_METADATA_HANDLE); + } finally { + rmSync(parent, { recursive: true, force: true }); + } + }); + test("rejects an official export whose accounts do not match the bound auth subject", async () => { const parent = privateDirectory("wrench-beeper-source-subject-test."); const working = join(parent, "working"); diff --git a/src/beeper-message-like-me-source.ts b/src/beeper-message-like-me-source.ts index 67184cc..1c285f5 100644 --- a/src/beeper-message-like-me-source.ts +++ b/src/beeper-message-like-me-source.ts @@ -83,6 +83,7 @@ const MAX_RAW_CACHE_SYMLINK_TARGET_BYTES = 512; const RAW_WORKING_MONITOR_INTERVAL_MS = 500; const RAW_WORKING_RECOVERY_GRACE_MS = 5 * 60 * 1_000; const DEFAULT_MAX_PARTICIPANTS = 500; +const MAX_PARTICIPANT_OCCURRENCES = 250_000; const DEFAULT_TIMEOUT_MS = 6 * 60 * 60 * 1_000; type JsonRecord = Readonly>; @@ -184,6 +185,10 @@ export type BeeperMessageLikeMeSourceDependencies = Readonly<{ maxBundleBytes?: number; /** Test-only seam for exercising the per-chat JSON allocation cap. */ maxMessagesJsonBytes?: number; + /** Test-only seam for exercising the participant-occurrence work cap. */ + maxParticipantOccurrences?: number; + /** Test-only seam for mutating a fixture between bounded alias passes. */ + onSelfAliasRefinementPass?: (pass: number) => Promise; runCli?: ( invocation: BeeperExportCliInvocation, ) => Promise; @@ -207,14 +212,14 @@ type ParsedLimits = Readonly<{ timeoutMs: number; }>; -type ParticipantFact = { +type ParticipantFact = Readonly<{ readonly id: string; readonly accountId: string; readonly providerId: string; - displayName: string | null; - handle: string | null; - isSelf: boolean | null; -}; + readonly displayName: string | null; + readonly handle: string | null; + readonly isSelf: boolean | null; +}>; type PrivateDirectoryIdentity = Readonly<{ device: number; @@ -231,6 +236,7 @@ type ConversationScan = Readonly<{ messagesSha256: string; observedAt: string; participantIds: readonly string[]; + participantFactChanges: readonly ParticipantFact[]; participantsComplete: boolean; startedAt: string | null; lastMessageAt: string | null; @@ -261,8 +267,13 @@ type ValidatedAccountShard = Readonly<{ type SelfAliasPrepass = Readonly<{ aliasesByAccount: ReadonlyMap>; coveredChats: readonly ValidatedShardChat[]; + bundleByteLimitReached: boolean; + bundleRecordLimitReached: boolean; evidenceLimitReached: boolean; messagesSha256ByPath: ReadonlyMap; + participantOccurrenceLimitReached: boolean; + provisionalRecordBytes: number; + provisionalRecordCount: number; }>; type BeeperCliStoreSnapshot = Readonly<{ @@ -1852,16 +1863,14 @@ async function prepassSelfAliases( accounts: readonly BeeperAccountProjection[], accountsById: ReadonlyMap, maxMessagesJsonBytes: number, - maxEvidence: number, + maxBundleRecords: number, + maxBundleBytes: number, + maxParticipantOccurrences: number, + observedAtByAccount: ReadonlyMap, signal: AbortSignal | undefined, heartbeat: () => void, + onRefinementPass: ((pass: number) => Promise) | undefined, ): Promise { - const aliasesByAccount = new Map>(); - const peerEvidenceByAccount = new Map>(); - const evidenceIdsByAccount = new Map>(); - const coveredChats: ValidatedShardChat[] = []; - const messagesSha256ByPath = new Map(); - let evidenceCount = 0; const evidenceSet = ( collection: Map>, accountId: string, @@ -1873,116 +1882,523 @@ async function prepassSelfAliases( } return values; }; - for (const account of accounts) { - if (account.user.isSelf === false) { - return fail("Beeper account user contradicts its self identity anchor"); - } - const coordinate = digest([ - "beeper-self-alias-v1", - account.accountId, - account.user.id, - ]); - evidenceSet(aliasesByAccount, account.accountId).add(coordinate); - evidenceSet(evidenceIdsByAccount, account.accountId).add(coordinate); - evidenceCount += 1; - } - - let evidenceLimitReached = evidenceCount > maxEvidence; - for (const validated of chats) { - if (evidenceLimitReached) break; - heartbeat(); - throwIfAborted(signal); - const chatDocument = await readOwnedJsonDocument( - validated.chatPath, - validated.root, - MAX_CHAT_JSON_BYTES, - false, - signal, - ); - if (chatDocument.sha256 !== validated.chatSha256) { - return fail("official export chat changed before self-alias prepass"); - } - const chat = parseBeeperExportConversation(chatDocument.value, accounts); - if ( - chat.id !== validated.chat.id - || chat.accountId !== validated.chat.accountId - ) return fail("official export chat identity changed before self-alias prepass"); - const account = accountsById.get(chat.accountId); - if (account === undefined) return fail("official export chat references an unknown account"); - const positiveEvidence = new Set(); - const peerEvidence = new Set(); - for (const participant of chat.participants.items) { + type EvidencePass = Readonly<{ + aliasesByAccount: ReadonlyMap>; + coveredChats: readonly ValidatedShardChat[]; + evidenceLimitReached: boolean; + identityConflict: boolean; + messagesSha256ByPath: ReadonlyMap; + oversizedMessagesPaths: ReadonlySet; + participantOccurrenceLimitReached: boolean; + }>; + type BudgetPass = Readonly<{ + admittedChats: readonly ValidatedShardChat[]; + bundleByteLimitReached: boolean; + bundleRecordLimitReached: boolean; + provisionalRecordBytes: number; + provisionalRecordCount: number; + }>; + const boundMessagesSha256ByPath = new Map(); + const boundOversizedMessagesPaths = new Set(); + + const collectEvidence = async ( + candidateChats: readonly ValidatedShardChat[], + ): Promise => { + const aliasesByAccount = new Map>(); + const peerEvidenceByAccount = new Map>(); + const evidenceIdsByAccount = new Map>(); + const coveredChats: ValidatedShardChat[] = []; + const messagesSha256ByPath = new Map(); + const oversizedMessagesPaths = new Set(); + let evidenceCount = 0; + let participantOccurrenceCount = 0; + for (const account of accounts) { + if (account.user.isSelf === false) { + return fail("Beeper account user contradicts its self identity anchor"); + } const coordinate = digest([ "beeper-self-alias-v1", account.accountId, - participant.id, + account.user.id, ]); - if (participant.isSelf === true) positiveEvidence.add(coordinate); - if (participant.isSelf === false) peerEvidence.add(coordinate); + evidenceSet(aliasesByAccount, account.accountId).add(coordinate); + evidenceSet(evidenceIdsByAccount, account.accountId).add(coordinate); + evidenceCount += 1; + participantOccurrenceCount += 1; } - if (await ownedFileSize(validated.messagesPath, validated.root) > maxMessagesJsonBytes) { - coveredChats.push(validated); - continue; + if (participantOccurrenceCount > maxParticipantOccurrences) { + return fail("connected Beeper accounts exceed the participant work bound"); } - const messagesDocument = await readOwnedJsonDocument( - validated.messagesPath, - validated.root, - maxMessagesJsonBytes, - false, - signal, - ); - const messages = parseBeeperExportMessages( - messagesDocument.value, - chat.accountId, - chat.id, - MAX_EXPORT_MESSAGES_PER_CHAT, - ); - if (messages.length !== validated.expectedMessageCount) { - return fail("official export chat messages did not match completed state"); + let evidenceLimitReached = evidenceCount > maxBundleRecords; + let participantOccurrenceLimitReached = false; + for (const validated of candidateChats) { + if (evidenceLimitReached || participantOccurrenceLimitReached) break; + heartbeat(); + throwIfAborted(signal); + const chatDocument = await readOwnedJsonDocument( + validated.chatPath, + validated.root, + MAX_CHAT_JSON_BYTES, + false, + signal, + ); + if (chatDocument.sha256 !== validated.chatSha256) { + return fail("official export chat changed before self-alias prepass"); + } + const chat = parseBeeperExportConversation(chatDocument.value, accounts); + if ( + chat.id !== validated.chat.id + || chat.accountId !== validated.chat.accountId + ) return fail("official export chat identity changed before self-alias prepass"); + const account = accountsById.get(chat.accountId); + if (account === undefined) { + return fail("official export chat references an unknown account"); + } + const rosterOccurrences = chat.participants.items.length; + if (rosterOccurrences > maxParticipantOccurrences - participantOccurrenceCount) { + participantOccurrenceLimitReached = true; + break; + } + const messagesSize = await ownedFileSize(validated.messagesPath, validated.root); + const priorSha256 = boundMessagesSha256ByPath.get(validated.messagesPath); + if (messagesSize > maxMessagesJsonBytes) { + if (priorSha256 !== undefined) { + return fail("official export messages changed between self-alias passes"); + } + boundOversizedMessagesPaths.add(validated.messagesPath); + oversizedMessagesPaths.add(validated.messagesPath); + participantOccurrenceCount += rosterOccurrences; + coveredChats.push(validated); + continue; + } + if (boundOversizedMessagesPaths.has(validated.messagesPath)) { + return fail("official export messages changed between self-alias passes"); + } + const messagesDocument = await readOwnedJsonDocument( + validated.messagesPath, + validated.root, + maxMessagesJsonBytes, + false, + signal, + ); + if (priorSha256 !== undefined && priorSha256 !== messagesDocument.sha256) { + return fail("official export messages changed between self-alias passes"); + } + boundMessagesSha256ByPath.set(validated.messagesPath, messagesDocument.sha256); + const messages = parseBeeperExportMessages( + messagesDocument.value, + chat.accountId, + chat.id, + MAX_EXPORT_MESSAGES_PER_CHAT, + ); + if (messages.length !== validated.expectedMessageCount) { + return fail("official export chat messages did not match completed state"); + } + const reactionCount = messages.reduce( + (count, message) => count + message.reactions.length, + 0, + ); + const participantOccurrences = rosterOccurrences + + messages.length + + reactionCount + + (chat.type === "single" ? 1 : 0); + if ( + !Number.isSafeInteger(reactionCount) + || !Number.isSafeInteger(participantOccurrences) + ) return fail("official export derived record count overflowed"); + if (participantOccurrences > maxParticipantOccurrences - participantOccurrenceCount) { + participantOccurrenceLimitReached = true; + break; + } + const positiveEvidence = new Set(); + const peerEvidence = new Set(); + for (const participant of chat.participants.items) { + const coordinate = digest([ + "beeper-self-alias-v1", + account.accountId, + participant.id, + ]); + if (participant.isSelf === true) positiveEvidence.add(coordinate); + if (participant.isSelf === false) peerEvidence.add(coordinate); + } + for (const message of messages) { + (message.isSender ? positiveEvidence : peerEvidence).add(digest([ + "beeper-self-alias-v1", + account.accountId, + message.senderId, + ])); + } + const accountEvidenceIds = evidenceSet(evidenceIdsByAccount, account.accountId); + const candidateEvidence = new Set([...positiveEvidence, ...peerEvidence]); + let newEvidence = 0; + for (const sourceId of candidateEvidence) { + if (!accountEvidenceIds.has(sourceId)) newEvidence += 1; + } + if (newEvidence > maxBundleRecords - evidenceCount) { + evidenceLimitReached = true; + break; + } + const aliases = evidenceSet(aliasesByAccount, account.accountId); + const peers = evidenceSet(peerEvidenceByAccount, account.accountId); + for (const sourceId of positiveEvidence) aliases.add(sourceId); + for (const sourceId of peerEvidence) peers.add(sourceId); + for (const sourceId of candidateEvidence) accountEvidenceIds.add(sourceId); + evidenceCount += newEvidence; + participantOccurrenceCount += participantOccurrences; + messagesSha256ByPath.set(validated.messagesPath, messagesDocument.sha256); + coveredChats.push(validated); } - for (const message of messages) { - (message.isSender ? positiveEvidence : peerEvidence).add(digest([ - "beeper-self-alias-v1", - account.accountId, - message.senderId, - ])); + let identityConflict = false; + for (const [accountId, peerEvidence] of peerEvidenceByAccount) { + const aliases = aliasesByAccount.get(accountId); + if (aliases === undefined) { + return fail("Beeper account self-alias set disappeared"); + } + if ([...peerEvidence].some((sourceId) => aliases.has(sourceId))) { + identityConflict = true; + } } - const accountEvidenceIds = evidenceSet(evidenceIdsByAccount, account.accountId); - let newEvidence = 0; - for (const sourceId of new Set([...positiveEvidence, ...peerEvidence])) { - if (!accountEvidenceIds.has(sourceId)) newEvidence += 1; + return Object.freeze({ + aliasesByAccount, + coveredChats: Object.freeze(coveredChats), + evidenceLimitReached, + identityConflict, + messagesSha256ByPath, + oversizedMessagesPaths, + participantOccurrenceLimitReached, + }); + }; + + const planBudget = async (evidence: EvidencePass): Promise => { + const participantFacts = new Map(); + const selfParticipantByAccount = new Map(); + let recordCount = accounts.length; + let recordBytes = 0; + for (const account of accounts) { + const observedAt = observedAtByAccount.get(account.accountId); + if (observedAt === undefined) { + return fail("Beeper account observation time disappeared"); + } + const self = planningParticipantFact( + account, + account.user, + true, + evidence.aliasesByAccount, + ); + participantFacts.set(self.id, self); + selfParticipantByAccount.set(account.accountId, self.id); + recordCount += 1; + const network = normalizeNetwork(account.network, account.bridge.type); + recordBytes += bundleRecordBytes(accountRecord( + account, + network, + observedAt, + self.id, + )); + recordBytes += bundleRecordBytes(participantRecord( + self, + account, + network, + observedAt, + )); } - if (newEvidence > maxEvidence - evidenceCount) { - evidenceLimitReached = true; - break; + if (recordCount > maxBundleRecords || recordBytes > maxBundleBytes) { + return fail("connected Beeper accounts exceed the bounded bundle foundation"); } - const aliases = evidenceSet(aliasesByAccount, account.accountId); - const peers = evidenceSet(peerEvidenceByAccount, account.accountId); - for (const sourceId of positiveEvidence) aliases.add(sourceId); - for (const sourceId of peerEvidence) peers.add(sourceId); - for (const sourceId of new Set([...positiveEvidence, ...peerEvidence])) { - accountEvidenceIds.add(sourceId); + const admittedChats: ValidatedShardChat[] = []; + for (const validated of evidence.coveredChats) { + heartbeat(); + throwIfAborted(signal); + const messagesSha256 = evidence.messagesSha256ByPath.get(validated.messagesPath); + if (messagesSha256 === undefined) { + if (!evidence.oversizedMessagesPaths.has(validated.messagesPath)) { + return fail("official export message proof disappeared during bundle admission"); + } + if ( + await ownedFileSize(validated.messagesPath, validated.root) + <= maxMessagesJsonBytes + ) return fail("official export messages changed during bundle admission"); + admittedChats.push(validated); + continue; + } + const chatDocument = await readOwnedJsonDocument( + validated.chatPath, + validated.root, + MAX_CHAT_JSON_BYTES, + false, + signal, + ); + if (chatDocument.sha256 !== validated.chatSha256) { + return fail("official export chat changed during bundle admission"); + } + const chat = parseBeeperExportConversation(chatDocument.value, accounts); + if ( + chat.id !== validated.chat.id + || chat.accountId !== validated.chat.accountId + ) return fail("official export chat identity changed during bundle admission"); + const account = accountsById.get(chat.accountId); + if (account === undefined) { + return fail("official export chat references an unknown account"); + } + const messagesDocument = await readOwnedJsonDocument( + validated.messagesPath, + validated.root, + maxMessagesJsonBytes, + false, + signal, + ); + if (messagesDocument.sha256 !== messagesSha256) { + return fail("official export messages changed during bundle admission"); + } + const messages = parseBeeperExportMessages( + messagesDocument.value, + chat.accountId, + chat.id, + MAX_EXPORT_MESSAGES_PER_CHAT, + ); + if (messages.length !== validated.expectedMessageCount) { + return fail("official export chat messages did not match completed state"); + } + const selfParticipantId = selfParticipantByAccount.get(account.accountId); + const selfParticipant = selfParticipantId === undefined + ? undefined + : participantFacts.get(selfParticipantId); + if (selfParticipantId === undefined || selfParticipant === undefined) { + return fail("Beeper account self participant disappeared"); + } + const scanParticipantFacts = new Map([ + [selfParticipantId, selfParticipant], + ]); + const participantIds = new Set(); + const addPlanningFact = (fact: ParticipantFact): void => { + scanParticipantFacts.set( + fact.id, + mergePlanningParticipantFact(scanParticipantFacts.get(fact.id), fact), + ); + participantIds.add(fact.id); + }; + for (const participant of chat.participants.items) { + addPlanningFact(planningParticipantFact( + account, + participant, + participant.isSelf, + evidence.aliasesByAccount, + )); + } + if (chat.type === "single") participantIds.add(selfParticipantId); + const messageIds = new Set(messages.map((message) => message.id)); + for (const message of messages) { + addPlanningFact(planningParticipantFact(account, { + id: message.senderId, + fullName: message.senderName, + phoneNumber: null, + email: null, + username: null, + }, message.isSender, evidence.aliasesByAccount)); + for (const reaction of message.reactions) { + addPlanningFact(planningParticipantFact(account, { + id: reaction.participantId, + fullName: null, + phoneNumber: null, + email: null, + username: null, + }, null, evidence.aliasesByAccount)); + } + } + const reactionCount = messages.reduce( + (count, message) => count + message.reactions.length, + 0, + ); + const tombstoneCount = messages.reduce( + (count, message) => count + (message.isDeleted || message.isHidden ? 1 : 0), + 0, + ); + const reactionProviderIdNonUniqueGroups = messages.reduce( + (count, message) => count + new Set( + message.reactions + .filter((reaction) => reaction.providerIdNonUnique) + .map((reaction) => reaction.id), + ).size, + 0, + ); + const range = messageTimestampRange(messages); + const roster = [...participantIds].map((participantId) => { + const participant = scanParticipantFacts.get(participantId); + if (participant === undefined) { + return fail("Beeper conversation participant disappeared"); + } + return participant; + }); + const directRosterComplete = chat.type !== "single" + || ( + roster.length === 2 + && roster.filter((participant) => participant.isSelf === true).length === 1 + && roster.filter((participant) => participant.isSelf !== true).length === 1 + ); + const scan: ConversationScan = Object.freeze({ + chat: Object.freeze({ + id: chat.id, + accountId: chat.accountId, + lastActivity: chat.lastActivity, + title: chat.title, + type: chat.type, + }), + root: validated.root, + messagesPath: validated.messagesPath, + messagesSha256: messagesDocument.sha256, + observedAt: validated.observedAt, + participantIds: Object.freeze([...participantIds].sort()), + participantFactChanges: Object.freeze([]), + participantsComplete: !chat.participants.hasMore + && chat.participants.items.length === chat.participants.total + && directRosterComplete, + startedAt: range.first, + lastMessageAt: range.last, + messageCount: messages.length, + reactionCount, + reactionProviderIdNonUniqueGroups, + tombstoneCount, + nonParticipantRecordBytes: 0, + }); + const network = normalizeNetwork(account.network, account.bridge.type); + let nonParticipantBytes = bundleRecordBytes(conversationRecord( + scan, + account, + network, + validated.observedAt, + )); + for (const message of messages) { + nonParticipantBytes += bundleRecordBytes(messageRecord( + message, + scan, + messageIds, + account, + evidence.aliasesByAccount, + network, + validated.observedAt, + )); + for (const reaction of message.reactions) { + nonParticipantBytes += bundleRecordBytes(reactionRecord( + reaction, + message, + scan, + account, + evidence.aliasesByAccount, + network, + validated.observedAt, + )); + } + const tombstone = tombstoneRecord( + message, + scan, + account, + network, + validated.observedAt, + ); + if (tombstone !== null) nonParticipantBytes += bundleRecordBytes(tombstone); + } + const stagedParticipantFacts = new Map(); + let addedParticipants = 0; + let participantByteDelta = 0; + for (const participantId of participantIds) { + const incoming = scanParticipantFacts.get(participantId); + if (incoming === undefined) { + return fail("Beeper conversation participant disappeared"); + } + const current = participantFacts.get(participantId); + const merged = mergePlanningParticipantFact(current, incoming); + stagedParticipantFacts.set(participantId, merged); + if (current === undefined) addedParticipants += 1; + const mergedBytes = bundleRecordBytes(participantRecord( + merged, + account, + network, + validated.observedAt, + )); + const currentBytes = current === undefined + ? 0 + : bundleRecordBytes(participantRecord( + current, + account, + network, + validated.observedAt, + )); + participantByteDelta += mergedBytes - currentBytes; + } + const candidateRecords = 1 + + messages.length + + reactionCount + + tombstoneCount + + addedParticipants; + const candidateBytes = nonParticipantBytes + participantByteDelta; + if ( + !Number.isSafeInteger(candidateRecords) + || !Number.isSafeInteger(candidateBytes) + || candidateBytes < 0 + ) return fail("official export derived bundle budget overflowed"); + if (candidateRecords > maxBundleRecords - recordCount) { + return Object.freeze({ + admittedChats: Object.freeze(admittedChats), + bundleByteLimitReached: false, + bundleRecordLimitReached: true, + provisionalRecordBytes: recordBytes, + provisionalRecordCount: recordCount, + }); + } + if (candidateBytes > maxBundleBytes - recordBytes) { + return Object.freeze({ + admittedChats: Object.freeze(admittedChats), + bundleByteLimitReached: true, + bundleRecordLimitReached: false, + provisionalRecordBytes: recordBytes, + provisionalRecordCount: recordCount, + }); + } + admittedChats.push(validated); + recordCount += candidateRecords; + recordBytes += candidateBytes; + for (const [participantId, fact] of stagedParticipantFacts) { + participantFacts.set(participantId, fact); + } } - evidenceCount += newEvidence; - messagesSha256ByPath.set(validated.messagesPath, messagesDocument.sha256); - coveredChats.push(validated); - } + return Object.freeze({ + admittedChats: Object.freeze(admittedChats), + bundleByteLimitReached: false, + bundleRecordLimitReached: false, + provisionalRecordBytes: recordBytes, + provisionalRecordCount: recordCount, + }); + }; - for (const [accountId, peerEvidence] of peerEvidenceByAccount) { - const aliases = aliasesByAccount.get(accountId); - if (aliases === undefined) return fail("Beeper account self-alias set disappeared"); - for (const sourceId of peerEvidence) { - if (aliases.has(sourceId)) { + let candidateChats = chats; + let bundleByteLimitReached = false; + let bundleRecordLimitReached = false; + for (let pass = 0; pass < 16; pass += 1) { + const evidence = await collectEvidence(candidateChats); + const budget = await planBudget(evidence); + bundleByteLimitReached ||= budget.bundleByteLimitReached; + bundleRecordLimitReached ||= budget.bundleRecordLimitReached; + if (budget.admittedChats.length === evidence.coveredChats.length) { + if (evidence.identityConflict) { return fail("official export has peer evidence for an account self alias"); } + return Object.freeze({ + aliasesByAccount: evidence.aliasesByAccount, + coveredChats: evidence.coveredChats, + bundleByteLimitReached, + bundleRecordLimitReached, + evidenceLimitReached: evidence.evidenceLimitReached, + messagesSha256ByPath: evidence.messagesSha256ByPath, + participantOccurrenceLimitReached: + evidence.participantOccurrenceLimitReached, + provisionalRecordBytes: budget.provisionalRecordBytes, + provisionalRecordCount: budget.provisionalRecordCount, + }); } + await onRefinementPass?.(pass + 1); + candidateChats = budget.admittedChats; } - return Object.freeze({ - aliasesByAccount, - coveredChats: Object.freeze(coveredChats), - evidenceLimitReached, - messagesSha256ByPath, - }); + return fail("bounded chat prefix did not stabilize"); } function canonicalParticipantSourceId( @@ -2002,7 +2418,6 @@ function upsertParticipant( user: Pick, self: boolean | null, aliasesByAccount: ReadonlyMap>, - createdIds?: Set, ): ParticipantFact { const sourceId = canonicalParticipantSourceId(account, user.id, aliasesByAccount); const id = localId("participant", account.accountId, sourceId); @@ -2012,24 +2427,99 @@ function upsertParticipant( if (self !== null && current.isSelf !== null && current.isSelf !== self) { return fail("one Beeper participant has conflicting self-direction evidence"); } - current.displayName ??= user.fullName; - current.handle ??= handle; - current.isSelf ??= self; - return current; + const updated = Object.freeze({ + ...current, + displayName: current.displayName ?? user.fullName, + handle: current.handle ?? handle, + isSelf: current.isSelf ?? self, + }); + if ( + updated.displayName === current.displayName + && updated.handle === current.handle + && updated.isSelf === current.isSelf + ) return current; + facts.set(id, updated); + return updated; } - const created: ParticipantFact = { + const created: ParticipantFact = Object.freeze({ id, accountId: localId("account", account.accountId), providerId: providerId("participant", account.accountId, sourceId), displayName: user.fullName, handle, isSelf: self, - }; + }); facts.set(id, created); - createdIds?.add(id); return created; } +function mergeParticipantFact( + current: ParticipantFact | undefined, + incoming: ParticipantFact, +): ParticipantFact { + if (current === undefined) return incoming; + if ( + current.id !== incoming.id + || current.accountId !== incoming.accountId + || current.providerId !== incoming.providerId + ) return fail("one Beeper participant has conflicting source coordinates"); + if ( + current.isSelf !== null + && incoming.isSelf !== null + && current.isSelf !== incoming.isSelf + ) return fail("one Beeper participant has conflicting self-direction evidence"); + const merged = Object.freeze({ + ...current, + displayName: current.displayName ?? incoming.displayName, + handle: current.handle ?? incoming.handle, + isSelf: current.isSelf ?? incoming.isSelf, + }); + return merged.displayName === current.displayName + && merged.handle === current.handle + && merged.isSelf === current.isSelf + ? current + : merged; +} + +function planningParticipantFact( + account: BeeperAccountProjection, + user: Pick, + self: boolean | null, + aliasesByAccount: ReadonlyMap>, +): ParticipantFact { + const sourceId = canonicalParticipantSourceId(account, user.id, aliasesByAccount); + return Object.freeze({ + id: localId("participant", account.accountId, sourceId), + accountId: localId("account", account.accountId), + providerId: providerId("participant", account.accountId, sourceId), + displayName: user.fullName, + handle: user.phoneNumber ?? user.email ?? user.username, + isSelf: self, + }); +} + +function mergePlanningParticipantFact( + current: ParticipantFact | undefined, + incoming: ParticipantFact, +): ParticipantFact { + if (current === undefined) return incoming; + if ( + current.id !== incoming.id + || current.accountId !== incoming.accountId + || current.providerId !== incoming.providerId + ) return fail("one Beeper participant has conflicting source coordinates"); + return Object.freeze({ + ...current, + displayName: current.displayName ?? incoming.displayName, + handle: current.handle ?? incoming.handle, + isSelf: current.isSelf !== null + && incoming.isSelf !== null + && current.isSelf !== incoming.isSelf + ? false + : current.isSelf ?? incoming.isSelf, + }); +} + function messageTimestampRange( messages: readonly BeeperMessageProjection[], ): { readonly first: string | null; readonly last: string | null } { @@ -2447,6 +2937,14 @@ export function createBeeperMessageLikeMeSource( "test maxMessagesJsonBytes", MAX_MESSAGES_JSON_BYTES, ); + const maxParticipantOccurrences = + request.dependencies?.maxParticipantOccurrences === undefined + ? MAX_PARTICIPANT_OCCURRENCES + : positiveInteger( + request.dependencies.maxParticipantOccurrences, + "test maxParticipantOccurrences", + MAX_PARTICIPANT_OCCURRENCES, + ); if (!isAbsolute(binary)) return fail("Beeper CLI binary path must be absolute"); const customCreateWorking = request.dependencies?.createWorkingDirectory; const customRemoveWorking = request.dependencies?.removeWorkingDirectory; @@ -2773,8 +3271,12 @@ export function createBeeperMessageLikeMeSource( accountsById, maxMessagesJsonBytes, maxBundleRecords, + maxBundleBytes, + maxParticipantOccurrences, + accountObservedAt, request.signal, assertProgressHeartbeat, + request.dependencies?.onSelfAliasRefinementPass, ); const aliasesByAccount = selfAliasPrepass.aliasesByAccount; @@ -2798,6 +3300,7 @@ export function createBeeperMessageLikeMeSource( let messageLimitReached = false; let oversizedChatSkipped = false; let scannedRecordCount = accounts.length + selfParticipantByAccount.size; + const scannedParticipantIds = new Set(selfParticipantByAccount.values()); let scanRecordBudgetExhausted = false; const listedChatEntries = selfAliasPrepass.coveredChats; for (const { @@ -2868,22 +3371,29 @@ export function createBeeperMessageLikeMeSource( scanRecordBudgetExhausted = true; continue; } - const newlyCreatedParticipantIds = new Set(); + const selfParticipantId = selfParticipantByAccount.get(account.accountId); + if (selfParticipantId === undefined) { + return fail("Beeper account self participant disappeared"); + } + const selfParticipant = participantFacts.get(selfParticipantId); + if (selfParticipant === undefined) { + return fail("Beeper account self participant disappeared"); + } + const scanParticipantFacts = new Map([ + [selfParticipantId, selfParticipant], + ]); const participantIds = new Set(); for (const participant of chat.participants.items) { participantIds.add(upsertParticipant( - participantFacts, + scanParticipantFacts, account, participant, participant.isSelf, aliasesByAccount, - newlyCreatedParticipantIds, ).id); } if (chat.type === "single") { - const self = selfParticipantByAccount.get(account.accountId); - if (self === undefined) return fail("Beeper account self participant disappeared"); - participantIds.add(self); + participantIds.add(selfParticipantId); } const messageIds = new Set(messages.map((message) => message.id)); const reactionCount = messages.reduce( @@ -2907,36 +3417,39 @@ export function createBeeperMessageLikeMeSource( || !Number.isSafeInteger(tombstoneCount) ) return fail("official export derived record count overflowed"); for (const message of messages) { - participantIds.add(upsertParticipant(participantFacts, account, { + participantIds.add(upsertParticipant(scanParticipantFacts, account, { id: message.senderId, fullName: message.senderName, phoneNumber: null, email: null, username: null, - }, message.isSender, aliasesByAccount, newlyCreatedParticipantIds).id); + }, message.isSender, aliasesByAccount).id); for (const reaction of message.reactions) { - participantIds.add(upsertParticipant(participantFacts, account, { + participantIds.add(upsertParticipant(scanParticipantFacts, account, { id: reaction.participantId, fullName: null, phoneNumber: null, email: null, username: null, - }, null, aliasesByAccount, newlyCreatedParticipantIds).id); + }, null, aliasesByAccount).id); } } + const newlySeenParticipantIds = [...participantIds].filter( + (participantId) => !scannedParticipantIds.has(participantId), + ); const scanRecordCount = 1 + messages.length + reactionCount + tombstoneCount - + newlyCreatedParticipantIds.size; + + newlySeenParticipantIds.length; if (scanRecordCount > maxBundleRecords - scannedRecordCount) { - for (const participantId of newlyCreatedParticipantIds) { - participantFacts.delete(participantId); - } scanRecordBudgetExhausted = true; continue; } scannedRecordCount += scanRecordCount; + for (const participantId of newlySeenParticipantIds) { + scannedParticipantIds.add(participantId); + } const range = messageTimestampRange(messages); if (range.first !== null && (observedFrom === null || range.first < observedFrom)) { observedFrom = range.first; @@ -2945,7 +3458,7 @@ export function createBeeperMessageLikeMeSource( observedThrough = range.last; } const roster = [...participantIds].map((participantId) => { - const participant = participantFacts.get(participantId); + const participant = scanParticipantFacts.get(participantId); if (participant === undefined) { return fail("Beeper conversation participant disappeared"); } @@ -2974,6 +3487,15 @@ export function createBeeperMessageLikeMeSource( messagesSha256: messagesDocument.sha256, observedAt, participantIds: Object.freeze([...participantIds].sort()), + participantFactChanges: Object.freeze([...participantIds] + .sort() + .map((participantId) => { + const fact = scanParticipantFacts.get(participantId); + if (fact === undefined) { + return fail("Beeper conversation participant disappeared"); + } + return fact; + })), participantsComplete, startedAt: range.first, lastMessageAt: range.last, @@ -3033,7 +3555,8 @@ export function createBeeperMessageLikeMeSource( localId("conversation", left.chat.accountId, left.chat.id).localeCompare( localId("conversation", right.chat.accountId, right.chat.id), )); - const selectedParticipantIds = new Set(selfParticipantByAccount.values()); + const selectedParticipantFacts = new Map(participantFacts); + const selectedParticipantIds = new Set(selectedParticipantFacts.keys()); const selectedScans: ConversationScan[] = []; let selectedRecordCount = accounts.length + selectedParticipantIds.size; let selectedRecordBytes = 0; @@ -3058,34 +3581,56 @@ export function createBeeperMessageLikeMeSource( observedAtForAccount(account.accountId), )); } - let bundleRecordLimitReached = scanRecordBudgetExhausted; - let bundleByteLimitReached = false; + let bundleRecordLimitReached = selfAliasPrepass.bundleRecordLimitReached; + let bundleByteLimitReached = selfAliasPrepass.bundleByteLimitReached; for (const scan of orderedScans) { - let addedParticipants = 0; - let addedParticipantBytes = 0; - for (const participantId of scan.participantIds) { - if (selectedParticipantIds.has(participantId)) continue; - addedParticipants += 1; - const fact = participantFacts.get(participantId); - const account = fact === undefined - ? undefined - : accountsById.get(scan.chat.accountId); - if (fact === undefined || account === undefined) { - return fail("selected participant source identity disappeared"); + const account = accountsById.get(scan.chat.accountId); + if (account === undefined) { + return fail("selected participant source identity disappeared"); + } + const expectedParticipantAccountId = localId("account", account.accountId); + const stagedParticipantFacts = new Map(); + for (const incoming of scan.participantFactChanges) { + if (incoming.accountId !== expectedParticipantAccountId) { + return fail("selected participant source identity changed accounts"); } - addedParticipantBytes += bundleRecordBytes(participantRecord( + const current = stagedParticipantFacts.get(incoming.id) + ?? selectedParticipantFacts.get(incoming.id); + stagedParticipantFacts.set( + incoming.id, + mergeParticipantFact(current, incoming), + ); + } + let addedParticipants = 0; + let participantByteDelta = 0; + for (const [participantId, fact] of stagedParticipantFacts) { + const current = selectedParticipantFacts.get(participantId); + if (current === undefined) addedParticipants += 1; + const factBytes = bundleRecordBytes(participantRecord( fact, account, normalizeNetwork(account.network, account.bridge.type), - scan.observedAt, + observedAtForAccount(account.accountId), )); + const currentBytes = current === undefined + ? 0 + : bundleRecordBytes(participantRecord( + current, + account, + normalizeNetwork(account.network, account.bridge.type), + observedAtForAccount(account.accountId), + )); + participantByteDelta += factBytes - currentBytes; } const scanRecords = 1 + scan.messageCount + scan.reactionCount + scan.tombstoneCount + addedParticipants; - const scanBytes = scan.nonParticipantRecordBytes + addedParticipantBytes; + const scanBytes = scan.nonParticipantRecordBytes + participantByteDelta; + if (!Number.isSafeInteger(scanBytes) || scanBytes < 0) { + return fail("official export derived record bytes overflowed"); + } const exceedsRecords = scanRecords > maxBundleRecords - selectedRecordCount; const exceedsBytes = scanBytes > maxBundleBytes - selectedRecordBytes; if (exceedsRecords || exceedsBytes) { @@ -3096,10 +3641,21 @@ export function createBeeperMessageLikeMeSource( selectedScans.push(scan); selectedRecordCount += scanRecords; selectedRecordBytes += scanBytes; + for (const [participantId, fact] of stagedParticipantFacts) { + selectedParticipantFacts.set(participantId, fact); + } for (const participantId of scan.participantIds) { selectedParticipantIds.add(participantId); } } + if ( + scanRecordBudgetExhausted + || selectedScans.length !== orderedScans.length + ) return fail("bounded chat prefix exceeded its conservative admission proof"); + if ( + selectedRecordCount > selfAliasPrepass.provisionalRecordCount + || selectedRecordBytes > selfAliasPrepass.provisionalRecordBytes + ) return fail("exact bundle exceeded its provisional admission proof"); observedFrom = null; observedThrough = null; for (const scan of selectedScans) { @@ -3130,7 +3686,7 @@ export function createBeeperMessageLikeMeSource( selfParticipantId, ); } - for (const fact of [...participantFacts.values()] + for (const fact of [...selectedParticipantFacts.values()] .filter((candidate) => selectedParticipantIds.has(candidate.id)) .sort((left, right) => left.id.localeCompare(right.id))) { const account = accounts.find((candidate) => @@ -3247,6 +3803,9 @@ export function createBeeperMessageLikeMeSource( if (selfAliasPrepass.evidenceLimitReached) { warnings.add("self-alias-evidence-limit-reached"); } + if (selfAliasPrepass.participantOccurrenceLimitReached) { + warnings.add("participant-occurrence-limit-reached"); + } if (emittedReactionProviderIdNonUnique) { warnings.add("reaction-provider-id-non-unique"); } @@ -3255,7 +3814,8 @@ export function createBeeperMessageLikeMeSource( || oversizedChatSkipped || bundleRecordLimitReached || bundleByteLimitReached - || selfAliasPrepass.evidenceLimitReached; + || selfAliasPrepass.evidenceLimitReached + || selfAliasPrepass.participantOccurrenceLimitReached; stopProgressHeartbeat(); assertProgressHeartbeat(); completion = Object.freeze({ @@ -3267,6 +3827,8 @@ export function createBeeperMessageLikeMeSource( ? "bundle-byte-limit" : selfAliasPrepass.evidenceLimitReached ? "self-alias-evidence-limit" + : selfAliasPrepass.participantOccurrenceLimitReached + ? "participant-occurrence-limit" : oversizedChatSkipped ? "oversized-chat" : hardSourceLimitReached diff --git a/src/media/manifest.test.ts b/src/media/manifest.test.ts index 560a9a2..60a2dd4 100644 --- a/src/media/manifest.test.ts +++ b/src/media/manifest.test.ts @@ -465,7 +465,7 @@ function trackedYtDlpManifest( describe("Wrench media manifest", () => { test("uses one Wrench-owned schema and transcriber identity", () => { expect(WRENCH_MEDIA_SCHEMA_VERSION).toBe(1); - expect(WRENCH_MEDIA_VERSION).toBe("0.12.0"); + expect(WRENCH_MEDIA_VERSION).toBe("0.13.0"); expect(localTranscriptVariantSegments(localIdentity)).toEqual([ "transcript", "local", diff --git a/src/media/manifest.ts b/src/media/manifest.ts index fc790bc..e8d71ac 100644 --- a/src/media/manifest.ts +++ b/src/media/manifest.ts @@ -39,7 +39,7 @@ import { import { compareUtf8 } from "./utf8-order"; export const WRENCH_MEDIA_SCHEMA_VERSION = 1 as const; -export const WRENCH_MEDIA_VERSION = "0.12.0" as const; +export const WRENCH_MEDIA_VERSION = "0.13.0" as const; export const WRENCH_MEDIA_MANIFEST_FILE = "wrench-media.json" as const; export const WRENCH_MEDIA_CHECKSUM_FILE = "manifest-sha256.txt" as const; const MAX_ITEM_ENTRIES = 4_096; diff --git a/website/build.ts b/website/build.ts index 363d3b4..41091d3 100644 --- a/website/build.ts +++ b/website/build.ts @@ -19,7 +19,7 @@ export const REPOSITORY_URL = "https://github.com/hraness/wrench" as const; export const PUBLISHER_URL = "https://github.com/hraness" as const; export const SKILL_INSTALL_COMMAND = "npx skills add hraness/wrench" as const; export const SKILL_INSTALL_COMMAND_BUNX = "bunx skills add hraness/wrench" as const; -export const CONTENT_REVIEWED_RELEASE = "v0.12.0" as const; +export const CONTENT_REVIEWED_RELEASE = "v0.13.0" as const; export const DEFAULT_POSTHOG_HOST = "https://us.i.posthog.com" as const; export const PUBLIC_PAGES = [