diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index f061d3a1f..c3a9764c5 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -148,11 +148,21 @@ jobs: build-args: | PDPP_REFERENCE_REVISION=${{ github.sha }} labels: ${{ steps.meta.outputs.labels }} + load: ${{ matrix.image == 'core' }} platforms: linux/amd64 push: false - tags: ${{ steps.meta.outputs.tags }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ matrix.image == 'core' && 'pdpp:ci-core-test' || '' }} target: ${{ matrix.target }} + - name: Test core image (slackdump bundling, license, builder isolation) + if: (github.event_name != 'workflow_dispatch' || github.event.inputs.image == 'all' || github.event.inputs.image == 'core') && matrix.image == 'core' + env: + PDPP_CORE_IMAGE_TAG: pdpp:ci-core-test + run: | + node --test scripts/docker-slackdump-core-bundle.test.ts + publish: name: publish ${{ matrix.image }} if: github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') diff --git a/Dockerfile b/Dockerfile index 718409ad8..7cc04ab06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -105,6 +105,49 @@ EXPOSE 7662 7663 CMD ["sh", "-c", "export AS_PORT=\"${PORT:-${AS_PORT:-7662}}\"; export PDPP_RS_URL=\"${PDPP_RS_URL:-http://127.0.0.1:${RS_PORT:-7663}}\"; exec node reference-implementation/server/index.ts"] +# Isolated slackdump (v4.4.2, AGPL-3.0) builder stage. +# Downloads pre-built tarball, verifies SHA256, extracts binary and license. +# Only the binary (not build deps or Go) is copied to final image. +FROM debian:bookworm-slim AS slackdump-builder + +ARG TARGETARCH + +WORKDIR /build + +# Map Docker TARGETARCH to slackdump release tarball name +RUN case "${TARGETARCH}" in \ + x86_64|amd64) SLACKDUMP_TARBALL="slackdump_Linux_x86_64.tar.gz"; SLACKDUMP_SHA256="e2f386b2af30b0ba0ae98973f6a053225fba7d7127a20ad196cfdd96bf601052" ;; \ + arm64) SLACKDUMP_TARBALL="slackdump_Linux_arm64.tar.gz"; SLACKDUMP_SHA256="71d8b55b9132c0d39d6fe66e3542ee7d2ec6c032b7701928124c736611cc235e" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \ + esac && \ + echo "${SLACKDUMP_TARBALL}" > /tmp/tarball.txt && \ + echo "${SLACKDUMP_SHA256}" > /tmp/sha256.txt + +# Install only ca-certificates and curl; minimal runtime +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && \ + rm -rf /var/lib/apt/lists/* + +# Download from official GitHub release, verify SHA256, extract +RUN TARBALL=$(cat /tmp/tarball.txt) && \ + EXPECTED_SHA=$(cat /tmp/sha256.txt) && \ + curl -fsSL -o "${TARBALL}" "https://github.com/rusq/slackdump/releases/download/v4.4.2/${TARBALL}" && \ + ACTUAL_SHA=$(sha256sum "${TARBALL}" | awk '{print $1}') && \ + if [ "${EXPECTED_SHA}" != "${ACTUAL_SHA}" ]; then \ + echo "SHA256 mismatch for ${TARBALL}" >&2; \ + echo "Expected: ${EXPECTED_SHA}" >&2; \ + echo "Actual: ${ACTUAL_SHA}" >&2; \ + exit 1; \ + fi && \ + tar -xzf "${TARBALL}" && \ + test -x slackdump && \ + ./slackdump version + +# Download LICENSE and source reference from upstream +# AGPL section 6(d): Corresponding Source URL must resolve to exact versioned tree +RUN curl -fsSL -o LICENSE "https://raw.githubusercontent.com/rusq/slackdump/v4.4.2/LICENSE" && \ + echo "https://github.com/rusq/slackdump/tree/v4.4.2" > SOURCE_URL && \ + test -f LICENSE && test -s LICENSE + # Dedicated browsers stage. Patchright + bundled Chromium + (on amd64) Google # Chrome stable + their apt deps are baked into a stage whose cache key is # only the patchright version and target arch. This is independent of the @@ -174,9 +217,14 @@ CMD ["node", "reference-implementation/server/index.ts"] # tranche. See openspec/changes/split-public-site-and-operator-console. FROM base AS console +# The console image is paired with the current reference implementation, whose +# merged-timeline contract supports direction=asc. Keep the explicit capability +# gate enabled for that pairing; an older external RS can still fail closed by +# setting PDPP_EXPLORE_TIMELINE_DIRECTION=0. ENV NODE_ENV=production \ HOSTNAME=0.0.0.0 \ - PORT=3000 + PORT=3000 \ + PDPP_EXPLORE_TIMELINE_DIRECTION=1 COPY --from=console-builder /app/apps/console/.next/standalone ./ COPY --from=console-builder /app/apps/console/.next/static ./apps/console/.next/static @@ -222,6 +270,9 @@ ARG PDPP_REFERENCE_REVISION=unknown # [[restart]] override). If this stage is ever deployed through a path with # no restart policy, that deployment is the truthful gap to fix, not this # flag. +# Core bundles the matching reference implementation, including the +# direction=asc merged-timeline read contract. Keep the UI capability gate +# explicit; an older external RS can still fail closed with =0. ENV NODE_ENV=production \ HOSTNAME=0.0.0.0 \ PORT=3000 \ @@ -234,6 +285,7 @@ ENV NODE_ENV=production \ PDPP_BROWSER_PROFILE_ROOT=/var/lib/pdpp/browser-profiles \ PDPP_EMBEDDING_DOWNLOAD_ALLOWED=1 \ PDPP_EMBEDDING_CACHE_DIR=/var/lib/pdpp/transformers \ + PDPP_EXPLORE_TIMELINE_DIRECTION=1 \ PDPP_REFERENCE_OPERATIONAL_DEFAULTS=1 \ PDPP_LOCAL_TRANSFORMER_SUPERVISOR_RESTART_CONTRACT=1 \ PDPP_RECONCILE_POLYFILL_MANIFESTS=1 \ @@ -245,6 +297,15 @@ COPY --from=console-builder /app/apps/console/.next/standalone /console COPY --from=console-builder /app/apps/console/.next/static /console/apps/console/.next/static COPY --from=console-builder /app/apps/console/public /console/apps/console/public +# Copy slackdump binary (AGPL-3.0, v4.4.2) from builder stage. +# Binary required by Slack connector; upstream: https://github.com/rusq/slackdump/blob/v4.4.2 +COPY --from=slackdump-builder /build/slackdump /usr/local/bin/slackdump +COPY --from=slackdump-builder /build/LICENSE /usr/local/share/slackdump/LICENSE.agpl-3.0.txt +COPY --from=slackdump-builder /build/SOURCE_URL /usr/local/share/slackdump/SOURCE_URL + +# Verify slackdump is executable and functional +RUN chmod +x /usr/local/bin/slackdump && /usr/local/bin/slackdump version + EXPOSE 3000 CMD ["node", "--import", "tsx", "/app/deploy/railway/core-supervisor.ts"] diff --git a/apps/console/src/app/(console)/components/source-setup-catalog.invariants.test.ts b/apps/console/src/app/(console)/components/source-setup-catalog.invariants.test.ts new file mode 100644 index 000000000..9429f84c1 --- /dev/null +++ b/apps/console/src/app/(console)/components/source-setup-catalog.invariants.test.ts @@ -0,0 +1,28 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const SOURCE_SETUP_CATALOG_FILE = fileURLToPath(new URL("./source-setup-catalog.tsx", import.meta.url)); + +const EXTERNAL_DOCS = /externalDocs\.map/; +const NEW_TAB = /target="_blank"/; +const NEW_TAB_TITLE = /title="Opens in a new tab"/; +const NEW_TAB_COPY = /\(opens in a new tab\)/; +const NOREFERRER = /rel="noreferrer"/; + +test("source-setup-catalog renders external documentation links with new-tab forewarning", async () => { + const src = await readFile(SOURCE_SETUP_CATALOG_FILE, "utf8"); + assert.match(src, EXTERNAL_DOCS, "must render externalDocs links"); + assert.match(src, NEW_TAB, 'all external links must have target="_blank"'); + assert.match(src, NEW_TAB_COPY, "external documentation links must warn visibly before opening a new tab"); + assert.match( + src, + NEW_TAB_TITLE, + 'external documentation links must have title="Opens in a new tab" for accessibility/forewarning' + ); + assert.match(src, NOREFERRER, 'all external links must have rel="noreferrer" for security'); +}); diff --git a/apps/console/src/app/(console)/components/source-setup-catalog.tsx b/apps/console/src/app/(console)/components/source-setup-catalog.tsx index 8a0f38584..d8b8b5441 100644 --- a/apps/console/src/app/(console)/components/source-setup-catalog.tsx +++ b/apps/console/src/app/(console)/components/source-setup-catalog.tsx @@ -9,6 +9,7 @@ import type { RefCountState } from "../lib/ref-client.ts"; import { sourceSetupAction, sourceSetupAvailability, + sourceSetupContext, sourceSetupGuidance, sourceSetupRank, sourceSetupSecondaryAction, @@ -93,7 +94,7 @@ function SourceAcquisitionPaths({ paths }: { paths: readonly ConnectorAcquisitio const secondary = paths.filter((path) => !visibleLabels.has(path.label)); return (
-

Acquisition paths

+

Ways to add data

{sourceMethodLine(entry, existingSources.length)}

+
{action ? ( <> - Next + Next step {action.label} @@ -277,7 +299,7 @@ function SourceSetupCard({ className="pdpp-caption rounded-md border border-border/70 bg-muted/20 px-2.5 py-1 text-muted-foreground" data-testid="source-unavailable-fact" > - {unavailable ? "Not available from this page" : "No primary action"} + {unavailable ? "Not available from this page" : "No setup path available here"} )}
@@ -296,18 +318,37 @@ function ServerSetupSummary({ entries }: { entries: readonly ConnectorCatalogEnt

- These sources need provider app settings on this instance before an account can be added. + These sources need provider app settings on the instance before an account can be added. This dashboard shows + the missing requirements but does not edit provider applications here.

    {entries.map((entry) => ( -
  • +
  • {entry.displayName}

    {sourceMethodLine(entry, 0)}

    +
    - - Open server settings - +

    + {sourceSetupGuidance(entry)} +

    + {entry.externalDocs.length > 0 ? ( +
    + Provider documentation: + {entry.externalDocs.map((doc) => ( + + {doc.label} (opens in a new tab) + + ))} +
    + ) : null}
  • ))}
diff --git a/apps/console/src/app/(console)/components/views/acquisition-coverage-ui.invariants.test.ts b/apps/console/src/app/(console)/components/views/acquisition-coverage-ui.invariants.test.ts index 2194d943b..9bf024dcf 100644 --- a/apps/console/src/app/(console)/components/views/acquisition-coverage-ui.invariants.test.ts +++ b/apps/console/src/app/(console)/components/views/acquisition-coverage-ui.invariants.test.ts @@ -37,7 +37,7 @@ const STATUS_FILE = `${HERE}../../connect/status/[connectionId]/page.tsx`; const ONE_STATUS_AND_ACTION_COPY = /one status and one next action/; const COMPACT_METHOD_LINE = /function sourceMethodLine/; const SUPPORT_FACT_TEST_ID = /data-testid="source-support-fact"/; -const NEXT_COPY = />NextNext step { +test("manual upload page has owner-safe storage and validate-before-commit language", async () => { const pageSrc = await readFile(MANUAL_UPLOAD_FILE, "utf8"); const formSrc = await readFile(MANUAL_UPLOAD_FORM_FILE, "utf8"); - assert.match(pageSrc, MANIFEST_GENERATED_COPY); + assert.match(pageSrc, OWNER_SAFE_STORAGE_COPY); // Validates before durable commit when a validator exists. assert.match(formSrc, VALIDATES_BEFORE_COMMIT_COPY); // It speaks of a durable receipt the owner can revisit. diff --git a/apps/console/src/app/(console)/components/views/sources-ia.invariants.test.ts b/apps/console/src/app/(console)/components/views/sources-ia.invariants.test.ts index 7ad30a804..ea39c92e2 100644 --- a/apps/console/src/app/(console)/components/views/sources-ia.invariants.test.ts +++ b/apps/console/src/app/(console)/components/views/sources-ia.invariants.test.ts @@ -82,14 +82,15 @@ const SOURCES_PAGE_STATUS_HELPER_IMPORT_RE = /isActiveConnectorRunSummaryStatus[\s\S]*from "\.\.\/lib\/connector-run-summary-status\.ts"/; const SOURCES_PAGE_STATUS_HELPER_CALL_RE = /isActiveConnectorRunSummaryStatus\(\s*s\.last_run\.status\s*\)/; const LIST_CONNECTOR_MANIFESTS_RE = /listConnectorManifests\(\)/; -const BUILD_CONNECTOR_CATALOG_RE = /buildConnectorCatalog\(manifests\)/; +const LIST_OWNER_CONNECTOR_TEMPLATES_RE = /listOwnerConnectorTemplates\(\)/; +const BUILD_CONNECTOR_CATALOG_RE = /buildOwnerConnectorCatalog\(manifests, templates\)/; const SOURCE_SETUP_CATALOG_RE = / { const page = await readFile(RECORDS_ADD_PAGE_FILE, "utf8"); const catalog = await readFile(SOURCE_SETUP_CATALOG_FILE, "utf8"); assert.match(page, LIST_CONNECTOR_MANIFESTS_RE); + assert.match(page, LIST_OWNER_CONNECTOR_TEMPLATES_RE); assert.match(page, BUILD_CONNECTOR_CATALOG_RE); assert.match(page, SOURCE_SETUP_CATALOG_RE); assert.match(catalog, SOURCE_SETUP_SECTION_RE); @@ -228,7 +230,7 @@ test("source setup presentation has no connector-specific copy or examples", asy assert.doesNotMatch(src, FORBIDDEN_DEV_STRINGS_RE); }); -// ── 4. "Connect AI apps" is a separate read-access surface ────────────────── +// ── 4. "Connect apps" is a separate read-access surface ───────────────────── test("the nav names the inbound client surface 'Connect apps', distinct from Sources", async () => { const src = await readFile(SHELL_FILE, "utf8"); diff --git a/apps/console/src/app/(console)/components/views/standing-overview.tsx b/apps/console/src/app/(console)/components/views/standing-overview.tsx index 5cde2d7c6..5db6014fd 100644 --- a/apps/console/src/app/(console)/components/views/standing-overview.tsx +++ b/apps/console/src/app/(console)/components/views/standing-overview.tsx @@ -167,9 +167,7 @@ function RelationshipsBlock({ ))}
) : ( -

- No grant is out. Nothing is shared — only you and what you've given a token read this server. -

+

No grants are active. No connected app can read this instance yet.

)} ); diff --git a/apps/console/src/app/(console)/components/views/standing-view-model.ts b/apps/console/src/app/(console)/components/views/standing-view-model.ts index 086de1fb1..fcd05e66c 100644 --- a/apps/console/src/app/(console)/components/views/standing-view-model.ts +++ b/apps/console/src/app/(console)/components/views/standing-view-model.ts @@ -979,12 +979,12 @@ function buildDecideHero(pending: PendingApproval[], hrefs: StandingHrefs): Stan const more = pending.length - 1; const who = first ? clientLabel(first.client_id ?? null, first.approval_id) : "An app"; const reads = first ? approvalReads(first) : "parts of your data"; - const moreSub = `Nothing leaves until you say so — review each request one at a time. ${more} more after this one.`; + const moreSub = `No data is shared until you approve a request. Review each request one at a time. ${more} more after this one.`; return { cta: { href: hrefs.grants, human: true, label: "Review the request" }, kicker: pending.length === 1 ? "A request is waiting on you" : `${pending.length} requests are waiting`, line: { emphasis: reads, tail: ".", text: `${who} wants to read ` }, - sub: more > 0 ? moreSub : "Nothing leaves until you say so — approve it one piece at a time.", + sub: more > 0 ? moreSub : "No data is shared until you approve this request.", tone: "decide", }; } diff --git a/apps/console/src/app/(console)/connect/browser-session/[connectorId]/launch/start/route.ts b/apps/console/src/app/(console)/connect/browser-session/[connectorId]/launch/start/route.ts index 9177a5613..93c5830f6 100644 --- a/apps/console/src/app/(console)/connect/browser-session/[connectorId]/launch/start/route.ts +++ b/apps/console/src/app/(console)/connect/browser-session/[connectorId]/launch/start/route.ts @@ -6,6 +6,7 @@ import { isBrowserBoundConnector } from "../../../../../lib/connection-modality. import { requireDashboardAccess } from "../../../../../lib/dashboard-access.ts"; import { runConnectionNow } from "../../../../../lib/operator-runs.ts"; import { abandonBrowserEnrollmentShell } from "../../../../../lib/ref-client.ts"; +import { originMatchesHost } from "../../../../../lib/same-origin-route.ts"; import { type BrowserSessionRunStartResult, classifyBrowserSessionLaunchResult } from "../launch-result.ts"; export const dynamic = "force-dynamic"; @@ -19,22 +20,6 @@ function pagePath(connectorId: string): string { return `/connect/browser-session/${encodeURIComponent(connectorId)}/launch`; } -function originMatchesHost(request: Request): boolean { - const origin = request.headers.get("origin"); - if (!origin) { - return true; - } - const host = request.headers.get("host"); - if (!host) { - return false; - } - try { - return new URL(origin).host === host; - } catch { - return false; - } -} - function readRequiredStringField(formData: FormData, name: string): string { const value = formData.get(name); if (typeof value !== "string" || value.trim().length === 0) { diff --git a/apps/console/src/app/(console)/connect/browser-session/[connectorId]/page.tsx b/apps/console/src/app/(console)/connect/browser-session/[connectorId]/page.tsx index f9d96ed60..998d6fd6c 100644 --- a/apps/console/src/app/(console)/connect/browser-session/[connectorId]/page.tsx +++ b/apps/console/src/app/(console)/connect/browser-session/[connectorId]/page.tsx @@ -30,7 +30,13 @@ import Link from "next/link"; import { notFound } from "next/navigation"; import { RecordroomShellWithPalette } from "@/app/(console)/components/recordroom-shell-with-palette.tsx"; import { isBrowserBoundConnector, isSupportedBrowserCollectorConnector } from "../../../lib/connection-modality.ts"; -import { getStaticSecretSetup, type StaticSecretSetup, type StaticSecretSetupField } from "../../../lib/ref-client.ts"; +import { getStaticSecretSetup, type StaticSecretSetupField } from "../../../lib/ref-client.ts"; +import { + type BrowserOptionalCredentialContract, + browserSessionFormContract, + connectionNameFieldContract, + optionalCredentialFieldLabel, +} from "../../../lib/source-setup-form-contract.ts"; export const dynamic = "force-dynamic"; @@ -62,38 +68,33 @@ function inputType(field: StaticSecretSetupField): "email" | "password" | "text" } function OptionalStoredCredentialFields({ + credentials, searchParams, - setup, }: { + credentials: BrowserOptionalCredentialContract; searchParams: Record; - setup: StaticSecretSetup; }) { return (
- - Remember my sign-in details for automatic reconnection (optional) - -

- These manifest-defined details are encrypted and may help with initial sign-in or repair. CAPTCHA, OTP, - passkeys, and other human steps always stay in the secure browser; unattended reconnection is not guaranteed. -

+ {credentials.title} +

{credentials.description}

- {setup.credential_capture.fields.map((field) => ( + {credentials.fields.map((field) => (
diff --git a/apps/console/src/app/(console)/device-exporters/browser-bound-deep-link.test.ts b/apps/console/src/app/(console)/device-exporters/browser-bound-deep-link.test.ts index 2dbba0b75..9746ed6f4 100644 --- a/apps/console/src/app/(console)/device-exporters/browser-bound-deep-link.test.ts +++ b/apps/console/src/app/(console)/device-exporters/browser-bound-deep-link.test.ts @@ -37,8 +37,10 @@ const DEFAULT_ONLY_LOCAL_COLLECTOR = /const defaultConnectorId = isSupportedLocalCollectorConnector\(requestedConnector\) \? requestedConnector : undefined/; const DOES_NOT_USE_SUPPORTED_BROWSER_CLASSIFIER = /\bisSupportedBrowserCollectorConnector\b/; const RENDERS_NOTICE = /browserBoundRequest\s*\?\s*\s*Add source\s*<\/Link>/; +const CONNECT_APPS_LINK = /href="\/connect"[\s\S]{0,120}>\s*Connect apps\s*<\/Link>/; const FORBIDDEN_MONOREPO_COPY = /PDPP monorepo checkout|generated monorepo commands|Manual browser setup/; test("page classifies a browser-bound deep-link via the shared modality classifier (no scattered key checks)", async () => { @@ -67,7 +69,9 @@ test("browser-bound deep-link renders packaged-path-pending guidance, not monore const src = await read(PAGE_PATH); assert.match(src, RENDERS_NOTICE, "page must render packaged-path-pending browser guidance"); assert.match(src, PENDING_BROWSER_TITLE, "the notice must name the dashboard-browser setup boundary"); - assert.match(src, PACKAGED_PENDING_COPY, "the notice must say the owner-usable path is not packaged yet"); + assert.match(src, PACKAGED_PENDING_COPY, "the notice must say browser setup is not available yet"); + assert.match(src, ADD_SOURCE_LINK, "the notice must link to the Sources add page"); + assert.doesNotMatch(src, CONNECT_APPS_LINK, "browser setup must not send owners to Connect apps"); assert.doesNotMatch(src, FORBIDDEN_MONOREPO_COPY, "normal dashboard copy must not send owners to monorepo commands"); }); diff --git a/apps/console/src/app/(console)/device-exporters/enrollment-form.consistency.test.ts b/apps/console/src/app/(console)/device-exporters/enrollment-form.consistency.test.ts index 3c56ad41e..eb947aad1 100644 --- a/apps/console/src/app/(console)/device-exporters/enrollment-form.consistency.test.ts +++ b/apps/console/src/app/(console)/device-exporters/enrollment-form.consistency.test.ts @@ -29,17 +29,20 @@ function read(relPath: string): Promise { const FORM_PATH = "apps/console/src/app/(console)/device-exporters/enrollment-form.tsx"; const ACTIONS_PATH = "apps/console/src/app/(console)/device-exporters/actions.ts"; +const COLLECTOR_SETUP_HELPER = /pdppLocalCollectorSetupCommand/; const COLLECTOR_ENROLL_HELPER = /pdppLocalCollectorEnrollCommand/; const COLLECTOR_RUN_HELPER = /pdppLocalCollectorRunCommand/; const LOCAL_COLLECTOR_PACKAGE = /@pdpp\/local-collector/; const BROWSER_COLLECTOR_MONOREPO_COPY = /PDPP monorepo checkout|pnpm --dir|packages\/polyfill-connectors|browser-collector run command/; +const SETUP_TESTID = /data-testid="collector-setup-command"/; const ENROLL_TESTID = /data-testid="collector-enroll-command"/; const RUN_TESTID_CLAUDE = /data-testid={`collector-run-command-/; const SUPPORTED_CONNECTORS = /COLLECTOR_RUN_CONNECTORS\s*=\s*\["claude_code",\s*"codex"\]/; test("enrollment form derives the canonical local collector commands via shared helpers", async () => { const src = await read(FORM_PATH); + assert.match(src, COLLECTOR_SETUP_HELPER, "form must call pdppLocalCollectorSetupCommand as the primary path"); assert.match(src, COLLECTOR_ENROLL_HELPER, "form must call pdppLocalCollectorEnrollCommand"); assert.match(src, COLLECTOR_RUN_HELPER, "form must call pdppLocalCollectorRunCommand"); assert.match(src, LOCAL_COLLECTOR_PACKAGE, "form must surface the public @pdpp/local-collector path"); @@ -48,10 +51,20 @@ test("enrollment form derives the canonical local collector commands via shared test("enrollment form exposes stable test hooks for the rendered commands", async () => { const src = await read(FORM_PATH); + assert.match(src, SETUP_TESTID, "setup command must carry a stable data-testid"); assert.match(src, ENROLL_TESTID, "enroll command must carry a stable data-testid"); assert.match(src, RUN_TESTID_CLAUDE, "run command must carry a stable per-connector data-testid"); }); +test("guided setup command appears before the advanced enroll/run commands in the rendered form", async () => { + const src = await read(FORM_PATH); + const setupIndex = src.indexOf('data-testid="collector-setup-command"'); + const enrollIndex = src.indexOf('data-testid="collector-enroll-command"'); + assert.ok(setupIndex > -1, "setup command block must be present"); + assert.ok(enrollIndex > -1, "enroll command block must be present"); + assert.ok(setupIndex < enrollIndex, "setup must render before the low-level enroll/run commands"); +}); + test("enrollment form advertises claude_code and codex as the operator-ready connectors", async () => { const src = await read(FORM_PATH); assert.match(src, SUPPORTED_CONNECTORS, "claude_code and codex are the documented MVP collector lanes"); diff --git a/apps/console/src/app/(console)/device-exporters/enrollment-form.tsx b/apps/console/src/app/(console)/device-exporters/enrollment-form.tsx index cca2481e2..9ac3d5b28 100644 --- a/apps/console/src/app/(console)/device-exporters/enrollment-form.tsx +++ b/apps/console/src/app/(console)/device-exporters/enrollment-form.tsx @@ -6,11 +6,16 @@ import { IcButton, IcInput } from "@pdpp/brand-react"; import { CopyButton } from "@pdpp/operator-ui/components/copy-button"; import { Callout, ToolbarField } from "@pdpp/operator-ui/components/primitives"; -import { useActionState } from "react"; -import { pdppLocalCollectorEnrollCommand, pdppLocalCollectorRunCommand } from "@/lib/pdpp-cli-command.ts"; +import { useActionState, useState } from "react"; +import { + pdppLocalCollectorEnrollCommand, + pdppLocalCollectorRunCommand, + pdppLocalCollectorSetupCommand, +} from "@/lib/pdpp-cli-command.ts"; import { createEnrollmentCodeAction } from "./actions.ts"; const COLLECTOR_RUN_CONNECTORS = ["claude_code", "codex"] as const; +const SETUP_SAMPLE_SIZE = 20; export function EnrollmentForm({ referenceBaseUrl, @@ -28,9 +33,18 @@ export function EnrollmentForm({ defaultConnectorId?: string; }) { const [state, formAction, pending] = useActionState(createEnrollmentCodeAction, { ok: null }); + const [showAdvanced, setShowAdvanced] = useState(false); + let setupCommand: string | null = null; let enrollCommand: string | null = null; if (state.ok === true) { + setupCommand = pdppLocalCollectorSetupCommand({ + baseUrl: referenceBaseUrl, + code: state.code.enrollment_code, + connectorId: state.code.connector_id, + deviceLabel: state.deviceLabel, + sample: SETUP_SAMPLE_SIZE, + }); enrollCommand = pdppLocalCollectorEnrollCommand({ baseUrl: referenceBaseUrl, code: state.code.enrollment_code, @@ -40,7 +54,7 @@ export function EnrollmentForm({ return ( @@ -61,7 +75,7 @@ export function EnrollmentForm({ {state.ok === false ?

{state.message}

: null} - {state.ok === true && enrollCommand ? ( + {state.ok === true && setupCommand && enrollCommand ? (
Enrollment code
@@ -75,61 +89,97 @@ export function EnrollmentForm({
-
1. Enroll the host that has the data
+
Set up the device that has the data

- Run this @pdpp/local-collector command on the host with Claude Code or - Codex data. It uses the npx-launched pdpp-local-collector binary; no - PDPP source checkout is required. The JSON response returns device_id,{" "} - device_token, and source_instance_id{" "} - — persist all three without logging the token. + Run this @pdpp/local-collector command on the device with Claude Code + or Codex data. It exchanges the code, saves your device credentials to a local file only you can read + (never printed here or in your terminal), and runs a bounded {SETUP_SAMPLE_SIZE}-record proof pass so you + can see it working before it collects everything.

- {enrollCommand} + {setupCommand} - +
+

+ When that finishes, it prints the exact run command to collect the full + source — credentials are picked up automatically, no values to copy by hand. +

-
2. Run a connector pass
-

- Use the three values from the enrollment response. The command resumes from prior connector state via the - device-scoped STATE route; re-running is safe. -

-
- {COLLECTOR_RUN_CONNECTORS.map((connectorId) => { - const runCommand = pdppLocalCollectorRunCommand({ baseUrl: referenceBaseUrl, connectorId }); - const fullCommand = [ - "PDPP_LOCAL_DEVICE_ID= \\", - "PDPP_LOCAL_DEVICE_TOKEN= \\", - "PDPP_CONNECTION_ID= \\", - runCommand, - ].join("\n"); - return ( -
-
-
- {connectorId} -
- -
-
 setShowAdvanced((prev) => !prev)}
+              type="button"
+            >
+              {showAdvanced ? "Hide" : "Show"} advanced / scriptable commands
+            
+            {showAdvanced ? (
+              
+
+
1. Enroll the device (prints raw JSON)
+

+ Exchanges the code for a credential and prints it as JSON instead of saving a profile file — + for scripts that manage credentials themselves. The response returns{" "} + device_id, device_token, and{" "} + source_instance_id — save all three, and never log the + token. +

+
+ - {fullCommand} -
+ {enrollCommand} + +
- ); - })} -
+
+ +
+
2. Start collection with explicit env vars
+

+ Use the three values from the enrollment response. The collector resumes from its saved state, so + running it again is safe. +

+
+ {COLLECTOR_RUN_CONNECTORS.map((connectorId) => { + const runCommand = pdppLocalCollectorRunCommand({ baseUrl: referenceBaseUrl, connectorId }); + const fullCommand = [ + "PDPP_LOCAL_DEVICE_ID= \\", + "PDPP_LOCAL_DEVICE_TOKEN= \\", + "PDPP_CONNECTION_ID= \\", + runCommand, + ].join("\n"); + return ( +
+
+
+ {connectorId} +
+ +
+
+                            {fullCommand}
+                          
+
+ ); + })} +
+
+
+ ) : null} ) : null} diff --git a/apps/console/src/app/(console)/device-exporters/page.tsx b/apps/console/src/app/(console)/device-exporters/page.tsx index e3ebc96f8..d393c358d 100644 --- a/apps/console/src/app/(console)/device-exporters/page.tsx +++ b/apps/console/src/app/(console)/device-exporters/page.tsx @@ -85,7 +85,7 @@ export default async function DeviceExportersPage({ @@ -102,12 +102,12 @@ export default async function DeviceExportersPage({
{devices.length === 0 ? ( ) : ( @@ -140,15 +140,14 @@ function BrowserBoundEnrollmentNotice({ connectorId }: { connectorId: string }) return (

- Existing collected data remains usable. Adding another account is waiting on the packaged browser setup flow. - See the full add-source list on the{" "} - - Connect + Existing data remains available. Adding another account is not available yet. See available setup paths on the{" "} + + Add source {" "} page.

diff --git a/apps/console/src/app/(console)/device-exporters/reenroll-button.tsx b/apps/console/src/app/(console)/device-exporters/reenroll-button.tsx index c5b672446..d5d51833a 100644 --- a/apps/console/src/app/(console)/device-exporters/reenroll-button.tsx +++ b/apps/console/src/app/(console)/device-exporters/reenroll-button.tsx @@ -53,9 +53,9 @@ export function ReenrollButton({
1. Re-enroll this device

- Run on the target host. The response returns device_id,{" "} + Run on the target device. The response returns device_id,{" "} device_token, and source_instance_id — - persist all three. + save all three, and never log the token.

-
2. Run a connector pass
+
2. Start collection
diff --git a/apps/console/src/app/(console)/explore/explore-navigation.test.ts b/apps/console/src/app/(console)/explore/explore-navigation.test.ts index 388a949d0..6be775d53 100644 --- a/apps/console/src/app/(console)/explore/explore-navigation.test.ts +++ b/apps/console/src/app/(console)/explore/explore-navigation.test.ts @@ -153,6 +153,30 @@ test("order FLIP is feed-defining: re-pages from page 1 (drops anchor + cursor t assert.equal(params.get("ucursors"), null, "an order flip resets the upcoming trail too (fresh snapshot)"); }); +test("order FLIP preserves the active query and filters while resetting pagination", () => { + const state = accumulatingState({ + connectionIds: ["cin_ynab"], + excludeConnectionIds: ["cin_private"], + excludeStreams: ["secrets"], + query: "coffee", + since: "2026-06-01", + streams: ["transactions"], + until: "2026-07-01", + }); + const params = paramsOf(buildNavigateHref(EXPLORE, state, { order: "oldest" })); + + assert.equal(params.get("q"), "coffee", "the sort flip must keep the text query"); + assert.deepEqual(params.getAll("connection"), ["cin_ynab"], "the sort flip must keep included sources"); + assert.deepEqual(params.getAll("xconnection"), ["cin_private"], "the sort flip must keep excluded sources"); + assert.deepEqual(params.getAll("stream"), ["transactions"], "the sort flip must keep included streams"); + assert.deepEqual(params.getAll("xstream"), ["secrets"], "the sort flip must keep excluded streams"); + assert.equal(params.get("since"), "2026-06-01", "the sort flip must keep the lower time bound"); + assert.equal(params.get("until"), "2026-07-01", "the sort flip must keep the upper time bound"); + assert.equal(params.get("order"), "oldest", "the sort flip must select oldest-first"); + assert.equal(params.get("anchor"), null, "the sort flip must reset the old snapshot anchor"); + assert.equal(params.get("cursors"), null, "the sort flip must reset the old cursor trail"); +}); + test("a same-value order (carried forward by a peek) is NOT feed-defining (same feed)", () => { // A peek that carries the current order forward unchanged (newest == state.order) // is a pure same-feed move — the accumulated trail + anchor must survive. diff --git a/apps/console/src/app/(console)/explore/page.invariants.test.ts b/apps/console/src/app/(console)/explore/page.invariants.test.ts index 31949e417..84fb3c573 100644 --- a/apps/console/src/app/(console)/explore/page.invariants.test.ts +++ b/apps/console/src/app/(console)/explore/page.invariants.test.ts @@ -260,6 +260,12 @@ const FEED_PENDING_DIM_RE = const TYPEAHEAD_VIEWPORT_CLAMP_RE = /\.rr-x-typeahead \{(?=[\s\S]*?left: 0;)(?=[\s\S]*?right: 0;)(?=[\s\S]*?max-width: 100vw;)(?=[\s\S]*?max-height: min\(280px, 60vh\);)(?=[\s\S]*?overflow-y: auto;)[\s\S]*?\}/; +// Mobile: the options popover (operator-syntax legend) becomes a full-width sheet, +// same as the date popover, so it doesn't overflow when the trigger wraps near the +// left edge. The override turns off the absolute right:0 anchor and spans full width. +const OPTIONS_VIEWPORT_CLAMP_RE = + /\.rr-x-options__body \{(?=[\s\S]*?right: 0;)(?=[\s\S]*?left: 0;)(?=[\s\S]*?width: auto;)(?=[\s\S]*?max-width: none;)[\s\S]*?\}/; + // (#7) Motion communicates model state and is reduced-motion gated with a static // fallback. The shared reveal (Upcoming body / burst expand / day-group mount): // - base rule = STATIC visible fallback (opacity:1; transform:none) OUTSIDE any @@ -738,6 +744,14 @@ test("Slice 5 (#6): the operators/typeahead popover stays within the viewport (p TYPEAHEAD_VIEWPORT_CLAMP_RE, "the typeahead must be left:0/right:0 anchored to the input AND clamped by max-width:100vw + max-height:min(280px,60vh) + overflow-y:auto so it never runs off-screen" ); + // Mobile override: the options popover (operator-syntax legend) also becomes a + // full-width sheet to match DateChip's mobile pattern, preventing leftward overflow + // when the trigger wraps near the left edge on narrow screens. + assert.match( + css, + OPTIONS_VIEWPORT_CLAMP_RE, + "the options popover must have a mobile override with right:0 + left:0 + width:auto + max-width:none to become a full-width sheet and prevent overflow" + ); }); test("Slice 5 (#7): expansion/load-more motion is reduced-motion gated with a static fallback and causes no layout shift", async () => { diff --git a/apps/console/src/app/(console)/grants/page.tsx b/apps/console/src/app/(console)/grants/page.tsx index 33cbb395e..f01f7f407 100644 --- a/apps/console/src/app/(console)/grants/page.tsx +++ b/apps/console/src/app/(console)/grants/page.tsx @@ -134,7 +134,7 @@ export default async function GrantsPage({ searchParams }: { searchParams: Promi when there is something waiting. */} {approvals.data.length > 0 ? (
@@ -146,12 +146,11 @@ export default async function GrantsPage({ searchParams }: { searchParams: Promi ))}

- These dashboard shortcut buttons work in open local-dev mode. If placeholder owner auth is enabled, sign in - at{" "} + These approval shortcuts are available only in local approval mode. Otherwise, sign in through{" "} owner access {" "} - and approve there instead. + and approve the request there.

) : null} @@ -162,7 +161,7 @@ export default async function GrantsPage({ searchParams }: { searchParams: Promi activeFilterChips: activeFilters, buildListHref: (overrides) => listHref(params, overrides), description: "Issued authorizations and lifecycle decisions for client access to owner data.", - emptyHint: "Grant artifacts appear after client/provider-connect consent flows issue or reject grants.", + emptyHint: "Grants appear after a connected app requests access and the request is approved, denied, or fails.", emptyTitle: "No grants yet", filters: { query: { defaultValue: params.q ?? "", name: "q", placeholder: "id contains…" }, diff --git a/apps/console/src/app/(console)/lib/connection-catalog.test.ts b/apps/console/src/app/(console)/lib/connection-catalog.test.ts index 88a96cbcc..e4d2d82fc 100644 --- a/apps/console/src/app/(console)/lib/connection-catalog.test.ts +++ b/apps/console/src/app/(console)/lib/connection-catalog.test.ts @@ -19,18 +19,24 @@ import { browserBoundRunbookEntries, browserCollectorEntries, buildConnectorCatalog, + buildOwnerConnectorCatalog, type CatalogManifestLike, catalogModalityFromManifest, deploymentBlockedEntries, + isOwnerActionableEntry, localCollectorEntries, localCollectorUnprovenEntries, manualUploadConnectEntries, manualUploadPendingEntries, + type OwnerConnectorTemplateLike, + providerAuthConnectEntries, staticSecretConnectEntries, unsupportedNetworkEntries, } from "./connection-catalog.ts"; import { sourceSetupAction, + sourceSetupAvailability, + sourceSetupContext, sourceSetupGuidance, sourceSetupSecondaryAction, sourceSetupStatus, @@ -38,8 +44,14 @@ import { const FIRST_PARTY_REGISTRY_PREFIX = "https://registry.pdpp.org/connectors/"; const TRAILING_SLASH_RE = /\/$/; -const SECURE_BROWSER_SESSION_RE = /secure browser session/i; +const SECURE_BROWSER_RE = /secure browser/i; const SAVE_SIGN_IN_DETAILS_RE = /sign-in details/i; +const DATA_PORTABILITY_SEPARATE_RE = /separate from Google Maps Timeline Import/; +const TIMELINE_API_DISTINCTION_RE = /not exposed by Google's documented Data Portability API/i; +const TIMELINE_NO_SIGN_IN_RE = /no Google account sign-in is used/i; +const GOOGLE_DEPLOYMENT_BLOCKER_RE = /GOOGLE_DATAPORTABILITY_CLIENT_ID/; +const PROVIDER_BROWSER_GUIDANCE_RE = /provider's browser/; +const OWNER_INTENT_URL_RE = /\/v1\/owner\/connections\/intents$/; function canonicalKeyFromManifestId(connectorId: string): string { if (connectorId.startsWith(FIRST_PARTY_REGISTRY_PREFIX)) { @@ -64,6 +76,52 @@ async function loadCommittedManifests(): Promise { return parsed.filter((m) => m.connector_id); } +function ownerTemplate( + args: { + actionMethod?: string | null; + actionStatus?: string; + actionUrl?: string | null; + connectorKey?: string; + connectorModality?: string; + disposition?: string; + enrollmentKey?: string | null; + listingStatus?: string; + nextStepKind?: string; + ownerActionable?: boolean; + proofGate?: string | null; + setupModality?: string; + supportState?: string; + } = {} +): OwnerConnectorTemplateLike { + const connectorKey = args.connectorKey ?? "test-provider"; + return { + connector_key: connectorKey, + connector_modality: args.connectorModality ?? "api_network", + display_name: connectorKey, + public_listing: { listed: true, status: args.listingStatus ?? "proven" }, + registration_status: "registered", + setup_plan: { + catalog_disposition: args.disposition ?? "provider_auth_connect", + deployment_readiness: { blockers: [], guidance: null, state: "ready" }, + enrollment_key: args.enrollmentKey ?? null, + next_step_kind: args.nextStepKind ?? "open_provider_auth", + owner_actionable: args.ownerActionable ?? true, + proof_gate: args.proofGate ?? null, + runbook_path: null, + setup_modality: args.setupModality ?? "provider_authorization", + support_state: args.supportState ?? "supported", + }, + supported_actions: [ + { + family: "initiate_connection", + method: args.actionMethod === undefined ? "POST" : args.actionMethod, + status: args.actionStatus ?? "supported", + url: args.actionUrl === undefined ? "https://reference.test/v1/owner/connections/intents" : args.actionUrl, + }, + ], + }; +} + test("catalogModalityFromManifest mirrors the filesystem>browser>network precedence", () => { assert.equal(catalogModalityFromManifest({ connector_id: "x", runtime_requirements: { bindings: {} } }), "unknown"); assert.equal( @@ -119,7 +177,7 @@ test("only proven-creatable dispositions carry an enrollment deep-link key", asy } }); -test("heb is cataloged as one browser-bound account setup with optional stored credentials", async () => { +test("unproven browser-bound static-secret entries fail closed", async () => { const catalog = buildConnectorCatalog(await loadCommittedManifests()); const heb = catalog.find((entry) => entry.connectorKey === "heb"); if (!heb) { @@ -129,16 +187,17 @@ test("heb is cataloged as one browser-bound account setup with optional stored c assert.equal(heb.setupModality, "static_secret"); assert.equal(heb.disposition, "static_secret_connect"); assert.equal(heb.enrollmentKey, undefined); - assert.equal(sourceSetupStatus(heb).label, "Connect account"); - assert.equal(sourceSetupAction(heb)?.href, "/connect/browser-session/heb"); - assert.equal(sourceSetupAction(heb)?.label, "Connect account"); + assert.equal(heb.supportState, "proof_gated"); + assert.equal(heb.proofGate, "static_secret_live_proof_missing"); + assert.equal(sourceSetupStatus(heb).label, "Not available here"); + assert.equal(sourceSetupAction(heb), null); assert.equal(sourceSetupSecondaryAction(heb), null); - assert.match(sourceSetupGuidance(heb), SECURE_BROWSER_SESSION_RE); - assert.match(sourceSetupGuidance(heb), SAVE_SIGN_IN_DETAILS_RE); + assert.doesNotMatch(sourceSetupGuidance(heb), SECURE_BROWSER_RE); + assert.doesNotMatch(sourceSetupGuidance(heb), SAVE_SIGN_IN_DETAILS_RE); assert.deepEqual(browserCollectorEntries(catalog), []); }); -test("browser-bound static-secret-capable connectors get one primary choice generically", () => { +test("browser-bound static-secret capability is not enough to create an account", () => { const catalog = buildConnectorCatalog([ { connector_id: "https://registry.pdpp.org/connectors/browser-sample", @@ -159,9 +218,9 @@ test("browser-bound static-secret-capable connectors get one primary choice gene assert.equal(entry.modality, "browser_bound"); assert.equal(entry.setupModality, "static_secret"); assert.equal(entry.disposition, "static_secret_connect"); - assert.equal(sourceSetupAction(entry)?.href, "/connect/browser-session/browser-sample"); + assert.equal(sourceSetupAction(entry), null); assert.equal(sourceSetupSecondaryAction(entry), null); - assert.equal(sourceSetupStatus(entry).label, "Connect account"); + assert.equal(sourceSetupStatus(entry).label, "Not available here"); }); test("non-browser static-secret connectors keep the existing single capture path", () => { @@ -190,6 +249,24 @@ test("non-browser static-secret connectors keep the existing single capture path assert.equal(sourceSetupStatus(entry).label, "Add account"); }); +test("YNAB static-secret entry shows as actionable with draft-create path", async () => { + const manifests = await loadCommittedManifests(); + const catalog = buildConnectorCatalog(manifests); + const ynab = catalog.find((entry) => entry.connectorKey === "ynab"); + assert.ok(ynab, "ynab must be present in the catalog"); + assert.equal(ynab.modality, "api_network"); + assert.equal(ynab.setupModality, "static_secret"); + assert.equal(ynab.disposition, "static_secret_connect"); + assert.equal(ynab.enrollmentKey, undefined); + assert.equal(ynab.supportState, "supported"); + assert.equal(ynab.proofGate, null); + assert.equal(sourceSetupStatus(ynab).label, "Add account"); + assert.equal(sourceSetupAction(ynab)?.href, "/connect/static-secret/ynab"); + assert.equal(sourceSetupSecondaryAction(ynab), null); + assert.equal(sourceSetupAvailability(ynab), "available_now"); + assert.ok(sourceSetupGuidance(ynab).includes("protected setup form")); +}); + test("no browser-bound or API/network connector is one-click-creatable", async () => { const catalog = buildConnectorCatalog(await loadCommittedManifests()); for (const entry of catalog) { @@ -400,6 +477,8 @@ test("provider-authorization deployment blockers are separate from unsupported n ); assert.deepEqual(deploymentBlockedEntries(catalog), [entry]); assert.deepEqual(unsupportedNetworkEntries(catalog), []); + assert.equal(sourceSetupAction(entry), null); + assert.equal(sourceSetupAvailability(entry), "requires_server_setup"); assert.equal(entry.enrollmentKey, undefined); }); @@ -418,9 +497,198 @@ test("Google Maps Data Portability is the API-backed provider-auth source, not T entry.deploymentReadiness.blockers.map((blocker) => blocker.key), ["GOOGLE_DATAPORTABILITY_CLIENT_ID", "GOOGLE_DATAPORTABILITY_CLIENT_SECRET", "GOOGLE_DATAPORTABILITY_REDIRECT_URI"] ); + assert.equal(sourceSetupAction(entry), null, "provider settings must not link to diagnostics as a setup CTA"); + assert.match(sourceSetupGuidance(entry), GOOGLE_DEPLOYMENT_BLOCKER_RE); + assert.match(sourceSetupContext(entry) ?? "", DATA_PORTABILITY_SEPARATE_RE); + assert.ok(entry.externalDocs.length >= 1, "provider-auth manifest documentation should remain available"); assert.equal(entry.enrollmentKey, undefined); }); +test("Google Maps Timeline keeps its import/API distinction visible in the catalog", async () => { + const catalog = buildConnectorCatalog(await loadCommittedManifests()); + const entry = catalog.find((candidate) => candidate.connectorKey === "google-maps"); + assert.ok(entry, "google-maps must be in the committed catalog"); + assert.match(sourceSetupContext(entry) ?? "", TIMELINE_API_DISTINCTION_RE); + assert.match(sourceSetupContext(entry) ?? "", TIMELINE_NO_SIGN_IN_RE); +}); + +test("configured Google provider readiness exposes the existing owner authorization action", async () => { + const catalog = buildConnectorCatalog(await loadCommittedManifests(), ["google-maps-data-portability"]); + const entry = catalog.find((candidate) => candidate.connectorKey === "google-maps-data-portability"); + assert.ok(entry, "google-maps-data-portability must be in the committed catalog"); + assert.equal(entry.deploymentReadiness.state, "ready"); + assert.equal(entry.nextStepKind, "open_provider_auth"); + assert.equal(entry.supportState, "supported"); + assert.equal(entry.disposition, "provider_auth_connect"); + assert.equal(sourceSetupStatus(entry).label, "Authorize account"); + assert.match(sourceSetupGuidance(entry), PROVIDER_BROWSER_GUIDANCE_RE); + assert.deepEqual(sourceSetupAction(entry), { + href: "/connect/provider-auth/google-maps-data-portability", + label: "Authorize account", + }); + assert.equal(sourceSetupAvailability(entry), "available_now"); + assert.deepEqual(providerAuthConnectEntries(catalog), [entry]); +}); + +test("owner catalog fails closed for local-only, listed-unproven, and proof-gated static-secret entries", () => { + const staleLocalManifest: CatalogManifestLike = { + capabilities: { public_listing: { listed: true, status: "proven" } }, + connector_id: "stale-local-only", + display_name: "Stale local-only", + runtime_requirements: { bindings: { network: {} } }, + }; + assert.deepEqual(buildOwnerConnectorCatalog([staleLocalManifest], []), [], "local-only entries are not listed"); + + const listedUnproven = buildOwnerConnectorCatalog( + [], + [ + ownerTemplate({ + connectorKey: "listed-unproven", + connectorModality: "local_collector", + disposition: "local_collector_enroll", + listingStatus: "unproven", + nextStepKind: "enroll_local_collector", + ownerActionable: false, + setupModality: "local_collector", + actionMethod: null, + actionStatus: "unsupported", + actionUrl: null, + }), + ] + ); + const [listedUnprovenEntry] = listedUnproven; + assert.ok(listedUnprovenEntry, "server listing status controls visibility"); + assert.equal(listedUnprovenEntry.ownerActionable, false); + assert.equal(sourceSetupAction(listedUnprovenEntry), null); + assert.equal(sourceSetupAvailability(listedUnprovenEntry), "not_available_here"); + + const proofGatedStaticSecret = buildOwnerConnectorCatalog( + [], + [ + ownerTemplate({ + connectorKey: "unproven-static-secret", + disposition: "static_secret_connect", + nextStepKind: "capture_static_secret", + ownerActionable: false, + proofGate: "static_secret_live_proof_missing", + setupModality: "static_secret", + supportState: "proof_gated", + actionMethod: null, + actionStatus: "unsupported", + actionUrl: null, + }), + ] + ); + const [proofGatedEntry] = proofGatedStaticSecret; + assert.ok(proofGatedEntry, "proof-gated static-secret template should remain visible"); + assert.equal(proofGatedEntry.supportState, "proof_gated"); + assert.equal(sourceSetupStatus(proofGatedEntry).label, "Not available here"); + assert.equal(sourceSetupAction(proofGatedEntry), null); + assert.equal(sourceSetupAvailability(proofGatedEntry), "not_available_here"); +}); + +test("owner catalog exposes browser owner-session actions without inventing an owner-agent REST action", () => { + const catalog = buildOwnerConnectorCatalog( + [], + [ + ownerTemplate({ + connectorKey: "chatgpt", + connectorModality: "browser_bound", + disposition: "static_secret_connect", + nextStepKind: "capture_static_secret", + ownerActionable: true, + proofGate: "static_secret_live_proof_missing", + setupModality: "static_secret", + supportState: "proof_gated", + actionMethod: null, + actionStatus: "owner_mediated", + actionUrl: null, + }), + ownerTemplate({ + connectorKey: "chase", + connectorModality: "browser_bound", + disposition: "browser_collector_manual", + enrollmentKey: "chase", + nextStepKind: "enroll_browser_collector", + ownerActionable: true, + proofGate: "browser_collector_live_proof_missing", + setupModality: "browser_bound", + supportState: "proof_gated", + actionMethod: null, + actionStatus: "owner_mediated", + actionUrl: null, + }), + ownerTemplate({ + connectorKey: "doordash", + connectorModality: "browser_bound", + disposition: "browser_bound_runbook", + nextStepKind: "manual_runbook", + ownerActionable: false, + proofGate: "browser_collector_live_proof_missing", + setupModality: "browser_bound", + supportState: "proof_gated", + actionMethod: null, + actionStatus: "unsupported", + actionUrl: null, + }), + ] + ); + + const chatgpt = catalog.find((entry) => entry.connectorKey === "chatgpt"); + assert.ok(chatgpt); + assert.equal(chatgpt.ownerActionable, true); + assert.equal(chatgpt.ownerActionMethod, null); + assert.equal(chatgpt.ownerActionUrl, null); + assert.equal(sourceSetupAvailability(chatgpt), "available_now"); + assert.deepEqual(sourceSetupAction(chatgpt), { + href: "/connect/browser-session/chatgpt", + label: "Connect account", + }); + + const chase = catalog.find((entry) => entry.connectorKey === "chase"); + assert.ok(chase); + assert.equal(chase.ownerActionable, true); + assert.equal(sourceSetupAvailability(chase), "available_now"); + assert.deepEqual(sourceSetupAction(chase), { + href: "/connect/browser-session/chase", + label: "Connect account", + }); + + const doordash = catalog.find((entry) => entry.connectorKey === "doordash"); + assert.ok(doordash); + assert.equal(doordash.ownerActionable, false); + assert.equal(sourceSetupAvailability(doordash), "not_available_here"); + assert.equal(sourceSetupAction(doordash), null); +}); + +test("a supported owner action has an invariant actionable provider projection", () => { + const catalog = buildOwnerConnectorCatalog( + [ + { + connector_id: "test-provider", + display_name: "Provider display from manifest", + external_docs: [{ label: "Provider docs", url: "https://example.test/docs" }], + }, + ], + [ownerTemplate({ connectorKey: "test-provider", listingStatus: "needs_human_auth" })] + ); + const [entry] = catalog; + assert.ok(entry); + assert.equal(entry.ownerActionable, true); + assert.equal(entry.disposition, "provider_auth_connect"); + assert.equal(entry.supportState, "supported"); + assert.equal(entry.proofGate, null); + assert.equal(entry.ownerActionMethod, "POST"); + assert.match(entry.ownerActionUrl ?? "", OWNER_INTENT_URL_RE); + assert.equal(sourceSetupAvailability(entry), "available_now"); + assert.deepEqual(sourceSetupAction(entry), { + href: "/connect/provider-auth/test-provider", + label: "Authorize account", + }); + assert.deepEqual(entry.externalDocs, [{ label: "Provider docs", url: "https://example.test/docs" }]); + assert.equal(entry.displayName, "test-provider"); +}); + test("claude-code manifest slug maps to the claude_code enrollment key", async () => { // The manifest slug is `claude-code` (hyphen); the proven enrollment path and // the form's COLLECTOR_RUN_CONNECTORS literal use `claude_code` (underscore). @@ -452,6 +720,7 @@ test("the grouping helpers partition the catalog without overlap or loss", async manualUploadConnectEntries(catalog), manualUploadPendingEntries(catalog), deploymentBlockedEntries(catalog), + providerAuthConnectEntries(catalog), unsupportedNetworkEntries(catalog), ]; const total = groups.reduce((sum, g) => sum + g.length, 0); @@ -488,3 +757,138 @@ test("filesystem connectors outside the proven set are local-collector-unproven, assert.notEqual(entry.modality, "local_collector"); } }); + +test("ownerActionable field is the sole authority for live owner catalogs", () => { + // Live owner catalogs always compute ownerActionable once in + // buildOwnerConnectorCatalog and store it. isOwnerActionableEntry must + // trust that field, not re-derive it. + const catalog = buildOwnerConnectorCatalog( + [], + [ + ownerTemplate({ + connectorKey: "actionable-api", + disposition: "provider_auth_connect", + ownerActionable: true, + }), + ownerTemplate({ + connectorKey: "blocked-api", + disposition: "provider_auth_connect", + ownerActionable: false, + }), + ownerTemplate({ + connectorKey: "actionable-static-secret", + connectorModality: "browser_bound", + disposition: "static_secret_connect", + setupModality: "static_secret", + nextStepKind: "capture_static_secret", + ownerActionable: true, + actionMethod: null, + actionStatus: "owner_mediated", + actionUrl: null, + }), + ownerTemplate({ + connectorKey: "blocked-static-secret", + connectorModality: "browser_bound", + disposition: "static_secret_connect", + setupModality: "static_secret", + nextStepKind: "capture_static_secret", + ownerActionable: false, + actionMethod: null, + actionStatus: "unsupported", + actionUrl: null, + }), + ] + ); + + for (const entry of catalog) { + const actionable = isOwnerActionableEntry(entry); + assert.equal( + actionable, + entry.ownerActionable, + `isOwnerActionableEntry(${entry.connectorKey}) must return the ownerActionable field: ${entry.ownerActionable}` + ); + } +}); + +test("isOwnerActionableEntry respects demo/test fallback rules when ownerActionable is undefined", async () => { + // Pure manifest catalogs (buildConnectorCatalog) have no ownerActionable field. + // isOwnerActionableEntry must apply fallback rules for testing. YNAB is proven + // to be supported (not proof-gated), so it serves as a good fallback test. + const manifests = await loadCommittedManifests(); + const catalog = buildConnectorCatalog(manifests); + + const ynab = catalog.find((e) => e.connectorKey === "ynab"); + assert.ok(ynab); + assert.equal(ynab.ownerActionable, undefined); + assert.equal(ynab.setupModality, "static_secret"); + assert.equal(ynab.supportState, "supported"); + assert.equal(ynab.proofGate, null); + // Fallback rule: static_secret dispositions are actionable if supported and not proof-gated + assert.equal(isOwnerActionableEntry(ynab), true); +}); + +test("presentation consistency: helper functions agree with ownerActionable authority", async () => { + // Every fixture in the presentation test suite must have presentation functions + // that agree with isOwnerActionableEntry. This is the core maintainability check. + const manifests = await loadCommittedManifests(); + const catalog = buildConnectorCatalog(manifests); + + for (const entry of catalog) { + const isActionable = isOwnerActionableEntry(entry); + const hasAction = sourceSetupAction(entry) !== null; + + // The invariant: if isOwnerActionableEntry returns true, sourceSetupAction + // must have a non-null result. Mutations to either would break this. + assert.equal( + hasAction, + isActionable, + `${entry.connectorKey}: sourceSetupAction must match isOwnerActionableEntry. ` + + `Helper says ${isActionable}, action is ${hasAction ? "set" : "null"}` + ); + } +}); + +test("owner catalog: presentation consistency between actionability and availability", () => { + // For owner catalogs, ownerActionable gates both sourceSetupAvailability and + // sourceSetupAction. They must converge. + const catalog = buildOwnerConnectorCatalog( + [], + [ + ownerTemplate({ + connectorKey: "available", + disposition: "provider_auth_connect", + ownerActionable: true, + }), + ownerTemplate({ + connectorKey: "not-available", + disposition: "provider_auth_proof_gated", + ownerActionable: false, + proofGate: "missing_proof", + }), + ownerTemplate({ + connectorKey: "deployment-blocked", + disposition: "provider_auth_deployment_blocked", + ownerActionable: false, + }), + ] + ); + + const available = catalog.find((e) => e.connectorKey === "available"); + assert.ok(available); + assert.equal(isOwnerActionableEntry(available), true); + assert.equal(sourceSetupAction(available) !== null, true); + assert.equal(sourceSetupAvailability(available), "available_now"); + + const notAvailable = catalog.find((e) => e.connectorKey === "not-available"); + assert.ok(notAvailable); + assert.equal(isOwnerActionableEntry(notAvailable), false); + assert.equal(sourceSetupAction(notAvailable), null); + assert.equal(sourceSetupAvailability(notAvailable), "not_available_here"); + + const deploymentBlocked = catalog.find((e) => e.connectorKey === "deployment-blocked"); + assert.ok(deploymentBlocked); + assert.equal(isOwnerActionableEntry(deploymentBlocked), false); + assert.equal(sourceSetupAction(deploymentBlocked), null); + // Special case: deployment_blocked gets "requires_server_setup", not "not_available_here" + assert.equal(sourceSetupAvailability(deploymentBlocked), "requires_server_setup"); +}); diff --git a/apps/console/src/app/(console)/lib/connection-catalog.ts b/apps/console/src/app/(console)/lib/connection-catalog.ts index af65fb757..c206b6b6f 100644 --- a/apps/console/src/app/(console)/lib/connection-catalog.ts +++ b/apps/console/src/app/(console)/lib/connection-catalog.ts @@ -4,12 +4,10 @@ /** * Pure connector-catalog model for the console add-connection surface. * - * The add-connection surface is a server component, so it can read every shipped - * connector manifest cookie-side via `listConnectorManifests()` — each manifest - * carries `runtime_requirements.bindings`, which is all the binding-derived - * modality classifier needs. This module turns that manifest list into a catalog - * the picker renders: every connector, grouped by modality, routed to the honest - * next step the reference can complete today. + * The live add-connection surface consumes the authenticated owner-template + * projection. Local manifests are joined only for manifest-authored display, + * help, acquisition, and documentation fields; they never supply capability, + * listing, registration, proof, readiness, or action truth. * * This module introduces NO new classification truth. It projects the shared * reference setup planner (`pdpp-reference-implementation/connection-setup-plan`) @@ -29,6 +27,7 @@ import { enrollmentKeyForCanonicalKey, manualUploadSetupFromManifest, type StaticSecretSetupFieldLike, + staticSecretCredentialCaptureFromManifest, } from "pdpp-reference-implementation/connection-setup-plan"; /** @@ -45,10 +44,18 @@ export interface CatalogManifestLike { required?: readonly string[] | null; type?: string | null; } | null; + refresh_policy?: { + rationale?: string | null; + } | null; + public_listing?: { + listed?: boolean | null; + status?: string | null; + } | null; } | null; connector_id: string; connector_key?: string | null; display_name?: string | null; + external_docs?: readonly { label?: string | null; url?: string | null }[] | null; name?: string | null; runtime_requirements?: { bindings?: Record | null } | null; setup?: { @@ -83,6 +90,45 @@ export interface CatalogManifestLike { } | null; } +/** + * Server-owned capability projection returned by the owner-template route. + * Local manifests are joined only for display/help/documentation fields; these + * fields are the authority for registration, listing, setup capability, proof, + * readiness, and owner-facing action. An owner-mediated browser action has no + * owner-agent REST method or URL; `supported_actions` remains the authority + * for that separate API capability. + */ +export interface OwnerConnectorTemplateLike { + connector_key?: string | null; + connector_modality?: string | null; + display_name?: string | null; + public_listing?: { + listed?: boolean | null; + status?: string | null; + } | null; + registration_status?: string | null; + setup_plan?: { + catalog_disposition?: string | null; + deployment_readiness?: ConnectorSetupDeploymentReadiness | null; + enrollment_key?: string | null; + next_step_kind?: string | null; + /** True when an owner-facing setup path exists, including owner-mediated browser setup. */ + owner_actionable?: boolean | null; + proof_gate?: string | null; + runbook_path?: string | null; + setup_modality?: string | null; + support_state?: string | null; + } | null; + supported_actions?: + | readonly { + family?: string | null; + method?: string | null; + status?: string | null; + url?: string | null; + }[] + | null; +} + /** Binding-derived modality, matching the backend intent route's taxonomy. */ export type CatalogModality = ConnectorIntentModality; @@ -98,11 +144,9 @@ export type CatalogModality = ConnectorIntentModality; * path (deep-links to mint a code; the owner finishes the run locally). * - `browser_bound_runbook` — a browser-bound connector with no generated console * path yet; visible and pointed at the runbook, but NOT deep-linked. - * - `static_secret_connect` — a network-class connector whose first connection - * is created via the owner-session static-secret draft path. - * A real owner connect route exists; the picker links to that owner-session - * capture form, not to local-device enrollment, and the connection stays - * hidden until first ingest accepts records. + * - `static_secret_connect` — a network-class connector whose manifest declares + * static-secret capture. The live owner catalog supplies the proof/action + * gate; a capture form alone never makes this disposition actionable. * - `manual_upload_connect` — a manifest-declared file/import connector whose * owner-session upload route is packaged; the picker links to the generic * file-capture form and the connection stays hidden until first ingest @@ -125,6 +169,11 @@ export interface ConnectorAcquisitionPath { posture: string; } +export interface ConnectorExternalDoc { + readonly label: string; + readonly url: string; +} + export interface ConnectorCatalogEntry { /** Manifest-declared owner acquisition jobs, such as export/upload paths. */ acquisitionPaths: readonly ConnectorAcquisitionPath[]; @@ -143,14 +192,32 @@ export interface ConnectorCatalogEntry { * never renders an enrollment link the reference cannot complete. */ enrollmentKey?: string; + /** Manifest-authored external documentation links. */ + externalDocs: readonly ConnectorExternalDoc[]; /** Binding-derived modality. */ modality: CatalogModality; /** The next owner step selected by the shared planner. */ nextStepKind: ConnectorSetupNextStepKind; + /** Server-authorized owner-facing setup action; live owner catalogs always set it. */ + ownerActionable?: boolean; + /** Projected owner-agent method; null for owner-mediated browser setup. */ + ownerActionMethod?: string | null; + /** Projected owner-agent URL; null for owner-mediated browser setup. */ + ownerActionUrl?: string | null; /** Proof gate blocking support, if any. */ proofGate: string | null; + /** Server-owned public-listing state. */ + publicListingStatus?: string | null; + /** Existing capability rationale used for owner context where no setup copy exists. */ + refreshPolicyRationale: string | null; + /** Server-owned registration state. */ + registrationStatus?: string | null; /** Optional runbook path surfaced in advanced/details copy. */ runbookPath: string | null; + /** Manifest-authored setup description, when the setup modality provides one. */ + setupDescription: string | null; + /** Manifest-authored setup help text, when the setup modality provides one. */ + setupHelpText: string | null; /** The owner setup modality selected by the shared planner. */ setupModality: ConnectorSetupModality; /** Support state selected by the shared planner. */ @@ -178,6 +245,33 @@ function displayNameFor(manifest: CatalogManifestLike, connectorKey: string): st return connectorKey; } +function cleanManifestText(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function setupCopyFromManifest(manifest: CatalogManifestLike): { + description: string | null; + helpText: string | null; +} { + const uploadSetup = manualUploadSetupFromManifest(manifest); + const credentialSetup = staticSecretCredentialCaptureFromManifest(manifest); + return { + description: uploadSetup?.description ?? credentialSetup?.description ?? null, + helpText: uploadSetup?.helpText ?? null, + }; +} + +function externalDocsFromManifest(manifest: CatalogManifestLike): ConnectorExternalDoc[] { + if (!Array.isArray(manifest.external_docs)) { + return []; + } + return manifest.external_docs.flatMap((doc) => { + const label = cleanManifestText(doc?.label); + const url = cleanManifestText(doc?.url); + return label && url ? [{ label, url }] : []; + }); +} + function acquisitionPathsFromManifest(manifest: CatalogManifestLike): ConnectorAcquisitionPath[] { const uploadSetup = manualUploadSetupFromManifest(manifest); if (!uploadSetup) { @@ -193,29 +287,36 @@ function acquisitionPathsFromManifest(manifest: CatalogManifestLike): ConnectorA } /** - * Build the connector catalog from the shipped manifests. One entry per manifest - * with a `connector_id`, sorted by display name so the picker is stable across - * renders. Entries only carry an `enrollmentKey` for dispositions the console can - * actually start, so a caller cannot accidentally deep-link a gated connector. + * Build the pure manifest/planner projection used by tests and demo data. The + * live Add Source page uses `buildOwnerConnectorCatalog` so local manifests + * cannot supply registration, listing, proof, readiness, or action authority. */ -export function buildConnectorCatalog(manifests: readonly CatalogManifestLike[]): ConnectorCatalogEntry[] { +export function buildConnectorCatalog( + manifests: readonly CatalogManifestLike[], + configuredProviderAuthConnectorKeys: readonly string[] = [] +): ConnectorCatalogEntry[] { const entries: ConnectorCatalogEntry[] = []; for (const manifest of manifests) { if (!manifest.connector_id) { continue; } const connectorKey = canonicalConnectorKey(manifest.connector_id); - const plan = buildConnectionSetupPlan({ connectorKey, manifest }); + const plan = buildConnectionSetupPlan({ connectorKey, configuredProviderAuthConnectorKeys, manifest }); + const setupCopy = setupCopyFromManifest(manifest); const entry: ConnectorCatalogEntry = { acquisitionPaths: acquisitionPathsFromManifest(manifest), connectorKey, deploymentReadiness: plan.deploymentReadiness, displayName: displayNameFor(manifest, connectorKey), disposition: plan.catalogDisposition, + externalDocs: externalDocsFromManifest(manifest), modality: plan.connectorModality, nextStepKind: plan.nextStepKind, proofGate: plan.proofGate, + refreshPolicyRationale: cleanManifestText(manifest.capabilities?.refresh_policy?.rationale), runbookPath: plan.runbookPath, + setupDescription: setupCopy.description, + setupHelpText: setupCopy.helpText, setupModality: plan.setupModality, supportState: plan.supportState, }; @@ -230,6 +331,215 @@ export function buildConnectorCatalog(manifests: readonly CatalogManifestLike[]) return entries; } +const OWNER_ACTIONABLE_PUBLIC_LISTING_STATUSES = new Set(["proven", "needs_human_auth"]); +const CATALOG_INTENT_MODALITIES = new Set([ + "local_collector", + "browser_bound", + "api_network", + "unknown", +]); +const CATALOG_MODALITIES = new Set([ + "local_collector", + "browser_bound", + "static_secret", + "provider_authorization", + "manual_or_upload", + "unsupported", + "unknown", +]); +const CATALOG_NEXT_STEPS = new Set([ + "enroll_local_collector", + "enroll_browser_collector", + "capture_static_secret", + "open_provider_auth", + "needs_deployment_config", + "provide_import_file", + "manual_runbook", + "unsupported", +]); +const CATALOG_SUPPORT_STATES = new Set([ + "supported", + "proof_gated", + "unsupported", + "needs_deployment_config", +]); +const CATALOG_READINESS_STATES = new Set([ + "not_applicable", + "ready", + "needs_config", +]); +const CATALOG_DISPOSITIONS = new Set([ + "local_collector_enroll", + "local_collector_unproven", + "browser_collector_manual", + "browser_bound_runbook", + "static_secret_connect", + "manual_upload_connect", + "manual_upload_pending", + "provider_auth_deployment_blocked", + "provider_auth_connect", + "provider_auth_proof_gated", + "api_network_unsupported", + "unknown_unsupported", +]); + +function isCatalogValue(values: ReadonlySet, value: unknown): value is T { + return typeof value === "string" && values.has(value as T); +} + +function actionableOwnerActionFromTemplate( + template: OwnerConnectorTemplateLike, + entry: { + connectorModality: ConnectorIntentModality; + disposition: CatalogDisposition; + enrollmentKey?: string; + nextStepKind: ConnectorSetupNextStepKind; + proofGate: string | null; + setupModality: ConnectorSetupModality; + supportState: ConnectorSetupSupportState; + } +): { actionable: boolean; method: string | null; url: string | null } { + const action = template.supported_actions?.find((candidate) => candidate.family === "initiate_connection"); + const method = typeof action?.method === "string" ? action.method : null; + const url = typeof action?.url === "string" && action.url.trim() ? action.url : null; + const authority = + template.registration_status === "registered" && + template.public_listing?.listed === true && + typeof template.public_listing.status === "string" && + OWNER_ACTIONABLE_PUBLIC_LISTING_STATUSES.has(template.public_listing.status) && + template.setup_plan?.owner_actionable === true && + entry.nextStepKind === template.setup_plan?.next_step_kind; + const ownerAgentActionable = + authority && + entry.supportState === "supported" && + entry.proofGate === null && + action?.status === "supported" && + method !== null && + url !== null && + (entry.disposition !== "local_collector_enroll" || typeof entry.enrollmentKey === "string"); + const ownerSessionBrowserActionable = + authority && + entry.connectorModality === "browser_bound" && + ((entry.disposition === "browser_collector_manual" && + entry.nextStepKind === "enroll_browser_collector" && + typeof entry.enrollmentKey === "string") || + (entry.disposition === "static_secret_connect" && entry.setupModality === "static_secret")) && + action?.status === "owner_mediated" && + method === null && + url === null; + return { actionable: ownerAgentActionable || ownerSessionBrowserActionable, method, url }; +} + +/** + * Build the live catalog from the authenticated server projection. A template + * with missing authority fields is dropped rather than reconstructed from a + * local manifest. Local data is a display/help/docs join only. + */ +export function buildOwnerConnectorCatalog( + manifests: readonly CatalogManifestLike[], + templates: readonly OwnerConnectorTemplateLike[] +): ConnectorCatalogEntry[] { + const manifestsByKey = new Map(); + for (const manifest of manifests) { + if (manifest.connector_id) { + manifestsByKey.set(canonicalConnectorKey(manifest.connector_id), manifest); + } + } + + const entries: ConnectorCatalogEntry[] = []; + for (const template of templates) { + const connectorKey = cleanManifestText(template.connector_key); + const setupPlan = template.setup_plan; + if (!connectorKey || template.registration_status !== "registered" || template.public_listing?.listed !== true) { + continue; + } + const disposition = setupPlan?.catalog_disposition; + const connectorModality = template.connector_modality; + const setupModality = setupPlan?.setup_modality; + const nextStepKind = setupPlan?.next_step_kind; + const supportState = setupPlan?.support_state; + const deploymentReadiness = setupPlan?.deployment_readiness; + if ( + !( + isCatalogValue(CATALOG_DISPOSITIONS, disposition) && + isCatalogValue(CATALOG_INTENT_MODALITIES, connectorModality) && + isCatalogValue(CATALOG_MODALITIES, setupModality) && + isCatalogValue(CATALOG_NEXT_STEPS, nextStepKind) && + isCatalogValue(CATALOG_SUPPORT_STATES, supportState) && + deploymentReadiness && + isCatalogValue(CATALOG_READINESS_STATES, deploymentReadiness.state) + ) + ) { + continue; + } + + const localManifest = manifestsByKey.get(canonicalConnectorKey(connectorKey)); + const manifestForCopy = localManifest ?? { connector_id: connectorKey }; + const proofGate = typeof setupPlan.proof_gate === "string" ? setupPlan.proof_gate : null; + const enrollmentKey = cleanManifestText(setupPlan.enrollment_key) ?? undefined; + const capability = actionableOwnerActionFromTemplate(template, { + connectorModality, + disposition, + enrollmentKey, + nextStepKind, + proofGate, + setupModality, + supportState, + }); + const setupCopy = setupCopyFromManifest(manifestForCopy); + const entry: ConnectorCatalogEntry = { + acquisitionPaths: acquisitionPathsFromManifest(manifestForCopy), + connectorKey: canonicalConnectorKey(connectorKey), + deploymentReadiness, + displayName: cleanManifestText(template.display_name) ?? displayNameFor(manifestForCopy, connectorKey), + disposition, + externalDocs: externalDocsFromManifest(manifestForCopy), + modality: connectorModality, + nextStepKind, + ownerActionable: capability.actionable, + ownerActionMethod: capability.method, + ownerActionUrl: capability.url, + proofGate, + publicListingStatus: cleanManifestText(template.public_listing?.status), + refreshPolicyRationale: cleanManifestText(localManifest?.capabilities?.refresh_policy?.rationale), + registrationStatus: template.registration_status, + runbookPath: cleanManifestText(setupPlan.runbook_path), + setupDescription: setupCopy.description, + setupHelpText: setupCopy.helpText, + setupModality, + supportState, + }; + if (enrollmentKey) { + entry.enrollmentKey = enrollmentKey; + } + entries.push(entry); + } + entries.sort((a, b) => a.displayName.localeCompare(b.displayName)); + return entries; +} + +/** + * Check if this entry is owner-actionable. + * + * For live owner-catalog entries, ownerActionable is the authoritative field + * computed once during buildOwnerConnectorCatalog. For demo/test entries from + * buildConnectorCatalog, fall back to explicit rules since they carry no + * owner-session authority. + */ +export function isOwnerActionableEntry(entry: ConnectorCatalogEntry): boolean { + if (entry.ownerActionable !== undefined) { + return entry.ownerActionable; + } + // `buildConnectorCatalog` remains a pure manifest/planner projection for + // tests and demo data. Its static-secret and provider branches still fail + // closed on the planner's proof fields; live pages must use the owner + // projection above, which also supplies registration and listing authority. + if (entry.setupModality === "static_secret" || entry.disposition === "provider_auth_connect") { + return entry.supportState === "supported" && entry.proofGate === null; + } + return entry.supportState === "supported" || entry.disposition === "browser_collector_manual"; +} + /** Catalog entries the console can start as a one-click local-collector enroll. */ export function localCollectorEntries(catalog: readonly ConnectorCatalogEntry[]): ConnectorCatalogEntry[] { return catalog.filter((e) => e.disposition === "local_collector_enroll"); @@ -273,6 +583,11 @@ export function manualUploadPendingEntries(catalog: readonly ConnectorCatalogEnt return catalog.filter((e) => e.disposition === "manual_upload_pending"); } +/** Provider-auth entries whose shared plan authorizes an owner action now. */ +export function providerAuthConnectEntries(catalog: readonly ConnectorCatalogEntry[]): ConnectorCatalogEntry[] { + return catalog.filter((e) => e.disposition === "provider_auth_connect"); +} + /** Provider-authorization entries blocked on instance-level deployment config. */ export function deploymentBlockedEntries(catalog: readonly ConnectorCatalogEntry[]): ConnectorCatalogEntry[] { return catalog.filter((e) => e.disposition === "provider_auth_deployment_blocked"); diff --git a/apps/console/src/app/(console)/lib/connection-evidence.test.ts b/apps/console/src/app/(console)/lib/connection-evidence.test.ts index 6ae253fcf..02d5614be 100644 --- a/apps/console/src/app/(console)/lib/connection-evidence.test.ts +++ b/apps/console/src/app/(console)/lib/connection-evidence.test.ts @@ -2237,7 +2237,7 @@ test("formatCollectionRateReadout fails closed for partial projection payloads", ceiling_rate_per_min: 240, current_interval_ms: 250, effective_rate_per_min: 240, - last_backoff: { reason: "throttle" } as never, + last_backoff: { reason: "throttle" }, }); assert.ok(partialBackoff); assert.equal(partialBackoff.backoffLabel, null, "partial back-off → omit optional line"); diff --git a/apps/console/src/app/(console)/lib/connection-evidence.ts b/apps/console/src/app/(console)/lib/connection-evidence.ts index ff6cadcde..417696981 100644 --- a/apps/console/src/app/(console)/lib/connection-evidence.ts +++ b/apps/console/src/app/(console)/lib/connection-evidence.ts @@ -1923,7 +1923,10 @@ export function formatCollectionRateReadout( } const backoff = rate.last_backoff; const backoffLabel = - backoff && Number.isFinite(backoff.at_interval_ms) && typeof backoff.reason === "string" + backoff && + typeof backoff.at_interval_ms === "number" && + Number.isFinite(backoff.at_interval_ms) && + typeof backoff.reason === "string" ? `last backed off to ${backoff.at_interval_ms.toLocaleString()}ms (${backoff.reason})` : null; return { diff --git a/apps/console/src/app/(console)/lib/connector-run-summary-status.test.ts b/apps/console/src/app/(console)/lib/connector-run-summary-status.test.ts index fb7c693e5..223d55ead 100644 --- a/apps/console/src/app/(console)/lib/connector-run-summary-status.test.ts +++ b/apps/console/src/app/(console)/lib/connector-run-summary-status.test.ts @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { isActiveConnectorRunSummaryStatus } from "./connector-run-summary-status.ts"; +import { connectorRunSummaryId, isActiveConnectorRunSummaryStatus } from "./connector-run-summary-status.ts"; test("isActiveConnectorRunSummaryStatus accepts only the current connector-summary active states", () => { for (const status of ["pending", "started", "in_progress"]) { @@ -14,3 +14,10 @@ test("isActiveConnectorRunSummaryStatus accepts only the current connector-summa assert.equal(isActiveConnectorRunSummaryStatus(status), false, `${status} should not be active`); } }); + +test("connectorRunSummaryId rejects scheduler decisions that are not real runs", () => { + assert.equal(connectorRunSummaryId("run_123"), "run_123"); + assert.equal(connectorRunSummaryId(undefined), null); + assert.equal(connectorRunSummaryId(null), null); + assert.equal(connectorRunSummaryId(" "), null); +}); diff --git a/apps/console/src/app/(console)/lib/connector-run-summary-status.ts b/apps/console/src/app/(console)/lib/connector-run-summary-status.ts index 00cbca5d9..2df980a8f 100644 --- a/apps/console/src/app/(console)/lib/connector-run-summary-status.ts +++ b/apps/console/src/app/(console)/lib/connector-run-summary-status.ts @@ -7,3 +7,8 @@ const ACTIVE_RUN_SUMMARY_STATUSES = new Set(["pending", "started", "in_progress" export function isActiveConnectorRunSummaryStatus(status: string): boolean { return ACTIVE_RUN_SUMMARY_STATUSES.has(status); } + +/** Synthetic scheduler gate decisions have no run id and are not navigable syncs. */ +export function connectorRunSummaryId(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} diff --git a/apps/console/src/app/(console)/lib/operator-runs.test.ts b/apps/console/src/app/(console)/lib/operator-runs.test.ts index 40142a448..5ab6cc363 100644 --- a/apps/console/src/app/(console)/lib/operator-runs.test.ts +++ b/apps/console/src/app/(console)/lib/operator-runs.test.ts @@ -14,6 +14,7 @@ const CONNECTION_CONTROL_PATH_TEMPLATE_RE = /`\/_ref\/connections\/\$\{encodeURIComponent\(connectionId\)\}\$\{suffix\}`/; const RUN_CONNECTION_EXPORT_RE = /export function runConnectionNow\(connectionId: string, options: RunNowOptions = \{\}\)/; +const RUN_ADMISSION_TYPE_RE = /export type RunAdmission = "browser_enrollment" \| "setup";/; const RUN_NOW_FORCE_BODY_RE = /forceOption === true \? \{ force: true \}/; const RUN_NOW_ADMISSION_BODY_RE = /runAdmission \? \{ run_admission: runAdmission \}/; const RUN_CONNECTOR_OPTIONS_RE = @@ -33,6 +34,7 @@ test("operator run helpers expose connection-scoped control paths", async () => assert.match(src, CONNECTION_CONTROL_PATH_TEMPLATE_RE); assert.match(src, RUN_CONNECTOR_OPTIONS_RE); assert.match(src, RUN_CONNECTION_EXPORT_RE); + assert.match(src, RUN_ADMISSION_TYPE_RE); assert.match(src, RUN_NOW_FORCE_BODY_RE); assert.match(src, RUN_NOW_ADMISSION_BODY_RE); assert.match(src, SAVE_CONNECTION_SCHEDULE_EXPORT_RE); diff --git a/apps/console/src/app/(console)/lib/operator-runs.ts b/apps/console/src/app/(console)/lib/operator-runs.ts index 733770db5..09bfc0cc0 100644 --- a/apps/console/src/app/(console)/lib/operator-runs.ts +++ b/apps/console/src/app/(console)/lib/operator-runs.ts @@ -83,9 +83,11 @@ function connectionControlPath(connectionId: string, suffix: string): string { return `/_ref/connections/${encodeURIComponent(connectionId)}${suffix}`; } -interface RunNowOptions { +export type RunAdmission = "browser_enrollment" | "setup"; + +export interface RunNowOptions { force?: boolean; - runAdmission?: "browser_enrollment"; + runAdmission?: RunAdmission; } async function runNowAt(path: string, options: RunNowOptions = {}) { diff --git a/apps/console/src/app/(console)/lib/ref-client.ts b/apps/console/src/app/(console)/lib/ref-client.ts index 2d1b9af1c..345ce5c7e 100644 --- a/apps/console/src/app/(console)/lib/ref-client.ts +++ b/apps/console/src/app/(console)/lib/ref-client.ts @@ -93,7 +93,7 @@ export interface RunStatusEnvelope { } | null; links: { timeline: string }; object: "run_status"; - run_id: string; + run_id?: string | null; started_at: string | null; status: RunHandleStatus; terminal_reason: string | null; @@ -317,11 +317,16 @@ export interface RefConnectorRunSummary { first_at: string; known_gaps?: unknown[]; last_at: string; + records_emitted?: number | null; + reported_records_emitted?: number | null; run_id: string; started_at: string; status: string; + yield_counts_present?: boolean; } +export type RefTerminalSetupDisposition = "verified_empty" | "unverified_missing_counts" | "unverified_zero"; + export interface RefreshPolicy { assisted_after_owner_auth?: boolean; background_safe?: boolean; @@ -676,6 +681,8 @@ export interface RefConnectorSummary { * the same reason as `manifest_declaration`. */ terminal_facts?: RefTerminalFactsState | null; + /** Shared connection-scoped terminal setup disposition for a draft. */ + terminal_setup_disposition?: RefTerminalSetupDisposition | null; total_records: number; /** * Orthogonal state for `total_records` (`reconcile-active-summary-evidence` @@ -1114,8 +1121,12 @@ export interface RefCollectionRateSnapshot { current_interval_ms: number; /** Current effective rate (requests/min). */ effective_rate_per_min: number; - /** Most recent back-off, or null when none. */ - last_backoff: { at?: string | null; at_interval_ms: number; reason: string } | null; + /** Most recent back-off, or null when none; legacy projections may be partial. */ + last_backoff?: { + at?: string | null; + at_interval_ms?: number | null; + reason?: string | null; + } | null; } export type RefRemoteSurfaceAxis = "failed" | "idle" | "leased" | "none" | "unknown" | "waiting"; @@ -1304,6 +1315,29 @@ export async function refFetch( return res.json(); } +export interface ProviderAuthInitiateResponse { + authorization_url: string; + connector_id: string; + expires_at: string; + next_step: { + authorization_url: string; + expires_at: string; + kind: "open_provider_auth"; + reason: string; + redirect_uri: string; + }; + object: "provider_auth_initiate"; + setup_modality: "provider_authorization"; +} + +export async function initiateProviderAuthorization(connectorId: string): Promise { + return (await refFetch(`/_ref/connectors/${encodeURIComponent(connectorId)}/provider-auth-initiate`, undefined, { + body: "{}", + headers: { "content-type": "application/json" }, + method: "POST", + })) as ProviderAuthInitiateResponse; +} + export { RefNotFoundError, RefRequestError }; // Thrown when the owner-session static-secret capture route rejects a credential @@ -2009,6 +2043,7 @@ export interface StaticSecretDraftConnection { connector_id: string; connector_instance_id: string; credential_kind: string; + display_name: string; next_step: { kind: "capture_static_secret_credential"; method: "POST"; @@ -2016,7 +2051,7 @@ export interface StaticSecretDraftConnection { url: string; }; object: "static_secret_draft_connection"; - status: "draft"; + status: "active" | "draft"; } export interface StaticSecretCredentialCapture { @@ -2040,6 +2075,7 @@ export interface StaticSecretCredentialCapture { rotated_at: string | null; status: string | null; }; + deduplicated?: boolean; // Non-secret account identity from a synchronous credential probe ("Connected // as {identity}"). Null when the connector has no probe (first-sync path). identity: { account_identity: string; detail: string | null } | null; @@ -2133,6 +2169,7 @@ export async function captureStaticSecretCredential(input: { connectionId: string; credentialKind: string; secret: string; + setupFields?: Record; }): Promise { try { return (await refFetch( @@ -2142,6 +2179,7 @@ export async function captureStaticSecretCredential(input: { body: JSON.stringify({ credential_kind: input.credentialKind, secret: input.secret, + ...(input.setupFields ? { setup_fields: input.setupFields } : {}), }), headers: { "content-type": "application/json" }, method: "POST", @@ -2173,6 +2211,10 @@ export type StaticSecretSetupStateValue = | "first_sync_failed" | "first_sync_pending" | "first_sync_running" + | "first_sync_unverified_missing_counts" + | "first_sync_unverified_zero" + | "first_sync_verified_empty" + /** @deprecated Kept for references that still emit the pre-disposition state. */ | "first_sync_zero_yield" | "paused" | "revoked" @@ -2242,6 +2284,7 @@ export interface ConnectionSetupStatus { }; setup_state: StaticSecretSetupStateValue; status: string; + terminal_setup_disposition: RefTerminalSetupDisposition | null; updated_at: string | null; } diff --git a/apps/console/src/app/(console)/lib/rs-client.ts b/apps/console/src/app/(console)/lib/rs-client.ts index 7f7c998cf..f5c5fb896 100644 --- a/apps/console/src/app/(console)/lib/rs-client.ts +++ b/apps/console/src/app/(console)/lib/rs-client.ts @@ -16,6 +16,7 @@ import { readdir, readFile } from "node:fs/promises"; import { join } from "node:path"; import { findManifestForConnectorId } from "../sources/lib/relationships.ts"; +import type { OwnerConnectorTemplateLike } from "./connection-catalog.ts"; import { isActiveConnectorRunSummaryStatus } from "./connector-run-summary-status.ts"; import { getOwnerToken, @@ -142,6 +143,19 @@ export interface StreamMetadata { } export interface ConnectorManifest { + capabilities?: { + auth?: { + deployment_config?: readonly string[] | null; + kind?: string | null; + mode?: string | null; + required?: readonly string[] | null; + type?: string | null; + } | null; + public_listing?: { + listed?: boolean | null; + status?: string | null; + } | null; + } | null; connector_id: string; connector_key?: string; display_name?: string; @@ -789,6 +803,14 @@ export async function listConnectorManifests(): Promise { return manifests; } +/** Server-owned catalog projection consumed by the live Add Source surface. */ +export type OwnerConnectorTemplate = OwnerConnectorTemplateLike; + +export async function listOwnerConnectorTemplates(): Promise { + const body = (await authedFetch("/v1/owner/connector-templates")) as { data?: OwnerConnectorTemplate[] }; + return Array.isArray(body.data) ? body.data : []; +} + export interface ConnectorOverview { acquisitionCoverage?: RefAcquisitionCoverageSummary | null; /** diff --git a/apps/console/src/app/(console)/lib/run-assistance.test.ts b/apps/console/src/app/(console)/lib/run-assistance.test.ts index 7bf2cbce3..5d91333b0 100644 --- a/apps/console/src/app/(console)/lib/run-assistance.test.ts +++ b/apps/console/src/app/(console)/lib/run-assistance.test.ts @@ -9,6 +9,7 @@ import { getCurrentRunAssistance, hasActiveBrowserSurface, hasAvailableBrowserSurfaceAttachment, + hasResolvedBrowserSurfaceAssistance, requiresBrowserSurfaceAssistance, } from "./run-assistance.ts"; @@ -177,3 +178,74 @@ test("terminal browser-surface events do not keep stream fallback in the browser assert.equal(hasActiveBrowserSurface(events), false); }); + +// fr-setup-status-lifecycle-0806: an H-E-B-style browser login that already +// resolved must read as "browser step complete," not the same "nothing has +// ever happened" signal a fresh run reports before assistance is requested. + +test("a never-requested run reports no resolved browser-surface assistance", () => { + const events = [event("run.started", { automation_mode: "assisted" })]; + + assert.equal(hasResolvedBrowserSurfaceAssistance(events), false); +}); + +test("a currently-open browser-surface assistance request is not yet resolved", () => { + const events = [ + event("run.assistance_requested", { + assistance_request_id: "assist_1", + attachments: [{ kind: "browser_surface", ref: "surface_1", role: "streaming_companion" }], + message: "Log in to continue.", + owner_action: "operate_attachment", + progress_posture: "blocked", + response_contract: "response_required", + }), + ]; + + assert.equal(hasResolvedBrowserSurfaceAssistance(events), false); +}); + +test("a resolved structured browser-surface assistance request reports handoff-ready", () => { + const events = [ + event("run.assistance_requested", { + assistance_request_id: "assist_1", + attachments: [{ kind: "browser_surface", ref: "surface_1", role: "streaming_companion" }], + message: "Log in to continue.", + owner_action: "operate_attachment", + progress_posture: "blocked", + response_contract: "response_required", + }), + event("run.assistance_resolved", { assistance_request_id: "assist_1" }), + ]; + + assert.equal(hasResolvedBrowserSurfaceAssistance(events), true); + // The run keeps going — this must not read as fully resolved/ended. + assert.equal(getCurrentBrowserSurfaceAssistance(events), null); +}); + +test("a resolved legacy manual_action interaction reports handoff-ready", () => { + const events = [ + event("run.interaction_required", { + interaction_id: "int_1", + kind: "manual_action", + message: "Log in to continue.", + }), + event("run.interaction_completed", { interaction_id: "int_1" }), + ]; + + assert.equal(hasResolvedBrowserSurfaceAssistance(events), true); +}); + +test("a resolved non-browser (app-push) assistance request is not treated as browser handoff", () => { + const events = [ + event("run.assistance_requested", { + assistance_request_id: "assist_1", + message: "Approve the sign-in in the app.", + owner_action: "act_elsewhere", + progress_posture: "running", + response_contract: "none", + }), + event("run.assistance_resolved", { assistance_request_id: "assist_1" }), + ]; + + assert.equal(hasResolvedBrowserSurfaceAssistance(events), false); +}); diff --git a/apps/console/src/app/(console)/lib/run-assistance.ts b/apps/console/src/app/(console)/lib/run-assistance.ts index 094f4a55b..043b23d0a 100644 --- a/apps/console/src/app/(console)/lib/run-assistance.ts +++ b/apps/console/src/app/(console)/lib/run-assistance.ts @@ -98,6 +98,42 @@ export function hasActiveBrowserSurface(events: SpineEvent[]): boolean { return false; } +/** + * True once a browser-surface-shaped assistance request has ever been raised + * for this run and is now terminal (resolved/cancelled/escalated/timed out) — + * the H-E-B "owner finished login" signal. Distinct from + * `hasActiveBrowserSurface`, which answers "is a browser surface open right + * now": this answers "did the owner already do the browser step," so the + * stream page can hand off to a background-continuing surface instead of + * re-showing the same no-action-waiting copy it would show for a run that + * never needed browser input at all (fr-setup-status-lifecycle-0806). + */ +export function hasResolvedBrowserSurfaceAssistance(events: SpineEvent[]): boolean { + const terminalState = getTerminalAssistanceState(events); + for (const event of events) { + if (event.event_type === "run.assistance_requested") { + const id = getEventAssistanceId(event); + if (id && terminalState.ids.has(id) && requiresBrowserSurfaceAssistance(assistanceFromEvent(event, id))) { + return true; + } + continue; + } + if (event.event_type === "run.interaction_completed") { + const id = getEventAssistanceId(event); + if (!id) { + continue; + } + const requested = events.find( + (candidate) => candidate.event_type === "run.interaction_required" && getEventAssistanceId(candidate) === id + ); + if (requested && requiresBrowserSurfaceAssistance(assistanceFromLegacyInteraction(requested, id))) { + return true; + } + } + } + return false; +} + function getCompletedLegacyInteractions(events: SpineEvent[]): Set { return new Set( events diff --git a/apps/console/src/app/(console)/lib/same-origin-route.test.ts b/apps/console/src/app/(console)/lib/same-origin-route.test.ts new file mode 100644 index 000000000..54e10877c --- /dev/null +++ b/apps/console/src/app/(console)/lib/same-origin-route.test.ts @@ -0,0 +1,30 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import test from "node:test"; +import { originMatchesHost, publicOrigin, redirectToPublicPath } from "./same-origin-route.ts"; + +function requestWith(headers: Record = {}, url = "https://internal.example/submit"): Request { + return new Request(url, { headers }); +} + +test("same-origin route helper preserves forwarded public origin for redirects", () => { + const request = requestWith({ + host: "internal.example", + "x-forwarded-host": "console.example", + "x-forwarded-proto": "https", + }); + assert.equal(publicOrigin(request), "https://console.example"); + const response = redirectToPublicPath(request, "/sources/add?error=failed"); + assert.equal(response.status, 303); + assert.equal(response.headers.get("location"), "https://console.example/sources/add?error=failed"); +}); + +test("same-origin route helper accepts absent or matching origins and rejects mismatches", () => { + assert.equal(originMatchesHost(requestWith({ host: "console.example" })), true); + assert.equal(originMatchesHost(requestWith({ host: "console.example", origin: "https://console.example" })), true); + assert.equal(originMatchesHost(requestWith({ host: "console.example", origin: "https://attacker.example" })), false); + assert.equal(originMatchesHost(requestWith({ host: "console.example", origin: "not a URL" })), false); + assert.equal(originMatchesHost(requestWith({ origin: "https://console.example" })), false); +}); diff --git a/apps/console/src/app/(console)/lib/same-origin-route.ts b/apps/console/src/app/(console)/lib/same-origin-route.ts new file mode 100644 index 000000000..cb937dbdc --- /dev/null +++ b/apps/console/src/app/(console)/lib/same-origin-route.ts @@ -0,0 +1,35 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import { NextResponse } from "next/server"; + +/** Resolve the browser-facing origin using the same proxy headers as setup routes. */ +export function publicOrigin(request: Request): string { + const host = request.headers.get("x-forwarded-host") ?? request.headers.get("host") ?? new URL(request.url).host; + const proto = + request.headers.get("x-forwarded-proto") ?? + (host.startsWith("localhost") || host.startsWith("127.") ? "http" : "https"); + return `${proto}://${host}`; +} + +/** Redirect to a local route while preserving the public host/protocol. */ +export function redirectToPublicPath(request: Request, path: string): NextResponse { + return NextResponse.redirect(new URL(path, publicOrigin(request)), 303); +} + +/** Accept same-origin form posts and reject cross-origin or malformed Origin headers. */ +export function originMatchesHost(request: Request): boolean { + const origin = request.headers.get("origin"); + if (!origin) { + return true; + } + const host = request.headers.get("host"); + if (!host) { + return false; + } + try { + return new URL(origin).host === host; + } catch { + return false; + } +} diff --git a/apps/console/src/app/(console)/lib/schedule-evidence.test.ts b/apps/console/src/app/(console)/lib/schedule-evidence.test.ts new file mode 100644 index 000000000..9ec511dfb --- /dev/null +++ b/apps/console/src/app/(console)/lib/schedule-evidence.test.ts @@ -0,0 +1,29 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { activeScheduleRunId, scheduleEnabled, scheduleIntervalSeconds } from "./schedule-evidence.ts"; + +test("schedule interval evidence fails closed for missing or invalid values", () => { + for (const value of [undefined, null, Number.NaN, Number.POSITIVE_INFINITY, 0, -1, "3600"]) { + assert.equal(scheduleIntervalSeconds(value), null); + } + assert.equal(scheduleIntervalSeconds(3600), 3600); +}); + +test("schedule enabled evidence fails closed when the flag is missing", () => { + assert.equal(scheduleEnabled(undefined), null); + assert.equal(scheduleEnabled("true"), null); + assert.equal(scheduleEnabled(true), true); + assert.equal(scheduleEnabled(false), false); +}); + +test("active schedule run evidence only accepts a non-empty string id", () => { + assert.equal(activeScheduleRunId(null), null); + assert.equal(activeScheduleRunId({ active_run_id: undefined }), null); + assert.equal(activeScheduleRunId({ active_run_id: 42 }), null); + assert.equal(activeScheduleRunId({ active_run_id: "" }), null); + assert.equal(activeScheduleRunId({ active_run_id: "run_1" }), "run_1"); +}); diff --git a/apps/console/src/app/(console)/lib/schedule-evidence.ts b/apps/console/src/app/(console)/lib/schedule-evidence.ts new file mode 100644 index 000000000..6cf9fb8bc --- /dev/null +++ b/apps/console/src/app/(console)/lib/schedule-evidence.ts @@ -0,0 +1,22 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Runtime guards for additive/legacy schedule projections. The wire type is + * complete on current references, but old or partially materialized summary + * rows can omit fields. Callers should use null as unknown, never format an + * omitted value into NaN or treat an arbitrary truthy active-run value as an + * href. + */ +export function scheduleIntervalSeconds(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null; +} + +export function scheduleEnabled(value: unknown): boolean | null { + return typeof value === "boolean" ? value : null; +} + +export function activeScheduleRunId(schedule: { readonly active_run_id?: unknown } | null | undefined): string | null { + const value = schedule?.active_run_id; + return typeof value === "string" && value.length > 0 ? value : null; +} diff --git a/apps/console/src/app/(console)/lib/source-actionability.test.ts b/apps/console/src/app/(console)/lib/source-actionability.test.ts index e1b1eb61e..c90c6f530 100644 --- a/apps/console/src/app/(console)/lib/source-actionability.test.ts +++ b/apps/console/src/app/(console)/lib/source-actionability.test.ts @@ -20,6 +20,7 @@ import { SOURCE_WORK_GROUP_COPY, sourceAttentionHeadline, sourceWorkFromConnectors, + TERMINAL_SETUP_DISPOSITION_COPY, verdictRequiresOwnerNow, } from "./source-actionability.ts"; @@ -437,6 +438,25 @@ test("source actionability projects a draft connection as needs-you setup_in_pro assert.equal(actionability.work?.statusLabel, "needs you"); }); +test("terminal setup dispositions replace generic draft copy while staying inactive setup work", () => { + for (const disposition of ["verified_empty", "unverified_zero", "unverified_missing_counts"] as const) { + const copy = TERMINAL_SETUP_DISPOSITION_COPY[disposition]; + const actionability = projectSourceActionability(draftConnector({ terminal_setup_disposition: disposition })); + + assert.equal(isSetupInProgressConnector(draftConnector({ terminal_setup_disposition: disposition })), true); + assert.equal(actionability.renderedStatus.label, copy.statusLabel); + assert.equal(actionability.renderedStatus.kind, "degraded"); + assert.equal(actionability.nextAction?.label, copy.actionLabel); + assert.equal(actionability.primaryVerdictAction?.cta, copy.actionLabel); + assert.equal(actionability.primaryVerdictAction?.terminal, true); + assert.equal(actionability.ownerActionCue?.label, copy.actionLabel); + assert.equal(actionability.work?.group, "needsOwner"); + assert.equal(actionability.work?.actionLabel, copy.actionLabel); + assert.equal(actionability.work?.statusLabel, copy.statusLabel); + assert.equal(actionability.work?.what, copy.what); + } +}); + test("source actionability: revoked outranks draft — a revoked connection never reads as setup_in_progress", () => { const actionability = projectSourceActionability( draftConnector({ revoked_at: "2026-07-10T00:00:00Z", status: "revoked" }) diff --git a/apps/console/src/app/(console)/lib/source-actionability.ts b/apps/console/src/app/(console)/lib/source-actionability.ts index 1750c12ed..ab3be21f1 100644 --- a/apps/console/src/app/(console)/lib/source-actionability.ts +++ b/apps/console/src/app/(console)/lib/source-actionability.ts @@ -8,6 +8,7 @@ import type { RefConnectorSummary, RefRenderedVerdict, RefRequiredAction, + RefTerminalSetupDisposition, RefVerdictTone, } from "./ref-client.ts"; import { @@ -121,6 +122,31 @@ export const SOURCE_WORK_GROUP_COPY: Record = { + unverified_missing_counts: { + actionLabel: "Review setup", + statusLabel: "needs review", + what: "The first sync completed without durable count evidence. Review the connection before retrying.", + }, + unverified_zero: { + actionLabel: "Retry first sync", + statusLabel: "needs review", + what: "The first sync returned zero records without proving the account was empty. Review the connection and retry.", + }, + verified_empty: { + actionLabel: "Review empty result", + statusLabel: "verified empty", + what: "The first sync verified that this source has no records. Review the setup result before trying again.", + }, +}; + /** The one owner-facing meaning of the headline "needs you" attention number. */ export interface SourceAttentionHeadline { /** Count of sources genuinely blocked on the owner's action (the needs-you group). */ @@ -231,11 +257,21 @@ function labelWithFreshness(base: string, note: string | null): string { export function deriveRenderedSourceStatus( verdict: RefRenderedVerdict | null | undefined, revoked: boolean, - pending = false + pending = false, + terminalSetupDisposition: RefTerminalSetupDisposition | null = null ): SourceStatusFlag { if (revoked) { return { dot: "⊘", freshnessNote: null, kind: "revoked", label: "Revoked", tone: "muted" }; } + if (terminalSetupDisposition) { + return { + dot: "◐", + freshnessNote: null, + kind: "degraded", + label: TERMINAL_SETUP_DISPOSITION_COPY[terminalSetupDisposition].statusLabel, + tone: "warning", + }; + } // Setup-in-progress overrides any verdict shape, same priority as revoked: // a draft has no meaningful health/coverage evidence yet (see // `isSetupInProgressConnector`), so its verdict tone (if any) must never be @@ -276,8 +312,18 @@ export const SETUP_IN_PROGRESS_CTA_LABEL = "Continue setup"; */ export function formatRenderedRequiredAction( verdict: RefRenderedVerdict | null | undefined, - pending = false + pending = false, + terminalSetupDisposition: RefTerminalSetupDisposition | null = null ): FormattedNextAction | null { + if (terminalSetupDisposition) { + return { + actionTarget: "connection_detail", + caveat: null, + label: TERMINAL_SETUP_DISPOSITION_COPY[terminalSetupDisposition].actionLabel, + notificationHint: null, + variant: "structured", + }; + } if (pending) { return { actionTarget: "connection_detail", @@ -320,10 +366,26 @@ function setupInProgressPrimaryVerdictAction(): SourcePrimaryVerdictAction { }; } +function terminalSetupPrimaryVerdictAction(disposition: RefTerminalSetupDisposition): SourcePrimaryVerdictAction { + return { + audience: "owner", + channel: "attention", + cta: TERMINAL_SETUP_DISPOSITION_COPY[disposition].actionLabel, + kind: "reauth", + ownerRunnable: true, + satisfiedWhenKind: "attention_resolved", + terminal: true, + }; +} + export function formatPrimaryVerdictAction( verdict: RefRenderedVerdict | null | undefined, - pending = false + pending = false, + terminalSetupDisposition: RefTerminalSetupDisposition | null = null ): SourcePrimaryVerdictAction | null { + if (terminalSetupDisposition) { + return terminalSetupPrimaryVerdictAction(terminalSetupDisposition); + } if (pending) { return setupInProgressPrimaryVerdictAction(); } @@ -476,6 +538,16 @@ export function sourceWorkItemFromConnector(connector: RefConnectorSummary): Sou return null; } + const terminalSetupDisposition = connector.terminal_setup_disposition ?? null; + if (isSetupInProgressConnector(connector) && terminalSetupDisposition) { + const copy = TERMINAL_SETUP_DISPOSITION_COPY[terminalSetupDisposition]; + return itemFromConnector(connector, "needsOwner", { + actionLabel: copy.actionLabel, + statusLabel: copy.statusLabel, + what: copy.what, + }); + } + // Setup-in-progress outranks the verdict (see `isSetupInProgressConnector`): // a draft has no health/coverage evidence to derive work from, and the // owner genuinely has something to finish, so it always surfaces in the @@ -578,20 +650,25 @@ export function projectSourceActionability(connector: RefConnectorSummary): Sour const routeId = connectionRouteId(connector); const label = connectorLabel(connector); const revoked = isRevokedConnector(connector); - const pending = !revoked && isSetupInProgressConnector(connector); + const terminalSetupDisposition = connector.terminal_setup_disposition ?? null; + const pending = !revoked && isSetupInProgressConnector(connector) && terminalSetupDisposition === null; const primaryAction = pending ? null : primaryRequiredAction(connector.rendered_verdict); - const primaryVerdictAction = formatPrimaryVerdictAction(connector.rendered_verdict, pending); + const primaryVerdictAction = formatPrimaryVerdictAction( + connector.rendered_verdict, + pending, + terminalSetupDisposition + ); return { failureSummary: pending ? null : deriveFailureSummary(connector.connection_health, connector.rendered_verdict ?? null), label, - nextAction: formatRenderedRequiredAction(connector.rendered_verdict, pending), + nextAction: formatRenderedRequiredAction(connector.rendered_verdict, pending, terminalSetupDisposition), ownerActionByStream: pending ? {} : ownerActionAvailabilityByStream(connector.rendered_verdict ?? null), ownerActionCue: ownerActionCueFromVerdictAction(primaryVerdictAction), primaryAction, primaryVerdictAction, - renderedStatus: deriveRenderedSourceStatus(connector.rendered_verdict, revoked, pending), + renderedStatus: deriveRenderedSourceStatus(connector.rendered_verdict, revoked, pending, terminalSetupDisposition), revoked, routeId, work: sourceWorkItemFromConnector(connector), diff --git a/apps/console/src/app/(console)/lib/source-add-support.test.ts b/apps/console/src/app/(console)/lib/source-add-support.test.ts index 5be813152..bc5b89100 100644 --- a/apps/console/src/app/(console)/lib/source-add-support.test.ts +++ b/apps/console/src/app/(console)/lib/source-add-support.test.ts @@ -20,10 +20,10 @@ import { buildSourceAddSupport, resolveSourceAddSupport } from "./source-add-sup const ADD_ANOTHER_ACCOUNT_LABEL_RE = /add another account/i; const CONNECT_BROWSER_SESSION_ROUTE_RE = /\/connect\/browser-session\/amazon/; -const STATIC_SECRET_ROUTE_RE = /\/connect\/static-secret\/ynab/; +const STATIC_SECRET_ROUTE_RE = /\/connect\/static-secret\/gmail/; const DEVICE_EXPORTER_ROUTE_RE = /\/device-exporters\?connector=/; const MANUAL_UPLOAD_ROUTE_RE = /\/connect\/manual-upload\/google-maps/; -const PACKAGED_PATH_PENDING_RE = /^Add path not packaged$/; +const NOT_SELF_SERVICE_RE = /^Adding another account is not available here$/; const DEMOTION_COPY_RE = /not self-service|not supported|track only|developer proof/i; const DEV_JARGON_RE = /pnpm --dir|packages\/|monorepo|env var|connector_instance_id|PDPP_/; @@ -114,9 +114,9 @@ function providerAuthManifest(connectorId: string): CatalogManifestLike { }; } -test("static-secret source supports self-service add-another-account with an action", () => { - const map = buildSourceAddSupport([staticSecretManifest("ynab")]); - const support = resolveSourceAddSupport(map, "ynab"); +test("proven static-secret source supports self-service add-another-account with an action", () => { + const map = buildSourceAddSupport([staticSecretManifest("gmail")]); + const support = resolveSourceAddSupport(map, "gmail"); assert.ok(support, "static-secret connector must appear in the support map"); assert.equal(support.support, "self_service"); assert.ok(support.action, "self-service add must carry a next action"); @@ -146,23 +146,23 @@ test("supported browser collector source is self-service and routes to browser-s assert.doesNotMatch(support.supportLabel, DEMOTION_COPY_RE); }); -test("browser-bound runbook source stays packaged-path-pending", () => { +test("proof-gated browser runbook stays unavailable for self-service setup", () => { const map = buildSourceAddSupport([browserBoundManifest("some_browser_source")]); const support = resolveSourceAddSupport(map, "some_browser_source"); assert.ok(support); - assert.equal(support.support, "packaged_path_pending"); + assert.equal(support.support, "not_self_service"); assert.equal(support.action, null); - assert.match(support.supportLabel, PACKAGED_PATH_PENDING_RE); + assert.match(support.supportLabel, NOT_SELF_SERVICE_RE); assert.doesNotMatch(support.supportLabel, DEMOTION_COPY_RE); }); -test("manual/upload pending source stays packaged-path-pending", () => { +test("unshipped manual upload path stays unavailable for self-service setup", () => { const map = buildSourceAddSupport([manualUploadPendingManifest("google-maps-pending")]); const support = resolveSourceAddSupport(map, "google-maps-pending"); assert.ok(support); - assert.equal(support.support, "packaged_path_pending"); + assert.equal(support.support, "not_self_service"); assert.equal(support.action, null); - assert.match(support.supportLabel, PACKAGED_PATH_PENDING_RE); + assert.match(support.supportLabel, NOT_SELF_SERVICE_RE); assert.doesNotMatch(support.supportLabel, DEMOTION_COPY_RE); }); @@ -187,8 +187,8 @@ test("provider-auth source reports deployment prerequisite, not static-secret se }); test("a connection's raw registry-prefixed connector_id resolves to the canonical key", () => { - const map = buildSourceAddSupport([staticSecretManifest("ynab")]); - const support = resolveSourceAddSupport(map, "https://registry.pdpp.org/connectors/ynab"); + const map = buildSourceAddSupport([staticSecretManifest("gmail")]); + const support = resolveSourceAddSupport(map, "https://registry.pdpp.org/connectors/gmail"); assert.ok(support, "registry-URL connector_id must resolve via canonicalConnectorKey"); assert.equal(support.support, "self_service"); }); diff --git a/apps/console/src/app/(console)/lib/source-add-support.ts b/apps/console/src/app/(console)/lib/source-add-support.ts index 891fa313d..7fe5c3952 100644 --- a/apps/console/src/app/(console)/lib/source-add-support.ts +++ b/apps/console/src/app/(console)/lib/source-add-support.ts @@ -46,8 +46,8 @@ export interface SourceAddSupport { const SUPPORT_LABELS: Record = { deployment_prerequisite: "Server setup required to add another account", - not_self_service: "Add path not available here", - packaged_path_pending: "Add path not packaged", + not_self_service: "Adding another account is not available here", + packaged_path_pending: "Adding another account is not available yet", self_service: "Add another account", }; diff --git a/apps/console/src/app/(console)/lib/source-copy-negative.test.ts b/apps/console/src/app/(console)/lib/source-copy-negative.test.ts index c3377ae27..962a58a7a 100644 --- a/apps/console/src/app/(console)/lib/source-copy-negative.test.ts +++ b/apps/console/src/app/(console)/lib/source-copy-negative.test.ts @@ -84,10 +84,75 @@ function stripComments(src: string): string { /** A minimal catalog entry stub for a given disposition. */ function entryForDisposition(disposition: ConnectorCatalogEntry["disposition"]): ConnectorCatalogEntry { + const supported = new Set([ + "local_collector_enroll", + "static_secret_connect", + "manual_upload_connect", + "browser_collector_manual", + "provider_auth_connect", + ]).has(disposition); + const proofGated = new Set([ + "local_collector_unproven", + "provider_auth_proof_gated", + "browser_bound_runbook", + "manual_upload_pending", + ]).has(disposition); + const providerDeploymentBlocked = disposition === "provider_auth_deployment_blocked"; + let setupModality = "unsupported"; + if (disposition === "static_secret_connect") { + setupModality = "static_secret"; + } else if (disposition === "manual_upload_connect" || disposition === "manual_upload_pending") { + setupModality = "manual_or_upload"; + } else if ( + disposition === "provider_auth_connect" || + providerDeploymentBlocked || + disposition === "provider_auth_proof_gated" + ) { + setupModality = "provider_authorization"; + } else if (disposition === "browser_collector_manual" || disposition === "browser_bound_runbook") { + setupModality = "browser_bound"; + } else if (disposition === "local_collector_enroll" || disposition === "local_collector_unproven") { + setupModality = "local_collector"; + } + + let nextStepKind = "unsupported"; + if (disposition === "local_collector_enroll") { + nextStepKind = "enroll_local_collector"; + } else if (disposition === "browser_collector_manual") { + nextStepKind = "enroll_browser_collector"; + } else if (disposition === "static_secret_connect") { + nextStepKind = "capture_static_secret"; + } else if (disposition === "provider_auth_connect") { + nextStepKind = "open_provider_auth"; + } else if (providerDeploymentBlocked) { + nextStepKind = "needs_deployment_config"; + } else if (disposition === "manual_upload_connect" || disposition === "manual_upload_pending") { + nextStepKind = "provide_import_file"; + } else if (disposition === "browser_bound_runbook" || disposition === "provider_auth_proof_gated") { + nextStepKind = "manual_runbook"; + } + let supportState = "unsupported"; + if (providerDeploymentBlocked) { + supportState = "needs_deployment_config"; + } else if (supported) { + supportState = "supported"; + } else if (proofGated) { + supportState = "proof_gated"; + } return { connectorKey: `stub-${disposition}`, - deploymentReadiness: { blockers: [], ready: true }, + deploymentReadiness: { + blockers: [], + guidance: null, + state: providerDeploymentBlocked ? "needs_config" : "ready", + }, disposition, + nextStepKind, + ownerActionMethod: supported ? "POST" : null, + ownerActionUrl: supported ? "/v1/owner/connections/intents" : null, + proofGate: proofGated ? "setup_proof_missing" : null, + setupModality, + supportState, } as unknown as ConnectorCatalogEntry; } @@ -99,6 +164,7 @@ const ALL_DISPOSITIONS: readonly ConnectorCatalogEntry["disposition"][] = [ "manual_upload_connect", "manual_upload_pending", "provider_auth_deployment_blocked", + "provider_auth_connect", "browser_bound_runbook", "local_collector_unproven", "provider_auth_proof_gated", @@ -127,8 +193,9 @@ test("only self-service and server-setup dispositions expose primary actions", ( "static_secret_connect", "manual_upload_connect", "browser_collector_manual", - "provider_auth_deployment_blocked", + "provider_auth_connect", ]); + const serverSetupDisposition = "provider_auth_deployment_blocked" as const; for (const disposition of ALL_DISPOSITIONS) { const entry = entryForDisposition(disposition); const action = sourceSetupAction(entry); @@ -138,6 +205,11 @@ test("only self-service and server-setup dispositions expose primary actions", ( assert.notEqual(availability, "not_available_here"); continue; } + if (disposition === serverSetupDisposition) { + assert.equal(action, null, "blocked provider authorization must not render a dead action"); + assert.equal(availability, "requires_server_setup"); + continue; + } assert.equal(action, null, `${disposition} must not render a fake primary setup action`); assert.equal(availability, "not_available_here", `${disposition} must be separated from available setup`); } @@ -186,9 +258,9 @@ test("the agreed add-account labels are exactly the realignment-plan vocabulary" assert.ok(labels.length > 0, "projection must produce at least one label"); const AGREED = new Set([ "Add another account", - "Add path not packaged", + "Adding another account is not available yet", "Server setup required to add another account", - "Add path not available here", + "Adding another account is not available here", ]); for (const label of labels) { assertCleanCopy(label, "addAccountSupport label"); diff --git a/apps/console/src/app/(console)/lib/source-setup-form-contract.test.ts b/apps/console/src/app/(console)/lib/source-setup-form-contract.test.ts new file mode 100644 index 000000000..d122c1833 --- /dev/null +++ b/apps/console/src/app/(console)/lib/source-setup-form-contract.test.ts @@ -0,0 +1,100 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import test from "node:test"; +import type { StaticSecretSetup } from "./ref-client.ts"; +import { + browserSessionFormContract, + connectionNameFieldContract, + optionalCredentialFieldLabel, + staticSecretFormContract, +} from "./source-setup-form-contract.ts"; + +const INTERACTIVE_SIGN_IN_RE = /Interactive sign-in is valid/; +const LEAVE_FIELDS_BLANK_RE = /Leave these fields blank/; +const UNATTENDED_RECONNECTION_RE = /unattended reconnection is not guaranteed/; +const NO_PROVIDER_CREDENTIALS_RE = /does not collect provider credentials/; +const NO_UNATTENDED_RECONNECTION_RE = /does not promise unattended reconnection/; +const AUTOMATIC_LOGIN_RE = /automatic login/i; + +const SETUP: StaticSecretSetup = { + connector_id: "synthetic-browser-source", + credential_capture: { + description: "Manifest-authored sign-in details.", + fields: [ + { + autocomplete: "username", + description: null, + help_text: null, + help_url: null, + identity: false, + label: "Username", + name: "username", + placeholder: null, + required: true, + secret: true, + type: "text", + }, + { + autocomplete: "current-password", + description: null, + help_text: null, + help_url: null, + identity: false, + label: "Password", + name: "password", + placeholder: null, + required: true, + secret: true, + type: "password", + }, + ], + kind: "username_password", + label: "Sign-in details", + submit_label: "Save details", + }, + credential_kind: "username_password", + deployment_readiness: { blockers: [], guidance: null, state: "ready" }, + display_name: "Synthetic browser source", + object: "static_secret_setup", + validation: "first_sync", +}; + +test("connection-name contract is app-owned and shared across source forms", () => { + assert.deepEqual(connectionNameFieldContract("Synthetic source"), { + helpText: "Used only when creating a new source. You can rename it later.", + label: "Connection name (optional)", + maxLength: 200, + name: "display_name", + placeholder: "Synthetic source personal", + }); +}); + +test("browser credential contract makes interactive sign-in and optional fields explicit", () => { + const contract = browserSessionFormContract(SETUP); + assert.ok(contract.optionalCredentials); + assert.match(contract.setupDescription, INTERACTIVE_SIGN_IN_RE); + assert.match(contract.optionalCredentials.description, LEAVE_FIELDS_BLANK_RE); + assert.match(contract.optionalCredentials.description, UNATTENDED_RECONNECTION_RE); + const usernameField = SETUP.credential_capture.fields.find((field) => field.name === "username"); + assert.ok(usernameField); + assert.equal(optionalCredentialFieldLabel(usernameField), "Username (optional)"); + assert.deepEqual(contract.optionalCredentials.fields, SETUP.credential_capture.fields); +}); + +test("browser-only start has no optional credential section or automatic-login promise", () => { + const contract = browserSessionFormContract(null); + assert.equal(contract.optionalCredentials, null); + assert.match(contract.setupDescription, NO_PROVIDER_CREDENTIALS_RE); + assert.match(contract.setupDescription, NO_UNATTENDED_RECONNECTION_RE); + assert.doesNotMatch(contract.setupDescription, AUTOMATIC_LOGIN_RE); +}); + +test("static-secret form keeps manifest fields and submit label while adding the shared name field", () => { + const contract = staticSecretFormContract(SETUP, false); + assert.equal(contract.connectionName.name, "display_name"); + assert.deepEqual(contract.credentialFields, SETUP.credential_capture.fields); + assert.equal(contract.primaryActionLabel, "Save details"); + assert.equal(staticSecretFormContract(SETUP, true).primaryActionLabel, "Reconnect account and run sync"); +}); diff --git a/apps/console/src/app/(console)/lib/source-setup-form-contract.ts b/apps/console/src/app/(console)/lib/source-setup-form-contract.ts new file mode 100644 index 000000000..f6fe66dc2 --- /dev/null +++ b/apps/console/src/app/(console)/lib/source-setup-form-contract.ts @@ -0,0 +1,103 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Shared owner-form presentation for source setup. + * + * Connector manifests own provider credential fields and their metadata. The + * console owns the small amount of workflow state that is common to every + * source setup form: the optional connection name and, for browser sessions, + * the choice to save optional sign-in details. Keeping those two contracts + * here prevents the three capture forms from quietly inventing different + * labels or promises. + */ + +import type { StaticSecretSetup, StaticSecretSetupField } from "./ref-client.ts"; + +export interface ConnectionNameFieldContract { + readonly helpText: string; + readonly label: "Connection name (optional)"; + readonly maxLength: 200; + readonly name: "display_name"; + readonly placeholder: string; +} + +export function connectionNameFieldContract(displayName: string): ConnectionNameFieldContract { + return { + helpText: "Used only when creating a new source. You can rename it later.", + label: "Connection name (optional)", + maxLength: 200, + name: "display_name", + placeholder: `${displayName} personal`, + }; +} + +export interface BrowserOptionalCredentialContract { + readonly checkboxLabel: "Save these details to assist initial sign-in or repair."; + readonly checkboxName: "remember_sign_in_details"; + readonly description: string; + readonly fields: readonly StaticSecretSetupField[]; + readonly title: "Optional saved sign-in details"; +} + +export interface BrowserSessionFormContract { + readonly optionalCredentials: BrowserOptionalCredentialContract | null; + readonly repairLoginDescription: string; + readonly setupDescription: string; +} + +/** + * Project the presence of manifest credential capture into owner-safe browser + * copy. A missing setup descriptor is meaningful: the owner signs in in the + * browser and this flow makes no automatic-login promise. + */ +export function browserSessionFormContract(setup: StaticSecretSetup | null): BrowserSessionFormContract { + if (!setup) { + return { + optionalCredentials: null, + repairLoginDescription: + "This flow uses the browser session directly; it does not collect provider credentials and does not promise unattended reconnection.", + setupDescription: + "Create a new account in a secure browser. Sign in interactively; this flow does not collect provider credentials and does not promise unattended reconnection.", + }; + } + + return { + optionalCredentials: { + checkboxLabel: "Save these details to assist initial sign-in or repair.", + checkboxName: "remember_sign_in_details", + description: + "Interactive sign-in is valid. Leave these fields blank to sign in in the secure browser; save them only if they may help with initial sign-in or repair. CAPTCHA, OTP, passkeys, and other human steps stay in the browser, and unattended reconnection is not guaranteed.", + fields: setup.credential_capture.fields, + title: "Optional saved sign-in details", + }, + repairLoginDescription: + "Optional encrypted sign-in details may assist repair, but they do not replace the secure browser or guarantee unattended reconnection.", + setupDescription: + "Create a new account in a secure browser. Interactive sign-in is valid; optional saved sign-in details can assist initial sign-in or repair, but they do not guarantee unattended reconnection.", + }; +} + +export function optionalCredentialFieldLabel(field: StaticSecretSetupField): string { + return `${field.label} (optional)`; +} + +export interface StaticSecretFormContract { + readonly connectionName: ConnectionNameFieldContract; + readonly credentialFields: readonly StaticSecretSetupField[]; + readonly credentialSectionDescription: string; + readonly primaryActionLabel: string; +} + +export function staticSecretFormContract(setup: StaticSecretSetup, isReplaceMode: boolean): StaticSecretFormContract { + return { + connectionName: connectionNameFieldContract(setup.display_name), + credentialFields: setup.credential_capture.fields, + credentialSectionDescription: + setup.credential_capture.description ?? + "This form is generated from the connector manifest. Secrets are submitted to the owner-session capture route and are not returned to agents, MCP clients, REST reads, audit payloads, or the dashboard.", + primaryActionLabel: isReplaceMode + ? "Reconnect account and run sync" + : (setup.credential_capture.submit_label ?? "Create connection and start first sync"), + }; +} diff --git a/apps/console/src/app/(console)/lib/source-setup-presentation.ts b/apps/console/src/app/(console)/lib/source-setup-presentation.ts index 429d65f15..3613f9ada 100644 --- a/apps/console/src/app/(console)/lib/source-setup-presentation.ts +++ b/apps/console/src/app/(console)/lib/source-setup-presentation.ts @@ -23,7 +23,7 @@ * `owner-journey-slvp-realignment-plan-2026-06-10.md`. */ -import type { ConnectorCatalogEntry } from "./connection-catalog.ts"; +import { type ConnectorCatalogEntry, isOwnerActionableEntry } from "./connection-catalog.ts"; export interface SourceSetupStatus { /** One short owner-facing status label. */ @@ -37,10 +37,35 @@ export interface SourceSetupAction { label: string; } +/** + * Owner context projected from manifest setup/capability metadata. The + * fallback is intentionally setup-modality based: it keeps provider auth + * distinct from file import without naming a connector or claiming a new + * protocol behavior. + */ +export function sourceSetupContext(entry: ConnectorCatalogEntry): string | null { + if (entry.setupHelpText) { + return entry.setupHelpText; + } + if (entry.setupModality === "provider_authorization") { + return ( + entry.refreshPolicyRationale ?? + "This source uses provider authorization, not a file import. Provider app settings must be configured on the instance before an owner can authorize an account." + ); + } + return entry.setupDescription; +} + function browserBoundWithStoredCredentials(entry: ConnectorCatalogEntry): boolean { return entry.modality === "browser_bound" && entry.setupModality === "static_secret"; } +function isUnavailableSetupEntry(entry: ConnectorCatalogEntry): boolean { + return ( + !isOwnerActionableEntry(entry) && entry.disposition !== "provider_auth_deployment_blocked" + ); +} + /** * Whether adding a new account for this disposition is self-service today. * @@ -66,6 +91,9 @@ export type SourceSetupAvailability = "available_now" | "requires_server_setup" /** Owner-facing picker order: actionable dispositions first, unsupported last. */ export function sourceSetupRank(entry: ConnectorCatalogEntry): number { + if (isUnavailableSetupEntry(entry)) { + return 8; + } switch (entry.disposition) { case "local_collector_enroll": return 0; @@ -77,22 +105,27 @@ export function sourceSetupRank(entry: ConnectorCatalogEntry): number { return 3; case "manual_upload_pending": return 4; - case "provider_auth_deployment_blocked": + case "provider_auth_connect": return 5; + case "provider_auth_deployment_blocked": + return 6; case "browser_bound_runbook": case "local_collector_unproven": case "provider_auth_proof_gated": - return 6; + return 7; case "api_network_unsupported": case "unknown_unsupported": - return 7; - default: return 8; + default: + return 9; } } /** The owner-facing status label + tone for first-account setup. */ export function sourceSetupStatus(entry: ConnectorCatalogEntry): SourceSetupStatus { + if (isUnavailableSetupEntry(entry)) { + return { label: "Not available here", tone: "border-border bg-muted/30 text-muted-foreground" }; + } if (browserBoundWithStoredCredentials(entry)) { return { label: "Connect account", @@ -117,9 +150,14 @@ export function sourceSetupStatus(entry: ConnectorCatalogEntry): SourceSetupStat label: "Import file", tone: "border-[color:var(--success)]/30 bg-status-success-bg text-status-success-fg", }; + case "provider_auth_connect": + return { + label: "Authorize account", + tone: "border-[color:var(--success)]/30 bg-status-success-bg text-status-success-fg", + }; case "manual_upload_pending": return { - label: "Import not packaged", + label: "Import not available yet", tone: "border-[color:var(--warning)]/30 bg-status-warning-bg text-status-warning-fg", }; case "provider_auth_deployment_blocked": @@ -129,7 +167,7 @@ export function sourceSetupStatus(entry: ConnectorCatalogEntry): SourceSetupStat }; case "browser_bound_runbook": return { - label: "Browser setup not packaged", + label: "Browser setup not available yet", tone: "border-[color:var(--warning)]/30 bg-status-warning-bg text-status-warning-fg", }; case "local_collector_unproven": @@ -146,30 +184,35 @@ export function sourceSetupStatus(entry: ConnectorCatalogEntry): SourceSetupStat /** One short owner-facing guidance line for first-account setup. */ export function sourceSetupGuidance(entry: ConnectorCatalogEntry): string { + if (isUnavailableSetupEntry(entry)) { + return "This dashboard cannot add this source yet."; + } if (browserBoundWithStoredCredentials(entry)) { - return "Connect in a secure browser session. You can optionally remember sign-in details for automatic reconnection; they may help with initial sign-in or repair, but CAPTCHA, OTP, passkeys, and other human steps stay in the secure browser and unattended reconnection is not guaranteed."; + return "Sign in in the secure browser. Saving sign-in details is optional and may help with setup or repair, but one-time codes, passkeys, and other human steps still happen in the browser. Automatic reconnection is not guaranteed."; } switch (entry.disposition) { case "local_collector_enroll": return "Set up the local collector on the machine that has this data. Repeat setup to add another device or account."; case "browser_collector_manual": - return "Create a browser-session shell to connect a new account. Add an optional source label so the source is easy to distinguish; if you need to reconnect an existing source, go back to Sources and open that source's reconnect flow."; + return "Connect a new account in a secure browser. Add an optional source label to distinguish it later; to reconnect an existing source, go back to Sources and open its reconnect flow."; case "static_secret_connect": return "Enter the required provider credential in the protected setup form. Submit again to add another account."; case "manual_upload_connect": return "Upload an owner-exported file. Reuse an existing source for another export from the same identity; create a new source only for a different account, profile, device, or source identity."; + case "provider_auth_connect": + return "Authorize this provider account in the provider's browser. The connection activates after authorization and account inventory succeed."; case "manual_upload_pending": - return "This source imports an owner-provided file, but the dashboard upload step is not packaged yet."; + return "This source accepts an owner-provided file, but file import is not available in this dashboard yet."; case "provider_auth_deployment_blocked": - return `Configure instance-level provider app material first: ${entry.deploymentReadiness.blockers + return `Finish the server setup first: ${entry.deploymentReadiness.blockers .map((blocker) => blocker.label || blocker.key) .join(", ")}.`; case "browser_bound_runbook": - return "This source can collect through a logged-in browser, but this dashboard does not yet package the add-account path safely."; + return "This source can collect through a logged-in browser, but this dashboard cannot start a new account from here yet."; case "local_collector_unproven": - return "This connector needs a packaged collector path before it can be started from this dashboard."; + return "This source needs a local collection setup before it can start from this dashboard."; case "provider_auth_proof_gated": - return "Provider authorization is not packaged in this dashboard yet."; + return "Provider authorization is not available in this dashboard yet."; case "api_network_unsupported": return "This dashboard cannot add this source yet."; default: @@ -181,6 +224,9 @@ export function sourceSetupGuidance(entry: ConnectorCatalogEntry): string { /** The primary next action for first-account setup, or null when none exists. */ export function sourceSetupAction(entry: ConnectorCatalogEntry): SourceSetupAction | null { + if (!isOwnerActionableEntry(entry)) { + return null; + } // Browser-bound connectors that also declare credential capture still start // from the one browser-session path. The optional saved-sign-in-details // fields live inside that page rather than becoming a second picker choice. @@ -211,8 +257,11 @@ export function sourceSetupAction(entry: ConnectorCatalogEntry): SourceSetupActi href: `/connect/browser-session/${encodeURIComponent(entry.enrollmentKey ?? entry.connectorKey)}`, label: "Connect account", }; - case "provider_auth_deployment_blocked": - return { href: "/deployment", label: "Open server settings" }; + case "provider_auth_connect": + return { + href: `/connect/provider-auth/${encodeURIComponent(entry.connectorKey)}`, + label: "Authorize account", + }; default: return null; } @@ -226,11 +275,15 @@ export function sourceSetupSecondaryAction(_entry: ConnectorCatalogEntry): Sourc } export function sourceSetupAvailability(entry: ConnectorCatalogEntry): SourceSetupAvailability { + if (isUnavailableSetupEntry(entry)) { + return "not_available_here"; + } switch (entry.disposition) { case "local_collector_enroll": case "static_secret_connect": case "manual_upload_connect": case "browser_collector_manual": + case "provider_auth_connect": return "available_now"; case "provider_auth_deployment_blocked": return "requires_server_setup"; @@ -246,11 +299,15 @@ export function sourceSetupAvailability(entry: ConnectorCatalogEntry): SourceSet * self-service add-another-account (the browser-bound dispositions). */ export function addAccountSupport(entry: ConnectorCatalogEntry): AddAccountSupport { + if (isUnavailableSetupEntry(entry)) { + return "not_self_service"; + } switch (entry.disposition) { case "local_collector_enroll": case "static_secret_connect": case "manual_upload_connect": case "browser_collector_manual": + case "provider_auth_connect": return "self_service"; case "browser_bound_runbook": case "manual_upload_pending": diff --git a/apps/console/src/app/(console)/schedules/page.tsx b/apps/console/src/app/(console)/schedules/page.tsx index e95d9a80a..558a93e5b 100644 --- a/apps/console/src/app/(console)/schedules/page.tsx +++ b/apps/console/src/app/(console)/schedules/page.tsx @@ -53,7 +53,9 @@ export default async function SchedulesPage({ searchParams }: { searchParams?: P } const summaries = page.items; - const hasActiveRun = summaries.some((s) => s.schedule?.active_run_id !== null); + const hasActiveRun = summaries.some( + (s) => typeof s.schedule?.active_run_id === "string" && s.schedule.active_run_id.length > 0 + ); return ( diff --git a/apps/console/src/app/(console)/schedules/schedule-row.tsx b/apps/console/src/app/(console)/schedules/schedule-row.tsx index 100f8a1bc..37aa89683 100644 --- a/apps/console/src/app/(console)/schedules/schedule-row.tsx +++ b/apps/console/src/app/(console)/schedules/schedule-row.tsx @@ -9,6 +9,7 @@ import Link from "next/link"; import { useRouter } from "next/navigation"; import { useCallback, useState, useTransition } from "react"; import type { RefConnectorSummary, RefSchedule } from "../lib/ref-client.ts"; +import { activeScheduleRunId, scheduleEnabled, scheduleIntervalSeconds } from "../lib/schedule-evidence.ts"; import { formatTotalRecordsLabel } from "../lib/total-records-label.ts"; import { deleteScheduleAction, pauseScheduleAction, resumeScheduleAction, upsertScheduleAction } from "./actions.ts"; @@ -19,7 +20,10 @@ interface ScheduleRowProps { type EditState = "idle" | "editing"; -function formatInterval(seconds: number): string { +function formatInterval(seconds: number | null): string { + if (seconds === null) { + return "—"; + } if (seconds < 60) { return `${seconds}s`; } @@ -32,7 +36,10 @@ function formatInterval(seconds: number): string { return `${Math.round(seconds / 86_400)}d`; } -function formatIntervalForInput(seconds: number): string { +function formatIntervalForInput(seconds: number | null): string { + if (seconds === null) { + return ""; + } if (seconds < 60) { return `${seconds}s`; } @@ -46,10 +53,11 @@ function formatIntervalForInput(seconds: number): string { } function recommendedIntervalLabel(policy: RefConnectorSummary["refresh_policy"]): string | null { - if (!policy?.recommended_interval_seconds) { + const seconds = scheduleIntervalSeconds(policy?.recommended_interval_seconds); + if (seconds === null) { return null; } - return formatInterval(policy.recommended_interval_seconds); + return formatInterval(seconds); } /** @@ -93,16 +101,37 @@ function remoteScheduleFor(summary: RefConnectorSummary): RefSchedule | null { return summary.source_kind === "local_device" ? null : summary.schedule; } +function ScheduleEditButton({ + intervalSeconds, + isPending, + jitterSeconds, + onEdit, +}: { + intervalSeconds: number | null; + isPending: boolean; + jitterSeconds: number | null; + onEdit: (every: string, jitter: string) => void; +}) { + return ( + onEdit(formatIntervalForInput(intervalSeconds), formatIntervalForInput(jitterSeconds))} + size="sm" + variant="ghost" + > + Edit + + ); +} + export function ScheduleRow({ summary, runsHref }: ScheduleRowProps) { const router = useRouter(); const [isPending, startTransition] = useTransition(); const [editState, setEditState] = useState("idle"); - const [every, setEvery] = useState(() => - summary.schedule ? formatIntervalForInput(summary.schedule.interval_seconds) : "1h" - ); - const [jitter, setJitter] = useState(() => - summary.schedule?.jitter_seconds ? formatIntervalForInput(summary.schedule.jitter_seconds) : "" - ); + const initialIntervalSeconds = scheduleIntervalSeconds(summary.schedule?.interval_seconds); + const initialJitterSeconds = scheduleIntervalSeconds(summary.schedule?.jitter_seconds); + const [every, setEvery] = useState(() => formatIntervalForInput(initialIntervalSeconds)); + const [jitter, setJitter] = useState(() => formatIntervalForInput(initialJitterSeconds)); const [toast, setToast] = useState<{ kind: "error" | "warning"; message: string } | null>(null); const showToast = useCallback((kind: "error" | "warning", message: string) => { @@ -194,7 +223,10 @@ export function ScheduleRow({ summary, runsHref }: ScheduleRowProps) { }); const connectorKey = formatConnectorKeyForDisplay(summary.connector_id); const recordsHref = recordsHrefForSummary(summary); - const activeRunId = schedule?.active_run_id; + const activeRunId = activeScheduleRunId(schedule); + const enabled = scheduleEnabled(schedule?.enabled); + const intervalSeconds = scheduleIntervalSeconds(schedule?.interval_seconds); + const jitterSeconds = scheduleIntervalSeconds(schedule?.jitter_seconds); // schedule is RefSchedule | null (remoteScheduleFor can return null); tsc // confirms schedule.human_attention_needed errors without the guard when // the fallback is removed. Biome's type-aware pass mis-resolves this one. @@ -247,9 +279,9 @@ export function ScheduleRow({ summary, runsHref }: ScheduleRowProps) { running → )} - {schedule && ( + {schedule && enabled !== null && ( <> - {schedule.enabled ? ( + {enabled ? ( Pause @@ -258,18 +290,16 @@ export function ScheduleRow({ summary, runsHref }: ScheduleRowProps) { Resume )} - { - setEvery(formatIntervalForInput(schedule.interval_seconds)); - setJitter(schedule.jitter_seconds ? formatIntervalForInput(schedule.jitter_seconds) : ""); + { + setEvery(nextEvery); + setJitter(nextJitter); setEditState(editState === "editing" ? "idle" : "editing"); }} - size="sm" - variant="ghost" - > - Edit - + /> Delete @@ -294,7 +324,7 @@ export function ScheduleRow({ summary, runsHref }: ScheduleRowProps) { Automation: {automationModeLabel(schedule.automation_mode)} - Every: {formatInterval(schedule.interval_seconds)} + Every: {formatInterval(intervalSeconds)} ) : ( diff --git a/apps/console/src/app/(console)/schedules/schedules-poller.tsx b/apps/console/src/app/(console)/schedules/schedules-poller.tsx index db472e7b2..c5d7d67b5 100644 --- a/apps/console/src/app/(console)/schedules/schedules-poller.tsx +++ b/apps/console/src/app/(console)/schedules/schedules-poller.tsx @@ -3,21 +3,8 @@ // Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 -import { useRouter } from "next/navigation"; -import { useEffect } from "react"; - -const ACTIVE_POLL_MS = 3000; +import { LivePoller } from "../components/live-poller.tsx"; export function SchedulesPoller({ enabled }: { enabled: boolean }) { - const router = useRouter(); - - useEffect(() => { - if (!enabled) { - return; - } - const id = setInterval(() => router.refresh(), ACTIVE_POLL_MS); - return () => clearInterval(id); - }, [enabled, router]); - - return null; + return ; } diff --git a/apps/console/src/app/(console)/sources/[connector]/[stream]/page.tsx b/apps/console/src/app/(console)/sources/[connector]/[stream]/page.tsx index 4bb9ec930..428edf76f 100644 --- a/apps/console/src/app/(console)/sources/[connector]/[stream]/page.tsx +++ b/apps/console/src/app/(console)/sources/[connector]/[stream]/page.tsx @@ -21,6 +21,7 @@ import { RecordroomShellWithPalette } from "@/app/(console)/components/recordroo import { ServerUnreachable } from "../../../components/shell.tsx"; import { WarningsBanner } from "../../../components/warnings-banner.tsx"; import { formatStreamCollectionFacts, type StreamCollectionFacts } from "../../../lib/collection-report.ts"; +import { connectorRunSummaryId } from "../../../lib/connector-run-summary-status.ts"; import { ReferenceServerUnreachableError, ResourceServerHttpError } from "../../../lib/owner-token.ts"; import { pickSemanticTimestamp, primaryTimestamp } from "../../../lib/record-timestamps.ts"; import type { RefCollectionReportEntry, RefConnectorRunSummary } from "../../../lib/ref-client.ts"; @@ -664,7 +665,8 @@ function latestStreamRunEvidence( facts: StreamCollectionFacts | null, latestRun: RefConnectorRunSummary | null ): { detail: string; href: string | null; value: string } { - const href = latestRun ? `/syncs/${encodeURIComponent(latestRun.run_id)}` : null; + const runId = connectorRunSummaryId(latestRun?.run_id); + const href = runId ? `/syncs/${encodeURIComponent(runId)}` : null; if (!facts) { return { detail: latestRun @@ -675,7 +677,7 @@ function latestStreamRunEvidence( }; } const detailParts = [ - latestRun ? `run ${latestRun.run_id}` : null, + runId ? `run ${runId}` : null, `coverage ${facts.coverage.value}`, facts.disposition ? `next run: ${facts.disposition.label}` : null, facts.pendingDetailGapsLabel, diff --git a/apps/console/src/app/(console)/sources/[connector]/actions.ts b/apps/console/src/app/(console)/sources/[connector]/actions.ts index 0e3b43d2f..7012a9aca 100644 --- a/apps/console/src/app/(console)/sources/[connector]/actions.ts +++ b/apps/console/src/app/(console)/sources/[connector]/actions.ts @@ -59,7 +59,7 @@ export async function runConnectorNowAction(formData: FormData) { try { const runOptions = { force }; const result = (await (connectionId - ? runConnectionNow(connectionId, runOptions) + ? runConnectionNow(connectionId, { ...runOptions, runAdmission: "setup" }) : runConnectorNow(connectorId, runOptions))) as { run_id?: string; trace_id?: string; diff --git a/apps/console/src/app/(console)/sources/[connector]/page.tsx b/apps/console/src/app/(console)/sources/[connector]/page.tsx index b9bf0bed0..87f62979d 100644 --- a/apps/console/src/app/(console)/sources/[connector]/page.tsx +++ b/apps/console/src/app/(console)/sources/[connector]/page.tsx @@ -2,7 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 import { buttonVariants, IcButton, IcTimestamp } from "@pdpp/brand-react"; -import { formatConnectorKeyForDisplay, formatConnectorNameForDisplay, isFallbackConnectionLabel } from "@pdpp/display"; +import type { StreamManifestEntry } from "@pdpp/display"; +import { + formatConnectorKeyForDisplay, + formatConnectorNameForDisplay, + isFallbackConnectionLabel, + streamDisplayLabel, +} from "@pdpp/display"; import { CopyButton } from "@pdpp/operator-ui/components/copy-button"; import { DataList, PageHeader, Section, StatusBadge } from "@pdpp/operator-ui/components/primitives"; import Link from "next/link"; @@ -31,7 +37,7 @@ import { syncActionIdleLabel, } from "../../lib/connection-evidence.ts"; import { isBrowserBoundConnector, isBrowserSessionBoundConnection } from "../../lib/connection-modality.ts"; -import { isActiveConnectorRunSummaryStatus } from "../../lib/connector-run-summary-status.ts"; +import { connectorRunSummaryId, isActiveConnectorRunSummaryStatus } from "../../lib/connector-run-summary-status.ts"; import { getReferencePublicOrigin, ReferenceServerUnreachableError } from "../../lib/owner-token.ts"; import { isRevokedConnection } from "../../lib/records-list-classification.ts"; import { @@ -158,7 +164,8 @@ export interface ConnectorPageModel { } function toConnectorRunRef(summary: RefConnectorRunSummary | null) { - if (!summary) { + const runId = connectorRunSummaryId(summary?.run_id); + if (!(summary && runId)) { return null; } return { @@ -167,7 +174,7 @@ function toConnectorRunRef(summary: RefConnectorRunSummary | null) { first_at: summary.first_at, known_gaps: summary.known_gaps ?? [], last_at: summary.last_at, - run_id: summary.run_id, + run_id: runId, status: summary.status, }; } @@ -181,6 +188,10 @@ function toRunSummaryForConnection( if (!summary) { return null; } + const runId = connectorRunSummaryId(summary.run_id); + if (!runId) { + return null; + } const status = runStatusWithCollectionReportGaps(summary.status, collectionReport); return { connection_id: connectionId, @@ -194,7 +205,7 @@ function toRunSummaryForConnection( last_at: summary.last_at, needs_input: false, object: "run_summary", - run_id: summary.run_id, + run_id: runId, status, }; } @@ -482,13 +493,39 @@ function resolveActiveRunNavigation(input: { overview: ConnectorOverview; schedu running: boolean; } { const activeRunId = - input.scheduleActiveRunId ?? (input.overview.isRunning ? (input.overview.lastRun?.run_id ?? null) : null); + input.scheduleActiveRunId ?? + (input.overview.isRunning && input.overview.lastRun ? input.overview.lastRun.run_id : null); return { activeRunHref: activeRunId ? `/syncs/${encodeURIComponent(activeRunId)}` : null, running: activeRunId !== null || input.overview.isRunning, }; } +function StreamDisplayName({ + displayLabel, + name, + unexpected, +}: { + displayLabel: string; + name: string; + unexpected: boolean; +}) { + return ( + + {displayLabel} + {unexpected ? ( + + (undeclared) + + ) : null} + + ); +} + function ConnectorPageView({ model, dangerMessage, @@ -579,6 +616,12 @@ function ConnectorPageView({ const syncIdleLabel = syncActionIdleLabel(overview.lastRun?.status); const streakDots = deriveStreakDots(recentRuns); const autoPausedBanner = deriveAutoPausedBanner(schedule); + const manifestStreams = Array.isArray(manifest.streams) ? manifest.streams : []; + const streamLabelsByName = new Map( + manifestStreams.map( + (stream) => [stream.name, streamDisplayLabel(stream.name, stream as StreamManifestEntry)] as const + ) + ); return ( @@ -658,6 +701,7 @@ function ConnectorPageView({ const ownerActionAvailable = collectionOwnerActionByStream[s.name] ?? true; const countLabel = streamCountLabel(s); const unexpected = isUnexpectedStreamDeclaration(s.declaration_state); + const displayLabel = streamLabelsByName.get(s.name) ?? s.name; return (
  • - - {s.name} - {unexpected ? ( - - (undeclared) - - ) : null} - + { @@ -79,11 +80,11 @@ test("the already_running 409 branch preserves the full run id for linking", asy assert.match(src, ALREADY_RUNNING_RETURNS_FULL_RUN_ID); }); -test("run-now action forwards explicit force override to the operator client", async () => { +test("run-now action forwards explicit force override and runAdmission to the operator client", async () => { const src = await readFile(ACTIONS_FILE, "utf8"); assert.match(src, FORCE_OPTION_SIGNATURE); assert.match(src, FORCE_OPTION_BODY); - assert.match(src, RUN_CONNECTION_WITH_OPTIONS); + assert.match(src, RUN_CONNECTION_WITH_SETUP_ADMISSION); assert.match(src, RUN_CONNECTOR_WITH_OPTIONS); }); diff --git a/apps/console/src/app/(console)/sources/actions.ts b/apps/console/src/app/(console)/sources/actions.ts index fff1c07d7..c7719e216 100644 --- a/apps/console/src/app/(console)/sources/actions.ts +++ b/apps/console/src/app/(console)/sources/actions.ts @@ -51,7 +51,7 @@ export async function runConnectorNowAction( try { const runOptions = { force: options.force === true }; const body = (await (connectionId - ? runConnectionNow(connectionId, runOptions) + ? runConnectionNow(connectionId, { ...runOptions, runAdmission: "setup" }) : runConnectorNow(connectorId, runOptions))) as { run_id?: string; trace_id?: string; diff --git a/apps/console/src/app/(console)/sources/add/add-source-demo-data.ts b/apps/console/src/app/(console)/sources/add/add-source-demo-data.ts index de4d6b340..4df6f9888 100644 --- a/apps/console/src/app/(console)/sources/add/add-source-demo-data.ts +++ b/apps/console/src/app/(console)/sources/add/add-source-demo-data.ts @@ -16,10 +16,14 @@ export function buildAddSourceDemoCatalog(): { deploymentReadiness: { blockers: [], guidance: null, state: "ready" }, displayName: "ChatGPT", disposition: "static_secret_connect", + externalDocs: [], modality: "api_network", nextStepKind: "capture_static_secret", proofGate: null, + refreshPolicyRationale: null, runbookPath: null, + setupDescription: null, + setupHelpText: null, setupModality: "static_secret", supportState: "supported", }, @@ -37,10 +41,14 @@ export function buildAddSourceDemoCatalog(): { deploymentReadiness: { blockers: [], guidance: null, state: "ready" }, displayName: "Amazon", disposition: "manual_upload_connect", + externalDocs: [], modality: "api_network", nextStepKind: "provide_import_file", proofGate: null, + refreshPolicyRationale: null, runbookPath: null, + setupDescription: null, + setupHelpText: null, setupModality: "manual_or_upload", supportState: "supported", }, @@ -50,11 +58,15 @@ export function buildAddSourceDemoCatalog(): { deploymentReadiness: { blockers: [], guidance: null, state: "ready" }, displayName: "Claude Code", disposition: "local_collector_enroll", + externalDocs: [], enrollmentKey: "claude_code", modality: "local_collector", nextStepKind: "enroll_local_collector", proofGate: null, + refreshPolicyRationale: null, runbookPath: null, + setupDescription: null, + setupHelpText: null, setupModality: "local_collector", supportState: "supported", }, @@ -68,10 +80,14 @@ export function buildAddSourceDemoCatalog(): { }, displayName: "Calendar Demo", disposition: "provider_auth_deployment_blocked", + externalDocs: [], modality: "api_network", nextStepKind: "needs_deployment_config", proofGate: null, + refreshPolicyRationale: null, runbookPath: null, + setupDescription: null, + setupHelpText: null, setupModality: "provider_authorization", supportState: "needs_deployment_config", }, @@ -81,10 +97,14 @@ export function buildAddSourceDemoCatalog(): { deploymentReadiness: { blockers: [], guidance: null, state: "ready" }, displayName: "Browser Archive Demo", disposition: "browser_bound_runbook", + externalDocs: [], modality: "browser_bound", nextStepKind: "manual_runbook", proofGate: "browser_setup_package", + refreshPolicyRationale: null, runbookPath: "docs/connectors/browser-archive.md", + setupDescription: null, + setupHelpText: null, setupModality: "browser_bound", supportState: "proof_gated", }, diff --git a/apps/console/src/app/(console)/sources/add/page.tsx b/apps/console/src/app/(console)/sources/add/page.tsx index 67298f6e0..51bcb08c8 100644 --- a/apps/console/src/app/(console)/sources/add/page.tsx +++ b/apps/console/src/app/(console)/sources/add/page.tsx @@ -7,9 +7,9 @@ import { RecordroomShellWithPalette } from "@/app/(console)/components/recordroo import { existingSourcesByConnectorCatalog } from "../../components/existing-sources-by-connector.ts"; import { ServerUnreachable } from "../../components/shell.tsx"; import { type ExistingSourceSetupLink, SourceSetupCatalog } from "../../components/source-setup-catalog.tsx"; -import { buildConnectorCatalog, type ConnectorCatalogEntry } from "../../lib/connection-catalog.ts"; +import { buildOwnerConnectorCatalog, type ConnectorCatalogEntry } from "../../lib/connection-catalog.ts"; import { ReferenceServerUnreachableError } from "../../lib/owner-token.ts"; -import { listConnectorManifests } from "../../lib/rs-client.ts"; +import { listConnectorManifests, listOwnerConnectorTemplates } from "../../lib/rs-client.ts"; export const dynamic = "force-dynamic"; @@ -27,8 +27,8 @@ export default async function AddSourcePage({ searchParams }: { searchParams: Pr ({ catalog, existingSourcesByConnector } = demo.buildAddSourceDemoCatalog()); } else { try { - const manifests = await listConnectorManifests(); - catalog = buildConnectorCatalog(manifests); + const [manifests, templates] = await Promise.all([listConnectorManifests(), listOwnerConnectorTemplates()]); + catalog = buildOwnerConnectorCatalog(manifests, templates); // EXACT per-connector existing-sources lookup — one `GET // /_ref/connections?connector_id=` call per catalog entry (bounded by // the registered connector-type catalog size, a few dozen, never by @@ -58,7 +58,7 @@ export default async function AddSourcePage({ searchParams }: { searchParams: Pr { href: dashboardRoutes.section.records, label: "Sources" }, { label: "Add source" }, ]} - description="Add source accounts that populate this PDPP instance. App and agent access is configured separately under Connect apps." + description="Add sources that populate this PDPP instance. App and local-client access is configured separately under Connect apps." title="Add source" /> - + ); } diff --git a/apps/console/src/app/(console)/sources/page.tsx b/apps/console/src/app/(console)/sources/page.tsx index c07de7f96..06281ed53 100644 --- a/apps/console/src/app/(console)/sources/page.tsx +++ b/apps/console/src/app/(console)/sources/page.tsx @@ -41,6 +41,7 @@ import { getReferencePublicOrigin, ReferenceServerUnreachableError } from "../li import { listConnectorManifests } from "../lib/rs-client.ts"; import { reactivateConnectionAction, revokeConnectionAction } from "./[connector]/actions.ts"; import { RecordsPagePoller } from "./records-page-poller.tsx"; +import { SOURCE_ACCESS_NOTE } from "./sources-copy.ts"; import { SourcesView } from "./sources-view.tsx"; import { buildSourcesChurnAdvisory, @@ -179,7 +180,7 @@ function SourcesHeader({ error, message, notice }: { error?: string; message?: s margin: 0, }} > - your loading dock · each source pushes into your streams · nothing leaves + {SOURCE_ACCESS_NOTE}

    {notice ? (
    diff --git a/apps/console/src/app/(console)/sources/sources-copy.ts b/apps/console/src/app/(console)/sources/sources-copy.ts new file mode 100644 index 000000000..3c65ffd3a --- /dev/null +++ b/apps/console/src/app/(console)/sources/sources-copy.ts @@ -0,0 +1,6 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** Shared explanation of how source data and connected-app access relate. */ +export const SOURCE_ACCESS_NOTE = + "Sources populate streams in this instance. Connected apps read only what a grant allows."; diff --git a/apps/console/src/app/(console)/sources/sources-view-model.test.ts b/apps/console/src/app/(console)/sources/sources-view-model.test.ts index e4f467ce4..3c2538a9f 100644 --- a/apps/console/src/app/(console)/sources/sources-view-model.test.ts +++ b/apps/console/src/app/(console)/sources/sources-view-model.test.ts @@ -573,6 +573,8 @@ test("formatSchedule is honest about no schedule, paused, and policy-ineligible" assert.equal(formatSchedule({ ...base, enabled: false }), "paused"); assert.equal(formatSchedule({ ...base, effective_mode: "paused" }), "paused"); assert.equal(formatSchedule({ ...base, ineligibility_reason: "manifest_policy" }), "every 1d · paused by policy"); + assert.equal(formatSchedule({ ...base, enabled: undefined as never }), "schedule details unavailable"); + assert.equal(formatSchedule({ ...base, interval_seconds: undefined as never }), "schedule details unavailable"); }); test("exploreHrefFor encodes connection + stream into the Explore deep link", () => { @@ -823,7 +825,7 @@ test("toSourcesView disambiguates duplicate unnamed connections without exposing assert.equal(views[0]?.displayName, "Amazon · account 1"); assert.equal(views[1]?.displayName, "Amazon · account 2"); - assert.equal(views[0]?.accountLine, "Unnamed source · 100 records · 2 streams"); + assert.equal(views[0]?.accountLine, "Amazon source · 100 records · 2 streams"); assert.equal(views[2]?.displayName, "Amazon - Personal"); assert.equal(views[2]?.accountLine, "100 records · 2 streams"); assert.equal(views[2]?.listKind, null); @@ -1141,3 +1143,68 @@ test("toSourceInstanceView: the passport 'records' row preserves the exact prior const view = toSourceInstanceView(summary({ total_records: 42, total_records_state: undefined })); assert.equal(passportField(view, "records"), "42"); }); + +test("toSourceInstanceView: accountLine uses connector-derived fallback when display_name is a fallback", () => { + const view = toSourceInstanceView( + summary({ + connector_display_name: "Gmail", + connector_id: "gmail", + display_name: "Gmail", + }) + ); + assert.equal(view.accountLine, "Gmail source · 100 records · 2 streams"); +}); + +test("toSourceInstanceView: accountLine uses Amazon fallback correctly", () => { + const view = toSourceInstanceView( + summary({ + connector_display_name: "Amazon", + connector_id: "amazon", + display_name: "Amazon", + }) + ); + assert.equal(view.accountLine, "Amazon source · 100 records · 2 streams"); +}); + +test("toSourceInstanceView: accountLine preserves owned name (no fallback)", () => { + const view = toSourceInstanceView( + summary({ + connector_display_name: "Gmail", + connector_id: "gmail", + display_name: "Work Gmail", + }) + ); + assert.equal(view.accountLine, "100 records · 2 streams"); +}); + +test("toSourceInstanceView: Google Maps timeline_points displays human label, not protocol identifier", () => { + const manifest = { + connector_id: "google-maps", + streams: [ + { + name: "timeline_points", + display: { label: "Your Google Maps location points" }, + }, + ], + }; + const sum = summary({ connector_id: "google-maps", streams: ["timeline_points"] }); + const view = toSourceInstanceView(sum, { manifests: [manifest] }); + assert.equal(view.streams[0]?.displayLabel, "Your Google Maps location points"); + assert.notEqual(view.streams[0]?.displayLabel, "timeline_points"); +}); + +test("toSourceInstanceView: stream without manifest display.label falls back to name", () => { + const manifest = { + connector_id: "gmail", + streams: [{ name: "messages" }], + }; + const sum = summary({ connector_id: "gmail", streams: ["messages"] }); + const view = toSourceInstanceView(sum, { manifests: [manifest] }); + assert.equal(view.streams[0]?.displayLabel, "messages"); +}); + +test("toSourceInstanceView: stream with no manifest available falls back to name", () => { + const sum = summary({ streams: ["messages"] }); + const view = toSourceInstanceView(sum, { manifests: undefined }); + assert.equal(view.streams[0]?.displayLabel, "messages"); +}); diff --git a/apps/console/src/app/(console)/sources/sources-view-model.ts b/apps/console/src/app/(console)/sources/sources-view-model.ts index a381f0c61..21b7567df 100644 --- a/apps/console/src/app/(console)/sources/sources-view-model.ts +++ b/apps/console/src/app/(console)/sources/sources-view-model.ts @@ -27,7 +27,13 @@ * a false zero or green. */ -import { formatConnectorNameForDisplay, isFallbackConnectionLabel } from "@pdpp/display"; +import type { StreamManifestEntry } from "@pdpp/display"; +import { + deriveSourceDisplayNameFallback, + formatConnectorNameForDisplay, + isFallbackConnectionLabel, + streamDisplayLabel, +} from "@pdpp/display"; import { type ConnectorManifestLike, canonicalConnectorKey, @@ -44,6 +50,7 @@ import type { RefRecordVersionStatsRow, RefSchedule, } from "../lib/ref-client.ts"; +import { scheduleEnabled, scheduleIntervalSeconds } from "../lib/schedule-evidence.ts"; import { isRevokedConnector, isSetupInProgressConnector, @@ -81,6 +88,8 @@ export interface SourceStreamManifestRow { collection: SourceStreamCollectionFacts | null; /** Cursor/checkpoint hint, or null when none is exposed at the index level. */ cursor: string | null; + /** Human display label from manifest display.label, or stream name if absent. */ + displayLabel: string; /** Deep-link into Explore for this connection + stream. */ exploreHref: string; name: string; @@ -194,7 +203,10 @@ export interface SourcesRuntimeAdvisory { note: string; } -type SourceManifestLike = ConnectorManifestLike & { connector_id: string }; +type SourceManifestLike = ConnectorManifestLike & { + connector_id: string; + streams?: readonly StreamManifestEntry[]; +}; const DUPLICATE_SOURCE_GROUP_MIN_UNNAMED = 3; @@ -275,16 +287,21 @@ export function formatSchedule(schedule: RefSchedule | null): string { if (!schedule) { return "manual — no schedule"; } - if (schedule.effective_mode === "paused" || !schedule.enabled) { + const enabled = scheduleEnabled(schedule.enabled); + if (schedule.effective_mode === "paused" || enabled === false) { return "paused"; } + const interval = scheduleIntervalSeconds(schedule.interval_seconds); + if (enabled === null || interval === null) { + return "schedule details unavailable"; + } if (schedule.ineligibility_reason) { - return `every ${formatInterval(schedule.interval_seconds)} · paused by policy`; + return `every ${formatInterval(interval)} · paused by policy`; } if (schedule.effective_mode === "automatic") { - return `every ${formatInterval(schedule.interval_seconds)} · automatic`; + return `every ${formatInterval(interval)} · automatic`; } - return `every ${formatInterval(schedule.interval_seconds)} · manual`; + return `every ${formatInterval(interval)} · manual`; } /** @@ -498,7 +515,12 @@ export function toSourceInstanceView( const listKind = listKindForDisplayName(displayName, kind); let accountLine: string; if (hasFallbackLabel) { - accountLine = `Unnamed source · ${formatSourceListFacts(summary, sourceStreamNames.length)}`; + const fallbackName = deriveSourceDisplayNameFallback({ + connectorId, + displayName: summary.display_name, + name: summary.connector_display_name, + }); + accountLine = `${fallbackName} · ${formatSourceListFacts(summary, sourceStreamNames.length)}`; } else { accountLine = formatSourceListFacts(summary, sourceStreamNames.length); } @@ -507,9 +529,20 @@ export function toSourceInstanceView( const { ownerActionCue } = actionability; const status = actionability.renderedStatus; + const manifest = options.manifests + ? options.manifests.find((candidate) => manifestMatchesConnectorId(candidate, connectorId)) + : undefined; + const streamsByName = new Map(); + if (manifest?.streams) { + for (const stream of manifest.streams) { + streamsByName.set(stream.name, stream as StreamManifestEntry); + } + } + const streams: SourceStreamManifestRow[] = sourceStreamNames.map((name) => { const facts = collectionFactsByStream.get(name) ?? null; const retained = streamRecordsByStream.get(name) ?? null; + const streamDecl = streamsByName.get(name); return { collection: facts ? { @@ -526,10 +559,8 @@ export function toSourceInstanceView( tone: facts.tone, } : null, - // The index summary exposes no cursor or searchable flag per stream; - // render them as unknown rather than guessing. Collection-report facts - // are server-owned and safe to show here without another read. cursor: null, + displayLabel: streamDisplayLabel(name, streamDecl), exploreHref: exploreHrefFor(routeId, name), name, recordCount: retained ? retained.record_count : null, diff --git a/apps/console/src/app/(console)/sources/sources-view.tsx b/apps/console/src/app/(console)/sources/sources-view.tsx index a7af387d6..642b0851d 100644 --- a/apps/console/src/app/(console)/sources/sources-view.tsx +++ b/apps/console/src/app/(console)/sources/sources-view.tsx @@ -57,6 +57,7 @@ import Link from "next/link"; import { useRouter } from "next/navigation"; import { useCallback, useState, useTransition } from "react"; import { type RunNowResult, runConnectorNowAction } from "./actions.ts"; +import { SOURCE_ACCESS_NOTE } from "./sources-copy.ts"; import { buildDuplicateSourceReview, collapseDuplicateFallbackSources, @@ -168,7 +169,7 @@ export function SourcesView({ add a source → - a source pushes into your streams · nothing leaves + {SOURCE_ACCESS_NOTE}
    @@ -227,7 +228,7 @@ function DuplicateSourcesAdvisory({ reviews }: { reviews: readonly DuplicateSour const more = reviews.length > 1 ? ` ${reviews.length - 1} other source type needs the same review.` : ""; return ( ); @@ -832,10 +833,13 @@ function StreamManifest({ instance }: { instance: SourceInstanceView }) { function StreamManifestRow({ stream }: { stream: SourceInstanceView["streams"][number] }) { const { collection } = stream; + const isDisplayLabelDifferent = stream.displayLabel !== stream.name; return ( - {stream.name} + + {stream.displayLabel} + diff --git a/apps/console/src/app/(console)/syncs/[runId]/interaction-form.tsx b/apps/console/src/app/(console)/syncs/[runId]/interaction-form.tsx index 8f02c7c86..73de0ee1e 100644 --- a/apps/console/src/app/(console)/syncs/[runId]/interaction-form.tsx +++ b/apps/console/src/app/(console)/syncs/[runId]/interaction-form.tsx @@ -92,8 +92,7 @@ export function RunInteractionForm({ runId, interactionId, kind, message, fields
  • ) : null}

    - Values you submit here satisfy this run only. The reference server does not persist them as durable connector - credentials, env vars, or timeline payloads. + These values are used only for this run and are not saved as connection credentials.

    {state.error ? (

    diff --git a/apps/console/src/app/(console)/syncs/[runId]/page.tsx b/apps/console/src/app/(console)/syncs/[runId]/page.tsx index f9b13b2e5..068a7e78c 100644 --- a/apps/console/src/app/(console)/syncs/[runId]/page.tsx +++ b/apps/console/src/app/(console)/syncs/[runId]/page.tsx @@ -284,16 +284,16 @@ function getAssistanceTitle(assistance: CurrentRunAssistance, active: boolean): assistance.ownerAction === "act_elsewhere" && assistance.responseContract === "none" ) { - return "Waiting for external approval"; + return "Waiting for approval outside this dashboard"; } if ( assistance.progressPosture === "waiting_retry" && assistance.ownerAction === "none" && assistance.responseContract === "none" ) { - return "Waiting before retry"; + return "Retry scheduled"; } - return "Waiting on operator input"; + return "Waiting for your input"; } function getAssistanceDescription( @@ -309,14 +309,14 @@ function getAssistanceDescription( assistance.ownerAction === "act_elsewhere" && assistance.responseContract === "none" ) { - return "The connector is still running and watching for completion. No dashboard response is required."; + return "The source is still running and watching for completion. No dashboard response is required."; } if ( assistance.progressPosture === "waiting_retry" && assistance.ownerAction === "none" && assistance.responseContract === "none" ) { - return "The connector is waiting before retrying. No owner action is required right now."; + return "The source will retry automatically. No action is needed right now."; } if (supportsStreaming) { return "This run is blocked until the requested browser-surface action is completed."; @@ -334,9 +334,9 @@ function formatAssistanceAttachments(assistance: CurrentRunAssistance): string { return attachment.kind; } if (hasAvailableBrowserSurfaceAttachment(assistance)) { - return "browser_surface available"; + return "Secure browser ready"; } - return "browser_surface waiting for stream target"; + return "Waiting for the secure browser"; }) .join(", "); } @@ -625,7 +625,7 @@ function ViolationDiagnosis({ failure }: { failure: SpineEvent | undefined }) {

    Failure diagnosis

    - runtime-authored + Runtime message
    subtype
    diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/page.tsx b/apps/console/src/app/(console)/syncs/[runId]/stream/page.tsx index 8a3b6c44f..eca0d114a 100644 --- a/apps/console/src/app/(console)/syncs/[runId]/stream/page.tsx +++ b/apps/console/src/app/(console)/syncs/[runId]/stream/page.tsx @@ -18,6 +18,7 @@ import { getCurrentBrowserSurfaceAssistance, getCurrentRunAssistance, hasActiveBrowserSurface, + hasResolvedBrowserSurfaceAssistance, requiresBrowserSurfaceAssistance, } from "../../../lib/run-assistance.ts"; import { @@ -41,13 +42,28 @@ export const viewport: Viewport = { width: "device-width", }; -function RunDetailLink({ children, runId }: { children: string; runId: string }) { +function RunDetailLink({ runId }: { runId: string }) { return ( - {children} + View run details (optional) + + ); +} + +function SetupStatusLink({ connectionId, runId }: { connectionId: string | null; runId: string }) { + if (!connectionId) { + return ; + } + const query = new URLSearchParams({ run_id: runId }); + return ( + + View setup status ); } @@ -112,11 +128,13 @@ function renderNoAssistanceSurface({ connector, currentAssistance, envelope, + connectorInstanceId, runId, runStatus, }: { connector: ConnectorContext | null; currentAssistance: ReturnType; + connectorInstanceId: string | null; envelope: TimelineEnvelope; runId: string; runStatus: RunStatusEnvelope | null; @@ -125,7 +143,7 @@ function renderNoAssistanceSurface({ return ; } if (currentAssistance?.ownerAction === "act_elsewhere" && currentAssistance.responseContract === "none") { - return ; + return ; } const noAssistanceState = selectNoAssistanceStreamState({ // biome-ignore lint/suspicious/noUnnecessaryConditions: runStatus is nullable; tsc rejects removing this. @@ -149,9 +167,18 @@ function renderNoAssistanceSurface({ ); } if (hasActiveBrowserSurface(envelope.events)) { - return ; + return ; + } + // The owner already completed a browser step (e.g. H-E-B login) for this + // run, and no further browser action is currently open. Say so plainly + // instead of reusing the generic "No browser action is waiting" copy, + // which reads as a dead end right after a login the owner just finished + // (fr-setup-status-lifecycle-0806) — the run keeps going in the + // background and this page's job here is done. + if (hasResolvedBrowserSurfaceAssistance(envelope.events)) { + return ; } - return ; + return ; } export default async function RunInteractionStreamPage({ @@ -217,7 +244,14 @@ export default async function RunInteractionStreamPage({ const connector = await resolveConnectorContext(connectorId, connectorInstanceId); if (!streamableAssistance) { - return renderNoAssistanceSurface({ connector, currentAssistance, envelope, runId, runStatus }); + return renderNoAssistanceSurface({ + connector, + connectorInstanceId, + currentAssistance, + envelope, + runId, + runStatus, + }); } return ( @@ -246,7 +280,7 @@ function RunEndedSurface({ let statusLabel = "failed"; let title = `${subject} needs a look.`; let description = - "The browser step is no longer waiting, but the run did not complete successfully. Open the run timeline for the exact failure and next action."; + "The browser step is no longer waiting, but the run did not complete successfully. View run details for the exact failure and next action."; let sectionClass = "rounded-3xl border border-destructive/30 bg-destructive/5 p-6 shadow-2xl shadow-black/10"; if (terminalStatus === "cancelled") { statusLabel = "cancelled"; @@ -255,7 +289,7 @@ function RunEndedSurface({ } else if (terminalStatus === "deferred") { statusLabel = "browser deferred"; title = "Secure browser slot unavailable."; - description = `${subject} waited for a secure browser slot, but capacity stayed full. No connector work started. Retry when a browser slot is available.`; + description = `${subject} waited for a secure browser slot, but capacity stayed full. No connector work started. Try again when a secure browser slot is available.`; sectionClass = "rounded-3xl border border-border bg-card p-6 shadow-2xl shadow-black/10"; } return ( @@ -264,15 +298,19 @@ function RunEndedSurface({

    run {statusLabel}

    {title}

    {description}

    - Open run timeline +
    ); } -function RunContinuingSurface({ connector, runId }: { connector: ConnectorContext | null; runId: string }) { - // biome-ignore lint/suspicious/noUnnecessaryConditions: connector is nullable; tsc rejects removing this. - const subject = connector?.displayName ?? "This run"; +function RunContinuingSurface({ + connectionId, + runId, +}: { + connectionId: string | null; + runId: string; +}) { return (
    @@ -280,17 +318,38 @@ function RunContinuingSurface({ connector, runId }: { connector: ConnectorContex

    run continuing

    No browser action is waiting.

    - {subject} is still being checked. Open the run timeline to follow the latest status. + The run is still in progress. This page updates automatically when browser input is needed.

    - Open run timeline +
    ); } -function PreparingBrowserSurface({ connector, runId }: { connector: ConnectorContext | null; runId: string }) { - // biome-ignore lint/suspicious/noUnnecessaryConditions: connector is nullable; tsc rejects removing this. - const subject = connector?.displayName ?? "This run"; +function AssistanceCompleteSurface({ connectionId, runId }: { connectionId: string | null; runId: string }) { + return ( +
    +
    + +

    browser step complete

    +

    Browser step complete.

    +

    + Collection is continuing in the background. You can close this page — it updates automatically if browser + input is needed again. +

    + +
    +
    + ); +} + +function PreparingBrowserSurface({ + connectionId, + runId, +}: { + connectionId: string | null; + runId: string; +}) { return (
    @@ -298,10 +357,9 @@ function PreparingBrowserSurface({ connector, runId }: { connector: ConnectorCon

    secure browser starting

    Preparing the secure browser.

    - {subject} has started a browser-session repair. This page will open the browser controls as soon as the run - asks for your input. + Keep this page open. Browser controls will appear automatically when the run needs your input.

    - Open run timeline +
    ); @@ -309,27 +367,23 @@ function PreparingBrowserSurface({ connector, runId }: { connector: ConnectorCon function ExternalApprovalSurface({ assistance, - connector, runId, }: { assistance: NonNullable>; - connector: ConnectorContext | null; runId: string; }) { - // biome-ignore lint/suspicious/noUnnecessaryConditions: connector is nullable; tsc rejects removing this. - const subject = connector?.displayName ?? "This run"; return (

    approval waiting

    -

    Approve the prompt outside PDPP.

    +

    Approve the request with the provider.

    {assistance.message}

    - {subject} will continue automatically after the provider confirms the approval. No browser controls are - waiting on this page. + The run will continue automatically after the provider confirms your approval. No browser controls are waiting + here.

    - Open run timeline +
    ); @@ -343,11 +397,10 @@ function UnavailableStreamSurface({ connector, runId }: { connector: ConnectorCo

    stream unavailable

    Waiting for a browser surface

    - {connector ? `${connector.displayName} needs browser control, but ` : "This run needs browser control, but "} - no current stream target is registered for this assistance request. Keep the run open while the runtime - registers a browser surface, then return to the run detail page. + {connector ? `${connector.displayName} needs browser control. ` : "This run needs browser control. "}Keep this + page open while the secure browser prepares. The page updates automatically when it is ready.

    - Back to run detail +
    ); diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-keyboard-focus.test.ts b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-keyboard-focus.test.ts index 5267b9fa7..988e2bdd1 100644 --- a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-keyboard-focus.test.ts +++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-keyboard-focus.test.ts @@ -271,3 +271,62 @@ test("geometry extraction is fail-closed when the SSE element has no complete re assert.equal(readRemoteEditableRect({ width: 3, x: 1, y: 2 }), null); assert.equal(readRemoteEditableRect(null), null); }); + +test("remote-focus at mount then first tap after realistic delay focuses directly (warm cache covers mount→tap)", () => { + let state = createMobileKeyboardFocusState(); + // Remote page sends focus event at mount (t=0) + ({ state } = transition(state, { atMs: 0, rect: editableRect, type: "remote-focus" })); + assert.equal(state.editableRectCache?.confirmedAtMs, 0); + + // User taps ~2.5s later (realistic delay: page render + user reaction time) + const atRealisticTap = 2500; + ({ state } = transition(state, { + atMs: atRealisticTap, + pointerId: 1, + remotePoint: editablePoint, + type: "pointerdown", + })); + const released = transition(state, { + atMs: atRealisticTap + 50, + pointerId: 1, + remotePoint: editablePoint, + type: "pointerup", + }); + + // Should focus directly without needing a second remote-focus (warm cache). + assert.equal(released.effect, "focus-text-input", "one-tap path should fire with extended TTL"); + assert.equal(released.state.gesture, null); +}); + +test("negative case: tap at expired stale geometry still requires re-confirmation", () => { + let state = createMobileKeyboardFocusState(); + // Cache confirmed at t=0 + ({ state } = transition(state, { atMs: 0, rect: editableRect, type: "remote-focus" })); + + // Tap arrives after cache has expired (beyond 3.5s) + const atExpiredCache = MOBILE_KEYBOARD_EDITABLE_RECT_CACHE_TTL_MS + 100; + ({ state } = transition(state, { + atMs: atExpiredCache, + pointerId: 2, + remotePoint: editablePoint, + type: "pointerdown", + })); + const released = transition(state, { + atMs: atExpiredCache + 50, + pointerId: 2, + remotePoint: editablePoint, + type: "pointerup", + }); + + // Expired cache falls back to awaiting-confirmation pattern, requiring a second remote-focus + assert.equal(released.effect, "none"); + assert.equal( + released.state.gesture?.phase, + "awaiting-confirmation", + "expired cache must re-confirm via second remote-focus" + ); + + // Re-confirmation arrives and shows affordance + const confirmed = transition(released.state, { atMs: atExpiredCache + 51, rect: editableRect, type: "remote-focus" }); + assert.equal(confirmed.effect, "show-affordance"); +}); diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-keyboard-focus.ts b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-keyboard-focus.ts index 4d5d58277..dc4898336 100644 --- a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-keyboard-focus.ts +++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-keyboard-focus.ts @@ -2,10 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 export const MOBILE_KEYBOARD_GESTURE_EXPIRY_MS = 1500; -// Keep a confirmed rect only across the same short trusted-touch window. This -// covers the common warm follow-up tap without turning a past remote focus -// into a durable authority to summon the keyboard. -export const MOBILE_KEYBOARD_EDITABLE_RECT_CACHE_TTL_MS = 1500; +// Extend to cover mount → first tap window (~2-3.5s): remote page sends focus event +// at mount, realistic first tap arrives ~2-3s later after page render/user reaction time. +// Re-confirm at tap time if geometry has drifted since remote-focus; see test case. +export const MOBILE_KEYBOARD_EDITABLE_RECT_CACHE_TTL_MS = 3500; export const MOBILE_KEYBOARD_TAP_SLOP_PX = 12; export interface RemotePoint { diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-page-terminal-state.test.ts b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-page-terminal-state.test.ts index 5b7d20dbe..d394b37a2 100644 --- a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-page-terminal-state.test.ts +++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-page-terminal-state.test.ts @@ -18,13 +18,24 @@ const ENDED_SURFACE_GATE_RE = /noAssistanceState === "ended"[\s\S]{0,360}/; +const SETUP_STATUS_LINK_RE = + /function SetupStatusLink[\s\S]{0,520}\/connect\/status\/\$\{encodeURIComponent\(connectionId\)\}/; +const SETUP_STATUS_CTA_RE = //g; +const NO_PENDING_TIMELINE_DETOUR_RE = /still being checked[\s\S]{0,220}Open the run timeline/; const UNAVAILABLE_STREAM_POLLER_RE = /function UnavailableStreamSurface[\s\S]{0,520}/; const PREPARING_BROWSER_SURFACE_GATE_RE = /hasActiveBrowserSurface\(envelope\.events\)[\s\S]{0,120}/; +const EXTERNAL_APPROVAL_COPY_RE = /Approve the request with the provider\./; const EXTERNAL_APPROVAL_WAITING_COPY_RE = /No browser controls are\s+waiting/; +const OPTIONAL_RUN_DETAILS_COPY_RE = /View run details \(optional\)/; +const LEGACY_RUN_TIMELINE_CTA_RE = /Open run timeline/; const POLLER_TIMELINE_PROBE_RE = /fetch\(`\/_ref\/runs\/\$\{encodeURIComponent\(runId\)\}\/timeline`/; const POLLER_STREAM_READY_RE = /getCurrentBrowserSurfaceAssistance\(timelineEventsFrom\(body\)\) !== null/; const POLLER_HARD_RELOAD_RE = /window\.location\.reload\(\)/; @@ -92,6 +103,39 @@ test("stream page does not render resolved copy solely because assistance disapp assert.match(pageSource, PREPARING_BROWSER_SURFACE_COPY_RE); assert.match(pageSource, CONTINUING_SURFACE_RE); assert.match(pageSource, CONTINUING_POLLER_RE); + assert.match(pageSource, SETUP_STATUS_LINK_RE); + assert.equal([...pageSource.matchAll(SETUP_STATUS_CTA_RE)].length, 3); + assert.doesNotMatch(pageSource, NO_PENDING_TIMELINE_DETOUR_RE); +}); + +// fr-setup-status-lifecycle-0806: a browser-assistance connector (e.g. H-E-B) +// that already resolved its login step must not fall back to the generic +// "No browser action is waiting" copy — that copy is now reserved for a run +// that genuinely never raised browser assistance at all. Once assistance was +// requested and resolved, the page must say so plainly and keep polling for +// any FURTHER assistance request, not just for the run's own terminal state. +test("stream page hands off with explicit copy once browser assistance resolves, instead of the generic no-action copy", () => { + assert.match(pageSource, ASSISTANCE_COMPLETE_GATE_RE); + assert.match(pageSource, ASSISTANCE_COMPLETE_COPY_RE); + assert.match(pageSource, ASSISTANCE_COMPLETE_POLLER_RE); + // The resolved-assistance gate must be checked before the generic + // RunContinuingSurface fallback, so a resolved run never regresses to the + // ambiguous copy. + const assistanceCompleteGateIndex = pageSource.search(ASSISTANCE_COMPLETE_GATE_RE); + const continuingSurfaceIndex = pageSource.lastIndexOf("return { + assert.match(pageSource, OPTIONAL_RUN_DETAILS_COPY_RE); + assert.match(streamViewerSource, OPTIONAL_RUN_DETAILS_COPY_RE); + assert.doesNotMatch(pageSource, LEGACY_RUN_TIMELINE_CTA_RE); + assert.doesNotMatch(streamViewerSource, LEGACY_RUN_TIMELINE_CTA_RE); }); test("external provider approval does not render as a browser-session repair", () => { diff --git a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer.tsx b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer.tsx index 516c02d16..72de60fbc 100644 --- a/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer.tsx +++ b/apps/console/src/app/(console)/syncs/[runId]/stream/stream-viewer.tsx @@ -5979,13 +5979,13 @@ export function ResolvedSurface({ connector, runId }: { connector: ConnectorCont {subject} is back on it.

    - The browser step is complete. You can close this tab with your browser controls, or open the run timeline. + The browser step is complete. You can close this tab with your browser controls, or view run details.

    - Open run timeline + View run details (optional) @@ -6026,7 +6026,7 @@ function UnsupportedSurface({

    This step takes a credential, not a browser.

    - Open run timeline + View run details (optional) diff --git a/apps/console/src/app/(console)/syncs/loading.tsx b/apps/console/src/app/(console)/syncs/loading.tsx index 70999c419..ae2469002 100644 --- a/apps/console/src/app/(console)/syncs/loading.tsx +++ b/apps/console/src/app/(console)/syncs/loading.tsx @@ -15,7 +15,7 @@ import { ListLoadingSkeleton } from "../components/route-loading.tsx"; export default function RunsLoading() { return ( - + ); } diff --git a/apps/console/src/app/(console)/syncs/page.tsx b/apps/console/src/app/(console)/syncs/page.tsx index 80bd01ef2..047913b79 100644 --- a/apps/console/src/app/(console)/syncs/page.tsx +++ b/apps/console/src/app/(console)/syncs/page.tsx @@ -5,7 +5,7 @@ * Syncs — the Recordroom reskin of the Runs route. * * Health-first: this surface answers "what was recently collected, and what (in - * plain English) needs my hand?" It fuses three real reference contracts: + * needs my attention?" It fuses three real reference contracts: * - `_ref/runs` → the runs feed, for per-connection Rhythm + last result * - `_ref/connectors` → per-connection health + schedule + stream list * via the pure {@link buildSyncsViewModel}, then renders the {@link SyncsView} diff --git a/apps/console/src/app/(console)/syncs/syncs-model.test.ts b/apps/console/src/app/(console)/syncs/syncs-model.test.ts index 543d11f51..af098406c 100644 --- a/apps/console/src/app/(console)/syncs/syncs-model.test.ts +++ b/apps/console/src/app/(console)/syncs/syncs-model.test.ts @@ -28,6 +28,7 @@ const RECONNECT_PROMPT_RE = /reconnect|log in/i; const RESUME_FALSE_REASSURANCE_RE = /fills on the next successful run|resumes normally/i; const RESUME_NORMALLY_RE = /resume normally/i; const THROTTLING_RE = /throttling/i; +const UNVERIFIED_ZERO_COPY_RE = /without proving the account was empty/; const ACTIONABILITY_RENDERED_STATUS_RE = /actionability\.renderedStatus/; const RAW_VERDICT_TONE_RE = /rendered_verdict\.pill\.tone|verdict\.pill\.tone/; const SYNCS_PAGE_SOURCE = readFileSync(new URL("./page.tsx", import.meta.url), "utf8"); @@ -551,6 +552,22 @@ test("a draft connection produces a PendingSetupCard, not a SyncGroup or Failure assert.equal(card.continueHref, "/connect/status/cin_draft"); }); +test("terminal setup disposition keeps a draft out of sync groups and carries shared actionability copy", () => { + const model = buildSyncsViewModel({ + connectors: [draftConnector({ terminal_setup_disposition: "unverified_zero" })], + runs: [], + }); + + assert.equal(model.groups.length, 0); + assert.equal(model.failureCards.length, 0); + assert.equal(model.pendingSetupCards.length, 1); + const [card] = model.pendingSetupCards; + assert.ok(card); + assert.equal(card.statusLabel, "needs review"); + assert.equal(card.actionLabel, "Retry first sync"); + assert.match(card.what, UNVERIFIED_ZERO_COPY_RE); +}); + test("a draft connection counts toward needYourHand and inflates onSchedule by zero", () => { const model = buildSyncsViewModel({ connectors: [draftConnector(), connector({ connection_id: "cin_healthy" })], diff --git a/apps/console/src/app/(console)/syncs/syncs-model.ts b/apps/console/src/app/(console)/syncs/syncs-model.ts index 35ab27aa1..f2155e907 100644 --- a/apps/console/src/app/(console)/syncs/syncs-model.ts +++ b/apps/console/src/app/(console)/syncs/syncs-model.ts @@ -39,6 +39,7 @@ import { type SourceStatusFlag, type SourceWorkItem, sourceAttentionHeadline, + sourceWorkItemFromConnector, } from "../lib/source-actionability.ts"; // ─── Rhythm tick type (mirrors the kit's RhythmTick) ────────────────────────── @@ -157,6 +158,8 @@ export interface FailureCard { * fix-pending-connection-discovery design. */ export interface PendingSetupCard { + /** Shared owner action translated from the connection-scoped disposition. */ + actionLabel: string; /** Durable connection identity. */ connectionId: string; /** Connector key. */ @@ -165,6 +168,10 @@ export interface PendingSetupCard { continueHref: string; /** Connection display name (the card title). */ name: string; + /** Shared terminal/setup status label. */ + statusLabel: string; + /** Shared owner-facing explanation. */ + what: string; } /** @@ -663,11 +670,15 @@ function toPendingSetupCard(connector: RefConnectorSummary): PendingSetupCard { // recordsHrefForSummary for the same documented pattern). // biome-ignore lint/suspicious/noUnnecessaryConditions: see comment above. const routeId = connector.connection_id ?? connector.connector_instance_id ?? connector.connector_id; + const work = pendingSetupWorkItem(connector); return { + actionLabel: work.actionLabel ?? SETUP_IN_PROGRESS_CTA_LABEL, connectionId: connector.connection_id, connectorId: connector.connector_id, continueHref: `/connect/status/${encodeURIComponent(routeId)}`, name: connector.display_name, + statusLabel: work.statusLabel, + what: work.what, }; } @@ -676,6 +687,10 @@ function pendingSetupWorkItem(connector: RefConnectorSummary): SourceWorkItem { // connection_id is non-optional in the current contract; connector_instance_id // /connector_id are a real legacy-server fallback (see schedule-row.tsx's // recordsHrefForSummary for the same documented pattern). + const work = sourceWorkItemFromConnector(connector); + if (work) { + return work; + } // biome-ignore lint/suspicious/noUnnecessaryConditions: see comment above. const routeId = connector.connection_id ?? connector.connector_instance_id ?? connector.connector_id; return { diff --git a/apps/console/src/app/(console)/syncs/syncs-view.tsx b/apps/console/src/app/(console)/syncs/syncs-view.tsx index d3081c136..4a23addbc 100644 --- a/apps/console/src/app/(console)/syncs/syncs-view.tsx +++ b/apps/console/src/app/(console)/syncs/syncs-view.tsx @@ -68,7 +68,7 @@ function HealthBandStrip({ band }: { band: SyncsViewModel["band"] }) { const reviewValue = band.needYourHand > 0 ? band.needYourHand : band.needsReview; let reviewLabel = "need attention"; if (band.needYourHand > 0) { - reviewLabel = "need your hand"; + reviewLabel = "need your attention"; } else if (band.needsReview > 0) { reviewLabel = "need review"; } @@ -79,7 +79,9 @@ function HealthBandStrip({ band }: { band: SyncsViewModel["band"] }) { 0 ? "is-warn" : undefined} k={reviewLabel} v={reviewValue} />

    - {band.allClear ? `Nothing needs you right now. ${RESET_NOTE}` : `Review the cards below. ${RESET_NOTE}`} + {band.allClear + ? `Nothing needs your attention right now. ${RESET_NOTE}` + : `Review the cards below. ${RESET_NOTE}`}

    ); @@ -155,13 +157,15 @@ function PendingSetupCardPanel({ card }: { card: PendingSetupCard }) { return (
    -

    {card.name} — setup in progress

    -

    Finish connecting this source to start its first sync.

    +

    + {card.name} — {card.statusLabel} +

    +

    {card.what}

    - Continue setup + {card.actionLabel}
    @@ -216,14 +220,14 @@ function FailureCardSection({ cards, section }: { cards: FailureCard[]; section: function DuplicateSyncGroupPanel({ group }: { group: DuplicateSyncGroup }) { return ( ); @@ -399,7 +403,7 @@ export function SyncsView({ model, seeded = false }: { model: SyncsViewModel; se

    Syncs

    -

    What was recently collected, and what — in plain English — needs your hand.

    +

    What ran recently and what needs your attention.

    {seeded ? : null}
    diff --git a/apps/console/src/lib/pdpp-cli-command.test.ts b/apps/console/src/lib/pdpp-cli-command.test.ts index 91caba182..2531ac0cb 100644 --- a/apps/console/src/lib/pdpp-cli-command.test.ts +++ b/apps/console/src/lib/pdpp-cli-command.test.ts @@ -21,6 +21,7 @@ import { pdppCliPackageInfo, pdppLocalCollectorDoctorCommand, pdppLocalCollectorRetryDeadLettersCommand, + pdppLocalCollectorSetupCommand, pdppLocalCollectorStatusCommand, substituteCommandTemplate, } from "./pdpp-cli-command.ts"; @@ -96,6 +97,53 @@ test("pdppCliCollectorEnrollCommand ignores empty device labels", () => { ); }); +test("pdppLocalCollectorSetupCommand renders the guided one-command onboarding form", () => { + assert.equal( + pdppLocalCollectorSetupCommand({ + baseUrl: "http://127.0.0.1:7662", + code: "abc-123", + connectorId: "claude_code", + }), + "npx -y @pdpp/local-collector setup --base-url http://127.0.0.1:7662 --code abc-123 --connector claude_code" + ); +}); + +test("pdppLocalCollectorSetupCommand appends a quoted --device-label and --sample when provided", () => { + assert.equal( + pdppLocalCollectorSetupCommand({ + baseUrl: "https://ref.example.com", + code: "code-1", + connectorId: "codex", + deviceLabel: "the owner's laptop", + sample: 20, + }), + 'npx -y @pdpp/local-collector setup --base-url https://ref.example.com --code code-1 --connector codex --device-label "the owner\'s laptop" --sample 20' + ); +}); + +test("pdppLocalCollectorSetupCommand ignores an empty device label and omits --sample when absent", () => { + assert.equal( + pdppLocalCollectorSetupCommand({ + baseUrl: "https://ref.example.com", + code: "code-1", + connectorId: "claude_code", + deviceLabel: " ", + }), + "npx -y @pdpp/local-collector setup --base-url https://ref.example.com --code code-1 --connector claude_code" + ); +}); + +test("pdppLocalCollectorSetupCommand never embeds a device token or other secret", () => { + const rendered = pdppLocalCollectorSetupCommand({ + baseUrl: "https://ref.example.com", + code: "code-1", + connectorId: "claude_code", + sample: 20, + }); + // biome-ignore lint/performance/useTopLevelRegex: inline assertion literal scoped to this test case; hoisting would separate the pattern from the single call site it documents. + assert.doesNotMatch(rendered, /--device-token|--device-id/); +}); + test("pdppCliCollectorRunCommand renders the canonical run form", () => { assert.equal( pdppCliCollectorRunCommand({ baseUrl: "http://127.0.0.1:7662", connectorId: "claude_code" }), diff --git a/apps/console/src/lib/pdpp-cli-command.ts b/apps/console/src/lib/pdpp-cli-command.ts index 327ebc1ed..9bcddc3bd 100644 --- a/apps/console/src/lib/pdpp-cli-command.ts +++ b/apps/console/src/lib/pdpp-cli-command.ts @@ -71,11 +71,57 @@ export function pdppLocalCollectorEnrollCommand(args: { return parts.join(" "); } +/** + * Render the public `@pdpp/local-collector setup` command for a freshly + * minted enrollment code and a specific connector id. This is the guided, + * one-command onboarding path: it exchanges the code, saves device + * credentials to a local profile file (never printed/logged), and runs a + * bounded `--sample` proof pass so the operator sees real evidence the + * pairing works before deciding to collect the full source. This is the + * PRIMARY path the dashboard should render — it replaces the old two-step + * `enroll` (prints JSON) + hand-copy-into-env-vars `run` flow, which is a + * discoverability/legibility tax for a first-time operator, not a security + * requirement (the device token still never appears in dashboard-rendered + * text either way). + */ +export function pdppLocalCollectorSetupCommand(args: { + baseUrl: string; + code: string; + connectorId: string; + deviceLabel?: string | null | undefined; + sample?: number | undefined; +}): string { + const parts = [ + "npx", + "-y", + localCollectorPackageSpecifier, + "setup", + "--base-url", + args.baseUrl, + "--code", + args.code, + "--connector", + args.connectorId, + ]; + const label = args.deviceLabel?.trim(); + if (label) { + parts.push("--device-label", JSON.stringify(label)); + } + if (args.sample) { + parts.push("--sample", String(args.sample)); + } + return parts.join(" "); +} + /** * Render the public `@pdpp/local-collector` run command. The device id, device * token, and source instance id come from a prior enrollment response and are * passed as env vars so the dashboard never embeds secrets in generated * commands. + * + * Kept as the LOW-LEVEL alternative for operators who used `enroll` directly + * or manage credentials themselves; {@link pdppLocalCollectorSetupCommand} is + * the primary onboarding command the dashboard renders. */ export function pdppLocalCollectorRunCommand(args: { baseUrl: string; connectorId: string }): string { return [ diff --git a/apps/site/.source/dynamic.ts b/apps/site/.source/dynamic.ts index b66af00dc..1282aefee 100644 --- a/apps/site/.source/dynamic.ts +++ b/apps/site/.source/dynamic.ts @@ -9,4 +9,4 @@ const create = await dynamic(Config, {"environment":"dynamic","root":"","configPath":"source.config.ts","outDir":".source"}); -export const docs = await create.docs("docs", "content/docs", {"meta.json": __fd_glob_0, }, [{ absolutePath: path.resolve("content/docs/README.md"), info: {"fullPath":"content/docs/README.md","path":"README.md"}, data: {"title":"Web Spec Publishing Notes","description":"Internal notes for keeping canonical root specs and public web spec pages in sync."}, hash: "1ba07ddf8f89f32bb09ef593bff2fd69" }, { absolutePath: path.resolve("content/docs/index.mdx"), info: {"fullPath":"content/docs/index.mdx","path":"index.mdx"}, data: {"title":"PDPP Docs","description":"Protocol documentation for the Personal Data Portability Protocol."}, hash: "b9783b6bea9089a761a5c837bf933bfa" }, { absolutePath: path.resolve("content/docs/open-questions.md"), info: {"fullPath":"content/docs/open-questions.md","path":"open-questions.md"}, data: {"title":"Open Questions","description":"Open design questions for PDPP, ordered by importance. Consolidated from the deferred-concerns register, design notes, and review threads."}, hash: "df882328c808e8dd54db7256267872bb" }, { absolutePath: path.resolve("content/docs/reference-implementation-examples.md"), info: {"fullPath":"content/docs/reference-implementation-examples.md","path":"reference-implementation-examples.md"}, data: {"title":"Reference Implementation Examples","description":"Current end-to-end examples from the live PDPP reference implementation. Not normative protocol documentation."}, hash: "a31ccbdc9f80be6fed145b24598cbd31" }, { absolutePath: path.resolve("content/docs/reference-implementation.md"), info: {"fullPath":"content/docs/reference-implementation.md","path":"reference-implementation.md"}, data: {"title":"Reference Implementation Notes","description":"Current implementation behavior for the forkable PDPP reference stack. Not normative protocol documentation."}, hash: "a3be6f281ce67a241421cea5b47e8e33" }, { absolutePath: path.resolve("content/docs/spec-architecture.md"), info: {"fullPath":"content/docs/spec-architecture.md","path":"spec-architecture.md"}, data: {"title":"Reference Topology","description":"How the current PDPP reference components relate: native provider, polyfill path, runtime, and client flows."}, hash: "71c35f0f350f490def49943e7cb7fdad" }, { absolutePath: path.resolve("content/docs/spec-auth-design.md"), info: {"fullPath":"content/docs/spec-auth-design.md","path":"spec-auth-design.md"}, data: {"title":"Auth Design","description":"Bearer tokens at both boundaries: wire format and semantics. Identity provider and token issuance are out of scope."}, hash: "33dfd9d9a0a2e8c6923f4a8491020134" }, { absolutePath: path.resolve("content/docs/spec-change-tracking.md"), info: {"fullPath":"content/docs/spec-change-tracking.md","path":"spec-change-tracking.md"}, data: {"title":"Change Tracking","description":"Design rationale for grant-relative incremental sync via changes_since cursors, not canonical changelog streams."}, hash: "5616160a4f1f92edc946cb8b3c2890e3" }, { absolutePath: path.resolve("content/docs/spec-collection-profile.md"), info: {"fullPath":"content/docs/spec-collection-profile.md","path":"spec-collection-profile.md"}, data: {"title":"Profile","description":"Companion to the Personal Data Portability Protocol (PDPP) core spec."}, hash: "e7da41bf35a380e8e2f858970548a9c2" }, { absolutePath: path.resolve("content/docs/spec-connector-ecosystem.md"), info: {"fullPath":"content/docs/spec-connector-ecosystem.md","path":"spec-connector-ecosystem.md"}, data: {"title":"Connector Ecosystem","description":"Reference runtime notes for connectors: browser abstraction decisions and third-party source integration."}, hash: "1d4005862904051e0a7fb4ca87bc8de2" }, { absolutePath: path.resolve("content/docs/spec-core.md"), info: {"fullPath":"content/docs/spec-core.md","path":"spec-core.md"}, data: {"title":"Protocol Specification","description":"Authorization and disclosure semantics for personal data — record model, selection request, grant, manifest, and resource server interface."}, hash: "6600831d55abaa45add81dd8a4080820" }, { absolutePath: path.resolve("content/docs/spec-data-query-api.md"), info: {"fullPath":"content/docs/spec-data-query-api.md","path":"spec-data-query-api.md"}, data: {"title":"Data Query API (Superseded)","description":"Historical companion spec — Core spec Section 8 is now authoritative for the RS query interface."}, hash: "d68a3755c582a8f849d8fb4bb0ef8152" }, { absolutePath: path.resolve("content/docs/spec-deferred.md"), info: {"fullPath":"content/docs/spec-deferred.md","path":"spec-deferred.md"}, data: {"title":"Deferred Concerns","description":"Issues identified during design and review that are intentionally out of scope for v0.1. Each item is named precisely so it can be referenced from the core spec and tracked for future versions."}, hash: "a8f1026938ab3b386bf91918a13f3c29" }, { absolutePath: path.resolve("content/docs/spec-ext-aggregation.md"), info: {"fullPath":"content/docs/spec-ext-aggregation.md","path":"spec-ext-aggregation.md"}, data: {"title":"Extension Profile: Aggregation","description":"Optional companion profile to the Personal Data Portability Protocol (PDPP) core spec defining a discoverable, grant-safe, single-stream aggregation surface."}, hash: "bbd055c04646182a16b292b7bd74a681" }, { absolutePath: path.resolve("content/docs/spec-ext-lexical-search.md"), info: {"fullPath":"content/docs/spec-ext-lexical-search.md","path":"spec-ext-lexical-search.md"}, data: {"title":"Extension Profile: Lexical Search","description":"Optional companion profile to the Personal Data Portability Protocol (PDPP) core spec defining a discoverable, grant-safe lexical (full-text) search surface."}, hash: "89b5e1ad42f73c7836d9270ce205b013" }, { absolutePath: path.resolve("content/docs/spec-semantic-retrieval-extension.md"), info: {"fullPath":"content/docs/spec-semantic-retrieval-extension.md","path":"spec-semantic-retrieval-extension.md"}, data: {"title":"Semantic Retrieval Extension (Experimental)","description":"Experimental optional PDPP extension defining a semantic retrieval surface at GET /v1/search/semantic. Unstable."}, hash: "853fa323bd46198c205cd5a049e726bc" }]); \ No newline at end of file +export const docs = await create.docs("docs", "content/docs", {"meta.json": __fd_glob_0, }, [{ absolutePath: path.resolve("content/docs/README.md"), info: {"fullPath":"content/docs/README.md","path":"README.md"}, data: {"title":"Web Spec Publishing Notes","description":"Internal notes for keeping canonical root specs and public web spec pages in sync."}, hash: "1ba07ddf8f89f32bb09ef593bff2fd69" }, { absolutePath: path.resolve("content/docs/index.mdx"), info: {"fullPath":"content/docs/index.mdx","path":"index.mdx"}, data: {"title":"PDPP Docs","description":"Protocol documentation for the Personal Data Portability Protocol."}, hash: "b9783b6bea9089a761a5c837bf933bfa" }, { absolutePath: path.resolve("content/docs/open-questions.md"), info: {"fullPath":"content/docs/open-questions.md","path":"open-questions.md"}, data: {"title":"Open Questions","description":"Open design questions for PDPP, ordered by importance. Consolidated from the deferred-concerns register, design notes, and review threads."}, hash: "df882328c808e8dd54db7256267872bb" }, { absolutePath: path.resolve("content/docs/reference-implementation-examples.md"), info: {"fullPath":"content/docs/reference-implementation-examples.md","path":"reference-implementation-examples.md"}, data: {"title":"Reference Implementation Examples","description":"Current end-to-end examples from the live PDPP reference implementation. Not normative protocol documentation."}, hash: "a31ccbdc9f80be6fed145b24598cbd31" }, { absolutePath: path.resolve("content/docs/reference-implementation.md"), info: {"fullPath":"content/docs/reference-implementation.md","path":"reference-implementation.md"}, data: {"title":"Reference Implementation Notes","description":"Current implementation behavior for the forkable PDPP reference stack. Not normative protocol documentation."}, hash: "a3be6f281ce67a241421cea5b47e8e33" }, { absolutePath: path.resolve("content/docs/spec-architecture.md"), info: {"fullPath":"content/docs/spec-architecture.md","path":"spec-architecture.md"}, data: {"title":"Reference Topology","description":"How the current PDPP reference components relate: native provider, polyfill path, runtime, and client flows."}, hash: "71c35f0f350f490def49943e7cb7fdad" }, { absolutePath: path.resolve("content/docs/spec-auth-design.md"), info: {"fullPath":"content/docs/spec-auth-design.md","path":"spec-auth-design.md"}, data: {"title":"Auth Design","description":"Bearer tokens at both boundaries: wire format and semantics. Identity provider and token issuance are out of scope."}, hash: "33dfd9d9a0a2e8c6923f4a8491020134" }, { absolutePath: path.resolve("content/docs/spec-change-tracking.md"), info: {"fullPath":"content/docs/spec-change-tracking.md","path":"spec-change-tracking.md"}, data: {"title":"Change Tracking","description":"Design rationale for grant-relative incremental sync via changes_since cursors, not canonical changelog streams."}, hash: "5616160a4f1f92edc946cb8b3c2890e3" }, { absolutePath: path.resolve("content/docs/spec-collection-profile.md"), info: {"fullPath":"content/docs/spec-collection-profile.md","path":"spec-collection-profile.md"}, data: {"title":"Profile","description":"Companion to the Personal Data Portability Protocol (PDPP) core spec."}, hash: "e7da41bf35a380e8e2f858970548a9c2" }, { absolutePath: path.resolve("content/docs/spec-connector-ecosystem.md"), info: {"fullPath":"content/docs/spec-connector-ecosystem.md","path":"spec-connector-ecosystem.md"}, data: {"title":"Connector Ecosystem","description":"Reference runtime notes for connectors: browser abstraction decisions and third-party source integration."}, hash: "afc37a4e5cdee0bd05e9d5916abf660c" }, { absolutePath: path.resolve("content/docs/spec-core.md"), info: {"fullPath":"content/docs/spec-core.md","path":"spec-core.md"}, data: {"title":"Protocol Specification","description":"Authorization and disclosure semantics for personal data — record model, selection request, grant, manifest, and resource server interface."}, hash: "6600831d55abaa45add81dd8a4080820" }, { absolutePath: path.resolve("content/docs/spec-data-query-api.md"), info: {"fullPath":"content/docs/spec-data-query-api.md","path":"spec-data-query-api.md"}, data: {"title":"Data Query API (Superseded)","description":"Historical companion spec — Core spec Section 8 is now authoritative for the RS query interface."}, hash: "d68a3755c582a8f849d8fb4bb0ef8152" }, { absolutePath: path.resolve("content/docs/spec-deferred.md"), info: {"fullPath":"content/docs/spec-deferred.md","path":"spec-deferred.md"}, data: {"title":"Deferred Concerns","description":"Issues identified during design and review that are intentionally out of scope for v0.1. Each item is named precisely so it can be referenced from the core spec and tracked for future versions."}, hash: "a8f1026938ab3b386bf91918a13f3c29" }, { absolutePath: path.resolve("content/docs/spec-ext-aggregation.md"), info: {"fullPath":"content/docs/spec-ext-aggregation.md","path":"spec-ext-aggregation.md"}, data: {"title":"Extension Profile: Aggregation","description":"Optional companion profile to the Personal Data Portability Protocol (PDPP) core spec defining a discoverable, grant-safe, single-stream aggregation surface."}, hash: "bbd055c04646182a16b292b7bd74a681" }, { absolutePath: path.resolve("content/docs/spec-ext-lexical-search.md"), info: {"fullPath":"content/docs/spec-ext-lexical-search.md","path":"spec-ext-lexical-search.md"}, data: {"title":"Extension Profile: Lexical Search","description":"Optional companion profile to the Personal Data Portability Protocol (PDPP) core spec defining a discoverable, grant-safe lexical (full-text) search surface."}, hash: "89b5e1ad42f73c7836d9270ce205b013" }, { absolutePath: path.resolve("content/docs/spec-semantic-retrieval-extension.md"), info: {"fullPath":"content/docs/spec-semantic-retrieval-extension.md","path":"spec-semantic-retrieval-extension.md"}, data: {"title":"Semantic Retrieval Extension (Experimental)","description":"Experimental optional PDPP extension defining a semantic retrieval surface at GET /v1/search/semantic. Unstable."}, hash: "853fa323bd46198c205cd5a049e726bc" }]); \ No newline at end of file diff --git a/deploy/railway/core.env.example b/deploy/railway/core.env.example new file mode 100644 index 000000000..5dd23cea6 --- /dev/null +++ b/deploy/railway/core.env.example @@ -0,0 +1,19 @@ +# PDPP Railway Core app service +# +# Selected pushbutton path: one public app service named `core` runs the console +# on Railway's injected $PORT and runs the reference AS/RS privately on loopback +# inside the same container. Do not set PORT, AS_PORT, RS_PORT, PDPP_AS_URL, or +# PDPP_RS_URL as Railway service variables; the image/supervisor owns them. + +PDPP_REFERENCE_ORIGIN=https://${{core.RAILWAY_PUBLIC_DOMAIN}} +PDPP_OWNER_PASSWORD= +PDPP_CREDENTIAL_ENCRYPTION_KEY=${{ secret(64) }} +PDPP_DATABASE_URL=${{Postgres.DATABASE_URL}} + +# Optional Google Data Portability OAuth app for the API-backed Google Maps +# Data Portability source. These are deployment-level OAuth app settings, not +# per-account Google credentials and not Gmail app passwords. +# GOOGLE_DATAPORTABILITY_CLIENT_ID= +# GOOGLE_DATAPORTABILITY_CLIENT_SECRET= +# GOOGLE_DATAPORTABILITY_REDIRECT_URI=https://${{core.RAILWAY_PUBLIC_DOMAIN}}/_ref/provider-auth/callback +# GOOGLE_DATAPORTABILITY_RESOURCE_GROUPS= diff --git a/docs/operator/local-collector-runbook.md b/docs/operator/local-collector-runbook.md index 65509c214..93d073c91 100644 --- a/docs/operator/local-collector-runbook.md +++ b/docs/operator/local-collector-runbook.md @@ -79,9 +79,51 @@ After "Create code" the dashboard renders: You do not need to memorize the route or the env var names; the dashboard advertises the exact command. -## Step 3 — Enroll the host +## Step 3 — Set up the host (guided) -On the host with the data, paste the command the dashboard rendered. Example: +On the host with the data, run `setup` with the code the dashboard rendered: + +```bash +npx -y @pdpp/local-collector setup \ + --base-url https://your-pdpp-host.example.com \ + --code \ + --connector claude_code \ + --device-label "the owner's laptop" \ + --sample 20 +``` + +`setup` exchanges the code, saves the device id/device token/connection id to a +local profile file under `~/.config/pdpp/collectors/` (owner-only permissions: +`0600` on the file, `0700` on the directory — POSIX mode bits, so on +Windows the file lands under your user profile directory instead), and — +because `--sample 20` was passed — immediately runs a bounded 20-record +proof pass so you can see real evidence the pairing works before deciding to +collect the full source: + +```text +✓ Enrolled claude_code (device dev_...). +✓ Credentials saved to /home/you/.config/pdpp/collectors/claude_code.env (readable only by you). +✓ Sample stopped after 20 record(s) (limit 20). These records are durably queued + but this is NOT a complete collection — the connector was stopped before + finishing its scan, so no coverage checkpoint was recorded. Run `run` (without + --sample) to collect the full source, or `recover --apply` to drain what was + already queued. + +Next: pdpp-local-collector run --connection-id si_... +(the profile above is picked up automatically — no env vars to set by hand) +``` + +Swap `--connector claude_code` for `codex` to pair Codex CLI history/skills/etc. +Add `--json` for machine-readable output instead of the human summary above. + +Drop `--sample` to skip the proof pass and only enroll + save credentials. +`setup` never prints the raw device token to the terminal — it goes +straight into the permission-restricted profile file. + +**Manual / scriptable alternative.** `enroll` remains available as the +low-level primitive `setup` is built on — it prints the raw JSON +response instead of writing a profile, for scripts that manage credentials +themselves: ```bash npx -y @pdpp/local-collector enroll \ @@ -90,8 +132,6 @@ npx -y @pdpp/local-collector enroll \ --device-label "the owner's laptop" ``` -The JSON response shape: - ```json { "device_id": "dev_...", @@ -102,11 +142,29 @@ The JSON response shape: } ``` -Persist the device id, device token, and `source_instance_id`. `connector_instance_id` is the server-side connection id for owner-facing diagnostics; the collector command still passes the device-binding selector as `PDPP_CONNECTION_ID`. The device token is sensitive (device-scoped ingest only, but still write-capable on this lane). Treat it like an API key — never commit it. +`connector_instance_id` is the server-side connection id for owner-facing +diagnostics; the collector still passes the device-binding selector as +`PDPP_CONNECTION_ID`. The device token is sensitive (device-scoped ingest +only, but still write-capable on this lane) — treat it like an API key +and never commit or log it. ## Step 4 — Run a connector pass -Paste the `@pdpp/local-collector run` command from the dashboard, filling the three env vars from the enrollment response: +If you used `setup`, the profile it wrote is picked up automatically by +`--connection-id` alone: + +```bash +npx -y @pdpp/local-collector run --connection-id si_... +``` + +Live progress prints to stderr as the connector finds records (phase, running +counts, and a final summary), so a large local archive no longer looks stuck +— pass `--quiet` to suppress it (useful under a systemd unit where +stderr already goes to the journal). Add `--sample ` here too, any time +you want a bounded proof pass instead of collecting everything. + +**Manual / scriptable alternative.** If you used `enroll` directly, supply the +three values as env vars the way the dashboard's rendered command shows: ```bash PDPP_LOCAL_DEVICE_ID=dev_... \ @@ -148,6 +206,67 @@ Open `/device-exporters`. The device row updates with: - **`status: blocked` with `state_get_failed`**: the runner refuses to advance without prior state to avoid over-collecting. Inspect the dashboard for the underlying error (typically a transient AS reach issue or a removed source instance) before retrying. - **`status: retrying` with `state_put_failed`**: benign; the next pass re-reads state and re-emits records the connector child considered consumed. Server-side idempotency absorbs the duplicates. +## Stopping a run and interrupt safety + +`Ctrl+C` (`SIGINT`, and `SIGTERM`) during a plain `run` or `run --sample`/ +`setup --sample` is safe: the collector installs a real signal handler for +the duration of the run and aborts the same `abortSignal` `--sample ` uses +internally to stop after N records. The abort forwards to the connector child +(`SIGTERM`, then `SIGKILL` after a grace period if it does not exit), flushes +any records it had already parsed before the interrupt to the durable local +outbox, and records an honest recovery gap. Nothing already flushed is lost, +and nothing beyond what was flushed is claimed as collected — the +interrupted pass never advances the server-side checkpoint, and the CLI exits +non-zero so a supervising script does not mistake an interrupt for success. +Re-run the same command (or `recover --apply`) to pick up where it left off; +the durable outbox is exactly what makes that safe. + +## Removing a host + +To fully log out a host — revoke its device credential on the reference +server, then delete its local profile: + +```bash +npx -y @pdpp/local-collector logout --connector claude_code +# or, for a profile saved under a custom name: +npx -y @pdpp/local-collector logout --profile +``` + +`logout` calls the reference server first, using the device's own credential +to revoke itself (a device can only revoke itself, never another device), +and only deletes the local profile `.env` file after the server confirms the +credential is gone — either freshly revoked, or already revoked from a +prior `logout` attempt (idempotent). If the server call fails ambiguously +(network error, timeout, unexpected response) `logout` fails closed: it +prints an error and leaves the local profile in place so you can retry once +the server is reachable, rather than silently deleting the only local record +of a token that may still be live. + +If the server is unreachable or has been decommissioned and you need to +clear local state anyway, pass `--local-only`: + +```bash +npx -y @pdpp/local-collector logout --connector claude_code --local-only +``` + +This skips the server call entirely and deletes the local profile +unconditionally — the device token then remains valid against the +reference deployment until revoked some other way (a server admin +revoking it directly, or a future `logout` once the server is reachable +again). It is deliberately not what plain `logout` does, since it does not +close the server-side lane. + +After logout, also delete the durable outbox file (`status` reports its exact +path under `db.path`) if you no longer need the locally queued history for +that connection. + +To remove `@pdpp/local-collector` itself from a host: + +```bash +npm rm -g @pdpp/local-collector +rm -rf ~/.config/pdpp/collectors # all saved profiles, not just one +``` + ## Coverage and excluded stores A successful Claude Code or Codex run **does not** mean every file in your diff --git a/docs/reference/local-collector.md b/docs/reference/local-collector.md index ee4f180b5..1ce2d1ff3 100644 --- a/docs/reference/local-collector.md +++ b/docs/reference/local-collector.md @@ -167,12 +167,43 @@ versions, and capabilities such as `network`, `filesystem`, and `local_device`. Browser-bound connectors are intentionally not shipped in this package until each has its own publishability review. -## Enroll +## Setup (guided) Start the reference deployment and open the dashboard's local exporter enrollment form. Create an enrollment code for the connector id and local -binding you want to run, then exchange that short-lived code on the host that -has the local data: +binding you want to run, then run `setup` on the host that has the local +data: + +```bash +# @pdpp/local-collector package, npx-launched pdpp-local-collector binary +npx -y @pdpp/local-collector setup \ + --base-url https:// \ + --code \ + --connector claude_code \ + --device-label "" \ + --sample 20 +``` + +`setup` exchanges the code, writes the device id/device token/connection id +to a local profile `.env` file under `~/.config/pdpp/collectors/` +(`$PDPP_LOCAL_COLLECTOR_PROFILE_DIR` or `${XDG_CONFIG_HOME:-$HOME/.config}/pdpp/collectors`; +file permissions `0600`, directory `0700` — POSIX mode bits, inert on +Windows, where the file's protection comes from living under your own user +profile directory instead), and prints a human-readable summary — never the +raw device token. `--sample 20` runs a bounded 20-record proof pass +immediately after enrolling, so you get real evidence the pairing works +before deciding to collect the full source; omit it to only enroll and save +credentials. Add `--json` for machine-readable output. Connector ids are +case-insensitive and hyphens normalize to underscores (`claude-code` == +`claude_code`). + +Once `setup` has written a profile, `run --connection-id ` resolves +device id/device token/connector from it automatically — see "Run" below. + +**Low-level alternative.** `enroll` is the scriptable primitive `setup` is +built on: it performs the same exchange but prints the raw JSON response +instead of writing a profile, for callers that manage credentials themselves +(unchanged, still fully supported): ```bash # @pdpp/local-collector package, npx-launched pdpp-local-collector binary @@ -192,8 +223,24 @@ lane. ## Run -Run the connector with the enrollment response values supplied through -environment variables: +If you used `setup`, the profile it wrote is picked up automatically: + +```bash +# @pdpp/local-collector package, npx-launched pdpp-local-collector binary +npx -y @pdpp/local-collector run --connection-id +``` + +Live progress (phase, running record counts, a final summary) prints to +stderr as the connector finds records — stdout stays a pure JSON result, so +piping/parsing `run`'s output is unaffected. Pass `--quiet` to suppress +progress lines (for example under a systemd unit, where stderr already lands +in the journal). Pass `--sample ` any time to run a bounded proof pass +instead of a full collection — it stops the connector after `n` records, +still durably queues what it collected, and never marks the pass as a +complete/coverage-checkpointed run. + +**Low-level alternative.** Supply the enrollment response values directly +through environment variables — unchanged, still fully supported: ```bash # @pdpp/local-collector package, npx-launched pdpp-local-collector binary @@ -221,6 +268,31 @@ npx -y @pdpp/local-collector run \ `PDPP_CONNECTION_ID`, but new docs and scripts should use `PDPP_CONNECTION_ID`. +## Connectors and logout + +List the connector ids this build accepts: + +```bash +npx -y @pdpp/local-collector connectors +``` + +Revoke this device's own credential on the reference server, then remove its +saved local profile. Deletion only happens after the server confirms the +credential is revoked (or was already revoked); a network/server failure +leaves local credentials in place so you can retry: + +```bash +npx -y @pdpp/local-collector logout --connector claude_code +``` + +If the server is unreachable or decommissioned, `--local-only` skips the +server call and deletes local credentials unconditionally — the device token +then stays valid server-side until revoked some other way: + +```bash +npx -y @pdpp/local-collector logout --connector claude_code --local-only +``` + ## Recover A Stalled Collector When the dashboard says a local collector needs attention, run the recovery diff --git a/openspec/changes/close-setup-lifecycle-authority-0806/design.md b/openspec/changes/close-setup-lifecycle-authority-0806/design.md new file mode 100644 index 000000000..9fc3ba840 --- /dev/null +++ b/openspec/changes/close-setup-lifecycle-authority-0806/design.md @@ -0,0 +1,57 @@ +## Context + +`run_history` is already the durable run-grain reader for owner summaries and +already carries the bounded `facts_json` payload written from terminal runtime +events. `collection_facts` is already the canonical per-stream evidence block: +`considered` is never inferred from `collected`, and a missing fact is not +complete. The setup route should use those primitives rather than maintaining a +second setup ledger. + +## Decision + +Use a shared pure classifier with three terminal dispositions: + +- `verified_empty`: every required/in-scope stream has trusted + `considered: 0`, no skip or pending detail gap, and a `committed` or `disabled` + checkpoint; +- `unverified_zero`: terminal success has an observed zero count but the + collection facts do not prove a valid empty result; +- `unverified_missing_counts`: terminal success has no observed yield count and + no valid-empty collection facts. + +The classifier receives the manifest stream set and the parsed facts. It never +derives proof from an aggregate count. Aggregate zero is only a discriminator +for the unverified case. Count presence is retained in the existing bounded +`facts_json` payload so a generalized run-history writer's schema default of +zero cannot turn a missing runtime field into evidence. + +The setup route first checks the active-run table by connection. For terminal +evidence it uses exact connection-scoped run-history lookup when `run_id` is +provided; without `run_id` it reads the latest product run-history row for that +connection. It does not use the global spine terminal lookup for this projection. + +The server summary adds the same disposition to each draft connection. The +console's existing shared source-actionability function owns the copy and CTA +for Dashboard, Sources, and Syncs. Drafts continue to route to the setup-status +page and remain excluded from active sync groups and scheduler enrollment. + +## Alternatives rejected + +- A setup-specific table or enum was rejected because it would create a second + lifecycle authority and drift from run-history evidence. +- Treating `records_emitted === 0` as empty was rejected because it cannot + distinguish a silent runtime from a connector that proved an empty account. +- A global `run_id` terminal lookup was rejected because run ids are legitimate + duplicates across connection instances. +- Activating or scheduling a draft after a terminal zero was rejected because + accepted records remain the existing activation boundary. + +## Acceptance checks + +- Setup status returns distinct state and disposition for verified-empty, + unverified-zero, and missing-count terminal success. +- A duplicate run id cannot cross connection boundaries, and a no-query revisit + resolves the durable latest result for the addressed connection. +- Dashboard, Sources, and Syncs share the same terminal disposition copy and + CTA while the draft remains inactive and unscheduled. +- Focused tests plus OpenSpec strict validation pass. diff --git a/openspec/changes/close-setup-lifecycle-authority-0806/proposal.md b/openspec/changes/close-setup-lifecycle-authority-0806/proposal.md new file mode 100644 index 000000000..126154b7c --- /dev/null +++ b/openspec/changes/close-setup-lifecycle-authority-0806/proposal.md @@ -0,0 +1,38 @@ +## Why + +The static-secret setup projection currently treats every terminal zero as the +same outcome, reads terminal evidence by an unscoped `run_id`, and leaves a +draft's terminal setup result indistinguishable from setup that has not run. +Those seams allow a sibling connection's run to leak into the status page and +make Dashboard, Sources, and Syncs disagree about a completed zero-yield setup. + +## What Changes + +- Make one pure setup terminal-disposition projection over connection-scoped + `run_history` and canonical `collection_facts`, distinguishing verified empty, + unverified zero, and missing counts. +- Preserve count-presence evidence in the existing bounded `facts_json` payload + so a missing runtime count is not rewritten as a proven zero. +- Resolve setup terminal evidence by `(connector_instance_id, run_id)` when a + run is requested and by the latest product `run_history` row for the owner + connection when no run id is supplied. +- Carry the resulting connection-scoped disposition into the existing summary + contract and shared owner actionability projection. Drafts remain visible for + setup review but never become active or scheduled from a zero-yield result. +- Add deterministic regression coverage for valid-empty, silent-zero, + missing-count, duplicate-run-id isolation, run-id-less revisit, and + cross-surface zero-yield behavior. + +## Capabilities + +Modified: + +- reference-connection-health +- reference-implementation-runtime + +## Out of Scope + +- No new lifecycle storage or parallel state model. +- No aggregate-count inference in place of canonical collection facts. +- No run admission, catalog, schedule activation, deployment, live mutation, + push, or PR. diff --git a/openspec/changes/close-setup-lifecycle-authority-0806/specs/reference-connection-health/spec.md b/openspec/changes/close-setup-lifecycle-authority-0806/specs/reference-connection-health/spec.md new file mode 100644 index 000000000..5ecc102ea --- /dev/null +++ b/openspec/changes/close-setup-lifecycle-authority-0806/specs/reference-connection-health/spec.md @@ -0,0 +1,35 @@ +## ADDED Requirements + +### Requirement: Setup terminal disposition SHALL be connection-scoped and evidence-backed + +The owner-facing setup projection SHALL reuse the connection's durable +`run_history` row and canonical `collection_facts` evidence. It SHALL expose +distinct terminal dispositions for a verified empty result, an unverified zero, +and missing yield counts. It SHALL NOT infer a verified empty result from an +aggregate count alone. + +#### Scenario: Valid empty is proven + +- **WHEN** terminal success has facts for every required/in-scope stream +- **AND** each fact has trusted `considered: 0`, no skip or unresolved detail + gap, and a `committed` or `disabled` checkpoint +- **THEN** setup status SHALL report `verified_empty` + +#### Scenario: Silent zero is not proven empty + +- **WHEN** terminal success reports an observed aggregate zero +- **AND** canonical collection facts are absent or incomplete +- **THEN** setup status SHALL report `unverified_zero`, not a verified empty + +#### Scenario: Missing counts stay missing + +- **WHEN** terminal success has no observed yield count and no valid-empty facts +- **THEN** setup status SHALL report `unverified_missing_counts` +- **AND** it SHALL remain terminal attention rather than pending forever + +#### Scenario: Owner surfaces share the draft disposition + +- **WHEN** a draft has a terminal setup disposition +- **THEN** Dashboard, Sources, and Syncs SHALL consume the same connection-scoped + disposition and owner action copy +- **AND** the draft SHALL remain inactive and unscheduled diff --git a/openspec/changes/close-setup-lifecycle-authority-0806/specs/reference-implementation-runtime/spec.md b/openspec/changes/close-setup-lifecycle-authority-0806/specs/reference-implementation-runtime/spec.md new file mode 100644 index 000000000..973393e98 --- /dev/null +++ b/openspec/changes/close-setup-lifecycle-authority-0806/specs/reference-implementation-runtime/spec.md @@ -0,0 +1,20 @@ +## ADDED Requirements + +### Requirement: Setup terminal evidence SHALL use composite run identity + +The reference implementation SHALL resolve setup terminal evidence by the pair +`(connector_instance_id, run_id)` when a run id is supplied, and SHALL resolve +the latest terminal product run from `run_history` scoped to the addressed +connection when no run id is supplied. A global run-id-only terminal read SHALL +not be used for setup status. + +#### Scenario: Duplicate run ids remain isolated + +- **WHEN** two connections have terminal run-history rows with the same `run_id` +- **THEN** each setup-status request SHALL return only its own connection's row + +#### Scenario: Revisiting without a run id is durable + +- **WHEN** an owner revisits setup status without `run_id` after terminal success +- **THEN** the connection's latest durable run-history row SHALL supply the + terminal setup disposition diff --git a/openspec/changes/close-setup-lifecycle-authority-0806/tasks.md b/openspec/changes/close-setup-lifecycle-authority-0806/tasks.md new file mode 100644 index 000000000..3c26676e9 --- /dev/null +++ b/openspec/changes/close-setup-lifecycle-authority-0806/tasks.md @@ -0,0 +1,20 @@ +## 1. Contract and authority + +- [x] 1.1 Add the shared terminal setup-disposition contract and count-presence + evidence to the existing run-history facts payload. +- [x] 1.2 Add connection-scoped exact/latest run-history readers and wire setup + status to them. +- [x] 1.3 Carry the shared disposition through the owner summary contract. + +## 2. Owner surfaces + +- [x] 2.1 Distinguish terminal setup states and status-page copy/actions. +- [x] 2.2 Make Dashboard, Sources, and Syncs consume the same disposition copy + while retaining draft discoverability and inactive scheduling behavior. + +## 3. Verification + +- [x] 3.1 Add deterministic valid-empty, silent-zero, and missing-count tests. +- [x] 3.2 Add duplicate-run-id, no-run-id revisit, and cross-surface zero-yield + tests. +- [x] 3.3 Run focused tests, lint/type checks, and strict OpenSpec validation. diff --git a/openspec/changes/fix-static-secret-retry-idempotency-0806/design.md b/openspec/changes/fix-static-secret-retry-idempotency-0806/design.md new file mode 100644 index 000000000..d4a4ea9d3 --- /dev/null +++ b/openspec/changes/fix-static-secret-retry-idempotency-0806/design.md @@ -0,0 +1,79 @@ +## Decision + +Use the existing `(owner, connector, source_kind, source_binding_key)` unique +constraint as the identity claim, rather than adding a second identity table or +partial index. Static-secret binding keys have explicit phases: + +- `static_secret_draft_identity_` is derived from the owner-supplied + non-secret identity field when one is declared; connectors without one keep + a random draft key so distinct accounts are never collapsed on insufficient + evidence. +- `static_secret_verified_identity_` is derived from the non-secret + identity returned by a successful synchronous probe. The binding also keeps + that provider identity as non-secret metadata for future status/retry + decisions. The connector-instance id does not change when its binding key is + re-keyed. + +The capture route accepts optional `setup_fields`. It validates them against +the manifest, rejects secret/unknown/required-field violations, updates only +the existing draft's non-secret binding, then probes. A rejected probe does +not mutate the draft status. On a successful probe the route claims the +verified binding key before credential capture. If the database unique binding +constraint reports a collision, the route resolves the winner under the same +owner and connector: a draft target may converge onto the winner, but an active +target refuses an identity conflict. This leaves at most one newly-managed +draft/active instance for a verified identity without merging distinct +accounts. + +For an active row already created before this change, the draft route performs +a bounded owner+connector identity comparison against its stored non-secret +setup fields for synchronous-identity connectors. A single matching active +connection is reused; more than one is an ambiguity and fails closed. A later +successful probe claims the verified key, so future requests use the same +authoritative binding. Existing duplicate active rows are not auto-merged. + +Credential replacement remains connection-id scoped and uses the same capture +route. A replacement may claim the same verified identity on that connection; +attempting to retarget an already verified active connection to a different +identity fails closed and directs the owner to create a distinct account. + +## Alternatives rejected + +- **Client-side submit disabling:** does not converge retries after a timeout, + reload, second tab, or direct API request. +- **A new idempotency table:** duplicates an identity axis already enforced by + the connector-instance binding constraint and would require a new lifecycle + and cleanup policy. +- **Active-only post-hoc duplicate cleanup:** allows both rows to promote and + leaves a window where both can collect; the identity claim happens before + credential persistence instead. +- **Hashing the submitted secret as an account key:** secrets are not provider + identity, can rotate, and would risk collapsing distinct accounts. The secret + is never used for binding or round-tripped. +- **Guessing identity for first-sync-only connectors:** without a verified + provider identity, collapsing submissions is unsafe. Those connectors retain + random draft keys and fail closed until a provider identity exists. + +## Compatibility and migration + +No database migration is required. The change reuses the existing binding key +and JSON binding columns. Existing rows without a verified-identity key remain +valid and are not rewritten at startup. On a future synchronous capture, a +single matching legacy active row can claim the new key; ambiguous legacy rows +fail closed and require explicit owner/operator cleanup outside this change. + +## Acceptance checks + +- A synchronous typo rejection leaves one draft, preserves its id through the + Console redirect, and a corrected retry updates the draft's non-secret setup + fields before a successful probe/capture. +- Repeated draft creation and concurrent/same-identity capture converge to one + connector instance and one active identity; repeated capture does not create + a second credential row or active connection. +- Distinct identity fields create distinct connection ids and both can promote + independently. +- Existing connection-id credential replacement keeps id, schedule, records, + and history; no secret appears in bindings, responses, audits, or query + parameters. +- SQLite focused tests, Console invariant/type checks, OpenSpec strict + validation, and the repository's relevant final checks pass. diff --git a/openspec/changes/fix-static-secret-retry-idempotency-0806/proposal.md b/openspec/changes/fix-static-secret-retry-idempotency-0806/proposal.md new file mode 100644 index 000000000..eab2645c4 --- /dev/null +++ b/openspec/changes/fix-static-secret-retry-idempotency-0806/proposal.md @@ -0,0 +1,40 @@ +## Why + +The owner-session static-secret flow can fork one provider account into several +connector instances. Draft creation currently chooses a fresh random binding +key for every request, so retries cannot hit the existing upsert identity. A +synchronous probe rejection then revokes that draft and the Console's error +redirect drops its connection id, forcing the next submit through draft +creation again. Gmail UAT showed the resulting duplicate active connections +for one verified mailbox. + +The fix is connector-generic and stays inside the existing owner-session +static-secret lifecycle. It makes draft retries addressable, derives a safe +pre-validation identity key when a manifest supplies one, and promotes a +successful provider identity into the same binding uniqueness axis before the +credential is stored. Different identities retain separate connections; +ambiguous or conflicting identity state fails closed. + +## What Changes + +- Keep a synchronously rejected static-secret draft in `draft` state and carry + its connection id through the Console retry redirect. +- Let an owner-session capture submit corrected non-secret setup fields for a + draft before probing. Secrets remain request-only and are never stored in the + binding or returned by any response. +- Use a deterministic draft binding key for a non-secret manifest identity + field, while retaining random keys when no safe identity is available. +- After a synchronous probe succeeds, atomically re-key the existing instance + to a deterministic verified-identity binding. The existing database unique + binding constraint is the server authority for concurrent convergence; a + draft collision reuses the winner, while an active replacement conflict is + rejected rather than silently retargeting an account. +- Preserve same-connection credential replacement and allow distinct provider + identities to remain distinct. + +## Impact + +Owner-session static-secret API and Console setup actions, connector-instance +store binding updates, and focused reference-implementation tests. No live +data, containers, public protocol surface, or credential-table schema is +changed. diff --git a/openspec/changes/fix-static-secret-retry-idempotency-0806/specs/reference-implementation-static-secret/spec.md b/openspec/changes/fix-static-secret-retry-idempotency-0806/specs/reference-implementation-static-secret/spec.md new file mode 100644 index 000000000..0108256f7 --- /dev/null +++ b/openspec/changes/fix-static-secret-retry-idempotency-0806/specs/reference-implementation-static-secret/spec.md @@ -0,0 +1,68 @@ +## ADDED Requirements + +### Requirement: Owner-session static-secret retries SHALL preserve a safe draft identity + +The owner-session static-secret lifecycle SHALL keep a synchronously rejected +draft in `draft` state and SHALL allow a subsequent submission to update only +manifest-declared non-secret setup fields on that same connection before +re-probing. The Console retry surface SHALL preserve the connection id and +shall never round-trip the secret. + +#### Scenario: A corrected mailbox retry reuses the rejected draft + +- **GIVEN** a Gmail draft whose submitted credential is rejected +- **WHEN** the owner corrects the non-secret mailbox field and resubmits +- **THEN** the original connection id SHALL remain the target +- **AND** the draft SHALL be probed with the corrected setup fields +- **AND** a successful capture SHALL store the credential on that same + connection id +- **AND** the rejected credential SHALL not be stored. + +### Requirement: Verified static-secret identities SHALL converge through the server binding authority + +After a synchronous probe returns a non-secret provider identity, the server +SHALL claim a deterministic owner+connector+verified-identity binding before +storing the credential. Duplicate submissions or retries that race for that +binding SHALL converge to one draft/active connection. A verified active +identity SHALL not silently fork into another active connection. + +#### Scenario: Duplicate submission for the same verified identity + +- **WHEN** the owner submits the same static-secret account more than once +- **THEN** successful captures SHALL resolve to one connector instance +- **AND** at most one connection for that owner, connector, and verified + provider identity SHALL be active +- **AND** repeated capture may rotate that one connection's credential but + SHALL not create another connector instance. + +#### Scenario: Distinct provider identities remain separate + +- **WHEN** the same owner submits two distinct provider identities for one + static-secret connector +- **THEN** the server SHALL retain two distinct connection ids +- **AND** each identity SHALL be able to capture and promote independently. + +#### Scenario: Ambiguous identity state fails closed + +- **WHEN** an identity claim collides with an already verified active connection + while the requested target is a different active connection +- **THEN** the server SHALL reject the mutation with a typed conflict +- **AND** SHALL not store the submitted credential or silently retarget the + active connection. + +### Requirement: Static-secret credential replacement SHALL remain connection scoped + +Replacing a credential on an existing connection SHALL preserve its connection +id and existing records, schedule, and history. Secrets SHALL remain sealed in +the credential store and SHALL never be returned in setup fields, API responses, +audits, or retry URLs. + +#### Scenario: Replacing a credential does not fork the connection + +- **GIVEN** an existing active static-secret connection with collected records + and a schedule +- **WHEN** the owner replaces its credential with a valid credential for the + same verified provider identity +- **THEN** the capture response SHALL use the original connection id +- **AND** the records, schedule, and history SHALL remain attached to that id +- **AND** no second connector instance SHALL be created. diff --git a/openspec/changes/fix-static-secret-retry-idempotency-0806/tasks.md b/openspec/changes/fix-static-secret-retry-idempotency-0806/tasks.md new file mode 100644 index 000000000..5fcc42519 --- /dev/null +++ b/openspec/changes/fix-static-secret-retry-idempotency-0806/tasks.md @@ -0,0 +1,11 @@ +## Implementation + +- [x] Add the static-secret identity key/rekey and setup-field update contract. +- [x] Preserve drafts on validation rejection and carry retry state through the + Console action/page/API lifecycle. +- [x] Add focused route/store/Console tests for typo retry, duplicate submit, + same verified identity, distinct identities, replacement, and secret + non-round-tripping. +- [x] Run focused tests, type/lint checks, OpenSpec strict validation, and the + final diff/old-pattern audit. +- [x] Commit with DCO and `Assisted-by: AI`; write the requested report. diff --git a/packages/cli/scripts/package-contract.ts b/packages/cli/scripts/package-contract.ts index c6c937f8b..205222544 100644 --- a/packages/cli/scripts/package-contract.ts +++ b/packages/cli/scripts/package-contract.ts @@ -6,13 +6,46 @@ import { existsSync, readFileSync, statSync } from "node:fs"; import { relative, resolve, sep } from "node:path"; const TEST_ARTIFACT = /(^|\/)\.?.+\.test\.(?:js|mjs|cjs|ts|mts|cts)$/; -const NPM_PACK_JSON = /(\[\s*\{[\s\S]*\])\s*$/; +const WHITESPACE = /\s/; +const NPM_PACK_OUTPUT_MAX_BYTES = 8 * 1024 * 1024; interface ExportTarget { label: string; target: string; } +interface NpmPackResult { + filename: string; + files: Array<{ path: string }>; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function normalizeNpmPackPayload(payload: unknown): NpmPackResult[] { + let entries: unknown[]; + if (Array.isArray(payload)) { + entries = payload; + } else if (isRecord(payload)) { + entries = typeof payload.filename === "string" ? [payload] : Object.values(payload); + } else { + entries = []; + } + + assert.ok(entries.length > 0, "npm pack did not produce a non-empty JSON payload"); + return entries.map((entry, index) => { + assert.ok(isRecord(entry), `npm pack result ${index} is not an object`); + assert.equal(typeof entry.filename, "string", `npm pack result ${index} has no filename`); + assert.ok(Array.isArray(entry.files), `npm pack result ${index} has no files list`); + for (const [fileIndex, file] of entry.files.entries()) { + assert.ok(isRecord(file), `npm pack result ${index} file ${fileIndex} is not an object`); + assert.equal(typeof file.path, "string", `npm pack result ${index} file ${fileIndex} has no path`); + } + return entry as unknown as NpmPackResult; + }); +} + function assertInsideDist(packageRoot: string, target: string, label: string): { target: string; targetPath: string } { assert.equal(typeof target, "string", `${label} must be a string target`); assert.equal(target.startsWith("./dist/"), true, `${label} must point into ./dist/: ${target}`); @@ -113,8 +146,40 @@ export function assertPackedFiles(manifest: PackageManifest, packedFiles: string } } -export function parseNpmPackOutput(output: string): Array<{ filename: string }> { - const match = output.match(NPM_PACK_JSON); - assert.ok(match, "npm pack did not produce a trailing JSON payload"); - return JSON.parse(match[1]) as Array<{ filename: string }>; +export function parseNpmPackOutput(output: string): NpmPackResult[] { + assert.ok( + Buffer.byteLength(output, "utf8") <= NPM_PACK_OUTPUT_MAX_BYTES, + `npm pack output exceeds the ${NPM_PACK_OUTPUT_MAX_BYTES}-byte limit` + ); + + const trimmed = output.trimEnd(); + let payload: unknown; + try { + payload = JSON.parse(trimmed); + } catch { + let searchEnd = trimmed.length; + while (searchEnd > 0 && payload === undefined) { + const newline = trimmed.lastIndexOf("\n", searchEnd - 1); + const lineStart = newline + 1; + const lineEnd = searchEnd; + let candidateStart = lineStart; + while (candidateStart < lineEnd && WHITESPACE.test(trimmed[candidateStart] ?? "")) { + candidateStart += 1; + } + + if (trimmed[candidateStart] === "[" || trimmed[candidateStart] === "{") { + try { + payload = JSON.parse(trimmed.slice(candidateStart)); + } catch { + // A nested array/object line is not the root payload; keep looking + // toward the beginning of the bounded output. + } + } + + searchEnd = newline >= 0 ? newline : 0; + } + } + + assert.ok(payload !== undefined, "npm pack did not produce a trailing JSON payload"); + return normalizeNpmPackPayload(payload); } diff --git a/packages/cli/src/collector/commands.ts b/packages/cli/src/collector/commands.ts index a1052acdc..b8b685bb7 100644 --- a/packages/cli/src/collector/commands.ts +++ b/packages/cli/src/collector/commands.ts @@ -23,39 +23,48 @@ Distribution: See openspec/changes/publish-pdpp-local-collector/design.md. Usage: + ${PDPP_CLI_BIN_NAME} collector setup --base-url --code + --connector [--sample ] + ${PDPP_CLI_BIN_NAME} collector run --connection-id [--connector ] + [--sample ] [--quiet] + ${PDPP_CLI_BIN_NAME} collector connectors + ${PDPP_CLI_BIN_NAME} collector logout --connector ${PDPP_CLI_BIN_NAME} collector advertise ${PDPP_CLI_BIN_NAME} collector enroll --base-url --code [--device-label
    ); diff --git a/packages/operator-ui/src/explore/explore-data-assembler.ts b/packages/operator-ui/src/explore/explore-data-assembler.ts index ffeadc3c8..0e2952463 100644 --- a/packages/operator-ui/src/explore/explore-data-assembler.ts +++ b/packages/operator-ui/src/explore/explore-data-assembler.ts @@ -6,9 +6,11 @@ import { classifyRecordKind, type DeclaredFieldRoles, type DeclaredFieldTypes, + deriveSourceDisplayNameFallback, EMPTY_DECLARED_FIELD_ROLES, type FieldRole, formatConnectorNameForDisplay, + isFallbackConnectionLabel, parseFieldRole, } from "@pdpp/display"; import { validateListEnvelope } from "@pdpp/list-envelope"; @@ -197,11 +199,16 @@ function toConnectionFacet(summary: RefConnectorIdentitySummary): ExplorerConnec } function connectorSummaryDisplayName(summary: RefConnectorIdentitySummary): string { - return formatConnectorNameForDisplay({ + const displayInput = { connectorId: summary.connector_id, displayName: summary.display_name, name: summary.connector_display_name, - }); + }; + const display = formatConnectorNameForDisplay(displayInput); + if (isFallbackConnectionLabel(displayInput)) { + return deriveSourceDisplayNameFallback(displayInput); + } + return display; } function summaryByConnectionId( @@ -1612,7 +1619,8 @@ async function loadTimeRangeFeed( */ function detectSingleStreamDoor( filtered: Array<{ connector_id: string; stream: string }>, - filteredSummaries: RefConnectorIdentitySummary[] + filteredSummaries: RefConnectorIdentitySummary[], + streamDisplayLabels: ReadonlyMap ): ExplorerStreamDoor | null { if (filtered.length === 0) { return null; @@ -1633,14 +1641,24 @@ function detectSingleStreamDoor( return null; } const [summary] = matchingSummaries; + const connectorKey = manifestConnectorKey({ connector_id: sharedConnector } as { connector_id: string }); + const streamLabel = resolveStreamDisplayLabel(streamDisplayLabels, connectorKey, sharedStream); return { connectionId: summary.connection_id, connectorId: sharedConnector, - displayName: `${connectorSummaryDisplayName(summary)} - ${sharedStream}`, + displayName: `${connectorSummaryDisplayName(summary)} - ${streamLabel}`, stream: sharedStream, }; } +export function resolveStreamDisplayLabel( + streamDisplayLabels: ReadonlyMap, + connectorId: string, + streamName: string +): string { + return streamDisplayLabels.get(searchTimestampMetadataKey(connectorId, streamName)) ?? streamName; +} + /** * Most-recent mode for a single connection+stream using LEXICAL search in recency * order (F2 fix). This replaces the former queryRecords-without-query path which @@ -1859,6 +1877,7 @@ async function loadSearchFeed( declaredFieldTypes: ReadonlyMap, declaredFieldRoles: ReadonlyMap, selectedConnectionIds: ReadonlySet, + streamDisplayLabels: ReadonlyMap, // EXCLUDE scope ("is not" / `-con:`/`-stream:`): drop excluded hits BEFORE counts/ // descriptors are built so "everything except X" is honest in search too. exclude: { instanceIds: ReadonlySet; streams: ReadonlySet }, @@ -1946,7 +1965,7 @@ async function loadSearchFeed( // Detect single-entity case (all hits share same connection+stream) for the // per-source browse door and for Most-recent single-stream pagination. - const streamDoor = detectSingleStreamDoor(filtered, filteredSummaries); + const streamDoor = detectSingleStreamDoor(filtered, filteredSummaries, streamDisplayLabels); // ── Most-recent mode: chronological, exhaustively pageable ───────────────── // @@ -2067,7 +2086,7 @@ async function loadSearchFeed( stream: hit.stream, }; }); - const fallbackDoor = detectSingleStreamDoor(fallbackFiltered, filteredSummaries); + const fallbackDoor = detectSingleStreamDoor(fallbackFiltered, filteredSummaries, streamDisplayLabels); const fallbackNextCursor = fallbackPage.next_cursor ?? null; return { // keyword_pageable ordered by time: multi-stream Most-recent path uses @@ -2183,6 +2202,8 @@ async function dispatchFeed(args: { declaredFieldTypes: ReadonlyMap; /** Declared presentation ROLES per connector::stream (parallel to declaredFieldTypes). */ declaredFieldRoles: ReadonlyMap; + /** Human stream labels keyed by connector::stream. */ + streamDisplayLabels: ReadonlyMap; filterConnectionSet: ReadonlySet; /** EXCLUDED connection ids (facet "is not" / `-con:`). Applied on the recent lens. */ excludeConnectionSet: ReadonlySet; @@ -2213,6 +2234,7 @@ async function dispatchFeed(args: { manifestFieldNames, declaredFieldTypes, declaredFieldRoles, + streamDisplayLabels, filterConnectionSet, excludeConnectionSet, excludeStreamSet, @@ -2242,6 +2264,7 @@ async function dispatchFeed(args: { declaredFieldTypes, declaredFieldRoles, filterConnectionSet, + streamDisplayLabels, exclude, dataSource ); @@ -2314,12 +2337,15 @@ interface ManifestMetadata { * object/array), mirroring how the records list page filters declared fields. */ serverFilterableFields: Map>; + /** Stream display labels from manifest display.label, keyed by connector::stream. */ + streamDisplayLabels: Map; timestampMetadata: Map; } interface ManifestStream { consent_time_field?: unknown; cursor_field?: unknown; + display?: { label?: string }; fields?: unknown; name: string; schema?: { properties?: Record; fields?: unknown }; @@ -2480,10 +2506,14 @@ function indexManifestStream( declaredFieldTypes: Map; manifestFieldNames: Map; serverFilterableFields: Map>; + streamDisplayLabels: Map; timestampMetadata: Map; } ): void { const key = searchTimestampMetadataKey(connectorId, stream.name); + if (stream.display?.label) { + maps.streamDisplayLabels.set(key, stream.display.label); + } maps.timestampMetadata.set(key, { consent_time_field: typeof stream.consent_time_field === "string" ? stream.consent_time_field : null, cursor_field: typeof stream.cursor_field === "string" ? stream.cursor_field : null, @@ -2539,6 +2569,7 @@ async function buildManifestMetadata(dataSource: DashboardDataSource): Promise(), manifestFieldNames: new Map(), serverFilterableFields: new Map>(), + streamDisplayLabels: new Map(), timestampMetadata: new Map(), }; for (const manifest of await dataSource.listConnectorManifests()) { @@ -3077,6 +3108,7 @@ export async function assembleExplorerData( since, snapshotAnchorParam: rawAnchor, summaries, + streamDisplayLabels: manifestMetadata.streamDisplayLabels, timestampMetadata, until, upcomingTrail, diff --git a/packages/operator-ui/src/explore/explore-sort-direction.test.ts b/packages/operator-ui/src/explore/explore-sort-direction.test.ts index f52cf5639..7f4248c82 100644 --- a/packages/operator-ui/src/explore/explore-sort-direction.test.ts +++ b/packages/operator-ui/src/explore/explore-sort-direction.test.ts @@ -40,6 +40,16 @@ function ynabSummary(): RefConnectorSummary { } as RefConnectorSummary; } +function excludedSummary(): RefConnectorSummary { + return { + ...ynabSummary(), + connection_id: "cin_excluded", + connector_instance_id: "cin_excluded", + display_name: "Private", + streams: ["private"], + } as RefConnectorSummary; +} + function ynabManifest(): ConnectorManifest { return { connector_id: "ynab", @@ -63,7 +73,14 @@ const notStubbed = () => Promise.reject(new Error("not stubbed")); /** A fake source that records the `direction` opt of each listExploreTimeline call. */ function makeDirectionCapturingSource( captured: Array<"asc" | "desc" | undefined>, - supportsTimelineDirection = true + supportsTimelineDirection = true, + capturedRequests: Array<{ + connectionIds: readonly string[]; + direction: "asc" | "desc" | undefined; + excludeConnectionIds: readonly string[]; + excludeStreams: readonly string[]; + streams: readonly string[]; + }> = [] ): DashboardDataSource { return { aggregateRecordsByTime: notStubbed, @@ -80,10 +97,17 @@ function makeDirectionCapturingSource( isSemanticRetrievalAdvertised: () => Promise.resolve(false), kind: "live", listConnectorManifests: () => Promise.resolve([ynabManifest()]), - listConnectorSummaries: mockListConnectorSummaries([ynabSummary()]), + listConnectorSummaries: mockListConnectorSummaries([ynabSummary(), excludedSummary()]), listExploreRecordBuckets: notStubbed, listExploreTimeline: (opts): Promise => { captured.push(opts?.direction); + capturedRequests.push({ + connectionIds: [...(opts?.connectionIds ?? [])], + direction: opts?.direction, + excludeConnectionIds: [...(opts?.excludeConnectionIds ?? [])], + excludeStreams: [...(opts?.excludeStreams ?? [])], + streams: [...(opts?.streams ?? [])], + }); return Promise.resolve(emptyTimelinePage()); }, listGrants: () => Promise.resolve({ data: [], has_more: false, object: "list" as const }), @@ -140,6 +164,35 @@ test("oldest re-page also threads through a multi-page Load-more trail (every pa } }); +test("oldest pagination preserves the Explore query scope on every direction-bound request", async () => { + const captured: Array<"asc" | "desc" | undefined> = []; + const capturedRequests: Parameters[2] = []; + const ds = makeDirectionCapturingSource(captured, true, capturedRequests); + + await assembleExplorerData( + { + anchor: SNAPSHOT_AT, + connection: "cin_ynab", + cursors: "c1,c2", + order: "oldest", + stream: "transactions", + xconnection: "cin_excluded", + xstream: "private", + }, + ds, + "https://rs.test" + ); + + assert.ok(capturedRequests.length >= 3, "the query must make page-1 and cursor-trail requests"); + for (const request of capturedRequests) { + assert.equal(request.direction, "asc", "the query's oldest direction must survive pagination"); + assert.deepEqual(request.connectionIds, ["cin_ynab"], "the selected connection must survive pagination"); + assert.deepEqual(request.streams, ["transactions"], "the selected stream must survive pagination"); + assert.deepEqual(request.excludeConnectionIds, ["cin_excluded"], "the excluded connection must survive pagination"); + assert.deepEqual(request.excludeStreams, ["private"], "the excluded stream must survive pagination"); + } +}); + test("oldest no-ops to newest-first when the server direction substrate is not advertised", async () => { const captured: Array<"asc" | "desc" | undefined> = []; const ds = makeDirectionCapturingSource(captured, false); diff --git a/packages/operator-ui/src/explore/explore-stream-display-labels.test.ts b/packages/operator-ui/src/explore/explore-stream-display-labels.test.ts new file mode 100644 index 000000000..026571f1a --- /dev/null +++ b/packages/operator-ui/src/explore/explore-stream-display-labels.test.ts @@ -0,0 +1,32 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { resolveStreamDisplayLabel } from "./explore-data-assembler.ts"; + +test("Stream display labels: timeline_points displays as human label, not protocol ID", () => { + const streamDisplayLabels = new Map([ + ["google-maps::timeline_points", "Your Google Maps location points"], + ["google-maps::timeline_segments", "Your Google Maps visits and activities"], + ]); + + const label = resolveStreamDisplayLabel(streamDisplayLabels, "google-maps", "timeline_points"); + assert.equal(label, "Your Google Maps location points"); + assert.notEqual(label, "timeline_points"); +}); + +test("Stream display labels: missing label falls back to stream name in filter", () => { + const streamDisplayLabels = new Map(); + const label = resolveStreamDisplayLabel(streamDisplayLabels, "gmail", "messages"); + assert.equal(label, "messages"); +}); + +test("Stream display labels: preserves protocol stream names when no label exists", () => { + const streamDisplayLabels = new Map([ + ["google-maps::timeline_points", "Your Google Maps location points"], + ]); + + assert.equal(resolveStreamDisplayLabel(streamDisplayLabels, "google-maps", "unknown_stream"), "unknown_stream"); +}); diff --git a/packages/pdpp-brand-react/src/components.css b/packages/pdpp-brand-react/src/components.css index 614d04d71..bf430b61f 100644 --- a/packages/pdpp-brand-react/src/components.css +++ b/packages/pdpp-brand-react/src/components.css @@ -4662,6 +4662,15 @@ a.rr-x-row { width: auto; max-width: none; } + /* Mobile: the Options popover (operator-syntax legend) also becomes a full-width + sheet, same as DateChip. Without this override, right:0 on a left-positioned + trigger near the wrap boundary grows leftward past x=0, overflowing the viewport. */ + .rr-x-options__body { + right: 0; + left: 0; + width: auto; + max-width: none; + } .rr-x-datechip__preset { /* Comfortable touch targets in the sheet. */ min-height: 40px; diff --git a/packages/polyfill-connectors/connectors/chase/download-button-click.test.ts b/packages/polyfill-connectors/connectors/chase/download-button-click.test.ts new file mode 100644 index 000000000..21bb719bc --- /dev/null +++ b/packages/polyfill-connectors/connectors/chase/download-button-click.test.ts @@ -0,0 +1,123 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Regression coverage for the QFX Download button click path. + * + * Live evidence (run_1786072649511, UAT container pdpp-pr81-uat, connection + * cin_c2f766b7166a6184adf021aa, 2026-08-07T03:18:42.473Z): the connector's + * own error message proves the CSS host locator RESOLVED — + * "waiting for locator('mds-button#download') - locator resolved" + * — yet the subsequent `.click({ timeout: 10000 })` still timed out. + * `mds-button` is a custom element (see + * __fixtures__/current-activity-download-form-no-rows.html:29 — + * `` with no light-DOM + * children): its accessible name and interactive target live in shadow DOM. + * A resolved host-element locator only proves the host is attached/visible, + * not that Playwright's actionability checks (stable, receives pointer + * events, enabled) can be satisfied against it — exactly the same + * locator-vs-actionability gap already fixed for the Activity and File Type + * controls (see clickActivityControl/clickFileTypeControl above), which use + * a CSS-id-first, semantic-role-fallback strategy because Chase's MDS + * elements are known to be unreliable for pure CSS-locator interaction. + * + * clickDownloadButton() applies that same two-tier strategy to the Download + * button. These tests prove: + * - the CSS locator is tried first (cheap path stays cheap), + * - a CSS click failure (the observed failure mode) falls back to the + * semantic role locator and succeeds when the shadow-DOM button is + * reachable that way, + * - both locators failing surfaces a diagnostic error naming both + * failure reasons, not just the first one. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import type { Locator, Page } from "playwright"; +import { clickDownloadButton } from "./index.ts"; + +interface ClickCall { + kind: "css" | "role"; +} + +interface FakeLocatorOptions { + clickError?: Error | undefined; +} + +function makeClickOnlyLocator(calls: ClickCall[], kind: ClickCall["kind"], opts: FakeLocatorOptions): Locator { + const base: Pick = { + click(_options?: Parameters[0]): ReturnType { + calls.push({ kind }); + if (opts.clickError) { + return Promise.reject(opts.clickError); + } + return Promise.resolve(); + }, + first(): Locator { + return base as Locator; + }, + }; + return base as Locator; +} + +function makePage(opts: { cssClickError?: Error; roleClickError?: Error }): { calls: ClickCall[]; page: Page } { + const calls: ClickCall[] = []; + const fake: Pick = { + getByRole(role: Parameters[0], options?: Parameters[1]): Locator { + assert.equal(role, "button"); + assert.match(String((options as { name?: RegExp })?.name), /download/i); + return makeClickOnlyLocator(calls, "role", { clickError: opts.roleClickError }); + }, + locator(selector: Parameters[0]): Locator { + assert.equal(selector, "mds-button#download"); + return makeClickOnlyLocator(calls, "css", { clickError: opts.cssClickError }); + }, + }; + return { calls, page: fake as Page }; +} + +test("clickDownloadButton clicks the CSS host locator directly when it succeeds", async () => { + const { calls, page } = makePage({}); + + await clickDownloadButton(page); + + assert.deepEqual( + calls.map((call) => call.kind), + ["css"], + "expected only the CSS host click, no semantic-role fallback needed" + ); +}); + +test("clickDownloadButton falls back to the semantic role locator when the CSS host click times out", async () => { + // Reproduces the exact observed failure: the host-element locator + // resolves (attached + visible) but .click()'s actionability wait times + // out against the shadow-DOM button underneath it. + const cssClickError = new Error( + "locator.click: Timeout 10000ms exceeded.\nCall log:\n - waiting for locator('mds-button#download')\n - locator resolved" + ); + const { calls, page } = makePage({ cssClickError }); + + await clickDownloadButton(page); + + assert.deepEqual( + calls.map((call) => call.kind), + ["css", "role"], + "expected the CSS click to be attempted, fail, then the role-based click to succeed" + ); +}); + +test("clickDownloadButton surfaces both failure reasons when the CSS and role click both fail", async () => { + const cssClickError = new Error("locator.click: Timeout 10000ms exceeded (css)"); + const roleClickError = new Error("locator.click: Timeout 10000ms exceeded (role)"); + const { page } = makePage({ cssClickError, roleClickError }); + + await assert.rejects(clickDownloadButton(page), (err: Error) => { + assert.match(err.message, /download_button_unavailable/); + assert.match(err.message, /selector=mds-button#download/); + assert.match(err.message, /\(css\)/); + assert.match(err.message, /role=/); + assert.match(err.message, /\(role\)/); + assert.equal(err.cause, roleClickError); + return true; + }); +}); diff --git a/packages/polyfill-connectors/connectors/chase/index.ts b/packages/polyfill-connectors/connectors/chase/index.ts index f94c66174..5715020ca 100644 --- a/packages/polyfill-connectors/connectors/chase/index.ts +++ b/packages/polyfill-connectors/connectors/chase/index.ts @@ -130,6 +130,7 @@ const CHASE_DOWNLOAD_ROUTE_RE = /downloadAccountTransactions|confirmDownloadAcco const NO_ACTIVITY_CONFIRMATION_RE = /we couldn't find any activity that matched the date range you chose/iu; const CHASE_QFX_FILE_TYPE_COMBOBOX_NAME_RE = /file type/i; const CHASE_QFX_ACTIVITY_COMBOBOX_NAME_RE = /activity/i; +const CHASE_QFX_DOWNLOAD_BUTTON_NAME_RE = /download/i; const DASHBOARD_ACCOUNT_SELECTOR = '[id^="accounts-name-link-button-"][id$="-label"], button[id^="accounts-name-link-button-"], button[data-testid^="accounts-name-link-button-"]'; export const CHASE_CURRENT_ACTIVITY_ROW_SELECTOR = @@ -443,6 +444,39 @@ async function clickFileTypeControl(page: Page): Promise { } } +/** + * Click the QFX Download button. `mds-button` is a custom element — its + * label and interactive target live in shadow DOM, so a host-element CSS + * click (`mds-button#download`) is only as reliable as the host's own + * click-forwarding. Mirror the two-tier strategy already used for the + * Activity/File Type controls: CSS id first, then the semantic role + * locator (`getByRole` pierces shadow DOM), so a host element that resolves + * but does not forward the click to its shadow-DOM button still finds the + * button via its accessible role. + */ +export async function clickDownloadButton(page: Page): Promise { + try { + await page.locator("mds-button#download").click({ timeout: CLICK_TIMEOUT_MS }); + } catch (selectorErr) { + try { + await page + .getByRole("button", { + name: CHASE_QFX_DOWNLOAD_BUTTON_NAME_RE, + }) + .first() + .click({ timeout: CLICK_TIMEOUT_MS }); + } catch (semanticErr) { + throw new Error( + `download_button_unavailable: selector=mds-button#download: ${truncate( + errMessage(selectorErr), + ERROR_MESSAGE_SLICE + )}; role=${truncate(errMessage(semanticErr), ERROR_MESSAGE_SLICE)}`, + { cause: semanticErr } + ); + } + } +} + function isLikelyQfxResponseBody(body: Buffer, headers: Record): boolean { if (body.length === 0) { return false; @@ -740,7 +774,7 @@ async function downloadQfx( const qfxResponseQueue = attachQfxResponseQueue(page); await qfxResponseQueue.ready; try { - await page.locator("mds-button#download").click({ timeout: CLICK_TIMEOUT_MS }); + await clickDownloadButton(page); } catch (err) { await capturePageCheckpoint(capture, page, `download-qfx-${account.internal_id}-${activity}-download-click-failed`); downloadQueue.detach(); diff --git a/packages/polyfill-connectors/connectors/slack/README.md b/packages/polyfill-connectors/connectors/slack/README.md index 4d50f2c29..ed2acb98d 100644 --- a/packages/polyfill-connectors/connectors/slack/README.md +++ b/packages/polyfill-connectors/connectors/slack/README.md @@ -157,10 +157,10 @@ See `openspec/changes/complete-slack-bundled-connector-coverage` for the evidenc ## Auth -Requires `SLACK_TOKEN`, `SLACK_COOKIE`, `SLACK_WORKSPACE` in env. Capture `SLACK_TOKEN` (an `xoxc-` token) and `SLACK_COOKIE` (the `d=...` cookie value) from a logged-in browser session against your workspace. +Requires `SLACK_TOKEN`, `SLACK_COOKIE`, `SLACK_WORKSPACE` in env. Capture `SLACK_TOKEN` (the browser web-client `xoxc-` token) and `SLACK_COOKIE` (the value of the cookie named `d`, usually `xoxd-...`; paste the value without `d=` and preserve URL escapes) from a logged-in browser session against your workspace. See the official [Slackdump manual authentication guide](https://github.com/rusq/slackdump/blob/5ecece6b7fa63f6e1a71e049900b9ccc61f6b1e7/doc/login-manual.md). Slackdump resolution: - Host runs: put `slackdump` on `PATH` or set `SLACKDUMP_BIN` to the binary path. -- Docker runs: the stock PDPP reference image does not bundle AGPL-3.0 `slackdump`. Build a derived image that installs it, or mount the binary into the container and set `SLACKDUMP_BIN` to that in-container path. +- Docker runs: `core`, `core-browser`, `railway-core`, and `platform-core` images ship `slackdump` v4.4.2 (AGPL-3.0) bundled by default at `/usr/local/bin/slackdump`. To override, set `SLACKDUMP_BIN` to an alternative executable path or mount it as described in the reference-implementation README. - Missing binary failures are reported before credentials are printed; do not paste Slack tokens into logs. diff --git a/packages/polyfill-connectors/connectors/slack/archive-reclaim.test.ts b/packages/polyfill-connectors/connectors/slack/archive-reclaim.test.ts index 0774352db..454ce0b28 100644 --- a/packages/polyfill-connectors/connectors/slack/archive-reclaim.test.ts +++ b/packages/polyfill-connectors/connectors/slack/archive-reclaim.test.ts @@ -144,8 +144,8 @@ test("base archive resume is throttled on the 90-minute follow-up without invoki entrypoint: SLACK_ENTRYPOINT, env: { HOME: homeDir, - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, SLACKDUMP_BIN: fakeSlackdump.path, TEST_SLACKDUMP_CALL_LOG: fakeSlackdump.callLog, @@ -184,8 +184,8 @@ test("a failed base archive resume remains owed and retries successfully on the entrypoint: SLACK_ENTRYPOINT, env: { HOME: homeDir, - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, SLACKDUMP_BIN: fakeSlackdump.path, TEST_SLACKDUMP_CALL_LOG: fakeSlackdump.callLog, @@ -228,8 +228,8 @@ test("base archive resume runs again after the seven-day lookback expires", asyn entrypoint: SLACK_ENTRYPOINT, env: { HOME: homeDir, - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, SLACKDUMP_BIN: fakeSlackdump.path, TEST_SLACKDUMP_CALL_LOG: fakeSlackdump.callLog, @@ -284,8 +284,8 @@ test("upgrade compatibility: a pre-upgrade successful base archive is throttled entrypoint: SLACK_ENTRYPOINT, env: { HOME: homeDir, - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, SLACKDUMP_BIN: fakeSlackdump.path, TEST_SLACKDUMP_CALL_LOG: fakeSlackdump.callLog, @@ -313,8 +313,8 @@ test("upgrade compatibility: a pre-upgrade successful base archive is throttled entrypoint: SLACK_ENTRYPOINT, env: { HOME: homeDir, - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, SLACKDUMP_BIN: fakeSlackdump.path, TEST_SLACKDUMP_CALL_LOG: fakeSlackdump.callLog, @@ -358,8 +358,8 @@ test("upgrade compatibility does NOT seed the throttle from archive existence al entrypoint: SLACK_ENTRYPOINT, env: { HOME: homeDir, - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, SLACKDUMP_BIN: fakeSlackdump.path, TEST_SLACKDUMP_CALL_LOG: fakeSlackdump.callLog, @@ -444,8 +444,8 @@ test("connector emits phase-timing and archive-size PROGRESS every run", async ( env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: "timing-ws", }, start: { type: "START", scope: { streams: [{ name: "messages" }] } }, @@ -482,8 +482,8 @@ test("__uploads reclaim is OFF by default: a normal run leaves __uploads intact" env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: "off-ws", }, start: { type: "START", scope: { streams: [{ name: "messages" }] } }, @@ -512,9 +512,9 @@ test("SLACK_RECLAIM_UPLOADS=1 does NOT reclaim when the run fails (gate honored) env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", + SLACK_COOKIE: "xoxd-fake", SLACK_RECLAIM_UPLOADS: "1", - SLACK_TOKEN: "xoxc-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: "fail-ws", }, start: { type: "START", scope: { streams: [{ name: "messages" }] } }, @@ -537,9 +537,9 @@ test("SLACK_RECLAIM_UPLOADS=1 removes __uploads after a successful run, sqlite i env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", + SLACK_COOKIE: "xoxd-fake", SLACK_RECLAIM_UPLOADS: "1", - SLACK_TOKEN: "xoxc-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: "on-ws", }, start: { type: "START", scope: { streams: [{ name: "messages" }] } }, @@ -617,9 +617,9 @@ test("SLACK_RECLAIM_UPLOADS=1 reclaims __uploads/ in every archive the run actua env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", + SLACK_COOKIE: "xoxd-fake", SLACK_RECLAIM_UPLOADS: "1", - SLACK_TOKEN: "xoxc-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { @@ -708,9 +708,9 @@ test("SLACK_RECLAIM_UPLOADS=1 reclaims a repair archive that was successfully cr env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", + SLACK_COOKIE: "xoxd-fake", SLACK_RECLAIM_UPLOADS: "1", - SLACK_TOKEN: "xoxc-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { @@ -784,9 +784,9 @@ test("SLACK_RECLAIM_UPLOADS=1 does NOT reclaim a repair archive when the repair env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", + SLACK_COOKIE: "xoxd-fake", SLACK_RECLAIM_UPLOADS: "1", - SLACK_TOKEN: "xoxc-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { @@ -863,8 +863,8 @@ test("scoped-archive-reconcile phase timing is reported when source-cache healin env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { @@ -940,8 +940,8 @@ test("scoped-archive-reconcile declares 0 selected repair units and does no work env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { @@ -1019,8 +1019,8 @@ test("scoped-archive-reconcile throttles a scoped archive's resume to at most on env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { @@ -1119,8 +1119,8 @@ test("scoped-archive-reconcile resumes a scoped archive again once its lookback env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { @@ -1232,8 +1232,8 @@ process.exit(0); entrypoint: SLACK_ENTRYPOINT, env: { HOME: homeDir, - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, SLACKDUMP_BIN: fakeSlackdumpPath, }, @@ -1345,8 +1345,8 @@ test("scoped-archive-reconcile emits DETAIL_GAP_RECOVERED when a previously-fail env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { @@ -1465,9 +1465,9 @@ process.exit(0); entrypoint: SLACK_ENTRYPOINT, env: { HOME: homeDir, - SLACK_COOKIE: "d=fake", + SLACK_COOKIE: "xoxd-fake", SLACK_RECLAIM_UPLOADS: "1", - SLACK_TOKEN: "xoxc-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, SLACKDUMP_BIN: fakeSlackdumpPath, }, @@ -1590,8 +1590,8 @@ test("a later successful NEW-repair attempt emits DETAIL_GAP_RECOVERED for a pre env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { diff --git a/packages/polyfill-connectors/connectors/slack/index.ts b/packages/polyfill-connectors/connectors/slack/index.ts index fba97f262..0030660fa 100755 --- a/packages/polyfill-connectors/connectors/slack/index.ts +++ b/packages/polyfill-connectors/connectors/slack/index.ts @@ -372,6 +372,18 @@ function formatSlackdumpProgress(label: string, snapshot: SlackdumpProgressSnaps return `Slack slackdump ${label} progress: ${facts.join(" ")}`; } +function redactSlackdumpOutput(output: string, env: NodeJS.ProcessEnv): string { + let redacted = output; + for (const secret of [env.SLACK_TOKEN, env.SLACK_COOKIE]) { + if (secret) { + redacted = redacted.replaceAll(secret, "[REDACTED]"); + } + } + // Keep diagnostics safe even if a child prints only a token-shaped value or + // wraps the known credential before the exact replacement above can match. + return redacted.replace(/xox[a-z]-[^\s"'`]+/giu, "[REDACTED]"); +} + // Default timeout accommodates long-lived workspaces (10+ years) where a // first-run archive of DMs + history can run 6-20h depending on file count // and Slack rate-limit bursts. The cost of a too-high default is only "late @@ -442,7 +454,8 @@ export function runSlackdump( if (code === 0) { resolve({ stdout, stderr }); } else { - reject(new Error(`slackdump_exit_${code}: ${stderr.slice(0, 400) || stdout.slice(0, 400)}`)); + const detail = redactSlackdumpOutput(`${stderr}\n${stdout}`, env).slice(0, 400); + reject(new Error(`slackdump_exit_${code}${detail ? `: ${detail}` : ""}`)); } }); child.on("error", (e) => { @@ -509,14 +522,75 @@ interface SlackOpts { export const SLACK_RETRYABLE_FAILURE_RE = /ECONN|ETIMEDOUT|timeout|slackdump_exit_6|slack_rate_limited/i; -function extractCredentials(credentials: Record): SlackCredentials { - const workspace = credentials.SLACK_WORKSPACE; - const token = credentials.SLACK_TOKEN; - const cookie = credentials.SLACK_COOKIE; - if (!(workspace && token && cookie)) { +const SLACKDUMP_CLIENT_TOKEN_PREFIX = "xoxc-"; +const SLACKDUMP_D_COOKIE_PREFIX = "xoxd-"; +const SLACK_WORKSPACE_RE = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/u; +// Slackdump's value path preserves URL-safe percent escapes and QueryEscapes +// raw unsafe cookie characters. Keep provider values opaque here: only enforce +// the documented xoxc/xoxd prefixes, transport-safe control characters, and a +// bounded input size. +const SLACKDUMP_CREDENTIAL_MAX_LENGTH = 4096; +const INVALID_PERCENT_ESCAPE_RE = /%(?![0-9a-f]{2})/iu; + +function hasControlCharacter(value: string): boolean { + for (const character of value) { + const codePoint = character.codePointAt(0) ?? 0; + if (codePoint <= 0x1f || (codePoint >= 0x7f && codePoint <= 0x9f)) { + return true; + } + } + return false; +} + +function normalizeSlackOpaqueValue(raw: string, prefix: string | null, invalidCode: string): string { + const value = raw.trim(); + if ( + !value || + (prefix !== null && !value.startsWith(prefix)) || + value.length > SLACKDUMP_CREDENTIAL_MAX_LENGTH || + hasControlCharacter(value) + ) { + throw new Error(invalidCode); + } + return value; +} + +export function normalizeSlackWorkspace(raw: string): string { + const workspace = raw.trim().toLowerCase(); + if (!SLACK_WORKSPACE_RE.test(workspace)) { + throw new Error("slack_workspace_invalid"); + } + return workspace; +} + +export function normalizeSlackToken(raw: string): string { + return normalizeSlackOpaqueValue(raw, SLACKDUMP_CLIENT_TOKEN_PREFIX, "slack_token_invalid"); +} + +export function normalizeSlackCookie(raw: string): string { + let cookie = raw.trim(); + if (cookie.startsWith("d=")) { + cookie = cookie.slice(2).trim(); + } + const normalized = normalizeSlackOpaqueValue(cookie, SLACKDUMP_D_COOKIE_PREFIX, "slack_cookie_invalid"); + if (INVALID_PERCENT_ESCAPE_RE.test(normalized)) { + throw new Error("slack_cookie_invalid"); + } + return normalized; +} + +export function extractSlackCredentials(credentials: Record): SlackCredentials { + const rawWorkspace = typeof credentials.SLACK_WORKSPACE === "string" ? credentials.SLACK_WORKSPACE : ""; + const rawToken = typeof credentials.SLACK_TOKEN === "string" ? credentials.SLACK_TOKEN : ""; + const rawCookie = typeof credentials.SLACK_COOKIE === "string" ? credentials.SLACK_COOKIE : ""; + if (!(rawWorkspace.trim() && rawToken.trim() && rawCookie.trim())) { throw new Error("slack_credentials_missing"); } - return { workspace, token, cookie }; + return { + workspace: normalizeSlackWorkspace(rawWorkspace), + token: normalizeSlackToken(rawToken), + cookie: normalizeSlackCookie(rawCookie), + }; } function readSlackOptions(): SlackOpts { @@ -2393,7 +2467,7 @@ if (isMainModule(import.meta.url)) { async collect(ctx: CollectContext): Promise { const { state, requested, credentials, emit, progress } = ctx; - const { workspace, token, cookie } = extractCredentials(credentials); + const { workspace, token, cookie } = extractSlackCredentials(credentials); const opts = readSlackOptions(); // Resource filters (pre-fetch: pass as positional args; post-fetch: enforce too) diff --git a/packages/polyfill-connectors/connectors/slack/slackdump-runtime.test.ts b/packages/polyfill-connectors/connectors/slack/slackdump-runtime.test.ts index 1fee0f99d..fac8439a8 100644 --- a/packages/polyfill-connectors/connectors/slack/slackdump-runtime.test.ts +++ b/packages/polyfill-connectors/connectors/slack/slackdump-runtime.test.ts @@ -12,7 +12,11 @@ import { fileURLToPath } from "node:url"; import type { EmittedMessage } from "../../src/connector-runtime.ts"; import { runConnectorProtocolSubprocess } from "../../src/test-harness.ts"; import { + extractSlackCredentials, formatSlackdumpMissingError, + normalizeSlackCookie, + normalizeSlackToken, + normalizeSlackWorkspace, runSlackdump, SLACK_RETRYABLE_FAILURE_RE, slackdumpProgressChanged, @@ -22,6 +26,45 @@ const __dirname = dirname(fileURLToPath(import.meta.url)); const PACKAGE_ROOT = resolve(__dirname, "../.."); const SLACK_ENTRYPOINT = join(PACKAGE_ROOT, "connectors", "slack", "index.ts"); const SLACK_MANIFEST = join(PACKAGE_ROOT, "manifests", "slack.json"); +const VALID_SLACK_TOKEN = "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + +test("Slack credential normalization preserves URL-encoded d-cookie bytes", () => { + const encodedCookie = "xoxd-session-a%2Bb%2Fc%3D%3D"; + const rawCookie = "xoxd-session-a+b/c=="; + const opaqueToken = "xoxc-enterprise/session.v2+opaque=="; + const opaqueCookie = "xoxd-enterprise/session.v2+opaque==?&!"; + + assert.equal(normalizeSlackCookie(` d=${encodedCookie} `), encodedCookie); + assert.equal(normalizeSlackCookie(rawCookie), rawCookie); + assert.equal(normalizeSlackToken(` ${VALID_SLACK_TOKEN} `), VALID_SLACK_TOKEN); + assert.equal(normalizeSlackToken(` ${opaqueToken} `), opaqueToken); + assert.equal(normalizeSlackCookie(opaqueCookie), opaqueCookie); + assert.equal(normalizeSlackWorkspace(" MyTeam "), "myteam"); + + assert.deepEqual( + extractSlackCredentials({ + SLACK_WORKSPACE: " myteam ", + SLACK_TOKEN: ` ${opaqueToken} `, + SLACK_COOKIE: `d=${opaqueCookie}`, + }), + { workspace: "myteam", token: opaqueToken, cookie: opaqueCookie } + ); +}); + +test("Slack credential normalization rejects empty, control, malformed, and oversized values", () => { + assert.throws(() => normalizeSlackToken(" "), /slack_token_invalid/); + assert.throws(() => normalizeSlackToken("xoxp-not-a-client-token"), /slack_token_invalid/); + assert.throws(() => normalizeSlackToken("xoxc-valid\u0000opaque"), /slack_token_invalid/); + assert.throws(() => normalizeSlackToken(`xoxc-${"a".repeat(4092)}`), /slack_token_invalid/); + assert.throws(() => normalizeSlackCookie("d= "), /slack_cookie_invalid/); + assert.throws(() => normalizeSlackCookie("xoxd-session-%2"), /slack_cookie_invalid/); + assert.throws(() => normalizeSlackCookie("xoxd-valid\u0001opaque"), /slack_cookie_invalid/); + assert.throws(() => normalizeSlackCookie(`xoxd-${"a".repeat(4092)}`), /slack_cookie_invalid/); + assert.throws(() => normalizeSlackWorkspace("../outside"), /slack_workspace_invalid/); + assert.throws(() => extractSlackCredentials({ SLACK_WORKSPACE: "myteam", SLACK_TOKEN: "", SLACK_COOKIE: "" }), { + message: "slack_credentials_missing", + }); +}); function createSlackArchiveSchema(db: DatabaseSync): void { db.exec(` @@ -116,6 +159,50 @@ test("runSlackdump: maps ENOENT to actionable missing-binary guidance", async () } }); +test("runSlackdump: redacts session credentials from child failure output", async () => { + const tmpDir = await mkdtemp(join(tmpdir(), "pdpp-slackdump-redaction-")); + const fakeSlackdump = join(tmpDir, "fake-slackdump.mjs"); + const token = VALID_SLACK_TOKEN; + const cookie = "xoxd-session-secret%2Bvalue"; + const priorBin = process.env.SLACKDUMP_BIN; + + await writeFile( + fakeSlackdump, + `#!/usr/bin/env node +process.stderr.write("token=" + process.env.SLACK_TOKEN + " cookie=" + process.env.SLACK_COOKIE); +process.stdout.write("stdout-token=" + process.env.SLACK_TOKEN); +process.exit(7); +`, + "utf8" + ); + await chmod(fakeSlackdump, 0o755); + process.env.SLACKDUMP_BIN = fakeSlackdump; + + try { + await assert.rejects( + runSlackdump(["workspace", "new"], { + env: { ...process.env, SLACK_TOKEN: token, SLACK_COOKIE: cookie }, + timeoutMs: 1000, + }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.match(error.message, /slackdump_exit_7/); + assert.doesNotMatch(error.message, new RegExp(token)); + assert.doesNotMatch(error.message, new RegExp(cookie)); + assert.match(error.message, /\[REDACTED\]/); + return true; + } + ); + } finally { + if (priorBin === undefined) { + delete process.env.SLACKDUMP_BIN; + } else { + process.env.SLACKDUMP_BIN = priorBin; + } + await rm(tmpDir, { recursive: true, force: true }); + } +}); + test("slack retry classification treats slackdump exit 6 as resumable", () => { assert.equal(SLACK_RETRYABLE_FAILURE_RE.test("slackdump failed: slackdump_exit_6: conversations.history 500"), true); assert.equal(SLACK_RETRYABLE_FAILURE_RE.test("parser error: unexpected token in archive"), false); @@ -309,6 +396,33 @@ test("slack manifest declares no unsupported-in-mode streams (all four gap strea } }); +test("slack manifest explains the xoxc token and d-cookie fields with the official manual", async () => { + const manifest = JSON.parse(await readFile(SLACK_MANIFEST, "utf8")) as { + setup?: { + credential_capture?: { + description?: string; + fields?: Array<{ help_text?: string; help_url?: string; label?: string; name?: string }>; + }; + }; + }; + const setup = manifest.setup?.credential_capture; + const token = setup?.fields?.find((field) => field.name === "slack_token"); + const cookie = setup?.fields?.find((field) => field.name === "slack_cookie"); + assert.match(setup?.description ?? "", /not an OAuth app token/); + assert.match(token?.label ?? "", /web-client session token/); + assert.match(token?.help_text ?? "", /localConfig_v2/); + assert.match(cookie?.label ?? "", /d cookie value/); + assert.match(cookie?.help_text ?? "", /cookie named exactly d/); + assert.match(cookie?.help_text ?? "", /not d=/); + assert.match(cookie?.help_text ?? "", /%2F.*%2B/); + assert.equal(token?.help_url, cookie?.help_url); + assert.match( + token?.help_url ?? "", + /github\.com\/rusq\/slackdump\/blob\/5ecece6b7fa63f6e1a71e049900b9ccc61f6b1e7\/doc\/login-manual\.md/ + ); + assert.doesNotMatch(token?.help_url ?? "", /wiki\/How-to-get-your-Slack-credentials/); +}); + test("slack connector reports DONE.records_emitted from runtime-counted RECORDs", async () => { const homeDir = await mkdtemp(join(tmpdir(), "pdpp-slack-counter-")); try { @@ -354,8 +468,8 @@ test("slack connector reports DONE.records_emitted from runtime-counted RECORDs" env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { @@ -441,8 +555,8 @@ test("slack connector counts channel-scoped message RECORDs in DONE.records_emit env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { @@ -488,8 +602,8 @@ test("slack connector emits a bounded source-partition diagnostic when a prior c env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { @@ -565,8 +679,8 @@ test("slack connector heals a missing prior channel from an existing scoped arch env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { @@ -638,8 +752,8 @@ test("slack connector does not emit a missing-partition diagnostic when prior ch env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { @@ -692,8 +806,8 @@ test("slack connector uses per-channel message cursors with legacy global fallba env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { @@ -777,8 +891,8 @@ test("slack connector uses an isolated scoped archive for targeted channel backf env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { @@ -839,8 +953,8 @@ test("slack connector emits scoped archive rows even when they are older than th env: { HOME: homeDir, PDPP_SLACK_SKIP_SLACKDUMP: "1", - SLACK_COOKIE: "d=fake", - SLACK_TOKEN: "xoxc-fake", + SLACK_COOKIE: "xoxd-fake", + SLACK_TOKEN: "xoxc-1-2-3-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SLACK_WORKSPACE: workspace, }, start: { diff --git a/packages/polyfill-connectors/manifests/google_maps_data_portability.json b/packages/polyfill-connectors/manifests/google_maps_data_portability.json index 53181e855..ce4230287 100644 --- a/packages/polyfill-connectors/manifests/google_maps_data_portability.json +++ b/packages/polyfill-connectors/manifests/google_maps_data_portability.json @@ -69,8 +69,9 @@ "rationale": "Google Data Portability API exports require a deployment-level Google OAuth/Data Portability application plus owner authorization. Timeline points and segments are not documented Data Portability resources, so this source is separate from Google Maps Timeline Import." }, "public_listing": { - "listed": true, - "status": "needs_human_auth" + "listed": false, + "status": "unproven", + "proof_gate": "Archive download and Maps resource group parsing not yet implemented. Currently only archive job metadata (control plane) is available." } }, "external_docs": [ diff --git a/packages/polyfill-connectors/manifests/slack.json b/packages/polyfill-connectors/manifests/slack.json index 39a528584..8167a7542 100644 --- a/packages/polyfill-connectors/manifests/slack.json +++ b/packages/polyfill-connectors/manifests/slack.json @@ -33,7 +33,7 @@ "credential_capture": { "kind": "secret_bundle", "label": "Slack workspace credentials", - "description": "Paste the session credentials from your Slack browser session. These are captured and sealed per-connection; the deployment does not need env vars.", + "description": "Slackdump uses a logged-in Slack web session, not an OAuth app token: enter your workspace subdomain, the xoxc web-client session token, and the value of the browser cookie named d. These are captured and sealed per-connection; the deployment does not need env vars.", "submit_label": "Create Slack connection and start first sync", "fields": [ { @@ -50,23 +50,24 @@ }, { "name": "slack_token", - "label": "Slack session token (xoxc-…)", + "label": "Slack web-client session token (xoxc-…)", "type": "password", "required": true, "secret": true, "autocomplete": "off", - "help_url": "https://github.com/rusq/slackdump/wiki/How-to-get-your-Slack-credentials", - "help_text": "Copy the xoxc-… token from your browser's Slack JS context. See the runbook link for exact steps.", + "help_url": "https://github.com/rusq/slackdump/blob/5ecece6b7fa63f6e1a71e049900b9ccc61f6b1e7/doc/login-manual.md", + "help_text": "In Slack on the web, open Developer Tools → Console and run JSON.parse(localStorage.localConfig_v2).teams[document.location.pathname.match(/^\\/client\/([A-Z0-9]+)/)[1]].token. Paste the returned xoxc-… value.", "env": ["SLACK_TOKEN"] }, { "name": "slack_cookie", - "label": "Slack 'd' cookie", + "label": "Slack browser d cookie value (xoxd-…)", "type": "password", "required": true, "secret": true, "autocomplete": "off", - "help_text": "Copy the 'd' cookie from your browser while logged in to Slack. Same runbook as above.", + "help_url": "https://github.com/rusq/slackdump/blob/5ecece6b7fa63f6e1a71e049900b9ccc61f6b1e7/doc/login-manual.md", + "help_text": "In Developer Tools → Application/Storage → Cookies, select your Slack domain, find the cookie named exactly d, and copy its Value. Paste the value only (usually starts xoxd-), not d=; preserve %2F/%2B escapes.", "env": ["SLACK_COOKIE"] } ] @@ -83,7 +84,7 @@ "rate_limit_sensitivity": "medium", "bot_detection_sensitivity": "low", "background_safe": true, - "rationale": "Slack credentials are durable and the Docker reference deployment includes the slackdump runtime; refresh moderately." + "rationale": "Slack credentials are durable. Slackdump v4.4.2 (AGPL-3.0, https://github.com/rusq/slackdump/tree/v4.4.2) is bundled in core-browser Docker image; refresh moderately." }, "public_listing": { "listed": true, diff --git a/packages/polyfill-connectors/manifests/strava.json b/packages/polyfill-connectors/manifests/strava.json index 974b0f0a3..91fa4c887 100644 --- a/packages/polyfill-connectors/manifests/strava.json +++ b/packages/polyfill-connectors/manifests/strava.json @@ -15,19 +15,19 @@ "capabilities": { "human_interaction": [], "refresh_policy": { - "recommended_mode": "automatic", + "recommended_mode": "manual", "recommended_interval_seconds": 21600, "minimum_interval_seconds": 3600, "maximum_staleness_seconds": 86400, "interaction_posture": "none", "rate_limit_sensitivity": "medium", "bot_detection_sensitivity": "low", - "background_safe": true, - "rationale": "Strava activities are uploaded after workouts; six-hourly refresh stays well inside rate limits." + "background_safe": false, + "rationale": "Manual refresh is retained while the owner setup path remains unproven." }, "public_listing": { - "listed": true, - "status": "proven" + "listed": false, + "status": "unproven" }, "auth": { "kind": "env", diff --git a/packages/polyfill-connectors/scripts/no-await-in-loops-allowlist.ts b/packages/polyfill-connectors/scripts/no-await-in-loops-allowlist.ts index c4b3fda47..22810a3d2 100644 --- a/packages/polyfill-connectors/scripts/no-await-in-loops-allowlist.ts +++ b/packages/polyfill-connectors/scripts/no-await-in-loops-allowlist.ts @@ -265,42 +265,42 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/chase/index.ts", - line: 1290, + line: 1324, column: 7, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chase/index.ts", - line: 1480, + line: 1514, column: 7, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chase/index.ts", - line: 1534, + line: 1568, column: 7, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chase/index.ts", - line: 2094, + line: 2128, column: 5, category: "ordered_protocol_emission", note: "deps.emit(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/chase/index.ts", - line: 2126, + line: 2160, column: 21, category: "ordered_browser_interaction", note: "processAccountDownload(): sequential Playwright action against the shared page/context", }, { path: "connectors/chase/index.ts", - line: 2397, + line: 2431, column: 7, category: "ordered_protocol_emission", note: "deps.emit(): Collection Profile protocol emission requiring in-order delivery", @@ -1000,91 +1000,91 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "connectors/slack/index.ts", - line: 1154, + line: 1228, column: 20, category: "shared_mutable_accumulator", note: "refreshScopedArchive(): loop body mutates a shared accumulator the next iteration reads", }, { path: "connectors/slack/index.ts", - line: 1252, + line: 1326, column: 9, category: "shared_mutable_accumulator", note: "runRequestedStreams(): loop body mutates a shared accumulator the next iteration reads", }, { path: "connectors/slack/index.ts", - line: 1483, + line: 1557, column: 7, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 1487, + line: 1561, column: 9, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 1492, + line: 1566, column: 9, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 1687, + line: 1761, column: 5, category: "ordered_protocol_emission", note: "emitWithFingerprint(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 1724, + line: 1798, column: 7, category: "ordered_protocol_emission", note: "emitWithFingerprint(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 1956, + line: 2030, column: 5, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 1980, + line: 2054, column: 5, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 1988, + line: 2062, column: 5, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 1996, + line: 2070, column: 5, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 2037, + line: 2111, column: 5, category: "ordered_protocol_emission", note: "deps.emitRecord(): Collection Profile protocol emission requiring in-order delivery", }, { path: "connectors/slack/index.ts", - line: 2386, + line: 2460, column: 32, category: "dependent_file_cursor", note: "reclaimUploads(): sequential file/dir walk, incremental byte-offset read, or one-way fs mutation", @@ -1462,28 +1462,28 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "src/auto-login/heb.test.ts", - line: 803, + line: 806, column: 7, category: "test_assertion_sequencing", note: "assert.rejects(): test drives/asserts an ordered per-case side effect", }, { path: "src/auto-login/heb.test.ts", - line: 852, + line: 855, column: 7, category: "test_assertion_sequencing", note: "assert.rejects(): test drives/asserts an ordered per-case side effect", }, { path: "src/auto-login/heb.test.ts", - line: 919, + line: 922, column: 7, category: "test_assertion_sequencing", note: "assert.rejects(): test drives/asserts an ordered per-case side effect", }, { path: "src/auto-login/heb.test.ts", - line: 948, + line: 951, column: 16, category: "test_assertion_sequencing", note: "ensureHebSession(): test drives/asserts an ordered per-case side effect", @@ -1544,6 +1544,13 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ category: "ordered_browser_interaction", note: "inspectPostSubmitAuthSurface(): sequential Playwright action against the shared page/context", }, + { + path: "src/auto-login/heb.ts", + line: 616, + column: 22, + category: "bounded_retry_polling", + note: "waitForUniqueVerificationCodeFormRoot(): retry/poll until the remounted OTP surface is uniquely actionable", + }, { path: "src/auto-login/manual-action-copy.test.ts", line: 42, @@ -1551,9 +1558,16 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ category: "test_assertion_sequencing", note: "readFile(): test drives/asserts an ordered per-case side effect", }, + { + path: "src/auto-login/reddit.test.ts", + line: 94, + column: 9, + category: "bounded_retry_polling", + note: "makeLocator().waitFor(): bounded test double polling until the simulated locator attaches", + }, { path: "src/auto-login/reddit.ts", - line: 268, + line: 275, column: 10, category: "bounded_retry_polling", note: "hasSessionCookie(): retry/backoff/poll loop gated on the prior attempt's outcome", @@ -1616,21 +1630,21 @@ export const NO_AWAIT_IN_LOOPS_ALLOWLIST: readonly NoAwaitInLoopsAllowlistEntry[ }, { path: "src/collector-runner.ts", - line: 1677, + line: 1700, column: 7, category: "provider_pacing_backpressure", note: "input.client.recoverLocalCollectorGap(): rate-limited/budget-gated external call", }, { path: "src/collector-runner.ts", - line: 1984, + line: 2007, column: 7, category: "shared_mutable_accumulator", note: "drainClaimedOutboxItem(): loop body mutates a shared accumulator the next iteration reads", }, { path: "src/collector-runner.ts", - line: 2484, + line: 2507, column: 18, category: "shared_mutable_accumulator", note: "input.queue.dequeueReady(): loop body mutates a shared accumulator the next iteration reads", diff --git a/packages/polyfill-connectors/src/auto-login/heb.test.ts b/packages/polyfill-connectors/src/auto-login/heb.test.ts index ea134ed45..8ecade981 100644 --- a/packages/polyfill-connectors/src/auto-login/heb.test.ts +++ b/packages/polyfill-connectors/src/auto-login/heb.test.ts @@ -132,6 +132,7 @@ interface FakePageState { live: boolean; loginHtml: string; nowMs: number; + onWaitForTimeout: (() => void) | undefined; postSubmitOutcomes: PostSubmitTransition[]; submitClicks: number; url: string; @@ -467,6 +468,7 @@ function makePage(initial: FakePageInit = {}): Page { live: initial.live ?? false, loginHtml: initial.html ?? SIGNIN_HTML, nowMs: 0, + onWaitForTimeout: initial.onWaitForTimeout, postSubmitOutcomes: initial.postSubmitOutcomes ?? (initial.postSubmitOutcome ? [initial.postSubmitOutcome] : []), submitClicks: 0, url: initial.url ?? SIGNIN_URL, @@ -519,6 +521,7 @@ function makePage(initial: FakePageInit = {}): Page { waitForTimeout: (ms: number): Promise => { state.nowMs += ms; maybeApplyPostSubmitOutcome(); + state.onWaitForTimeout?.(); return Promise.resolve(); }, }; @@ -1099,3 +1102,114 @@ test("ensureHebSession times out on a stable unknown post-submit page", async () assert.ok(state.gotoEvents[1]?.atMs !== undefined && state.gotoEvents[1].atMs >= 8000); }); }); + +test("ensureHebSession recognizes authenticated evidence that appears after the old eight-second window", async () => { + await withHebCredentials(async () => { + const page = makePage({ + html: SIGNIN_HTML, + live: false, + postSubmitOutcomes: [ + { + atMs: 9500, + html: LIVE_HTML, + kind: "live", + url: ORDERS_URL, + }, + ], + url: SIGNIN_URL, + view: "login", + }); + const harness = makeInteractionHarness(); + + const ok = await ensureHebSession({ + page, + postSubmitWaitClock: makePostSubmitWaitClock(page), + sendInteraction: harness.sendInteraction, + }); + + assert.equal(ok, true); + assert.equal(harness.requests.length, 0); + assert.equal(state.submitClicks, 1); + assert.equal(state.live, true); + assert.ok(state.nowMs >= 9500); + }); +}); + +test("ensureHebSession re-resolves a remounted OTP form after a delayed owner response", async () => { + await withHebCredentials(async () => { + const page = makePage({ + html: VERIFICATION_HTML, + live: false, + postSubmitOutcome: { + atMs: 200, + html: LIVE_HTML, + kind: "live", + url: ORDERS_URL, + }, + url: SIGNIN_URL, + view: "verification", + }); + const harness = makeInteractionHarness({ + responseForRequest: (req: InteractionRequest): InteractionResponse => { + assert.equal(req.kind, "otp"); + // Model the UAT shape: the owner response arrives after the page has + // had time to replace the original OTP root, but before the new root + // is available to the resumed connector. + state.nowMs += 19_000; + state.forms = []; + state.onWaitForTimeout = () => { + state.forms = [createForm({ codeControls: [createControl(true)], submitControls: [] })]; + state.onWaitForTimeout = undefined; + }; + return { + data: { code: "123456" }, + request_id: req.request_id ?? "test_interaction", + status: "success", + type: "INTERACTION_RESPONSE", + }; + }, + }); + + const ok = await ensureHebSession({ + page, + postSubmitWaitClock: makePostSubmitWaitClock(page), + sendInteraction: harness.sendInteraction, + }); + + assert.equal(ok, true); + assert.equal(harness.requests.length, 1); + assert.equal(harness.requests[0]?.kind, "otp"); + assert.equal(state.submitClicks, 1); + assert.equal(state.live, true); + assert.ok(state.nowMs >= 19_000); + }); +}); + +test("ensureHebSession keeps OTP root ambiguity fail-closed after a valid response", async () => { + await withHebCredentials(async () => { + const page = makePage({ + html: VERIFICATION_HTML, + forms: [ + createForm({ codeControls: [createControl(true)], submitControls: [] }), + createForm({ codeControls: [createControl(true)], submitControls: [] }), + ], + live: false, + url: SIGNIN_URL, + view: "verification", + }); + const harness = makeInteractionHarness(); + + await assert.rejects( + ensureHebSession({ + page, + postSubmitWaitClock: makePostSubmitWaitClock(page), + sendInteraction: harness.sendInteraction, + }), + /heb_verification_code_input_missing/ + ); + assert.equal(harness.requests.length, 1); + assert.equal(harness.requests[0]?.kind, "otp"); + assert.equal(state.submitClicks, 0); + assert.equal(state.live, false); + }); +}); diff --git a/packages/polyfill-connectors/src/auto-login/heb.ts b/packages/polyfill-connectors/src/auto-login/heb.ts index ac82ffce1..74aa8edb3 100644 --- a/packages/polyfill-connectors/src/auto-login/heb.ts +++ b/packages/polyfill-connectors/src/auto-login/heb.ts @@ -28,7 +28,7 @@ import type { CaptureSession } from "../fixture-capture.ts"; const ORDERS_URL = "https://www.heb.com/my-account/your-orders"; const SESSION_PROBE_WAIT_MS = 2000; const POST_SUBMIT_POLL_INTERVAL_MS = 200; -const POST_SUBMIT_TIMEOUT_MS = 8000; +const POST_SUBMIT_TIMEOUT_MS = 12_000; const FIELD_TIMEOUT_MS = 15_000; const EMAIL_SELECTOR = 'input[name="email"], input[type="email"], input[autocomplete="username"], input[name="username"]'; @@ -501,7 +501,8 @@ async function handleVerificationCodeSubmission({ throw new Error("heb_verification_code_not_provided"); } - const verificationCodeRoot = await resolveUniqueVerificationCodeFormRoot(page); + const waitClock = postSubmitWaitClock ?? defaultPostSubmitWaitClock(page); + const verificationCodeRoot = await waitForUniqueVerificationCodeFormRoot(page, waitClock); if (!verificationCodeRoot) { throw new Error("heb_verification_code_input_missing"); } @@ -512,12 +513,9 @@ async function handleVerificationCodeSubmission({ } await checkpoint?.("heb-verification-code-submitted"); - const postSubmitSurface = await waitForPostSubmitAuthSurface( - page, - postSubmitWaitClock ?? defaultPostSubmitWaitClock(page), - checkpoint, - { ignoreVerificationCode: true } - ); + const postSubmitSurface = await waitForPostSubmitAuthSurface(page, waitClock, checkpoint, { + ignoreVerificationCode: true, + }); if (postSubmitSurface.kind === "live") { await checkpoint?.("heb-post-submit-live"); await checkpoint?.("heb-verification-code-reprobe"); @@ -611,3 +609,20 @@ export async function ensureHebSession({ throw new Error("heb_login_unexpected_ui"); } + +async function waitForUniqueVerificationCodeFormRoot(page: Page, clock: PostSubmitWaitClock): Promise { + const deadline = clock.now() + FIELD_TIMEOUT_MS; + while (clock.now() <= deadline) { + const resolved = await resolveUniqueVerificationCodeFormRoot(page); + if (resolved) { + return resolved; + } + + const remainingMs = deadline - clock.now(); + if (remainingMs <= 0) { + return null; + } + await clock.wait(Math.min(POST_SUBMIT_POLL_INTERVAL_MS, remainingMs)); + } + return null; +} diff --git a/packages/polyfill-connectors/src/auto-login/reddit.test.ts b/packages/polyfill-connectors/src/auto-login/reddit.test.ts index 40b15aee5..73a10c9e3 100644 --- a/packages/polyfill-connectors/src/auto-login/reddit.test.ts +++ b/packages/polyfill-connectors/src/auto-login/reddit.test.ts @@ -25,11 +25,14 @@ function makeContext(cookies: BrowserCookie[] = []): BrowserContext { } function makePageWithoutLoginInputs(): Page { - const emptyLocator: Pick = { + // Mirrors real Playwright: `waitFor` rejects on timeout when the element + // never attaches (never resolves `undefined` the way a stubbed no-op would). + const emptyLocator: Pick = { count: (): Promise => Promise.resolve(0), first(): Locator { return emptyLocator as Locator; }, + waitFor: (): Promise => Promise.reject(new Error("Timeout waiting for locator")), }; const fake: Pick = { goto(_url: string, _options?: Parameters[1]): ReturnType { @@ -43,7 +46,7 @@ function makePageWithoutLoginInputs(): Page { } function makeLocator({ count = 1, visible = true }: { count?: number; visible?: boolean } = {}): Locator { - const fake: Pick = { + const fake: Pick = { click: (): Promise => Promise.resolve(), count: (): Promise => Promise.resolve(count), fill: (_value: string): Promise => Promise.resolve(), @@ -53,10 +56,48 @@ function makeLocator({ count = 1, visible = true }: { count?: number; visible?: isVisible(): Promise { return Promise.resolve(visible); }, + waitFor(): Promise { + return count > 0 ? Promise.resolve() : Promise.reject(new Error("Timeout waiting for locator")); + }, }; return fake as Locator; } +/** Models the login input attaching to the DOM after a render delay. */ +function makeDelayedAttachLocator({ attachesAfterMs }: { attachesAfterMs: number }): { + fillCalls: string[]; + locator: Locator; +} { + const start = Date.now(); + const fillCalls: string[] = []; + const attached = (): boolean => Date.now() - start >= attachesAfterMs; + const fake: Pick = { + click: (): Promise => Promise.resolve(), + count: (): Promise => Promise.resolve(attached() ? 1 : 0), + fill: (value: string): Promise => { + fillCalls.push(value); + return Promise.resolve(); + }, + first(): Locator { + return fake as Locator; + }, + isVisible(): Promise { + return Promise.resolve(attached()); + }, + async waitFor(options?: Parameters[0]): Promise { + const timeout = options?.timeout ?? 30_000; + const deadline = Date.now() + timeout; + while (!attached()) { + if (Date.now() >= deadline) { + throw new Error("Timeout waiting for locator to be attached"); + } + await new Promise((resolve) => setTimeout(resolve, 5)); + } + }, + }; + return { fillCalls, locator: fake as Locator }; +} + function makePageWithHiddenOtp(): Page { const username = makeLocator(); const password = makeLocator(); @@ -234,6 +275,62 @@ test("ensureRedditSession emits manual_action when login inputs are blocked", as }); }); +test("ensureRedditSession waits past a slow client-side render instead of treating it as blocked", async () => { + await withRedditCredentials(async () => { + const requests: InteractionRequest[] = []; + const { fillCalls, locator: username } = makeDelayedAttachLocator({ attachesAfterMs: 150 }); + const password = makeLocator(); + const submit = makeLocator(); + const empty = makeLocator({ count: 0, visible: false }); + const page: Pick = { + getByRole(_role: Parameters[0], _options?: Parameters[1]): Locator { + return submit; + }, + goto(_url: string, _options?: Parameters[1]): ReturnType { + return Promise.resolve(null); + }, + locator(selector: string, _options?: Parameters[1]): Locator { + if (selector.includes("username")) { + return username; + } + if (selector.includes("password")) { + return password; + } + return empty; + }, + waitForLoadState(): ReturnType { + return Promise.resolve(); + }, + waitForTimeout(): ReturnType { + return Promise.resolve(); + }, + }; + + // The run doesn't reach a live session in this fixture (no cookie + // machinery wired up) — the assertion is about the fill, not the outcome. + await assert.rejects( + ensureRedditSession({ + context: makeContext(), + page: page as Page, + sendInteraction(req: InteractionRequest): Promise { + requests.push(req); + return Promise.resolve({ + request_id: req.request_id ?? "test_interaction", + status: "success", + type: "INTERACTION_RESPONSE", + }); + }, + }) + ); + + // The pre-fix `count()` snapshot would have read 0 at t=0 and handed off + // to the operator without ever calling fill(); the correct behavior is + // to wait past the render delay and fill the real value. + assert.deepEqual(fillCalls, ["test-user"]); + assert.equal(requests.length, 0, "must not hand off to the operator for a field that arrives within budget"); + }); +}); + test("ensureRedditSession ignores hidden OTP fields instead of asking the owner too early", async () => { await withRedditCredentials(async () => { const requests: InteractionRequest[] = []; diff --git a/packages/polyfill-connectors/src/auto-login/reddit.ts b/packages/polyfill-connectors/src/auto-login/reddit.ts index 28a8502b7..11109b6a4 100644 --- a/packages/polyfill-connectors/src/auto-login/reddit.ts +++ b/packages/polyfill-connectors/src/auto-login/reddit.ts @@ -213,7 +213,14 @@ export async function ensureRedditSession({ await captureLoginState(capture, page, "reddit-login-page"); const userIn = page.locator(USERNAME_SELECTOR).first(); - if (!(await userIn.count().catch(() => 0))) { + // `count()` is a one-shot DOM snapshot with no wait; on Reddit's + // client-rendered login page it can read 0 before the field has painted. + // `waitFor` gives the render a real, bounded chance instead. + const usernameAppeared = await userIn + .waitFor({ state: "attached", timeout: 10_000 }) + .then((): true => true) + .catch((): false => false); + if (!usernameAppeared) { // Cloudflare challenge, shadow DOM change, or redirect loop — hand off. // Earn the diagnosis via the shared detector instead of guessing "possible // Cloudflare challenge" from absence of inputs alone. diff --git a/packages/polyfill-connectors/src/collector-runner.test.ts b/packages/polyfill-connectors/src/collector-runner.test.ts index 604c61634..16fa0d369 100644 --- a/packages/polyfill-connectors/src/collector-runner.test.ts +++ b/packages/polyfill-connectors/src/collector-runner.test.ts @@ -282,6 +282,80 @@ test("runCollectorConnector reports null completeness when no coverage diagnosti } }); +test("runCollectorConnector.onMessage observes every protocol message in emission order without altering the result", async () => { + const harness = await startCollectorHarness({ priorState: {} }); + try { + const fixture = await writeFixtureConnector({ + script: ` + await new Promise((r) => { let b = ""; process.stdin.on("data", (c) => { b += c; if (b.includes("\\n")) r(); }); }); + process.stdout.write(JSON.stringify({ type: "RECORD", stream: "messages", key: "m-1", data: { id: "m-1" }, emitted_at: new Date().toISOString() }) + "\\n"); + process.stdout.write(JSON.stringify({ type: "RECORD", stream: "messages", key: "m-2", data: { id: "m-2" }, emitted_at: new Date().toISOString() }) + "\\n"); + process.stdout.write(JSON.stringify({ type: "STATE", stream: "messages", cursor: { fetched_at: new Date().toISOString() } }) + "\\n"); + process.stdout.write(JSON.stringify({ type: "DONE", status: "succeeded", records_emitted: 2 }) + "\\n"); + `, + }); + + const observed: string[] = []; + const result = await runCollectorConnector({ + baseUrl: harness.url, + connector: { + args: [fixture], + command: "node", + connector_id: "fixture-onmessage", + runtime_requirements: { bindings: {} }, + streams: ["messages"], + }, + deviceId: "device-1", + deviceToken: "device-token", + onMessage: (message) => observed.push(message.type), + queuePath: await tempQueuePath(), + sourceInstanceId: "src-onmessage", + }); + + assert.deepEqual(observed, ["RECORD", "RECORD", "STATE", "DONE"]); + assert.equal(result.done?.status, "succeeded"); + assert.equal(result.recordsQueued, 2); + } finally { + await harness.close(); + } +}); + +test("runCollectorConnector.onMessage errors are swallowed and never break the run", async () => { + const harness = await startCollectorHarness({ priorState: {} }); + try { + const fixture = await writeFixtureConnector({ + script: ` + await new Promise((r) => { let b = ""; process.stdin.on("data", (c) => { b += c; if (b.includes("\\n")) r(); }); }); + process.stdout.write(JSON.stringify({ type: "RECORD", stream: "messages", key: "m-1", data: { id: "m-1" }, emitted_at: new Date().toISOString() }) + "\\n"); + process.stdout.write(JSON.stringify({ type: "DONE", status: "succeeded", records_emitted: 1 }) + "\\n"); + `, + }); + + const result = await runCollectorConnector({ + baseUrl: harness.url, + connector: { + args: [fixture], + command: "node", + connector_id: "fixture-onmessage-throws", + runtime_requirements: { bindings: {} }, + streams: ["messages"], + }, + deviceId: "device-1", + deviceToken: "device-token", + onMessage: () => { + throw new Error("reporter bug"); + }, + queuePath: await tempQueuePath(), + sourceInstanceId: "src-onmessage-throws", + }); + + assert.equal(result.done?.status, "succeeded"); + assert.equal(result.recordsQueued, 1); + } finally { + await harness.close(); + } +}); + test("runCollectorConnector rejects failed terminal DONE, preserves records, and leaves a durable recovery gap", async () => { const harness = await startCollectorHarness({ priorState: {} }); try { diff --git a/packages/polyfill-connectors/src/collector-runner.ts b/packages/polyfill-connectors/src/collector-runner.ts index 782cc7661..38f79710d 100644 --- a/packages/polyfill-connectors/src/collector-runner.ts +++ b/packages/polyfill-connectors/src/collector-runner.ts @@ -479,6 +479,16 @@ export interface CollectorRunConfig { connector: CollectorConnectorSpec; deviceId: string; deviceToken: string; + /** + * Optional observer invoked for every protocol message the connector + * child emits (RECORD, STATE, PROGRESS, DONE, etc.), in emission order, + * before the message is applied to the durable outbox. Purely a + * read-only tap for live progress reporting (e.g. the CLI's terminal + * output) — it MUST NOT be relied on for correctness, is never awaited, + * and any error it throws is swallowed so a reporting bug cannot break + * the run. + */ + onMessage?: (message: EmittedMessage) => void; /** * Path to the durable SQLite outbox. The legacy `queuePath` field is * accepted as a fallback when this is omitted so existing call sites @@ -1290,7 +1300,20 @@ async function streamConnectorIntoOutbox( coverageByStore.set(entry.store, { status: entry.status, stream: entry.stream }); }; + const notifyOnMessage = (message: EmittedMessage): void => { + if (!input.config.onMessage) { + return; + } + try { + input.config.onMessage(message); + } catch { + // The observer is a read-only reporting tap (e.g. CLI progress + // output); a bug in it must never break the run. + } + }; + const handleMessage = (message: EmittedMessage): void => { + notifyOnMessage(message); if (done !== null) { throw new Error(`${input.config.connector.connector_id} emitted ${message.type} after terminal DONE`); } diff --git a/packages/polyfill-connectors/src/connector-conformance-roster.ts b/packages/polyfill-connectors/src/connector-conformance-roster.ts index f14f00403..ae4ceabb0 100644 --- a/packages/polyfill-connectors/src/connector-conformance-roster.ts +++ b/packages/polyfill-connectors/src/connector-conformance-roster.ts @@ -44,12 +44,10 @@ export const PRODUCTION_READY_CONNECTORS: Record = gmail: { testFile: "connectors/gmail/integration.test.ts" }, heb: { testFile: "connectors/heb/index.test.ts" }, google_maps: { testFile: "connectors/google_maps/parsers.test.ts" }, - google_maps_data_portability: { testFile: "connectors/google_maps_data_portability/api.test.ts" }, notion: { testFile: "connectors/notion/schemas.test.ts" }, oura: { testFile: "connectors/oura/schemas.test.ts" }, reddit: { testFile: "connectors/reddit/integration.test.ts" }, slack: { testFile: "connectors/slack/integration.test.ts" }, - strava: { testFile: "connectors/strava/schemas.test.ts" }, usaa: { testFile: "connectors/usaa/integration.test.ts" }, whatsapp: { testFile: "connectors/whatsapp/integration.test.ts" }, ynab: { testFile: "connectors/ynab/integration.test.ts" }, @@ -88,10 +86,12 @@ export const KNOWN_SCAFFOLD_CONNECTORS = [ */ export const REAL_UNLISTED_CONNECTORS: Record = { apple_health: { testFile: "connectors/apple_health/parsers.test.ts" }, + google_maps_data_portability: { testFile: "connectors/google_maps_data_portability/api.test.ts" }, google_takeout: { testFile: "connectors/google_takeout/schemas.test.ts" }, ical: { testFile: "connectors/ical/parsers.test.ts" }, imessage: { testFile: "connectors/imessage/integration.test.ts" }, spotify: { testFile: "connectors/spotify/schemas.test.ts" }, + strava: { testFile: "connectors/strava/schemas.test.ts" }, twitter_archive: { testFile: "connectors/twitter_archive/parsers.test.ts" }, }; diff --git a/packages/polyfill-connectors/src/local-device-client.test.ts b/packages/polyfill-connectors/src/local-device-client.test.ts index 51a7b6be3..794c0d517 100644 --- a/packages/polyfill-connectors/src/local-device-client.test.ts +++ b/packages/polyfill-connectors/src/local-device-client.test.ts @@ -83,6 +83,23 @@ test("LocalDeviceClient sends bearer-authenticated heartbeat and ingest batch sh } }); +test("LocalDeviceClient sends bearer-authenticated self-revoke with no body", async () => { + const seen: SeenRequest[] = []; + const server = await startJsonServer(seen); + try { + const client = new LocalDeviceClient({ baseUrl: server.url, deviceId: "device-1", deviceToken: "device-token" }); + await client.selfRevoke(); + + assert.equal(seen[0]?.method, "POST"); + assert.equal(seen[0]?.path, LOCAL_DEVICE_ENDPOINTS.selfRevoke("device-1")); + assert.equal(seen[0]?.authorization, "Bearer device-token"); + assert.equal(seen[0]?.collectorProtocol, COLLECTOR_PROTOCOL_VERSION); + assert.equal(seen[0]?.body, null); + } finally { + await server.close(); + } +}); + test("LocalDeviceClient GET source-instance state hits the device-scoped state route with the bearer", async () => { const seen: SeenRequest[] = []; const server = await startJsonServer(seen); diff --git a/packages/polyfill-connectors/src/local-device-client.ts b/packages/polyfill-connectors/src/local-device-client.ts index a84b2e1fb..e51ed0c18 100644 --- a/packages/polyfill-connectors/src/local-device-client.ts +++ b/packages/polyfill-connectors/src/local-device-client.ts @@ -7,6 +7,7 @@ import type { LocalDeviceIngestBatchRequest } from "./local-device-envelope.ts"; export const LOCAL_DEVICE_ENDPOINTS = { exchangeEnrollment: "/_ref/device-exporters/enroll", heartbeat: (deviceId: string) => `/_ref/device-exporters/${encodeURIComponent(deviceId)}/heartbeat`, + selfRevoke: (deviceId: string) => `/_ref/device-exporters/${encodeURIComponent(deviceId)}/self-revoke`, ingestBatch: (deviceId: string) => `/_ref/device-exporters/${encodeURIComponent(deviceId)}/ingest-batches`, terminalCollection: (deviceId: string, sourceInstanceId: string) => `/_ref/device-exporters/${encodeURIComponent(deviceId)}/source-instances/${encodeURIComponent(sourceInstanceId)}/terminal-collection`, @@ -199,6 +200,12 @@ export interface RecoverLocalCollectorGapRequest { stream_boundary?: string; } +export interface SelfRevokeDeviceResponse { + device_id: string; + object: "device_exporter_revocation"; + revoked_at: string; +} + export class LocalDeviceHttpError extends Error { readonly body: string; readonly envelopeMessage: string | null; @@ -321,6 +328,19 @@ export class LocalDeviceClient { }); } + /** + * Revoke this device's own credential using its own bearer token. A + * device may only revoke itself — the server rejects any deviceId that + * does not match the authenticated credential. Called by `logout` before + * deleting local credentials so the server-side lane closes with them. + */ + selfRevoke(): Promise { + return this.#request(LOCAL_DEVICE_ENDPOINTS.selfRevoke(this.#requireDeviceId()), { + authenticate: true, + method: "POST", + }); + } + ingestBatch(request: IngestBatchRequest): Promise<{ ok: true }> { return this.#request(LOCAL_DEVICE_ENDPOINTS.ingestBatch(this.#requireDeviceId()), { authenticate: true, diff --git a/packages/polyfill-connectors/src/public-listing-manifest-honesty.test.ts b/packages/polyfill-connectors/src/public-listing-manifest-honesty.test.ts index 1f053497b..7622d02e7 100644 --- a/packages/polyfill-connectors/src/public-listing-manifest-honesty.test.ts +++ b/packages/polyfill-connectors/src/public-listing-manifest-honesty.test.ts @@ -28,7 +28,6 @@ const PILOT_FIXTURE_EXEMPT: Record = { notion: "Runtime ships without a connectors/notion/schemas.ts validator; pilot-fixture-test-helper has nothing to assert against.", oura: "Runtime ships without a connectors/oura/schemas.ts validator; same gap as notion.", - strava: "Runtime ships without a connectors/strava/schemas.ts validator; same gap as notion.", }; interface PublicListing { diff --git a/packages/reference-contract/src/reference/index.ts b/packages/reference-contract/src/reference/index.ts index d5ca247f0..dabe73133 100644 --- a/packages/reference-contract/src/reference/index.ts +++ b/packages/reference-contract/src/reference/index.ts @@ -2366,6 +2366,33 @@ export const referenceManifests = [ surface: "reference", tags: ["reference", "device-exporters"], }, + { + id: "refSelfRevokeDeviceExporter", + method: "POST", + path: "/_ref/device-exporters/{deviceId}/self-revoke", + request: { params: DeviceIdParamSchema }, + responses: { + 200: { + schema: { + additionalProperties: false, + properties: { + device_id: { type: "string" }, + object: { const: "device_exporter_revocation" }, + revoked_at: { type: "string" }, + }, + required: ["object", "device_id", "revoked_at"], + type: "object", + }, + }, + ...DeviceExporterErrors, + }, + summary: + "Revoke a local device exporter's own credential using its own device bearer token. A device credential may " + + "only revoke itself, never another device; the path deviceId must match the authenticated credential's " + + "device. Used by local-collector `logout` to close the server-side lane before deleting local credentials.", + surface: "reference", + tags: ["reference", "device-exporters"], + }, { id: "refHeartbeatDeviceExporter", method: "POST", diff --git a/reference-implementation/README.md b/reference-implementation/README.md index 6a6270093..bd7a68867 100644 --- a/reference-implementation/README.md +++ b/reference-implementation/README.md @@ -516,19 +516,22 @@ persistent browser profiles and remain subject to upstream anti-bot behavior. Mount any optional local connector inputs, such as Slack archives, explicitly when testing those connectors. -Slack imports additionally require a host-provided `slackdump` executable. The -stock reference image intentionally does not bundle `slackdump` because it is an -AGPL-licensed external tool. For Docker runs, set -`PDPP_DOCKER_SLACKDUMP_DIR` in `.env.docker` to a host directory containing a -`slackdump` executable, and keep `SLACKDUMP_BIN` pointed at the stable -in-container path: +Slack imports require `slackdump` v4.4.2 (AGPL-3.0). The `core`, `core-browser`, +`railway-core`, and `platform-core` Docker images ship `slackdump` bundled by default. +Source tree and license: bundled at `/usr/local/share/slackdump/` in the image +(LICENSE.agpl-3.0.txt, SOURCE_URL, SLACKDUMP_BIN=/usr/local/bin/slackdump). + +To override with a different `slackdump` build or version, set `SLACKDUMP_BIN` +to point at an alternative executable on `PATH` or an in-container path. For +Docker runs, you can mount a host directory via `PDPP_DOCKER_SLACKDUMP_DIR` +in `.env.docker`: ```env PDPP_DOCKER_SLACKDUMP_DIR=/home/user/go/bin SLACKDUMP_BIN=/opt/pdpp-tools/slackdump/slackdump ``` -Alternatively, place or symlink the executable at +Alternatively, place or symlink an alternative executable at `packages/polyfill-connectors/.pdpp-tools/slackdump/slackdump`; that local runtime directory is gitignored and is the default compose mount source. diff --git a/reference-implementation/docs/generated/reference-ref-routes.md b/reference-implementation/docs/generated/reference-ref-routes.md index 0bfefa684..8f09331b0 100644 --- a/reference-implementation/docs/generated/reference-ref-routes.md +++ b/reference-implementation/docs/generated/reference-ref-routes.md @@ -41,6 +41,7 @@ Generated from `packages/reference-contract/src/reference/`. Reference-designate | **GET** | `/_ref/device-exporters/source-instances` | `refListDeviceExporterSourceInstances` | List local device exporter source instances without promoting source-instance identity to the public PDPP contract. | | **GET** | `/_ref/device-exporters/diagnostics` | `refListDeviceExporterDiagnostics` | List owner/operator diagnostics for local device exporters, including heartbeat and ingest freshness. | | **POST** | `/_ref/device-exporters/{deviceId}/revoke` | `refRevokeDeviceExporter` | Revoke a local device exporter credential and stop future heartbeats or ingest from that device. | +| **POST** | `/_ref/device-exporters/{deviceId}/self-revoke` | `refSelfRevokeDeviceExporter` | Revoke a local device exporter's own credential using its own device bearer token. A device credential may only revoke itself, never another device; the path deviceId must match the authenticated credential's device. Used by local-collector `logout` to close the server-side lane before deleting local credentials. | | **POST** | `/_ref/device-exporters/{deviceId}/heartbeat` | `refHeartbeatDeviceExporter` | Accept a heartbeat from a device-scoped local exporter credential. | | **POST** | `/_ref/device-exporters/{deviceId}/ingest-batches` | `refIngestDeviceExporterBatch` | Accept an idempotent source-instance-aware ingest batch from a local device exporter. | | **GET** | `/_ref/device-exporters/{deviceId}/source-instances/{sourceInstanceId}/state` | `refGetDeviceExporterSourceInstanceState` | Read device-scoped local collector state for a source instance. Owner-token and client-token routes do not accept device credentials and vice versa. | @@ -725,6 +726,25 @@ Revoke a local device exporter credential and stop future heartbeats or ingest f - `404` — Not found - `409` — Conflict (e.g. run_already_active) +## refSelfRevokeDeviceExporter + +`POST /_ref/device-exporters/{deviceId}/self-revoke` + +Revoke a local device exporter's own credential using its own device bearer token. A device credential may only revoke itself, never another device; the path deviceId must match the authenticated credential's device. Used by local-collector `logout` to close the server-side lane before deleting local credentials. + +### Path parameters + +- `deviceId` — string + +### Responses + +- `200` — JSON body +- `400` — Invalid request +- `401` — Authentication required +- `403` — Permission denied +- `404` — Not found +- `409` — Conflict (e.g. run_already_active) + ## refHeartbeatDeviceExporter `POST /_ref/device-exporters/{deviceId}/heartbeat` diff --git a/reference-implementation/openapi/reference-full.openapi.json b/reference-implementation/openapi/reference-full.openapi.json index 48814716a..b7f2001e9 100644 --- a/reference-implementation/openapi/reference-full.openapi.json +++ b/reference-implementation/openapi/reference-full.openapi.json @@ -27842,6 +27842,496 @@ ] } }, + "/_ref/device-exporters/{deviceId}/self-revoke": { + "post": { + "operationId": "refSelfRevokeDeviceExporter", + "parameters": [ + { + "in": "path", + "name": "deviceId", + "required": true, + "schema": { + "minLength": 1, + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "additionalProperties": false, + "properties": { + "device_id": { + "type": "string" + }, + "object": { + "const": "device_exporter_revocation" + }, + "revoked_at": { + "type": "string" + } + }, + "required": [ + "object", + "device_id", + "revoked_at" + ], + "type": "object" + } + } + } + }, + "400": { + "description": "Invalid request", + "content": { + "application/json": { + "schema": { + "$id": "pdpp/common/PdppError", + "additionalProperties": false, + "properties": { + "error": { + "additionalProperties": false, + "properties": { + "available_connections": { + "items": { + "$id": "pdpp/common/ErrorAvailableConnection", + "additionalProperties": false, + "properties": { + "connection_id": { + "type": "string" + }, + "connector_id": { + "type": "string" + }, + "connector_key": { + "type": "string" + }, + "display_name": { + "type": [ + "string", + "null" + ] + }, + "label_status": { + "enum": [ + "owner_set", + "fallback" + ], + "type": "string" + } + }, + "required": [ + "connection_id" + ], + "type": "object" + }, + "type": "array" + }, + "code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "next_step": { + "type": "string" + }, + "param": { + "type": "string" + }, + "request_id": { + "type": "string" + }, + "resource_metadata": { + "type": "string" + }, + "retry_with": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": [ + "type", + "code", + "message", + "request_id" + ], + "type": "object" + } + }, + "required": [ + "error" + ], + "type": "object" + } + } + } + }, + "401": { + "description": "Authentication required", + "content": { + "application/json": { + "schema": { + "$id": "pdpp/common/PdppError", + "additionalProperties": false, + "properties": { + "error": { + "additionalProperties": false, + "properties": { + "available_connections": { + "items": { + "$id": "pdpp/common/ErrorAvailableConnection", + "additionalProperties": false, + "properties": { + "connection_id": { + "type": "string" + }, + "connector_id": { + "type": "string" + }, + "connector_key": { + "type": "string" + }, + "display_name": { + "type": [ + "string", + "null" + ] + }, + "label_status": { + "enum": [ + "owner_set", + "fallback" + ], + "type": "string" + } + }, + "required": [ + "connection_id" + ], + "type": "object" + }, + "type": "array" + }, + "code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "next_step": { + "type": "string" + }, + "param": { + "type": "string" + }, + "request_id": { + "type": "string" + }, + "resource_metadata": { + "type": "string" + }, + "retry_with": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": [ + "type", + "code", + "message", + "request_id" + ], + "type": "object" + } + }, + "required": [ + "error" + ], + "type": "object" + } + } + } + }, + "403": { + "description": "Permission denied", + "content": { + "application/json": { + "schema": { + "$id": "pdpp/common/PdppError", + "additionalProperties": false, + "properties": { + "error": { + "additionalProperties": false, + "properties": { + "available_connections": { + "items": { + "$id": "pdpp/common/ErrorAvailableConnection", + "additionalProperties": false, + "properties": { + "connection_id": { + "type": "string" + }, + "connector_id": { + "type": "string" + }, + "connector_key": { + "type": "string" + }, + "display_name": { + "type": [ + "string", + "null" + ] + }, + "label_status": { + "enum": [ + "owner_set", + "fallback" + ], + "type": "string" + } + }, + "required": [ + "connection_id" + ], + "type": "object" + }, + "type": "array" + }, + "code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "next_step": { + "type": "string" + }, + "param": { + "type": "string" + }, + "request_id": { + "type": "string" + }, + "resource_metadata": { + "type": "string" + }, + "retry_with": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": [ + "type", + "code", + "message", + "request_id" + ], + "type": "object" + } + }, + "required": [ + "error" + ], + "type": "object" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$id": "pdpp/common/PdppError", + "additionalProperties": false, + "properties": { + "error": { + "additionalProperties": false, + "properties": { + "available_connections": { + "items": { + "$id": "pdpp/common/ErrorAvailableConnection", + "additionalProperties": false, + "properties": { + "connection_id": { + "type": "string" + }, + "connector_id": { + "type": "string" + }, + "connector_key": { + "type": "string" + }, + "display_name": { + "type": [ + "string", + "null" + ] + }, + "label_status": { + "enum": [ + "owner_set", + "fallback" + ], + "type": "string" + } + }, + "required": [ + "connection_id" + ], + "type": "object" + }, + "type": "array" + }, + "code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "next_step": { + "type": "string" + }, + "param": { + "type": "string" + }, + "request_id": { + "type": "string" + }, + "resource_metadata": { + "type": "string" + }, + "retry_with": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": [ + "type", + "code", + "message", + "request_id" + ], + "type": "object" + } + }, + "required": [ + "error" + ], + "type": "object" + } + } + } + }, + "409": { + "description": "Conflict (e.g. run_already_active)", + "content": { + "application/json": { + "schema": { + "$id": "pdpp/common/PdppError", + "additionalProperties": false, + "properties": { + "error": { + "additionalProperties": false, + "properties": { + "available_connections": { + "items": { + "$id": "pdpp/common/ErrorAvailableConnection", + "additionalProperties": false, + "properties": { + "connection_id": { + "type": "string" + }, + "connector_id": { + "type": "string" + }, + "connector_key": { + "type": "string" + }, + "display_name": { + "type": [ + "string", + "null" + ] + }, + "label_status": { + "enum": [ + "owner_set", + "fallback" + ], + "type": "string" + } + }, + "required": [ + "connection_id" + ], + "type": "object" + }, + "type": "array" + }, + "code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "next_step": { + "type": "string" + }, + "param": { + "type": "string" + }, + "request_id": { + "type": "string" + }, + "resource_metadata": { + "type": "string" + }, + "retry_with": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": [ + "type", + "code", + "message", + "request_id" + ], + "type": "object" + } + }, + "required": [ + "error" + ], + "type": "object" + } + } + } + } + }, + "summary": "Revoke a local device exporter's own credential using its own device bearer token. A device credential may only revoke itself, never another device; the path deviceId must match the authenticated credential's device. Used by local-collector `logout` to close the server-side lane before deleting local credentials.", + "tags": [ + "reference", + "device-exporters" + ] + } + }, "/_ref/device-exporters/{deviceId}/heartbeat": { "post": { "operationId": "refHeartbeatDeviceExporter", diff --git a/reference-implementation/operations/rs-records-ingest/index.ts b/reference-implementation/operations/rs-records-ingest/index.ts index a9c061ad8..2ec868c73 100644 --- a/reference-implementation/operations/rs-records-ingest/index.ts +++ b/reference-implementation/operations/rs-records-ingest/index.ts @@ -15,14 +15,16 @@ * - the public `{ stream, records_accepted, records_rejected, errors }` * response envelope. * - * Per-line ingest order is preserved exactly: the operation iterates lines - * sequentially and awaits each `ingestRecord` call before advancing. It MUST - * NOT parallelize ingest, batch ingests, or coalesce errors. + * The default capability is deliberately per-line and ordered: the operation + * awaits each `ingestRecord` call before advancing. Hosts that provide the + * optional `ingestRecords` capability may batch only the already-parsed + * records; they still return one result per input record so parse and ingest + * failures remain line-addressable and ordered. * * Atomicity and durable write ordering for each record remain the - * responsibility of the underlying `ingestRecord` capability. A failure on - * one line increments `records_rejected` and continues; it MUST NOT roll back - * earlier accepted records (matches the previous native route behavior). + * responsibility of the underlying ingest capability. A failure on one line + * increments `records_rejected` and continues; it MUST NOT roll back earlier + * accepted records (matches the previous native route behavior). * * Boundary rules: * - This module SHALL NOT import Fastify, Next, SQLite, Postgres, a raw SQL @@ -55,6 +57,18 @@ export interface RecordsIngestDependencies { connectorInstanceId: string | null, record: Record ) => unknown | Promise; + /** + * Optional host optimization for a single NDJSON request. The input is in + * line order and contains only successfully parsed records. Each result is + * either null (accepted) or the exact error message for that record. + * Hosts MUST preserve the same per-record durability and failure-isolation + * contract as `ingestRecord`. + */ + ingestRecords?: ( + connectorId: string, + connectorInstanceId: string | null, + records: readonly Record[] + ) => readonly (string | null)[] | Promise; } export interface RecordsIngestEnvelope { @@ -76,6 +90,12 @@ export interface RecordsIngestOutput { readonly submittedRecordCount: number; } +interface ParsedRecordLines { + readonly lineErrors: Array; + readonly parsedLineIndexes: number[]; + readonly parsedRecords: Record[]; +} + export class RecordsIngestInvalidRequestError extends Error { readonly code: "invalid_request"; @@ -109,6 +129,103 @@ export function parseLines(body: string | null | undefined): string[] { return body.split("\n").filter((line) => line.trim().length > 0); } +function parseRecordLines(lines: readonly string[], streamName: string): ParsedRecordLines { + const lineErrors = new Array(lines.length).fill(null); + const parsedRecords: Record[] = []; + const parsedLineIndexes: number[] = []; + + for (const [lineIndex, line] of lines.entries()) { + try { + const parsed = JSON.parse(line) as Record; + parsedRecords.push({ ...parsed, stream: streamName }); + parsedLineIndexes.push(lineIndex); + } catch (err) { + lineErrors[lineIndex] = err instanceof Error ? err.message : String(err); + } + } + + return { lineErrors, parsedLineIndexes, parsedRecords }; +} + +async function ingestParsedRecords( + connectorId: string, + connectorInstanceId: string | null, + parsedRecords: readonly Record[], + dependencies: RecordsIngestDependencies +): Promise { + if (parsedRecords.length === 0) { + return []; + } + if (dependencies.ingestRecords) { + return await ingestWithBatchCapability(connectorId, connectorInstanceId, parsedRecords, dependencies.ingestRecords); + } + return await ingestSequentially(connectorId, connectorInstanceId, parsedRecords, dependencies.ingestRecord); +} + +async function ingestWithBatchCapability( + connectorId: string, + connectorInstanceId: string | null, + parsedRecords: readonly Record[], + ingestRecords: NonNullable +): Promise { + try { + const results = await ingestRecords(connectorId, connectorInstanceId, parsedRecords); + if (results.length !== parsedRecords.length) { + throw new Error(`ingestRecords returned ${results.length} results for ${parsedRecords.length} records`); + } + return results; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return parsedRecords.map(() => message); + } +} + +async function ingestSequentially( + connectorId: string, + connectorInstanceId: string | null, + parsedRecords: readonly Record[], + ingestRecord: RecordsIngestDependencies["ingestRecord"] +): Promise { + const errors: Array = new Array(parsedRecords.length).fill(null); + for (const [recordIndex, record] of parsedRecords.entries()) { + try { + // biome-ignore lint/performance/noAwaitInLoops: Preserves established ordered async behavior when the host has no batch capability. + await ingestRecord(connectorId, connectorInstanceId, record); + } catch (err) { + errors[recordIndex] = err instanceof Error ? err.message : String(err); + } + } + return errors; +} + +function applyIngestErrors( + lineErrors: Array, + parsedLineIndexes: readonly number[], + ingestErrors: readonly (string | null)[] +): void { + for (const [recordIndex, error] of ingestErrors.entries()) { + const lineIndex = parsedLineIndexes[recordIndex]; + if (lineIndex !== undefined) { + lineErrors[lineIndex] = error; + } + } +} + +function buildIngestEnvelope(stream: string, lineErrors: readonly (string | null)[]): RecordsIngestEnvelope { + let recordsAccepted = 0; + let recordsRejected = 0; + const errors: string[] = []; + for (const error of lineErrors) { + if (error === null) { + recordsAccepted += 1; + } else { + recordsRejected += 1; + errors.push(error); + } + } + return { errors, records_accepted: recordsAccepted, records_rejected: recordsRejected, stream }; +} + /** * Execute the canonical `rs.records.ingest` operation. * @@ -116,9 +233,12 @@ export function parseLines(body: string | null | undefined): string[] { * 1. parse non-empty NDJSON lines. * 2. invalid_request when connector_id is missing/empty. * 3. not_found when the manifest does not declare the stream. - * 4. iterate lines sequentially. Each line is JSON.parsed and ingested - * under `{ ...record, stream }`. JSON.parse failures and ingest throws - * both increment records_rejected and append the message to errors. + * 4. JSON.parse each line and ingest under `{ ...record, stream }`. + * JSON.parse failures and ingest errors both increment records_rejected + * and append the message to errors. If the host exposes `ingestRecords`, + * valid records use that capability once while preserving line order in + * the returned results; otherwise the established sequential capability + * is used. * 5. return the envelope plus submitted_record_count for instrumentation. */ export async function executeRecordsIngest( @@ -138,32 +258,17 @@ export async function executeRecordsIngest( throw new RecordsIngestNotFoundError(`Stream '${input.streamName}' not found for connector ${connectorId}`); } - let recordsAccepted = 0; - let recordsRejected = 0; - const errors: string[] = []; - - for (const line of lines) { - try { - const parsed = JSON.parse(line) as Record; - // biome-ignore lint/performance/noAwaitInLoops: Preserves established ordered async behavior, boundary contract, or dynamic test-harness type where a mechanical rewrite would change semantics. - await dependencies.ingestRecord(connectorId, input.connectorInstanceId ?? null, { - ...parsed, - stream: input.streamName, - }); - recordsAccepted += 1; - } catch (err) { - recordsRejected += 1; - errors.push(err instanceof Error ? err.message : String(err)); - } - } + const parsed = parseRecordLines(lines, input.streamName); + const ingestErrors = await ingestParsedRecords( + connectorId, + input.connectorInstanceId ?? null, + parsed.parsedRecords, + dependencies + ); + applyIngestErrors(parsed.lineErrors, parsed.parsedLineIndexes, ingestErrors); return { - envelope: { - errors, - records_accepted: recordsAccepted, - records_rejected: recordsRejected, - stream: input.streamName, - }, + envelope: buildIngestEnvelope(input.streamName, parsed.lineErrors), submittedRecordCount, }; } diff --git a/reference-implementation/runtime/controller.ts b/reference-implementation/runtime/controller.ts index 1f8ef058f..9effc66e3 100644 --- a/reference-implementation/runtime/controller.ts +++ b/reference-implementation/runtime/controller.ts @@ -235,6 +235,8 @@ export interface ActiveRun { readonly trace_id: string; } +export type RunAdmission = "collection" | "setup" | "browser_enrollment"; + export interface RunNowOptions { connectorInstanceId?: string; /** @@ -254,10 +256,12 @@ export interface RunNowOptions { resources?: Readonly>; rsUrl?: string; /** - * Narrow owner-session admission for the first run of a browser enrollment - * shell. Omitted means the ordinary active-connection collection path. + * Narrow owner-session admission for setup lifecycle runs. `setup` admits an + * exact draft or active connection; `browser_enrollment` admits only an exact + * browser enrollment draft. Omitted means the ordinary active-connection + * collection path. */ - runAdmission?: "browser_enrollment"; + runAdmission?: RunAdmission; runId?: string; scenarioId?: string; traceContext?: SpineTraceContext; @@ -309,6 +313,10 @@ function buildAutoResumeRunNowOptions( const options: RunNowOptions = { connectorInstanceId, priorityClass: "interactive", + // Credential capture is an owner-session setup operation. Its exact draft + // is intentionally admitted through the setup capability before the run + // is created; ordinary collection remains active-only. + runAdmission: "setup", triggerKind: "manual", }; if (input.manifest !== undefined) { @@ -415,7 +423,7 @@ export interface ControllerOptions { connectorId: string; connectorInstanceId: string | null; ownerSubjectId: string; - runAdmission: "collection" | "browser_enrollment"; + runAdmission: RunAdmission; }) => Promise<{ connectorId: string; connectorInstanceId: string }>; asPublicUrl?: string; /** Awaited before a managed surface lease becomes reusable after run cleanup. */ @@ -551,7 +559,7 @@ function resolveAdmittedRunConnection( connectorId: string, connectorInstanceId: string | undefined, ownerSubjectId: string, - runAdmission: "collection" | "browser_enrollment" + runAdmission: RunAdmission ): Promise<{ connectorId: string; connectorInstanceId: string }> { if (controllerOptions.admitRunConnection) { return controllerOptions.admitRunConnection({ @@ -567,7 +575,7 @@ function resolveAdmittedRunConnection( ); } -function runAdmissionFor(options: RunNowOptions): "collection" | "browser_enrollment" { +function runAdmissionFor(options: RunNowOptions): RunAdmission { return options.runAdmission ?? "collection"; } diff --git a/reference-implementation/runtime/index.ts b/reference-implementation/runtime/index.ts index 88046ec4f..56fb735fd 100644 --- a/reference-implementation/runtime/index.ts +++ b/reference-implementation/runtime/index.ts @@ -2545,6 +2545,7 @@ export async function runConnector(opts: RuntimeRunConnectorOptions): Promise ownerCancelRequested || runTimedOut; const knownGaps: Record[] = []; // Streams whose batch ingest was rejected as not_found for a stream the runtime // already validated present in the manifest at START (transient manifest drift @@ -3009,6 +3010,14 @@ export async function runConnector(opts: RuntimeRunConnectorOptions): Promise { + // Cancellation owns the terminal outcome. Do not start another ingest for + // a RECORD that was already buffered in the parent after the child was + // stopped; doing so makes terminalization wait behind an unbounded Gmail + // message/attachment queue. + if (terminalStopRequested()) { + recordBatch[stream] = []; + return; + } // Already deferred this run for transient manifest drift: don't re-POST (it // would just 404 again). Drop any further buffered records for the stream. if (driftSkippedStreams.has(stream)) { @@ -3025,6 +3034,12 @@ export async function runConnector(opts: RuntimeRunConnectorOptions): Promise>; try { result = await readIngestResponse(resp, stream, batch.length); @@ -3246,7 +3266,7 @@ export async function runConnector(opts: RuntimeRunConnectorOptions): Promise { + if (terminalStopRequested()) { + msgQueue.length = 0; + if (!processing) { + return Promise.resolve(); + } + } if (!(msgQueue.length || processing)) { return Promise.resolve(); } @@ -3417,6 +3447,11 @@ export async function runConnector(opts: RuntimeRunConnectorOptions): Promise { + if (terminalStopRequested()) { + msgQueue.length = 0; + notifyQueueDrained(); + return; + } if (processing || !msgQueue.length) { return; } @@ -3428,6 +3463,9 @@ export async function runConnector(opts: RuntimeRunConnectorOptions): Promise { @@ -4793,12 +4836,12 @@ export async function runConnector(opts: RuntimeRunConnectorOptions): Promise idle // first_sync_running setup material or browser login, run in flight -> idle (+ syncing activity) // first_sync_pending setup material or browser login, no run yet -> idle +// first_sync_verified_empty first run proved every in-scope stream empty -> needs_attention +// first_sync_unverified_zero first run observed zero without proof -> needs_attention +// first_sync_unverified_missing_counts terminal run omitted yield counts -> needs_attention // first_sync_failed last run failed, still a draft -> needs_attention // active first ingest accepted records -> healthy // paused owner-paused connection -> idle @@ -50,6 +53,11 @@ export type StaticSecretSetupState = | "first_sync_failed" | "first_sync_pending" | "first_sync_running" + | "first_sync_unverified_missing_counts" + | "first_sync_unverified_zero" + | "first_sync_verified_empty" + /** @deprecated Kept for clients that still decode the pre-disposition state. */ + | "first_sync_zero_yield" | "paused" | "revoked" | "unknown"; @@ -61,6 +69,10 @@ const SETUP_STATE_HEALTH: Record first_sync_failed: "needs_attention", first_sync_pending: "idle", first_sync_running: "idle", + first_sync_unverified_missing_counts: "needs_attention", + first_sync_unverified_zero: "needs_attention", + first_sync_verified_empty: "needs_attention", + first_sync_zero_yield: "idle", paused: "idle", revoked: "idle", unknown: "unknown", @@ -115,11 +127,36 @@ export interface SetupStatusImportReceipt { // (started/in_progress/succeeded/failed/...). `failureReason` is the terminal // failure reason when the run failed. export interface SetupStatusRun { + readonly collectionFacts?: SetupStatusCollectionFacts | null; readonly failureReason?: string | null; readonly finishedAt?: string | null; + readonly recordsEmitted?: number | null; + readonly reportedRecordsEmitted?: number | null; readonly runId: string | null; readonly startedAt?: string | null; readonly status: string | null; + /** False means the generalized writer preserved that the runtime omitted both counts. */ + readonly yieldCountsPresent?: boolean; +} + +export type SetupTerminalDisposition = "verified_empty" | "unverified_missing_counts" | "unverified_zero"; + +export interface SetupStatusCollectionFact { + readonly checkpoint: string | null; + readonly considered: number | null; + readonly covered?: number | null; + readonly pending_detail_gaps: number; + readonly skipped: unknown; + readonly stream: string; +} + +export interface SetupStatusCollectionFacts { + readonly streams: readonly SetupStatusCollectionFact[]; +} + +export interface SetupStatusManifestStream { + readonly name: string; + readonly required?: boolean; } export interface SetupStatusInstance { @@ -139,6 +176,7 @@ export interface ProjectConnectionSetupStatusInput { // The currently in-flight run for this connection, if any // (`controller_active_runs` keyed on connector_instance_id). readonly activeRun: SetupStatusRun | null; + readonly collectionFacts?: SetupStatusCollectionFacts | null; readonly credential: SetupStatusCredentialMetadata | null; // The identity field name (a non-secret manifest setup field marked // `identity: true`), used to pull the account label out of `setupFields`. @@ -149,6 +187,7 @@ export interface ProjectConnectionSetupStatusInput { // The most recent run for this connection (terminal or otherwise), if known. // Used to surface a failed first sync after the run leaves the active table. readonly lastRun: SetupStatusRun | null; + readonly manifestStreams?: readonly SetupStatusManifestStream[]; readonly setupKind?: ConnectionSetupKind; readonly setupMaterial?: SetupStatusMaterialMetadata | null; } @@ -201,8 +240,8 @@ export interface ConnectionSetupStatus { readonly remediation: string; } | null; readonly object: "connection_setup_status"; - // True while the connection is not yet a working connection (draft) and the - // owner still has a setup action to complete or await. + // True while setup is transitional. A draft with a terminal zero-yield run + // or terminal failure is not pending merely because it remains draft. readonly pending: boolean; // The current/last run, for the owner to follow progress or read a failure. readonly run: { @@ -210,6 +249,8 @@ export interface ConnectionSetupStatus { readonly status: string | null; readonly started_at: string | null; readonly finished_at: string | null; + readonly records_emitted: number | null; + readonly reported_records_emitted: number | null; } | null; // True while a first sync run is in flight. readonly running: boolean; @@ -224,11 +265,25 @@ export interface ConnectionSetupStatus { readonly setup_state: StaticSecretSetupState; // The real connector-instance status (draft/active/paused/revoked). readonly status: string; + readonly terminal_setup_disposition: SetupTerminalDisposition | null; readonly updated_at: string | null; } const TERMINAL_FAILURE_STATUSES = new Set(["failed", "errored", "error", "cancelled", "canceled", "aborted"]); const RUNNING_STATUSES = new Set(["started", "in_progress", "running", "pending"]); +const TERMINAL_SUCCESS_STATUSES = new Set(["completed", "complete", "succeeded", "success"]); +const TRANSITIONAL_SETUP_STATES = new Set([ + "awaiting_browser_login", + "awaiting_credential", + "first_sync_pending", + "first_sync_running", +]); +const TERMINAL_DRAFT_STATE_BY_DISPOSITION: Record = { + unverified_missing_counts: "first_sync_unverified_missing_counts", + unverified_zero: "first_sync_unverified_zero", + verified_empty: "first_sync_verified_empty", +}; +const TRUSTED_EMPTY_CHECKPOINTS = new Set(["committed", "disabled"]); function runIsFailure(run: SetupStatusRun | null): boolean { return run !== null && typeof run.status === "string" && TERMINAL_FAILURE_STATUSES.has(run.status); @@ -238,6 +293,129 @@ function runIsRunning(run: SetupStatusRun | null): boolean { return run !== null && typeof run.status === "string" && RUNNING_STATUSES.has(run.status); } +function runIsTerminalSuccess(run: SetupStatusRun | null): boolean { + return run !== null && typeof run.status === "string" && TERMINAL_SUCCESS_STATUSES.has(run.status); +} + +function terminalDraftState( + run: SetupStatusRun | null, + disposition: SetupTerminalDisposition | null +): StaticSecretSetupState | null { + if (runIsFailure(run)) { + return "first_sync_failed"; + } + if (!runIsTerminalSuccess(run) || disposition === null) { + return null; + } + return TERMINAL_DRAFT_STATE_BY_DISPOSITION[disposition]; +} + +function manifestStreamNames(manifestStreams: readonly SetupStatusManifestStream[]): Set { + return new Set(manifestStreams.map((stream) => stream.name).filter((name) => name.length > 0)); +} + +function indexSetupFactsByStream( + facts: readonly SetupStatusCollectionFact[], + knownManifestStreamNames: ReadonlySet +): Map | null { + const factByStream = new Map(); + for (const fact of facts) { + if (!knownManifestStreamNames.has(fact.stream) || factByStream.has(fact.stream)) { + return null; + } + factByStream.set(fact.stream, fact); + } + return factByStream; +} + +function hasRequiredManifestFacts( + manifestStreams: readonly SetupStatusManifestStream[], + factByStream: ReadonlyMap +): boolean { + for (const stream of manifestStreams) { + if (stream.required !== false && stream.name.length > 0 && !factByStream.has(stream.name)) { + return false; + } + } + return true; +} + +function isCoveredEmpty(covered: number | null | undefined): boolean { + return covered === undefined || covered === null || covered === 0; +} + +function isTrustedEmptyFact(fact: SetupStatusCollectionFact): boolean { + if (fact.considered !== 0) { + return false; + } + if (!isCoveredEmpty(fact.covered)) { + return false; + } + if (fact.pending_detail_gaps !== 0) { + return false; + } + if (fact.skipped !== null) { + return false; + } + return TRUSTED_EMPTY_CHECKPOINTS.has(fact.checkpoint ?? ""); +} + +function hasTrustedEmptyStreamFacts( + collectionFacts: SetupStatusCollectionFacts | null | undefined, + manifestStreams: readonly SetupStatusManifestStream[] +): boolean { + const facts = collectionFacts?.streams ?? []; + const names = manifestStreamNames(manifestStreams); + if (facts.length === 0 || names.size === 0) { + return false; + } + const factByStream = indexSetupFactsByStream(facts, names); + if (factByStream === null) { + return false; + } + return hasRequiredManifestFacts(manifestStreams, factByStream) && facts.every(isTrustedEmptyFact); +} + +function isTerminalSuccessStatus(status: string | null): boolean { + return typeof status === "string" && TERMINAL_SUCCESS_STATUSES.has(status); +} + +function observedYieldCount(input: { + readonly recordsEmitted?: number | null | undefined; + readonly reportedRecordsEmitted?: number | null | undefined; + readonly yieldCountsPresent?: boolean | undefined; +}): number | null { + if (input.yieldCountsPresent === false) { + return null; + } + return input.recordsEmitted ?? input.reportedRecordsEmitted ?? null; +} + +function unverifiedDispositionForCount(observedCount: number | null): SetupTerminalDisposition { + return observedCount === 0 ? "unverified_zero" : "unverified_missing_counts"; +} + +export function classifyTerminalSetupDisposition(input: { + readonly collectionFacts?: SetupStatusCollectionFacts | null | undefined; + readonly manifestStreams?: readonly SetupStatusManifestStream[] | undefined; + readonly recordsEmitted?: number | null | undefined; + readonly reportedRecordsEmitted?: number | null | undefined; + readonly status: string | null; + readonly yieldCountsPresent?: boolean | undefined; +}): SetupTerminalDisposition | null { + if (!isTerminalSuccessStatus(input.status)) { + return null; + } + if (hasTrustedEmptyStreamFacts(input.collectionFacts, input.manifestStreams ?? [])) { + return "verified_empty"; + } + return unverifiedDispositionForCount(observedYieldCount(input)); +} + +export function isTransitionalSetupState(state: StaticSecretSetupState): boolean { + return TRANSITIONAL_SETUP_STATES.has(state); +} + function credentialUpdatedAt(credential: SetupStatusCredentialMetadata | null): string | null { return credential?.rotatedAt ?? credential?.capturedAt ?? null; } @@ -286,6 +464,11 @@ const AWAITING_MATERIAL_STATE: Record> = { + active: "active", + paused: "paused", + revoked: "revoked", +}; // The no-material draft state before any first-sync run evidence exists. A // browser-session connection has no stored credential — its owner action is @@ -307,18 +490,14 @@ function deriveSetupState( input: ProjectConnectionSetupStatusInput, setupKind: ConnectionSetupKind, hasSetupMaterial: boolean, - running: boolean + running: boolean, + terminalSetupDisposition: SetupTerminalDisposition | null ): StaticSecretSetupState { // biome-ignore lint/style/useDestructuring: Explicit property or positional access documents this compatibility boundary. const status = input.instance.status; - if (status === "active") { - return "active"; - } - if (status === "paused") { - return "paused"; - } - if (status === "revoked") { - return "revoked"; + const fixedState = FIXED_SETUP_STATE_BY_INSTANCE_STATUS[status]; + if (fixedState) { + return fixedState; } if (status !== "draft") { return "unknown"; @@ -333,14 +512,12 @@ function deriveSetupState( if (running) { return "first_sync_running"; } - // No in-flight run. A terminal failure on the last run is a failed first sync. - if (runIsFailure(input.lastRun)) { - return "first_sync_failed"; - } + // No in-flight run. Terminal run evidence is authoritative for the draft. + const terminalState = terminalDraftState(input.lastRun, terminalSetupDisposition); // Credential/browser-login material present (or run evidence exists for a // browser session), run queued or just-submitted but not yet running and // not yet failed: the first sync is pending. - return "first_sync_pending"; + return terminalState ?? "first_sync_pending"; } function defaultSetupMaterial( @@ -412,59 +589,158 @@ function projectImportReceipt( }; } -export function projectConnectionSetupStatus(input: ProjectConnectionSetupStatusInput): ConnectionSetupStatus { +function terminalSetupDispositionForInput(input: ProjectConnectionSetupStatusInput): SetupTerminalDisposition | null { + if (input.instance.status !== "draft") { + return null; + } + return classifyTerminalSetupDisposition({ + collectionFacts: input.lastRun?.collectionFacts ?? input.collectionFacts, + manifestStreams: input.manifestStreams, + recordsEmitted: input.lastRun?.recordsEmitted, + reportedRecordsEmitted: input.lastRun?.reportedRecordsEmitted, + status: input.lastRun?.status ?? null, + yieldCountsPresent: input.lastRun?.yieldCountsPresent, + }); +} + +const SETUP_ERROR_REASON_BY_DISPOSITION: Partial> = { + unverified_missing_counts: "first_sync_unverified_missing_counts", + unverified_zero: "first_sync_unverified_zero", +}; +const SETUP_ERROR_REMEDIATION_BY_DISPOSITION: Partial> = { + unverified_missing_counts: + "Review the connection before retrying; the first sync did not leave durable count evidence.", + unverified_zero: "Review the connection and retry the first sync if you expected records.", +}; + +function setupErrorReason( + input: ProjectConnectionSetupStatusInput, + run: SetupStatusRun | null, + setupState: StaticSecretSetupState, + terminalSetupDisposition: SetupTerminalDisposition | null +): string | null { + if (setupState === "first_sync_failed") { + return run?.failureReason ?? input.lastRun?.failureReason ?? "first_sync_failed"; + } + return terminalSetupDisposition === null + ? null + : (SETUP_ERROR_REASON_BY_DISPOSITION[terminalSetupDisposition] ?? null); +} + +function setupErrorRemediation( + reason: string, + setupKind: ConnectionSetupKind, + terminalSetupDisposition: SetupTerminalDisposition | null +): string { + const dispositionRemediation = + terminalSetupDisposition === null + ? null + : (SETUP_ERROR_REMEDIATION_BY_DISPOSITION[terminalSetupDisposition] ?? null); + return dispositionRemediation ?? remediationForReason(reason, setupKind); +} + +function projectSetupLastError( + input: ProjectConnectionSetupStatusInput, + run: SetupStatusRun | null, + setupKind: ConnectionSetupKind, + setupState: StaticSecretSetupState, + terminalSetupDisposition: SetupTerminalDisposition | null +): ConnectionSetupStatus["last_error"] { + const reason = setupErrorReason(input, run, setupState, terminalSetupDisposition); + if (reason === null) { + return null; + } + return { reason, remediation: setupErrorRemediation(reason, setupKind, terminalSetupDisposition) }; +} + +function setupRunIsRunning(activeRun: SetupStatusRun | null, lastRun: SetupStatusRun | null): boolean { + return runIsRunning(activeRun) || (activeRun === null && runIsRunning(lastRun)); +} + +function projectSetupRun(run: SetupStatusRun | null): ConnectionSetupStatus["run"] { + if (run === null) { + return null; + } + return { + finished_at: run.finishedAt ?? null, + records_emitted: nullable(run.recordsEmitted), + reported_records_emitted: nullable(run.reportedRecordsEmitted), + run_id: run.runId, + started_at: run.startedAt ?? null, + status: run.status, + }; +} + +function projectSetupCredential(credential: SetupStatusCredentialMetadata | null): ConnectionSetupStatus["credential"] { + return { + captured_at: credential?.capturedAt ?? null, + credential_kind: credential?.credentialKind ?? null, + present: credential?.present === true, + rotated_at: credential?.rotatedAt ?? null, + }; +} + +function projectSetupMaterial(material: SetupStatusMaterialMetadata): ConnectionSetupStatus["setup_material"] { + return { + captured_at: material.capturedAt ?? null, + kind: material.kind, + label: material.label, + present: material.present, + }; +} + +interface SetupStatusProjection { + readonly lastError: ConnectionSetupStatus["last_error"]; + readonly material: SetupStatusMaterialMetadata; + readonly run: SetupStatusRun | null; + readonly running: boolean; + readonly setupKind: ConnectionSetupKind; + readonly setupState: StaticSecretSetupState; + readonly terminalSetupDisposition: SetupTerminalDisposition | null; +} + +function buildSetupStatusProjection(input: ProjectConnectionSetupStatusInput): SetupStatusProjection { const setupKind = input.setupKind ?? "static_secret"; const material = input.setupMaterial ?? defaultSetupMaterial(setupKind, input.credential); const hasSetupMaterial = material.present === true; // A run is "running" when the active-run table holds it OR the last-run // summary still reports a non-terminal status (covers the window between // submit and the active-run row landing). - const running = runIsRunning(input.activeRun) || (input.activeRun === null && runIsRunning(input.lastRun)); - const setupState = deriveSetupState(input, setupKind, hasSetupMaterial, running); + const running = setupRunIsRunning(input.activeRun, input.lastRun); + const terminalSetupDisposition = terminalSetupDispositionForInput(input); + const setupState = deriveSetupState(input, setupKind, hasSetupMaterial, running, terminalSetupDisposition); const run = input.activeRun ?? input.lastRun ?? null; + const lastError = projectSetupLastError(input, run, setupKind, setupState, terminalSetupDisposition); + return { lastError, material, run, running, setupKind, setupState, terminalSetupDisposition }; +} - const failed = setupState === "first_sync_failed"; - const failureReason = - (failed ? (run?.failureReason ?? input.lastRun?.failureReason) : null) || (failed ? "first_sync_failed" : null); - const lastError = failureReason - ? { reason: failureReason, remediation: remediationForReason(failureReason, setupKind) } - : null; +export function projectConnectionSetupStatus(input: ProjectConnectionSetupStatusInput): ConnectionSetupStatus { + const { lastError, material, run, running, setupKind, setupState, terminalSetupDisposition } = + buildSetupStatusProjection(input); return { account_identity: accountIdentity(input), connection_id: input.instance.connectorInstanceId, connector_id: input.instance.connectorId, created_at: input.instance.createdAt, - credential: { - captured_at: input.credential?.capturedAt ?? null, - credential_kind: input.credential?.credentialKind ?? null, - present: input.credential?.present === true, - rotated_at: input.credential?.rotatedAt ?? null, - }, + credential: projectSetupCredential(input.credential), display_name: input.instance.displayName, health_state: SETUP_STATE_HEALTH[setupState], import_receipt: projectImportReceipt(setupKind, input.importReceipt), last_error: lastError, object: "connection_setup_status", - pending: input.instance.status === "draft", - run: run - ? { - finished_at: run.finishedAt ?? null, - run_id: run.runId, - started_at: run.startedAt ?? null, - status: run.status, - } - : null, + // Pending is a lifecycle state, not a storage-status synonym. A draft with + // a terminal first-sync failure must stop polling and expose the failure as + // terminal; otherwise every owner surface can remain indefinitely pending + // after the run has already failed. + pending: isTransitionalSetupState(setupState), + run: projectSetupRun(run), running, setup_kind: setupKind, - setup_material: { - captured_at: material.capturedAt ?? null, - kind: material.kind, - label: material.label, - present: material.present, - }, + setup_material: projectSetupMaterial(material), setup_state: setupState, status: input.instance.status, + terminal_setup_disposition: terminalSetupDisposition, updated_at: input.instance.updatedAt, }; } diff --git a/reference-implementation/scripts/child-process-output.ts b/reference-implementation/scripts/child-process-output.ts new file mode 100644 index 000000000..9234791d1 --- /dev/null +++ b/reference-implementation/scripts/child-process-output.ts @@ -0,0 +1,24 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import type { EventEmitter } from "node:events"; +import type { Readable } from "node:stream"; + +interface ChildWithPipedOutput extends EventEmitter { + stderr: Readable | null; + stdout: Readable | null; +} + +/** Collect both output streams until Node confirms the process and stdio are closed. */ +export function collectChildProcessOutput(child: ChildWithPipedOutput): Promise { + return new Promise((resolve) => { + let output = ""; + child.stdout?.on("data", (chunk: Buffer) => { + output += chunk.toString(); + }); + child.stderr?.on("data", (chunk: Buffer) => { + output += chunk.toString(); + }); + child.once("close", () => resolve(output)); + }); +} diff --git a/reference-implementation/scripts/direct-prepare-allowlist.ts b/reference-implementation/scripts/direct-prepare-allowlist.ts index 301a76b12..a52a3f01e 100644 --- a/reference-implementation/scripts/direct-prepare-allowlist.ts +++ b/reference-implementation/scripts/direct-prepare-allowlist.ts @@ -95,7 +95,7 @@ export const DIRECT_PREPARE_ALLOWLIST: readonly DirectPrepareAllowlistEntry[] = }, { category: "grandfathered_pre_wrapper", - line: 7272, + line: 7600, note: "backfillSqliteRecordSemanticTimesForManifest: prepared UPDATE records SET semantic_time = ? (per-record backfill write inside the writeTransaction)", path: "reference-implementation/server/records.ts", }, diff --git a/reference-implementation/scripts/quality-ratchet/mass-baseline.json b/reference-implementation/scripts/quality-ratchet/mass-baseline.json index 11f3a6178..7210e4de7 100644 --- a/reference-implementation/scripts/quality-ratchet/mass-baseline.json +++ b/reference-implementation/scripts/quality-ratchet/mass-baseline.json @@ -141,7 +141,7 @@ "server/routes/ref-run-status.ts": 18, "server/routes/ref-spine-correlations.ts": 2, "server/routes/ref-spine-timelines.ts": 12, - "server/routes/ref-static-secret-credentials.ts": 25, + "server/routes/ref-static-secret-credentials.ts": 24, "server/routes/ref-static-secret-draft-connection.ts": 21, "server/routes/ref-static-secret-setup-status.ts": 9, "server/routes/root-and-discovery.ts": 27, @@ -189,7 +189,7 @@ "server/version-disposition.ts": 6, "server/web-push-notifications.ts": 11 }, - "total": 6960, + "total": 6959, "meta": { "biomeVersion": "2.5.6", "maxAllowedComplexity": 5 diff --git a/reference-implementation/scripts/quality-ratchet/mass-justifications.json b/reference-implementation/scripts/quality-ratchet/mass-justifications.json index bf693b258..b7e497f09 100644 --- a/reference-implementation/scripts/quality-ratchet/mass-justifications.json +++ b/reference-implementation/scripts/quality-ratchet/mass-justifications.json @@ -1,4 +1,9 @@ { + "runtime/index.ts": { + "allowed_mass": 422, + "reason": "Owner-run cancellation terminalization fix (2026-08-07): after the proven queue-drain repro, the runtime must clear buffered child messages, abort the active ingest transport, and let cancellation win over a queued DONE while preserving the already-started ingest. The queue-drop/in-flight-preservation oracle in test/runtime-cancel-queue.test.ts distinguishes cooperative and SIGTERM-ignoring children by requiring no ingest after abort, at least one ingest before abort, and sub-1.5s terminalization; runtime-cancel-run.test.ts separately proves pre-cancel records remain durable and staged state is not committed. The terminalStopRequested extraction lowers the measured mass from 428 to 422; further decomposition would split the queue-drop, transport-cancel, child-close ordering invariant across shallow helpers.", + "date": "2026-08-07" + }, "server/routes/run-interaction.ts": { "allowed_mass": 29, "reason": "Direct-CDP terminal-barrier closure (2026-08-05): run-final cleanup must invoke the presentation terminal barrier for every run, including direct-CDP runs without a managed n.eko lease, before clearing the run nonce. The added hook is the single lifecycle seam that prevents a connector finalization path from skipping target purge; extracting it would obscure the required ordering.", @@ -165,9 +170,14 @@ "date": "2026-07-27" }, "server/stores/connector-instance-credential-store.ts": { - "allowed_mass": 7, - "reason": "Auth/store leaf TypeScript migration (2026-07-27): encrypted credential persistence retains validation for credential kinds, ownership, status, and secret recovery. The checks protect the fail-closed store contract and both backend paths; replacing them with casts would weaken the typed storage boundary.", - "date": "2026-07-27" + "allowed_mass": 13, + "reason": "Auth/store leaf TypeScript migration (2026-07-27): encrypted credential persistence retains validation for credential kinds, ownership, status, and secret recovery. The checks protect the fail-closed store contract and both backend paths; replacing them with casts would weaken the typed storage boundary. PR #84 static-identity fail-closed fix (2026-08-07): re-measuring this file surfaced the true mass was already 13, not 7 -- the committed baseline had drifted stale since the 2026-07-27 entry and was never regenerated by an intervening change to this file (confirmed via measure-mass.ts against the unmodified pre-PR HEAD: 13, identical to post-PR). The new fingerprintCandidate accessor added by this PR is a single-line pure delegation to the existing cipher() factory and contributes zero measured mass; this entry corrects the stale ceiling to match reality rather than papering over an unrelated pre-existing drift.", + "date": "2026-08-07" + }, + "server/static-secret-identity.ts": { + "allowed_mass": 4, + "reason": "PR #84 P1 fix (2026-08-07): assertStaticSecretActiveCredentialReplacementAllowed is the terminal authority for replacing the credential behind an active static-secret connection. It must fail closed rather than infer permission from absent data: skip only non-active/non-pipeline/no-existing-credential rows, then require either a synchronous-probe-verified identity matching the durable verified_identity on record, or (when no such durable identity exists to compare against) an exact key-derived fingerprint match proving the submitted secret is byte-for-byte the one already stored -- never trusting owner-typed setup_fields for either side of that comparison, since they are trivially resubmittable alongside a stolen credential. Each branch is a distinct, load-bearing security predicate proven by dedicated discriminating tests (active-verified mismatch, first-sync no-probe fail-closed, probed-identity-alone-insufficient, same-claimed-setup-field-different-secret x2); collapsing them into a shallower helper would hide which channel of proof was actually satisfied.", + "date": "2026-08-07" }, "server/stores/credential-encryption.ts": { "allowed_mass": 12, @@ -205,9 +215,14 @@ "date": "2026-07-28" }, "server/records.ts": { - "allowed_mass": 456, - "reason": "TypeScript migration closure (2026-07-28): server/records.ts strict-plus type fixes add 7 conditional branches for null/undefined guards (string|null|undefined -> string narrowing), DevicePreparePlanEntry/DeviceRecordPlanEntry alignment, and exactOptionalPropertyTypes spreads. Each branch is an honest narrowing guard that expresses a real nullable boundary; the alternative is a cast or suppression that violates the strict-plus baseline.", - "date": "2026-07-28" + "allowed_mass": 490, + "reason": "TypeScript migration closure (2026-07-28): server/records.ts strict-plus type fixes add 7 conditional branches for null/undefined guards (string|null|undefined -> string narrowing), DevicePreparePlanEntry/DeviceRecordPlanEntry alignment, and exactOptionalPropertyTypes spreads. Each branch is an honest narrowing guard that expresses a real nullable boundary; the alternative is a cast or suppression that violates the strict-plus baseline. Shared ingest throughput fix (2026-08-06): ingestRecords holds one connector-instance ownership capability across the bounded request while preserving the existing ordered per-record durable/index phase, per-record failure isolation, and dual-backend dispatch. The added loop and outcome guards are the minimum control flow for those invariants and are proven by the focused two-stream fault oracle; removing them would either reacquire the bottleneck per record or collapse a failed record into a batch-wide rollback. Follow-up ordering correction (2026-08-06): the generic afterRecord completion callback is awaited between each record's storage phase and the next record, so acquisition provenance cannot be moved after the batch; its explicit error boundary and ordered callback path are proven by the route-level event-order oracle. Batch liveness correction (2026-08-07): ingestRecords keeps one authoritative connector-instance fence and schedules ordered derived-index work on a separate per-instance lane behind a release barrier, preserving batch lock reuse while allowing blob writers to proceed during embedding/index waits. The lane, per-record derived-error isolation, and release-barrier ordering are proven by the SQLite blob-route and lock-count oracles; removing them either restores blob starvation or lets newer derived work overtake older durable batches. Run-admission fence (2026-08-07, harden-ingest-run-admission-fence): PR #84 red-team found that a durable ingest write already admitted into the per-connector-instance write coordinator before owner-cancellation still committed after run_history recorded the run terminal -- runtime/index.ts's own cancellation is a client-side AbortSignal that cannot retroactively refuse a write the server already accepted. assertSqliteRunStillAdmitted checks run_history for (runId, connectorInstanceId) inside the same writeTransaction as the durable mutation it guards, so the check-then-write is atomic on SQLite's single writer connection; it fails CLOSED on a run_id with no matching row (spoofed/mistyped/foreign), not admitted, since a genuine run-bound write is always preceded by an awaited run.started spine insert. Opt-in via options.runId -- owner/API ingestion that never threads a run_id is unaffected. Checked once per record inside ingestRecordsWithinCoordinator's existing loop (not once per batch): the coordinator lock ingestRecords already holds for the whole batch does not serialize against the run's terminal write (a fully separate call path, runtime/index.ts's proc.on('close', ...) handler), and making the terminal writer share that lock was evaluated and rejected -- it would force cancellation to wait behind an unbounded in-flight batch, reintroducing the exact defect commit 73708a720 fixed and violating the sub-second terminalization contract runtime-cancel-run.test.ts/runtime-cancel-queue.test.ts already enforce. Measured cost: ~20.5us/record (~13% relative) on SQLite, a warmed 5-round 200-record benchmark. Proven by test/runtime-cancel-ingest-commit-boundary-probe.test.ts (9 tests): cancel-before-release refusal, release-before-cancel preservation, cross-connection run_id-collision spoof resistance, fail-closed on an unrecognized run_id, HTTP wire-through from the real POST /v1/ingest/:stream route, and Postgres parity for all of the above.", + "date": "2026-08-07" + }, + "server/routes/rs-mutation.ts": { + "allowed_mass": 102, + "reason": "Shared ingest throughput fix (2026-08-06): the route adapter adds the optional common batch capability while retaining the existing per-line fallback, malformed-outcome rejection, ordered acquisition provenance, and request-scoped namespace resolution. These branches are the load-bearing boundary that maps the batch storage outcomes back to the canonical line-level error contract; removing them would either regress to one coordinator acquisition per record or hide provenance/index failures.", + "date": "2026-08-06" }, "server/connector-summary-evidence-engine.ts": { "allowed_mass": 75, diff --git a/reference-implementation/scripts/run-tests.lifecycle.test.ts b/reference-implementation/scripts/run-tests.lifecycle.test.ts new file mode 100644 index 000000000..7f8a2bebd --- /dev/null +++ b/reference-implementation/scripts/run-tests.lifecycle.test.ts @@ -0,0 +1,25 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { EventEmitter } from "node:events"; +import { PassThrough } from "node:stream"; +import test from "node:test"; +import { collectChildProcessOutput } from "./child-process-output.ts"; + +test("test output is retained when process exit precedes stdio close", async () => { + const child = new EventEmitter() as EventEmitter & { + stderr: PassThrough; + stdout: PassThrough; + }; + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + + const captured = collectChildProcessOutput(child); + child.emit("exit", 0, null); + child.stdout.end("stdout after exit\n"); + child.stderr.end("stderr after exit\n"); + child.emit("close", 0, null); + + assert.equal(await captured, "stdout after exit\nstderr after exit\n"); +}); diff --git a/reference-implementation/scripts/run-tests.ts b/reference-implementation/scripts/run-tests.ts index 9749ee31a..6549170e0 100644 --- a/reference-implementation/scripts/run-tests.ts +++ b/reference-implementation/scripts/run-tests.ts @@ -23,6 +23,7 @@ import { dedicatedPostgresTestUrl, isDedicatedPostgresTestDatabaseName, } from "../test/helpers/dedicated-postgres-test-url.ts"; +import { collectChildProcessOutput } from "./child-process-output.ts"; import { deriveDedicatedPostgresDbNameForFile } from "./dedicated-postgres-db-name.ts"; import type { ProcessEnvLike } from "./test-env.ts"; import { buildScrubbedTestEnv } from "./test-env.ts"; @@ -293,8 +294,9 @@ async function runNodeTest(filePath: string, extraArgs: string[]): Promise { - output += chunk.toString(); - }); - child.stderr?.on("data", (chunk: Buffer) => { - output += chunk.toString(); - }); - - child.on("error", (err) => { + const settleAfterRelease = async (finish: () => void | Promise) => { + if (settled) { + return; + } + settled = true; clearTimeout(watchdog); if (allocation) { - allocation.release().finally(() => reject(err)); - } else { - reject(err); + await allocation.release(); } + await finish(); + }; + + child.on("error", (err) => { + settleAfterRelease(() => reject(err)).catch(reject); }); - child.on("exit", (code, signal) => { - clearTimeout(watchdog); - const finish = () => { + child.on("close", (code, signal) => { + settleAfterRelease(async () => { + const output = await outputPromise; if (timedOut) { reject(new Error(`Test process for ${filePath} timed out after ${PER_FILE_TIMEOUT_MS}ms and was killed`)); return; @@ -342,12 +344,7 @@ async function runNodeTest(filePath: string, extraArgs: string[]): Promise ${filePath}\n${output}`, }); - }; - if (allocation) { - allocation.release().finally(finish); - } else { - finish(); - } + }).catch(reject); }); }); } diff --git a/reference-implementation/server/connection-setup-plan.ts b/reference-implementation/server/connection-setup-plan.ts index 834861015..dd424682d 100644 --- a/reference-implementation/server/connection-setup-plan.ts +++ b/reference-implementation/server/connection-setup-plan.ts @@ -39,6 +39,7 @@ export type ConnectorCatalogDisposition = | "manual_upload_connect" | "manual_upload_pending" | "provider_auth_deployment_blocked" + | "provider_auth_connect" | "provider_auth_proof_gated" | "api_network_unsupported" | "unknown_unsupported"; @@ -277,8 +278,8 @@ export function isProviderAuthLifecycleProven(connectorKey: string): boolean { // github — run_1781131195649 completed/succeeded, env-free container // + run_1781131489458 trigger_kind=scheduled unattended succeeded (4 records) // slack — run_1781131204868 completed/succeeded, env-free container -// (ynab store path also proven; token is provider-side dead — not a capture-path failure) -export const STATIC_SECRET_LIVE_PROVEN_CONNECTOR_KEYS = ["gmail", "github", "slack"] as const; +// ynab — store path proven; captured credentials validated successfully +export const STATIC_SECRET_LIVE_PROVEN_CONNECTOR_KEYS = ["gmail", "github", "slack", "ynab"] as const; export type StaticSecretLiveProvenConnector = (typeof STATIC_SECRET_LIVE_PROVEN_CONNECTOR_KEYS)[number]; @@ -775,7 +776,7 @@ function buildProviderAuthorizationSetupPlan(ctx: ConnectionSetupPlanContext): C const lifecycleProven = !deploymentBlocked && isProviderAuthLifecycleProven(ctx.connectorKey); if (lifecycleProven) { return { - catalogDisposition: "provider_auth_proof_gated", + catalogDisposition: "provider_auth_connect", connectorKey: ctx.connectorKey, connectorModality: ctx.connectorModality, deploymentReadiness: ctx.deploymentReadiness, diff --git a/reference-implementation/server/db.ts b/reference-implementation/server/db.ts index bad66451b..3797f912c 100644 --- a/reference-implementation/server/db.ts +++ b/reference-implementation/server/db.ts @@ -1556,7 +1556,8 @@ CREATE INDEX IF NOT EXISTS idx_connector_attention_dedupe -- additive; existing rows are backfilled in initDb post-schema. New -- inserts compute event_seq via a (SELECT MAX(event_seq) + 1 FROM ...) -- subquery inside the INSERT, which is safe under SQLite's single-writer --- lock model. +-- lock model. The startup backfill preserves any sequence already assigned +-- by an interleaved writer and allocates the remaining legacy rows above it. -- Spec: openspec/changes/replace-spine-rowid-cursor-with-event-seq/specs/ -- reference-implementation-architecture/spec.md CREATE TABLE IF NOT EXISTS spine_events ( @@ -5082,14 +5083,31 @@ export function initDb(path = ":memory:", opts: InitDbOptions = {}): DatabaseHan }); // Disclosure-spine `event_seq` migration. Pre-existing reference DBs were // created before `event_seq` existed; add the column non-destructively and - // seed it for any rows that lack a value. The seed orders by `rowid` — - // SQLite's physical row identity at the moment of backfill — purely as a - // one-shot reconstruction of historical append order. After backfill, - // `event_seq` is the only ordering surface readers and cursors consult; - // the cursor contract no longer reads `rowid`. + // seed any rows that lack a value. An interrupted boot can leave the column + // present while a concurrent writer has already assigned sequences, so + // preserve those values and allocate NULL rows above the current maximum, + // ordered by `rowid`. After backfill, `event_seq` is the only ordering + // surface readers and cursors consult; the cursor contract no longer reads + // `rowid`. runWithSqliteBusyRetrySync(() => addColumnIfMissing(raw, "spine_events", "event_seq", "INTEGER")); runWithSqliteBusyRetrySync(() => { - raw.exec("UPDATE spine_events SET event_seq = rowid WHERE event_seq IS NULL"); + raw.exec(` + WITH pending AS ( + SELECT rowid, ROW_NUMBER() OVER (ORDER BY rowid) AS ordinal + FROM spine_events + WHERE event_seq IS NULL + ), current AS ( + SELECT COALESCE(MAX(event_seq), 0) AS max_seq + FROM spine_events + ) + UPDATE spine_events + SET event_seq = ( + SELECT current.max_seq + pending.ordinal + FROM pending, current + WHERE pending.rowid = spine_events.rowid + ) + WHERE rowid IN (SELECT rowid FROM pending) + `); }); runWithSqliteBusyRetrySync(() => migrateSpineSourceColumns(raw, opts)); // blob_bindings gains a json_path column (RFC 6901 JSON Pointer or diff --git a/reference-implementation/server/index.ts b/reference-implementation/server/index.ts index b6c8e797e..f83f1b011 100644 --- a/reference-implementation/server/index.ts +++ b/reference-implementation/server/index.ts @@ -213,6 +213,7 @@ import { import { buildRecordVersionStatsEnvelope } from "./record-version-stats.ts"; import { aggregateRecordsAcrossBindings, + assertConnectorInstanceWritable, deleteAllRecords, deleteConnectionRecordRowsPostgres, deleteConnectionRecordRowsSqlite, @@ -228,6 +229,7 @@ import { getRecordFieldWindowAcrossBindings, getSyncState, ingestRecord, + ingestRecords, listAllStreams, listDatasetSummaryStreamProjectionSeeds, listDatasetTopConnectorCandidates, @@ -308,7 +310,11 @@ import { mountRefSchedules, mountRefSearch, } from "./routes/ref-admin.ts"; -import { mountRefBrowserEnrollmentShell } from "./routes/ref-browser-enrollment-shell.ts"; +import { + type BrowserEnrollmentShellSourceBinding, + mountRefBrowserEnrollmentShell, + promoteBrowserEnrollmentShellBinding, +} from "./routes/ref-browser-enrollment-shell.ts"; import { mountRefConnectionDelete, mountRefConnectionDetail, @@ -357,6 +363,7 @@ import { mountRefDeviceExporterLocalCollectorGaps, mountRefDeviceExporterLocalCollectorGapsRecovered, mountRefDeviceExporterRevoke, + mountRefDeviceExporterSelfRevoke, mountRefDeviceExporterSourceInstanceStateGet, mountRefDeviceExporterSourceInstanceStatePut, mountRefDeviceExporterSourceInstances, @@ -373,7 +380,11 @@ import { mountRefGrantPackagesList, mountRefGrantPackagesRevoke, } from "./routes/ref-grants.ts"; -import { mountRefManualUploadDraftConnection } from "./routes/ref-manual-upload-draft-connection.ts"; +import { + type ManualUploadDraftSourceBinding, + mountRefManualUploadDraftConnection, + promoteManualUploadDraftBinding, +} from "./routes/ref-manual-upload-draft-connection.ts"; import { createInProcessPendingAuthStore, mountRefProviderAuthCallback, @@ -384,7 +395,11 @@ import { mountRefRunStatus } from "./routes/ref-run-status.ts"; import { mountRefGrants, mountRefRuns, mountRefTraces } from "./routes/ref-spine-correlations.ts"; import { mountRefGrantTimeline, mountRefRunTimeline, mountRefTraceTimeline } from "./routes/ref-spine-timelines.ts"; import { mountRefStaticSecretCredentialCapture } from "./routes/ref-static-secret-credentials.ts"; -import { mountRefStaticSecretDraftConnection } from "./routes/ref-static-secret-draft-connection.ts"; +import { + mountRefStaticSecretDraftConnection, + promoteStaticSecretDraftBinding, + type StaticSecretDraftSourceBinding, +} from "./routes/ref-static-secret-draft-connection.ts"; import { mountRefStaticSecretSetupStatus } from "./routes/ref-static-secret-setup-status.ts"; import { mountAsAuthorizationServerMetadata, @@ -1160,12 +1175,6 @@ function readReferenceLocalConnectorCatalogManifest(connectorId: string) { } } -function listReferenceLocalConnectorCatalogManifests() { - return Array.from(REFERENCE_LOCAL_CONNECTOR_CATALOG_MANIFESTS.keys()) - .map((connectorId) => readReferenceLocalConnectorCatalogManifest(connectorId)) - .filter(Boolean); -} - async function ensureReferenceConnectorCatalogEntry( connectorId: string, connectorDisplayName: string | null | undefined @@ -2467,6 +2476,78 @@ async function resolveRegisteredConnectorManifest(connectorId: string) { return manifest; } +// Keyed by the current setup binding kind; each builder returns the durable +// replacement binding. A new setup-binding kind only needs an entry here. +const SETUP_BINDING_PROMOTIONS: Record< + string, + (currentBinding: Record, now: string) => Record +> = { + browser_enrollment_shell: (binding, now) => + promoteBrowserEnrollmentShellBinding( + binding as unknown as BrowserEnrollmentShellSourceBinding, + now + ) as unknown as Record, + manual_upload_draft: (binding, now) => + promoteManualUploadDraftBinding(binding as unknown as ManualUploadDraftSourceBinding, now) as unknown as Record< + string, + unknown + >, + static_secret_draft: (binding, now) => + promoteStaticSecretDraftBinding(binding as unknown as StaticSecretDraftSourceBinding, now) as unknown as Record< + string, + unknown + >, +}; + +interface ActivateDraftConnectionStore { + activateDraft: (connectorInstanceId: string) => unknown | Promise; + get: ( + connectorInstanceId: string + ) => + | { status?: string; sourceBinding?: unknown } + | null + | Promise<{ status?: string; sourceBinding?: unknown } | null>; + promoteSetupBinding: ( + connectorInstanceId: string, + args: { fromKind: string; sourceBinding: Record; updatedAt: string } + ) => { instance: unknown; promoted: boolean } | Promise<{ instance: unknown; promoted: boolean }>; +} + +// Extracted from its `rsMutationContext.activateDraftConnection` call site +// so it's unit-testable against a fake store without a full server — see +// test/activate-draft-connection.test.ts. +type ActivationScheduleAttacher = ( + instance: { connectorId?: string; connectorInstanceId?: string; status?: string } | null | undefined +) => Promise; + +export async function activateDraftConnection( + connectorInstanceId: string, + store: ActivateDraftConnectionStore, + attachSchedule: ActivationScheduleAttacher +): Promise { + const current = await store.get(connectorInstanceId); + const bindingKind = + current?.sourceBinding && typeof current.sourceBinding === "object" + ? (current.sourceBinding as { kind?: unknown }).kind + : null; + const promotion = typeof bindingKind === "string" ? SETUP_BINDING_PROMOTIONS[bindingKind] : undefined; + const now = new Date().toISOString(); + const { instance, promoted } = + current?.status === "draft" && promotion + ? await store.promoteSetupBinding(connectorInstanceId, { + fromKind: bindingKind as string, + sourceBinding: promotion(current.sourceBinding as Record, now), + updatedAt: now, + }) + : { instance: await store.activateDraft(connectorInstanceId), promoted: true }; + if (!promoted) { + return null; + } + return await attachSchedule( + instance as { connectorId?: string; connectorInstanceId?: string; status?: string } | null | undefined + ); +} + function createActivationScheduleAttacher(controller: unknown) { return async ( instance: { connectorId?: string; connectorInstanceId?: string; status?: string } | null | undefined @@ -3331,8 +3412,14 @@ async function persistContentAddressedBlob({ mimeType: string; data: Buffer; }) { - return withConnectorInstanceWrite(connectorInstanceId, (ownership) => - persistContentAddressedBlobWithinFence({ + return withConnectorInstanceWrite(connectorInstanceId, async (ownership) => { + // `persistContentAddressedBlob` is only reached via the HTTP blob-write + // route (not called directly by tests, unlike `ingestRecord`), so the + // existence re-check runs unconditionally, inside the fence, right + // before the write. See `assertConnectorInstanceWritable` in records.ts + // for the delete/write TOCTOU this closes. + await assertConnectorInstanceWritable(connectorInstanceId); + return persistContentAddressedBlobWithinFence({ connectorId, connectorInstanceId, coordinatorOwnership: ownership, @@ -3340,8 +3427,8 @@ async function persistContentAddressedBlob({ mimeType, recordKey, stream, - }) - ); + }); + }); } async function persistContentAddressedBlobWithinFence({ @@ -5093,9 +5180,11 @@ export function buildAsApp(opts: ServerOpts = {}) { createRequestAcquisitionBatchStore, createRequestConnectorInstanceCredentialStore, createRequestConnectorInstanceStore, + getLatestRunHistoryForProductByConnectionId: (connectorInstanceId: string) => + getDefaultSchedulerStore().getLatestRunHistoryForProductByConnectionId?.(connectorInstanceId) ?? null, getOwnerSubjectId, - getRunStartedAt: async (runId: string) => (await getRunStartedEvent(runId))?.occurred_at ?? null, - getRunTerminalStatus, + getProductRunHistoryForConnectionRunId: (connectorInstanceId: string, runId: string) => + getDefaultSchedulerStore().getProductRunHistoryForConnectionRunId?.(connectorInstanceId, runId) ?? null, handleError, pdppError, requireOwnerSession: ownerAuth.requireOwnerSession, @@ -5168,6 +5257,7 @@ export function buildAsApp(opts: ServerOpts = {}) { // enforcement; the adapter owns all route logic. const refDeviceExportersContext = { acceptedCollectorProtocolVersions, + assertConnectorInstanceWritable, canonicalConnectorKey, createRequestConnectorInstanceStore, DeviceBatchConflictError, @@ -5225,6 +5315,10 @@ export function buildAsApp(opts: ServerOpts = {}) { app, refDeviceExportersContext as unknown as Parameters[1] ); + mountRefDeviceExporterSelfRevoke( + app, + refDeviceExportersContext as unknown as Parameters[1] + ); mountRefDeviceExporterHeartbeat( app, refDeviceExportersContext as unknown as Parameters[1] @@ -5612,13 +5706,13 @@ function buildRsApp(opts: ServerOpts = {}) { // before mountRsReadQueries) and mountRsBlobsUpload / mountRsMutation // (registered after) share the same context object. const rsMutationContext = { - // First-ingest activation for static-secret drafts: flip draft → active - // once a record lands. No-op on a non-draft row. See - // add-static-secret-owner-session-connect-path design Decision 5. - activateDraftConnection: async (connectorInstanceId: string) => { - const instance = await createRequestConnectorInstanceStore().activateDraft(connectorInstanceId); - return await attachActivationScheduleForConnection(instance); - }, + // See add-static-secret-owner-session-connect-path design Decision 5. + activateDraftConnection: (connectorInstanceId: string) => + activateDraftConnection( + connectorInstanceId, + createRequestConnectorInstanceStore(), + attachActivationScheduleForConnection + ), buildMutationContext, buildStateContext, deleteAllRecords, @@ -5635,8 +5729,24 @@ function buildRsApp(opts: ServerOpts = {}) { )[0] ?? null, getSyncState, handleError, - ingestRecord: (target: unknown, record: unknown) => - ingestRecord(target as Parameters[0], record as Parameters[1]), + ingestRecord: ( + target: unknown, + record: unknown, + options?: { requireConnectionAdmission?: boolean; runId?: string | null } + ) => + ingestRecord(target as Parameters[0], record as Parameters[1], options), + ingestRecords: ( + target: unknown, + records: readonly unknown[], + afterRecord: ((record: unknown, outcome: unknown) => Promise) | undefined, + options?: { requireConnectionAdmission?: boolean; runId?: string | null } + ) => + ingestRecords( + target as Parameters[0], + records as Parameters[1], + afterRecord, + options + ), // Same cache the mutation routes below already invalidate on every other // connection-mutating action (revoke, reactivate, schedule, run, rename, // delete). `maybeActivateDraftAfterIngest` (rs-mutation.ts) calls this @@ -6244,17 +6354,16 @@ function buildRsApp(opts: ServerOpts = {}) { // GET /v1/owner/connector-templates is the bearer-authed owner-agent template // catalog. It separates connector implementation metadata from configured // connection instances, embeds related connection summaries, and reports - // template-level `initiate_connection` support truthfully: proven - // local-collector templates can create an enrollment intent; browser-bound and - // API/network-only templates name the missing primitive instead of pretending - // an owner bearer can add a provider account. + // template-level `initiate_connection` support truthfully: only registered + // templates whose server-owned listing, proof, readiness, and planner + // contract support an owner action receive a supported intent. mountOwnerConnectorTemplates(app, { canonicalConnectorKey, + configuredProviderAuthConnectorKeys: opts.configuredProviderAuthConnectorKeys ?? [], createRequestConnectorInstanceStore, getConnectorManifest: (connectorId: string) => getConnectorManifest(connectorId), getOwnerTokenSubjectId, handleError, - listReferenceLocalConnectorCatalogManifests, listRegisteredConnectorIds, projectStorageDisplayName, requireOwner, @@ -6620,6 +6729,7 @@ export async function startServer(opts: ServerOpts = {}) { ownerSubjectId, }) : await admitOwnerRunConnection({ + allowDraft: runAdmission === "setup", connectorId, connectorInstanceId, connectorInstanceStore: createRequestConnectorInstanceStore(), @@ -6819,6 +6929,15 @@ export async function startServer(opts: ServerOpts = {}) { // (it lazily imports the connector package's probe + live transport). Tests // may inject their own via `opts.staticSecretCredentialProber`. const staticSecretCredentialProber = opts.staticSecretCredentialProber ?? (await buildStaticSecretCredentialProber()); + const configuredProviderAuthConnectorKeys = + opts.configuredProviderAuthConnectorKeys ?? configuredGoogleDataPortabilityProviderAuthConnectorKeys(process.env); + const providerAuthExchanger = + opts.providerAuthExchanger ?? + (configuredProviderAuthConnectorKeys.includes("google-maps-data-portability") + ? createGoogleDataPortabilityProviderAuthExchanger({ + credentialStoreFactory: createRequestConnectorInstanceCredentialStore, + }) + : null); const asApp = buildAsApp({ acceptedCollectorProtocolVersions: opts.acceptedCollectorProtocolVersions, @@ -6865,10 +6984,10 @@ export async function startServer(opts: ServerOpts = {}) { ...(browserSurfaceControllerOptions as Record), // biome-ignore lint/suspicious/noUnnecessaryConditions: TypeScript boundary permits nullish input; this guard preserves runtime behavior. cancelScheduledRun: (runId: string) => schedulerManager?.cancelRun?.(runId) ?? null, - configuredProviderAuthConnectorKeys: opts.configuredProviderAuthConnectorKeys ?? [], + configuredProviderAuthConnectorKeys, logger, onScheduleMutation: () => schedulerManager?.refresh(), - providerAuthExchanger: opts.providerAuthExchanger ?? null, + providerAuthExchanger, runTargetRegistry, staticSecretAutoResume: opts.staticSecretAutoResume, } as unknown as ServerOpts); @@ -6917,6 +7036,7 @@ export async function startServer(opts: ServerOpts = {}) { asIssuer: configuredAsIssuer || asPublicUrl, asPort, asPublicUrl, + configuredProviderAuthConnectorKeys, controller, hybridRetrievalCapability: opts.hybridRetrievalCapability, // Hybrid retrieval experimental extension knobs — see search-hybrid.js + diff --git a/reference-implementation/server/postgres-records.ts b/reference-implementation/server/postgres-records.ts index bbc118f56..52903a149 100644 --- a/reference-implementation/server/postgres-records.ts +++ b/reference-implementation/server/postgres-records.ts @@ -241,6 +241,8 @@ interface IngestRecord { interface IngestOptions { attemptContext?: DeviceAttemptContext | null; deviceReservation?: JsonObject & { inputIndex: number }; + /** See RecordIngestOptions.runId in server/records.ts. */ + runId?: string | null; } interface IngestOutcome { @@ -1763,6 +1765,42 @@ async function writePostgresIngestMutation({ return nextRecordJsonBytes; } +type PostgresTransactionClient = Parameters[0]>[0]; + +// Run-admission fence: `FOR UPDATE` takes Postgres' row lock on this run's +// run_history row, so this SELECT blocks until any concurrent terminal write +// (writePostgresRunHistoryForSpineEvent's `UPDATE ... WHERE status='running'`, +// which takes the same row lock implicitly) commits or rolls back — +// whichever transaction reaches the row first wins, giving a linearized +// ordering rather than a check-then-write race. Fails CLOSED: a caller that +// supplies a runId is asserting run-bound ingestion, and runtime/index.ts +// always awaits the run.started spine write (which durably inserts this row +// with status='running') before spawning the child that could ever call +// flushBatch — so a genuine run-bound write is guaranteed to find its row. A +// missing row for a supplied runId means the id is spoofed, mistyped, or +// belongs to a run this process never started; none of those should be +// admitted. Must be called from inside the same transaction as the mutation +// it guards. See harden-ingest-run-admission-fence. +async function assertPostgresRunStillAdmitted( + client: PostgresTransactionClient, + runId: string | null | undefined, + connectorInstanceId: string +): Promise { + if (!runId) { + return; + } + const runStatusResult = await client.query<{ status: string }>( + "SELECT status FROM run_history WHERE run_id = $1 AND connector_instance_id = $2 FOR UPDATE", + [runId, connectorInstanceId] + ); + const runStatus = runStatusResult.rows[0]?.status; + if (!runStatus || runStatus !== "running") { + throw new Error( + `run ${runId} is already terminal; refusing to commit an ingest write admitted before cancellation` + ); + } +} + export async function postgresIngestRecord( storageTarget: StorageTarget, record: IngestRecord, @@ -1802,6 +1840,7 @@ export async function postgresIngestRecord( op === "delete" ? null : semanticTimeValue(data ?? null, manifestStream, effectiveEmittedAt); const outcome = await withPostgresTransaction(async (client) => { + await assertPostgresRunStillAdmitted(client, options.runId, connectorInstanceId); const finishDurableOutcome = async (value: DurableIngestOutcome): Promise => { if (options.deviceReservation) { await advancePostgresDeviceIngestPrefix( diff --git a/reference-implementation/server/queries/connector-instances/promote-setup-binding.sql b/reference-implementation/server/queries/connector-instances/promote-setup-binding.sql new file mode 100644 index 000000000..44d4870c0 --- /dev/null +++ b/reference-implementation/server/queries/connector-instances/promote-setup-binding.sql @@ -0,0 +1,8 @@ +-- @terminator: exec +UPDATE connector_instances +SET source_binding_json = ?, + status = ?, + updated_at = ? +WHERE connector_instance_id = ? + AND status = 'draft' + AND json_extract(source_binding_json, '$.kind') = ?; diff --git a/reference-implementation/server/queries/connector-instances/update-static-secret-binding.sql b/reference-implementation/server/queries/connector-instances/update-static-secret-binding.sql new file mode 100644 index 000000000..2c74b115b --- /dev/null +++ b/reference-implementation/server/queries/connector-instances/update-static-secret-binding.sql @@ -0,0 +1,9 @@ +-- @terminator: exec +UPDATE connector_instances +SET source_binding_key = ?, + source_binding_json = ?, + updated_at = ? +WHERE connector_instance_id = ? + AND owner_subject_id = ? + AND connector_id = ? + AND status IN ('active', 'draft'); diff --git a/reference-implementation/server/queries/controller/get-run-history-status-for-run.sql b/reference-implementation/server/queries/controller/get-run-history-status-for-run.sql new file mode 100644 index 000000000..efd229fa1 --- /dev/null +++ b/reference-implementation/server/queries/controller/get-run-history-status-for-run.sql @@ -0,0 +1,13 @@ +-- @terminator: one +-- Read the current status of exactly one (run_id, connector_instance_id) +-- run_history row. Used by the record-ingest write path to fence a write +-- against a run that reached a terminal state (owner-cancelled, timed out) +-- while the write was already admitted into the per-connector-instance +-- write coordinator. `connector_instance_id` is required, not optional: +-- run_id alone is NOT globally unique (see run-history-writer.ts header) — +-- a bare `WHERE run_id = ?` could match a DIFFERENT connection's row that +-- happens to share this run_id. See harden-ingest-run-admission-fence. +SELECT status +FROM run_history +WHERE run_id = ? + AND connector_instance_id = ? diff --git a/reference-implementation/server/queries/index.ts b/reference-implementation/server/queries/index.ts index 83551a611..a38c9bc16 100644 --- a/reference-implementation/server/queries/index.ts +++ b/reference-implementation/server/queries/index.ts @@ -252,13 +252,16 @@ export interface ReferenceQueryRegistry extends Readonly; + +type RecordIngestBatchOutcome = RecordIngestOutcome & { + /** Present when the durable write or its derived-index phase failed. */ + error?: string; +}; +type RecordIngestAfterRecord = (record: RecordEnvelope, outcome: RecordIngestBatchOutcome) => void | Promise; +interface DeferredRecordIndex { + index: number; + record: RecordEnvelope; +} +interface IngestRecordsWithinCoordinatorResult { + changedRecords: DeferredRecordIndex[]; + outcomes: RecordIngestBatchOutcome[]; +} interface JsonSchema { format?: string; properties?: Record; @@ -766,6 +808,21 @@ export class RecordIndexAdmissionError extends Error { } } +// A write already admitted into the per-connector-instance write coordinator +// for a run that has since reached a terminal state (owner-cancelled, timed +// out, or otherwise closed). The runtime's own cancellation signal is a +// client-side AbortSignal that cannot retroactively un-admit a write the +// server already accepted; this is the storage-layer fence that refuses it +// instead. See harden-ingest-run-admission-fence. +export class RecordIngestRunTerminalError extends Error { + code: string; + constructor(runId: string) { + super(`run ${runId} is already terminal; refusing to commit an ingest write admitted before cancellation`); + this.name = "RecordIngestRunTerminalError"; + this.code = "run_terminal"; + } +} + let activeIndexWork = 0; const indexWorkWaiters: IndexWorkWaiter[] = []; @@ -844,6 +901,43 @@ async function withIndexWork(operation: () => Promise): Promise { } } +// Index work is derived state. Keep it ordered for one connector instance so +// a newer record writer cannot race an older batch's lexical/semantic repair, +// but do not make that derived queue another connector-instance writer fence. +// A batch schedules its work before releasing the authoritative fence and +// supplies a start barrier, so later batches cannot overtake it while blobs +// remain free to acquire the writer fence. +const connectorInstanceIndexTails = new Map>(); + +function enqueueConnectorInstanceIndexWork( + connectorInstanceId: string, + operation: () => Promise, + startAfter?: Promise +): Promise { + const previous = connectorInstanceIndexTails.get(connectorInstanceId) ?? Promise.resolve(); + const next = previous.then(async () => { + if (startAfter) { + await startAfter; + } + await operation(); + }); + let tail: Promise; + tail = next.then( + () => { + if (connectorInstanceIndexTails.get(connectorInstanceId) === tail) { + connectorInstanceIndexTails.delete(connectorInstanceId); + } + }, + () => { + if (connectorInstanceIndexTails.get(connectorInstanceId) === tail) { + connectorInstanceIndexTails.delete(connectorInstanceId); + } + } + ); + connectorInstanceIndexTails.set(connectorInstanceId, tail); + return next; +} + export function recordIndexWorkStatsForTests(): { active: number; queued: number } { return { active: activeIndexWork, queued: indexWorkWaiters.length }; } @@ -1065,6 +1159,56 @@ function maybeRecordIndexFault(point: string, ctx: HookContext): void { } } +/** + * Refuse a record/blob write for a `connector_instance_id` whose + * `connector_instances` row is gone — closing a delete/write TOCTOU the + * coordinator fence's mutual exclusion alone does not close: the fence only + * serializes a delete against a write for the SAME identity, it does not + * reject a write that acquires the fence AFTER a delete already committed + * and removed the row. Neither the SQLite nor the Postgres schema declares a + * foreign key from `records`/`record_changes`/`blobs`/`blob_bindings` to + * `connector_instances`, so without this check a post-delete write silently + * resurrects a live `records` row for a tombstoned, no-longer-existent + * connection. + * + * `ingestRecord`/`ingestRecords` stay a connector-agnostic durable storage + * primitive for direct callers by default (internal repair paths, and + * dozens of existing tests that ingest without ever enrolling a + * `connector_instances` row) — this function is called from inside them + * ONLY when the caller opts in via `RecordIngestOptions.requireConnectionAdmission`. + * Owner HTTP ingest (server/routes/rs-mutation.ts) opts in on every call + * that resolved a real connection. Device-exporter ingest + * (server/routes/ref-device-exporters.ts) calls this function directly, + * once per batch, immediately after acquiring the batch's coordinator fence + * and before its first mutation — every record write in that batch reuses + * the SAME held fence via `coordinatorOwnership`, so the one check covers + * the whole batch. `persistContentAddressedBlob` (blob writes, + * server/index.ts) calls this function directly and unconditionally inside + * its own fence, since it is reached only via the HTTP blob-write route. + * Source-webhook ingest (server/routes/source-webhooks.ts) is deliberately + * NOT wired to this check: it is connector-id-only generic ingest with no + * per-connection admission concept at that layer, so it is out of scope for + * this fix. + * + * Reuses `ConnectorInstanceResolutionError`'s `connector_instance_not_found` + * code — the same typed outcome `deleteConnection`'s own ownership check + * raises — so callers already handling that code (e.g. a route's + * `handleError` mapping to 404) require no new branch. + */ +export async function assertConnectorInstanceWritable(connectorInstanceId: string): Promise { + const exists = isPostgresStorageBackend() + ? (await postgresQuery("SELECT 1 FROM connector_instances WHERE connector_instance_id = $1", [connectorInstanceId])) + .rows.length > 0 + : Boolean(getOne(referenceQueries.connectorInstancesGetById, [connectorInstanceId])); + if (!exists) { + throw new ConnectorInstanceResolutionError( + "connector_instance_not_found", + `Connector instance '${connectorInstanceId}' does not exist; it may have been deleted concurrently with this write.`, + { connectorInstanceId } + ); + } +} + /** * Ingest a RECORD envelope (owner-authenticated). * @@ -1095,12 +1239,156 @@ export async function ingestRecord( const coordinationInstanceId = resolveStorageConnectorInstanceId(storageTarget, coordinationConnectorId); return await withConnectorInstanceWrite( coordinationInstanceId, - (coordinatorOwnership) => - ingestRecordWithinCoordinator(storageTarget, record, { ...options, coordinatorOwnership }), + async (coordinatorOwnership) => { + if (options.requireConnectionAdmission) { + await assertConnectorInstanceWritable(coordinationInstanceId); + } + return ingestRecordWithinCoordinator(storageTarget, record, { ...options, coordinatorOwnership }); + }, options.coordinatorOwnership ); } +/** + * Ingest one HTTP batch under a single connector-instance fence. + * + * Durable record mutations stay in one ordered phase under one + * connector-instance ownership capability. Derived index work is serialized + * on a separate per-instance lane after that fence releases, so embedding and + * index latency cannot starve blob writers. When supplied, `afterRecord` is + * awaited between a record's storage completion and the next record. + */ +export async function ingestRecords( + storageTarget: RecordStorageTarget, + records: readonly RecordEnvelope[], + afterRecord?: RecordIngestAfterRecord, + options: Pick = {} +): Promise { + const coordinationConnectorId = connectorIdForStorageTarget(storageTarget); + const coordinationInstanceId = resolveStorageConnectorInstanceId(storageTarget, coordinationConnectorId); + let deferredIndexWork: Promise | undefined; + let releaseFence: (() => void) | undefined; + const fenceReleasedPromise = new Promise((resolve) => { + releaseFence = resolve; + }); + try { + const result = await withConnectorInstanceWrite( + coordinationInstanceId, + async (coordinatorOwnership) => { + if (options.requireConnectionAdmission) { + await assertConnectorInstanceWritable(coordinationInstanceId); + } + const batch = await ingestRecordsWithinCoordinator( + storageTarget, + records, + coordinatorOwnership, + afterRecord, + options.runId + ); + if (batch.changedRecords.length > 0) { + deferredIndexWork = enqueueConnectorInstanceIndexWork( + coordinationInstanceId, + () => runDeferredRecordIndexes(storageTarget, batch), + fenceReleasedPromise + ); + } + return batch; + }, + undefined + ); + releaseFence?.(); + await deferredIndexWork; + return result.outcomes; + } finally { + releaseFence?.(); + } +} + +async function runDeferredRecordIndexes( + storageTarget: RecordStorageTarget, + batch: IngestRecordsWithinCoordinatorResult +): Promise { + for (const { index, record } of batch.changedRecords) { + try { + // biome-ignore lint/performance/noAwaitInLoops: One connector instance's derived index repairs are intentionally ordered. + await withIndexWork(() => maintainRecordIndexesWithinPermit(storageTarget, record, {})); + } catch (err) { + const outcome = batch.outcomes[index]; + if (outcome?.accepted) { + batch.outcomes[index] = { + accepted: false, + changed: false, + error: err instanceof Error ? err.message : String(err), + }; + } + } + } +} + +async function ingestRecordsWithinCoordinator( + storageTarget: RecordStorageTarget, + records: readonly RecordEnvelope[], + coordinatorOwnership: ConnectorInstanceWriteOwnership, + afterRecord?: RecordIngestAfterRecord, + runId?: string | null +): Promise { + const outcomes: Array = new Array(records.length); + const changedRecords: DeferredRecordIndex[] = []; + const perRecordOptions: RecordIngestOptions = { + coordinatorOwnership, + deferIndexes: true, + ...(runId ? { runId } : {}), + }; + + for (const [index, record] of records.entries()) { + let outcome: RecordIngestBatchOutcome; + try { + // Versions, current-state transitions, summaries, and after-commit + // notifications stay inside the one authoritative batch fence. Derived + // indexes are scheduled after that fence so expensive embedding work + // cannot starve an unrelated blob writer. + // biome-ignore lint/performance/noAwaitInLoops: Durable version allocation and same-instance state transitions are intentionally ordered. + outcome = await ingestRecordWithinCoordinator(storageTarget, record, perRecordOptions); + if (outcome.accepted && outcome.changed) { + changedRecords.push({ index, record }); + } + } catch (err) { + outcome = { + accepted: false, + changed: false, + error: err instanceof Error ? err.message : String(err), + }; + } + if (afterRecord && outcome.accepted) { + try { + // The callback is part of the per-record completion boundary. It must + // finish before the next record enters storage so host-side provenance + // and similar effects preserve the established store->effect order. + await afterRecord(record, outcome); + } catch (err) { + outcome = { + accepted: false, + changed: false, + error: err instanceof Error ? err.message : String(err), + }; + } + } + outcomes[index] = outcome; + } + + return { + changedRecords, + outcomes: outcomes.map( + (outcome) => + outcome ?? { + accepted: false, + changed: false, + error: "ingest batch did not produce a result", + } + ), + }; +} + function ingestRecordWithinCoordinator( storageTarget: RecordStorageTarget, record: RecordEnvelope, @@ -1158,6 +1446,7 @@ function toPostgresIngestOptions(options: RecordIngestOptions) { return { ...(attemptContext ? { attemptContext } : {}), ...(options.deviceReservation ? { deviceReservation: options.deviceReservation } : {}), + ...(options.runId ? { runId: options.runId } : {}), }; } @@ -1343,6 +1632,31 @@ function pruneRecordChangeHistory( return { prunedBytesForDelta, prunedRowsForDelta }; } +// Run-admission fence: SQLite's single writer connection makes this +// read-then-write atomic with run_history's terminal write (both go through +// the same synchronous handle — see run-history-writer.ts header). Fails +// CLOSED: a caller that supplies a runId is asserting run-bound ingestion, +// and runtime/index.ts always awaits the run.started spine write (which +// durably inserts this row with status='running') before spawning the child +// that could ever call flushBatch — so a genuine run-bound write is +// guaranteed to find its row. A missing row for a supplied runId means the +// id is spoofed, mistyped, or belongs to a run this process never started; +// none of those should be admitted. Must be called from inside the durable +// write transaction so no other write can land between this check and the +// mutation it guards. See harden-ingest-run-admission-fence. +function assertSqliteRunStillAdmitted(runId: string | null | undefined, connectorInstanceId: string): void { + if (!runId) { + return; + } + const runStatus = getOne<{ status: string }>(referenceQueries.controllerGetRunHistoryStatusForRun, [ + runId, + connectorInstanceId, + ]); + if (runStatus?.status !== "running") { + throw new RecordIngestRunTerminalError(runId); + } +} + async function ingestSqliteRecord( storageTarget: RecordStorageTarget, record: RecordEnvelope, @@ -1383,6 +1697,7 @@ async function ingestSqliteRecord( // Durable mutation unit: returns the operation outcome so derived index // maintenance can run *after* the commit succeeds. const outcome = writeTransaction(() => { + assertSqliteRunStillAdmitted(options.runId, connectorInstanceId); const finishDurableOutcome = (value: DurableIngestOutcome): DurableIngestOutcome => { if (options.deviceReservation) { advanceSqliteDeviceIngestPrefix(options.deviceReservation, options.deviceReservation.inputIndex); @@ -5657,6 +5972,10 @@ export function deleteConnectionRecordRowsSqlite(connectorInstanceId: string) { exec(referenceQueries.recordsDeleteDeleteRecordChangesByInstance, [connectorInstanceId]); exec(referenceQueries.recordsDeleteDeleteVersionCounterByInstance, [connectorInstanceId]); exec(referenceQueries.recordsDeleteDeleteBlobBindingsByInstance, [connectorInstanceId]); + // Blob rows are content-addressed and can have a binding from a sibling + // connection. The registered delete query removes only unreferenced rows, + // after this connection's bindings are gone, so the sibling binding remains + // valid under SQLite's blob_bindings foreign key. exec(referenceQueries.recordsDeleteDeleteBlobsByInstance, [connectorInstanceId]); exec(referenceQueries.recordsDeleteDeleteAttentionRecordsByInstance, [connectorInstanceId]); exec(referenceQueries.recordsDeleteDeleteRecordsByInstance, [connectorInstanceId]); @@ -5678,7 +5997,16 @@ export async function deleteConnectionRecordRowsPostgres(client: PostgresClient, await client.query("DELETE FROM record_changes WHERE connector_instance_id = $1", [connectorInstanceId]); await client.query("DELETE FROM version_counter WHERE connector_instance_id = $1", [connectorInstanceId]); await client.query("DELETE FROM blob_bindings WHERE connector_instance_id = $1", [connectorInstanceId]); - await client.query("DELETE FROM blobs WHERE connector_instance_id = $1", [connectorInstanceId]); + await client.query( + `DELETE FROM blobs + WHERE connector_instance_id = $1 + AND NOT EXISTS ( + SELECT 1 + FROM blob_bindings + WHERE blob_bindings.blob_id = blobs.blob_id + )`, + [connectorInstanceId] + ); await client.query("DELETE FROM connector_attention_records WHERE connector_instance_id = $1", [connectorInstanceId]); await client.query("DELETE FROM records WHERE connector_instance_id = $1", [connectorInstanceId]); return count; diff --git a/reference-implementation/server/ref-control.ts b/reference-implementation/server/ref-control.ts index ca1addb73..391da003d 100644 --- a/reference-implementation/server/ref-control.ts +++ b/reference-implementation/server/ref-control.ts @@ -82,6 +82,10 @@ import { } from "../runtime/recovery-decision.ts"; import type { RenderedVerdict, ScheduleEvidence } from "../runtime/rendered-verdict.ts"; import { SOURCE_PRESSURE_GAP_REASONS } from "../runtime/scheduler-source-pressure-cooldown.ts"; +import { + classifyTerminalSetupDisposition, + type SetupTerminalDisposition, +} from "../runtime/static-secret-setup-status.ts"; import { pickMostUrgentAttention } from "./attention-urgency.ts"; import { getConnectorManifest } from "./auth.ts"; import { @@ -479,6 +483,7 @@ export interface ConnectorRunSummary { readonly first_at: string; readonly known_gaps: unknown[]; readonly last_at: string; + readonly records_emitted?: number | null; /** * Whether this run was dispatched `recovery_only` (drains pending detail * gaps only; performs no forward/list-pass inventory scan). Read directly @@ -493,10 +498,12 @@ export interface ConnectorRunSummary { * verdict that wipes prior proven coverage. */ readonly recovery_only: boolean; + readonly reported_records_emitted?: number | null; readonly run_id: string | undefined; readonly started_at: string; readonly status: string; readonly terminal_reason: string | null; + readonly yield_counts_present?: boolean; } export interface PendingDetailGapSummary { @@ -802,6 +809,8 @@ export interface ConnectorSummary { readonly as_of: string | null; readonly reason_code: string | null; }; + /** Shared terminal setup disposition for a draft, or null otherwise. */ + readonly terminal_setup_disposition: SetupTerminalDisposition | null; readonly total_records: number; /** * Orthogonal state for `total_records`, the same `count_state` contract @@ -1142,6 +1151,27 @@ function isActiveRunSummaryStatus(status: string): boolean { return status === "pending" || status === "started" || status === "in_progress"; } +function finiteNumberOrNull(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function productRunYieldCounts(history: ProductRunHistoryRecord): { + readonly present: boolean; + readonly recordsEmitted: number | null; + readonly reportedRecordsEmitted: number | null; +} { + const facts = history.factsJson; + const present = + facts === null || facts === undefined + ? true + : Object.hasOwn(facts, "records_emitted") || Object.hasOwn(facts, "reported_records_emitted"); + return { + present, + recordsEmitted: present ? finiteNumberOrNull(history.recordsEmitted) : null, + reportedRecordsEmitted: present ? finiteNumberOrNull(history.reportedRecordsEmitted) : null, + }; +} + /** * Product LIST/detail composition (terminal-read-architecture-fable-0730.md * §9/R9.2): a `run_history` row for ANY run kind, composed with the @@ -1166,6 +1196,7 @@ function productRunHistoryToConnectorRunSummary( return null; } const facts = history.factsJson ?? null; + const yieldCounts = productRunYieldCounts(history); const isLive = Boolean(activeRun && history.runId && activeRun.run_id === history.runId); let status: string = history.status; if (status === "running") { @@ -1187,11 +1218,14 @@ function productRunHistoryToConnectorRunSummary( first_at: history.startedAt, known_gaps: terminalKnownGaps.length > 0 ? terminalKnownGaps : [...history.knownGaps], last_at: isActiveRunSummaryStatus(status) ? history.startedAt : history.completedAt, + records_emitted: yieldCounts.recordsEmitted, recovery_only: facts?.recovery_only === true, + reported_records_emitted: yieldCounts.reportedRecordsEmitted, run_id: history.runId || undefined, started_at: history.startedAt, status, terminal_reason: history.terminalReason ?? null, + yield_counts_present: yieldCounts.present, }; } @@ -4626,8 +4660,10 @@ function shouldHydrateRunSummariesForInstance( // still refuses connector-wide fallback unless the connector has exactly one // active visible source, but skipping hydration here would also drop exact // `connector_instance_id` / browser-profile matches for multi-account - // connectors and render them as indefinitely "checking". - return instance.status === "active"; + // connectors and render them as indefinitely "checking". Drafts are also + // hydrated so owner surfaces can reuse the same terminal setup disposition + // without making a second lifecycle projection. + return instance.status === "active" || instance.status === "draft"; } function groupRetainedSizeRowsByInstance( @@ -4867,6 +4903,20 @@ function synthesizeConnectorSummary(input: ConnectorSummarySynthesisInput): Conn const authoritativeLastRun = localDeviceBacked ? null : lastRun; const authoritativeLastSuccessfulRun = localDeviceBacked ? null : lastSuccessfulRun; const authoritativeLatestStreamFacts = localDeviceBacked ? null : latestStreamFacts; + const terminalSetupDisposition = + instance.status === "draft" && authoritativeLastRun + ? classifyTerminalSetupDisposition({ + collectionFacts: authoritativeLastRun.collection_facts, + manifestStreams: (manifest.streams ?? []).map((stream) => ({ + name: stream.name, + ...(stream.required === undefined ? {} : { required: stream.required }), + })), + recordsEmitted: authoritativeLastRun.records_emitted, + reportedRecordsEmitted: authoritativeLastRun.reported_records_emitted, + status: authoritativeLastRun.status, + yieldCountsPresent: authoritativeLastRun.yield_counts_present, + }) + : null; const healthRemoteSurface = connectionHealthRemoteSurface({ remoteSurface, runtime: authoritativeEphemeralBrowserRuntime, @@ -5102,6 +5152,7 @@ function synthesizeConnectorSummary(input: ConnectorSummarySynthesisInput): Conn terminal_facts: evidence ? evidence.terminal_facts : { as_of: null, event_seq: null, reason_code: "summary_evidence_unavailable", state: "unobserved" }, + terminal_setup_disposition: terminalSetupDisposition, total_records: totalRecords, total_records_state: totalRecordsState, total_retained_bytes: totalRetainedBytes, diff --git a/reference-implementation/server/routes/owner-connector-templates.ts b/reference-implementation/server/routes/owner-connector-templates.ts index 847fcab87..9d400a7a1 100644 --- a/reference-implementation/server/routes/owner-connector-templates.ts +++ b/reference-implementation/server/routes/owner-connector-templates.ts @@ -5,13 +5,14 @@ // route `GET /v1/owner/connector-templates`. // // This route is intentionally template-level. It tells a trusted owner agent -// what connector implementations exist and which configured connection -// instances currently belong to each template. Stateful work still targets -// `connection_id` through `/v1/owner/connections`; adding a new connection is -// exposed only as a typed intent and is marked unsupported when this reference -// build lacks a proven provider primitive. +// what registered connector implementations exist and which configured +// connection instances currently belong to each template. Stateful work still +// targets `connection_id` through `/v1/owner/connections`; adding a new +// connection is exposed as a typed owner-agent intent only when the +// server-owned planner and proof/listing contract mark that REST action +// supported. Interactive browser setup remains owner-mediated in Console. -import { buildConnectionSetupPlan } from "../connection-setup-plan.ts"; +import { buildConnectionSetupPlan, isSupportedBrowserCollectorConnector } from "../connection-setup-plan.ts"; import type { OwnerAgentControlAction } from "../metadata.ts"; import type { MiddlewareHandler, RouteArg } from "./_route-contract.ts"; @@ -32,6 +33,19 @@ interface AppLike { } interface ConnectorManifestLike { + readonly capabilities?: { + readonly auth?: { + readonly deployment_config?: readonly string[] | null; + readonly kind?: string | null; + readonly mode?: string | null; + readonly required?: readonly string[] | null; + readonly type?: string | null; + } | null; + readonly public_listing?: { + readonly listed?: boolean | null; + readonly status?: string | null; + } | null; + } | null; readonly connector_id?: string | null; readonly connector_key?: string | null; readonly display_name?: string | null; @@ -60,11 +74,11 @@ interface ConnectorInstanceStore { export interface MountOwnerConnectorTemplatesContext { canonicalConnectorKey: (value: string | null | undefined) => string | null; + configuredProviderAuthConnectorKeys?: readonly string[]; createRequestConnectorInstanceStore: () => ConnectorInstanceStore; getConnectorManifest: (connectorId: string) => Promise | ConnectorManifestLike | null; getOwnerTokenSubjectId: (req: unknown) => string; handleError: (res: unknown, err: unknown) => void; - listReferenceLocalConnectorCatalogManifests: () => readonly ConnectorManifestLike[]; listRegisteredConnectorIds: () => Promise | readonly string[]; projectStorageDisplayName: ( displayName: string | null | undefined, @@ -121,13 +135,74 @@ function projectConnectionSummary( }; } +const ACTIONABLE_PUBLIC_LISTING_STATUSES = new Set(["proven", "needs_human_auth"]); +// These are the dispositions with a supported owner-agent REST intent. Browser +// setup is intentionally handled by the separate owner-session projection +// below, because the REST intent route cannot launch interactive login. +const ACTIONABLE_CATALOG_DISPOSITIONS = new Set([ + "local_collector_enroll", + "manual_upload_connect", + "provider_auth_connect", + "static_secret_connect", +]); + +const OWNER_SESSION_BROWSER_ACTION_REASON = + "Connect this account from the owner's secure browser-session dashboard. Owner-agent REST does not launch interactive browser setup."; + +function isActionablePublicListing(manifest: ConnectorManifestLike): boolean { + // `needs_human_auth` is an explicitly actionable listing state for sources + // whose owner-mediated setup still requires an interactive provider step. + const listing = manifest.capabilities?.public_listing; + return ( + listing?.listed === true && + typeof listing.status === "string" && + ACTIONABLE_PUBLIC_LISTING_STATUSES.has(listing.status) + ); +} + +export function isSupportedOwnerActionPlan(plan: ReturnType): boolean { + return ( + ACTIONABLE_CATALOG_DISPOSITIONS.has(plan.catalogDisposition) && + plan.ownerAgentIntent.status === "supported" && + plan.ownerAgentIntent.method !== null && + plan.ownerAgentIntent.nextStepKind === plan.nextStepKind && + plan.supportState === "supported" && + plan.proofGate === null + ); +} + +/** + * Browser setup has a shipped owner-session route, but it is not an + * owner-agent REST primitive: the owner must complete interactive login in + * the secure browser. The planner's production-ready browser roster is the + * proof for browser-backed static-secret entries; the manual disposition is + * already the planner's proof-backed browser classification. + */ +export function isOwnerSessionBrowserActionPlan(plan: ReturnType): boolean { + if (plan.connectorModality !== "browser_bound") { + return false; + } + if (plan.catalogDisposition === "browser_collector_manual") { + return plan.nextStepKind === "enroll_browser_collector" && typeof plan.enrollmentKey === "string"; + } + return ( + plan.catalogDisposition === "static_secret_connect" && + plan.setupModality === "static_secret" && + isSupportedBrowserCollectorConnector(plan.connectorKey) + ); +} + +function isOwnerActionablePlan(plan: ReturnType): boolean { + return isSupportedOwnerActionPlan(plan) || isOwnerSessionBrowserActionPlan(plan); +} + function buildTemplateSupportedActions(args: { - connectorKey: string; + manifest: ConnectorManifestLike; plan: ReturnType; resource: string; }): OwnerAgentControlAction[] { const rs = stripTrailingSlash(args.resource); - if (args.plan.ownerAgentIntent.status === "supported") { + if (isActionablePublicListing(args.manifest) && isSupportedOwnerActionPlan(args.plan)) { return [ { family: "initiate_connection", @@ -138,6 +213,17 @@ function buildTemplateSupportedActions(args: { }, ]; } + if (isActionablePublicListing(args.manifest) && isOwnerSessionBrowserActionPlan(args.plan)) { + return [ + { + family: "initiate_connection", + method: null, + reason: OWNER_SESSION_BROWSER_ACTION_REASON, + status: "owner_mediated", + url: null, + }, + ]; + } return [ { family: "initiate_connection", @@ -149,10 +235,19 @@ function buildTemplateSupportedActions(args: { ]; } -function projectSetupPlan(plan: ReturnType): Record { +function projectSetupPlan( + manifest: ConnectorManifestLike, + plan: ReturnType +): Record { return { + catalog_disposition: plan.catalogDisposition, deployment_readiness: plan.deploymentReadiness, + enrollment_key: plan.enrollmentKey ?? null, next_step_kind: plan.nextStepKind, + // This is owner-facing actionability, not owner-agent REST support. A + // browser action is represented in supported_actions as owner_mediated + // with no method or URL because interactive setup stays in Console. + owner_actionable: isActionablePublicListing(manifest) && isOwnerActionablePlan(plan), proof_gate: plan.proofGate, runbook_path: plan.runbookPath, setup_modality: plan.setupModality, @@ -171,7 +266,11 @@ function projectTemplate( if (!connectorKey) { return null; } - const plan = buildConnectionSetupPlan({ connectorKey, manifest }); + const plan = buildConnectionSetupPlan({ + configuredProviderAuthConnectorKeys: ctx.configuredProviderAuthConnectorKeys ?? [], + connectorKey, + manifest, + }); const modality = plan.connectorModality; const connections = (connectionsByConnector.get(connectorKey) ?? []).map((instance) => projectConnectionSummary(ctx, instance) @@ -184,21 +283,17 @@ function projectTemplate( connector_modality: modality, display_name: displayNameForTemplate(connectorKey, manifest), object: "owner_connector_template", - setup_plan: projectSetupPlan(plan), + public_listing: manifest.capabilities?.public_listing ?? null, + registration_status: "registered", + setup_plan: projectSetupPlan(manifest, plan), stream_count: Array.isArray(manifest.streams) ? manifest.streams.length : 0, - supported_actions: buildTemplateSupportedActions({ connectorKey, plan, resource }), + supported_actions: buildTemplateSupportedActions({ manifest, plan, resource }), version: manifest.version ?? null, }; } async function collectConnectorTemplates(ctx: MountOwnerConnectorTemplatesContext): Promise { const byConnectorKey = new Map(); - for (const manifest of ctx.listReferenceLocalConnectorCatalogManifests()) { - const key = connectorKeyFromManifest(ctx, manifest); - if (key) { - byConnectorKey.set(key, manifest); - } - } for (const connectorId of await ctx.listRegisteredConnectorIds()) { const connectorKey = ctx.canonicalConnectorKey(connectorId) ?? connectorId; try { diff --git a/reference-implementation/server/routes/ref-browser-enrollment-shell.ts b/reference-implementation/server/routes/ref-browser-enrollment-shell.ts index 48cb45149..9cb88210e 100644 --- a/reference-implementation/server/routes/ref-browser-enrollment-shell.ts +++ b/reference-implementation/server/routes/ref-browser-enrollment-shell.ts @@ -53,6 +53,28 @@ export interface BrowserEnrollmentShellSourceBinding { readonly kind: "browser_enrollment_shell"; } +// The durable binding a browser-enrollment shell promotes to — no TTL, so +// exempt from browser-enrollment-shell-retirement.ts by construction. +export interface BrowserCollectorSourceBinding { + readonly connector_id: string; + readonly kind: "browser_collector"; + readonly promoted_at: string; + readonly promoted_from: "browser_enrollment_shell"; +} + +// Pure — no I/O — so it's independently unit-testable. +export function promoteBrowserEnrollmentShellBinding( + shellBinding: BrowserEnrollmentShellSourceBinding, + now: string +): BrowserCollectorSourceBinding { + return { + connector_id: shellBinding.connector_id, + kind: "browser_collector", + promoted_at: now, + promoted_from: "browser_enrollment_shell", + }; +} + interface RouteRequest { readonly body?: unknown; ownerSession?: { readonly sub?: string | null } | null; diff --git a/reference-implementation/server/routes/ref-connectors.ts b/reference-implementation/server/routes/ref-connectors.ts index 2a9d95b28..4b58b7978 100644 --- a/reference-implementation/server/routes/ref-connectors.ts +++ b/reference-implementation/server/routes/ref-connectors.ts @@ -44,6 +44,7 @@ import { ConnectorSummaryPageRequestError, parseConnectorSummaryPageRequest, } from "../../operations/ref-connectors-list/pagination.ts"; +import type { RunAdmission } from "../../runtime/controller.ts"; import type { FleetHealthVerdict } from "../fleet-health.ts"; import type { MiddlewareHandler, PdppErrorFn, RouteArg } from "./_route-contract.ts"; import { assertRemoteControlSupported } from "./_route-contract.ts"; @@ -221,7 +222,7 @@ export interface MountRefConnectorsContext { options: { connectorInstanceId?: string | null; force?: boolean; - runAdmission?: "browser_enrollment"; + runAdmission?: RunAdmission; resources?: Readonly>; } ) => Promise; @@ -743,8 +744,6 @@ function readExplicitRunForce(req: RouteRequest): boolean { ); } -type RunAdmission = "collection" | "browser_enrollment"; - function readRunAdmission(req: RouteRequest): RunAdmission { const { body } = req; if (!(body && typeof body === "object" && !Array.isArray(body))) { @@ -754,14 +753,24 @@ function readRunAdmission(req: RouteRequest): RunAdmission { if (raw === undefined) { return "collection"; } - if (raw === "browser_enrollment") { + if (raw === "setup" || raw === "browser_enrollment") { return raw; } - const err = new Error("run_admission must be browser_enrollment when provided") as Error & { code: string }; + const err = new Error("run_admission must be setup or browser_enrollment when provided") as Error & { code: string }; err.code = "invalid_request"; throw err; } +function runAdmissionStatuses(runAdmission: RunAdmission): readonly string[] { + if (runAdmission === "browser_enrollment") { + return ["draft"]; + } + if (runAdmission === "setup") { + return ["active", "draft"]; + } + return ["active"]; +} + function readRunId(started: unknown): string | null { if (!started || typeof started !== "object") { return null; @@ -850,7 +859,7 @@ async function executeRunNow( connectorInstanceId: namespace.connectorInstanceId, force: audit.force, ...(audit.ownerSubjectId ? { ownerSubjectId: audit.ownerSubjectId } : {}), - ...(audit.runAdmission === "browser_enrollment" ? { runAdmission: audit.runAdmission } : {}), + ...(audit.runAdmission === "collection" ? {} : { runAdmission: audit.runAdmission }), ...(resources ? { resources } : {}), }); ctx.invalidateConnectorSummariesCache?.(); @@ -928,7 +937,7 @@ export function mountRefConnectionRun(app: AppLike, ctx: MountRefConnectorsConte connectionId = connectorInstanceId; runAdmission = readRunAdmission(req); const namespace = await resolveRefConnectionNamespace(ctx, req, connectorInstanceId, { - allowStatuses: runAdmission === "browser_enrollment" ? ["draft"] : ["active", "draft"], + allowStatuses: runAdmissionStatuses(runAdmission), }); connectionId = namespace.connectorInstanceId; connectorKey = ctx.canonicalConnectorKey(namespace.connectorId) ?? namespace.connectorId; diff --git a/reference-implementation/server/routes/ref-device-exporters.ts b/reference-implementation/server/routes/ref-device-exporters.ts index 5cf9dd5a7..d6218a5fa 100644 --- a/reference-implementation/server/routes/ref-device-exporters.ts +++ b/reference-implementation/server/routes/ref-device-exporters.ts @@ -432,6 +432,12 @@ interface ConnectorDetailGapStore { export interface MountRefDeviceExportersContext { acceptedCollectorProtocolVersions: readonly string[]; + // Re-checks the connection once per device-ingest batch after its + // coordinator fence is acquired. Every record write reuses that held + // fence, so one check covers the whole loop. Closes the delete/write TOCTOU documented on + // `assertConnectorInstanceWritable` in server/records.ts. + assertConnectorInstanceWritable: (connectorInstanceId: string) => Promise; + // Canonical key resolution canonicalConnectorKey: (value: string | null | undefined) => string | null; createRequestConnectorInstanceStore: () => ConnectorInstanceStore; @@ -2115,6 +2121,37 @@ export function mountRefDeviceExporterRevoke(app: AppLike, ctx: MountRefDeviceEx ); } +// POST /_ref/device-exporters/:deviceId/self-revoke +// +// A device credential may revoke itself, never another device: auth is the +// device's own bearer token (not an owner session), and the path deviceId +// must match the credential that authenticated the request. This is the +// route `pdpp-local-collector logout` calls before deleting its local +// profile — without it, a local device has no way to close its own +// server-side lane, and logout could only ever delete local state while the +// device token stayed live against the reference deployment indefinitely. +export function mountRefDeviceExporterSelfRevoke(app: AppLike, ctx: MountRefDeviceExportersContext): void { + app.post( + "/_ref/device-exporters/:deviceId/self-revoke", + { contract: "refSelfRevokeDeviceExporter" }, + ctx.requireDeviceExporterCredential, + async (req: RouteRequest, res: RouteResponse) => { + try { + const deviceId = decodeURIComponent(req.params.deviceId as string); + if (deviceId !== req.deviceExporter?.deviceId) { + ctx.pdppError(res, 403, "permission_error", "Device credential is not valid for this device"); + return; + } + const revokedAt = new Date().toISOString(); + await ctx.deviceExporterStore.revokeDevice(deviceId, revokedAt); + res.json({ device_id: deviceId, object: "device_exporter_revocation", revoked_at: revokedAt }); + } catch (err) { + ctx.handleError(res, err); + } + } + ); +} + async function markHeartbeatSourceInstance(input: { ctx: MountRefDeviceExportersContext; deviceId: string; @@ -2367,6 +2404,15 @@ async function processDeviceIngestBatch( return; } + // Re-check the connection still exists before any new mutation below. + // Runs once, under the SAME held fence every write in this batch (the + // reservation, the per-record ctx.ingestRecord loop, and + // prepareDeviceFinalRecords) reuses via `coordinatorOwnership` — closes + // the delete/write TOCTOU for the whole batch in one check, not one per + // record. An already-accepted replay (returned above) is a historical + // read, not a new write, so it is intentionally NOT gated by this check. + await ctx.assertConnectorInstanceWritable(connectorInstanceId); + // Accepted replays returned above intentionally do not consult the current // manifest or semantic backend. Every new/processing attempt does. const attemptContext = await compileDeviceAttemptContext(ctx, connectorId, records); diff --git a/reference-implementation/server/routes/ref-error-status.ts b/reference-implementation/server/routes/ref-error-status.ts index 08105f888..19fc53a31 100644 --- a/reference-implementation/server/routes/ref-error-status.ts +++ b/reference-implementation/server/routes/ref-error-status.ts @@ -96,6 +96,7 @@ export const codeToStatus: Readonly> = { connection_not_found: 404, connection_run_active: 409, connection_tombstoned: 409, + connector_instance_busy: 503, connector_instance_connector_mismatch: 400, connector_instance_inactive: 400, connector_instance_not_found: 404, @@ -139,6 +140,15 @@ export const codeToStatus: Readonly> = { query_not_found: 404, run_already_active: 409, run_owner_mismatch: 403, + static_secret_binding_invalid: 409, + static_secret_draft_required: 409, + static_secret_identity_ambiguous: 409, + static_secret_identity_conflict: 409, + static_secret_identity_mismatch: 409, + static_secret_identity_missing: 502, + static_secret_identity_revoked: 409, + static_secret_identity_unavailable: 503, + static_secret_identity_unverified_replacement: 409, unknown_field: 400, unsupported_version: 400, }; diff --git a/reference-implementation/server/routes/ref-manual-upload-draft-connection.ts b/reference-implementation/server/routes/ref-manual-upload-draft-connection.ts index adc9df36b..cc92d9c45 100644 --- a/reference-implementation/server/routes/ref-manual-upload-draft-connection.ts +++ b/reference-implementation/server/routes/ref-manual-upload-draft-connection.ts @@ -87,6 +87,51 @@ interface ConnectorInstanceStore { }) => Promise | ConnectorInstance; } +// Written by both createManualUploadDraftConnection and +// validateAndStageArtifact below; each sets a different subset of the +// optional fields. +export interface ManualUploadDraftSourceBinding { + readonly acquisition_method: "owner_artifact"; + readonly import_dir: string; + readonly import_dir_env_var: string; + readonly import_validation?: unknown; + readonly kind: "manual_upload_draft"; + readonly staged_upload?: boolean; + readonly uploaded_file_name?: string; +} + +// import_dir/import_dir_env_var are read on every run (connection-scoped-run-env.ts, +// buildControllerManualUploadRunEnvResolver), not just at setup — they must +// survive promotion. +export interface ManualUploadDurableSourceBinding { + readonly acquisition_method: "owner_artifact"; + readonly import_dir: string; + readonly import_dir_env_var: string; + readonly import_validation?: unknown; + readonly kind: "manual_upload"; + readonly promoted_at: string; + readonly promoted_from: "manual_upload_draft"; + readonly uploaded_file_name?: string; +} + +// Pure — no I/O. `staged_upload` does not carry over: it meant "no file +// staged yet," no longer true once records have ingested. +export function promoteManualUploadDraftBinding( + draftBinding: ManualUploadDraftSourceBinding, + now: string +): ManualUploadDurableSourceBinding { + return { + acquisition_method: draftBinding.acquisition_method, + import_dir: draftBinding.import_dir, + import_dir_env_var: draftBinding.import_dir_env_var, + kind: "manual_upload", + promoted_at: now, + promoted_from: "manual_upload_draft", + ...(draftBinding.import_validation === undefined ? {} : { import_validation: draftBinding.import_validation }), + ...(draftBinding.uploaded_file_name === undefined ? {} : { uploaded_file_name: draftBinding.uploaded_file_name }), + }; +} + interface AcquisitionBatch { readonly acceptedCount?: number | null; readonly artifactSha256?: string | null; diff --git a/reference-implementation/server/routes/ref-static-secret-credentials.ts b/reference-implementation/server/routes/ref-static-secret-credentials.ts index d42f4a4e8..ed47d3d1e 100644 --- a/reference-implementation/server/routes/ref-static-secret-credentials.ts +++ b/reference-implementation/server/routes/ref-static-secret-credentials.ts @@ -8,7 +8,21 @@ // route and it never returns the submitted secret. Owner-agent intent may point // at the owner-session capture page, but it never carries the credential itself. -import { type ConnectorManifestLike, expectedStaticSecretCredentialKind } from "../connection-setup-plan.ts"; +import { + type ConnectorManifestLike, + expectedStaticSecretCredentialKind, + type StaticSecretSetupField, + staticSecretCredentialCaptureFromManifest, +} from "../connection-setup-plan.ts"; +import { + assertStaticSecretActiveCredentialReplacementAllowed, + isStaticSecretBindingUniqueConflict, + isStaticSecretPipelineBinding, + parseStaticSecretSetupFields, + staticSecretBindingRecord, + staticSecretIdentityClaim, + staticSecretSetupFieldsFromBinding, +} from "../static-secret-identity.ts"; import { isCredentialEncryptionConfigured } from "../stores/credential-encryption.ts"; import type { MiddlewareHandler, PdppErrorFn, RouteArg } from "./_route-contract.ts"; import { codeToStatus } from "./ref-error-status.ts"; @@ -111,18 +125,38 @@ interface ConnectorInstanceCredentialStore { secret: string; now: string; }) => Promise | CredentialMetadata; + // Non-secret, key-derived fingerprint of a candidate plaintext — used to + // prove "is this the exact same credential already stored" without + // sealing/persisting anything. See connector-instance-credential-store.ts. + fingerprintCandidate: (secret: string) => string | null; getMetadata: (connectorInstanceId: string) => Promise | CredentialMetadata | null; } interface ConnectorInstanceRow { readonly connectorId: string; readonly connectorInstanceId: string; + readonly ownerSubjectId: string; readonly sourceBinding?: unknown; + readonly sourceBindingKey: string; readonly status: string; } interface ConnectorInstanceStore { get: (connectorInstanceId: string) => Promise | ConnectorInstanceRow | null; + getByBinding: (input: { + ownerSubjectId: string; + connectorId: string; + sourceKind: string; + sourceBindingKey: string; + }) => Promise | ConnectorInstanceRow | null; + updateStaticSecretBinding: (input: { + connectorInstanceId: string; + connectorId: string; + ownerSubjectId: string; + sourceBinding: Record; + sourceBindingKey: string; + updatedAt: string; + }) => Promise | ConnectorInstanceRow | null; updateStatus: ( connectorInstanceId: string, args: { readonly revokedAt?: string | null; readonly status: string; readonly updatedAt: string } @@ -155,11 +189,9 @@ export interface MountRefStaticSecretCredentialsContext { // given (matching the existing draft-route fallback). canonicalConnectorKey?: (value: string | null | undefined) => string | null; createRequestConnectorInstanceCredentialStore: () => ConnectorInstanceCredentialStore; - // Connector-instance store, used to recover the draft's non-secret setup - // fields for the probe context and to retire rejected first-time draft setup - // rows. Optional: when absent the probe runs with no setup-field context - // (fine for connectors whose probe needs none, e.g. GitHub) and cannot - // perform draft cleanup. + // Connector-instance store, used to recover/update non-secret setup fields + // and claim a verified provider identity. Optional only for narrow injected + // callers that do not use setup-field or identity-aware probing. createRequestConnectorInstanceStore?: () => ConnectorInstanceStore; createTraceContext: (input?: { scenarioId?: string }) => TraceContext; emitSpineEvent: (event: Record) => Promise; @@ -201,12 +233,33 @@ function errWithCode(code: string): { code: string } { return { code }; } -async function expectedCredentialKindForConnector( +function codedError(code: string, message: string): Error & { code: string } { + const error = new Error(message) as Error & { code: string }; + error.code = code; + return error; +} + +function throwCodedError(code: string, message: string): never { + throw codedError(code, message); +} + +function nowFor(ctx: MountRefStaticSecretCredentialsContext): string { + return ctx.now ? ctx.now() : new Date().toISOString(); +} + +interface StaticSecretCredentialContract { + readonly credentialKind: string; + readonly fields: readonly StaticSecretSetupField[]; +} + +async function staticSecretCredentialContract( ctx: MountRefStaticSecretCredentialsContext, connectorId: string -): Promise { +): Promise { const manifest = await ctx.resolveRegisteredConnectorManifest(connectorId); - return expectedStaticSecretCredentialKind(connectorId, manifest); + const credentialKind = expectedStaticSecretCredentialKind(connectorId, manifest); + const capture = staticSecretCredentialCaptureFromManifest(manifest); + return credentialKind && capture ? { credentialKind, fields: capture.fields } : null; } function projectCredentialMetadata(meta: CredentialMetadata): Record { @@ -337,67 +390,17 @@ async function emitCaptureAudit( }); } -// Pull the non-secret setup fields out of the draft's source binding. The draft -// binding is `{ kind: "static_secret_draft", setup_fields: {...} }`; only -// non-secret fields are ever stored there (the secret lives in the credential -// store), so this is safe to read for the probe context. -function setupFieldsFromBinding(sourceBinding: unknown): Record | null { - if (!sourceBinding || typeof sourceBinding !== "object" || Array.isArray(sourceBinding)) { - return null; - } - const raw = (sourceBinding as { setup_fields?: unknown }).setup_fields; - if (!raw || typeof raw !== "object" || Array.isArray(raw)) { - return null; - } - const fields: Record = {}; - for (const [key, value] of Object.entries(raw)) { - if (typeof value === "string" && value.length > 0) { - fields[key] = value; - } - } - return Object.keys(fields).length > 0 ? fields : null; -} - -function isStaticSecretDraftInstance(instance: ConnectorInstanceRow | null): instance is ConnectorInstanceRow { - if (instance?.status !== "draft") { - return false; - } - const binding = instance.sourceBinding; - return Boolean( - binding && - typeof binding === "object" && - !Array.isArray(binding) && - (binding as { kind?: unknown }).kind === "static_secret_draft" - ); -} - -async function retireRejectedStaticSecretDraft( - ctx: MountRefStaticSecretCredentialsContext, - connectorInstanceId: string, - now: string -): Promise { - if (typeof ctx.createRequestConnectorInstanceStore !== "function") { - return; - } - const store = ctx.createRequestConnectorInstanceStore(); - const instance = await store.get(connectorInstanceId); - if (!isStaticSecretDraftInstance(instance)) { - return; - } - await store.updateStatus(connectorInstanceId, { - revokedAt: now, - status: "revoked", - updatedAt: now, - }); -} - // Resolve the non-secret setup-field context for a connector's probe by reading // the draft instance's source binding. Best-effort: a connector whose probe // needs no setup fields (e.g. GitHub) is unaffected when this returns null. async function probeContextForInstance( ctx: MountRefStaticSecretCredentialsContext, - connectorInstanceId: string + connectorInstanceId: string, + setupFieldsOverride?: Record ): Promise { + if (setupFieldsOverride) { + return { connectorInstanceId, setupFields: setupFieldsOverride }; + } if (typeof ctx.createRequestConnectorInstanceStore !== "function") { return { connectorInstanceId, setupFields: null }; } @@ -405,7 +408,7 @@ async function probeContextForInstance( const instance = await store.get(connectorInstanceId); return { connectorInstanceId, - setupFields: instance ? setupFieldsFromBinding(instance.sourceBinding) : null, + setupFields: instance ? staticSecretSetupFieldsFromBinding(instance.sourceBinding) : null, }; } @@ -413,7 +416,7 @@ function parseCaptureBody( ctx: MountRefStaticSecretCredentialsContext, res: RouteResponse, body: unknown -): { credentialKind: string | null; secret: string } | null { +): { credentialKind: string | null; secret: string; setupFieldsRaw?: unknown } | null { const objectBody = (body as Record | null) || {}; // biome-ignore lint/style/useDestructuring: Explicit property or positional access documents this compatibility boundary. const secret = objectBody.secret; @@ -430,12 +433,209 @@ function parseCaptureBody( return { credentialKind: typeof objectBody.credential_kind === "string" ? objectBody.credential_kind.trim() : null, secret, + ...(Object.hasOwn(objectBody, "setup_fields") ? { setupFieldsRaw: objectBody.setup_fields } : {}), }; } +async function updateDraftSetupFieldsBeforeProbe( + ctx: MountRefStaticSecretCredentialsContext, + input: { + connectorId: string; + connectorInstanceId: string; + ownerSubjectId: string; + setupFields?: Record; + } +): Promise { + if (!input.setupFields || typeof ctx.createRequestConnectorInstanceStore !== "function") { + return; + } + const store = ctx.createRequestConnectorInstanceStore(); + const instance = await store.get(input.connectorInstanceId); + if (instance?.status !== "draft") { + return; + } + const binding = staticSecretBindingRecord(instance.sourceBinding); + if (binding?.kind !== "static_secret_draft") { + throwCodedError("static_secret_draft_required", "Only a static-secret draft can update setup fields during retry."); + } + binding.setup_fields = input.setupFields; + await store.updateStaticSecretBinding({ + connectorId: input.connectorId, + connectorInstanceId: input.connectorInstanceId, + ownerSubjectId: input.ownerSubjectId, + sourceBinding: binding, + sourceBindingKey: instance.sourceBindingKey, + updatedAt: ctx.now ? ctx.now() : new Date().toISOString(), + }); +} + +interface ClaimedStaticSecretIdentity { + readonly deduplicated: boolean; + readonly instance: ConnectorInstanceRow; +} + +function requireConnectorInstanceStore(ctx: MountRefStaticSecretCredentialsContext): ConnectorInstanceStore { + if (typeof ctx.createRequestConnectorInstanceStore !== "function") { + throwCodedError( + "static_secret_identity_unavailable", + "A connector-instance store is required to claim a verified provider identity." + ); + } + return ctx.createRequestConnectorInstanceStore(); +} + +async function currentInstanceOrThrow( + store: ConnectorInstanceStore, + connectorInstanceId: string +): Promise { + const current = await store.get(connectorInstanceId); + if (!current) { + throwCodedError("connector_instance_not_found", `Connection '${connectorInstanceId}' does not exist.`); + } + return current; +} + +function currentBindingOrThrow(current: ConnectorInstanceRow): Record { + const binding = staticSecretBindingRecord(current.sourceBinding); + if (!binding) { + throwCodedError( + "static_secret_binding_invalid", + "The connection has no valid static-secret binding; refusing to store the credential." + ); + } + return binding; +} + +async function resolveIdentityBindingConflict( + ctx: MountRefStaticSecretCredentialsContext, + store: ConnectorInstanceStore, + current: ConnectorInstanceRow, + sourceBindingKey: string, + originalError: unknown, + input: { connectorId: string; ownerSubjectId: string } +): Promise { + const winner = await store.getByBinding({ + connectorId: input.connectorId, + ownerSubjectId: input.ownerSubjectId, + sourceBindingKey, + sourceKind: "account", + }); + if (!winner) { + throw originalError; + } + if (winner.status === "revoked") { + throwCodedError( + "static_secret_identity_revoked", + "This provider identity belongs to a revoked connection; refusing to create or reactivate another connection silently." + ); + } + if (current.status === "active" && winner.connectorInstanceId !== current.connectorInstanceId) { + throwCodedError( + "static_secret_identity_conflict", + "Another active connection already owns this verified provider identity; refusing to retarget this active connection." + ); + } + if (winner.connectorInstanceId !== current.connectorInstanceId && current.status === "draft") { + const now = nowFor(ctx); + await store.updateStatus(current.connectorInstanceId, { + revokedAt: now, + status: "revoked", + updatedAt: now, + }); + } + return { deduplicated: true, instance: winner }; +} + +async function claimProbedStaticSecretIdentity( + ctx: MountRefStaticSecretCredentialsContext, + input: { + connectorId: string; + connectorInstanceId: string; + ownerSubjectId: string; + probedIdentity: string; + secret: string; + setupFields?: Record; + } +): Promise { + const store = requireConnectorInstanceStore(ctx); + const { identity, sourceBindingKey } = staticSecretIdentityClaim(input); + const current = await currentInstanceOrThrow(store, input.connectorInstanceId); + const binding = currentBindingOrThrow(current); + if (current.status === "active" && isStaticSecretPipelineBinding(current.sourceBinding)) { + const credentialStore = ctx.createRequestConnectorInstanceCredentialStore(); + const existingCredential = await credentialStore.getMetadata(input.connectorInstanceId); + assertStaticSecretActiveCredentialReplacementAllowed({ + existingCredentialFingerprint: existingCredential?.fingerprint ?? null, + hasExistingCredential: existingCredential !== null, + newSecretFingerprint: credentialStore.fingerprintCandidate(input.secret), + probedIdentity: identity, + sourceBinding: current.sourceBinding, + status: current.status, + }); + } + binding.verified_identity = identity; + if (input.setupFields) { + binding.setup_fields = input.setupFields; + } + try { + const updated = await store.updateStaticSecretBinding({ + connectorId: input.connectorId, + connectorInstanceId: input.connectorInstanceId, + ownerSubjectId: input.ownerSubjectId, + sourceBinding: binding, + sourceBindingKey, + updatedAt: nowFor(ctx), + }); + if (!updated) { + throwCodedError( + "connector_instance_not_found", + `Connection '${input.connectorInstanceId}' is no longer capturable.` + ); + } + return { deduplicated: false, instance: updated }; + } catch (err) { + if (!isStaticSecretBindingUniqueConflict(err)) { + throw err; + } + return resolveIdentityBindingConflict(ctx, store, current, sourceBindingKey, err, input); + } +} + +// Guards a credential replacement when there is no probed identity to claim +// (no-probe connector, or a probe that self-reported skipped). The active- +// connection fail-closed rule still applies here — this is the primary +// reproduction path for PR #84's P1: a connection that reached `active` +// through first-sync with no probe ever running has no durable identity +// signal on its binding, and nothing about that absence licenses a silent +// credential swap. +// +// Deliberately never passes anything derived from `setup_fields` as identity +// proof here: those are owner-typed, non-secret, and trivially resubmittable +// by an attacker alongside a stolen secret. With no probe, the only channel +// this route can offer is the credential fingerprint. +async function assertActiveReplacementAllowedWithoutProbe( + ctx: MountRefStaticSecretCredentialsContext, + input: { connectorInstanceId: string; secret: string } +): Promise { + const store = requireConnectorInstanceStore(ctx); + const current = await currentInstanceOrThrow(store, input.connectorInstanceId); + if (current.status !== "active" || !isStaticSecretPipelineBinding(current.sourceBinding)) { + return; + } + const credentialStore = ctx.createRequestConnectorInstanceCredentialStore(); + const existingCredential = await credentialStore.getMetadata(input.connectorInstanceId); + assertStaticSecretActiveCredentialReplacementAllowed({ + existingCredentialFingerprint: existingCredential?.fingerprint ?? null, + hasExistingCredential: existingCredential !== null, + newSecretFingerprint: credentialStore.fingerprintCandidate(input.secret), + sourceBinding: current.sourceBinding, + status: current.status, + }); +} + // Runs the synchronous credential probe when one is configured. Returns the // probed identity on success, null-with-side-effects on rejection (audit emitted, -// error sent, draft retired), or `{ probedIdentity: null }` when the probe is +// error sent, draft remains resumable), or `{ probedIdentity: null }` when the probe is // absent or skipped. async function runCredentialProbe( ctx: MountRefStaticSecretCredentialsContext, @@ -448,20 +648,19 @@ async function runCredentialProbe( credentialKind: string | null; ownerSubjectId: string | null; secret: string; + setupFields?: Record; } ): Promise<{ probedIdentity: { detail: string | null; identity: string } | null } | null> { if (typeof ctx.probeStaticSecretCredential !== "function") { return { probedIdentity: null }; } - const probeContext = await probeContextForInstance(ctx, args.connectorInstanceId); + const probeContext = await probeContextForInstance(ctx, args.connectorInstanceId, args.setupFields); const probeResult = await ctx.probeStaticSecretCredential({ connectorKey: args.connectorKey, context: probeContext, secret: args.secret, }); if (!probeResult.ok) { - const now = ctx.now ? ctx.now() : new Date().toISOString(); - await retireRejectedStaticSecretDraft(ctx, args.connectorInstanceId, now); await emitCaptureAudit(ctx, req, res, { connectionId: args.connectorInstanceId, connectorId: args.connectorId, @@ -481,7 +680,7 @@ async function runCredentialProbe( // Validates the expected credential kind for a namespace and that encryption is // configured. Emits audit + sends the error response on failure and returns -// false; returns true when all checks pass. +// null; returns the manifest's non-secret setup fields when all checks pass. async function validateCredentialKind( ctx: MountRefStaticSecretCredentialsContext, req: RouteRequest, @@ -489,9 +688,9 @@ async function validateCredentialKind( namespace: ConnectorNamespace, credentialKind: string | null, ownerSubjectId: string | null -): Promise { - const expectedKind = await expectedCredentialKindForConnector(ctx, namespace.connectorId); - if (!expectedKind) { +): Promise { + const contract = await staticSecretCredentialContract(ctx, namespace.connectorId); + if (!contract) { await emitCaptureAudit(ctx, req, res, { connectionId: namespace.connectorInstanceId, connectorId: namespace.connectorId, @@ -506,9 +705,9 @@ async function validateCredentialKind( "static_secret_credential_unsupported", `Connection '${namespace.connectorInstanceId}' belongs to connector '${namespace.connectorId}', which is not a static-secret connector.` ); - return false; + return null; } - if (credentialKind !== expectedKind) { + if (credentialKind !== contract.credentialKind) { await emitCaptureAudit(ctx, req, res, { connectionId: namespace.connectorInstanceId, connectorId: namespace.connectorId, @@ -521,10 +720,10 @@ async function validateCredentialKind( res, 400, "credential_kind_mismatch", - `credential_kind must be '${expectedKind}' for connector '${namespace.connectorId}'.`, + `credential_kind must be '${contract.credentialKind}' for connector '${namespace.connectorId}'.`, "credential_kind" ); - return false; + return null; } // Fail closed before probing when the instance-level credential key // provider is missing: there is no point validating a credential we @@ -545,9 +744,9 @@ async function validateCredentialKind( "credential_encryption_key_missing", "Credential encryption is required but no instance-level key provider is configured. Configure it before capturing a static-secret credential. No credential was validated or stored." ); - return false; + return null; } - return true; + return contract; } // Stores the validated credential and sends the success response. @@ -557,6 +756,7 @@ async function storeAndRespond( res: RouteResponse, args: { credentialKind: string | null; + deduplicated?: boolean; namespace: ConnectorNamespace; ownerSubjectId: string | null; probedIdentity: { detail: string | null; identity: string } | null; @@ -595,6 +795,7 @@ async function storeAndRespond( identity: args.probedIdentity ? { account_identity: args.probedIdentity.identity, detail: args.probedIdentity.detail } : null, + ...(args.deduplicated ? { deduplicated: true } : {}), next_step: { kind: "run_connection", method: "POST", @@ -613,92 +814,169 @@ async function storeAndRespond( // Owner-session-only credential capture for one existing connection. The // plaintext appears only in the request body and the store's sealing call; the // response and audit event contain non-secret metadata only. +interface CaptureRequestState { + credentialKind: string | null; + namespace: ConnectorNamespace | null; + ownerSubjectId: string | null; +} + +async function runStaticSecretCredentialCapture( + ctx: MountRefStaticSecretCredentialsContext, + req: RouteRequest, + res: RouteResponse, + connectorInstanceId: string, + state: CaptureRequestState +): Promise { + const ownerSubjectId = ctx.getOwnerSubjectId(req); + state.ownerSubjectId = ownerSubjectId; + const capture = parseCaptureBody(ctx, res, req.body); + if (!capture) { + await emitCaptureAudit(ctx, req, res, { + connectionId: connectorInstanceId, + credentialKind: state.credentialKind, + error: errWithCode("invalid_request"), + outcome: "failed", + ownerSubjectId, + }); + return; + } + const { credentialKind, secret, setupFieldsRaw } = capture; + state.credentialKind = credentialKind; + const namespace = await ctx.resolveOwnerConnectorNamespace(req, null, { + allowDefaultAccount: false, + // Admit a `draft` target so the owner can seal a credential onto a + // not-yet-ingested first static-secret connection. This is owner- + // session-only; no bearer/agent path passes allowStatuses. See + // add-static-secret-owner-session-connect-path design Decisions 3 & 5. + allowStatuses: ["active", "draft"], + connectorInstanceId, + ownerSubjectId, + }); + state.namespace = namespace; + const contract = await validateCredentialKind(ctx, req, res, namespace, credentialKind, ownerSubjectId); + if (!contract) { + return; + } + const submittedSetupFields = parseStaticSecretSetupFields(setupFieldsRaw, contract.fields, (code, message, param) => + ctx.pdppError(res, 400, code, message, param) + ); + if (submittedSetupFields === null) { + return; + } + await updateDraftSetupFieldsBeforeProbe(ctx, { + connectorId: namespace.connectorId, + connectorInstanceId: namespace.connectorInstanceId, + ownerSubjectId, + ...(submittedSetupFields ? { setupFields: submittedSetupFields } : {}), + }); + // Synchronous validation moment (owner-journey flow design B1). When a + // probe is injected, validate the credential against the provider BEFORE + // storing it. A known-bad credential is rejected and NOTHING is written. + // A skipped probe preserves the first-sync path. + const probeConnectorKey = ctx.canonicalConnectorKey + ? (ctx.canonicalConnectorKey(namespace.connectorId) ?? namespace.connectorId) + : namespace.connectorId; + const probeOutcome = await runCredentialProbe(ctx, req, res, { + connectorId: namespace.connectorId, + connectorInstanceId: namespace.connectorInstanceId, + connectorKey: probeConnectorKey, + credentialKind, + ownerSubjectId, + secret, + ...(submittedSetupFields ? { setupFields: submittedSetupFields } : {}), + }); + if (probeOutcome === null) { + return; + } + const { probedIdentity } = probeOutcome; + const { deduplicated, responseNamespace } = await claimOrGuardReplacement(ctx, { + namespace, + ownerSubjectId, + probedIdentity, + secret, + ...(submittedSetupFields ? { submittedSetupFields } : {}), + }); + await storeAndRespond(ctx, req, res, { + credentialKind, + ...(deduplicated ? { deduplicated: true } : {}), + namespace: responseNamespace, + ownerSubjectId, + probedIdentity, + secret, + }); +} + +// After a probe either names an identity or is skipped, either claims that +// identity (existing behavior) or — with no probed identity to claim — still +// runs the active-replacement guard. An active connection with no durable, +// provider-verified identity is never silently retargetable just because +// nothing on record contradicts the new secret; owner-typed `setup_fields` +// are never consulted as proof here (see assertActiveReplacementAllowedWithoutProbe). +async function claimOrGuardReplacement( + ctx: MountRefStaticSecretCredentialsContext, + input: { + namespace: ConnectorNamespace; + ownerSubjectId: string; + probedIdentity: { detail: string | null; identity: string } | null; + secret: string; + submittedSetupFields?: Record; + } +): Promise<{ deduplicated: boolean; responseNamespace: ConnectorNamespace }> { + if (input.probedIdentity) { + const claim = await claimProbedStaticSecretIdentity(ctx, { + connectorId: input.namespace.connectorId, + connectorInstanceId: input.namespace.connectorInstanceId, + ownerSubjectId: input.ownerSubjectId, + probedIdentity: input.probedIdentity.identity, + secret: input.secret, + ...(input.submittedSetupFields ? { setupFields: input.submittedSetupFields } : {}), + }); + return { + deduplicated: claim.deduplicated, + responseNamespace: { ...input.namespace, connectorInstanceId: claim.instance.connectorInstanceId }, + }; + } + await assertActiveReplacementAllowedWithoutProbe(ctx, { + connectorInstanceId: input.namespace.connectorInstanceId, + secret: input.secret, + }); + return { deduplicated: false, responseNamespace: input.namespace }; +} + +async function handleStaticSecretCredentialCapture( + ctx: MountRefStaticSecretCredentialsContext, + req: RouteRequest, + res: RouteResponse, + connectorInstanceId: string +): Promise { + const state: CaptureRequestState = { credentialKind: null, namespace: null, ownerSubjectId: null }; + try { + await runStaticSecretCredentialCapture(ctx, req, res, connectorInstanceId, state); + } catch (err) { + const auditNamespace = state.namespace; + await emitCaptureAudit(ctx, req, res, { + connectionId: auditNamespace ? auditNamespace.connectorInstanceId : connectorInstanceId, + connectorId: auditNamespace ? auditNamespace.connectorId : null, + credentialKind: state.credentialKind, + error: err, + outcome: "failed", + ownerSubjectId: state.ownerSubjectId, + }); + const status = credentialCaptureErrorStatus(err); + const { code } = err as { code?: unknown }; + if (typeof code === "string" && status !== 500) { + ctx.pdppError(res, status, code, err instanceof Error ? err.message : String(err)); + return; + } + ctx.handleError(res, err); + } +} + export function mountRefStaticSecretCredentialCapture(app: AppLike, ctx: MountRefStaticSecretCredentialsContext): void { app.post( "/_ref/connections/:connectorInstanceId/static-secret-credential", ctx.requireOwnerSession, - async (req: RouteRequest, res: RouteResponse) => { - const connectorInstanceId = decodeURIComponent(req.params.connectorInstanceId as string); - let ownerSubjectId: string | null = null; - let namespace: ConnectorNamespace | null = null; - let credentialKind: string | null = null; - try { - ownerSubjectId = ctx.getOwnerSubjectId(req); - const capture = parseCaptureBody(ctx, res, req.body); - if (!capture) { - await emitCaptureAudit(ctx, req, res, { - connectionId: connectorInstanceId, - credentialKind, - error: errWithCode("invalid_request"), - outcome: "failed", - ownerSubjectId, - }); - return; - } - // biome-ignore lint/style/useDestructuring: Explicit property or positional access documents this compatibility boundary. - credentialKind = capture.credentialKind; - namespace = await ctx.resolveOwnerConnectorNamespace(req, null, { - allowDefaultAccount: false, - // Admit a `draft` target so the owner can seal a credential onto a - // not-yet-ingested first static-secret connection. This is owner- - // session-only; no bearer/agent path passes allowStatuses. See - // add-static-secret-owner-session-connect-path design Decisions 3 & 5. - allowStatuses: ["active", "draft"], - connectorInstanceId, - ownerSubjectId, - }); - const kindOk = await validateCredentialKind(ctx, req, res, namespace, credentialKind, ownerSubjectId); - if (!kindOk) { - return; - } - // Synchronous validation moment (owner-journey flow design B1). When a - // probe is injected, validate the credential against the provider BEFORE - // storing it: a known-bad credential is rejected with a provider-named, - // owner-causal message and NOTHING is written to the credential store. - // The prober self-reports `skipped: true` for a connector with no probe, - // so the first-sync path is preserved without a separate gate. The probe - // is injected, so no live provider call happens under test. - const probeConnectorKey = ctx.canonicalConnectorKey - ? (ctx.canonicalConnectorKey(namespace.connectorId) ?? namespace.connectorId) - : namespace.connectorId; - const probeOutcome = await runCredentialProbe(ctx, req, res, { - connectorId: namespace.connectorId, - connectorInstanceId: namespace.connectorInstanceId, - connectorKey: probeConnectorKey, - credentialKind, - ownerSubjectId, - secret: capture.secret, - }); - if (probeOutcome === null) { - return; - } - await storeAndRespond(ctx, req, res, { - credentialKind, - namespace, - ownerSubjectId, - probedIdentity: probeOutcome.probedIdentity, - secret: capture.secret, - }); - } catch (err) { - await emitCaptureAudit(ctx, req, res, { - // biome-ignore lint/suspicious/noUnnecessaryConditions: TypeScript boundary permits nullish input; this guard preserves runtime behavior. - connectionId: namespace?.connectorInstanceId ?? connectorInstanceId, - // biome-ignore lint/suspicious/noUnnecessaryConditions: TypeScript boundary permits nullish input; this guard preserves runtime behavior. - connectorId: namespace?.connectorId ?? null, - credentialKind, - error: err, - outcome: "failed", - ownerSubjectId, - }); - const status = credentialCaptureErrorStatus(err); - // biome-ignore lint/suspicious/noUnnecessaryConditions: TypeScript boundary permits nullish input; this guard preserves runtime behavior. - const code = (err as { code?: unknown })?.code; - if (typeof code === "string" && status !== 500) { - ctx.pdppError(res, status, code, (err as Error).message); - return; - } - ctx.handleError(res, err); - } - } + async (req: RouteRequest, res: RouteResponse) => + handleStaticSecretCredentialCapture(ctx, req, res, decodeURIComponent(req.params.connectorInstanceId as string)) ); } diff --git a/reference-implementation/server/routes/ref-static-secret-draft-connection.ts b/reference-implementation/server/routes/ref-static-secret-draft-connection.ts index 73bcaf38f..91c72a577 100644 --- a/reference-implementation/server/routes/ref-static-secret-draft-connection.ts +++ b/reference-implementation/server/routes/ref-static-secret-draft-connection.ts @@ -12,9 +12,9 @@ // // It is NOT an owner-agent bearer route: `requireOwnerSession` (cookie) gates // it, and it never accepts or returns a provider secret. Non-static-secret -// connectors are refused. Each call mints a fresh random source-binding key, so -// two mailboxes become two distinct `connection_id`s. See -// add-static-secret-owner-session-connect-path design Decision 4. +// connectors are refused. Manifest-declared identities get a deterministic +// draft binding key so retries converge; connectors without a safe identity +// retain a random key so distinct accounts cannot be collapsed. import { randomBytes } from "node:crypto"; @@ -26,6 +26,12 @@ import { type StaticSecretSetupField, staticSecretCredentialCaptureFromManifest, } from "../connection-setup-plan.ts"; +import { + findExistingStaticSecretIdentity, + parseStaticSecretDraftSetupFields, + staticSecretDraftIdentityBindingKey, + staticSecretSetupIdentity, +} from "../static-secret-identity.ts"; import { CREDENTIAL_ENCRYPTION_KEY_ENV, CREDENTIAL_ENCRYPTION_KEY_FILE_ENV, @@ -63,10 +69,24 @@ interface ConnectorInstance { readonly connectorId: string; readonly connectorInstanceId: string; readonly displayName?: string | null; + readonly ownerSubjectId: string; + readonly sourceBinding?: unknown; + readonly sourceBindingKey?: string; readonly status: string; } interface ConnectorInstanceStore { + getByBinding: (input: { + ownerSubjectId: string; + connectorId: string; + sourceKind: string; + sourceBindingKey: string; + }) => Promise | ConnectorInstance | null; + listActiveByConnector: ( + ownerSubjectId: string, + connectorId: string, + options?: { limit?: number } + ) => Promise | ConnectorInstance[]; upsert: (record: { ownerSubjectId: string; connectorId: string; @@ -80,6 +100,51 @@ interface ConnectorInstanceStore { }) => Promise | ConnectorInstance; } +// The binding `createDraftConnection` below writes at draft-creation time. +export interface StaticSecretDraftSourceBinding { + readonly kind: "static_secret_draft"; + readonly setup_fields: Record; + readonly verified_identity?: string; +} + +// The credential itself lives in connector-instance-credential-store, never +// in this binding; setup_fields (non-secret manifest fields) is read on +// every credential probe and run, so it must survive promotion. +export interface StaticSecretDurableSourceBinding { + readonly kind: "static_secret"; + readonly promoted_at: string; + readonly promoted_from: "static_secret_draft"; + readonly setup_fields: Record; + readonly verified_identity?: string; +} + +// Pure — no I/O. +export function promoteStaticSecretDraftBinding( + draftBinding: StaticSecretDraftSourceBinding, + now: string +): StaticSecretDurableSourceBinding { + return { + kind: "static_secret", + promoted_at: now, + promoted_from: "static_secret_draft", + setup_fields: draftBinding.setup_fields, + ...(draftBinding.verified_identity ? { verified_identity: draftBinding.verified_identity } : {}), + }; +} + +interface ParsedDisplayName { + readonly displayName: string | null; + readonly ok: true; +} + +interface InvalidDisplayName { + readonly error: { + readonly message: string; + readonly param: "display_name"; + }; + readonly ok: false; +} + export interface MountRefStaticSecretDraftConnectionContext { canonicalConnectorKey: (value: string | null | undefined) => string | null; createRequestConnectorInstanceStore: () => ConnectorInstanceStore; @@ -191,43 +256,73 @@ function projectSetup(connectorId: string, manifest: ConnectorManifestLike): Rec }; } -function parseSetupFields( +function bodyRecord(body: unknown): Record { + return body && typeof body === "object" && !Array.isArray(body) ? (body as Record) : {}; +} + +function parseDisplayNameText(raw: string): ParsedDisplayName | InvalidDisplayName { + const displayName = raw.trim(); + if (!displayName) { + return { + displayName: null, + ok: true, + }; + } + if (displayName.length > 200) { + return { + error: { message: "display_name must be 200 characters or fewer", param: "display_name" }, + ok: false, + }; + } + return { displayName, ok: true }; +} + +function parseDisplayNameValue(raw: unknown): ParsedDisplayName | InvalidDisplayName { + if (raw === null || raw === undefined) { + return { displayName: null, ok: true }; + } + if (typeof raw !== "string") { + return { + error: { message: "display_name must be a string when provided", param: "display_name" }, + ok: false, + }; + } + return parseDisplayNameText(raw); +} + +function parseOptionalDisplayName(body: unknown): ParsedDisplayName | InvalidDisplayName { + const objectBody = bodyRecord(body); + if (!Object.hasOwn(objectBody, "display_name")) { + return { displayName: null, ok: true }; + } + return parseDisplayNameValue(objectBody.display_name); +} + +interface ParsedDraftSetup { + readonly displayName: string | null; + readonly setupFields: Record; +} + +function parseDraftSetup( ctx: MountRefStaticSecretDraftConnectionContext, res: RouteResponse, body: unknown, fields: readonly StaticSecretSetupField[] -): Record | null { - const objectBody = (body as Record | null) || {}; - const raw = objectBody.setup_fields; - const provided = raw && typeof raw === "object" && !Array.isArray(raw) ? (raw as Record) : {}; - const allowed = new Set(fields.filter((field) => !field.secret).map((field) => field.name)); - const output: Record = {}; - for (const key of Object.keys(provided)) { - if (!allowed.has(key)) { - ctx.pdppError(res, 400, "unknown_setup_field", `Unknown setup field: ${key}`, `setup_fields.${key}`); - return null; - } - } - for (const field of fields) { - if (field.secret) { - continue; - } - const value = provided[field.name]; - const text = typeof value === "string" ? value.trim() : ""; - if (field.required && !text) { - ctx.pdppError(res, 400, "missing_setup_field", `${field.label} is required.`, `setup_fields.${field.name}`); - return null; - } - if (text) { - output[field.name] = text; - } +): ParsedDraftSetup | null { + const objectBody = bodyRecord(body); + const setupFields = parseStaticSecretDraftSetupFields(objectBody.setup_fields, fields, (code, message, param) => + ctx.pdppError(res, 400, code, message, param) + ); + if (setupFields === null) { + return null; } - return output; -} -function identityValue(fields: readonly StaticSecretSetupField[], setupFields: Record): string | null { - const field = fields.find((candidate) => candidate.identity && !candidate.secret); - return field ? (setupFields[field.name] ?? null) : null; + const parsedDisplayName = parseOptionalDisplayName(body); + if (!parsedDisplayName.ok) { + ctx.pdppError(res, 400, "invalid_request", parsedDisplayName.error.message, parsedDisplayName.error.param); + return null; + } + return { displayName: parsedDisplayName.displayName, setupFields }; } async function emitDraftAudit( @@ -281,22 +376,21 @@ function createDraftConnection( connectorId: string; manifest: ConnectorManifestLike; captureSetup: NonNullable>; + displayName: string | null; setupFields: Record; ownerSubjectId: string; } ): { displayName: string; instance: ReturnType } { - // A fresh random binding key makes every draft a distinct connection - // identity (two mailboxes → two connection_ids) and deliberately avoids - // the deterministic default-account key, which is the phantom- - // resurrection key. The store derives the connector_instance_id from - // the binding key. - const sourceBindingKey = `draft_${randomBytes(24).toString("hex")}`; const now = ctx.now ? ctx.now() : new Date().toISOString(); const store = ctx.createRequestConnectorInstanceStore(); - const idValue = identityValue(input.captureSetup.fields, input.setupFields); - const displayName = idValue + const idValue = staticSecretSetupIdentity(input.captureSetup.fields, input.setupFields); + const sourceBindingKey = + staticSecretDraftIdentityBindingKey(input.ownerSubjectId, input.connectorId, idValue ?? "") || + `draft_${randomBytes(24).toString("hex")}`; + const fallbackDisplayName = idValue ? `${displayNameForConnector(input.connectorId, input.manifest)} - ${idValue}` : displayNameForConnector(input.connectorId, input.manifest); + const displayName = input.displayName ?? fallbackDisplayName; const instance = store.upsert({ connectorId: input.connectorId, createdAt: now, @@ -401,8 +495,8 @@ export function mountRefStaticSecretDraftConnection( ctx.pdppError(res, 503, "credential_encryption_key_missing", staticSecretSetupErrorMessage()); return; } - const setupFields = parseSetupFields(ctx, res, req.body, captureSetup.fields); - if (setupFields === null) { + const parsedSetup = parseDraftSetup(ctx, res, req.body, captureSetup.fields); + if (parsedSetup === null) { await emitDraftAudit(ctx, req, res, { connectorId, credentialKind, @@ -412,15 +506,38 @@ export function mountRefStaticSecretDraftConnection( }); return; } + const { displayName: requestedDisplayName, setupFields } = parsedSetup; - const { displayName, instance: pendingInstance } = createDraftConnection(ctx, { - captureSetup, - connectorId, - manifest, - ownerSubjectId, - setupFields, - }); - const instance = await pendingInstance; + let existingIdentity: ConnectorInstance | null = null; + if (credentialValidationMode(connectorId) === "synchronous") { + existingIdentity = await findExistingStaticSecretIdentity({ + connectorId, + fields: captureSetup.fields, + ownerSubjectId, + setupFields, + store: ctx.createRequestConnectorInstanceStore(), + }); + } + let displayName: string; + let instance: ConnectorInstance; + let created = false; + if (existingIdentity) { + instance = existingIdentity; + displayName = existingIdentity.displayName ?? displayNameForConnector(connectorId, manifest); + } else { + const createdDraft = createDraftConnection(ctx, { + captureSetup, + connectorId, + displayName: requestedDisplayName, + manifest, + ownerSubjectId, + setupFields, + }); + const { displayName: createdDisplayName, instance: createdInstance } = createdDraft; + instance = await createdInstance; + displayName = createdDisplayName; + created = true; + } await emitDraftAudit(ctx, req, res, { connectionId: instance.connectorInstanceId, @@ -430,7 +547,7 @@ export function mountRefStaticSecretDraftConnection( ownerSubjectId, }); - res.status(201).json({ + res.status(created ? 201 : 200).json({ connection_id: instance.connectorInstanceId, connector_id: connectorId, connector_instance_id: instance.connectorInstanceId, diff --git a/reference-implementation/server/routes/ref-static-secret-setup-status.ts b/reference-implementation/server/routes/ref-static-secret-setup-status.ts index 0dae17f45..53a51a0e5 100644 --- a/reference-implementation/server/routes/ref-static-secret-setup-status.ts +++ b/reference-implementation/server/routes/ref-static-secret-setup-status.ts @@ -27,10 +27,12 @@ import { type ConnectionSetupKind, projectConnectionSetupStatus, type SetupStatusImportReceipt, + type SetupStatusManifestStream, type SetupStatusMaterialMetadata, type SetupStatusRun, } from "../../runtime/static-secret-setup-status.ts"; import { type ConnectorManifestLike, staticSecretCredentialCaptureFromManifest } from "../connection-setup-plan.ts"; +import { readCollectionFactsFromTerminalData } from "../runtime-collection-facts.ts"; import type { MiddlewareHandler, PdppErrorFn, RouteArg } from "./_route-contract.ts"; interface RouteRequest { @@ -110,18 +112,34 @@ interface ConnectorNamespace { readonly connectorInstanceId: string; } +interface ProductRunHistoryRecord { + readonly completedAt: string | null; + readonly connectorInstanceId?: string | null; + readonly error?: string; + readonly factsJson?: Record | null; + readonly failureReason?: string | null; + readonly recordsEmitted: number; + readonly reportedRecordsEmitted?: number | null; + readonly runId?: string | null; + readonly startedAt: string; + readonly status: string; + readonly terminalReason?: string | null; +} + export interface MountRefStaticSecretSetupStatusContext { canonicalConnectorKey: (value: string | null | undefined) => string | null; createRequestAcquisitionBatchStore: () => AcquisitionBatchStore; createRequestConnectorInstanceCredentialStore: () => ConnectorInstanceCredentialStore; createRequestConnectorInstanceStore: () => ConnectorInstanceStore; + /** Product run-history readers are both durable and connection-scoped. */ + getLatestRunHistoryForProductByConnectionId?: ( + connectorInstanceId: string + ) => Promise | ProductRunHistoryRecord | null; getOwnerSubjectId: (req: unknown) => string; - // Bounded lookup of the run.start event timestamp, used to prove whether a - // terminal verification run belongs to the current credential rotation. - getRunStartedAt: (runId: string) => Promise; - // Window-independent terminal status for a run by run_id: "failed" | - // "completed" | "cancelled" | "abandoned" | null (still running / unknown). - getRunTerminalStatus: (runId: string) => Promise; + getProductRunHistoryForConnectionRunId?: ( + connectorInstanceId: string, + runId: string + ) => Promise | ProductRunHistoryRecord | null; handleError: (res: unknown, err: unknown) => void; pdppError: PdppErrorFn; requireOwnerSession: MiddlewareHandler; @@ -150,6 +168,29 @@ function identityFieldName(manifest: ConnectorManifestLike): string | null { return field?.name ?? null; } +function setupStatusManifestStreamFromUnknown(value: unknown): SetupStatusManifestStream | null { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + const stream = value as { readonly name?: unknown; readonly required?: unknown }; + if (typeof stream.name !== "string" || stream.name.length === 0) { + return null; + } + return { name: stream.name, required: stream.required !== false }; +} + +function isSetupStatusManifestStream(value: SetupStatusManifestStream | null): value is SetupStatusManifestStream { + return value !== null; +} + +function manifestStreamsForSetupStatus(manifest: ConnectorManifestLike): readonly SetupStatusManifestStream[] { + const { streams } = manifest as { readonly streams?: unknown }; + if (!Array.isArray(streams)) { + return []; + } + return streams.map(setupStatusManifestStreamFromUnknown).filter(isSetupStatusManifestStream); +} + // Pull the non-secret setup fields out of the draft's source binding. The draft // binding is `{ kind: "static_secret_draft", setup_fields: {...} }`; only the // non-secret fields are ever stored there (the secret goes to the credential @@ -186,6 +227,7 @@ const SETUP_KIND_BY_BINDING_KIND: Partial> = browser_enrollment_shell: "browser_session", manual_upload: "manual_upload", manual_upload_draft: "manual_upload", + static_secret: "static_secret", static_secret_draft: "static_secret", }; @@ -211,7 +253,9 @@ function nonCredentialSetupMaterial(setupKind: ConnectionSetupKind): SetupStatus return UNKNOWN_SETUP_MATERIAL; } -function setupKindForConnection(sourceBinding: unknown, manifest: ConnectorManifestLike): ConnectionSetupKind { +// Exported for direct unit coverage of the binding-kind -> setup-kind +// mapping, isolated from the manifest-fallback branch below. +export function setupKindForConnection(sourceBinding: unknown, manifest: ConnectorManifestLike): ConnectionSetupKind { const kind = bindingKind(sourceBinding); const bindingSetupKind = SETUP_KIND_BY_BINDING_KIND[kind ?? ""]; if (bindingSetupKind) { @@ -360,13 +404,84 @@ function importReceiptFromBatch(batch: AcquisitionBatch | null): SetupStatusImpo }; } -const TERMINAL_FAILURE = new Set(["failed", "cancelled", "abandoned"]); +const TERMINAL_FAILURE = new Set([ + "failed", + "cancelled", + "canceled", + "abandoned", + "errored", + "error", + "surface_failed", +]); + +function historyYieldCountsPresent(facts: Record | null | undefined): boolean { + if (facts === null || facts === undefined) { + return true; + } + return Object.hasOwn(facts, "records_emitted") || Object.hasOwn(facts, "reported_records_emitted"); +} + +function historyYieldCount( + facts: Record | null | undefined, + key: string, + fallback: number | null | undefined +): number | null { + return asFiniteNumberOrNull(facts?.[key]) ?? asFiniteNumberOrNull(fallback); +} + +function runYieldFromHistory(history: ProductRunHistoryRecord): { + readonly recordsEmitted: number | null; + readonly reportedRecordsEmitted: number | null; + readonly present: boolean; +} { + const facts = history.factsJson; + const present = historyYieldCountsPresent(facts); + if (!present) { + return { present: false, recordsEmitted: null, reportedRecordsEmitted: null }; + } + return { + present, + recordsEmitted: historyYieldCount(facts, "records_emitted", history.recordsEmitted), + reportedRecordsEmitted: historyYieldCount(facts, "reported_records_emitted", history.reportedRecordsEmitted), + }; +} // Resolve the run evidence for the setup-status projection. // - an in-flight run is the active-run row keyed on connector_instance_id; -// - otherwise, if a run id is known (in-flight earlier, or supplied by the -// owner surface that started the run), its terminal status answers whether -// the first sync failed. +// - otherwise, read the exact requested run or the latest product run-history +// row for this connection. There is intentionally no global run-id lookup: +// run ids are not connection identity. +function lookupRunHistory( + ctx: MountRefStaticSecretSetupStatusContext, + connectorInstanceId: string, + requestedRunId: string | null +) { + if (requestedRunId) { + return ctx.getProductRunHistoryForConnectionRunId?.(connectorInstanceId, requestedRunId); + } + return ctx.getLatestRunHistoryForProductByConnectionId?.(connectorInstanceId); +} + +function historyFailureReason(history: ProductRunHistoryRecord): string { + return history.failureReason ?? history.error ?? history.terminalReason ?? history.status; +} + +function setupStatusRunFromHistory(history: ProductRunHistoryRecord, requestedRunId: string | null): SetupStatusRun { + const failed = TERMINAL_FAILURE.has(history.status); + const yieldCounts = runYieldFromHistory(history); + return { + collectionFacts: readCollectionFactsFromTerminalData(history.factsJson ?? null), + failureReason: failed ? historyFailureReason(history) : null, + finishedAt: history.completedAt, + recordsEmitted: yieldCounts.recordsEmitted, + reportedRecordsEmitted: yieldCounts.reportedRecordsEmitted, + runId: history.runId ?? requestedRunId, + startedAt: history.startedAt, + status: failed ? "failed" : history.status, + yieldCountsPresent: yieldCounts.present, + }; +} + async function resolveRunEvidence( ctx: MountRefStaticSecretSetupStatusContext, store: ConnectorInstanceStore, @@ -380,23 +495,25 @@ async function resolveRunEvidence( lastRun: null, }; } - if (!requestedRunId) { + const history = await lookupRunHistory(ctx, connectorInstanceId, requestedRunId); + if (!history) { return { activeRun: null, lastRun: null }; } - const terminal = await ctx.getRunTerminalStatus(requestedRunId); - if (!terminal) { - return { activeRun: null, lastRun: null }; - } - const failed = TERMINAL_FAILURE.has(terminal); - const startedAt = await ctx.getRunStartedAt(requestedRunId); + // A history row can legitimately still read `"running"` in the window + // between the controller writing it and `controller_active_runs` + // gaining (or losing) its row for this connection — e.g. right after + // dispatch, or right after the active-run row clears but the history + // row's terminal write hasn't landed yet. Surfacing it as `lastRun` + // (rather than discarding it) lets `setupRunIsRunning` classify it as + // running, matching this route's own doc comment above ("otherwise, + // read... the latest... run-history row"). Discarding it here previously + // made the projection fall back to `first_sync_pending` with a stale + // read that never advanced until the history row went terminal + // (fr-setup-status-lifecycle-0806 — Slack/YNAB setup reading stuck on + // "First sync pending" while a run was genuinely in flight). return { activeRun: null, - lastRun: { - failureReason: failed ? terminal : null, - runId: requestedRunId, - startedAt, - status: failed ? "failed" : terminal, - }, + lastRun: setupStatusRunFromHistory(history, requestedRunId), }; } @@ -497,6 +614,7 @@ function projectSetupStatus( updatedAt: instance.updatedAt ?? null, }, lastRun, + manifestStreams: manifestStreamsForSetupStatus(manifest), setupKind, setupMaterial: setupMaterialFromBinding(setupKind, instance.sourceBinding, credentialMeta), }); diff --git a/reference-implementation/server/routes/rs-mutation.ts b/reference-implementation/server/routes/rs-mutation.ts index f93d76c49..463f0a589 100644 --- a/reference-implementation/server/routes/rs-mutation.ts +++ b/reference-implementation/server/routes/rs-mutation.ts @@ -220,6 +220,27 @@ async function maybeRecordAcquisitionProvenance( }); } +function batchOutcomeError(outcome: unknown): string | null { + if (!outcome || typeof outcome !== "object" || Array.isArray(outcome)) { + return "ingest batch returned a malformed record outcome"; + } + const result = outcome as { accepted?: unknown; error?: unknown }; + if (typeof result.error === "string") { + return result.error; + } + return result.accepted === true ? null : "ingest batch returned a rejected record without an error"; +} + +function mapBatchIngestOutcomes( + records: readonly Record[], + outcomes: readonly unknown[] +): readonly (string | null)[] { + if (outcomes.length !== records.length) { + throw new Error(`ingestRecords returned ${outcomes.length} results for ${records.length} records`); + } + return outcomes.map(batchOutcomeError); +} + interface TokenInfo { readonly client_id?: string | null; readonly grant?: GrantLike | null; @@ -354,7 +375,22 @@ export interface MountRsMutationContext { // Capability: error handler for untyped errors readonly handleError: (res: RouteResponse, err: unknown) => void; - readonly ingestRecord: (target: StorageTargetLike, record: unknown) => Promise; + readonly ingestRecord: ( + target: StorageTargetLike, + record: unknown, + options?: { requireConnectionAdmission?: boolean; runId?: string | null } + ) => Promise; + /** + * Optional common-path batch capability; hosts without it use the ordered + * fallback. Hosts that provide it must await `afterRecord` after each + * accepted store and before starting the next record. + */ + readonly ingestRecords?: ( + target: StorageTargetLike, + records: readonly unknown[], + afterRecord?: (record: unknown, outcome: unknown) => Promise, + options?: { requireConnectionAdmission?: boolean; runId?: string | null } + ) => Promise; // Every other owner-connection mutation route (revoke, reactivate, // schedule, run, rename, delete — see routes/owner-connection-*.ts, // ref-connectors.ts) invalidates the dashboard/Sources/Syncs summary cache @@ -982,6 +1018,12 @@ export function mountRsRecordsIngest(app: AppLike, ctx: MountRsMutationContext): app.post("/v1/ingest/:stream", ctx.requireToken, ctx.requireOwner, async (req: RouteRequest, res: RouteResponse) => { const connectorId = canonicalizeConnectorId(ctx.resolveSingleConnectorIdQueryValue(req.query.connector_id)); const connectorInstanceId = ctx.resolveSingleConnectorIdQueryValue(req.query.connector_instance_id); + // Run-bound connector ingestion threads its run_id through so the storage + // layer can fence a write already admitted before cancellation against + // the run's own terminal state (see harden-ingest-run-admission-fence). + // Absent for owner/API ingestion that has no run concept — the fence is + // opt-in and those callers are unaffected. + const runId = ctx.resolveSingleConnectorIdQueryValue(req.query.run_id); // parseLines is imported inside executeRecordsIngest; the line-count for // the mutation context must be computed here using the same parser. // Index.js imported `parseLines as parseIngestLines` from the operation @@ -1009,6 +1051,7 @@ export function mountRsRecordsIngest(app: AppLike, ctx: MountRsMutationContext): // target; the connector-only path stays active-only. `allowStatuses` is // omitted (not set to undefined) so it doesn't trip exactOptionalPropertyTypes. const draftAdmission = (cin: string | null) => (cin ? { allowStatuses: ["active", "draft"] as const } : {}); + const { ingestRecords } = ctx; const dependencies: RecordsIngestDependencies = { hasManifestStream: async (cid: string, streamName: string) => { const manifest = await ctx.resolveRegisteredConnectorManifest(cid); @@ -1028,7 +1071,10 @@ export function mountRsRecordsIngest(app: AppLike, ctx: MountRsMutationContext): ...draftAdmission(cin), connectorInstanceId: cin, })); - const result = await ctx.ingestRecord(ctx.storageTargetForConnectorNamespace(namespace), record); + const result = await ctx.ingestRecord(ctx.storageTargetForConnectorNamespace(namespace), record, { + requireConnectionAdmission: Boolean(namespace.connectorInstanceId), + runId, + }); if (ctx.getLatestAcquisitionBatchForConnection && namespace.connectorInstanceId) { acquisitionBatchPromise ??= Promise.resolve( ctx.getLatestAcquisitionBatchForConnection(namespace.connectorInstanceId) @@ -1044,6 +1090,52 @@ export function mountRsRecordsIngest(app: AppLike, ctx: MountRsMutationContext): } return result; }, + ...(ingestRecords + ? { + ingestRecords: async ( + cid: string, + cin: string | null, + records: readonly Record[] + ): Promise => { + const namespace = + storageNamespace ?? + (await ctx.resolveOwnerConnectorNamespace(req, cid, { + ...draftAdmission(cin), + connectorInstanceId: cin, + })); + // biome-ignore lint/suspicious/noUnnecessaryConditions: Express route params are nullable at runtime even though the local adapter type narrows them. + const streamName = req.params.stream ?? ""; + const getLatestAcquisitionBatch = ctx.getLatestAcquisitionBatchForConnection; + const connectorInstanceIdForBatch = namespace.connectorInstanceId; + const afterRecord = async (record: unknown, outcome: unknown): Promise => { + const outcomeError = batchOutcomeError(outcome); + if (outcomeError) { + throw new Error(outcomeError); + } + if (!(getLatestAcquisitionBatch && connectorInstanceIdForBatch)) { + return; + } + acquisitionBatchPromise ??= Promise.resolve( + getLatestAcquisitionBatch(connectorInstanceIdForBatch) ?? null + ); + await maybeRecordAcquisitionProvenance( + ctx, + namespace, + await acquisitionBatchPromise, + streamName, + record + ); + }; + const outcomes = await ingestRecords( + ctx.storageTargetForConnectorNamespace(namespace), + records, + afterRecord, + { requireConnectionAdmission: Boolean(namespace.connectorInstanceId), runId } + ); + return mapBatchIngestOutcomes(records, outcomes); + }, + } + : {}), }; let output: RecordsIngestOutput; try { diff --git a/reference-implementation/server/routes/source-webhooks.ts b/reference-implementation/server/routes/source-webhooks.ts index 914d1767d..852b57722 100644 --- a/reference-implementation/server/routes/source-webhooks.ts +++ b/reference-implementation/server/routes/source-webhooks.ts @@ -88,6 +88,9 @@ export interface MountRefSourceWebhooksContext { getSchedulerStore: () => SourceWebhookSchedulerStore; getSourceWebhookEventStore: () => SourceWebhookEventStoreLike; handleError: (res: unknown, err: unknown) => void; + // This route is intentionally connector-scoped, not connection-scoped. If + // it ever accepts a connector_instance_id, it must use the lifecycle + // admission fence documented by assertConnectorInstanceWritable. ingestRecord: (connectorId: string, record: Record) => unknown | Promise; parseSourceWebhookSecrets: () => SourceWebhookSecretsMap; pdppError: (res: unknown, status: number, code: string, message: string | undefined) => unknown; diff --git a/reference-implementation/server/runtime-collection-facts.ts b/reference-implementation/server/runtime-collection-facts.ts index 30a800eaf..e14740e07 100644 --- a/reference-implementation/server/runtime-collection-facts.ts +++ b/reference-implementation/server/runtime-collection-facts.ts @@ -118,14 +118,21 @@ function readCollectionRateNumbers(entry: Record): CollectionRa return Object.fromEntries(values) as CollectionRateNumbers; } +function readCollectionRateBackoff(value: unknown): Record | null { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + return value as Record; +} + function readCollectionRateLastBackoff(entry: Record): CollectionRateSnapshot["last_backoff"] { - if (entry.last_backoff === null) { + const backoff = readCollectionRateBackoff(entry.last_backoff); + if (backoff === null) { return null; } - const backoff = entry.last_backoff as Record; - const atIntervalMs = typeof backoff.at_interval_ms === "number" ? backoff.at_interval_ms : null; + const atIntervalMs = readFiniteNumber(backoff.at_interval_ms, Number.NaN); const reason = typeof backoff.reason === "string" ? backoff.reason : null; - return atIntervalMs !== null && reason !== null ? { at_interval_ms: atIntervalMs, reason } : null; + return Number.isFinite(atIntervalMs) && reason !== null ? { at_interval_ms: atIntervalMs, reason } : null; } /** diff --git a/reference-implementation/server/static-secret-identity.ts b/reference-implementation/server/static-secret-identity.ts new file mode 100644 index 000000000..f303adcca --- /dev/null +++ b/reference-implementation/server/static-secret-identity.ts @@ -0,0 +1,406 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import { fingerprintsEqual } from "./stores/credential-encryption.ts"; + +const DRAFT_IDENTITY_PREFIX = "static_secret_draft_identity_"; +const VERIFIED_IDENTITY_PREFIX = "static_secret_verified_identity_"; + +export interface StaticSecretIdentityField { + readonly identity?: boolean; + readonly label?: string; + readonly name: string; + readonly required?: boolean; + readonly secret?: boolean; +} + +/** + * Provider identities are non-secret account labels, not credentials. Keep + * normalization deliberately conservative: whitespace is transport noise; + * changing case or Unicode semantics could collapse two provider identities + * that the manifest has not declared equivalent. + */ +export function normalizeStaticSecretIdentity(value: string): string { + return value.trim(); +} + +function identityBindingKey(prefix: string, ownerSubjectId: string, connectorId: string, identity: string): string { + const normalized = normalizeStaticSecretIdentity(identity); + if (!normalized) { + return ""; + } + const digest = createHash("sha256") + .update(prefix) + .update("\u0000") + .update(ownerSubjectId) + .update("\u0000") + .update(connectorId) + .update("\u0000") + .update(normalized) + .digest("hex"); + return `${prefix}${digest}`; +} + +export function staticSecretDraftIdentityBindingKey( + ownerSubjectId: string, + connectorId: string, + identity: string +): string { + return identityBindingKey(DRAFT_IDENTITY_PREFIX, ownerSubjectId, connectorId, identity); +} + +export function staticSecretVerifiedIdentityBindingKey( + ownerSubjectId: string, + connectorId: string, + identity: string +): string { + return identityBindingKey(VERIFIED_IDENTITY_PREFIX, ownerSubjectId, connectorId, identity); +} + +export function isStaticSecretVerifiedIdentityBindingKey(value: string): boolean { + return value.startsWith(VERIFIED_IDENTITY_PREFIX); +} + +export function staticSecretSetupIdentity( + fields: readonly StaticSecretIdentityField[], + setupFields: Record +): string | null { + const field = fields.find((candidate) => candidate.identity && !candidate.secret); + return field ? (setupFields[field.name] ?? null) : null; +} + +export function staticSecretSetupFieldsFromBinding(sourceBinding: unknown): Record | null { + const source = objectRecord(sourceBinding); + const raw = objectRecord(source?.setup_fields); + if (!raw) { + return null; + } + const fields = stringFields(raw); + return Object.keys(fields).length > 0 ? fields : null; +} + +export function staticSecretVerifiedIdentityFromBinding(sourceBinding: unknown): string | null { + const value = objectRecord(sourceBinding)?.verified_identity; + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +export function staticSecretBindingRecord(sourceBinding: unknown): Record | null { + const source = objectRecord(sourceBinding); + return source ? { ...source } : null; +} + +export function staticSecretIdentityConflictError( + message: string, + code = "static_secret_identity_conflict" +): Error & { code: string } { + const error = new Error(message) as Error & { code: string }; + error.code = code; + return error; +} + +export function isStaticSecretBindingUniqueConflict(err: unknown): boolean { + const code = (err as { code?: unknown } | null)?.code; + if (typeof code === "string" && new Set(["23505", "SQLITE_CONSTRAINT_UNIQUE"]).has(code)) { + return true; + } + const message = err instanceof Error ? err.message : String(err); + return ( + message.includes("UNIQUE constraint failed") || message.includes("duplicate key value violates unique constraint") + ); +} + +export function staticSecretIdentityClaim(input: { + connectorId: string; + ownerSubjectId: string; + probedIdentity: string; +}): { identity: string; sourceBindingKey: string } { + const identity = normalizeStaticSecretIdentity(input.probedIdentity); + const sourceBindingKey = staticSecretVerifiedIdentityBindingKey(input.ownerSubjectId, input.connectorId, identity); + if (!sourceBindingKey) { + throw staticSecretIdentityConflictError( + "The provider returned no verified account identity; refusing to store the credential.", + "static_secret_identity_missing" + ); + } + return { identity, sourceBindingKey }; +} + +export function parseStaticSecretSetupFields( + raw: unknown, + fields: readonly StaticSecretIdentityField[], + onError: (code: string, message: string, param: string) => void +): Record | undefined | null { + if (raw === undefined) { + return; + } + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + onError("invalid_request", "setup_fields must be an object when provided.", "setup_fields"); + return null; + } + const provided = objectRecord(raw); + if (!provided) { + return null; + } + return parseSetupFieldRecord(provided, fields, onError); +} + +export function parseStaticSecretDraftSetupFields( + raw: unknown, + fields: readonly StaticSecretIdentityField[], + onError: (code: string, message: string, param: string) => void +): Record | null { + return parseSetupFieldRecord(objectRecord(raw) ?? {}, fields, onError); +} + +function parseSetupFieldRecord( + provided: Record, + fields: readonly StaticSecretIdentityField[], + onError: (code: string, message: string, param: string) => void +): Record | null { + const unknown = unknownStaticSecretSetupField(provided, fields); + if (unknown) { + onError("unknown_setup_field", `Unknown setup field: ${unknown}`, `setup_fields.${unknown}`); + return null; + } + const missing = missingStaticSecretSetupField(provided, fields); + if (missing) { + onError("missing_setup_field", `${missing.label ?? missing.name} is required.`, `setup_fields.${missing.name}`); + return null; + } + return collectStaticSecretSetupFields(provided, fields); +} + +function objectRecord(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : null; +} + +function stringFields(raw: Record): Record { + return Object.fromEntries( + Object.entries(raw).flatMap(([key, value]) => (typeof value === "string" ? [[key, value.trim()]] : [])) + ); +} + +function unknownStaticSecretSetupField( + provided: Record, + fields: readonly StaticSecretIdentityField[] +): string | null { + const allowed = new Set(fields.filter((field) => !field.secret).map((field) => field.name)); + return Object.keys(provided).find((key) => !allowed.has(key)) ?? null; +} + +function fieldText(field: StaticSecretIdentityField, provided: Record): string { + const value = provided[field.name]; + return typeof value === "string" ? value.trim() : ""; +} + +function missingStaticSecretSetupField( + provided: Record, + fields: readonly StaticSecretIdentityField[] +): StaticSecretIdentityField | null { + return fields.find((field) => !field.secret && field.required && !fieldText(field, provided)) ?? null; +} + +function collectStaticSecretSetupFields( + provided: Record, + fields: readonly StaticSecretIdentityField[] +): Record { + return Object.fromEntries( + fields + .filter((field) => !field.secret) + .map((field) => [field.name, fieldText(field, provided)]) + .filter(([, value]) => value) + ); +} + +// A binding only carries retarget-protection authority once it has gone +// through the real static-secret setup pipeline — captured as a +// `static_secret_draft` and, once promoted, a `static_secret` binding. A row +// with neither kind — e.g. an `account_hint`-only legacy shape that predates +// this pipeline entirely — never had the chance to record a durable identity +// signal and must remain repairable exactly as before. This is the one +// legacy carve-out; every pipeline-shaped active row is subject to the +// fail-closed rule below, whether or not `activateDraft` has yet rewritten +// its binding `kind` from `static_secret_draft` to `static_secret`. +const STATIC_SECRET_PIPELINE_BINDING_KINDS = new Set(["static_secret", "static_secret_draft"]); + +export function isStaticSecretPipelineBinding(sourceBinding: unknown): boolean { + const kind = objectRecord(sourceBinding)?.kind; + return typeof kind === "string" && STATIC_SECRET_PIPELINE_BINDING_KINDS.has(kind); +} + +function staticSecretRetargetRefusedError(): Error & { code: string } { + return staticSecretIdentityConflictError( + "This active connection has no verified provider identity on record, so a credential replacement cannot be " + + "proven safe. To connect a different account, disconnect this connection and create a new one.", + "static_secret_identity_unverified_replacement" + ); +} + +// Terminal authority for REPLACING the credential behind an ACTIVE +// static-secret connection. Only fires when a credential already exists — +// the very first capture on an active row (no credential yet, e.g. a +// connection seeded active before its owner ever sealed a secret) is +// establishment, not replacement, and always proceeds. +// +// Once a credential exists, absence of proof is never read as permission, +// and PROOF MEANS EVIDENCE THE OWNER DID NOT AUTHOR: a replacement is +// allowed only when THIS request affirmatively proves sameness via one of +// two channels, both independent of anything the caller typed into the +// request body — +// 1. `identity` is a value a synchronous PROVIDER PROBE just returned for +// the new secret, and it matches the durable `verified_identity` +// already on the binding — itself only ever written from an earlier +// successful probe. `setup_fields` (owner-typed, non-secret, resubmit- +// table alongside any stolen secret) is NEVER an acceptable source for +// either side of this comparison — an attacker who already knows the +// account email can trivially resubmit it. +// 2. A non-secret, key-derived fingerprint proves the submitted secret is +// byte-for-byte the same credential already stored — computed from the +// plaintext itself, never from anything the request "claims". +// A provider-probed identity that CONTRADICTS the durable one is a hard +// reject — fingerprint sameness cannot override an explicit, proven account +// mismatch. Anything that proves neither channel is refused; the sanctioned +// path for a genuine account change is an explicit disconnect + new +// connection. +export function assertStaticSecretActiveCredentialReplacementAllowed(input: { + existingCredentialFingerprint: string | null; + hasExistingCredential: boolean; + newSecretFingerprint: string | null; + probedIdentity?: string | undefined; + sourceBinding: unknown; + status: string; +}): void { + if ( + input.status !== "active" || + !isStaticSecretPipelineBinding(input.sourceBinding) || + !input.hasExistingCredential + ) { + return; + } + const durableIdentity = staticSecretVerifiedIdentityFromBinding(input.sourceBinding); + const probedIdentity = + input.probedIdentity === undefined ? null : normalizeStaticSecretIdentity(input.probedIdentity); + if (durableIdentity !== null && probedIdentity !== null) { + if (probedIdentity !== normalizeStaticSecretIdentity(durableIdentity)) { + throw staticSecretIdentityConflictError( + "This active connection is already verified for a different provider identity. Create a separate connection for the other account.", + "static_secret_identity_mismatch" + ); + } + return; + } + // No affirmative provider-verified identity match: either nothing durable + // is on record yet, or this request had no probe to compare against (a + // no-probe connector never reaches the branch above at all). Fall back to + // exact-credential proof — fingerprints are derived from the plaintext + // under the operator key, so a match proves sameness without decrypting + // the stored secret or trusting anything the request merely claims. + if ( + input.existingCredentialFingerprint && + input.newSecretFingerprint && + fingerprintsEqual(input.existingCredentialFingerprint, input.newSecretFingerprint) + ) { + return; + } + throw staticSecretRetargetRefusedError(); +} + +export interface StaticSecretIdentityInstance { + readonly connectorInstanceId: string; + readonly displayName?: string | null; + readonly sourceBinding?: unknown; + readonly status: string; +} + +type MaybePromise = T | Promise; + +export interface StaticSecretIdentityStore { + getByBinding: (input: { + connectorId: string; + ownerSubjectId: string; + sourceBindingKey: string; + sourceKind: string; + }) => MaybePromise; + listActiveByConnector: (ownerSubjectId: string, connectorId: string, options: { limit: number }) => MaybePromise; +} + +export async function findExistingStaticSecretIdentity(input: { + connectorId: string; + fields: readonly StaticSecretIdentityField[]; + ownerSubjectId: string; + setupFields: Record; + store: StaticSecretIdentityStore; +}): Promise { + const identity = normalizeStaticSecretIdentity(staticSecretSetupIdentity(input.fields, input.setupFields) ?? ""); + const identityField = input.fields.find((field) => field.identity && !field.secret); + if (!(identity && identityField)) { + return null; + } + const draft = await input.store.getByBinding({ + connectorId: input.connectorId, + ownerSubjectId: input.ownerSubjectId, + sourceBindingKey: staticSecretDraftIdentityBindingKey(input.ownerSubjectId, input.connectorId, identity), + sourceKind: "account", + }); + if (draft) { + return existingIdentityBindingOrThrow(draft, input.connectorId, identity); + } + const verified = await input.store.getByBinding({ + connectorId: input.connectorId, + ownerSubjectId: input.ownerSubjectId, + sourceBindingKey: staticSecretVerifiedIdentityBindingKey(input.ownerSubjectId, input.connectorId, identity), + sourceKind: "account", + }); + if (verified) { + return existingIdentityBindingOrThrow(verified, input.connectorId, identity); + } + return findLegacyStaticSecretIdentity(input.store, input.connectorId, input.ownerSubjectId, identity, identityField); +} + +function existingIdentityBindingOrThrow( + existing: T, + connectorId: string, + identity: string +): T { + if (existing.status === "revoked") { + throw staticSecretIdentityConflictError( + `The '${connectorId}' connection for provider identity '${identity}' is revoked; refusing to reactivate it silently.`, + "static_secret_identity_revoked" + ); + } + return existing; +} + +async function findLegacyStaticSecretIdentity( + store: StaticSecretIdentityStore, + connectorId: string, + ownerSubjectId: string, + identity: string, + identityField: StaticSecretIdentityField +): Promise { + const active = await store.listActiveByConnector(ownerSubjectId, connectorId, { limit: 500 }); + const matches = active.filter((instance) => matchesStaticSecretIdentity(instance, identity, identityField.name)); + if (matches.length > 1) { + throw staticSecretIdentityConflictError( + `More than one active '${connectorId}' connection already claims this provider identity; refusing to create another connection.`, + "static_secret_identity_ambiguous" + ); + } + return matches[0] ?? null; +} + +function matchesStaticSecretIdentity( + instance: StaticSecretIdentityInstance, + identity: string, + identityFieldName: string +): boolean { + const binding = objectRecord(instance.sourceBinding); + if (binding?.kind !== "static_secret") { + return false; + } + const verified = staticSecretVerifiedIdentityFromBinding(binding); + return verified + ? verified === identity + : staticSecretSetupFieldsFromBinding(binding)?.[identityFieldName] === identity; +} diff --git a/reference-implementation/server/stores/connector-instance-credential-store.ts b/reference-implementation/server/stores/connector-instance-credential-store.ts index 8ab090c33..0fbb33bf2 100644 --- a/reference-implementation/server/stores/connector-instance-credential-store.ts +++ b/reference-implementation/server/stores/connector-instance-credential-store.ts @@ -81,6 +81,12 @@ interface CredentialStoreRun { export interface ConnectorInstanceCredentialStore { capture: (args: CaptureCredentialArgs) => Promise; delete: (connectorInstanceId: string) => Promise; + /** + * Non-secret, key-derived fingerprint of a candidate plaintext, for proving + * "is this the exact same credential already stored" without sealing or + * persisting anything. Same derivation `capture` uses; a pure read. + */ + fingerprintCandidate: (secret: string) => string | null; getMetadata: (connectorInstanceId: string) => Promise; /** Non-secret metadata keyed by exact instance id. Empty input performs no SQL. */ getMetadataByInstanceIds: (connectorInstanceIds: readonly string[]) => Promise>; @@ -258,6 +264,10 @@ function buildStore({ return existed; }, + fingerprintCandidate(secret: string) { + return cipher().fingerprint(secret); + }, + /** Non-secret metadata for one instance, or null when no credential exists. */ async getMetadata(connectorInstanceId: string) { return projectMetadata(await read.getRaw(connectorInstanceId)); diff --git a/reference-implementation/server/stores/connector-instance-store.ts b/reference-implementation/server/stores/connector-instance-store.ts index 7423e31cd..bd75c164c 100644 --- a/reference-implementation/server/stores/connector-instance-store.ts +++ b/reference-implementation/server/stores/connector-instance-store.ts @@ -1066,6 +1066,7 @@ export async function resolveOwnerConnectorInstanceNamespace({ * connector type (or another owner's deterministic id) as a capability. */ export function admitOwnerRunConnection({ + allowDraft = false, ownerSubjectId, connectorId, connectorInstanceId = null, @@ -1073,6 +1074,8 @@ export function admitOwnerRunConnection({ displayName = null, now, }: { + /** Setup routes may explicitly admit the exact draft they just created. */ + allowDraft?: boolean; ownerSubjectId: string; connectorId: string; connectorInstanceId?: string | null; @@ -1084,6 +1087,7 @@ export function admitOwnerRunConnection({ // Explicit selectors never materialize or fall through. The broader // resolver still supports legacy read compatibility independently. allowDefaultAccount: !connectorInstanceId, + allowStatuses: allowDraft ? ["active", "draft"] : ["active"], connectorId, connectorInstanceId, connectorInstanceStore, @@ -1510,6 +1514,39 @@ export function createSqliteConnectorInstanceStore() { return { hasMore, rows: rows.slice(0, limit).map(mapInstance) }; }, + // Promotes a temporary setup binding to its durable sibling kind. + // Guarded by `status = 'draft' AND binding.kind = fromKind`: a + // concurrent revoke racing this UPDATE loses safely (no row change, + // `promoted: false`). Never writes the identity tuple + // (connector_instance_id/owner_subject_id/source_kind/source_binding_key). + promoteSetupBinding( + connectorInstanceId: string, + { + fromKind, + sourceBinding, + updatedAt, + }: { fromKind: string; sourceBinding: Record; updatedAt: string } + ): { instance: ConnectorInstance | null; promoted: boolean } { + let promoted = false; + writeTransaction(() => { + const result = exec(referenceQueries.connectorInstancesPromoteSetupBinding, [ + stableJson(sourceBinding), + "active", + updatedAt, + connectorInstanceId, + fromKind, + ]); + promoted = Boolean(result.changes); + if (promoted) { + exec(referenceQueries.connectorSummaryEvidenceMarkDirtyByConnectorInstance, [ + `connector instance promoted from ${fromKind}`, + connectorInstanceId, + ]); + } + }); + return { instance: this.get(connectorInstanceId), promoted }; + }, + resolveActiveByConnector(ownerSubjectId: string, connectorId: string): ConnectorInstance { const rows = getMany( referenceQueries.connectorInstancesListActiveByOwnerConnector, @@ -1559,6 +1596,47 @@ export function createSqliteConnectorInstanceStore() { return this.get(connectorInstanceId); }, + // Re-key one owner-session static-secret instance after a synchronous + // provider probe proves its account identity. The connector instance id + // is intentionally preserved: records, schedules, history, and callers + // all address that id. The existing binding unique constraint is the + // cross-request identity claim; a concurrent claim for the same verified + // identity raises the backend's normal unique-constraint error for the + // route to resolve to the winner. + updateStaticSecretBinding({ + connectorInstanceId, + connectorId, + ownerSubjectId, + sourceBinding, + sourceBindingKey, + updatedAt, + }: { + connectorId: string; + connectorInstanceId: string; + ownerSubjectId: string; + sourceBinding: Record; + sourceBindingKey: string; + updatedAt: string; + }): ConnectorInstance | null { + writeTransaction(() => { + const result = exec(referenceQueries.connectorInstancesUpdateStaticSecretBinding, [ + sourceBindingKey, + stableJson(sourceBinding), + updatedAt, + connectorInstanceId, + ownerSubjectId, + connectorId, + ]); + if (result.changes) { + exec(referenceQueries.connectorSummaryEvidenceMarkDirtyByConnectorInstance, [ + "static-secret binding updated", + connectorInstanceId, + ]); + } + }); + return this.get(connectorInstanceId); + }, + // Terminal-gate revision (2026-07-29): the status write and its // summary-evidence dirty marker commit in ONE transaction, matching // `deleteConnection`'s existing precedent for the same table. A marker @@ -2177,6 +2255,40 @@ export function createPostgresConnectorInstanceStore() { return { hasMore, rows: rows.slice(0, limit).map(mapInstance) }; }, + // Postgres mirror of the SQLite arm above — same `status = 'draft' AND + // binding.kind = fromKind` guard, same `promoted` result, same + // identity-preserving contract. + async promoteSetupBinding( + connectorInstanceId: string, + { + fromKind, + sourceBinding, + updatedAt, + }: { fromKind: string; sourceBinding: Record; updatedAt: string } + ): Promise<{ instance: ConnectorInstance | null; promoted: boolean }> { + let promoted = false; + await withPostgresTransaction( + async (client: { query: (sql: string, params?: unknown[]) => Promise<{ rowCount?: number | null }> }) => { + const result = await client.query( + `UPDATE connector_instances + SET source_binding_json = $1::jsonb, status = $2, updated_at = $3 + WHERE connector_instance_id = $4 + AND status = 'draft' + AND source_binding_json->>'kind' = $5`, + [stableJson(sourceBinding), "active", updatedAt, connectorInstanceId, fromKind] + ); + promoted = Boolean(result.rowCount); + if (promoted) { + await client.query( + `UPDATE connector_summary_evidence SET dirty = 1, state = 'stale', last_error = $1 WHERE connector_instance_id = $2`, + [`connector instance promoted from ${fromKind}`, connectorInstanceId] + ); + } + } + ); + return { instance: await this.get(connectorInstanceId), promoted }; + }, + async resolveActiveByConnector(ownerSubjectId: string, connectorId: string): Promise { const result = await postgresQuery( `SELECT connector_instance_id, owner_subject_id, connector_id, display_name, status, source_kind, source_binding_key, source_binding_json, created_at, updated_at, revoked_at @@ -2230,6 +2342,49 @@ export function createPostgresConnectorInstanceStore() { return await this.get(connectorInstanceId); }, + // Postgres mirror of the SQLite static-secret identity claim. The binding + // unique constraint is enforced by the database, not by a process-local + // read/then-write sequence, so concurrent owners/processes cannot both + // claim one verified identity. + async updateStaticSecretBinding({ + connectorInstanceId, + connectorId, + ownerSubjectId, + sourceBinding, + sourceBindingKey, + updatedAt, + }: { + connectorId: string; + connectorInstanceId: string; + ownerSubjectId: string; + sourceBinding: Record; + sourceBindingKey: string; + updatedAt: string; + }): Promise { + await withPostgresTransaction( + async (client: { query: (sql: string, params?: unknown[]) => Promise<{ rowCount?: number | null }> }) => { + const result = await client.query( + `UPDATE connector_instances + SET source_binding_key = $1, + source_binding_json = $2::jsonb, + updated_at = $3 + WHERE connector_instance_id = $4 + AND owner_subject_id = $5 + AND connector_id = $6 + AND status IN ('active', 'draft')`, + [sourceBindingKey, stableJson(sourceBinding), updatedAt, connectorInstanceId, ownerSubjectId, connectorId] + ); + if (result.rowCount) { + await client.query( + `UPDATE connector_summary_evidence SET dirty = 1, state = 'stale', last_error = $1 WHERE connector_instance_id = $2`, + ["static-secret binding updated", connectorInstanceId] + ); + } + } + ); + return await this.get(connectorInstanceId); + }, + async updateStatus( connectorInstanceId: string, { diff --git a/reference-implementation/server/stores/run-history-writer.ts b/reference-implementation/server/stores/run-history-writer.ts index dd41d0697..f2517a5f2 100644 --- a/reference-implementation/server/stores/run-history-writer.ts +++ b/reference-implementation/server/stores/run-history-writer.ts @@ -120,6 +120,10 @@ function toTerminalStatus(eventType: string, status: string): string { const FACTS_JSON_KEYS = [ "collection_facts", "needs_input", + // Preserve presence separately from the schema's records_emitted DEFAULT 0. + // A missing runtime field is not evidence of zero yield. + "records_emitted", + "reported_records_emitted", "browser_surface_lease_id", "browser_surface_profile_key", "browser_surface_status", diff --git a/reference-implementation/server/stores/scheduler-store.ts b/reference-implementation/server/stores/scheduler-store.ts index 9817cfecb..a25fa3751 100644 --- a/reference-implementation/server/stores/scheduler-store.ts +++ b/reference-implementation/server/stores/scheduler-store.ts @@ -148,6 +148,11 @@ export interface SchedulerStore { connectorInstanceId: string, status?: string | null ) => Promise | ProductRunHistoryRecord | null; + /** Exact product run-history lookup fenced by the addressed connection. */ + getProductRunHistoryForConnectionRunId?: ( + connectorInstanceId: string, + runId: string + ) => Promise | ProductRunHistoryRecord | null; getSchedule: (connectorInstanceId: string) => Promise | ScheduleRecord | null; listActiveRuns: () => Promise | readonly ActiveRunRecord[]; listLastRunTimes: () => Promise | readonly SchedulerLastRunTimeRecord[]; @@ -444,6 +449,24 @@ export function createSqliteSchedulerStore(): SchedulerStore { return row ? rowToProductRunHistoryRecord(row) : null; }, + getProductRunHistoryForConnectionRunId(connectorInstanceId, runId) { + // REVIEWED-DYNAMIC: both values are bound; the fixed projection is the + // existing product run-history reader. The composite predicate is the + // identity fence because run_id is not globally unique. + const row = [ + ...iterateDynamicSqlAcknowledged( + `SELECT ${PRODUCT_RUN_HISTORY_COLUMNS} + FROM run_history + WHERE connector_instance_id = ? + AND run_id = ? + ORDER BY id DESC + LIMIT 1`, + [connectorInstanceId, runId] + ), + ].at(0); + return row ? rowToProductRunHistoryRecord(row) : null; + }, + getSchedule(connectorInstanceId) { const row = getOne(referenceQueries.controllerGetScheduleByConnector, [connectorInstanceId]); return row ? rowToScheduleRecord(row) : null; @@ -809,6 +832,19 @@ export function createPostgresSchedulerStore(): SchedulerStore { return result.rows[0] ? rowToProductRunHistoryRecord(result.rows[0] as SchedulerRunHistoryRow) : null; }, + async getProductRunHistoryForConnectionRunId(connectorInstanceId, runId) { + const result = await postgresQuery( + `SELECT ${PRODUCT_RUN_HISTORY_COLUMNS} + FROM run_history + WHERE connector_instance_id = $1 + AND run_id = $2 + ORDER BY id DESC + LIMIT 1`, + [connectorInstanceId, runId] + ); + return result.rows[0] ? rowToProductRunHistoryRecord(result.rows[0] as SchedulerRunHistoryRow) : null; + }, + async getSchedule(connectorInstanceId) { const result = await postgresQuery( `SELECT connector_instance_id, connector_id, interval_seconds, jitter_seconds, enabled, created_at, updated_at diff --git a/reference-implementation/test/activate-draft-connection.test.ts b/reference-implementation/test/activate-draft-connection.test.ts new file mode 100644 index 000000000..93a081a37 --- /dev/null +++ b/reference-implementation/test/activate-draft-connection.test.ts @@ -0,0 +1,95 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { activateDraftConnection } from "../server/index.ts"; + +// On a lost promotion race (`promoted: false`), the caller must return null +// without calling the schedule attacher. The store-level guard itself is +// proven in test/setup-binding-promotion.test.ts. + +function fakeStore(overrides: { + current: { status?: string; sourceBinding?: unknown } | null; + promoted: boolean; + promotedInstance?: unknown; +}) { + return { + activateDraft: () => ({ status: "active" }), + get: () => overrides.current, + promoteSetupBinding: () => ({ + instance: overrides.promotedInstance ?? { status: "active" }, + promoted: overrides.promoted, + }), + }; +} + +test("lost race: promoted=false returns null and never attaches a schedule", async () => { + const store = fakeStore({ + current: { sourceBinding: { kind: "browser_enrollment_shell" }, status: "draft" }, + promoted: false, + }); + let attachCalled = false; + const attachSchedule = (instance: unknown) => { + attachCalled = true; + return Promise.resolve(instance); + }; + + const result = await activateDraftConnection("cin_race", store, attachSchedule); + + assert.equal(result, null, "a lost race returns null"); + assert.equal(attachCalled, false, "a lost race never attaches an activation schedule"); +}); + +test("won race: promoted=true attaches a schedule and returns its result", async () => { + const promotedInstance = { connectorInstanceId: "cin_won", status: "active" }; + const store = fakeStore({ + current: { sourceBinding: { kind: "browser_enrollment_shell" }, status: "draft" }, + promoted: true, + promotedInstance, + }); + let attachedWith: unknown; + const attachSchedule = (instance: unknown) => { + attachedWith = instance; + return Promise.resolve("schedule_result"); + }; + + const result = await activateDraftConnection("cin_won", store, attachSchedule); + + assert.equal(result, "schedule_result", "a won race returns the schedule attacher's result"); + assert.deepEqual(attachedWith, promotedInstance, "the schedule attacher receives the promoted instance"); +}); + +test("non-setup binding: falls through to plain activateDraft and attaches a schedule", async () => { + const store = fakeStore({ + current: { sourceBinding: { kind: "account" }, status: "draft" }, + promoted: false, + }); + let attachCalled = false; + const attachSchedule = (instance: unknown) => { + attachCalled = true; + return Promise.resolve(instance); + }; + + const result = await activateDraftConnection("cin_plain", store, attachSchedule); + + assert.deepEqual(result, { status: "active" }, "falls through to activateDraft's result"); + assert.equal(attachCalled, true, "the plain-activation path still attaches a schedule"); +}); + +test("already-active row: falls through to activateDraft (no-op) without ever calling promoteSetupBinding", async () => { + let promoteCalled = false; + const store = { + activateDraft: () => ({ status: "active" }), + get: () => ({ sourceBinding: { kind: "browser_enrollment_shell" }, status: "active" }), + promoteSetupBinding: () => { + promoteCalled = true; + return { instance: null, promoted: false }; + }, + }; + + await activateDraftConnection("cin_already_active", store, (instance) => Promise.resolve(instance)); + + assert.equal(promoteCalled, false, "an already-active row never reaches promoteSetupBinding"); +}); diff --git a/reference-implementation/test/blob-store-route-regression.test.ts b/reference-implementation/test/blob-store-route-regression.test.ts index c21ad6e13..bda5551f9 100644 --- a/reference-implementation/test/blob-store-route-regression.test.ts +++ b/reference-implementation/test/blob-store-route-regression.test.ts @@ -26,6 +26,8 @@ import test from "node:test"; import { fileURLToPath } from "node:url"; import { startServer } from "../server/index.ts"; +import { ingestRecords, recordIndexWorkStatsForTests, withRecordIndexWorkForTests } from "../server/records.ts"; +import { makeDefaultAccountConnectorInstanceId } from "../server/stores/connector-instance-store.ts"; const __dirname = dirname(fileURLToPath(import.meta.url)); const REFERENCE_IMPL_DIR = join(__dirname, ".."); @@ -105,6 +107,26 @@ async function withHarness(fn: (urls: { asUrl: string; rsUrl: string }) => Promi } } +function deferred(): { promise: Promise; resolve: () => void } { + let resolve: (() => void) | undefined; + const promise = new Promise((done) => { + resolve = done; + }); + assert.ok(resolve, "Promise executor runs synchronously, so resolve is always assigned here"); + return { promise, resolve }; +} + +async function waitFor(predicate: () => boolean, timeoutMs = 1000): Promise { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + if (Date.now() >= deadline) { + throw new Error("timed out waiting for the deterministic test condition"); + } + // biome-ignore lint/performance/noAwaitInLoops: Polling is intentionally sequential for a deterministic gate. + await new Promise((resolve) => setTimeout(resolve, 1)); + } +} + interface ConnectorManifest { connector_id: string; [key: string]: unknown; @@ -142,6 +164,79 @@ test("GET /v1/blobs/:blob_id returns 404 blob_not_found for unknown blob_id", as }); }); +test("POST /v1/blobs stays live while same-instance batch indexing waits", async () => { + const previous = { + indexLimit: process.env.PDPP_INGEST_INDEX_WORK_LIMIT, + lockWait: process.env.PDPP_INGEST_LOCK_WAIT_MS, + }; + process.env.PDPP_INGEST_INDEX_WORK_LIMIT = "1"; + process.env.PDPP_INGEST_LOCK_WAIT_MS = "100"; + try { + await withHarness(async ({ asUrl, rsUrl }) => { + const manifest = loadGmailManifest(); + await registerConnector(asUrl, manifest); + const ownerToken = await issueOwnerToken(asUrl); + const storageConnectorId = "gmail"; + const connectorInstanceId = makeDefaultAccountConnectorInstanceId("owner_local", storageConnectorId); + const indexEntered = deferred(); + const indexRelease = deferred(); + const heldIndexPermit = withRecordIndexWorkForTests(async () => { + indexEntered.resolve(); + await indexRelease.promise; + }); + let batch: Promise>> | undefined; + try { + await indexEntered.promise; + batch = ingestRecords({ connector_id: storageConnectorId, connector_instance_id: connectorInstanceId }, [ + { + data: { id: "thread-liveness-1", subject: "index wait" }, + emitted_at: "2026-08-07T00:00:00.000Z", + key: "thread-liveness-1", + stream: "threads", + }, + ]); + await waitFor(() => recordIndexWorkStatsForTests().queued >= 1); + + const upload = await fetch( + `${rsUrl}/v1/blobs?${new URLSearchParams({ + connector_id: manifest.connector_id, + connector_instance_id: connectorInstanceId, + record_key: "attachment-liveness-1", + stream: "attachments", + })}`, + { + body: Buffer.from("blob-liveness", "utf8"), + headers: { + Authorization: `Bearer ${ownerToken}`, + "Content-Type": "text/plain", + }, + method: "POST", + } + ); + assert.equal(upload.status, 200, `blob upload must not wait on derived indexes (${await upload.text()})`); + + indexRelease.resolve(); + const outcomes = await batch; + assert.equal(outcomes[0]?.accepted, true); + } finally { + indexRelease.resolve(); + await Promise.allSettled([heldIndexPermit, ...(batch ? [batch] : [])]); + } + }); + } finally { + if (previous.indexLimit === undefined) { + delete process.env.PDPP_INGEST_INDEX_WORK_LIMIT; + } else { + process.env.PDPP_INGEST_INDEX_WORK_LIMIT = previous.indexLimit; + } + if (previous.lockWait === undefined) { + delete process.env.PDPP_INGEST_LOCK_WAIT_MS; + } else { + process.env.PDPP_INGEST_LOCK_WAIT_MS = previous.lockWait; + } + } +}); + test("GET /v1/blobs/:blob_id returns 404 when blob exists but no visible record references it", async () => { await withHarness(async ({ asUrl, rsUrl }) => { const manifest = loadGmailManifest(); diff --git a/reference-implementation/test/browser-enrollment-shell-route.test.ts b/reference-implementation/test/browser-enrollment-shell-route.test.ts index 684f0a0b1..2dd26f892 100644 --- a/reference-implementation/test/browser-enrollment-shell-route.test.ts +++ b/reference-implementation/test/browser-enrollment-shell-route.test.ts @@ -14,6 +14,7 @@ import { import { getDb } from "../server/db.ts"; import { startServer } from "../server/index.ts"; import { BROWSER_ENROLLMENT_SHELL_TTL_MS } from "../server/routes/ref-browser-enrollment-shell.ts"; +import { createSqliteConnectorInstanceStore } from "../server/stores/connector-instance-store.ts"; // Integration coverage for the browser-enrollment shell routes: // POST /_ref/connectors/:connectorId/browser-enrollment-shell (on AS) @@ -106,6 +107,32 @@ async function withServer(fn: (urls: { asUrl: string; rsUrl: string }) => Promis } } +// Owner-auth-disabled harness for the end-to-end promotion test below, which +// needs an owner BEARER token (device flow) in addition to the owner-session +// shell-creation surface. With an empty owner password the default owner +// session is active with no cookie needed at all, so `/device/approve` +// (owner-session + CSRF gated) and `/_ref/...` both work with an empty +// cookie — same pattern as static-secret-draft-connection-route.test.ts's +// `withOpenServer`. +async function withOpenServer(fn: (urls: { asUrl: string; rsUrl: string }) => Promise): Promise { + const server = await startServer({ + asPort: 0, + autoEnrollEligibleSchedules: false, + dbPath: ":memory:", + ownerAuthPassword: "", + ownerAuthSubjectId: OWNER_SUBJECT_ID, + quiet: true, + rsPort: 0, + }); + const asUrl = `http://localhost:${server.asPort}`; + const rsUrl = `http://localhost:${server.rsPort}`; + try { + await fn({ asUrl, rsUrl }); + } finally { + await closeServer(server); + } +} + // Owner login is on the AS (same as /_ref/... routes). async function ownerLogin(asUrl: string, password: string = OWNER_PASSWORD): Promise { const res = await fetch(`${asUrl}/owner/login`, { @@ -118,6 +145,58 @@ async function ownerLogin(asUrl: string, password: string = OWNER_PASSWORD): Pro return cookie.split(";")[0] ?? ""; } +// Owner bearer token via the device-authorization flow, same as the +// static-secret-draft ingest-activation coverage (see +// static-secret-draft-connection-route.test.ts's `issueOwnerToken`) — the RS +// ingest endpoint below is bearer-authenticated, not cookie-authenticated. +async function issueOwnerToken(asUrl: string, subjectId: string = OWNER_SUBJECT_ID): Promise { + const clientId = "cli_longview"; + const deviceRes = await fetch(`${asUrl}/oauth/device_authorization`, { + body: new URLSearchParams({ client_id: clientId }).toString(), + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + method: "POST", + }); + const deviceBody = await jsonBody(deviceRes); + await fetch(`${asUrl}/device/approve`, { + body: new URLSearchParams({ subject_id: subjectId, user_code: String(deviceBody.user_code) }).toString(), + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + method: "POST", + }); + const tokenRes = await fetch(`${asUrl}/oauth/token`, { + body: new URLSearchParams({ + client_id: clientId, + device_code: String(deviceBody.device_code), + grant_type: "urn:ietf:params:oauth:grant-type:device_code", + }).toString(), + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + method: "POST", + }); + const tokenBody = await jsonBody(tokenRes); + return String(tokenBody.access_token); +} + +async function ingestNdjson( + rsUrl: string, + ownerToken: string, + connectorId: string, + connectionId: string, + stream: string, + records: Array<{ id: string; emitted_at: string; [key: string]: unknown }> +): Promise { + const lines = records + .map((record) => JSON.stringify({ data: record, emitted_at: record.emitted_at, key: record.id })) + .join("\n"); + const url = + `${rsUrl}/v1/ingest/${encodeURIComponent(stream)}` + + `?connector_id=${encodeURIComponent(connectorId)}` + + `&connector_instance_id=${encodeURIComponent(connectionId)}`; + return await fetch(url, { + body: lines, + headers: { Authorization: `Bearer ${ownerToken}`, "Content-Type": "application/x-ndjson" }, + method: "POST", + }); +} + // --- POST /_ref/connectors/:connectorId/browser-enrollment-shell --- test("browser-enrollment shell: creates draft for supported browser collector connector", async () => { @@ -485,3 +564,90 @@ test("retireExpiredBrowserEnrollmentShells flips expired draft/active shell bind }, ]); }); + +// --- End-to-end promotion: shell creation -> real ingest -> durable binding --- +// +// Live repro this closes: a ChatGPT connector_instance with 9,163 records had +// status revoked while source_binding_json still read +// `{kind: browser_enrollment_shell, ...}` — Sources hid it (RETIRED_SETUP_ +// SHELL_BINDING_KINDS) while Explore still showed its records, because +// nothing had ever promoted the binding off `browser_enrollment_shell` on +// successful first collection. This drives the REAL HTTP path (shell create +// -> RS ingest -> the same `activateDraftConnection` capability the +// static-secret-draft flow uses) rather than calling the store directly. +test("browser-enrollment shell: a successful first ingest promotes the shell to a durable browser_collector binding, and TTL retirement afterward never revokes it", async () => { + await withOpenServer(async ({ asUrl, rsUrl }) => { + await registerConnector(asUrl, "chatgpt"); + const cookie = ""; + + const created = await fetch(`${asUrl}/_ref/connectors/chatgpt/browser-enrollment-shell`, { + headers: { cookie }, + method: "POST", + }); + assert.equal(created.status, 201); + const createdBody = await jsonBody(created); + const connectionId = asString(createdBody.connection_id); + + // Before ingest: still a shell, hidden from /_ref/connections (mirrors + // the static-secret-draft pattern this flow was modeled on). + const preIngestRow = getDb() + .prepare("SELECT status, source_binding_json FROM connector_instances WHERE connector_instance_id = ?") + .get(connectionId) as { status: string; source_binding_json: string }; + assert.equal(preIngestRow.status, "draft"); + assert.equal(JSON.parse(preIngestRow.source_binding_json).kind, "browser_enrollment_shell"); + + const ownerToken = await issueOwnerToken(asUrl); + const ingestRes = await ingestNdjson(rsUrl, ownerToken, "chatgpt", connectionId, "conversations", [ + { emitted_at: "2026-08-06T09:00:00.000Z", id: "conv_1", title: "hello" }, + ]); + assert.equal(ingestRes.status, 200, `ingest into shell should succeed: ${await ingestRes.text()}`); + + // After ingest: promoted — durable binding, active, and NOT the shell + // kind anymore. + const postIngestRow = getDb() + .prepare("SELECT status, source_binding_json FROM connector_instances WHERE connector_instance_id = ?") + .get(connectionId) as { status: string; source_binding_json: string }; + assert.equal(postIngestRow.status, "active", "promoted connection is active"); + const postIngestBinding = JSON.parse(postIngestRow.source_binding_json); + assert.equal(postIngestBinding.kind, "browser_collector", "binding kind moved off browser_enrollment_shell"); + assert.equal(postIngestBinding.connector_id, "chatgpt", "connector_id carried over from the shell binding"); + + // Now visible on the owner-facing raw connection list. + const listRes = await fetch(`${asUrl}/_ref/connections`, { headers: { cookie } }); + const listBody = (await jsonBody(listRes)) as { data?: Record[] }; + const visible = (listBody.data ?? []).find( + (c) => c.connection_id === connectionId || c.connector_instance_id === connectionId + ); + assert.ok(visible, "promoted connection is visible on /_ref/connections"); + assert.equal(visible?.status, "active"); + + // The exact live-bug shape: run the REAL retirement sweep well past the + // shell's original 2h TTL. A promoted connection must survive untouched + // — this is the assertion that would have failed against the code + // before this fix (the sweep would have revoked it). + const store = createSqliteConnectorInstanceStore(); + const farFuture = new Date(Date.now() + BROWSER_ENROLLMENT_SHELL_TTL_MS * 10).toISOString(); + const retiredIds = await retireExpiredBrowserEnrollmentShells( + { + listDraftBrowserEnrollmentShells: (ownerSubjectId) => + Promise.resolve( + store.listDraftBrowserEnrollmentShells(ownerSubjectId) as unknown as { + connectorInstanceId: string; + sourceBinding?: Record | null; + status: string; + }[] + ), + updateStatus: (connectorInstanceId, args) => Promise.resolve(store.updateStatus(connectorInstanceId, args)), + }, + { now: farFuture, ownerSubjectId: OWNER_SUBJECT_ID } + ); + assert.ok( + !retiredIds.includes(connectionId), + "TTL retirement run long past the shell's original TTL does not revoke the promoted connection" + ); + const finalRow = getDb() + .prepare("SELECT status FROM connector_instances WHERE connector_instance_id = ?") + .get(connectionId) as { status: string }; + assert.equal(finalRow.status, "active", "connection remains active after the retirement sweep"); + }); +}); diff --git a/reference-implementation/test/connection-setup-plan.test.ts b/reference-implementation/test/connection-setup-plan.test.ts index 4ca2e742b..a2b93e66a 100644 --- a/reference-implementation/test/connection-setup-plan.test.ts +++ b/reference-implementation/test/connection-setup-plan.test.ts @@ -204,6 +204,24 @@ test("setup planner marks live-proven static-secret connectors as supported", () } }); +test("YNAB static-secret setup supports owner-session capture route", () => { + // YNAB regression: static-secret capture form was unreachable from the + // catalog picker when YNAB was not in the live-proven roster. Verify the fix. + const plan = buildConnectionSetupPlan({ + connectorKey: "ynab", + manifest: staticSecretManifest("ynab", "personal_access_token"), + }); + assert.equal(plan.connectorModality, "api_network"); + assert.equal(plan.setupModality, "static_secret"); + assert.equal(plan.supportState, "supported"); + assert.equal(plan.catalogDisposition, "static_secret_connect"); + assert.equal(plan.nextStepKind, "capture_static_secret"); + assert.equal(plan.ownerAgentIntent.status, "supported"); + assert.equal(plan.ownerAgentIntent.method, "POST"); + assert.equal(plan.proofGate, null); + assert.equal(plan.validationMode, "first_sync"); +}); + test("setup planner treats hybrid filesystem static-secret connectors as credential capture setup", () => { const plan = buildConnectionSetupPlan({ connectorKey: "slack", diff --git a/reference-implementation/test/connector-instance-delete-vs-queued-write-fence.test.ts b/reference-implementation/test/connector-instance-delete-vs-queued-write-fence.test.ts new file mode 100644 index 000000000..af00d00f8 --- /dev/null +++ b/reference-implementation/test/connector-instance-delete-vs-queued-write-fence.test.ts @@ -0,0 +1,514 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * PR #84 red-team claim: connection deletion versus a queued record/blob + * write for the SAME connector_instance_id. + * + * `deleteConnection` and `ingestRecord`/`ingestRecords` both serialize + * through the identical per-instance gate (`withConnectorInstanceWrite` in + * `connector-instance-write-coordinator.ts`), so the two operations can + * never run concurrently for one identity. What this test isolates is the + * ORDERING once they are serialized: + * + * A) write-first: the write acquires the gate, runs to completion, THEN + * the delete acquires the gate and purges everything. Expected/sane: + * the write's rows exist transiently, then the delete leaves a clean + * terminal state (no `records`/`record_changes`/`blobs`/`blob_bindings` + * rows, no `connector_instances` row, a tombstone present). + * + * B) delete-first: the delete acquires the gate first, purges the + * connection (including any records the write's caller believed were + * already ingested is irrelevant here — the write hasn't run yet), and + * releases. The queued write THEN acquires the same gate afterward and + * runs `ingestRecord` against the now-deleted `connector_instance_id`. + * + * The coordinator only provides MUTUAL EXCLUSION, not an ordering-aware + * REJECTION. `ingestSqliteRecord`/`ingestPostgresRecord` (server/records.ts) + * perform zero existence/active-state check against `connector_instances` + * before writing `records`/`record_changes`/`version_counter`. Neither the + * SQLite schema (server/db.ts) nor the Postgres schema + * (server/postgres-storage.ts) declares a foreign key from + * `records`/`record_changes`/`blobs`/`blob_bindings` to + * `connector_instances` (Postgres DOES declare FKs with ON DELETE CASCADE + * for `connector_instance_credentials`, `acquisition_batches`, and + * `record_acquisition_provenance` — the record/blob family is conspicuously + * NOT among them). So ordering B is hypothesized to silently resurrect a + * live `records` row (and, transitively, `blobs`/`blob_bindings`) for a + * connector_instance_id that has a tombstone and no owning + * `connector_instances` row — a zombie record invisible to + * `deleteConnection`'s own purge (which already ran) and to any owner UI + * that resolves connections by joining through `connector_instances`. + * + * The fix (`RecordIngestOptions.requireConnectionAdmission`) is opt-in: + * `ingestRecord`/`ingestRecords` stay a connector-agnostic durable storage + * primitive for direct callers (dozens of existing tests, internal repair + * paths) that never enroll a `connector_instances` row. Only HTTP routes + * that admit an external caller after resolving a real connection set + * `requireConnectionAdmission: true` — this test exercises that opted-in + * path directly, matching how `server/routes/rs-mutation.ts` calls it. + * + * Both orderings are driven deterministically in a single process via + * `__setConnectorInstanceWritePhaseHookForTest`, which fires immediately + * before the per-instance gate is acquired — the same seam + * `connector-instance-write-coordinator.test.ts` uses for deterministic + * interleaving. This is a genuine production-path exercise (the real + * `deleteConnection` store method, the real `ingestRecord` ingest path, the + * real coordinator, the real schema) — not a source-text assertion. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; +// biome-ignore lint/performance/noNamespaceImport: `server/auth.ts` is untyped-boundary legacy JS at several call sites; matches the records.ts import convention below. +import * as authModule from "../server/auth.ts"; +import { __setConnectorInstanceWritePhaseHookForTest } from "../server/connector-instance-write-coordinator.ts"; +import { closeDb, getDb, initDb } from "../server/db.ts"; +import { OWNER_AUTH_DEFAULT_SUBJECT_ID } from "../server/owner-auth.ts"; +import { closePostgresStorage, initPostgresStorage, postgresQuery } from "../server/postgres-storage.ts"; +// biome-ignore lint/performance/noNamespaceImport: `server/records.ts` is untyped-boundary legacy JS at several call sites; the namespace-import + local-type-recast pattern matches the established convention (see aggregate-time-buckets.test.ts). +import * as recordsModule from "../server/records.ts"; +import { + createPostgresConnectorInstanceStore, + createSqliteConnectorInstanceStore, +} from "../server/stores/connector-instance-store.ts"; +import { dedicatedPostgresTestUrl } from "./helpers/dedicated-postgres-test-url.ts"; + +const DEDICATED_POSTGRES_URL = dedicatedPostgresTestUrl(process.env.PDPP_TEST_POSTGRES_URL); + +const CONNECTOR_ID = "delete_fence_probe"; +const STREAM = "events"; +const NOW = "2026-08-07T00:00:00.000Z"; + +interface StorageTarget { + connector_id: string; + connector_instance_id: string; +} + +type IngestRecordFn = ( + storageTarget: StorageTarget, + record: { data: Record; emitted_at: string; key: string; stream: string }, + options?: { requireConnectionAdmission?: boolean } +) => Promise<{ accepted: boolean; changed: boolean }>; + +const ingestRecord = recordsModule.ingestRecord as unknown as IngestRecordFn; +const registerConnector = authModule.registerConnector as unknown as (manifest: object) => Promise; + +/** + * Drives `first` to full completion (acquiring and releasing the + * per-instance gate) BEFORE `second` even attempts to acquire it. This + * deterministically produces the "queued write" ordering under test: since + * `deleteConnection` and `ingestRecord` both serialize through the SAME + * `withConnectorInstanceWrite` gate for one `connector_instance_id`, a + * strict await-then-await sequence on the SAME identity is equivalent (for + * outcome purposes) to `second` having been queued behind `first` and + * dequeued only once `first` released — the coordinator provides no other + * ordering-sensitive behavior between a released gate and a fresh + * acquisition. `__setConnectorInstanceWritePhaseHookForTest` (the same seam + * `connector-instance-write-coordinator.test.ts` uses) instruments both + * acquisitions so the recorded order is asserted, not merely assumed. + */ +async function sequenceThroughGate( + connectorInstanceId: string, + first: () => Promise, + second: () => Promise +): Promise<{ acquisitionOrder: string[]; firstResult: A; secondResult: B }> { + const acquisitionOrder: string[] = []; + let phase: "first" | "second" = "first"; + + __setConnectorInstanceWritePhaseHookForTest((stage, context) => { + if (context.connectorInstanceId !== connectorInstanceId || stage !== "before_key_acquire") { + return; + } + acquisitionOrder.push(phase); + }); + + try { + const firstResult = await first(); + phase = "second"; + const secondResult = await second(); + return { acquisitionOrder, firstResult, secondResult }; + } finally { + __setConnectorInstanceWritePhaseHookForTest(null); + } +} + +function recordEnvelope(id: string) { + return { + data: { id, value: "probe" }, + emitted_at: NOW, + key: id, + stream: STREAM, + }; +} + +function manifest() { + return { + capabilities: { human_interaction: [] }, + connector_id: CONNECTOR_ID, + display_name: "Delete Fence Probe Connector", + protocol_version: "0.1.0", + streams: [ + { + name: STREAM, + primary_key: ["id"], + schema: { + properties: { id: { type: "string" }, value: { type: ["string", "null"] } }, + required: ["id"], + type: "object", + }, + }, + ], + version: "1.0.0", + }; +} + +function sqlitePurge() { + return { + deleteRecordRowsPostgres: () => { + throw new Error("deleteRecordRowsPostgres must not be called by the SQLite store"); + }, + deleteRecordRowsSqlite: (connectorInstanceId: string) => + recordsModule.deleteConnectionRecordRowsSqlite(connectorInstanceId), + enumerateStreams: (storageTarget: StorageTarget) => recordsModule.enumerateConnectionStreams(storageTarget), + teardownProjection: (args: { + connectorId: string; + connectorInstanceId: string; + streams: string[]; + deletedRecordCount: number; + }) => recordsModule.teardownConnectionSearchProjection(args), + }; +} + +function postgresPurge() { + return { + deleteRecordRowsPostgres: (client: unknown, connectorInstanceId: string) => + recordsModule.deleteConnectionRecordRowsPostgres(client as never, connectorInstanceId), + deleteRecordRowsSqlite: () => { + throw new Error("deleteRecordRowsSqlite must not be called by the Postgres store"); + }, + enumerateStreams: (storageTarget: StorageTarget) => recordsModule.enumerateConnectionStreams(storageTarget), + teardownProjection: (args: { + connectorId: string; + connectorInstanceId: string; + streams: string[]; + deletedRecordCount: number; + }) => recordsModule.teardownConnectionSearchProjection(args), + }; +} + +// --------------------------------------------------------------------------- +// SQLite +// --------------------------------------------------------------------------- + +async function seedSqliteInstance(connectorInstanceId: string) { + const store = createSqliteConnectorInstanceStore(); + await store.upsert({ + connectorId: CONNECTOR_ID, + connectorInstanceId, + createdAt: NOW, + displayName: "Probe", + ownerSubjectId: OWNER_AUTH_DEFAULT_SUBJECT_ID, + sourceBinding: { account: "probe@example.com" }, + sourceBindingKey: `probe-${connectorInstanceId}@example.com`, + sourceKind: "account", + status: "active", + updatedAt: NOW, + }); +} + +function sqliteRowCounts(connectorInstanceId: string) { + const db = getDb(); + const count = (table: string) => + ( + db.prepare(`SELECT COUNT(*) AS n FROM ${table} WHERE connector_instance_id = ?`).get(connectorInstanceId) as { + n: number; + } + ).n; + return { + blobBindings: count("blob_bindings"), + blobs: count("blobs"), + connectorInstance: ( + db + .prepare("SELECT COUNT(*) AS n FROM connector_instances WHERE connector_instance_id = ?") + .get(connectorInstanceId) as { n: number } + ).n, + recordChanges: count("record_changes"), + records: count("records"), + tombstone: ( + db + .prepare("SELECT COUNT(*) AS n FROM connector_instance_tombstones WHERE connector_instance_id = ?") + .get(connectorInstanceId) as { n: number } + ).n, + }; +} + +test("SQLite: write-admitted-first then delete leaves a clean terminal state (ordering A)", async () => { + initDb(); + try { + await registerConnector(manifest()); + const connectorInstanceId = "cin_delete_fence_a"; + await seedSqliteInstance(connectorInstanceId); + const storageTarget = { connector_id: CONNECTOR_ID, connector_instance_id: connectorInstanceId }; + const store = createSqliteConnectorInstanceStore(); + + const { acquisitionOrder } = await sequenceThroughGate( + connectorInstanceId, + () => ingestRecord(storageTarget, recordEnvelope("rec_a"), { requireConnectionAdmission: true }), + () => + store.deleteConnection(connectorInstanceId, { + now: NOW, + ownerSubjectId: OWNER_AUTH_DEFAULT_SUBJECT_ID, + purge: sqlitePurge(), + }) + ); + assert.deepEqual(acquisitionOrder, ["first", "second"], "the write must acquire the gate before the delete"); + + const counts = sqliteRowCounts(connectorInstanceId); + assert.deepEqual( + counts, + { + blobBindings: 0, + blobs: 0, + connectorInstance: 0, + recordChanges: 0, + records: 0, + tombstone: 1, + }, + `ordering A must leave a clean terminal state — got ${JSON.stringify(counts)}` + ); + } finally { + closeDb(); + } +}); + +test("SQLite: delete-commits-first then a queued write is refused, never creating post-delete zombie state (ordering B)", async () => { + initDb(); + try { + await registerConnector(manifest()); + const connectorInstanceId = "cin_delete_fence_b"; + await seedSqliteInstance(connectorInstanceId); + const storageTarget = { connector_id: CONNECTOR_ID, connector_instance_id: connectorInstanceId }; + const store = createSqliteConnectorInstanceStore(); + + const acquisitionOrder: string[] = []; + let phase: "first" | "second" = "first"; + __setConnectorInstanceWritePhaseHookForTest((stage, context) => { + if (context.connectorInstanceId === connectorInstanceId && stage === "before_key_acquire") { + acquisitionOrder.push(phase); + } + }); + + let deleteOutcome: Awaited> | undefined; + let writeError: unknown; + try { + deleteOutcome = await store.deleteConnection(connectorInstanceId, { + now: NOW, + ownerSubjectId: OWNER_AUTH_DEFAULT_SUBJECT_ID, + purge: sqlitePurge(), + }); + phase = "second"; + await ingestRecord(storageTarget, recordEnvelope("rec_b"), { requireConnectionAdmission: true }); + } catch (err) { + writeError = err; + } finally { + __setConnectorInstanceWritePhaseHookForTest(null); + } + assert.deepEqual(acquisitionOrder, ["first", "second"], "the delete must acquire the gate before the queued write"); + assert.ok(deleteOutcome, "delete must complete and report a summary before the queued write runs"); + + // Without `assertConnectorInstanceWritable` in server/records.ts, this + // queued write silently succeeds (accepted=true, changed=true) and + // inserts a live `records` + `record_changes` row for a + // connector_instance_id with no `connector_instances` row — a zombie + // record. WITH the fix, it must throw the same typed + // `connector_instance_not_found` the delete route's own ownership check + // raises. + assert.ok(writeError instanceof Error, "the queued write must throw, not silently succeed"); + assert.equal( + (writeError as { code?: string }).code, + "connector_instance_not_found", + `queued write must be refused with connector_instance_not_found — got ${String(writeError)}` + ); + + const counts = sqliteRowCounts(connectorInstanceId); + assert.deepEqual( + counts, + { blobBindings: 0, blobs: 0, connectorInstance: 0, recordChanges: 0, records: 0, tombstone: 1 }, + `no zombie row may exist after the refused write — got ${JSON.stringify(counts)}` + ); + } finally { + closeDb(); + } +}); + +test("SQLite negative control: generic ingest remains ungated unless a lifecycle-aware caller opts in", async () => { + initDb(); + try { + await registerConnector(manifest()); + const connectorInstanceId = "cin_delete_fence_negative_control"; + await seedSqliteInstance(connectorInstanceId); + const storageTarget = { connector_id: CONNECTOR_ID, connector_instance_id: connectorInstanceId }; + const store = createSqliteConnectorInstanceStore(); + + await store.deleteConnection(connectorInstanceId, { + now: NOW, + ownerSubjectId: OWNER_AUTH_DEFAULT_SUBJECT_ID, + purge: sqlitePurge(), + }); + const outcome = await ingestRecord(storageTarget, recordEnvelope("rec_negative_control")); + + assert.deepEqual(outcome, { accepted: true, changed: true }); + assert.equal(sqliteRowCounts(connectorInstanceId).records, 1); + } finally { + closeDb(); + } +}); + +// --------------------------------------------------------------------------- +// Postgres (skipped unless PDPP_TEST_POSTGRES_URL targets the dedicated, +// loopback-only test listener — see test/helpers/dedicated-postgres-test-url.ts) +// --------------------------------------------------------------------------- + +async function seedPostgresInstance(connectorInstanceId: string) { + await postgresQuery( + "INSERT INTO connectors(connector_id, manifest, created_at) VALUES($1, $2::jsonb, $3) ON CONFLICT(connector_id) DO NOTHING", + [CONNECTOR_ID, JSON.stringify(manifest()), NOW] + ); + const store = createPostgresConnectorInstanceStore(); + await store.upsert({ + connectorId: CONNECTOR_ID, + connectorInstanceId, + createdAt: NOW, + displayName: "Probe", + ownerSubjectId: OWNER_AUTH_DEFAULT_SUBJECT_ID, + sourceBinding: { account: "probe@example.com" }, + sourceBindingKey: `probe-${connectorInstanceId}@example.com`, + sourceKind: "account", + status: "active", + updatedAt: NOW, + }); +} + +async function postgresRowCounts(connectorInstanceId: string) { + const count = async (table: string) => { + const result = await postgresQuery<{ n: string }>( + `SELECT COUNT(*)::text AS n FROM ${table} WHERE connector_instance_id = $1`, + [connectorInstanceId] + ); + return Number(result.rows[0]?.n ?? 0); + }; + return { + blobBindings: await count("blob_bindings"), + blobs: await count("blobs"), + connectorInstance: await count("connector_instances"), + recordChanges: await count("record_changes"), + records: await count("records"), + tombstone: await count("connector_instance_tombstones"), + }; +} + +async function cleanupPostgresIdentity(connectorInstanceId: string) { + await postgresQuery("DELETE FROM connector_instance_tombstones WHERE connector_instance_id = $1", [ + connectorInstanceId, + ]); + await postgresQuery("DELETE FROM records WHERE connector_instance_id = $1", [connectorInstanceId]); + await postgresQuery("DELETE FROM record_changes WHERE connector_instance_id = $1", [connectorInstanceId]); + await postgresQuery("DELETE FROM version_counter WHERE connector_instance_id = $1", [connectorInstanceId]); + await postgresQuery("DELETE FROM connector_instances WHERE connector_instance_id = $1", [connectorInstanceId]); + await postgresQuery("DELETE FROM connectors WHERE connector_id = $1", [CONNECTOR_ID]); +} + +test("Postgres: write-admitted-first then delete leaves a clean terminal state (ordering A) (skipped: PDPP_TEST_POSTGRES_URL unset or non-dedicated)", { + skip: !DEDICATED_POSTGRES_URL, +}, async () => { + const databaseUrl = DEDICATED_POSTGRES_URL; + assert.ok(databaseUrl, "dedicated Postgres test URL is configured when this test runs"); + await initPostgresStorage({ backend: "postgres", databaseUrl }); + const connectorInstanceId = "cin_delete_fence_pg_a"; + try { + await cleanupPostgresIdentity(connectorInstanceId); + await seedPostgresInstance(connectorInstanceId); + const storageTarget = { connector_id: CONNECTOR_ID, connector_instance_id: connectorInstanceId }; + const store = createPostgresConnectorInstanceStore(); + + const { acquisitionOrder } = await sequenceThroughGate( + connectorInstanceId, + () => ingestRecord(storageTarget, recordEnvelope("rec_pg_a"), { requireConnectionAdmission: true }), + () => + store.deleteConnection(connectorInstanceId, { + now: NOW, + ownerSubjectId: OWNER_AUTH_DEFAULT_SUBJECT_ID, + purge: postgresPurge(), + }) + ); + assert.deepEqual(acquisitionOrder, ["first", "second"], "the write must acquire the gate before the delete"); + + const counts = await postgresRowCounts(connectorInstanceId); + assert.deepEqual( + counts, + { blobBindings: 0, blobs: 0, connectorInstance: 0, recordChanges: 0, records: 0, tombstone: 1 }, + `ordering A must leave a clean terminal state — got ${JSON.stringify(counts)}` + ); + } finally { + await cleanupPostgresIdentity(connectorInstanceId); + await closePostgresStorage(); + } +}); + +test("Postgres: delete-commits-first then a queued write is refused, never creating post-delete zombie state (ordering B) (skipped: PDPP_TEST_POSTGRES_URL unset or non-dedicated)", { + skip: !DEDICATED_POSTGRES_URL, +}, async () => { + const databaseUrl = DEDICATED_POSTGRES_URL; + assert.ok(databaseUrl, "dedicated Postgres test URL is configured when this test runs"); + await initPostgresStorage({ backend: "postgres", databaseUrl }); + const connectorInstanceId = "cin_delete_fence_pg_b"; + try { + await cleanupPostgresIdentity(connectorInstanceId); + await seedPostgresInstance(connectorInstanceId); + const storageTarget = { connector_id: CONNECTOR_ID, connector_instance_id: connectorInstanceId }; + const store = createPostgresConnectorInstanceStore(); + + const acquisitionOrder: string[] = []; + let phase: "first" | "second" = "first"; + __setConnectorInstanceWritePhaseHookForTest((stage, context) => { + if (context.connectorInstanceId === connectorInstanceId && stage === "before_key_acquire") { + acquisitionOrder.push(phase); + } + }); + + let deleteOutcome: Awaited> | undefined; + let writeError: unknown; + try { + deleteOutcome = await store.deleteConnection(connectorInstanceId, { + now: NOW, + ownerSubjectId: OWNER_AUTH_DEFAULT_SUBJECT_ID, + purge: postgresPurge(), + }); + phase = "second"; + await ingestRecord(storageTarget, recordEnvelope("rec_pg_b"), { requireConnectionAdmission: true }); + } catch (err) { + writeError = err; + } finally { + __setConnectorInstanceWritePhaseHookForTest(null); + } + assert.deepEqual(acquisitionOrder, ["first", "second"], "the delete must acquire the gate before the queued write"); + assert.ok(deleteOutcome, "delete must complete and report a summary before the queued write runs"); + + assert.ok(writeError instanceof Error, "the queued write must throw, not silently succeed"); + assert.equal( + (writeError as { code?: string }).code, + "connector_instance_not_found", + `queued write must be refused with connector_instance_not_found — got ${String(writeError)}` + ); + + const counts = await postgresRowCounts(connectorInstanceId); + assert.deepEqual( + counts, + { blobBindings: 0, blobs: 0, connectorInstance: 0, recordChanges: 0, records: 0, tombstone: 1 }, + `no zombie row may exist after the refused write — got ${JSON.stringify(counts)}` + ); + } finally { + await cleanupPostgresIdentity(connectorInstanceId); + await closePostgresStorage(); + } +}); diff --git a/reference-implementation/test/connector-instance-draft-status.test.ts b/reference-implementation/test/connector-instance-draft-status.test.ts index 1f6ec951a..473361d16 100644 --- a/reference-implementation/test/connector-instance-draft-status.test.ts +++ b/reference-implementation/test/connector-instance-draft-status.test.ts @@ -197,6 +197,15 @@ test("browser enrollment admission accepts only an exact owner-owned shell draft const staticDraft = makeDraft(store, { connectorId: "amazon", sourceBindingKey: "static_secret_draft" }); assert.ok(staticDraft, "upsert returned the static-secret draft"); + const admittedStaticDraft = await admitOwnerRunConnection({ + allowDraft: true, + connectorId: "amazon", + connectorInstanceId: staticDraft.connectorInstanceId, + connectorInstanceStore: store, + ownerSubjectId: "owner_1", + }); + assert.equal(admittedStaticDraft.connectorInstanceId, staticDraft.connectorInstanceId); + assert.equal(admittedStaticDraft.status, "draft"); await assert.rejects( () => admitOwnerBrowserEnrollmentRunConnection({ diff --git a/reference-implementation/test/control-actions.test.ts b/reference-implementation/test/control-actions.test.ts index 5a905c473..136ba3c63 100644 --- a/reference-implementation/test/control-actions.test.ts +++ b/reference-implementation/test/control-actions.test.ts @@ -582,8 +582,8 @@ test("GET /_ref/connectors projects known gaps from the latest run summary", asy .get("run_spotify_known_gap") as RunHistoryTerminalRow | undefined; assert.equal(history?.status, "succeeded", "terminal event finalizes its matching run-history row"); assert.deepEqual( - JSON.parse(history?.facts_json ?? "{}") as { known_gaps?: unknown }, - { known_gaps: knownGaps }, + JSON.parse(history?.facts_json ?? "{}") as { known_gaps?: unknown; records_emitted?: number }, + { known_gaps: knownGaps, records_emitted: 0 }, "terminal facts persist on the run-history authority" ); diff --git a/reference-implementation/test/controller-satisfaction-watcher.test.ts b/reference-implementation/test/controller-satisfaction-watcher.test.ts index b930d2763..2850c71fb 100644 --- a/reference-implementation/test/controller-satisfaction-watcher.test.ts +++ b/reference-implementation/test/controller-satisfaction-watcher.test.ts @@ -144,12 +144,16 @@ function detailGapBacklog(overrides: Partial = {}): DetailGapB // calls below always pass an explicit `connectorInstanceId: INSTANCE_ID`) — // the same authority shape `admitOwnerRunConnection` enforces in production, // without a real store. -function fakeAdmitRunConnection(): (input: { +function fakeAdmitRunConnection( + admissions: string[] = [] +): (input: { connectorId: string; connectorInstanceId: string | null; ownerSubjectId: string | null; + runAdmission: "browser_enrollment" | "collection" | "setup"; }) => Promise<{ connectorId: string; connectorInstanceId: string; ownerSubjectId: string }> { - return ({ connectorId, connectorInstanceId, ownerSubjectId: requestedOwnerSubjectId }) => { + return ({ connectorId, connectorInstanceId, ownerSubjectId: requestedOwnerSubjectId, runAdmission }) => { + admissions.push(runAdmission); const ownerSubjectId = requestedOwnerSubjectId || "owner_local"; const exactId = connectorInstanceId ?? `cin_${ownerSubjectId}_${connectorId.replace(/[^a-z0-9]+/gi, "_")}`; return Promise.resolve({ connectorId, connectorInstanceId: exactId, ownerSubjectId }); @@ -310,8 +314,9 @@ test("satisfaction watcher evaluates every unified contract kind from durable ev test("satisfying a reauth action auto-resumes on the existing connection and can flip green", async (t) => { freshDb(t); const calls: RuntimeRunConnectorOptions[] = []; + const admissions: string[] = []; const controller = createController({ - admitRunConnection: fakeAdmitRunConnection(), + admitRunConnection: fakeAdmitRunConnection(admissions), connectorPathResolver: () => "/tmp/connector.js", logger: { error: () => undefined, warn: () => undefined }, runConnectorImpl: completeRunConnector(calls), @@ -347,6 +352,7 @@ test("satisfying a reauth action auto-resumes on the existing connection and can assert.ok(firstCall); assert.equal(firstCall.connectorInstanceId, INSTANCE_ID, "connection_id is preserved"); assert.equal(firstCall.triggerKind, "manual", "owner repair clears owner-attention state without a second click"); + assert.deepEqual(admissions, ["setup"], "credential-repair auto-resume uses the setup admission capability"); const after = synthesizeRenderedVerdict( snapshot({ last_success_at: "2026-06-15T12:00:00.000Z" }), diff --git a/reference-implementation/test/device-exporter-routes.test.ts b/reference-implementation/test/device-exporter-routes.test.ts index 121f32f14..431050e76 100644 --- a/reference-implementation/test/device-exporter-routes.test.ts +++ b/reference-implementation/test/device-exporter-routes.test.ts @@ -1511,6 +1511,115 @@ test("device exporter routes enroll, heartbeat, ingest idempotently, isolate sou }); }); +test("self-revoke lets a device close its own credential using only its own bearer token", async () => { + await withServer(async ({ asUrl }) => { + const device = await enrollDevice(asUrl, "self-revoke-laptop"); + + const selfRevokeResp = await postJson( + `${asUrl}/_ref/device-exporters/${encodeURIComponent(device.device_id)}/self-revoke`, + {}, + authHeaders(device.device_token) + ); + assert.equal(selfRevokeResp.status, 200); + const body = bodyOf(selfRevokeResp); + assert.equal(body.object, "device_exporter_revocation"); + assert.equal(body.device_id, device.device_id); + assert.ok(typeof body.revoked_at === "string" && Number.isFinite(Date.parse(body.revoked_at as string))); + + const heartbeatAfterRevoke = await postJson( + `${asUrl}/_ref/device-exporters/${encodeURIComponent(device.device_id)}/heartbeat`, + { source_instances: [{ source_instance_id: device.source_instance_id }] }, + authHeaders(device.device_token) + ); + assert.equal(heartbeatAfterRevoke.status, 401, "the revoked credential must not authenticate any further request"); + }); +}); + +test("self-revoke rejects an owner session and any other unauthenticated caller", async () => { + await withServer(async ({ asUrl }) => { + const device = await enrollDevice(asUrl, "self-revoke-no-owner"); + + const missingAuth = await postJson( + `${asUrl}/_ref/device-exporters/${encodeURIComponent(device.device_id)}/self-revoke`, + {}, + PROTOCOL_HEADERS + ); + assert.equal(missingAuth.status, 401); + assert.equal(errorCode(missingAuth), "authentication_error"); + + getDb() + .prepare( + `INSERT INTO tokens(token_id, grant_id, subject_id, client_id, token_kind, expires_at) + VALUES(?, NULL, ?, NULL, 'owner', ?)` + ) + .run("owner-token-for-self-revoke-test", "owner_ref", "2999-01-01T00:00:00.000Z"); + const ownerTokenRejected = await postJson( + `${asUrl}/_ref/device-exporters/${encodeURIComponent(device.device_id)}/self-revoke`, + {}, + authHeaders("owner-token-for-self-revoke-test") + ); + assert.equal( + ownerTokenRejected.status, + 403, + "an owner/client bearer token is not a valid device exporter credential" + ); + assert.equal(errorCode(ownerTokenRejected), "permission_error"); + }); +}); + +test("self-revoke is scoped to the authenticated device: a device cannot revoke a different device", async () => { + await withServer(async ({ asUrl }) => { + const first = await enrollDevice(asUrl, "self-revoke-victim"); + const second = await enrollDevice(asUrl, "self-revoke-attacker"); + + const crossDeviceRevoke = await postJson( + `${asUrl}/_ref/device-exporters/${encodeURIComponent(first.device_id)}/self-revoke`, + {}, + authHeaders(second.device_token) + ); + assert.equal( + crossDeviceRevoke.status, + 403, + "a device credential must never be able to revoke a different device by URL id" + ); + assert.equal(errorCode(crossDeviceRevoke), "permission_error"); + + // The victim device's credential must still be live — the cross-device + // attempt above must not have revoked it as a side effect. + const heartbeatStillWorks = await postJson( + `${asUrl}/_ref/device-exporters/${encodeURIComponent(first.device_id)}/heartbeat`, + { source_instances: [{ source_instance_id: first.source_instance_id }] }, + authHeaders(first.device_token) + ); + assert.equal(heartbeatStillWorks.status, 200); + }); +}); + +test("self-revoke retried after the credential is already revoked fails closed with 401, not a crash", async () => { + await withServer(async ({ asUrl }) => { + const device = await enrollDevice(asUrl, "self-revoke-retry"); + + const first = await postJson( + `${asUrl}/_ref/device-exporters/${encodeURIComponent(device.device_id)}/self-revoke`, + {}, + authHeaders(device.device_token) + ); + assert.equal(first.status, 200); + + // A retry with the same (now-revoked) token cannot re-authenticate — the + // device-credential middleware itself rejects a revoked credential before + // the route body runs. This 401 is the exact signal `logout` on the CLI + // side treats as "already revoked" and proceeds to delete local state. + const retry = await postJson( + `${asUrl}/_ref/device-exporters/${encodeURIComponent(device.device_id)}/self-revoke`, + {}, + authHeaders(device.device_token) + ); + assert.equal(retry.status, 401); + assert.equal(errorCode(retry), "authentication_error"); + }); +}); + test("two claude-code source homes ingest the same connector-local key without overwriting each other", async () => { // complete-local-agent-collectors task 3.4 (Claude Code half). Two Claude // Code source homes for the same owner legitimately share connector-local diff --git a/reference-implementation/test/error-code-status-table-exhaustive.test.ts b/reference-implementation/test/error-code-status-table-exhaustive.test.ts index f46de29fc..352754407 100644 --- a/reference-implementation/test/error-code-status-table-exhaustive.test.ts +++ b/reference-implementation/test/error-code-status-table-exhaustive.test.ts @@ -41,6 +41,7 @@ const EXPECTED_CODE_TO_STATUS = { connection_not_found: 404, connection_run_active: 409, connection_tombstoned: 409, + connector_instance_busy: 503, connector_instance_connector_mismatch: 400, connector_instance_inactive: 400, connector_instance_not_found: 404, @@ -81,6 +82,15 @@ const EXPECTED_CODE_TO_STATUS = { query_not_found: 404, run_already_active: 409, run_owner_mismatch: 403, + static_secret_binding_invalid: 409, + static_secret_draft_required: 409, + static_secret_identity_ambiguous: 409, + static_secret_identity_conflict: 409, + static_secret_identity_mismatch: 409, + static_secret_identity_missing: 502, + static_secret_identity_revoked: 409, + static_secret_identity_unavailable: 503, + static_secret_identity_unverified_replacement: 409, unknown_field: 400, unsupported_version: 400, }; diff --git a/reference-implementation/test/event-spine.test.ts b/reference-implementation/test/event-spine.test.ts index 833526baf..796e3b981 100644 --- a/reference-implementation/test/event-spine.test.ts +++ b/reference-implementation/test/event-spine.test.ts @@ -663,6 +663,127 @@ test("event spine", async (t) => { } }); + await t.test("backfills event_seq safely after an interrupted concurrent Gmail append", () => { + const dir = mkdtempSync(join(tmpdir(), "pdpp-spine-event-seq-concurrent-migration-")); + const dbPath = join(dir, "legacy.sqlite"); + const legacyDb = new Database(dbPath); + + try { + legacyDb.exec(` + CREATE TABLE spine_events ( + event_id TEXT PRIMARY KEY, + event_type TEXT NOT NULL, + occurred_at TEXT NOT NULL, + recorded_at TEXT NOT NULL, + scenario_id TEXT NOT NULL, + trace_id TEXT NOT NULL, + actor_type TEXT NOT NULL, + actor_id TEXT NOT NULL, + subject_type TEXT, + subject_id TEXT, + object_type TEXT NOT NULL, + object_id TEXT NOT NULL, + status TEXT NOT NULL, + request_id TEXT, + grant_id TEXT, + run_id TEXT, + source_kind TEXT, + source_id TEXT, + client_id TEXT, + stream_id TEXT, + token_id TEXT, + interaction_id TEXT, + data_json TEXT NOT NULL, + version TEXT NOT NULL + ) + `); + const insertLegacyEvent = legacyDb.prepare(` + INSERT INTO spine_events( + event_id, event_type, occurred_at, recorded_at, scenario_id, trace_id, + actor_type, actor_id, object_type, object_id, status, run_id, data_json, version + ) VALUES (@event_id, @event_type, @occurred_at, @recorded_at, 'gmail', @trace_id, + 'runtime', 'gmail', 'run', @run_id, @status, @run_id, @data_json, 'spine.v1') + `); + for (const event of [ + { + data_json: JSON.stringify({ connector_id: "gmail", connector_instance_id: "gmail-a" }), + event_id: "evt_gmail_legacy_a", + event_type: "run.started", + occurred_at: "2026-08-07T12:00:00.000Z", + recorded_at: "2026-08-07T12:00:00.000Z", + run_id: "run_gmail_a", + status: "running", + trace_id: "trace_gmail_a", + }, + { + data_json: JSON.stringify({ connector_id: "gmail", connector_instance_id: "gmail-b" }), + event_id: "evt_gmail_legacy_b", + event_type: "run.started", + occurred_at: "2026-08-07T12:00:01.000Z", + recorded_at: "2026-08-07T12:00:01.000Z", + run_id: "run_gmail_b", + status: "running", + trace_id: "trace_gmail_b", + }, + ]) { + insertLegacyEvent.run(event); + } + } finally { + legacyDb.close(); + } + + // The interrupted first boot committed the additive column but not its + // backfill. A concurrent Gmail writer then used the normal MAX()+1 + // allocator while the legacy rows still had NULL event_seq values. + const interruptedBootDb = new Database(dbPath); + interruptedBootDb.exec("ALTER TABLE spine_events ADD COLUMN event_seq INTEGER"); + const concurrentGmailDb = new Database(dbPath); + try { + concurrentGmailDb + .prepare(` + INSERT INTO spine_events( + event_id, event_seq, event_type, occurred_at, recorded_at, scenario_id, trace_id, + actor_type, actor_id, object_type, object_id, status, run_id, data_json, version + ) VALUES ( + @event_id, (SELECT COALESCE(MAX(event_seq), 0) + 1 FROM spine_events), + @event_type, @occurred_at, @recorded_at, 'gmail', @trace_id, + 'runtime', 'gmail', 'run', @run_id, @status, @run_id, @data_json, 'spine.v1' + ) + `) + .run({ + data_json: JSON.stringify({ connector_id: "gmail", connector_instance_id: "gmail-a" }), + event_id: "evt_gmail_concurrent_append", + event_type: "run.failed", + occurred_at: "2026-08-07T12:00:02.000Z", + recorded_at: "2026-08-07T12:00:02.000Z", + run_id: "run_gmail_a", + status: "failed", + trace_id: "trace_gmail_a", + }); + } finally { + concurrentGmailDb.close(); + interruptedBootDb.close(); + } + + try { + initDb(dbPath); + const db = getDb(); + const rows = db + .prepare("SELECT event_id, event_seq FROM spine_events ORDER BY event_seq") + .all<{ event_id: string; event_seq: number }>(); + + assert.deepEqual(rows, [ + { event_id: "evt_gmail_concurrent_append", event_seq: 1 }, + { event_id: "evt_gmail_legacy_a", event_seq: 2 }, + { event_id: "evt_gmail_legacy_b", event_seq: 3 }, + ]); + assert.equal(new Set(rows.map((row) => row.event_seq)).size, rows.length); + } finally { + closeDb(); + rmSync(dir, { force: true, recursive: true }); + } + }); + await t.test("captures dynamic client registration success and rejection as trace artifacts", async () => { await withHarness(async ({ asUrl }) => { const registration = await registerDynamicClient(asUrl, { diff --git a/reference-implementation/test/google-data-portability-provider-auth.test.ts b/reference-implementation/test/google-data-portability-provider-auth.test.ts index cee4d961b..158953121 100644 --- a/reference-implementation/test/google-data-portability-provider-auth.test.ts +++ b/reference-implementation/test/google-data-portability-provider-auth.test.ts @@ -27,7 +27,14 @@ const READY_ENV = Object.freeze({ const TEST_KEY = "google-data-portability-test-key"; interface Manifest { + capabilities?: { + public_listing?: { + listed?: boolean; + proof_gate?: string; + }; + }; connector_id?: string; + streams?: Array<{ name?: string }>; [key: string]: unknown; } @@ -376,3 +383,25 @@ test("Google Data Portability provider-auth route materializes an active connect await closeServer(server); } }); + +test("Google Maps Data Portability must be unlisted because it only exposes control-plane metadata (archive jobs), not actual Maps data", () => { + const manifest = readManifest(); + const publicListing = manifest.capabilities?.public_listing; + + assert.ok(publicListing, "manifest declares capabilities.public_listing"); + assert.equal( + publicListing.listed, + false, + "google_maps_data_portability must be unlisted: only archive_jobs (control-plane job status) is implemented; no archive download, Maps resource group parsing, or user data exposure." + ); + assert.ok(publicListing.proof_gate, "manifest must explain why it is unlisted via proof_gate field"); + + const streams = manifest.streams ?? []; + const archiveJobsStream = streams.find((s) => s.name === "archive_jobs"); + assert.ok(archiveJobsStream, "the only stream present is archive_jobs (control plane)"); + assert.equal( + streams.length, + 1, + "only one stream implemented: archive_jobs control-plane metadata. Maps resource group data (reviews, places, activity, etc.) not yet parsed/exposed." + ); +}); diff --git a/reference-implementation/test/owner-connection-delete.test.ts b/reference-implementation/test/owner-connection-delete.test.ts index 7f5438c5f..c3d0ad59c 100644 --- a/reference-implementation/test/owner-connection-delete.test.ts +++ b/reference-implementation/test/owner-connection-delete.test.ts @@ -57,7 +57,7 @@ import { listSpineEventsPage, type SpineEventRecord } from "../lib/spine.ts"; import { canonicalConnectorKey } from "../server/connector-key.ts"; import { getDb } from "../server/db.ts"; import { startServer } from "../server/index.ts"; -import { ingestRecord } from "../server/records.ts"; +import { deleteConnectionRecordRowsSqlite, ingestRecord } from "../server/records.ts"; import { createSqliteConnectorInstanceStore, makeDefaultAccountConnectorInstanceId, @@ -70,6 +70,7 @@ const OWNER_SUBJECT_ID = "owner_local"; const OTHER_SUBJECT_ID = "owner_other"; const OWNER_CLIENT_ID = "cli_longview"; const NOW = "2026-05-31T00:00:00.000Z"; +const POST_PURGE_FAILURE = /injected post-purge failure after 1 records/; function mustRow>(value: T | undefined, description: string): T { assert.ok(value, description); @@ -222,6 +223,7 @@ interface SeedInstanceOptions { sourceBinding?: Record; sourceBindingKey: string; sourceKind?: string; + status?: string; } async function seedInstance({ @@ -232,6 +234,7 @@ async function seedInstance({ sourceKind = "account", sourceBinding, ownerSubjectId = OWNER_SUBJECT_ID, + status = "active", }: SeedInstanceOptions): Promise { const store = createSqliteConnectorInstanceStore(); await store.upsert({ @@ -243,7 +246,7 @@ async function seedInstance({ sourceBinding: sourceBinding ?? { account_hint: sourceBindingKey }, sourceBindingKey, sourceKind, - status: "active", + status, updatedAt: NOW, }); } @@ -318,6 +321,110 @@ function seedBlob({ connectorId, connectorInstanceId, stream, recordKey, blobId .run(blobId, connectorId, connectorInstanceId, stream, recordKey); } +// Gmail attachments are content-addressed. A duplicate Gmail connection can +// bind the same blob_id even though the blob row records the first connection +// that stored the bytes. This fixture keeps one shared blob plus one +// target-only blob, with durable record and summary-evidence rows for both +// connections, so deletion exercises the actual cross-connection FK edge. +function seedSharedGmailFixture({ + connectorId, + stream, + targetConnectionId, + siblingConnectionId, +}: { + connectorId: string; + stream: string; + targetConnectionId: string; + siblingConnectionId: string; +}): void { + const db = getDb(); + assert.equal(db.pragma("foreign_keys", { simple: true }), 1, "disposable SQLite enforces foreign keys"); + const messages = [ + [targetConnectionId, "gmail_primary_message"], + [siblingConnectionId, "gmail_duplicate_message"], + ] as const; + for (const [connectorInstanceId, recordKey] of messages) { + db.prepare( + `INSERT INTO records( + connector_id, connector_instance_id, stream, record_key, record_json, emitted_at, version + ) VALUES(?, ?, ?, ?, ?, ?, 1)` + ).run( + connectorId, + connectorInstanceId, + stream, + recordKey, + JSON.stringify({ + id: recordKey, + labelIds: ["INBOX"], + payload: { headers: [{ name: "Subject", value: "Quarterly report" }] }, + snippet: "Synthetic Gmail message for the connection-delete FK proof.", + threadId: "thread_duplicate_fixture", + }), + NOW + ); + db.prepare( + `INSERT INTO record_changes( + connector_id, connector_instance_id, stream, record_key, version, record_json, emitted_at + ) VALUES(?, ?, ?, ?, 1, ?, ?)` + ).run(connectorId, connectorInstanceId, stream, recordKey, JSON.stringify({ id: recordKey }), NOW); + db.prepare( + `INSERT INTO version_counter(connector_id, connector_instance_id, stream, max_version) + VALUES(?, ?, ?, 1)` + ).run(connectorId, connectorInstanceId, stream); + db.prepare( + `INSERT INTO connector_summary_evidence( + connector_instance_id, connector_id, display_name, total_records, stream_count, stream_records_json + ) VALUES(?, ?, ?, 1, 1, ?)` + ).run(connectorInstanceId, connectorId, connectorInstanceId, JSON.stringify([{ record_count: 1, stream }])); + } + + const sharedBytes = Buffer.from("From: automated@example.test\\r\\nSubject: Quarterly report\\r\\n", "utf8"); + db.prepare( + `INSERT INTO blobs( + blob_id, connector_id, connector_instance_id, stream, record_key, + mime_type, size_bytes, sha256, data + ) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + "blob_gmail_shared", + connectorId, + targetConnectionId, + stream, + "gmail_primary_message", + "message/rfc822", + sharedBytes.byteLength, + "sha256_gmail_shared", + sharedBytes + ); + const targetOnlyBytes = Buffer.from("target-only Gmail bytes", "utf8"); + db.prepare( + `INSERT INTO blobs( + blob_id, connector_id, connector_instance_id, stream, record_key, + mime_type, size_bytes, sha256, data + ) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + "blob_gmail_target_only", + connectorId, + targetConnectionId, + stream, + "gmail_primary_message", + "application/octet-stream", + targetOnlyBytes.byteLength, + "sha256_gmail_target_only", + targetOnlyBytes + ); + for (const [connectorInstanceId, recordKey, blobId] of [ + [targetConnectionId, "gmail_primary_message", "blob_gmail_shared"], + [siblingConnectionId, "gmail_duplicate_message", "blob_gmail_shared"], + [targetConnectionId, "gmail_primary_message", "blob_gmail_target_only"], + ] as const) { + db.prepare( + `INSERT INTO blob_bindings( + blob_id, connector_id, connector_instance_id, stream, record_key, json_path + ) VALUES(?, ?, ?, ?, ?, '@record')` + ).run(blobId, connectorId, connectorInstanceId, stream, recordKey); + } +} + interface SeedAttentionOptions { attentionId: string; connectorId: string; @@ -541,6 +648,145 @@ test("owner-agent delete erases a connection completely: records, history, blobs }); }); +test("owner-agent delete removes revoked Gmail data without deleting a shared blob or sibling duplicate", async () => { + await withServer(async ({ asUrl, rsUrl }) => { + const manifest = JSON.parse( + readFileSync(join(__dirname, "../../packages/polyfill-connectors/manifests/gmail.json"), "utf8") + ) as ReferenceManifest; + const registeredManifest = await registerConnector(asUrl, manifest); + const connectorKey = canonicalConnectorKey(registeredManifest.connector_id); + assert.ok(connectorKey, "expected a canonical Gmail connector key"); + // biome-ignore lint/style/useDestructuring: index access documents the asserted ordered position + const firstStream = registeredManifest.streams[0]; + assert.ok(firstStream, "expected the Gmail manifest to declare at least one stream"); + const stream = firstStream.name; + const target = "cin_gmail_primary"; + const sibling = "cin_gmail_duplicate"; + await seedInstance({ + connectorId: connectorKey, + connectorInstanceId: target, + displayName: "Gmail primary", + sourceBindingKey: "primary@example.test", + status: "revoked", + }); + await seedInstance({ + connectorId: connectorKey, + connectorInstanceId: sibling, + displayName: "Gmail duplicate", + sourceBindingKey: "duplicate@example.test", + }); + seedSharedGmailFixture({ + connectorId: connectorKey, + siblingConnectionId: sibling, + stream, + targetConnectionId: target, + }); + + const blobCount = (blobId: string): number => + (getDb().prepare("SELECT COUNT(*) AS n FROM blobs WHERE blob_id = ?").get(blobId) as { n: number }).n; + assert.equal(countRows("blob_bindings", target), 2, "target has shared and target-only bindings before"); + assert.equal(countRows("blob_bindings", sibling), 1, "sibling shares the content-addressed blob before"); + assert.equal(blobCount("blob_gmail_shared"), 1, "shared Gmail blob exists before"); + assert.equal(blobCount("blob_gmail_target_only"), 1, "target-only Gmail blob exists before"); + + const ownerToken = await issueOwnerToken(asUrl); + const del = await deleteConnection(rsUrl, ownerToken, `/v1/owner/connections/${target}`); + assert.equal(del.status, 200); + assert.equal(deleteBody(del).deleted_record_count, 1); + + assert.equal(getInstance(target), null, "revoked target connection is deleted"); + assert.equal(countRows("records", target), 0, "target Gmail records erased"); + assert.equal(countRows("record_changes", target), 0, "target Gmail history erased"); + assert.equal(countRows("version_counter", target), 0, "target Gmail version state erased"); + assert.equal(countRows("blob_bindings", target), 0, "target Gmail blob bindings erased"); + assert.equal(countRows("connector_summary_evidence", target), 0, "target summary evidence erased"); + assert.equal(blobCount("blob_gmail_target_only"), 0, "unreferenced target blob erased"); + + assert.equal(getInstance(sibling)?.status, "active", "sibling connection remains active"); + assert.equal(countRows("records", sibling), 1, "sibling Gmail record remains"); + assert.equal(countRows("blob_bindings", sibling), 1, "sibling Gmail binding remains"); + assert.equal(countRows("connector_summary_evidence", sibling), 1, "sibling summary evidence remains"); + assert.equal(blobCount("blob_gmail_shared"), 1, "shared blob remains for sibling"); + }); +}); + +test("shared-blob connection deletion rolls back the full SQLite cascade after the record purge", async () => { + await withServer(async ({ asUrl }) => { + const manifest = JSON.parse( + readFileSync(join(__dirname, "../../packages/polyfill-connectors/manifests/gmail.json"), "utf8") + ) as ReferenceManifest; + const registeredManifest = await registerConnector(asUrl, manifest); + const connectorKey = canonicalConnectorKey(registeredManifest.connector_id); + assert.ok(connectorKey, "expected a canonical Gmail connector key"); + // biome-ignore lint/style/useDestructuring: index access documents the asserted ordered position + const firstStream = registeredManifest.streams[0]; + assert.ok(firstStream, "expected the Gmail manifest to declare at least one stream"); + const target = "cin_gmail_rollback_target"; + const sibling = "cin_gmail_rollback_sibling"; + await seedInstance({ + connectorId: connectorKey, + connectorInstanceId: target, + displayName: "Gmail rollback target", + sourceBindingKey: "rollback-target@example.test", + status: "revoked", + }); + await seedInstance({ + connectorId: connectorKey, + connectorInstanceId: sibling, + displayName: "Gmail rollback sibling", + sourceBindingKey: "rollback-sibling@example.test", + }); + seedSharedGmailFixture({ + connectorId: connectorKey, + siblingConnectionId: sibling, + stream: firstStream.name, + targetConnectionId: target, + }); + + const store = createSqliteConnectorInstanceStore(); + await assert.rejects( + () => + store.deleteConnection(target, { + now: NOW, + ownerSubjectId: OWNER_SUBJECT_ID, + purge: { + deleteRecordRowsPostgres: (): Promise => { + throw new Error("unreachable: the SQLite connection-purge path never calls deleteRecordRowsPostgres"); + }, + deleteRecordRowsSqlite: (id: string) => { + const deleted = deleteConnectionRecordRowsSqlite(id); + assert.equal(countRows("records", id), 0, "record purge ran before injected later failure"); + throw new Error(`injected post-purge failure after ${deleted} records`); + }, + enumerateStreams: async () => ({ + connectorId: connectorKey, + connectorInstanceId: target, + streams: [firstStream.name], + }), + teardownProjection: () => Promise.resolve(), + }, + }), + POST_PURGE_FAILURE + ); + + // The record purge and terminal row delete share one transaction. The + // injected failure therefore restores the target data, evidence, shared + // blob, and sibling rows instead of leaving a half-deleted connection. + assert.ok(getInstance(target), "target instance survives rollback"); + assert.equal(countRows("records", target), 1, "target records restored"); + assert.equal(countRows("record_changes", target), 1, "target history restored"); + assert.equal(countRows("blob_bindings", target), 2, "target bindings restored"); + assert.equal(countRows("connector_summary_evidence", target), 1, "target evidence restored"); + assert.equal(countRows("records", sibling), 1, "sibling records remain after rollback"); + assert.equal(countRows("blob_bindings", sibling), 1, "sibling binding remains after rollback"); + assert.equal( + (getDb().prepare("SELECT COUNT(*) AS n FROM blobs").get() as { n: number }).n, + 2, + "shared and target-only blobs both survive rollback" + ); + }); +}); + test("owner-agent delete does not over-reach: a sibling connection of the same connector stays intact (I1)", async () => { await withServer(async ({ asUrl, rsUrl }) => { const manifest = await registerConnector(asUrl, loadReferenceManifest("spotify")); diff --git a/reference-implementation/test/owner-connector-templates.test.ts b/reference-implementation/test/owner-connector-templates.test.ts index 0c001f115..4413d1ff5 100644 --- a/reference-implementation/test/owner-connector-templates.test.ts +++ b/reference-implementation/test/owner-connector-templates.test.ts @@ -73,8 +73,20 @@ function asRecord(value: unknown): Record { return value && typeof value === "object" ? (value as Record) : {}; } -async function withServer(fn: (ctx: { asUrl: string; rsUrl: string }) => Promise): Promise { - const server = await startServer({ asPort: 0, dbPath: ":memory:", ownerAuthPassword: "", quiet: true, rsPort: 0 }); +async function withServer( + fn: (ctx: { asUrl: string; rsUrl: string }) => Promise, + options: { configuredProviderAuthConnectorKeys?: readonly string[] } = {} +): Promise { + const server = await startServer({ + asPort: 0, + ...(options.configuredProviderAuthConnectorKeys === undefined + ? {} + : { configuredProviderAuthConnectorKeys: options.configuredProviderAuthConnectorKeys }), + dbPath: ":memory:", + ownerAuthPassword: "", + quiet: true, + rsPort: 0, + }); const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; try { @@ -213,6 +225,12 @@ function actionByFamily(row: Record, family: string): Record { await withServer(async ({ asUrl, rsUrl }) => { const amazonManifest = await registerConnector(asUrl, loadManifest("amazon")); + const listedUnprovenManifest = loadManifest("doordash"); + listedUnprovenManifest.capabilities = { + ...asRecord(listedUnprovenManifest.capabilities), + public_listing: { listed: true, status: "unproven" }, + }; + await registerConnector(asUrl, listedUnprovenManifest); const amazonKey = canonicalConnectorKey(amazonManifest.connector_id); assert.ok(amazonKey, "amazon manifest must resolve a canonical connector key"); await seedInstance({ @@ -235,11 +253,14 @@ test("owner-agent bearer lists connector templates with related connection summa assert.equal(amazon.connector_id, "amazon"); assert.equal(amazon.display_name, "Amazon"); assert.equal(amazon.connector_modality, "browser_bound"); + assert.equal(amazon.registration_status, "registered"); + assert.deepEqual(amazon.public_listing, { listed: true, status: "needs_human_auth" }); const amazonSetupPlan = asRecord(amazon.setup_plan); assert.equal(amazonSetupPlan.setup_modality, "static_secret"); assert.equal(amazonSetupPlan.support_state, "proof_gated"); assert.equal(amazonSetupPlan.next_step_kind, "capture_static_secret"); assert.equal(amazonSetupPlan.proof_gate, "static_secret_live_proof_missing"); + assert.equal(amazonSetupPlan.owner_actionable, true); assert.equal(amazonSetupPlan.runbook_path, null); assert.equal(amazon.connection_count, 1); const amazonConnections = amazon.connections; @@ -252,28 +273,115 @@ test("owner-agent bearer lists connector templates with related connection summa assert.equal(amazonConnection.label_status, "owner_set"); const amazonInitiate = actionByFamily(amazon, "initiate_connection"); - assert.equal(amazonInitiate.status, "unsupported"); + assert.equal(amazonInitiate.status, "owner_mediated"); assert.equal(amazonInitiate.method, null); assert.equal(amazonInitiate.url, null); // biome-ignore lint/performance/useTopLevelRegex: test assertion patterns remain colocated with the assertion they explain. - assert.match(String(amazonInitiate.reason), /static provider secret|static-secret/i); - - // Local-collector templates are discoverable even before a connection is - // registered, because they live in the reference local-collector catalog. - const codex = byConnector(body, "codex"); - assert.equal(codex.connector_modality, "local_collector"); - const codexSetupPlan = asRecord(codex.setup_plan); - assert.equal(codexSetupPlan.support_state, "supported"); - assert.equal(codexSetupPlan.next_step_kind, "enroll_local_collector"); - assert.equal(codex.connection_count, 0); - const codexInitiate = actionByFamily(codex, "initiate_connection"); - assert.equal(codexInitiate.status, "supported"); - assert.equal(codexInitiate.method, "POST"); - // biome-ignore lint/performance/useTopLevelRegex: test assertion patterns remain colocated with the assertion they explain. - assert.match(String(codexInitiate.url), /\/v1\/owner\/connections\/intents$/); + assert.match(String(amazonInitiate.reason), /secure browser-session dashboard/i); + + const templates = asRecord(body).data; + assert.ok(Array.isArray(templates)); + assert.equal( + templates.some((item) => asRecord(item).connector_key === "codex"), + false, + "a local-only manifest must not create a server catalog entry" + ); + + const doordash = byConnector(body, "doordash"); + assert.deepEqual(doordash.public_listing, { listed: true, status: "unproven" }); + const doordashSetupPlan = asRecord(doordash.setup_plan); + assert.equal(doordashSetupPlan.owner_actionable, false); + const doordashInitiate = actionByFamily(doordash, "initiate_connection"); + assert.equal(doordashInitiate.status, "unsupported"); + assert.equal(doordashInitiate.method, null); + assert.equal(doordashInitiate.url, null); + }); +}); + +test("owner-template projection separates browser owner-session setup from owner-agent REST support", async () => { + await withServer(async ({ asUrl, rsUrl }) => { + await registerConnector(asUrl, loadManifest("chatgpt")); + const browserManualManifest = loadManifest("chase"); + browserManualManifest.setup = undefined; + await registerConnector(asUrl, browserManualManifest); + const browserRunbookManifest = loadManifest("doordash"); + browserRunbookManifest.capabilities = { + ...asRecord(browserRunbookManifest.capabilities), + public_listing: { listed: true, status: "proven" }, + }; + await registerConnector(asUrl, browserRunbookManifest); + + const ownerToken = await issueOwnerToken(asUrl); + const { status, body } = await fetchJson(`${rsUrl}/v1/owner/connector-templates`, { + headers: { Authorization: `Bearer ${ownerToken}` }, + }); + assert.equal(status, 200); + + const chatgpt = byConnector(body, "chatgpt"); + const chatgptSetupPlan = asRecord(chatgpt.setup_plan); + assert.equal(chatgptSetupPlan.catalog_disposition, "static_secret_connect"); + assert.equal(chatgptSetupPlan.owner_actionable, true); + const chatgptInitiate = actionByFamily(chatgpt, "initiate_connection"); + assert.equal(chatgptInitiate.status, "owner_mediated"); + assert.equal(chatgptInitiate.method, null); + assert.equal(chatgptInitiate.url, null); + + const browserManual = byConnector(body, "chase"); + const browserManualSetupPlan = asRecord(browserManual.setup_plan); + assert.equal(browserManualSetupPlan.catalog_disposition, "browser_collector_manual"); + assert.equal(browserManualSetupPlan.next_step_kind, "enroll_browser_collector"); + assert.equal(browserManualSetupPlan.owner_actionable, true); + const browserManualInitiate = actionByFamily(browserManual, "initiate_connection"); + assert.equal(browserManualInitiate.status, "owner_mediated"); + assert.equal(browserManualInitiate.method, null); + assert.equal(browserManualInitiate.url, null); + + const browserRunbook = byConnector(body, "doordash"); + const browserRunbookSetupPlan = asRecord(browserRunbook.setup_plan); + assert.equal(browserRunbookSetupPlan.catalog_disposition, "browser_bound_runbook"); + assert.equal(browserRunbookSetupPlan.next_step_kind, "manual_runbook"); + assert.equal(browserRunbookSetupPlan.owner_actionable, false); + const browserRunbookInitiate = actionByFamily(browserRunbook, "initiate_connection"); + assert.equal(browserRunbookInitiate.status, "unsupported"); + assert.equal(browserRunbookInitiate.method, null); + assert.equal(browserRunbookInitiate.url, null); }); }); +test("owner-template readiness reflects configured provider authorization", async () => { + await withServer( + async ({ asUrl, rsUrl }) => { + const manifest = await registerConnector(asUrl, loadManifest("google_maps_data_portability")); + const connectorKey = canonicalConnectorKey(manifest.connector_id); + assert.equal(connectorKey, "google-maps-data-portability"); + + const ownerToken = await issueOwnerToken(asUrl); + const { status, body } = await fetchJson(`${rsUrl}/v1/owner/connector-templates`, { + headers: { Authorization: `Bearer ${ownerToken}` }, + }); + assert.equal(status, 200); + + const google = byConnector(body, "google-maps-data-portability"); + const publicListing = asRecord(google.public_listing); + assert.equal(publicListing.listed, false); + assert.equal(publicListing.status, "unproven"); + const setupPlan = asRecord(google.setup_plan); + assert.equal(setupPlan.catalog_disposition, "provider_auth_connect"); + const deploymentReadiness = asRecord(setupPlan.deployment_readiness); + assert.equal(deploymentReadiness.state, "ready"); + assert.equal(setupPlan.next_step_kind, "open_provider_auth"); + assert.equal(setupPlan.support_state, "supported"); + assert.equal(setupPlan.proof_gate, null); + assert.equal(setupPlan.owner_actionable, false); + const initiate = actionByFamily(google, "initiate_connection"); + assert.equal(initiate.method, null); + assert.equal(initiate.status, "unsupported"); + assert.equal(initiate.url, null); + }, + { configuredProviderAuthConnectorKeys: ["google-maps-data-portability"] } + ); +}); + test("GET /v1/owner/control advertises list_connector_templates with the template route", async () => { await withServer(async ({ asUrl, rsUrl }) => { const ownerToken = await issueOwnerToken(asUrl); diff --git a/reference-implementation/test/owner-source-to-mcp-closure.test.ts b/reference-implementation/test/owner-source-to-mcp-closure.test.ts new file mode 100644 index 000000000..699bd16d2 --- /dev/null +++ b/reference-implementation/test/owner-source-to-mcp-closure.test.ts @@ -0,0 +1,689 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { createHash, randomBytes } from "node:crypto"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { LocalDeviceClient } from "../../packages/polyfill-connectors/src/local-device-client.ts"; +import { + buildLocalDeviceIngestBatchRequest, + buildLocalDeviceRecordEnvelope, +} from "../../packages/polyfill-connectors/src/local-device-envelope.ts"; +import { canonicalConnectorKeyFromManifest } from "../server/connector-key.ts"; +import { startServer } from "../server/index.ts"; +import { CREDENTIAL_ENCRYPTION_KEY_ENV } from "../server/stores/credential-encryption.ts"; + +const OWNER_PASSWORD = "owner-source-to-mcp-closure-password"; +const OWNER_SUBJECT_ID = "owner_local"; +const CREDENTIAL_KEY = "owner-source-to-mcp-closure-test-key"; +const STATIC_SECRET = "synthetic fixture app password"; +const FIXTURE_TIME = "2026-08-06T12:00:00.000Z"; +const CSRF_FIELD_RE = //; +const CLOSURE_MCP_MISSING_RE = /scoped MCP must read exactly the newly accepted fixture record/; +const GMAIL_FIXTURE_PATH = + "../../packages/polyfill-connectors/fixtures/gmail/scrubbed/pilot-real-shape/records/messages.jsonl"; +const CODEX_FIXTURE_PATH = + "../../packages/polyfill-connectors/fixtures/codex/scrubbed/pilot-real-shape/records/messages.jsonl"; + +type StartedServer = Awaited>; +type JsonRecord = Record; + +interface JsonResponse { + body: unknown; + response: Response; + status: number; + text: string; +} + +interface ConnectorManifest extends JsonRecord { + connector_id: string; + streams: Array<{ name: string; [key: string]: unknown }>; +} + +interface OwnerSession { + cookie: string; + csrfField: string; +} + +interface ClosureEvidence { + acceptedRecordReadableBeforeRevoke: boolean; + grantActiveAfterGrantRevoke: boolean; + grantActiveAfterSourceRevoke: boolean; + mcpRecordIds: string[]; + mcpStatusAfterGrantRevoke: number; + refVisibleStatuses: Record; + sourceRevokeStatus: string; + stoppedIngestStatus: number; +} + +function asRecord(value: unknown, description: string): JsonRecord { + assert.ok(value && typeof value === "object" && !Array.isArray(value), description); + return value as JsonRecord; +} + +function stringField(value: unknown, field: string): string { + const record = asRecord(value, `expected an object containing ${field}`); + assert.equal(typeof record[field], "string", `${field} must be a string`); + return record[field] as string; +} + +function objectField(value: unknown, field: string): JsonRecord { + const record = asRecord(value, `expected an object containing ${field}`); + return asRecord(record[field], `${field} must be an object`); +} + +async function fetchJson(url: string | URL, init: RequestInit = {}): Promise { + const response = await fetch(url, init); + const text = await response.text(); + let body: unknown = null; + try { + body = text ? JSON.parse(text) : null; + } catch { + body = text; + } + return { body, response, status: response.status, text }; +} + +function getSetCookies(response: Response): string[] { + if (typeof response.headers.getSetCookie === "function") { + return response.headers.getSetCookie(); + } + const single = response.headers.get("set-cookie"); + return single ? [single] : []; +} + +function cookiePair(headers: readonly string[], name: string): string | null { + for (const header of headers) { + const [pair] = header.split(";", 1); + if (pair?.startsWith(`${name}=`)) { + return pair; + } + } + return null; +} + +function csrfFieldFromHtml(html: string): string { + const match = html.match(CSRF_FIELD_RE); + assert.ok(match?.[1], "owner login must render a CSRF field"); + return match[1]; +} + +async function login(asUrl: string): Promise { + const loginPage = await fetch(`${asUrl}/owner/login`, { headers: { Accept: "text/html" }, redirect: "manual" }); + const csrfCookie = cookiePair(getSetCookies(loginPage), "pdpp_owner_csrf"); + const csrfField = csrfFieldFromHtml(await loginPage.text()); + assert.ok(csrfCookie, "owner login must set a CSRF cookie"); + + const loginResponse = await fetch(`${asUrl}/owner/login`, { + body: new URLSearchParams({ _csrf: csrfField, password: OWNER_PASSWORD, return_to: "/" }).toString(), + headers: { + Accept: "text/html", + "Content-Type": "application/x-www-form-urlencoded", + Cookie: csrfCookie, + }, + method: "POST", + redirect: "manual", + }); + const sessionCookie = cookiePair(getSetCookies(loginResponse), "pdpp_owner_session"); + assert.ok(sessionCookie, `owner login must issue a session cookie (${loginResponse.status})`); + return { cookie: `${sessionCookie}; ${csrfCookie}`, csrfField }; +} + +async function withCredentialKey(fn: () => Promise): Promise { + const previous = process.env[CREDENTIAL_ENCRYPTION_KEY_ENV]; + process.env[CREDENTIAL_ENCRYPTION_KEY_ENV] = CREDENTIAL_KEY; + try { + return await fn(); + } finally { + if (previous === undefined) { + delete process.env[CREDENTIAL_ENCRYPTION_KEY_ENV]; + } else { + process.env[CREDENTIAL_ENCRYPTION_KEY_ENV] = previous; + } + } +} + +function permissiveCredentialProber() { + return async ({ context }: { context?: { setupFields?: JsonRecord } }) => ({ + detail: null, + identity: context?.setupFields?.account_email ?? "fixture@example.com", + ok: true, + }); +} + +async function closeServer(server: StartedServer): Promise { + server.schedulerManager?.stop?.(); + (server.asServer as unknown as { closeAllConnections?: () => void }).closeAllConnections?.(); + (server.rsServer as unknown as { closeAllConnections?: () => void }).closeAllConnections?.(); + await Promise.allSettled([ + new Promise((resolve) => server.asServer.close(() => resolve())), + new Promise((resolve) => server.rsServer.close(() => resolve())), + ]); +} + +function startClosureServer(): Promise { + return startServer({ + asPort: 0, + autoEnrollEligibleSchedules: false, + dbPath: ":memory:", + ownerAuthPassword: OWNER_PASSWORD, + ownerAuthSubjectId: OWNER_SUBJECT_ID, + quiet: true, + rsPort: 0, + staticSecretAutoResume: false, + staticSecretCredentialProber: permissiveCredentialProber(), + }); +} + +function loadManifest(name: string): ConnectorManifest { + const raw = JSON.parse( + readFileSync(new URL(`../../packages/polyfill-connectors/manifests/${name}.json`, import.meta.url), "utf8") + ) as ConnectorManifest; + const canonical = canonicalConnectorKeyFromManifest(raw); + assert.ok(canonical, `${name} manifest must have a canonical connector key`); + return { ...raw, connector_id: canonical }; +} + +async function registerConnector(asUrl: string, session: OwnerSession, name: string): Promise { + const manifest = loadManifest(name); + const registered = await fetchJson(`${asUrl}/connectors`, { + body: JSON.stringify(manifest), + headers: { "Content-Type": "application/json", Cookie: session.cookie }, + method: "POST", + }); + assert.equal(registered.status, 201, `register ${name}: ${registered.text}`); + return manifest; +} + +async function createStaticDraft(asUrl: string, session: OwnerSession): Promise { + const created = await fetchJson(`${asUrl}/_ref/connectors/gmail/draft-connection`, { + body: JSON.stringify({ setup_fields: { account_email: "fixture-owner@example.com" } }), + headers: { "Content-Type": "application/json", Cookie: session.cookie }, + method: "POST", + }); + assert.equal(created.status, 201, created.text); + return stringField(created.body, "connection_id"); +} + +async function captureStaticCredential( + asUrl: string, + session: OwnerSession, + sourceConnectionId: string +): Promise { + const captured = await fetchJson( + `${asUrl}/_ref/connections/${encodeURIComponent(sourceConnectionId)}/static-secret-credential`, + { + body: JSON.stringify({ credential_kind: "app_password", secret: STATIC_SECRET }), + headers: { "Content-Type": "application/json", Cookie: session.cookie }, + method: "POST", + } + ); + assert.ok(captured.status === 200 || captured.status === 201, captured.text); +} + +async function createManualDraft(asUrl: string, session: OwnerSession): Promise { + const url = new URL(`${asUrl}/_ref/connectors/google-maps/manual-upload-draft-connection`); + url.searchParams.set("file_name", "Timeline.json"); + const created = await fetchJson(url, { + body: JSON.stringify({ + locations: [{ latitudeE7: 377_749_000, longitudeE7: -1_224_194_000, timestampMs: "1717595122000" }], + }), + headers: { "Content-Type": "application/octet-stream", Cookie: session.cookie }, + method: "POST", + }); + assert.equal(created.status, 201, created.text); + return stringField(created.body, "connection_id"); +} + +async function issueOwnerToken(asUrl: string, session: OwnerSession): Promise { + const device = await fetchJson(`${asUrl}/oauth/device_authorization`, { + body: new URLSearchParams({ client_id: "cli_longview" }).toString(), + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + method: "POST", + }); + assert.equal(device.status, 200, device.text); + const approved = await fetchJson(`${asUrl}/device/approve`, { + body: new URLSearchParams({ + _csrf: session.csrfField, + subject_id: OWNER_SUBJECT_ID, + user_code: stringField(device.body, "user_code"), + }).toString(), + headers: { "Content-Type": "application/x-www-form-urlencoded", Cookie: session.cookie }, + method: "POST", + }); + assert.equal(approved.status, 200, approved.text); + const token = await fetchJson(`${asUrl}/oauth/token`, { + body: new URLSearchParams({ + client_id: "cli_longview", + device_code: stringField(device.body, "device_code"), + grant_type: "urn:ietf:params:oauth:grant-type:device_code", + }).toString(), + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + method: "POST", + }); + assert.equal(token.status, 200, token.text); + return stringField(token.body, "access_token"); +} + +function fixtureRecord(relativePath: string): JsonRecord { + const line = readFileSync(new URL(relativePath, import.meta.url), "utf8") + .split("\n") + .find((candidate) => candidate.trim()); + assert.ok(line, `fixture ${relativePath} must contain a record`); + return JSON.parse(line) as JsonRecord; +} + +function ingestNdjson( + rsUrl: string, + ownerToken: string, + connectorId: string, + sourceConnectionId: string, + stream: string, + data: JsonRecord, + emittedAt = FIXTURE_TIME +): Promise { + const recordKey = stringField(data, "id"); + return fetchJson( + `${rsUrl}/v1/ingest/${encodeURIComponent(stream)}?connector_id=${encodeURIComponent(connectorId)}&connector_instance_id=${encodeURIComponent(sourceConnectionId)}`, + { + body: JSON.stringify({ data, emitted_at: emittedAt, key: recordKey }), + headers: { Authorization: `Bearer ${ownerToken}`, "Content-Type": "application/x-ndjson" }, + method: "POST", + } + ); +} + +interface EnrolledLocalDevice { + connector_id: string; + device_id: string; + device_token: string; + source_instance_id: string; +} + +async function enrollLocalDevice( + asUrl: string, + session: OwnerSession, + client: LocalDeviceClient +): Promise { + const code = await fetchJson(`${asUrl}/_ref/device-exporters/enrollment-codes`, { + body: JSON.stringify({ connector_id: "codex", local_binding_name: "closure-codex-laptop" }), + headers: { "Content-Type": "application/json", Cookie: session.cookie }, + method: "POST", + }); + assert.equal(code.status, 201, code.text); + const enrolled = await client.exchangeEnrollment({ + device_label: "closure fixture device", + enrollment_code: stringField(code.body, "enrollment_code"), + }); + return enrolled as EnrolledLocalDevice; +} + +async function ownerConnections(rsUrl: string, ownerToken: string): Promise { + const listed = await fetchJson(`${rsUrl}/v1/owner/connections`, { + headers: { Authorization: `Bearer ${ownerToken}` }, + }); + assert.equal(listed.status, 200, listed.text); + const { data } = asRecord(listed.body, "owner connections response"); + assert.ok(Array.isArray(data), "owner connections response must contain data[]"); + return data as JsonRecord[]; +} + +function connectionIdFromRow(row: JsonRecord): string { + return String(row.connection_id ?? row.connector_instance_id ?? ""); +} + +function assertClosureEvidence(evidence: ClosureEvidence, expectedMcpId: string): void { + assert.deepEqual( + Object.values(evidence.refVisibleStatuses).sort(), + ["active", "active", "active"], + "all three accepted sources must be visible and active before revoke" + ); + assert.deepEqual( + evidence.mcpRecordIds, + [expectedMcpId], + "scoped MCP must read exactly the newly accepted fixture record" + ); + assert.equal(evidence.sourceRevokeStatus, "revoked"); + assert.ok(evidence.stoppedIngestStatus >= 400, "source revoke must reject a new collection attempt"); + assert.equal( + evidence.acceptedRecordReadableBeforeRevoke, + true, + "the accepted record must be readable before source revoke" + ); + assert.equal(evidence.grantActiveAfterSourceRevoke, true, "source revoke must not revoke the app grant"); + assert.equal(evidence.grantActiveAfterGrantRevoke, false, "grant revoke must deactivate the app token"); + assert.notEqual(evidence.mcpStatusAfterGrantRevoke, 200, "grant revoke must stop the MCP token"); +} + +function mcpRecordIds(responseBody: unknown): string[] { + const result = objectField(responseBody, "result"); + const structured = objectField(result, "structuredContent"); + const { data } = structured; + if (Array.isArray(data)) { + return data.map((candidate) => stringField(candidate, "id")); + } + const dataRecord = data && typeof data === "object" && !Array.isArray(data) ? (data as JsonRecord) : {}; + for (const key of ["records", "items", "data"]) { + const candidates = dataRecord[key]; + if (Array.isArray(candidates)) { + return candidates.map((candidate) => stringField(candidate, "id")); + } + } + return []; +} + +async function registerAuthCodeClient(asUrl: string, session: OwnerSession): Promise { + const registered = await fetchJson(`${asUrl}/oauth/register`, { + body: JSON.stringify({ + application_type: "web", + client_name: "owner closure fixture client", + grant_types: ["authorization_code", "refresh_token"], + redirect_uris: ["https://client.example/callback"], + response_types: ["code"], + token_endpoint_auth_method: "none", + }), + headers: { "Content-Type": "application/json", Cookie: session.cookie }, + method: "POST", + }); + assert.equal(registered.status, 201, registered.text); + return stringField(registered.body, "client_id"); +} + +function pkceChallenge(verifier: string): string { + return createHash("sha256").update(verifier).digest("base64url"); +} + +async function completeScopedMcpFlow( + asUrl: string, + session: OwnerSession, + connectorId: string, + sourceConnectionId: string +): Promise<{ accessToken: string; grantId: string }> { + const clientId = await registerAuthCodeClient(asUrl, session); + const verifier = randomBytes(32).toString("base64url"); + const authorizationDetails = [ + { + access_mode: "continuous", + purpose_code: "https://pdpp.org/purpose/personal_ai_assistant", + purpose_description: "Read the fixture mailbox through hosted MCP.", + source: { id: connectorId, kind: "connector" }, + streams: [{ connection_id: sourceConnectionId, fields: ["id", "subject"], name: "messages" }], + type: "https://pdpp.org/data-access", + }, + ]; + const authorizeUrl = new URL(`${asUrl}/oauth/authorize`); + authorizeUrl.searchParams.set("authorization_details", JSON.stringify(authorizationDetails)); + authorizeUrl.searchParams.set("client_id", clientId); + authorizeUrl.searchParams.set("code_challenge", pkceChallenge(verifier)); + authorizeUrl.searchParams.set("code_challenge_method", "S256"); + authorizeUrl.searchParams.set("redirect_uri", "https://client.example/callback"); + authorizeUrl.searchParams.set("response_type", "code"); + authorizeUrl.searchParams.set("state", "owner-closure-state"); + + const authorized = await fetch(authorizeUrl, { + headers: { Cookie: session.cookie }, + redirect: "manual", + }); + assert.equal(authorized.status, 302); + const consent = new URL(String(authorized.headers.get("location")), asUrl); + const requestUri = consent.searchParams.get("request_uri"); + assert.ok(requestUri, "OAuth authorize must redirect to consent"); + + const approved = await fetch(`${asUrl}/consent/approve`, { + body: new URLSearchParams({ + _csrf: session.csrfField, + request_uri: requestUri, + subject_id: OWNER_SUBJECT_ID, + }).toString(), + headers: { "Content-Type": "application/x-www-form-urlencoded", Cookie: session.cookie }, + method: "POST", + redirect: "manual", + }); + assert.equal(approved.status, 302); + const callback = new URL(String(approved.headers.get("location")), asUrl); + const code = callback.searchParams.get("code"); + assert.ok(code, "consent approval must return an authorization code"); + + const token = await fetchJson(`${asUrl}/oauth/token`, { + body: new URLSearchParams({ + client_id: clientId, + code, + code_verifier: verifier, + grant_type: "authorization_code", + redirect_uri: "https://client.example/callback", + }).toString(), + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + method: "POST", + }); + assert.equal(token.status, 200, token.text); + return { accessToken: stringField(token.body, "access_token"), grantId: stringField(token.body, "grant_id") }; +} + +function postMcp(rsUrl: string, accessToken: string, id: number, method: string, params: JsonRecord) { + return fetchJson(`${rsUrl}/mcp`, { + body: JSON.stringify({ id, jsonrpc: "2.0", method, params }), + headers: { + Accept: "application/json, text/event-stream", + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + method: "POST", + }); +} + +function introspectionActive(body: unknown): boolean { + return asRecord(body, "introspection response").active === true; +} + +test("owner-source-to-mcp-closure", async () => { + await withCredentialKey(async () => { + const server = await startClosureServer(); + const asUrl = `http://localhost:${server.asPort}`; + const rsUrl = `http://localhost:${server.rsPort}`; + + try { + // 1. Owner sign-in is real: all subsequent setup and source visibility + // calls use the session produced by /owner/login. + const session = await login(asUrl); + const gmail = await registerConnector(asUrl, session, "gmail"); + await registerConnector(asUrl, session, "google_maps"); + const codex = await registerConnector(asUrl, session, "codex"); + const ownerToken = await issueOwnerToken(asUrl, session); + + // 2. Static-secret journey: draft -> credential capture -> accepted + // parser-derived fixture record. The injected prober is deterministic + // and cannot contact Gmail. + const gmailConnectionId = await createStaticDraft(asUrl, session); + await captureStaticCredential(asUrl, session, gmailConnectionId); + const gmailFixture = fixtureRecord(GMAIL_FIXTURE_PATH); + const gmailIngest = await ingestNdjson( + rsUrl, + ownerToken, + gmail.connector_id, + gmailConnectionId, + "messages", + gmailFixture + ); + assert.equal(gmailIngest.status, 200, gmailIngest.text); + + // 3. Manual/upload journey: Timeline.json validation -> accepted point + // record. The body is the existing synthetic Timeline fixture shape. + const mapsConnectionId = await createManualDraft(asUrl, session); + const mapsFixture = { + id: "closure-timeline-point-1", + latitude: 37.7749, + longitude: -122.4194, + source_format: "legacy_records", + source_kind: "raw_location", + timestamp: "2024-06-05T13:45:22.000Z", + }; + const mapsIngest = await ingestNdjson( + rsUrl, + ownerToken, + "google-maps", + mapsConnectionId, + "timeline_points", + mapsFixture, + "2024-06-05T13:45:22.000Z" + ); + assert.equal(mapsIngest.status, 200, mapsIngest.text); + + // 4. Local-device journey: owner creates an enrollment code, the + // shipped LocalDeviceClient exchanges it, and the shipped durable + // envelope helper sends a scrubbed Codex fixture record. + const enrollmentClient = new LocalDeviceClient({ baseUrl: asUrl, requestTimeoutMs: 5000 }); + const localDevice = await enrollLocalDevice(asUrl, session, enrollmentClient); + const localClient = new LocalDeviceClient({ + baseUrl: asUrl, + deviceId: localDevice.device_id, + deviceToken: localDevice.device_token, + requestTimeoutMs: 5000, + }); + const codexFixture = fixtureRecord(CODEX_FIXTURE_PATH); + const localEnvelope = buildLocalDeviceRecordEnvelope({ + batchId: "closure-codex-batch-1", + batchSeq: 1, + connectorId: localDevice.connector_id, + deviceId: localDevice.device_id, + record: { + data: codexFixture, + emitted_at: FIXTURE_TIME, + key: stringField(codexFixture, "id"), + stream: "messages", + type: "RECORD", + }, + sourceInstanceId: localDevice.source_instance_id, + }); + const localAccepted = await localClient.ingestBatch( + buildLocalDeviceIngestBatchRequest({ + batchId: "closure-codex-batch-1", + batchSeq: 1, + connectorId: localDevice.connector_id, + deviceId: localDevice.device_id, + records: [localEnvelope], + sourceInstanceId: localDevice.source_instance_id, + }) + ); + assert.ok(localAccepted, "local-device ingest must return an acceptance response"); + + // 5. Source visibility is checked through both owner surfaces after + // accepted records have activated every source. + const refVisible = await fetchJson(`${asUrl}/_ref/connectors?limit=100`, { + headers: { Cookie: session.cookie }, + }); + assert.equal(refVisible.status, 200, refVisible.text); + const refData = asRecord(refVisible.body, "ref connector list").data; + assert.ok(Array.isArray(refData), "ref connector list must contain data[]"); + const refRows = refData as JsonRecord[]; + const refVisibleStatuses: Record = {}; + for (const id of [gmailConnectionId, mapsConnectionId]) { + const row = refRows.find((candidate) => connectionIdFromRow(candidate) === id); + assert.ok(row, `owner-session source list must show ${id}`); + refVisibleStatuses[id] = String(row.status); + } + const localOwnerRows = await ownerConnections(rsUrl, ownerToken); + const localRow = localOwnerRows.find( + (candidate) => candidate.connector_key === codex.connector_id && candidate.source_kind === "local_device" + ); + assert.ok(localRow, "owner-agent source list must show the enrolled Codex local source"); + const localConnectionId = connectionIdFromRow(localRow); + assert.ok(localConnectionId, "local source list row must expose connection_id"); + const localFixtureRead = await fetchJson( + `${rsUrl}/v1/streams/messages/records?connector_id=${encodeURIComponent(codex.connector_id)}&connection_id=${encodeURIComponent(localConnectionId)}`, + { headers: { Authorization: `Bearer ${ownerToken}` } } + ); + assert.equal(localFixtureRead.status, 200, localFixtureRead.text); + assert.ok( + localFixtureRead.text.includes(stringField(codexFixture, "id")), + "local fixture record must be publicly readable" + ); + refVisibleStatuses[localConnectionId] = String(localRow.status); + assert.equal(localConnectionId.length > 0, true); + assert.equal(Object.keys(refVisibleStatuses).length, 3); + + // 6–7. The app receives only the Gmail/messages connection and stream; + // read the accepted fixture through the live scoped MCP route. + const oauth = await completeScopedMcpFlow(asUrl, session, gmail.connector_id, gmailConnectionId); + const initialized = await postMcp(rsUrl, oauth.accessToken, 1, "initialize", { + capabilities: {}, + clientInfo: { name: "owner-closure-fixture-client", version: "0.0.0" }, + protocolVersion: "2025-06-18", + }); + assert.equal(initialized.status, 200, initialized.text); + const queried = await postMcp(rsUrl, oauth.accessToken, 2, "tools/call", { + arguments: { connection_id: gmailConnectionId, limit: 10, stream: "messages" }, + name: "query_records", + }); + assert.equal(queried.status, 200, queried.text); + assert.equal(objectField(queried.body, "result").isError, undefined); + const gmailFixtureId = stringField(gmailFixture, "id"); + + // 8. Revoke the source through the public owner-agent route. A new + // collection attempt is rejected; the accepted record was already + // proven readable through the scoped MCP path above. + const sourceRevoke = await fetchJson( + `${rsUrl}/v1/owner/connections/${encodeURIComponent(gmailConnectionId)}/revoke`, + { + headers: { Authorization: `Bearer ${ownerToken}`, "Content-Type": "application/json" }, + method: "POST", + } + ); + assert.equal(sourceRevoke.status, 200, sourceRevoke.text); + const sourceRevokeStatus = stringField(sourceRevoke.body, "status"); + const stoppedIngest = await ingestNdjson(rsUrl, ownerToken, gmail.connector_id, gmailConnectionId, "messages", { + ...gmailFixture, + id: `${gmailFixtureId}-after-revoke`, + }); + assert.ok(stoppedIngest.status >= 400, stoppedIngest.text); + // Source revoke leaves the app grant active. Grant revoke then stops + // the same MCP token independently. + const afterSourceRevoke = await fetchJson(`${asUrl}/introspect`, { + body: new URLSearchParams({ token: oauth.accessToken }).toString(), + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + method: "POST", + }); + assert.equal(afterSourceRevoke.status, 200, afterSourceRevoke.text); + const grantActiveAfterSourceRevoke = introspectionActive(afterSourceRevoke.body); + + const grantRevoke = await fetchJson(`${asUrl}/grants/${encodeURIComponent(oauth.grantId)}/revoke`, { + headers: { Authorization: `Bearer ${ownerToken}`, "Content-Type": "application/json" }, + method: "POST", + }); + assert.equal(grantRevoke.status, 200, grantRevoke.text); + const afterGrantRevoke = await fetchJson(`${asUrl}/introspect`, { + body: new URLSearchParams({ token: oauth.accessToken }).toString(), + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + method: "POST", + }); + assert.equal(afterGrantRevoke.status, 200, afterGrantRevoke.text); + const stoppedMcp = await postMcp(rsUrl, oauth.accessToken, 3, "tools/list", {}); + + const afterOwnerRows = await ownerConnections(rsUrl, ownerToken); + const revokedRow = afterOwnerRows.find((candidate) => connectionIdFromRow(candidate) === gmailConnectionId); + assert.equal(revokedRow?.status, "revoked"); + + const evidence: ClosureEvidence = { + acceptedRecordReadableBeforeRevoke: mcpRecordIds(queried.body).includes(gmailFixtureId), + grantActiveAfterGrantRevoke: introspectionActive(afterGrantRevoke.body), + grantActiveAfterSourceRevoke, + mcpRecordIds: mcpRecordIds(queried.body), + mcpStatusAfterGrantRevoke: stoppedMcp.status, + refVisibleStatuses, + sourceRevokeStatus, + stoppedIngestStatus: stoppedIngest.status, + }; + assertClosureEvidence(evidence, gmailFixtureId); + + // Controlled mutation: the oracle must fail when the public read loses + // the accepted fixture, proving this is a discriminator rather than a + // journey-only smoke test. + assert.throws( + () => assertClosureEvidence({ ...evidence, mcpRecordIds: [] }, gmailFixtureId), + CLOSURE_MCP_MISSING_RE + ); + } finally { + await closeServer(server); + } + }); +}); diff --git a/reference-implementation/test/owner-state.test.ts b/reference-implementation/test/owner-state.test.ts index 960838187..b5bc09d83 100644 --- a/reference-implementation/test/owner-state.test.ts +++ b/reference-implementation/test/owner-state.test.ts @@ -371,6 +371,61 @@ test("gate: no lifecycle evidence never resolves setup_in_progress, even for a n assert.notEqual(state.resolver, "setup_in_progress"); }); +// ─── gate: Chase-shaped draft mid-first-sync (fr-setup-status-lifecycle-0806) +// +// Discriminates the exact UAT sequence: connection cin_c2f766b7166a6184adf021aa +// emits `run.started`, then ~1 minute later `run.interaction_required` +// (kind=otp). Before the interaction event lands, the Sources UI must render +// connecting/working — never "needs you" — even though the connection is +// still a draft. Once the interaction lands, it must render the exact owner +// action, not the generic draft "Finish connecting this source" copy. + +test("gate: draft + active run + no open attention resolves collecting (run.started-only window), never setup_in_progress", () => { + const snap = snapshot({ last_success_at: null, state: "idle" }); + const { state } = ownerStateFor(snap, [], { + active: true, + lifecycle: { status: "draft" }, + schedule: null, + source: "active_progress", + }); + assert.equal(state.resolver, "collecting"); + assert.notEqual(state.resolver, "setup_in_progress"); + assert.equal(state.owner_of_state, "system"); +}); + +test("gate: draft + active run + open OTP attention resolves needs_owner with the exact action, not the generic draft copy", () => { + const snap = snapshot({ + axes: { attention: "open" }, + last_success_at: null, + state: "needs_attention", + }); + const { state, verdict } = ownerStateFor(snap, [], { + active: true, + lifecycle: { status: "draft" }, + schedule: null, + source: "active_progress", + }); + assert.equal(state.resolver, "needs_owner"); + assert.notEqual(state.resolver, "setup_in_progress"); + assert.equal(state.owner_of_state, "owner"); + // biome-ignore lint/style/useDestructuring: localized test assertion preserves its explicit contract. + const primaryAction = verdict.required_actions[0]; + assert.ok(primaryAction, "expected a primary required action"); + assert.equal(primaryAction.kind, "add_info"); + assert.equal(primaryAction.surface?.kind, "provider_interaction"); +}); + +test("gate: draft + no active run + no attention still resolves the generic setup_in_progress (nothing else to say yet)", () => { + const snap = snapshot({ last_success_at: null, state: "idle" }); + const { state } = ownerStateFor(snap, [], { + active: false, + lifecycle: { status: "draft" }, + schedule: null, + source: "none", + }); + assert.equal(state.resolver, "setup_in_progress"); +}); + test("gate: never-run (idle, no prior success) resolves to healthy, system-owned, observed", () => { const snap = snapshot({ last_success_at: null, state: "idle" }); const { state } = ownerStateFor(snap, [], { schedule: scheduleRow(), source: "last_successful_freshness" }); @@ -554,8 +609,18 @@ test("exhaustive cross-product: every (state, coverage, schedule, last_success, // biome-ignore lint/suspicious/noEvolvingTypes: localized test assertion preserves its explicit contract. // biome-ignore lint/suspicious/noImplicitAnyLet: localized test assertion preserves its explicit contract. let state2; + // biome-ignore lint/suspicious/noEvolvingTypes: localized test assertion preserves its explicit contract. + // biome-ignore lint/suspicious/noImplicitAnyLet: localized test assertion preserves its explicit contract. + let verdict1; try { - ({ state: state1 } = crossProductCase(state, coverage, schedule, lastSuccessAt, lifecycle, active)); + ({ state: state1, verdict: verdict1 } = crossProductCase( + state, + coverage, + schedule, + lastSuccessAt, + lifecycle, + active + )); ({ state: state2 } = crossProductCase(state, coverage, schedule, lastSuccessAt, lifecycle, active)); } catch { // A small number of state/coverage combinations violate the @@ -590,12 +655,46 @@ test("exhaustive cross-product: every (state, coverage, schedule, last_success, assert.equal(state1.owner_of_state, "owner"); assert.equal(state1.posture, "observed"); } + // A draft with an active run and no open owner-attention action + // must read `collecting`, not the generic `setup_in_progress` + // copy (fr-setup-status-lifecycle-0806: a Chase draft between + // `run.started` and `run.interaction_required` must render + // connecting/working, never needs-you). A draft with an open + // owner-attention action must read `needs_owner` with the + // exact requested action, never the generic draft copy. Only a + // draft with neither resolves the generic `setup_in_progress`. if (lifecycle?.status === "draft") { - assert.equal( - state1.resolver, - "setup_in_progress", - "draft lifecycle must always resolve setup_in_progress" - ); + const primary1 = verdict1.required_actions[0] ?? null; + const hasOwnerAttention = verdict1.channel === "attention" && primary1?.audience === "owner"; + const hasMaintainerAction = primary1?.audience === "maintainer"; + // `owner_paused` requires a real disabled schedule row plus a + // prior success — not a realistic draft shape, but reachable + // in this synthetic matrix, and it legitimately outranks a + // bare `progress.active` per the existing "paused must not + // mask an active/urgent state" precedence; exclude it here + // rather than assert a resolver this module never claimed. + const ownerPausedEligible = schedule?.enabled === false && lastSuccessAt !== null; + if (hasMaintainerAction) { + assert.equal( + state1.resolver, + "blocked_maintainer", + "draft with a maintainer-audience primary action must resolve blocked_maintainer" + ); + } else if (!(active || ownerPausedEligible)) { + assert.equal( + state1.resolver, + "setup_in_progress", + "idle draft must resolve setup_in_progress before generic owner attention" + ); + } else if (hasOwnerAttention) { + assert.equal( + state1.resolver, + "needs_owner", + "draft with open owner attention must resolve needs_owner" + ); + } else if (active && !ownerPausedEligible) { + assert.equal(state1.resolver, "collecting", "draft with an active run must resolve collecting"); + } } // Design gate #3: owner_paused/refresh-schedule requires a real // schedule row that is disabled — never an absent (manual) schedule. diff --git a/reference-implementation/test/records-ingest-batch-coordination.test.ts b/reference-implementation/test/records-ingest-batch-coordination.test.ts new file mode 100644 index 000000000..93d2c709b --- /dev/null +++ b/reference-implementation/test/records-ingest-batch-coordination.test.ts @@ -0,0 +1,198 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Focused throughput oracle for the common ingest capability. + * + * The fake Postgres lock pool is enabled while storage remains SQLite. That + * isolates coordinator lifecycle calls from record-storage work: the old + * one-record path performs one advisory acquire and release per record, while + * `ingestRecords` must perform exactly one pair for the whole batch. The two + * stream shapes also prove the optimization is not connector-specific. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + __setConnectorInstancePostgresLockPoolForTest, + withConnectorInstanceWrite, +} from "../server/connector-instance-write-coordinator.ts"; +import { closeDb, getDb, initDb } from "../server/db.ts"; +import { + __setIngestFaultHookForTest, + ingestRecord, + ingestRecords, + recordIndexWorkStatsForTests, + withRecordIndexWorkForTests, +} from "../server/records.ts"; + +interface TestPostgresLockClient { + query: ( + sql: string, + params: readonly unknown[] + ) => Promise<{ rows: Array<{ acquired?: boolean; unlocked?: boolean }> }>; + release: (error?: boolean) => void; +} + +function records(stream: string, prefix: string) { + return Array.from({ length: 3 }, (_, index) => ({ + data: { id: `${prefix}-${index}`, text: `${stream}-${index}` }, + emitted_at: "2026-08-06T00:00:00.000Z", + key: `${prefix}-${index}`, + stream, + })); +} + +function countChanges(connectorId: string, stream: string): number { + const row = getDb() + .prepare( + `SELECT COUNT(*) AS count + FROM record_changes + WHERE connector_id = ? AND stream = ?` + ) + .get(connectorId, stream) as { count: number }; + return row.count; +} + +function changeRows(connectorId: string, stream: string): Array<{ record_key: string; version: number }> { + return getDb() + .prepare( + `SELECT record_key, version + FROM record_changes + WHERE connector_id = ? AND stream = ? + ORDER BY version` + ) + .all(connectorId, stream) as Array<{ record_key: string; version: number }>; +} + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve: (() => void) | undefined; + const promise = new Promise((done) => { + resolve = done; + }); + assert.ok(resolve, "Promise executor runs synchronously, so resolve is always assigned here"); + return { promise, resolve }; +} + +async function waitFor(predicate: () => boolean, timeoutMs = 1000): Promise { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + if (Date.now() >= deadline) { + throw new Error("timed out waiting for the deterministic test condition"); + } + // biome-ignore lint/performance/noAwaitInLoops: Polling is intentionally sequential for a deterministic gate. + await new Promise((resolve) => setTimeout(resolve, 1)); + } +} + +test("common ingest reuses coordinator ownership for messages and timeline_points", async () => { + initDb(); + let lockQueries = 0; + let clientReleases = 0; + const client: TestPostgresLockClient = { + query: (sql) => { + lockQueries += 1; + return Promise.resolve(sql.includes("unlock") ? { rows: [{ unlocked: true }] } : { rows: [{ acquired: true }] }); + }, + release: () => { + clientReleases += 1; + }, + }; + __setConnectorInstancePostgresLockPoolForTest({ + capacity: 8, + pool: { connect: async () => client }, + }); + + try { + const beforeConnector = "https://probe.example/connectors/before"; + for (const record of records("messages", "before")) { + // biome-ignore lint/performance/noAwaitInLoops: This is the serial baseline the oracle intentionally measures. + await ingestRecord(beforeConnector, record); + } + assert.equal(lockQueries, 6, "the pre-fix single-record path acquires and releases per record"); + assert.equal(countChanges(beforeConnector, "messages"), 3); + + const afterConnector = "https://probe.example/connectors/after"; + const messages = await ingestRecords(afterConnector, records("messages", "message")); + const timelinePoints = await ingestRecords(afterConnector, records("timeline_points", "point")); + assert.equal(messages.filter((outcome) => outcome.accepted).length, 3); + assert.equal(timelinePoints.filter((outcome) => outcome.accepted).length, 3); + assert.equal(lockQueries, 10, "two batches should add one acquire/release pair each"); + assert.equal(clientReleases, 5); + assert.equal(countChanges(afterConnector, "messages"), 3); + assert.equal(countChanges(afterConnector, "timeline_points"), 3); + + const failureConnector = "https://probe.example/connectors/failure-isolation"; + __setIngestFaultHookForTest((point: string, context: { recordKey?: string }) => { + if (point === "after-records-mutation" && context.recordKey === "fault-1") { + throw new Error("injected batch fault"); + } + }); + const failureOutcomes = await ingestRecords(failureConnector, records("timeline_points", "fault")); + assert.deepEqual( + failureOutcomes.map((outcome) => ({ accepted: outcome.accepted, error: outcome.error ?? null })), + [ + { accepted: true, error: null }, + { accepted: false, error: "injected batch fault" }, + { accepted: true, error: null }, + ] + ); + assert.deepEqual(changeRows(failureConnector, "timeline_points"), [ + { record_key: "fault-0", version: 1 }, + { record_key: "fault-2", version: 2 }, + ]); + } finally { + __setIngestFaultHookForTest(null); + __setConnectorInstancePostgresLockPoolForTest(null); + closeDb(); + } +}); + +test("batch releases the instance fence while its derived index lane is saturated", async () => { + const previousLimit = process.env.PDPP_INGEST_INDEX_WORK_LIMIT; + process.env.PDPP_INGEST_INDEX_WORK_LIMIT = "1"; + initDb(); + + const indexEntered = deferred(); + const indexRelease = deferred(); + const heldIndexPermit = withRecordIndexWorkForTests(async () => { + indexEntered.resolve(); + await indexRelease.promise; + }); + let batch: Promise>> | undefined; + let blobWriter: Promise | undefined; + try { + await indexEntered.promise; + assert.deepEqual(recordIndexWorkStatsForTests(), { active: 1, queued: 0 }); + + const connectorInstanceId = "cin_batch_blob_liveness"; + const target = { + connector_id: "https://probe.example/connectors/gmail", + connector_instance_id: connectorInstanceId, + }; + batch = ingestRecords(target, records("messages", "liveness")); + await waitFor(() => recordIndexWorkStatsForTests().queued === 1); + + blobWriter = withConnectorInstanceWrite(connectorInstanceId, async () => "blob-writer"); + const admission = await Promise.race([ + blobWriter.then(() => "completed" as const), + new Promise<"timed_out">((resolve) => setTimeout(() => resolve("timed_out"), 250)), + ]); + assert.equal(admission, "completed", "a blob-style writer must not wait behind deferred index work"); + + indexRelease.resolve(); + const outcomes = await batch; + assert.equal(outcomes.filter((outcome) => outcome.accepted).length, 3); + await blobWriter; + } finally { + indexRelease.resolve(); + await Promise.allSettled([heldIndexPermit, ...(batch ? [batch] : []), ...(blobWriter ? [blobWriter] : [])]); + closeDb(); + if (previousLimit === undefined) { + delete process.env.PDPP_INGEST_INDEX_WORK_LIMIT; + } else { + process.env.PDPP_INGEST_INDEX_WORK_LIMIT = previousLimit; + } + } +}); diff --git a/reference-implementation/test/ref-connectors-run-force-route.test.ts b/reference-implementation/test/ref-connectors-run-force-route.test.ts index 714f84706..db85a9910 100644 --- a/reference-implementation/test/ref-connectors-run-force-route.test.ts +++ b/reference-implementation/test/ref-connectors-run-force-route.test.ts @@ -2,12 +2,14 @@ const TOP_LEVEL_REGEX_1 = /run resources must include at least one resource id p const TOP_LEVEL_REGEX_2 = /run resources must map stream names to string arrays/; const TOP_LEVEL_REGEX_3 = /run resources must include at least one resource id per stream/; const TOP_LEVEL_REGEX_4 = /run resources must map stream names to string arrays/; +const DRAFT_NOT_ADMITTED = /draft connection not admitted/; // Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 import assert from "node:assert/strict"; import test from "node:test"; +import type { RunAdmission } from "../runtime/controller.ts"; import type { MountOwnerConnectionRunContext } from "../server/routes/owner-connection-run.ts"; import { mountOwnerConnectionRun } from "../server/routes/owner-connection-run.ts"; import type { MountRefConnectorsContext } from "../server/routes/ref-connectors.ts"; @@ -60,7 +62,7 @@ interface RunNowCall { readonly options: { connectorInstanceId?: string | null; force?: boolean; - runAdmission?: "browser_enrollment"; + runAdmission?: RunAdmission; resources?: Readonly>; }; } @@ -72,7 +74,7 @@ interface ResolveNamespaceCall { type MountRefRun = typeof mountRefConnectionRun | typeof mountRefConnectorRun; -function buildHarness(mount: MountRefRun) { +function buildHarness(mount: MountRefRun, harnessOptions: { draftConnectionId?: string } = {}) { const calls: { emitSpineEvent: SpineEvent[]; runNow: RunNowCall[]; @@ -124,6 +126,13 @@ function buildHarness(mount: MountRefRun) { requireOwnerSession: (_req, _res, next) => (typeof next === "function" ? next() : undefined), resolveOwnerConnectorNamespace(_req, connectorId, options = {}) { calls.resolveOwnerConnectorNamespace.push({ connectorId, options }); + if ( + harnessOptions.draftConnectionId !== undefined && + harnessOptions.draftConnectionId === options.connectorInstanceId && + !options.allowStatuses?.includes("draft") + ) { + throw new Error("draft connection not admitted"); + } return Promise.resolve({ connectorId: connectorId ?? "chatgpt", connectorInstanceId: options.connectorInstanceId ?? "cin_chatgpt", @@ -318,6 +327,35 @@ function buildOwnerHarness() { }; } +test("POST /_ref/connections/:id/run keeps omitted and empty bodies active-only", async () => { + await Promise.all( + [null, {}].map(async (body) => { + const harness = buildHarness(mountRefConnectionRun, { draftConnectionId: "cin_draft" }); + + await assert.rejects( + () => + harness.invoke({ + body, + params: { connectorInstanceId: "cin_draft" }, + }), + DRAFT_NOT_ADMITTED + ); + assert.deepEqual(harness.calls.resolveOwnerConnectorNamespace, [ + { + connectorId: null, + options: { + allowDefaultAccount: false, + allowStatuses: ["active"], + connectorInstanceId: "cin_draft", + ownerSubjectId: "owner_local", + }, + }, + ]); + assert.deepEqual(harness.calls.runNow, []); + }) + ); +}); + test("POST /_ref/connections/:id/run forwards explicit force override to the controller", async () => { const harness = buildHarness(mountRefConnectionRun); @@ -330,7 +368,11 @@ test("POST /_ref/connections/:id/run forwards explicit force override to the con assert.deepEqual(harness.calls.runNow, [ { connectorId: "chatgpt", - options: { connectorInstanceId: "cin_chatgpt", force: true, ownerSubjectId: "owner_local" }, + options: { + connectorInstanceId: "cin_chatgpt", + force: true, + ownerSubjectId: "owner_local", + }, }, ]); const [firstEvent] = harness.calls.emitSpineEvent; @@ -445,6 +487,38 @@ test("POST /_ref/connections/:id/run forwards scoped stream resources", async () ]); }); +test("POST /_ref/connections/:id/run accepts explicit setup admission", async () => { + const harness = buildHarness(mountRefConnectionRun, { draftConnectionId: "cin_draft" }); + const res = await harness.invoke({ + body: { run_admission: "setup" }, + params: { connectorInstanceId: "cin_draft" }, + }); + + assert.equal(res.statusCode, 202); + assert.deepEqual(harness.calls.resolveOwnerConnectorNamespace, [ + { + connectorId: null, + options: { + allowDefaultAccount: false, + allowStatuses: ["active", "draft"], + connectorInstanceId: "cin_draft", + ownerSubjectId: "owner_local", + }, + }, + ]); + assert.deepEqual(harness.calls.runNow, [ + { + connectorId: "chatgpt", + options: { + connectorInstanceId: "cin_draft", + force: false, + ownerSubjectId: "owner_local", + runAdmission: "setup", + }, + }, + ]); +}); + test("POST /_ref/connections/:id/run rejects prototype-polluting resource keys", async () => { const harness = buildHarness(mountRefConnectionRun); const body = JSON.parse('{"resources":{"__proto__":["C07JYF0U8BY"]}}') as Record; diff --git a/reference-implementation/test/ref-error-status.test.ts b/reference-implementation/test/ref-error-status.test.ts index 62aacf8c1..51615113d 100644 --- a/reference-implementation/test/ref-error-status.test.ts +++ b/reference-implementation/test/ref-error-status.test.ts @@ -56,6 +56,7 @@ test("codeToStatus pins grant/auth and connector-instance code statuses", () => ]) { assert.equal(codeToStatus[code], 403, `${code} must be 403`); } + assert.equal(codeToStatus.connector_instance_busy, 503); assert.equal(codeToStatus.authentication_error, 401); assert.equal(codeToStatus.connector_instance_store_required, 500); assert.equal(codeToStatus.run_already_active, 409); diff --git a/reference-implementation/test/rs-explore-upcoming-concurrency.test.ts b/reference-implementation/test/rs-explore-upcoming-concurrency.test.ts index b46f47177..a068248b0 100644 --- a/reference-implementation/test/rs-explore-upcoming-concurrency.test.ts +++ b/reference-implementation/test/rs-explore-upcoming-concurrency.test.ts @@ -73,12 +73,10 @@ test("postgresFetchUpcoming concurrency constant equals 4 and mapWithConcurrency ); }); -test("postgresFetchUpcoming: live Postgres in-flight partition workers never exceed the configured limit", async (t) => { - if (!POSTGRES_URL) { - t.skip("Skipped because PDPP_TEST_POSTGRES_URL is unset"); - return; - } - +test("postgresFetchUpcoming: live Postgres in-flight partition workers never exceed the configured limit", { + skip: !POSTGRES_URL, +}, async () => { + assert.ok(POSTGRES_URL); initDb(":memory:"); await initPostgresStorage({ backend: "postgres", databaseUrl: POSTGRES_URL }); @@ -116,7 +114,10 @@ test("postgresFetchUpcoming: live Postgres in-flight partition workers never exc } }); -test("sqliteFetchUpcoming & postgresFetchUpcoming: output is bit-identical and deterministic", async (t) => { +test("sqliteFetchUpcoming & postgresFetchUpcoming: output is bit-identical and deterministic", { + skip: !POSTGRES_URL, +}, async () => { + assert.ok(POSTGRES_URL); initDb(":memory:"); try { @@ -146,11 +147,7 @@ test("sqliteFetchUpcoming & postgresFetchUpcoming: output is bit-identical and d assert.ok(sqliteResult.total > 0, "Should have counted future records"); assert.equal(sqliteResult.rows.length, Math.min(50, sqliteResult.total)); - // Test Postgres backend if environment variable is available - if (!POSTGRES_URL) { - t.skip("Postgres parity check skipped because PDPP_TEST_POSTGRES_URL is unset"); - return; - } + // Compare against Postgres when the declaration-time availability gate admits this test. await initPostgresStorage({ backend: "postgres", databaseUrl: POSTGRES_URL }); try { diff --git a/reference-implementation/test/rs-records-ingest-operation.test.ts b/reference-implementation/test/rs-records-ingest-operation.test.ts index 9e4b3fc80..483283494 100644 --- a/reference-implementation/test/rs-records-ingest-operation.test.ts +++ b/reference-implementation/test/rs-records-ingest-operation.test.ts @@ -27,6 +27,7 @@ import { } from "../operations/rs-records-ingest/index.ts"; const REGEXP_1 = /store down/; +const DERIVED_INDEX_ERROR_REGEXP = /derived index failed/; function defaultDeps(overrides: Partial = {}): RecordsIngestDependencies { return { @@ -95,6 +96,44 @@ test("rs.records.ingest invokes ingestRecord sequentially in line order", async assert.deepEqual(seen, ["r1", "r2", "r3"]); }); +test("rs.records.ingest uses the bounded batch capability for distinct stream shapes", async () => { + const results = await Promise.all( + ["messages", "timeline_points"].map(async (streamName) => { + let batchCalls = 0; + let received: Record[] = []; + const out = await executeRecordsIngest( + defaultInput({ + body: '{"id":"r1"}\nNOT_JSON\n{"id":"r3"}', + streamName, + }), + defaultDeps({ + ingestRecord: () => { + throw new Error("ordered fallback should not run when batch capability is present"); + }, + ingestRecords: (_connectorId, _connectorInstanceId, records) => { + batchCalls += 1; + received = [...records]; + return [null, "derived index failed"]; + }, + }) + ); + + return { batchCalls, out, received, streamName }; + }) + ); + for (const { batchCalls, out, received, streamName } of results) { + assert.equal(batchCalls, 1, `${streamName} should use one common-path batch call`); + assert.deepEqual( + received.map((record) => record.stream), + [streamName, streamName] + ); + assert.equal(out.envelope.records_accepted, 1); + assert.equal(out.envelope.records_rejected, 2); + assert.equal(out.envelope.errors.length, 2); + assert.match(out.envelope.errors[1] ?? "", DERIVED_INDEX_ERROR_REGEXP); + } +}); + test("rs.records.ingest forwards { ...record, stream } to the dependency", async () => { let captured: { cid: string; cin: string | null; record: Record } | undefined; await executeRecordsIngest( diff --git a/reference-implementation/test/rs-records-ingest-provenance-order.test.ts b/reference-implementation/test/rs-records-ingest-provenance-order.test.ts new file mode 100644 index 000000000..1fa71e76c --- /dev/null +++ b/reference-implementation/test/rs-records-ingest-provenance-order.test.ts @@ -0,0 +1,133 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * The batch optimization must not move acquisition provenance after the whole + * request. The route's observable sequence is store(record), provenance(record), + * then the next record's store. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; + +import type { MountRsMutationContext } from "../server/routes/rs-mutation.ts"; +import { mountRsRecordsIngest } from "../server/routes/rs-mutation.ts"; + +type RouteHandler = (req: unknown, res: unknown) => unknown | Promise; +type MountApp = Parameters[0]; + +interface FakeResponse { + body: unknown; + end: () => void; + headers: Record; + json: (body: unknown) => unknown; + setHeader: (name: string, value: string) => unknown; + status: (code: number) => FakeResponse; + statusCode: number | null; +} + +function makeApp(): { app: MountApp; routes: Record } { + const routes: Record = {}; + const app = { + post(path: string, ...handlers: unknown[]) { + routes[path] = handlers as RouteHandler[]; + return app; + }, + } as unknown as MountApp; + return { app, routes }; +} + +function makeResponse(): FakeResponse { + const response: FakeResponse = { + body: undefined, + end() { + // This route returns JSON in the successful path. + }, + headers: {}, + json(body) { + response.body = body; + return body; + }, + setHeader(name, value) { + response.headers[name] = value; + }, + status(code) { + response.statusCode = code; + return response; + }, + statusCode: null, + }; + return response; +} + +test("rs.records.ingest preserves store/provenance order inside a batch", async () => { + const events: string[] = []; + const { app, routes } = makeApp(); + const ctx = { + buildMutationContext: () => ({ traceId: "trace-1" }), + emitMutationEvent: async () => undefined, + emitMutationRequested: async () => undefined, + getLatestAcquisitionBatchForConnection: async () => ({ + acquisitionMethod: "manual_upload", + batchId: "batch-1", + }), + ingestRecord: () => Promise.reject(new Error("single-record fallback must not run")), + ingestRecords: async ( + _target: unknown, + records: readonly Record[], + afterRecord?: (record: Record, outcome: unknown) => Promise + ) => { + const outcomes: Array<{ accepted: true; changed: true }> = []; + for (const record of records) { + const key = String(record.id); + events.push(`store:${key}`); + const outcome = { accepted: true as const, changed: true as const }; + outcomes.push(outcome); + // biome-ignore lint/performance/noAwaitInLoops: The oracle asserts the required per-record ordering. + await afterRecord?.(record, outcome); + } + return outcomes; + }, + recordAcquisitionProvenance: ({ recordKey }: { recordKey: string }) => { + events.push(`provenance:${recordKey}`); + }, + rejectMutation: (_res: unknown, _req: unknown, _ctx: unknown, err: Error) => Promise.reject(err), + requireOwner: (_req: unknown, _res: unknown, next: () => unknown) => next(), + requireToken: (_req: unknown, _res: unknown, next: () => unknown) => next(), + resolveOwnerConnectorNamespace: async () => ({ + connectorId: "connector-1", + connectorInstanceId: "instance-1", + }), + resolveRegisteredConnectorManifest: async () => ({ streams: [{ name: "messages" }] }), + resolveSingleConnectorIdQueryValue: (value: unknown) => (typeof value === "string" ? value : null), + setReferenceTraceId: () => undefined, + storageTargetForConnectorNamespace: () => ({ + connector_id: "connector-1", + connector_instance_id: "instance-1", + }), + } as unknown as MountRsMutationContext; + + mountRsRecordsIngest(app, ctx); + const routeHandlers = routes["/v1/ingest/:stream"]; + assert.ok(routeHandlers, "ingest route must be mounted"); + const handler = routeHandlers.at(-1); + assert.ok(handler, "ingest route handler must be mounted"); + const response = makeResponse(); + await handler( + { + body: '{"id":"r1","key":"r1"}\n{"id":"r2","key":"r2"}', + headers: {}, + params: { stream: "messages" }, + query: { connector_id: "connector-1", connector_instance_id: "instance-1" }, + }, + response + ); + + assert.deepEqual(events, ["store:r1", "provenance:r1", "store:r2", "provenance:r2"]); + assert.deepEqual(response.body, { + errors: [], + records_accepted: 2, + records_rejected: 0, + stream: "messages", + }); +}); diff --git a/reference-implementation/test/runtime-cancel-ingest-commit-boundary-probe.test.ts b/reference-implementation/test/runtime-cancel-ingest-commit-boundary-probe.test.ts new file mode 100644 index 000000000..4550c2483 --- /dev/null +++ b/reference-implementation/test/runtime-cancel-ingest-commit-boundary-probe.test.ts @@ -0,0 +1,604 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +// Red-team probe + regression proof for PR #84 (add-console-run-cancel-control): +// does a durable /v1/ingest write commit AFTER a run's canonical terminal state +// (run_history status='cancelled', completed_at set) has already been recorded? +// +// This is a production-path discriminator, not a mock-request counter. It +// drives the REAL storage write path — server/records.ts `ingestRecord` -> +// withConnectorInstanceWrite -> the same SQLite BEGIN IMMEDIATE transaction / +// Postgres transaction production traffic uses — and the REAL terminal-state +// writer (server/stores/run-history-writer.ts `writeSqliteRunHistoryForSpineEvent`, +// the exact function `emitSpineEvent` calls synchronously for run.cancelled). +// +// runtime/index.ts's own cancellation fence (flushBatch's terminalStopRequested +// check + `signal: cancelSignal` on fetch) is a CLIENT-side abort: it stops the +// runtime from *starting* new ingest requests and lets fetch's promise reject +// if the socket closes early. It says nothing about what the SERVER does with +// a write that was already admitted into the write-coordinator before cancel +// fired. This probe forces exactly that race deterministically, using the +// `__setConnectorInstanceWritePhaseHookForTest` seam (the same per-instance +// write-serialization gate every SQLite/Postgres ingest write passes through) +// to pause a write immediately after it acquires the coordinator lock — i.e., +// immediately before the durable transaction — record the run cancelled in +// the meantime, then release the write and observe whether it still lands in +// `records`. +// +// Distinguishing normal in-flight completion from a post-cancel commit: the +// existing coverage (runtime-cancel-run.test.ts, runtime-cancel-queue.test.ts) +// already proves the *legitimate* case — an ingest request that was already +// authorized and in flight when cancel fires is allowed to finish, and that is +// intentional. This probe asks a narrower, later question: is there a hard +// admission boundary such that a write cannot commit once the run's terminal +// state is already durable, or is the terminal state and the write simply +// racing with no ordering guarantee at all? +// +// Fix scope (harden-ingest-run-admission-fence): the fence in server/records.ts +// / server/postgres-records.ts activates ONLY when the caller supplies +// `options.runId` — i.e. only for run-bound connector ingestion. Owner/API +// ingestion that never threads a run_id (the pre-existing `ingestRecord(target, +// record)` two-arg call shape, still used by owner-agent tooling and the +// source-webhooks route) is completely unaffected; a record write with no +// `run_id` is admitted exactly as before this fix. +// +// Fails CLOSED on an unrecognized run_id: runtime/index.ts always awaits the +// run.started spine write (which durably inserts the run_history row with +// status='running') before spawning the child that could ever call +// flushBatch, so a genuine run-bound write is guaranteed to find its row. A +// run_id with no matching row is refused, not admitted — otherwise a +// spoofed/typo'd run_id would silently bypass the fence entirely. +// +// Design note — per-record cost, not per-batch: the coordinator lock +// `withConnectorInstanceWrite`/`ingestRecords` already holds for the whole +// batch does NOT serialize against the run's terminal write (a completely +// separate call path — runtime/index.ts's proc.on("close", ...) handler, +// never nested inside the coordinator's critical section). Making the +// terminal writer acquire that same lock to allow a cheaper once-per-batch +// check was evaluated and rejected: the lock is held for the FULL batch +// duration (in-process key gate AND, cross-process, the Postgres advisory +// lock), so cancellation would then have to wait for the entire in-flight +// batch — including an unbounded flood — to finish before it could even +// attempt to record run.cancelled. That is precisely the "terminalization +// waits behind an unbounded queue" defect commit 73708a720 fixed, and it +// would violate the sub-second terminalization contract both +// runtime-cancel-run.test.ts and runtime-cancel-queue.test.ts already +// enforce (`elapsedMs < 1500`). Per-record checking is therefore the +// necessary cost of the "no write commits after terminalization" invariant +// under the current architecture, not an unoptimized default. Measured cost +// (5-round warmed benchmark, 200-record batch): SQLite ~20.5us/record +// (~13% relative — an in-process prepared-statement lookup, no network hop); +// Postgres ~257us/record (~2.7% relative — a `FOR UPDATE` round trip against +// an already network-bound per-record baseline that already takes its own +// `FOR UPDATE` lock on the `records` row). +// +// Coverage in this file: +// 1. NO FIX (baseline): a run-bound write with no runId supplied — proves +// the fence is opt-in and pre-existing unscoped callers are unaffected. +// 2. FIXED — cancel-before-release: a write admitted while running, then +// the run terminalizes before the write reaches its transaction — the +// write is now REFUSED (not silently dropped as accepted). +// 3. FIXED — release-before-cancel: the legitimate ordering — a write that +// completes its transaction before cancellation is untouched (the +// existing "already-started ingest is preserved" guarantee still holds). +// 4. Spoof resistance — terminal status is scoped to (run_id, +// connector_instance_id); a write claiming a run_id that belongs to a +// DIFFERENT connector_instance_id cannot be fenced by that other run's +// cancellation, and cannot be used to bypass a fence on its own run. +// 5. Fail-closed — a run_id with no matching run_history row at all +// (spoofed, mistyped, or foreign) is refused, not admitted. +// 6. HTTP wire-through — the runtime's `?run_id=` query param on the real +// POST /v1/ingest/:stream route reaches the storage-layer fence. +// 7. Postgres parity for cases 2, 3, and 5, gated on PDPP_TEST_POSTGRES_URL +// (skips when unset, matching this repo's existing Postgres-parity +// test convention — see test/postgres-records-ingest-noop.test.ts). + +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test, { type TestContext } from "node:test"; +import { __setConnectorInstanceWritePhaseHookForTest } from "../server/connector-instance-write-coordinator.ts"; +import { closeDb, getDb, initDb } from "../server/db.ts"; +import { closePostgresStorage, initPostgresStorage, postgresQuery } from "../server/postgres-storage.ts"; +import { ingestRecord } from "../server/records.ts"; +import { writeSqliteRunHistoryForSpineEvent } from "../server/stores/run-history-writer.ts"; + +const STREAM = "items"; +const POSTGRES_URL = process.env.PDPP_TEST_POSTGRES_URL; +const RUN_TERMINAL_ERROR_RE = /run .* is already terminal/; + +function freshDb(t: TestContext): void { + const dir = mkdtempSync(join(tmpdir(), "pdpp-cancel-boundary-probe-")); + closeDb(); + initDb(join(dir, "pdpp.sqlite")); + t.after(() => { + closeDb(); + rmSync(dir, { force: true, recursive: true }); + }); +} + +function seedActiveConnection(connectorId: string, connectorInstanceId: string): void { + const now = new Date().toISOString(); + getDb() + .prepare("INSERT INTO connectors(connector_id, manifest, created_at) VALUES (?, '{}', ?)") + .run(connectorId, now); + getDb() + .prepare( + `INSERT INTO connector_instances( + connector_instance_id, owner_subject_id, connector_id, display_name, status, + source_kind, source_binding_key, source_binding_json, created_at, updated_at + ) VALUES (?, 'owner', ?, 'HTTP wire-through probe', 'active', 'account', ?, '{}', ?, ?)` + ) + .run(connectorInstanceId, connectorId, connectorInstanceId, now, now); +} + +function startRunSqlite(runId: string, connectorId: string, connectorInstanceId: string): void { + writeSqliteRunHistoryForSpineEvent({ + connectorId, + connectorInstanceId, + data: {}, + eventType: "run.started", + occurredAt: new Date().toISOString(), + runId, + status: "started", + }); +} + +function cancelRunSqlite(runId: string, connectorId: string, connectorInstanceId: string): void { + writeSqliteRunHistoryForSpineEvent({ + connectorId, + connectorInstanceId, + data: { reason: "owner_cancelled" }, + eventType: "run.cancelled", + occurredAt: new Date().toISOString(), + runId, + status: "cancelled", + }); +} + +interface RunHistoryRow { + completed_at: string | null; + status: string; +} + +function readRunHistorySqlite(runId: string, connectorInstanceId: string): RunHistoryRow { + const db = getDb(); + const row = db + .prepare("SELECT status, completed_at FROM run_history WHERE run_id = ? AND connector_instance_id = ?") + .get(runId, connectorInstanceId) as RunHistoryRow | undefined; + assert.ok(row, "run_history row must exist for the probe run"); + return row; +} + +function readCommittedRecordSqlite(connectorInstanceId: string, key: string): { record_key: string } | undefined { + const db = getDb(); + return db + .prepare( + "SELECT record_key FROM records WHERE connector_instance_id = ? AND stream = ? AND record_key = ? AND deleted = 0" + ) + .get(connectorInstanceId, STREAM, key) as { record_key: string } | undefined; +} + +test("no fix scope creep: a run-bound write with no runId supplied is admitted exactly as before (owner/API ingestion unaffected)", async (t) => { + freshDb(t); + const connectorId = "no-runid-scope-probe"; + const connectorInstanceId = "cin_no_runid_scope_probe"; + const runId = "run_no_runid_scope_probe"; + startRunSqlite(runId, connectorId, connectorInstanceId); + cancelRunSqlite(runId, connectorId, connectorInstanceId); + + // No `options.runId` — this is the exact call shape owner-agent tooling and + // the source-webhooks route use today. The fence must not activate. + const outcome = await ingestRecord( + { connector_id: connectorId, connector_instance_id: connectorInstanceId }, + { data: { id: "r1" }, emitted_at: new Date().toISOString(), key: "r1", stream: STREAM } + ); + + assert.equal( + outcome.accepted, + true, + "a write with no runId is admitted even though the connection's run_history shows a cancelled run" + ); + assert.ok( + readCommittedRecordSqlite(connectorInstanceId, "r1"), + "record committed — unscoped ingestion is unaffected by the fence" + ); +}); + +test("SQLite fixed: cancel-before-release — a write already admitted before terminalization is refused, not silently committed", async (t) => { + freshDb(t); + t.after(() => __setConnectorInstanceWritePhaseHookForTest(null)); + const connectorId = "cancel-before-release-probe"; + const connectorInstanceId = "cin_cancel_before_release_probe"; + const runId = "run_cancel_before_release_probe"; + startRunSqlite(runId, connectorId, connectorInstanceId); + + let releaseWrite!: () => void; + const writeHeld = new Promise((resolve) => { + releaseWrite = resolve; + }); + let sawRunningAtAcquire: string | null = null; + + __setConnectorInstanceWritePhaseHookForTest(async (stage) => { + if (stage !== "after_acquire") { + return; + } + sawRunningAtAcquire = readRunHistorySqlite(runId, connectorInstanceId).status; + await writeHeld; + }); + + const ingestPromise = ingestRecord( + { connector_id: connectorId, connector_instance_id: connectorInstanceId }, + { data: { id: "r1" }, emitted_at: new Date().toISOString(), key: "r1", stream: STREAM }, + { runId } + ); + + await new Promise((resolve) => setTimeout(resolve, 20)); + + cancelRunSqlite(runId, connectorId, connectorInstanceId); + const terminalSnapshot = readRunHistorySqlite(runId, connectorInstanceId); + assert.equal( + terminalSnapshot.status, + "cancelled", + "run_history is durably cancelled before the parked write is released" + ); + assert.ok(terminalSnapshot.completed_at, "cancellation stamped a terminal completed_at"); + + releaseWrite(); + await assert.rejects( + ingestPromise, + RUN_TERMINAL_ERROR_RE, + "FIX: the write-admission fence refuses a write for a run that terminalized while the write was already parked in the coordinator" + ); + + assert.equal( + sawRunningAtAcquire, + "running", + "the write was admitted into the coordinator while the run was still live" + ); + assert.equal( + readCommittedRecordSqlite(connectorInstanceId, "r1"), + undefined, + "no record committed — the fenced write never reached the durable mutation" + ); +}); + +test("SQLite legitimate ordering preserved: release-before-cancel — a write that completes before cancellation is untouched", async (t) => { + freshDb(t); + t.after(() => __setConnectorInstanceWritePhaseHookForTest(null)); + const connectorId = "release-before-cancel-probe"; + const connectorInstanceId = "cin_release_before_cancel_probe"; + const runId = "run_release_before_cancel_probe"; + startRunSqlite(runId, connectorId, connectorInstanceId); + + const outcome = await ingestRecord( + { connector_id: connectorId, connector_instance_id: connectorInstanceId }, + { data: { id: "r1" }, emitted_at: new Date().toISOString(), key: "r1", stream: STREAM }, + { runId } + ); + assert.equal(outcome.accepted, true, "a write that completes while the run is still running is accepted"); + + // Cancellation arrives strictly after the write's own transaction committed. + cancelRunSqlite(runId, connectorId, connectorInstanceId); + + assert.ok( + readCommittedRecordSqlite(connectorInstanceId, "r1"), + "the already-committed record is not retroactively rolled back by a later cancellation" + ); +}); + +test("spoof resistance: a runId scoped to a DIFFERENT connector_instance_id cannot be fenced by that other run's cancellation", async (t) => { + freshDb(t); + const sharedRunId = "run_shared_id_across_connections"; + const victimConnectorId = "spoof-victim"; + const victimInstanceId = "cin_spoof_victim"; + const attackerConnectorId = "spoof-attacker"; + const attackerInstanceId = "cin_spoof_attacker"; + + // Two different connections independently mint the SAME run_id — an + // explicitly documented possibility (run-history-writer.ts: run_id alone is + // NOT globally unique; only (run_id, connector_instance_id) is real + // identity). Cancel only the victim's run. + startRunSqlite(sharedRunId, victimConnectorId, victimInstanceId); + startRunSqlite(sharedRunId, attackerConnectorId, attackerInstanceId); + cancelRunSqlite(sharedRunId, victimConnectorId, victimInstanceId); + + // The attacker's own run (same run_id, different connector_instance_id) is + // still running and must be admitted — a bare `WHERE run_id = ?` fence + // would have wrongly refused this. + const attackerOutcome = await ingestRecord( + { connector_id: attackerConnectorId, connector_instance_id: attackerInstanceId }, + { data: { id: "a1" }, emitted_at: new Date().toISOString(), key: "a1", stream: STREAM }, + { runId: sharedRunId } + ); + assert.equal( + attackerOutcome.accepted, + true, + "a write for a live run under a DIFFERENT connector_instance_id is not fenced by a same-run_id cancellation on another connection" + ); + assert.ok(readCommittedRecordSqlite(attackerInstanceId, "a1")); + + // The victim's run is genuinely cancelled and must be fenced. + await assert.rejects( + ingestRecord( + { connector_id: victimConnectorId, connector_instance_id: victimInstanceId }, + { data: { id: "v1" }, emitted_at: new Date().toISOString(), key: "v1", stream: STREAM }, + { runId: sharedRunId } + ), + RUN_TERMINAL_ERROR_RE, + "the victim's cancelled run is correctly fenced despite the run_id collision" + ); + assert.equal(readCommittedRecordSqlite(victimInstanceId, "v1"), undefined); +}); + +test("fail closed: a runId with NO matching run_history row at all is refused, not admitted", async (t) => { + freshDb(t); + const connectorId = "fail-closed-probe"; + const connectorInstanceId = "cin_fail_closed_probe"; + + // No startRunSqlite call — run.started never wrote a row for this runId. + // Every genuine run-bound write is preceded by an awaited run.started + // spine write (runtime/index.ts), so this state is only reachable via a + // spoofed, mistyped, or foreign run_id — none of which should bypass the + // fence by exploiting a "no row found" fail-open default. + await assert.rejects( + ingestRecord( + { connector_id: connectorId, connector_instance_id: connectorInstanceId }, + { data: { id: "r1" }, emitted_at: new Date().toISOString(), key: "r1", stream: STREAM }, + { runId: "run_never_started_or_spoofed" } + ), + RUN_TERMINAL_ERROR_RE, + "a runId with no run_history row is refused (fail closed), not silently admitted" + ); + assert.equal(readCommittedRecordSqlite(connectorInstanceId, "r1"), undefined); +}); + +// --- HTTP wire-through: runtime's ?run_id= reaches the storage fence --- + +test("HTTP wire-through: POST /v1/ingest/:stream's ?run_id= query param reaches the storage-layer fence end to end", async (t) => { + freshDb(t); + t.after(() => __setConnectorInstanceWritePhaseHookForTest(null)); + const connectorId = "http-wire-through-probe"; + const connectorInstanceId = "cin_http_wire_through_probe"; + const runId = "run_http_wire_through_probe"; + seedActiveConnection(connectorId, connectorInstanceId); + startRunSqlite(runId, connectorId, connectorInstanceId); + cancelRunSqlite(runId, connectorId, connectorInstanceId); + + const routes: Record unknown)[]> = {}; + const app = { + post(path: string, ...handlers: unknown[]) { + routes[path] = handlers as ((req: unknown, res: unknown) => unknown)[]; + }, + }; + let jsonBody: unknown; + const res = { + json: (body: unknown) => { + jsonBody = body; + return body; + }, + setHeader: () => undefined, + status: () => res, + }; + const ctx = { + buildMutationContext: () => ({ traceId: "trace-http-wire-through" }), + emitMutationEvent: async () => undefined, + emitMutationRequested: async () => undefined, + handleError: (_res: unknown, err: unknown) => { + throw err; + }, + // The REAL storage function under test — not a mock. + ingestRecord, + rejectMutation: (_res: unknown, _req: unknown, _mctx: unknown, err: Error) => Promise.reject(err), + requireOwner: (_req: unknown, _res: unknown, next: () => unknown) => next(), + requireToken: (_req: unknown, _res: unknown, next: () => unknown) => next(), + resolveOwnerConnectorNamespace: async () => ({ connectorId, connectorInstanceId }), + resolveRegisteredConnectorManifest: async () => ({ streams: [{ name: STREAM }] }), + resolveSingleConnectorIdQueryValue: (value: unknown) => (typeof value === "string" ? value : null), + setReferenceTraceId: () => undefined, + storageTargetForConnectorNamespace: () => ({ + connector_id: connectorId, + connector_instance_id: connectorInstanceId, + }), + }; + + // dynamic import keeps this test file's static imports free of the route + // module's much larger MountRsMutationContext type surface. + const { mountRsRecordsIngest } = await import("../server/routes/rs-mutation.ts"); + type MountRsRecordsIngest = typeof mountRsRecordsIngest; + mountRsRecordsIngest( + app as unknown as Parameters[0], + ctx as unknown as Parameters[1] + ); + assert.ok("/v1/ingest/:stream" in routes, "ingest route must be mounted"); + const handler = routes["/v1/ingest/:stream"].at(-1); + assert.ok(handler, "ingest route handler must be registered"); + + await handler( + { + body: '{"id":"r1"}', + headers: {}, + params: { stream: STREAM }, + query: { connector_id: connectorId, connector_instance_id: connectorInstanceId, run_id: runId }, + }, + res + ); + + const envelope = jsonBody as { records_accepted: number; records_rejected: number; errors: readonly string[] }; + assert.equal(envelope.records_accepted, 0, "the HTTP route surfaces the fenced write as rejected, not accepted"); + assert.equal(envelope.records_rejected, 1); + assert.match(envelope.errors[0] ?? "", RUN_TERMINAL_ERROR_RE); + assert.equal( + readCommittedRecordSqlite(connectorInstanceId, "r1"), + undefined, + "no record committed through the real HTTP route for a cancelled run's run_id" + ); +}); + +// --- Postgres parity --------------------------------------------------- + +function postgresStorageConfig(): { backend: "postgres"; databaseUrl: string } { + assert.ok(POSTGRES_URL, "Postgres test requires PDPP_TEST_POSTGRES_URL"); + return { backend: "postgres", databaseUrl: POSTGRES_URL }; +} + +async function startRunPostgres(runId: string, connectorId: string, connectorInstanceId: string): Promise { + await postgresQuery( + `INSERT INTO run_history(run_id, connector_instance_id, connector_id, source_json, status, known_gaps_json, started_at, attempt) + VALUES($1, $2, $3, '{}'::jsonb, 'running', '[]'::jsonb, now(), 1)`, + [runId, connectorInstanceId, connectorId] + ); +} + +async function cancelRunPostgres(runId: string, connectorInstanceId: string): Promise { + const result = await postgresQuery<{ status: string }>( + `UPDATE run_history SET status = 'cancelled', completed_at = now() + WHERE run_id = $1 AND connector_instance_id = $2 AND status = 'running' + RETURNING status`, + [runId, connectorInstanceId] + ); + assert.equal(result.rows[0]?.status, "cancelled", "test setup: the run_history row transitioned to cancelled"); +} + +async function readCommittedRecordPostgres( + connectorInstanceId: string, + key: string +): Promise<{ record_key: string } | undefined> { + const result = await postgresQuery<{ record_key: string }>( + "SELECT record_key FROM records WHERE connector_instance_id = $1 AND stream = $2 AND record_key = $3 AND deleted = false", + [connectorInstanceId, STREAM, key] + ); + return result.rows[0]; +} + +async function cleanupPostgres(connectorInstanceId: string, runId: string): Promise { + try { + await postgresQuery("DELETE FROM record_changes WHERE connector_instance_id = $1", [connectorInstanceId]); + await postgresQuery("DELETE FROM records WHERE connector_instance_id = $1", [connectorInstanceId]); + await postgresQuery("DELETE FROM version_counter WHERE connector_instance_id = $1", [connectorInstanceId]); + await postgresQuery("DELETE FROM run_history WHERE run_id = $1 AND connector_instance_id = $2", [ + runId, + connectorInstanceId, + ]); + } catch { + // best-effort cleanup, matches this repo's existing Postgres test convention + } +} + +if (POSTGRES_URL) { + test("Postgres fixed: cancel-before-release — a write already admitted before terminalization is refused", async (t) => { + const suffix = `${Date.now()}_${Math.floor(Math.random() * 1e6)}`; + const connectorId = `pg_cancel_before_release_${suffix}`; + const connectorInstanceId = `cin_pg_cancel_before_release_${suffix}`; + const runId = `run_pg_cancel_before_release_${suffix}`; + + initDb(":memory:"); + await initPostgresStorage(postgresStorageConfig()); + t.after(async () => { + await cleanupPostgres(connectorInstanceId, runId); + await closePostgresStorage(); + closeDb(); + }); + t.after(() => __setConnectorInstanceWritePhaseHookForTest(null)); + + await startRunPostgres(runId, connectorId, connectorInstanceId); + + let releaseWrite!: () => void; + const writeHeld = new Promise((resolve) => { + releaseWrite = resolve; + }); + __setConnectorInstanceWritePhaseHookForTest(async (stage) => { + if (stage !== "after_acquire") { + return; + } + await writeHeld; + }); + + const ingestPromise = ingestRecord( + { connector_id: connectorId, connector_instance_id: connectorInstanceId }, + { data: { id: "r1" }, emitted_at: new Date().toISOString(), key: "r1", stream: STREAM }, + { runId } + ); + + await new Promise((resolve) => setTimeout(resolve, 20)); + await cancelRunPostgres(runId, connectorInstanceId); + + releaseWrite(); + await assert.rejects( + ingestPromise, + RUN_TERMINAL_ERROR_RE, + "FIX (Postgres): the write-admission fence refuses a write for a run that terminalized while the write was parked" + ); + + assert.equal( + await readCommittedRecordPostgres(connectorInstanceId, "r1"), + undefined, + "no record committed on Postgres — the fenced write never reached the durable mutation" + ); + }); + + test("Postgres legitimate ordering preserved: release-before-cancel — a write that completes before cancellation is untouched", async (t) => { + const suffix = `${Date.now()}_${Math.floor(Math.random() * 1e6)}`; + const connectorId = `pg_release_before_cancel_${suffix}`; + const connectorInstanceId = `cin_pg_release_before_cancel_${suffix}`; + const runId = `run_pg_release_before_cancel_${suffix}`; + + initDb(":memory:"); + await initPostgresStorage(postgresStorageConfig()); + t.after(async () => { + await cleanupPostgres(connectorInstanceId, runId); + await closePostgresStorage(); + closeDb(); + }); + + await startRunPostgres(runId, connectorId, connectorInstanceId); + + const outcome = await ingestRecord( + { connector_id: connectorId, connector_instance_id: connectorInstanceId }, + { data: { id: "r1" }, emitted_at: new Date().toISOString(), key: "r1", stream: STREAM }, + { runId } + ); + assert.equal( + outcome.accepted, + true, + "a write that completes while the run is still running is accepted (Postgres)" + ); + + await cancelRunPostgres(runId, connectorInstanceId); + + assert.ok( + await readCommittedRecordPostgres(connectorInstanceId, "r1"), + "the already-committed record is not retroactively rolled back by a later cancellation (Postgres)" + ); + }); + + test("Postgres fail closed: a runId with NO matching run_history row at all is refused, not admitted", async (t) => { + const suffix = `${Date.now()}_${Math.floor(Math.random() * 1e6)}`; + const connectorId = `pg_fail_closed_${suffix}`; + const connectorInstanceId = `cin_pg_fail_closed_${suffix}`; + + initDb(":memory:"); + await initPostgresStorage(postgresStorageConfig()); + t.after(async () => { + await cleanupPostgres(connectorInstanceId, `run_pg_never_started_${suffix}`); + await closePostgresStorage(); + closeDb(); + }); + + await assert.rejects( + ingestRecord( + { connector_id: connectorId, connector_instance_id: connectorInstanceId }, + { data: { id: "r1" }, emitted_at: new Date().toISOString(), key: "r1", stream: STREAM }, + { runId: `run_pg_never_started_${suffix}` } + ), + RUN_TERMINAL_ERROR_RE, + "a runId with no run_history row is refused on Postgres (fail closed), not silently admitted" + ); + assert.equal(await readCommittedRecordPostgres(connectorInstanceId, "r1"), undefined); + }); +} else { + test("Postgres cancel-boundary parity (skipped: PDPP_TEST_POSTGRES_URL unset)", { skip: true }, () => { + // See test/postgres-records-ingest-noop.test.ts for this repo's convention. + }); +} diff --git a/reference-implementation/test/runtime-cancel-queue.test.ts b/reference-implementation/test/runtime-cancel-queue.test.ts new file mode 100644 index 000000000..aad9d8319 --- /dev/null +++ b/reference-implementation/test/runtime-cancel-queue.test.ts @@ -0,0 +1,184 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +// Runtime cancellation tests for a connector that has already filled the +// parent runtime's ingest queue. Gmail can produce this shape while attachment +// hydration and message records are being forwarded to the resource server. + +import assert from "node:assert/strict"; +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test, { type TestContext } from "node:test"; +import { runConnector } from "../runtime/index.ts"; +import { closeDb, initDb } from "../server/db.ts"; + +const STREAM = "items"; +const MANIFEST = { + connector_id: "https://registry.pdpp.org/connectors/cancel-queue-regression", + runtime_requirements: {}, + streams: [ + { + name: STREAM, + primary_key: "id", + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + }, + ], + version: "0.1.0", +}; + +function freshDb(): string { + const dir = mkdtempSync(join(tmpdir(), "pdpp-runtime-cancel-queue-db-")); + closeDb(); + initDb(join(dir, "pdpp.sqlite")); + return dir; +} + +function writeFloodStub({ ignoreSigterm }: { ignoreSigterm: boolean }): { path: string; dir: string } { + const dir = mkdtempSync(join(tmpdir(), "pdpp-runtime-cancel-queue-")); + const path = join(dir, "stub.mjs"); + const sigtermHandler = ignoreSigterm + ? 'process.on("SIGTERM", () => { /* force the parent runtime fallback */ });' + : "// default SIGTERM disposition: exit immediately"; + writeFileSync( + path, + ` +import { createInterface } from "node:readline"; + +${sigtermHandler} + +await new Promise((resolve) => { + const reader = createInterface({ input: process.stdin, crlfDelay: Infinity }); + reader.once("line", () => { reader.close(); resolve(); }); +}); + +for (let i = 0; i < 120; i += 1) { + process.stdout.write(JSON.stringify({ + type: "RECORD", + stream: "${STREAM}", + key: String(i), + data: { id: String(i) }, + emitted_at: new Date().toISOString(), + }) + "\\n"); +} +setInterval(() => {}, 1000); +`, + "utf8" + ); + chmodSync(path, 0o755); + return { dir, path }; +} + +async function runCancelQueueScenario( + t: TestContext, + { ignoreSigterm, runId }: { ignoreSigterm: boolean; runId: string } +): Promise<{ elapsedMs: number; ingestCount: number; requestCountAtCancel: number | null; result: unknown }> { + const dbDir = freshDb(); + t.after(() => { + closeDb(); + rmSync(dbDir, { force: true, recursive: true }); + }); + const previousBatchSize = process.env.PDPP_RUNTIME_BATCH_SIZE; + process.env.PDPP_RUNTIME_BATCH_SIZE = "1"; + t.after(() => { + if (previousBatchSize === undefined) { + delete process.env.PDPP_RUNTIME_BATCH_SIZE; + } else { + process.env.PDPP_RUNTIME_BATCH_SIZE = previousBatchSize; + } + }); + + let ingestCount = 0; + let requestCountAtCancel: number | null = null; + let resolveFirstIngest!: () => void; + const firstIngest = new Promise((resolve) => { + resolveFirstIngest = resolve; + }); + const server = createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => { + body += chunk; + }); + req.on("end", () => { + const { method, url } = req; + const { pathname } = new URL(url ?? "/", "http://localhost"); + if (method === "POST" && pathname === `/v1/ingest/${STREAM}`) { + ingestCount += 1; + if (ingestCount === 1) { + resolveFirstIngest(); + } + setTimeout(() => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ records_accepted: body.split("\\n").filter(Boolean).length, records_rejected: 0 })); + }, 20).unref(); + return; + } + res.writeHead(200, { "content-type": "application/json" }); + res.end("{}"); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(() => server.close()); + const address = server.address(); + assert.ok(address && typeof address === "object"); + const { dir, path } = writeFloodStub({ ignoreSigterm }); + t.after(() => rmSync(dir, { force: true, recursive: true })); + + const cancellation = new AbortController(); + cancellation.signal.addEventListener("abort", () => { + requestCountAtCancel = ingestCount; + }); + firstIngest.then(() => cancellation.abort()); + const startedAt = Date.now(); + const result = await runConnector({ + admitRunConnection: async ({ connectorId, ownerSubjectId }) => ({ + connectorId, + connectorInstanceId: "cin_cancel_queue_regression", + ownerSubjectId: ownerSubjectId ?? "owner_local", + }), + cancelSignal: cancellation.signal, + collectionMode: "full_refresh", + connectorId: MANIFEST.connector_id, + connectorPath: path, + detailGapStore: { + listPendingGaps: async () => [], + markGapStatus: async () => null, + upsertPendingGap: async () => null, + }, + manifest: MANIFEST, + onInteraction: () => ({ status: "cancelled", type: "INTERACTION_RESPONSE" }), + onProgress: () => undefined, + ownerToken: "test-owner-token", + persistState: true, + rsUrl: `http://127.0.0.1:${address.port}`, + runId, + state: null, + }); + + return { elapsedMs: Date.now() - startedAt, ingestCount, requestCountAtCancel, result }; +} + +test("runtime cancellation drops queued ingest work for a cooperative child", async (t) => { + const { elapsedMs, ingestCount, requestCountAtCancel, result } = await runCancelQueueScenario(t, { + ignoreSigterm: false, + runId: "run_cancel_queue_graceful", + }); + assert.equal((result as { status: string }).status, "cancelled"); + assert.equal((result as { terminal_reason: string }).terminal_reason, "owner_cancelled"); + assert.equal(ingestCount, requestCountAtCancel, "no queued ingest starts after cooperative cancellation"); + assert.ok(ingestCount >= 1, "the first already-started ingest is preserved"); + assert.ok(elapsedMs < 1500, `terminalization should not drain the queued flood (${elapsedMs}ms)`); +}); + +test("runtime cancellation drops queued ingest work for an uncooperative child", async (t) => { + const { elapsedMs, ingestCount, requestCountAtCancel, result } = await runCancelQueueScenario(t, { + ignoreSigterm: true, + runId: "run_cancel_queue_forced", + }); + assert.equal((result as { status: string }).status, "cancelled"); + assert.equal((result as { terminal_reason: string }).terminal_reason, "owner_cancel_forced"); + assert.equal(ingestCount, requestCountAtCancel, "no queued ingest starts after forced cancellation"); + assert.ok(ingestCount >= 1, "the first already-started ingest is preserved"); + assert.ok(elapsedMs < 1500, `forced terminalization should not drain the queued flood (${elapsedMs}ms)`); +}); diff --git a/reference-implementation/test/runtime-collection-facts.test.ts b/reference-implementation/test/runtime-collection-facts.test.ts new file mode 100644 index 000000000..983442974 --- /dev/null +++ b/reference-implementation/test/runtime-collection-facts.test.ts @@ -0,0 +1,43 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { parseCollectionRatePayload } from "../server/runtime-collection-facts.ts"; + +const baseRate = { + ceiling_interval_ms: 60_000, + ceiling_rate_per_min: 60, + current_interval_ms: 1000, + effective_rate_per_min: 30, + object: "collection_rate", +}; + +test("collection-rate reader treats missing or malformed backoff as honest null", () => { + const malformedBackoffs: unknown[] = [ + undefined, + null, + {}, + { reason: "throttle" }, + { at_interval_ms: Number.NaN, reason: "throttle" }, + { at_interval_ms: 1000 }, + [], + "throttle", + ]; + + for (const last_backoff of malformedBackoffs) { + const parsed = parseCollectionRatePayload({ ...baseRate, last_backoff }); + assert.deepEqual(parsed?.last_backoff, null, `malformed backoff should be ignored: ${String(last_backoff)}`); + } +}); + +test("collection-rate reader preserves a complete backoff", () => { + const parsed = parseCollectionRatePayload({ + ...baseRate, + last_backoff: { at_interval_ms: 1000, reason: "throttle" }, + }); + assert.ok(parsed); + assert.equal(parsed.ceiling_interval_ms, 60_000); + assert.deepEqual(parsed.last_backoff, { at_interval_ms: 1000, reason: "throttle" }); +}); diff --git a/reference-implementation/test/setup-binding-promotion.test.ts b/reference-implementation/test/setup-binding-promotion.test.ts new file mode 100644 index 000000000..00c98813d --- /dev/null +++ b/reference-implementation/test/setup-binding-promotion.test.ts @@ -0,0 +1,489 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + expiredEnrollmentShellIds, + retireExpiredBrowserEnrollmentShells, +} from "../server/browser-enrollment-shell-retirement.ts"; +import { closeDb, getDb, initDb } from "../server/db.ts"; +import { closePostgresStorage, initPostgresStorage, postgresQuery } from "../server/postgres-storage.ts"; +import { promoteBrowserEnrollmentShellBinding } from "../server/routes/ref-browser-enrollment-shell.ts"; +import { + type ManualUploadDraftSourceBinding, + promoteManualUploadDraftBinding, +} from "../server/routes/ref-manual-upload-draft-connection.ts"; +import { + promoteStaticSecretDraftBinding, + type StaticSecretDraftSourceBinding, +} from "../server/routes/ref-static-secret-draft-connection.ts"; +import { + createPostgresConnectorInstanceStore, + createSqliteConnectorInstanceStore, + isOwnerVisibleConnectorInstance, +} from "../server/stores/connector-instance-store.ts"; + +// Every RETIRED_SETUP_SHELL_BINDING_KINDS member (browser_enrollment_shell, +// static_secret_draft, manual_upload_draft) must promote to its durable +// sibling on first successful ingest, or a later revoke wrongly hides a +// real, fully-collected connection. One parameterized conformance body +// drives the same scenario matrix — success, abandon, revoke-after-promote, +// and the race guard — against all three kinds. + +const NOW = "2026-08-06T09:00:00.000Z"; +const PROMOTED_AT = "2026-08-06T09:05:00.000Z"; + +interface StoreLike { + get: (id: string) => unknown | Promise; + listDraftBrowserEnrollmentShells: (ownerSubjectId: string | null) => unknown[] | Promise; + listOwnerVisibleIdentityPage: ( + ownerSubjectId: string, + args: { after?: unknown; limit: number } + ) => { hasMore: boolean; rows: readonly unknown[] } | Promise<{ hasMore: boolean; rows: readonly unknown[] }>; + promoteSetupBinding: ( + connectorInstanceId: string, + args: { fromKind: string; sourceBinding: Record; updatedAt: string } + ) => { instance: unknown; promoted: boolean } | Promise<{ instance: unknown; promoted: boolean }>; + updateStatus: ( + connectorInstanceId: string, + args: { status: string; updatedAt: string; revokedAt?: string | null } + ) => unknown | Promise; + upsert: (record: Record) => unknown | Promise; +} + +interface ConnectorInstanceLike { + connectorInstanceId: string; + ownerSubjectId: string; + revokedAt: string | null; + sourceBinding: Record | null; + sourceBindingKey: string; + sourceKind: string; + status: string; + updatedAt?: string; +} + +interface KindFixture { + readonly connectorId: string; + draftBinding: (variant?: string) => Record; + readonly draftKind: string; + readonly durableKind: string; + // Fields from the draft binding that MUST survive promotion unchanged + // (setup-specific durable metadata still needed for future runs). + durableMetadataAssertions: (draftBinding: Record, promotedBinding: Record) => void; + promote: (draftBinding: Record, now: string) => Record; + readonly sourceKind: string; +} + +const KIND_FIXTURES: readonly KindFixture[] = [ + { + connectorId: "chatgpt", + draftBinding: () => ({ + connector_id: "chatgpt", + enrollment_expires_at: "2026-08-06T11:00:00.000Z", + kind: "browser_enrollment_shell", + }), + draftKind: "browser_enrollment_shell", + durableKind: "browser_collector", + durableMetadataAssertions: (draftBinding, promotedBinding) => { + assert.equal(promotedBinding.connector_id, draftBinding.connector_id, "connector_id survives promotion"); + }, + promote: (draftBinding, now) => + promoteBrowserEnrollmentShellBinding( + draftBinding as unknown as Parameters[0], + now + ) as unknown as Record, + sourceKind: "account", + }, + { + connectorId: "gmail", + draftBinding: () => ({ + kind: "static_secret_draft", + setup_fields: { account_email: "owner@example.com" }, + }), + draftKind: "static_secret_draft", + durableKind: "static_secret", + durableMetadataAssertions: (draftBinding, promotedBinding) => { + assert.deepEqual( + promotedBinding.setup_fields, + draftBinding.setup_fields, + "setup_fields (non-secret manifest fields, read on every probe/run) survive promotion" + ); + }, + promote: (draftBinding, now) => + promoteStaticSecretDraftBinding( + draftBinding as unknown as StaticSecretDraftSourceBinding, + now + ) as unknown as Record, + sourceKind: "account", + }, + { + connectorId: "claude-code", + draftBinding: () => ({ + acquisition_method: "owner_artifact", + import_dir: "/tmp/pdpp-import/claude-code/abc123", + import_dir_env_var: "CLAUDE_CODE_EXPORT_DIR", + kind: "manual_upload_draft", + uploaded_file_name: "export.zip", + }), + draftKind: "manual_upload_draft", + durableKind: "manual_upload", + durableMetadataAssertions: (draftBinding, promotedBinding) => { + assert.equal( + promotedBinding.import_dir, + draftBinding.import_dir, + "import_dir survives promotion — the run-env resolver reads it on EVERY run, not just setup" + ); + assert.equal(promotedBinding.import_dir_env_var, draftBinding.import_dir_env_var); + assert.equal(promotedBinding.uploaded_file_name, draftBinding.uploaded_file_name); + }, + promote: (draftBinding, now) => + promoteManualUploadDraftBinding( + draftBinding as unknown as ManualUploadDraftSourceBinding, + now + ) as unknown as Record, + sourceKind: "manual", + }, +]; + +async function runPromotionConformanceForKind({ + fixture, + store, + seedConnector, + ownerSubjectId, +}: { + fixture: KindFixture; + store: StoreLike; + seedConnector: (connectorId: string) => Promise; + ownerSubjectId: string; +}): Promise { + await seedConnector(fixture.connectorId); + + // --- Fail-before semantics: promoting a row that is NOT currently this + // setup-binding kind must be a no-op, regardless of status. + const nonSetupBinding = { kind: "account" }; + const nonSetupRow = (await store.upsert({ + connectorId: fixture.connectorId, + createdAt: NOW, + displayName: "Connection", + ownerSubjectId, + sourceBinding: nonSetupBinding, + sourceBindingKey: `${ownerSubjectId}_${fixture.draftKind}_account_binding`, + sourceKind: fixture.sourceKind, + status: "active", + updatedAt: NOW, + })) as ConnectorInstanceLike; + const nonSetupResult = await store.promoteSetupBinding(nonSetupRow.connectorInstanceId, { + fromKind: fixture.draftKind, + sourceBinding: fixture.promote(fixture.draftBinding(), PROMOTED_AT), + updatedAt: PROMOTED_AT, + }); + assert.equal(nonSetupResult.promoted, false, "guard rejects a binding kind mismatch"); + const nonSetupAfter = (await store.get(nonSetupRow.connectorInstanceId)) as ConnectorInstanceLike; + assert.deepEqual(nonSetupAfter.sourceBinding, nonSetupBinding, "promotion never touches an unrelated binding kind"); + assert.equal(nonSetupAfter.updatedAt, NOW, "promotion guard rejected the write; updated_at is untouched"); + + // --- Success path: a draft that proves a successful first collection is + // promoted — binding kind flips to the durable sibling, status flips to + // active, identity is preserved exactly, and setup-specific durable + // metadata survives. + const draftKey = `${fixture.draftKind}_${ownerSubjectId}_success`; + const draftBinding = fixture.draftBinding(); + const draft = (await store.upsert({ + connectorId: fixture.connectorId, + createdAt: NOW, + displayName: "Connection", + ownerSubjectId, + sourceBinding: draftBinding, + sourceBindingKey: draftKey, + sourceKind: fixture.sourceKind, + status: "draft", + updatedAt: NOW, + })) as ConnectorInstanceLike; + + const promotedResult = await store.promoteSetupBinding(draft.connectorInstanceId, { + fromKind: fixture.draftKind, + sourceBinding: fixture.promote(draftBinding, PROMOTED_AT), + updatedAt: PROMOTED_AT, + }); + assert.equal(promotedResult.promoted, true, "guard admits a matching draft binding"); + const promoted = promotedResult.instance as ConnectorInstanceLike; + + assert.equal(promoted.connectorInstanceId, draft.connectorInstanceId, "connector_instance_id is preserved"); + assert.equal(promoted.ownerSubjectId, ownerSubjectId, "owner is preserved"); + assert.equal(promoted.sourceKind, fixture.sourceKind, "source_kind (identity axis) is preserved by promotion"); + assert.equal(promoted.sourceBindingKey, draftKey, "source_binding_key (identity axis) is preserved"); + assert.equal(promoted.status, "active", "promotion activates the connection"); + assert.equal(promoted.sourceBinding?.kind, fixture.durableKind, "binding kind moved to the durable sibling"); + fixture.durableMetadataAssertions(draftBinding, promoted.sourceBinding as Record); + + // --- Idempotency: a second promotion call is a safe no-op. + const promotedAgainResult = await store.promoteSetupBinding(draft.connectorInstanceId, { + fromKind: fixture.draftKind, + sourceBinding: fixture.promote(draftBinding, "2026-08-06T09:10:00.000Z"), + updatedAt: "2026-08-06T09:10:00.000Z", + }); + assert.equal(promotedAgainResult.promoted, false, "already-promoted row no longer matches status = draft"); + const promotedAgain = promotedAgainResult.instance as ConnectorInstanceLike; + assert.equal(promotedAgain.updatedAt, PROMOTED_AT, "second promotion call is a no-op; no re-stamp"); + + // --- The exact live-repro shape: a promoted connection that is LATER + // independently revoked stays visible on Sources — it is now an ordinary + // revoked connection, not retired setup residue (its kind is no longer in + // RETIRED_SETUP_SHELL_BINDING_KINDS). + const revokedAfterPromotion = (await store.updateStatus(promoted.connectorInstanceId, { + revokedAt: "2026-08-06T10:00:00.000Z", + status: "revoked", + updatedAt: "2026-08-06T10:00:00.000Z", + })) as ConnectorInstanceLike; + assert.ok( + isOwnerVisibleConnectorInstance({ + sourceBinding: revokedAfterPromotion.sourceBinding, + status: revokedAfterPromotion.status, + }), + `a revoked-after-promotion ${fixture.draftKind} connection stays visible on Sources` + ); + const page = await store.listOwnerVisibleIdentityPage(ownerSubjectId, { limit: 50 }); + const visibleIds = (page.rows as ConnectorInstanceLike[]).map((row) => row.connectorInstanceId); + assert.ok( + visibleIds.includes(promoted.connectorInstanceId), + "the promoted-then-revoked connection appears in the owner-visible identity page" + ); + + // --- Setup failure / abandon is unchanged: a draft that never collects a + // record and is revoked stays retired setup residue — hidden from + // Sources, exactly as before this fix. Promotion never fires for a row + // that never proved success. + const abandonedKey = `${fixture.draftKind}_${ownerSubjectId}_abandoned`; + const abandonedDraft = (await store.upsert({ + connectorId: fixture.connectorId, + createdAt: NOW, + displayName: "Connection", + ownerSubjectId, + sourceBinding: fixture.draftBinding(), + sourceBindingKey: abandonedKey, + sourceKind: fixture.sourceKind, + status: "draft", + updatedAt: NOW, + })) as ConnectorInstanceLike; + const abandoned = (await store.updateStatus(abandonedDraft.connectorInstanceId, { + revokedAt: NOW, + status: "revoked", + updatedAt: NOW, + })) as ConnectorInstanceLike; + assert.equal( + abandoned.sourceBinding?.kind, + fixture.draftKind, + "an abandoned draft's binding kind is untouched — it was never promoted" + ); + assert.ok( + !isOwnerVisibleConnectorInstance({ sourceBinding: abandoned.sourceBinding, status: abandoned.status }), + `an abandoned (never-promoted) ${fixture.draftKind} draft stays hidden from Sources, exactly as before this fix` + ); + const pageAfterAbandon = await store.listOwnerVisibleIdentityPage(ownerSubjectId, { limit: 50 }); + const idsAfterAbandon = (pageAfterAbandon.rows as ConnectorInstanceLike[]).map((row) => row.connectorInstanceId); + assert.ok( + !idsAfterAbandon.includes(abandonedDraft.connectorInstanceId), + "the abandoned draft does not leak into the owner-visible identity page" + ); +} + +// The browser-enrollment-shell kind additionally has a TTL sweep (the other +// two kinds are only ever revoked by explicit owner action — see the +// setup-shell audit); this proves that sweep specifically never revokes a +// promoted connection, closing the exact live-repro shape (a ChatGPT +// connection that ran past its shell TTL after already being promoted). +async function assertPromotedBrowserShellSurvivesTtlSweep({ + store, + ownerSubjectId, +}: { + store: StoreLike; + ownerSubjectId: string; +}): Promise { + const key = `browser_enrollment_shell_${ownerSubjectId}_ttl_survivor`; + const draft = (await store.upsert({ + connectorId: "chatgpt", + createdAt: NOW, + displayName: "ChatGPT", + ownerSubjectId, + sourceBinding: { + connector_id: "chatgpt", + enrollment_expires_at: "2026-08-06T11:00:00.000Z", + kind: "browser_enrollment_shell", + }, + sourceBindingKey: key, + sourceKind: "account", + status: "draft", + updatedAt: NOW, + })) as ConnectorInstanceLike; + const { instance: promotedInstance } = await store.promoteSetupBinding(draft.connectorInstanceId, { + fromKind: "browser_enrollment_shell", + sourceBinding: promoteBrowserEnrollmentShellBinding( + draft.sourceBinding as unknown as Parameters[0], + PROMOTED_AT + ) as unknown as Record, + updatedAt: PROMOTED_AT, + }); + const promoted = promotedInstance as ConnectorInstanceLike; + + const afterTtlExpiry = "2026-08-06T12:00:00.000Z"; + const survivorIds = expiredEnrollmentShellIds( + [{ connectorInstanceId: promoted.connectorInstanceId, sourceBinding: promoted.sourceBinding, status: "active" }], + afterTtlExpiry + ); + assert.deepEqual(survivorIds, [], "a promoted connection is never eligible for shell TTL retirement"); + + const retiredIds = await retireExpiredBrowserEnrollmentShells( + { + listDraftBrowserEnrollmentShells: (subjectId) => + Promise.resolve( + store.listDraftBrowserEnrollmentShells(subjectId) as unknown as { + connectorInstanceId: string; + sourceBinding?: Record | null; + status: string; + }[] + ), + updateStatus: (connectorInstanceId, args) => Promise.resolve(store.updateStatus(connectorInstanceId, args)), + }, + { now: afterTtlExpiry, ownerSubjectId } + ); + assert.ok( + !retiredIds.includes(promoted.connectorInstanceId), + "TTL retirement run past the shell TTL does not revoke the promoted connection" + ); + const promotedAfterSweep = (await store.get(promoted.connectorInstanceId)) as ConnectorInstanceLike; + assert.equal(promotedAfterSweep.status, "active", "promoted connection survives the TTL sweep untouched"); +} + +// Deterministic race oracle: revoke FIRST, then call promoteSetupBinding +// with a stale pre-revoke read — reproduces the exact interleaving +// (activateDraftConnection reads draft, revoke commits, then the UPDATE +// lands) without depending on real thread timing. +async function assertRevokeWinsRaceAgainstPromotion({ + store, + ownerSubjectId, +}: { + store: StoreLike; + ownerSubjectId: string; +}): Promise { + const key = `browser_enrollment_shell_${ownerSubjectId}_race`; + const draftBinding = { + connector_id: "chatgpt", + enrollment_expires_at: "2026-08-06T11:00:00.000Z", + kind: "browser_enrollment_shell", + }; + const draft = (await store.upsert({ + connectorId: "chatgpt", + createdAt: NOW, + displayName: "ChatGPT", + ownerSubjectId, + sourceBinding: draftBinding, + sourceBindingKey: key, + sourceKind: "account", + status: "draft", + updatedAt: NOW, + })) as ConnectorInstanceLike; + + // The race: an owner revoke commits between activateDraftConnection's read + // and promoteSetupBinding's UPDATE. + const revokedAt = "2026-08-06T09:03:00.000Z"; + const revoked = (await store.updateStatus(draft.connectorInstanceId, { + revokedAt, + status: "revoked", + updatedAt: revokedAt, + })) as ConnectorInstanceLike; + assert.equal(revoked.status, "revoked"); + + // promoteSetupBinding is called anyway with the STALE pre-revoke read + // (mirroring activateDraftConnection's actual sequencing: it read the + // draft binding before the revoke landed). + const raceResult = await store.promoteSetupBinding(draft.connectorInstanceId, { + fromKind: "browser_enrollment_shell", + sourceBinding: promoteBrowserEnrollmentShellBinding( + draftBinding as unknown as Parameters[0], + "2026-08-06T09:05:00.000Z" + ) as unknown as Record, + updatedAt: "2026-08-06T09:05:00.000Z", + }); + + assert.equal(raceResult.promoted, false, "the status = 'draft' guard rejects a row revoked mid-race"); + const finalRow = (await store.get(draft.connectorInstanceId)) as ConnectorInstanceLike; + assert.equal(finalRow.status, "revoked", "the row is NOT resurrected to active by the lost-race promotion attempt"); + assert.equal( + finalRow.sourceBinding?.kind, + "browser_enrollment_shell", + "the binding is NOT rewritten to the durable kind by the lost-race promotion attempt" + ); + assert.equal(finalRow.updatedAt, revokedAt, "the revoke's updated_at is not overwritten by the lost race"); +} + +async function runFullConformance({ + store, + seedConnector, + ownerSubjectId, +}: { + store: StoreLike; + seedConnector: (connectorId: string) => Promise; + ownerSubjectId: string; +}): Promise { + for (const fixture of KIND_FIXTURES) { + // biome-ignore lint/performance/noAwaitInLoops: Each kind's fixture rows must not interleave — sequential is correct here. + await runPromotionConformanceForKind({ + fixture, + ownerSubjectId: `${ownerSubjectId}_${fixture.draftKind}`, + seedConnector, + store, + }); + } + await assertPromotedBrowserShellSurvivesTtlSweep({ + ownerSubjectId: `${ownerSubjectId}_ttl`, + store, + }); + await assertRevokeWinsRaceAgainstPromotion({ + ownerSubjectId: `${ownerSubjectId}_race`, + store, + }); +} + +test("SQLite: every setup-binding kind promotes on success, stays hidden on abandon, survives its revoke path", async () => { + initDb(); + try { + for (const connectorId of ["chatgpt", "gmail", "claude-code"]) { + getDb() + .prepare("INSERT OR IGNORE INTO connectors(connector_id, manifest, created_at) VALUES (?, ?, ?)") + .run(connectorId, JSON.stringify({ connector_id: connectorId }), NOW); + } + const store = createSqliteConnectorInstanceStore() as unknown as StoreLike; + await runFullConformance({ ownerSubjectId: "owner_sqlite", seedConnector: () => Promise.resolve(), store }); + } finally { + closeDb(); + } +}); + +test("Postgres: every setup-binding kind promotes on success, stays hidden on abandon, survives its revoke path", { + skip: !process.env.PDPP_TEST_POSTGRES_URL, +}, async () => { + const databaseUrl = process.env.PDPP_TEST_POSTGRES_URL as string; + await initPostgresStorage({ backend: "postgres", databaseUrl }); + const ownerSubjectId = "owner_postgres"; + try { + await postgresQuery("DELETE FROM connector_instances WHERE owner_subject_id LIKE $1", [`${ownerSubjectId}%`]); + const store = createPostgresConnectorInstanceStore() as unknown as StoreLike; + await runFullConformance({ + ownerSubjectId, + seedConnector: async (connectorId: string) => { + await postgresQuery( + `INSERT INTO connectors(connector_id, manifest, created_at) + VALUES($1, $2::jsonb, $3) + ON CONFLICT(connector_id) DO NOTHING`, + [connectorId, JSON.stringify({ connector_id: connectorId }), NOW] + ); + }, + store, + }); + } finally { + await postgresQuery("DELETE FROM connector_instances WHERE owner_subject_id LIKE $1", [`${ownerSubjectId}%`]); + await closePostgresStorage(); + } +}); diff --git a/reference-implementation/test/setup-kind-for-connection.test.ts b/reference-implementation/test/setup-kind-for-connection.test.ts new file mode 100644 index 000000000..21fe99395 --- /dev/null +++ b/reference-implementation/test/setup-kind-for-connection.test.ts @@ -0,0 +1,35 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { setupKindForConnection } from "../server/routes/ref-static-secret-setup-status.ts"; + +// A manifest with no static-secret capture block means the legacy fallback +// (see setupKindForConnection) can never return "static_secret" on its own — +// isolates the binding-kind -> setup-kind map from that fallback. +const NO_CAPTURE_MANIFEST = { connector_id: "test_connector" }; + +test("a promoted static_secret binding classifies as static_secret via the map, not the manifest fallback", () => { + const kind = setupKindForConnection({ kind: "static_secret", setup_fields: {} }, NO_CAPTURE_MANIFEST); + assert.equal(kind, "static_secret"); +}); + +test("a promoted manual_upload binding classifies as manual_upload", () => { + const kind = setupKindForConnection( + { import_dir: "/tmp/x", import_dir_env_var: "X", kind: "manual_upload" }, + NO_CAPTURE_MANIFEST + ); + assert.equal(kind, "manual_upload"); +}); + +test("a promoted browser_collector binding classifies as browser_session", () => { + const kind = setupKindForConnection({ connector_id: "chatgpt", kind: "browser_collector" }, NO_CAPTURE_MANIFEST); + assert.equal(kind, "browser_session"); +}); + +test("an unrecognized binding kind with no manifest capture block falls through to unknown", () => { + const kind = setupKindForConnection({ kind: "account" }, NO_CAPTURE_MANIFEST); + assert.equal(kind, "unknown"); +}); diff --git a/reference-implementation/test/static-secret-credential-probe-route.test.ts b/reference-implementation/test/static-secret-credential-probe-route.test.ts index d56516f77..50f9d341f 100644 --- a/reference-implementation/test/static-secret-credential-probe-route.test.ts +++ b/reference-implementation/test/static-secret-credential-probe-route.test.ts @@ -18,7 +18,8 @@ import { readFileSync } from "node:fs"; import test from "node:test"; import { listSpineEventsPage } from "../lib/spine.ts"; -import { startServer } from "../server/index.ts"; +import { getDb } from "../server/db.ts"; +import { activateDraftConnection, startServer } from "../server/index.ts"; import { createSqliteConnectorInstanceCredentialStore } from "../server/stores/connector-instance-credential-store.ts"; import { createSqliteConnectorInstanceStore } from "../server/stores/connector-instance-store.ts"; import { CREDENTIAL_ENCRYPTION_KEY_ENV } from "../server/stores/credential-encryption.ts"; @@ -30,6 +31,7 @@ const OWNER_PASSWORD = "static-secret-probe-owner-password"; const OWNER_SUBJECT_ID = "owner_local"; const TEST_KEY = "static-secret-probe-test-key"; const GOOD_SECRET = "valid synthetic app password"; +const ALTERNATE_SECRET = "alternate synthetic app password"; const BAD_SECRET = "rejected synthetic app password"; const GOOD_PAT = "ghp_valid_synthetic_token"; const GMAIL_ADDRESS = "the owner@example.com"; @@ -301,10 +303,15 @@ async function capture( cookie: string, connectionId: string, secret: string, - credentialKind: string + credentialKind: string, + setupFields?: Record ): Promise { return fetchJson(`${asUrl}/_ref/connections/${encodeURIComponent(connectionId)}/static-secret-credential`, { - body: JSON.stringify({ credential_kind: credentialKind, secret }), + body: JSON.stringify({ + credential_kind: credentialKind, + secret, + ...(setupFields ? { setup_fields: setupFields } : {}), + }), headers: { Accept: "application/json", "Content-Type": "application/json", Cookie: cookie }, method: "POST", }); @@ -371,8 +378,8 @@ test("a rejected probe returns a typed validation error and stores no credential const instanceStore = createSqliteConnectorInstanceStore(); const instance = await instanceStore.get(connectionId); assert.ok(instance, "expected a connector instance for the draft"); - assert.equal(instance.status, "revoked", "a rejected first-time draft must be retired"); - assert.ok(instance.revokedAt, "retired draft should carry revokedAt"); + assert.equal(instance.status, "draft", "a rejected first-time draft must remain resumable"); + assert.equal(instance.revokedAt, null, "a resumable draft must not be tombstoned"); // The probe was given the non-secret mailbox context, never echoed back. assert.equal(proberCalls.length, 1); @@ -393,6 +400,148 @@ test("a rejected probe returns a typed validation error and stores no credential }); }); +test("a corrected mailbox retry updates and reuses the rejected draft", async () => { + await withCredentialKey(TEST_KEY, async () => { + await withServer(async ({ asUrl, proberCalls }) => { + await registerConnector(asUrl, "gmail"); + const cookie = await login(asUrl); + const draft = await createDraft(asUrl, cookie, "gmail", { account_email: "tim@opendatalabs.xzy" }); + const connectionId = requireString(draft.body.connection_id, "draft.body.connection_id"); + + const rejected = await capture(asUrl, cookie, connectionId, BAD_SECRET, "app_password"); + assert.equal(rejected.status, 400); + + const retried = await capture(asUrl, cookie, connectionId, GOOD_SECRET, "app_password", { + account_email: "tim@opendatalabs.xyz", + }); + assert.equal(retried.status, 201); + assert.equal(retried.body.connection_id, connectionId); + assert.equal(identityOf(retried.body).account_identity, "tim@opendatalabs.xyz"); + assert.equal(proberCalls.at(-1)?.context?.setupFields?.account_email, "tim@opendatalabs.xyz"); + + const instance = await createSqliteConnectorInstanceStore().get(connectionId); + assert.ok(instance, "expected the original draft to remain present"); + assert.equal(instance.status, "draft"); + const binding = instance.sourceBinding as { kind?: string; setup_fields?: Record }; + assert.equal(binding.kind, "static_secret_draft"); + assert.equal(binding.setup_fields?.account_email, "tim@opendatalabs.xyz"); + const retryCount = getDb() + .prepare("SELECT COUNT(*) AS count FROM connector_instances WHERE connector_id = 'gmail'") + .get() as { count: number }; + assert.equal(retryCount.count, 1, "retry must not create a second connector instance"); + assert.ok(await createSqliteConnectorInstanceCredentialStore().getMetadata(connectionId)); + }); + }); +}); + +test("duplicate Gmail draft submissions converge while distinct identities remain separate", async () => { + await withCredentialKey(TEST_KEY, async () => { + await withServer(async ({ asUrl }) => { + await registerConnector(asUrl, "gmail"); + const cookie = await login(asUrl); + const first = await createDraft(asUrl, cookie, "gmail", { account_email: "personal@example.com" }); + const duplicate = await createDraft(asUrl, cookie, "gmail", { account_email: "personal@example.com" }); + const personalId = requireString(first.body.connection_id, "personal connection id"); + assert.equal(duplicate.body.connection_id, personalId); + + const captured = await capture(asUrl, cookie, personalId, GOOD_SECRET, "app_password"); + assert.equal(captured.status, 201); + const reloaded = await createDraft(asUrl, cookie, "gmail", { account_email: "personal@example.com" }); + assert.equal(reloaded.status, 200, "reloading setup after capture must reuse the pending verified connection"); + assert.equal(reloaded.body.connection_id, personalId); + await createSqliteConnectorInstanceStore().activateDraft(personalId, { now: "2026-06-10T18:00:00.000Z" }); + const activeReload = await createDraft(asUrl, cookie, "gmail", { account_email: "personal@example.com" }); + assert.equal(activeReload.status, 200, "a verified active identity must not receive a duplicate connection"); + assert.equal(activeReload.body.connection_id, personalId); + + const work = await createDraft(asUrl, cookie, "gmail", { account_email: "work@example.com" }); + assert.notEqual(work.body.connection_id, personalId, "distinct identities must keep distinct connection ids"); + const identityCount = getDb() + .prepare("SELECT COUNT(*) AS count FROM connector_instances WHERE connector_id = 'gmail'") + .get() as { count: number }; + assert.equal(identityCount.count, 2); + const activeCount = getDb() + .prepare("SELECT COUNT(*) AS count FROM connector_instances WHERE connector_id = 'gmail' AND status = 'active'") + .get() as { count: number }; + assert.equal(activeCount.count, 1); + }); + }); +}); + +test("duplicate captures for one probed identity converge across random drafts", async () => { + await withCredentialKey(TEST_KEY, async () => { + await withServer(async ({ asUrl }) => { + await registerConnector(asUrl, "github"); + const cookie = await login(asUrl); + const first = await createDraft(asUrl, cookie, "github", {}); + const second = await createDraft(asUrl, cookie, "github", {}); + const firstId = requireString(first.body.connection_id, "first github connection id"); + const secondId = requireString(second.body.connection_id, "second github connection id"); + assert.notEqual( + firstId, + secondId, + "GitHub has no owner-entered identity field, so drafts stay separate until probe" + ); + + const [captured, deduplicated] = await Promise.all([ + capture(asUrl, cookie, firstId, GOOD_PAT, "personal_access_token"), + capture(asUrl, cookie, secondId, GOOD_PAT, "personal_access_token"), + ]); + assert.ok([201, 200].includes(captured.status), `unexpected first concurrent status: ${captured.status}`); + assert.ok( + [201, 200].includes(deduplicated.status), + `unexpected second concurrent status: ${deduplicated.status}` + ); + assert.equal(captured.body.connection_id, deduplicated.body.connection_id); + assert.ok( + [captured, deduplicated].some((result) => result.body.deduplicated === true), + "one concurrent capture must report the deduplication" + ); + const credentialCount = getDb().prepare("SELECT COUNT(*) AS count FROM connector_instance_credentials").get() as { + count: number; + }; + assert.equal(credentialCount.count, 1, "one verified identity must have one credential row"); + const winnerId = requireString(captured.body.connection_id, "converged winner connection id"); + const loserId = winnerId === firstId ? secondId : firstId; + assert.ok( + [firstId, secondId].includes(winnerId), + "the converged connection id must be one of the two racing drafts" + ); + const winnerInstance = await createSqliteConnectorInstanceStore().get(winnerId); + assert.notEqual(winnerInstance?.status, "revoked", "the winning draft must remain active for the identity"); + const loserInstance = await createSqliteConnectorInstanceStore().get(loserId); + assert.equal(loserInstance?.status, "revoked", "the losing draft is retired without creating an active fork"); + }); + }); +}); + +test("an active verified connection refuses credential replacement for another identity", async () => { + await withCredentialKey(TEST_KEY, async () => { + await withServer(async ({ asUrl }) => { + await registerConnector(asUrl, "gmail"); + const cookie = await login(asUrl); + const draft = await createDraft(asUrl, cookie, "gmail", { account_email: "personal@example.com" }); + const connectionId = requireString(draft.body.connection_id, "personal connection id"); + const captured = await capture(asUrl, cookie, connectionId, GOOD_SECRET, "app_password"); + assert.equal(captured.status, 201); + const before = await createSqliteConnectorInstanceCredentialStore().getMetadata(connectionId); + assert.ok(before?.fingerprint); + await createSqliteConnectorInstanceStore().activateDraft(connectionId, { now: "2026-06-10T18:00:00.000Z" }); + + const retargeted = await capture(asUrl, cookie, connectionId, ALTERNATE_SECRET, "app_password", { + account_email: "work@example.com", + }); + assert.equal(retargeted.status, 409); + assert.equal(errorOf(retargeted.body).code, "static_secret_identity_mismatch"); + const after = await createSqliteConnectorInstanceCredentialStore().getMetadata(connectionId); + assert.equal(after?.fingerprint, before.fingerprint, "a rejected retarget must not rotate the credential"); + const instance = await createSqliteConnectorInstanceStore().get(connectionId); + const binding = instance?.sourceBinding as { verified_identity?: string }; + assert.equal(binding.verified_identity, "personal@example.com"); + }); + }); +}); + test("a valid probe stores the credential and surfaces the account identity", async () => { await withCredentialKey(TEST_KEY, async () => { await withServer(async ({ asUrl, proberCalls }) => { @@ -457,7 +606,7 @@ test("github: a rejected token is refused and stores nothing", async () => { const instanceStore = createSqliteConnectorInstanceStore(); const instance = await instanceStore.get(connectionId); assert.ok(instance, "expected a connector instance for the draft"); - assert.equal(instance.status, "revoked", "a rejected github draft must be retired"); + assert.equal(instance.status, "draft", "a rejected github draft must remain resumable"); }); }); }); @@ -516,3 +665,282 @@ test("a connector with no probe keeps the first-sync path (stores, no identity e }); }); }); + +// PR #84 P1: an active connection that reached `active` via the real +// first-sync path (no synchronous probe ever ran, so `verified_identity` was +// never recorded) must NOT accept a silent credential replacement just +// because nothing on record contradicts it. Absence of proof is not +// permission. The only way to accept a replacement here is proof this +// request supplies: an exact-fingerprint match against the credential +// already stored (a legitimate resubmission), never a fresh probe result by +// itself. Drives the actual production activation function +// (`activateDraftConnection`, the same one `maybeActivateDraftAfterIngest` +// calls after the first accepted ingest), not a hand-seeded row shape. +test("an active first-sync connection with no recorded identity fails closed on credential replacement", async () => { + await withCredentialKey(TEST_KEY, async () => { + await withServer(async ({ asUrl }) => { + await registerManifest(asUrl, syntheticNoProbeStaticSecretManifest()); + const cookie = await login(asUrl); + + // 1. Owner creates a draft and captures a credential while the + // connector has no synchronous probe: stored, first_sync, no + // identity recorded anywhere on the binding. + const draft = await createDraft(asUrl, cookie, "ynab", {}); + const connectionId = requireString(draft.body.connection_id, "draft.body.connection_id"); + const firstCapture = await capture(asUrl, cookie, connectionId, GOOD_SECRET, "personal_access_token"); + assert.equal(firstCapture.status, 201); + assert.equal(firstCapture.body.validation, "first_sync"); + + const instanceStore = createSqliteConnectorInstanceStore(); + const draftInstance = await instanceStore.get(connectionId); + assert.ok(draftInstance, "expected a connector instance for the draft"); + assert.equal(draftInstance.status, "draft"); + const draftBinding = draftInstance.sourceBinding as { verified_identity?: string }; + assert.equal(draftBinding.verified_identity, undefined, "no probe ran, so no identity was ever recorded"); + + // 2. First accepted ingest activates the draft through the REAL + // production activation path — the same function + // `maybeActivateDraftAfterIngest` calls after a successful sync. + await activateDraftConnection( + connectionId, + instanceStore as unknown as Parameters[1], + () => Promise.resolve(null) + ); + const activeInstance = await instanceStore.get(connectionId); + assert.ok(activeInstance, "expected the connection to still exist after activation"); + assert.equal(activeInstance.status, "active", "first-sync activation must flip the connection active"); + const activeBinding = activeInstance.sourceBinding as { + kind?: string; + verified_identity?: string; + setup_fields?: Record; + }; + assert.equal(activeBinding.kind, "static_secret", "activation promotes to the real static-secret binding kind"); + assert.equal( + activeBinding.verified_identity, + undefined, + "the active connection has NO recorded identity signal — the exact P1 precondition" + ); + + // 3. An attacker with owner-session access (or a mistaken owner) tries + // to point this already-trusted, already-syncing connection at a + // DIFFERENT credential. There is no probe for this connector and no + // durable identity on record, so there is nothing to prove sameness + // except the credential bytes themselves — and they differ. This + // MUST be rejected, not silently accepted because nothing on file + // contradicts it. + const retargeted = await capture(asUrl, cookie, connectionId, ALTERNATE_SECRET, "personal_access_token"); + + assert.notEqual( + retargeted.status, + 201, + "an active connection with a live credential must fail closed on replacement, even with no prior verified_identity on record" + ); + assert.equal(errorOf(retargeted.body).code, "static_secret_identity_unverified_replacement"); + + // The original credential must survive untouched. + const credentialStore = createSqliteConnectorInstanceCredentialStore(); + const secret = await credentialStore.recoverSecret({ + connectorInstanceId: connectionId, + ownerSubjectId: OWNER_SUBJECT_ID, + }); + assert.equal(secret.secret, GOOD_SECRET, "a rejected replacement must not rotate the stored credential"); + + // 4. Resubmitting the EXACT SAME secret (a legitimate retry/resync, + // not an account swap) must still succeed: the fingerprint proves + // sameness without ever decrypting the stored secret or trusting a + // probe. + const resubmitted = await capture(asUrl, cookie, connectionId, GOOD_SECRET, "personal_access_token"); + assert.equal(resubmitted.status, 200, "resubmitting the identical credential is a proven no-op, not a retarget"); + }); + }); +}); + +// Companion to the P1 fail-before test: once a connector DOES have a probe, +// a matching probed identity is still not sufficient by itself to license a +// credential swap on an active, no-durable-identity connection — a probe +// result is exactly what an attacker holding a different, validly-probeable +// credential can also produce. Only the fingerprint (or, once one has been +// recorded, the durable identity) is trusted. +test("a probed identity match alone does not license a retarget when no durable identity is on record", async () => { + await withCredentialKey(TEST_KEY, async () => { + await withServer(async ({ asUrl }) => { + const gmail = loadManifest("gmail"); + await registerManifest(asUrl, gmail); + const cookie = await login(asUrl); + const connectionId = "cin_active_no_durable_identity"; + const instanceStore = createSqliteConnectorInstanceStore(); + await instanceStore.upsert({ + connectorId: gmail.connector_key, + connectorInstanceId: connectionId, + createdAt: "2026-06-10T18:00:00.000Z", + displayName: "Gmail - existing@example.com", + ownerSubjectId: OWNER_SUBJECT_ID, + sourceBinding: { + kind: "static_secret", + promoted_at: "2026-06-10T18:00:00.000Z", + promoted_from: "static_secret_draft", + setup_fields: {}, + }, + sourceBindingKey: connectionId, + sourceKind: "account", + status: "active", + updatedAt: "2026-06-10T18:00:00.000Z", + }); + await createSqliteConnectorInstanceCredentialStore().capture({ + connectorInstanceId: connectionId, + credentialKind: "app_password", + now: "2026-06-10T18:00:00.000Z", + ownerSubjectId: OWNER_SUBJECT_ID, + secret: GOOD_SECRET, + }); + + // The probe legitimately reports GMAIL_ADDRESS for this new secret — + // a real, valid identity, just not provably the SAME account as the + // one the original secret belongs to (nothing durable says what that + // was). This must still be refused. + const retargeted = await capture(asUrl, cookie, connectionId, ALTERNATE_SECRET, "app_password", { + account_email: GMAIL_ADDRESS, + }); + assert.equal(retargeted.status, 409); + assert.equal(errorOf(retargeted.body).code, "static_secret_identity_unverified_replacement"); + const credentialStore = createSqliteConnectorInstanceCredentialStore(); + const secret = await credentialStore.recoverSecret({ + connectorInstanceId: connectionId, + ownerSubjectId: OWNER_SUBJECT_ID, + }); + assert.equal(secret.secret, GOOD_SECRET, "a rejected retarget must not rotate the stored credential"); + }); + }); +}); + +// Discriminator: resubmitting the SAME claimed `setup_fields` identity +// alongside a DIFFERENT secret must still be refused when no durable +// verified_identity exists. `setup_fields` is owner-typed and non-secret — +// trivially resubmittable by anyone who already knows (or guesses) the +// account email, so it can never stand in for provider-verified proof. +// No-probe variant: nothing but the fingerprint can prove sameness here. +test("resubmitting the same claimed setup-field identity with a different secret is refused (no probe)", async () => { + await withCredentialKey(TEST_KEY, async () => { + await withServer(async ({ asUrl }) => { + const manifest = syntheticNoProbeStaticSecretManifest(); + // Give this connector an owner-entered, non-secret identity field so + // `setup_fields[identity_field]` is populated — exactly the shape an + // attacker would resubmit unchanged. + const withIdentityField = { + ...manifest, + setup: { + ...manifest.setup, + credential_capture: { + ...manifest.setup.credential_capture, + fields: [ + { identity: true, label: "Account email", name: "account_email", required: true, type: "text" }, + ...manifest.setup.credential_capture.fields, + ], + }, + }, + }; + await registerManifest(asUrl, withIdentityField); + const cookie = await login(asUrl); + + const draft = await createDraft(asUrl, cookie, "ynab", { account_email: "owner@example.com" }); + const connectionId = requireString(draft.body.connection_id, "draft.body.connection_id"); + const firstCapture = await capture(asUrl, cookie, connectionId, GOOD_SECRET, "personal_access_token", { + account_email: "owner@example.com", + }); + assert.equal(firstCapture.status, 201); + + const instanceStore = createSqliteConnectorInstanceStore(); + await activateDraftConnection( + connectionId, + instanceStore as unknown as Parameters[1], + () => Promise.resolve(null) + ); + const activeInstance = await instanceStore.get(connectionId); + const activeBinding = activeInstance?.sourceBinding as { verified_identity?: string } | undefined; + assert.equal( + activeBinding?.verified_identity, + undefined, + "no probe ever ran — no durable verified_identity exists to protect this row" + ); + + // Same claimed email, DIFFERENT secret. An attacker (or the API + // itself, absent this guard) could otherwise treat the matching email + // as proof. It must not be: only the fingerprint is trusted here, and + // it does not match. + const retargeted = await capture(asUrl, cookie, connectionId, ALTERNATE_SECRET, "personal_access_token", { + account_email: "owner@example.com", + }); + assert.equal(retargeted.status, 409); + assert.equal(errorOf(retargeted.body).code, "static_secret_identity_unverified_replacement"); + + const credentialStore = createSqliteConnectorInstanceCredentialStore(); + const secret = await credentialStore.recoverSecret({ + connectorInstanceId: connectionId, + ownerSubjectId: OWNER_SUBJECT_ID, + }); + assert.equal(secret.secret, GOOD_SECRET, "a rejected replacement must not rotate the stored credential"); + }); + }); +}); + +// Same discriminator, newly-probed variant: the probe returns the SAME +// claimed identity as before (an attacker who knows the account email can +// arrange this), but there is still no durable verified_identity on record +// to compare a probed result against — only the fingerprint can prove +// sameness, and a different secret fails it. +test("resubmitting the same claimed identity via a matching probe with a different secret is refused when unverified", async () => { + await withCredentialKey(TEST_KEY, async () => { + await withServer(async ({ asUrl }) => { + const gmail = loadManifest("gmail"); + await registerManifest(asUrl, gmail); + const cookie = await login(asUrl); + const connectionId = "cin_same_claimed_identity_new_probe"; + const instanceStore = createSqliteConnectorInstanceStore(); + // Active, real pipeline shape, `setup_fields.account_email` already + // populated (so a naive "trust setup_fields" implementation would see + // a "durable" identity here) — but NO verified_identity was ever + // written, because no probe has ever succeeded against this row. + await instanceStore.upsert({ + connectorId: gmail.connector_key, + connectorInstanceId: connectionId, + createdAt: "2026-06-10T18:00:00.000Z", + displayName: "Gmail - existing@example.com", + ownerSubjectId: OWNER_SUBJECT_ID, + sourceBinding: { + kind: "static_secret", + promoted_at: "2026-06-10T18:00:00.000Z", + promoted_from: "static_secret_draft", + setup_fields: { account_email: GMAIL_ADDRESS }, + }, + sourceBindingKey: connectionId, + sourceKind: "account", + status: "active", + updatedAt: "2026-06-10T18:00:00.000Z", + }); + await createSqliteConnectorInstanceCredentialStore().capture({ + connectorInstanceId: connectionId, + credentialKind: "app_password", + now: "2026-06-10T18:00:00.000Z", + ownerSubjectId: OWNER_SUBJECT_ID, + secret: GOOD_SECRET, + }); + + // The probe returns the SAME address as the stored setup_fields — an + // attacker submitting the known account email alongside a different, + // validly-probeable credential produces exactly this shape. With no + // durable verified_identity, this must still be refused. + const retargeted = await capture(asUrl, cookie, connectionId, ALTERNATE_SECRET, "app_password", { + account_email: GMAIL_ADDRESS, + }); + assert.equal(retargeted.status, 409); + assert.equal(errorOf(retargeted.body).code, "static_secret_identity_unverified_replacement"); + + const credentialStore = createSqliteConnectorInstanceCredentialStore(); + const secret = await credentialStore.recoverSecret({ + connectorInstanceId: connectionId, + ownerSubjectId: OWNER_SUBJECT_ID, + }); + assert.equal(secret.secret, GOOD_SECRET, "a rejected retarget must not rotate the stored credential"); + }); + }); +}); diff --git a/reference-implementation/test/static-secret-draft-connection-route.test.ts b/reference-implementation/test/static-secret-draft-connection-route.test.ts index c1e55092e..6f2ccea17 100644 --- a/reference-implementation/test/static-secret-draft-connection-route.test.ts +++ b/reference-implementation/test/static-secret-draft-connection-route.test.ts @@ -259,10 +259,14 @@ async function createDraft( asUrl: string, cookie: string, connectorId: string, - setupFields: Record = { account_email: "owner@example.com" } + setupFields: Record = { account_email: "owner@example.com" }, + displayName?: string ): Promise { return fetchJson(`${asUrl}/_ref/connectors/${encodeURIComponent(connectorId)}/draft-connection`, { - body: JSON.stringify({ setup_fields: setupFields }), + body: JSON.stringify({ + setup_fields: setupFields, + ...(displayName === undefined ? {} : { display_name: displayName }), + }), headers: { Accept: "application/json", "Content-Type": "application/json", Cookie: cookie }, method: "POST", }); @@ -459,6 +463,29 @@ test("static-secret setup descriptor is manifest-authored and readiness-gated", }); }); +test("owner-selected display name is stored on the static-secret draft", async () => { + await withCredentialKey(TEST_KEY, async () => { + await withServer(async ({ asUrl }) => { + await registerConnector(asUrl, "gmail"); + const cookie = await login(asUrl); + const created = await createDraft( + asUrl, + cookie, + "gmail", + { account_email: "owner@example.com" }, + "Primary account" + ); + assert.equal(created.status, 201, created.text); + assert.equal(created.body.display_name, "Primary account"); + + const row = getDb() + .prepare("SELECT display_name FROM connector_instances WHERE connector_instance_id = ?") + .get(created.body.connection_id) as { display_name?: string } | undefined; + assert.equal(row?.display_name, "Primary account"); + }); + }); +}); + test("draft create blocks before row creation when credential key provider is missing", async () => { await withCredentialKey(null, async () => { await withServer(async ({ asUrl }) => { @@ -499,6 +526,17 @@ test("draft create validates manifest-declared non-secret setup fields", async ( assert.equal(unknown.status, 400); assert.equal(errorOf(unknown.body).code, "unknown_setup_field"); + const overlongName = await createDraft( + asUrl, + cookie, + "gmail", + { account_email: "owner@example.com" }, + "x".repeat(201) + ); + assert.equal(overlongName.status, 400); + assert.equal(errorOf(overlongName.body).code, "invalid_request"); + assert.equal(errorOf(overlongName.body).param, "display_name"); + const list = await listConnections(asUrl, cookie); assert.equal(dataArrayOf(list.body).length, 0, "invalid setup fields must not create a draft"); }); @@ -510,8 +548,8 @@ test("two drafts for one connector are two distinct connection_ids", async () => await withServer(async ({ asUrl }) => { await registerConnector(asUrl, "gmail"); const cookie = await login(asUrl); - const a = await createDraft(asUrl, cookie, "gmail"); - const b = await createDraft(asUrl, cookie, "gmail"); + const a = await createDraft(asUrl, cookie, "gmail", { account_email: "personal@example.com" }); + const b = await createDraft(asUrl, cookie, "gmail", { account_email: "work@example.com" }); assert.equal(a.status, 201); assert.equal(b.status, 201); assert.notEqual(a.body.connection_id, b.body.connection_id); @@ -562,6 +600,11 @@ test("first ingest with records flips the draft to active and makes it visible", assert.equal(created.status, 201, `draft create: ${created.text}`); const connectionId = requireString(created.body.connection_id, "created.body.connection_id"); + const preIngestRow = getDb() + .prepare("SELECT source_binding_json FROM connector_instances WHERE connector_instance_id = ?") + .get(connectionId) as { source_binding_json: string }; + assert.equal(JSON.parse(preIngestRow.source_binding_json).kind, "static_secret_draft"); + const ownerToken = await issueOwnerToken(asUrl); const ingested = await ingest(rsUrl, ownerToken, "gmail", connectionId, "messages", [ { emitted_at: "2026-06-02T12:00:00.000Z", id: "m1", subject: "hello" }, @@ -576,6 +619,40 @@ test("first ingest with records flips the draft to active and makes it visible", ); assert.ok(visible, "connection is visible after first ingest"); assert.equal(visible.status, "active"); + + // Regression coverage for the setup-shell promotion gap: first + // successful ingest must promote the binding off `static_secret_draft` + // to the durable `static_secret` kind, preserving `setup_fields` — + // not just flip status. Otherwise a LATER revoke of this real, + // fully-collected connection would wrongly hide it from Sources + // (RETIRED_SETUP_SHELL_BINDING_KINDS), exactly like the browser + // enrollment shell bug this mirrors. + const postIngestRow = getDb() + .prepare("SELECT source_binding_json FROM connector_instances WHERE connector_instance_id = ?") + .get(connectionId) as { source_binding_json: string }; + const postIngestBinding = JSON.parse(postIngestRow.source_binding_json); + assert.equal(postIngestBinding.kind, "static_secret", "binding kind moved off static_secret_draft"); + assert.deepEqual( + postIngestBinding.setup_fields, + { account_email: "owner@example.com" }, + "setup_fields survive promotion — read on every credential probe/run, not just at setup" + ); + + // Revoking this now-real, fully-collected connection must NOT hide it + // from Sources — it is an ordinary revoked connection, not retired + // setup residue. + const revoked = await fetch(`${rsUrl}/v1/owner/connections/${encodeURIComponent(connectionId)}/revoke`, { + headers: { Authorization: `Bearer ${ownerToken}`, "Content-Type": "application/json" }, + method: "POST", + }); + assert.equal(revoked.status, 200, `revoke should succeed: ${await revoked.text()}`); + + const listAfterRevoke = await listConnections(asUrl, cookie); + const visibleAfterRevoke = dataArrayOf(listAfterRevoke.body).find( + (c) => c.connection_id === connectionId || c.connector_instance_id === connectionId + ); + assert.ok(visibleAfterRevoke, "revoked-after-promotion connection stays visible on /_ref/connections"); + assert.equal(visibleAfterRevoke.status, "revoked"); }); }); }); @@ -706,7 +783,7 @@ test("setup-status resolves a draft by its exact connection_id, not by connector }); }); -test("waiting owner action: credential captured but no ingest yet stays setup_in_progress on /_ref/connectors, not healthy or degraded", async () => { +test("credential captured with first sync active reads collecting on /_ref/connectors, not healthy or degraded", async () => { await withCredentialKey(TEST_KEY, async () => { await withServer(async ({ asUrl }) => { await registerConnector(asUrl, "gmail"); @@ -731,7 +808,7 @@ test("waiting owner action: credential captured but no ingest yet stays setup_in ); assert.ok(row, "draft with a captured credential but no run yet must still be discoverable"); assert.equal(row.status, "draft"); - assert.equal(ownerStateOf(row)?.resolver, "setup_in_progress"); + assert.equal(ownerStateOf(row)?.resolver, "collecting"); assert.notEqual(ownerStateOf(row)?.resolver, "healthy"); assert.notEqual(ownerStateOf(row)?.resolver, "system_degraded"); }); diff --git a/reference-implementation/test/static-secret-owner-capture-route.test.ts b/reference-implementation/test/static-secret-owner-capture-route.test.ts index 3dcddf0c8..512818473 100644 --- a/reference-implementation/test/static-secret-owner-capture-route.test.ts +++ b/reference-implementation/test/static-secret-owner-capture-route.test.ts @@ -189,11 +189,13 @@ async function seedInstance({ connectorId, ownerSubjectId = OWNER_SUBJECT_ID, displayName, + setupFields = {}, }: { connectorInstanceId: string; connectorId: string; ownerSubjectId?: string; displayName?: string; + setupFields?: Record; }): Promise { const store = createSqliteConnectorInstanceStore(); await store.upsert({ @@ -202,7 +204,7 @@ async function seedInstance({ createdAt: NOW, displayName: displayName ?? connectorInstanceId, ownerSubjectId, - sourceBinding: { account_hint: connectorInstanceId }, + sourceBinding: { account_hint: connectorInstanceId, kind: "static_secret", setup_fields: setupFields }, sourceBindingKey: connectorInstanceId, sourceKind: "account", status: "active", @@ -313,8 +315,16 @@ test("capture is per-connection and rotation preserves the connection id", async await withCredentialKey(TEST_KEY, async () => { await withServer(async ({ asUrl }) => { await registerConnector(asUrl, "gmail"); - await seedInstance({ connectorId: "gmail", connectorInstanceId: "cin_gmail_personal" }); - await seedInstance({ connectorId: "gmail", connectorInstanceId: "cin_gmail_work" }); + await seedInstance({ + connectorId: "gmail", + connectorInstanceId: "cin_gmail_personal", + setupFields: { account_email: "personal@example.com" }, + }); + await seedInstance({ + connectorId: "gmail", + connectorInstanceId: "cin_gmail_work", + setupFields: { account_email: "work@example.com" }, + }); const cookie = await login(asUrl); const first = await captureCredential(asUrl, cookie, "cin_gmail_personal", PERSONAL_SECRET); diff --git a/reference-implementation/test/static-secret-setup-status-projection.test.ts b/reference-implementation/test/static-secret-setup-status-projection.test.ts index 58a04d3b5..674537a10 100644 --- a/reference-implementation/test/static-secret-setup-status-projection.test.ts +++ b/reference-implementation/test/static-secret-setup-status-projection.test.ts @@ -85,12 +85,145 @@ test("draft with a failed last run projects first_sync_failed -> needs_attention }); assert.equal(status.setup_state, "first_sync_failed"); assert.equal(status.health_state, "needs_attention"); + assert.equal(status.pending, false); assert.ok(status.last_error); assert.equal(status.last_error.reason, "authentication_failed"); // biome-ignore lint/performance/useTopLevelRegex: localized test assertion preserves its explicit contract. assert.match(status.last_error.remediation, /credential/i); }); +test("a completed zero-yield first run is terminal and explicitly unverified", () => { + const status = projectStaticSecretSetupStatus({ + activeRun: null, + credential: { capturedAt: null, credentialKind: "app_password", present: true }, + identityFieldName: "account_email", + instance: baseInstance, + lastRun: { + finishedAt: "2026-06-10T00:04:00.000Z", + recordsEmitted: 0, + reportedRecordsEmitted: 0, + runId: "run_zero_yield", + status: "completed", + }, + }); + assert.equal(status.setup_state, "first_sync_unverified_zero"); + assert.equal(status.health_state, "needs_attention"); + assert.equal(status.pending, false); + assert.equal(status.terminal_setup_disposition, "unverified_zero"); + assert.equal(status.running, false); + assert.equal(status.run?.records_emitted, 0); + assert.equal(status.run?.reported_records_emitted, 0); +}); + +test("terminal collection facts distinguish a verified empty result from zero-count silence", () => { + const status = projectStaticSecretSetupStatus({ + activeRun: null, + credential: { present: true }, + identityFieldName: "account_email", + instance: baseInstance, + lastRun: { + collectionFacts: { + streams: [ + { + checkpoint: "committed", + considered: 0, + covered: 0, + pending_detail_gaps: 0, + skipped: null, + stream: "messages", + }, + ], + }, + recordsEmitted: 0, + reportedRecordsEmitted: 0, + runId: "run_verified_empty", + status: "succeeded", + yieldCountsPresent: true, + }, + manifestStreams: [{ name: "messages" }], + }); + + assert.equal(status.setup_state, "first_sync_verified_empty"); + assert.equal(status.health_state, "needs_attention"); + assert.equal(status.pending, false); + assert.equal(status.terminal_setup_disposition, "verified_empty"); + assert.equal(status.last_error, null); +}); + +test("incomplete collection facts cannot prove a verified empty result", () => { + const status = projectStaticSecretSetupStatus({ + activeRun: null, + credential: { present: true }, + identityFieldName: "account_email", + instance: baseInstance, + lastRun: { + collectionFacts: { + streams: [ + { + checkpoint: "committed", + considered: 0, + covered: 0, + pending_detail_gaps: 0, + skipped: null, + stream: "messages", + }, + ], + }, + recordsEmitted: 0, + reportedRecordsEmitted: 0, + runId: "run_incomplete_empty_facts", + status: "succeeded", + yieldCountsPresent: true, + }, + manifestStreams: [{ name: "messages" }, { name: "threads" }], + }); + + assert.equal(status.setup_state, "first_sync_unverified_zero"); + assert.equal(status.terminal_setup_disposition, "unverified_zero"); +}); + +test("a terminal zero without collection facts remains unverified", () => { + const status = projectStaticSecretSetupStatus({ + activeRun: null, + credential: { present: true }, + identityFieldName: "account_email", + instance: baseInstance, + lastRun: { + recordsEmitted: 0, + reportedRecordsEmitted: 0, + runId: "run_silent_zero", + status: "succeeded", + yieldCountsPresent: true, + }, + manifestStreams: [{ name: "messages" }], + }); + + assert.equal(status.setup_state, "first_sync_unverified_zero"); + assert.equal(status.terminal_setup_disposition, "unverified_zero"); + assert.equal(status.last_error?.reason, "first_sync_unverified_zero"); +}); + +test("a terminal run with omitted yield counts is distinct from an observed zero", () => { + const status = projectStaticSecretSetupStatus({ + activeRun: null, + credential: { present: true }, + identityFieldName: "account_email", + instance: baseInstance, + lastRun: { + recordsEmitted: null, + reportedRecordsEmitted: null, + runId: "run_missing_counts", + status: "succeeded", + yieldCountsPresent: false, + }, + manifestStreams: [{ name: "messages" }], + }); + + assert.equal(status.setup_state, "first_sync_unverified_missing_counts"); + assert.equal(status.terminal_setup_disposition, "unverified_missing_counts"); + assert.equal(status.last_error?.reason, "first_sync_unverified_missing_counts"); +}); + test("active instance projects active -> healthy and not pending", () => { const status = projectStaticSecretSetupStatus({ activeRun: null, @@ -102,6 +235,7 @@ test("active instance projects active -> healthy and not pending", () => { assert.equal(status.setup_state, "active"); assert.equal(status.health_state, "healthy"); assert.equal(status.pending, false); + assert.equal(status.terminal_setup_disposition, null); }); test("credential rotation metadata stays visible on setup status", () => { @@ -194,7 +328,7 @@ test("browser-session draft with an active run projects first_sync_running even assert.equal(status.setup_material.present, false); }); -test("browser-session draft with a completed last run (no active run) projects first_sync_pending, not awaiting_browser_login", () => { +test("browser-session draft with terminal counts missing projects unverified setup, not awaiting_browser_login", () => { const status = projectConnectionSetupStatus({ activeRun: null, credential: null, @@ -203,7 +337,9 @@ test("browser-session draft with a completed last run (no active run) projects f lastRun: { failureReason: null, runId: "run_browser_2", status: "completed" }, setupKind: "browser_session", }); - assert.equal(status.setup_state, "first_sync_pending"); + assert.equal(status.setup_state, "first_sync_unverified_missing_counts"); + assert.equal(status.terminal_setup_disposition, "unverified_missing_counts"); + assert.equal(status.pending, false); }); test("browser-session draft with a failed last run projects first_sync_failed with browser-safe remediation", () => { diff --git a/reference-implementation/test/static-secret-setup-status-route.test.ts b/reference-implementation/test/static-secret-setup-status-route.test.ts index 600922ed1..7a7f3f4a0 100644 --- a/reference-implementation/test/static-secret-setup-status-route.test.ts +++ b/reference-implementation/test/static-secret-setup-status-route.test.ts @@ -202,6 +202,24 @@ function loadManifest(name: string): ConnectorManifest { ) as ConnectorManifest; } +function verifiedEmptyCollectionFacts(connectorName: string): Record { + const manifest = loadManifest(connectorName); + const streams = Array.isArray(manifest.streams) + ? (manifest.streams as Array<{ name?: unknown; required?: unknown }>) + .filter((stream) => typeof stream.name === "string" && stream.name.length > 0) + .map((stream) => ({ + checkpoint: "committed", + collected: 0, + considered: 0, + covered: 0, + pending_detail_gaps: 0, + skipped: null, + stream: stream.name, + })) + : []; + return { collection_facts: { streams } }; +} + const VALID_TIMELINE_BODY = JSON.stringify({ locations: [ { @@ -348,11 +366,17 @@ function clearActiveRun(connectorInstanceId: string): void { getDb().prepare("DELETE FROM controller_active_runs WHERE connector_instance_id = ?").run(connectorInstanceId); } -async function emitTerminalRunEvent(connectorId: string, runId: string, status: string): Promise { +async function emitTerminalRunEvent( + connectorId: string, + runId: string, + status: string, + data: Readonly> = {}, + connectorInstanceId = `cin_${connectorId}` +): Promise { await emitSpineEvent({ actor_id: connectorId, actor_type: "runtime", - data: { source: { id: connectorId, kind: "connector" } }, + data: { connector_instance_id: connectorInstanceId, source: { id: connectorId, kind: "connector" }, ...data }, event_type: status === "failed" ? "run.failed" : "run.completed", object_id: runId, object_type: "run", @@ -368,14 +392,15 @@ async function emitTerminalRunEvent(connectorId: string, runId: string, status: async function emitStartedRunEvent( connectorId: string, runId: string, - occurredAt = "2026-06-10T00:02:00.000Z" + occurredAt = "2026-06-10T00:02:00.000Z", + connectorInstanceId = `cin_${connectorId}` ): Promise { await emitSpineEvent({ actor_id: connectorId, actor_type: "runtime", data: { boot_epoch: "11111111-1111-4111-8111-111111111111", - connector_instance_id: `cin_${connectorId}`, + connector_instance_id: connectorInstanceId, seq: 1, source: { id: connectorId, kind: "connector" }, }, @@ -463,6 +488,96 @@ test("pending static-secret setup is visible before any records are accepted", a assert.ok(!running.text.includes(SECRET), "status must not echo the secret"); assert.ok(!afterCapture.text.includes(SECRET), "status must not echo the secret"); clearActiveRun(connectionId); + + await emitTerminalRunEvent( + "gmail", + "run_status_zero_yield", + "succeeded", + { + records_emitted: 0, + reported_records_emitted: 0, + }, + connectionId + ); + const zeroYield = await getStatus(asUrl, cookie, connectionId, "run_status_zero_yield"); + assert.equal(zeroYield.status, 200, zeroYield.text); + assert.equal(zeroYield.body.setup_state, "first_sync_unverified_zero"); + assert.equal(zeroYield.body.health_state, "needs_attention"); + assert.equal(zeroYield.body.pending, false); + assert.equal(zeroYield.body.terminal_setup_disposition, "unverified_zero"); + assert.equal(subObject(zeroYield.body, "run").records_emitted, 0); + assert.equal(subObject(zeroYield.body, "run").reported_records_emitted, 0); + assert.equal( + getDb().prepare("SELECT 1 FROM connector_schedules WHERE connector_instance_id = ?").get(connectionId), + undefined, + "zero-yield draft must remain unscheduled" + ); + + // Revisit without a run_id resolves the latest terminal row through the + // connection-scoped run-history reader, rather than falling back to an + // unscoped spine lookup or returning to first_sync_pending. + const revisited = await getStatus(asUrl, cookie, connectionId); + assert.equal(revisited.body.setup_state, "first_sync_unverified_zero"); + assert.equal(revisited.body.terminal_setup_disposition, "unverified_zero"); + + const summaries = await listRefConnectors(asUrl, cookie); + assert.equal(summaries.status, 200, summaries.text); + const summary = dataArrayOf(summaries.body).find((item) => item.connection_id === connectionId); + assert.ok(summary, "draft terminal setup should remain owner-visible in connector summaries"); + assert.equal(summary.terminal_setup_disposition, "unverified_zero"); + }); + }); +}); + +// fr-setup-status-lifecycle-0806: Slack/YNAB setup read "First sync pending" +// and never advanced without a manual "Refresh Status" click. Root cause: the +// route discarded run-history evidence outright whenever it read status +// `"running"` and no `controller_active_runs` row existed yet for the +// connection — a real, reachable window between `run.started` writing the +// `run_history` row (`status: 'running'`) and the controller's active-run +// table row landing (or after it clears, before the terminal write commits). +// Every subsequent poll re-derived the same stale `first_sync_pending` +// forever, because the discarded evidence meant there was nothing to +// converge on until the run went fully terminal. +test("first sync in-flight evidence from run_history alone (no active-run row yet) reads first_sync_running, not stuck first_sync_pending", async () => { + await withCredentialKey(TEST_KEY, async () => { + await withServer(async ({ asUrl }) => { + await registerConnector(asUrl, "gmail"); + const cookie = await login(asUrl); + + const created = await createDraft(asUrl, cookie, "gmail", { account_email: "inflight@example.com" }); + assert.equal(created.status, 201); + const connectionId = requireString(created.body.connection_id, "created.body.connection_id"); + + const captured = await capture(asUrl, cookie, connectionId); + assert.equal(captured.status, 201, captured.text); + + // `run.started` writes a `run_history` row with status "running" — + // deliberately WITHOUT seeding `controller_active_runs`, modeling the + // window where the active-run table has no row for this connection yet + // (or no longer does) while the history row still legitimately reads + // "running". + await emitStartedRunEvent("gmail", "run_status_inflight_no_active_row", undefined, connectionId); + + const status = await getStatus(asUrl, cookie, connectionId); + assert.equal(status.status, 200, status.text); + assert.notEqual( + status.body.setup_state, + "first_sync_pending", + "an in-flight run must never read as a stuck first_sync_pending" + ); + assert.equal(status.body.setup_state, "first_sync_running"); + assert.equal(status.body.running, true); + assert.equal(status.body.pending, true); + assert.equal(subObject(status.body, "run").run_id, "run_status_inflight_no_active_row"); + assert.equal(subObject(status.body, "run").status, "running"); + + // Revisiting (the poller's own re-derivation, not a manual refresh + // click) must keep reading the same correct running state — never + // regress to first_sync_pending on a later read of the same evidence. + const revisited = await getStatus(asUrl, cookie, connectionId); + assert.equal(revisited.body.setup_state, "first_sync_running"); + assert.equal(revisited.body.running, true); }); }); }); @@ -508,6 +623,69 @@ test("pending manual/upload setup is visible without credential semantics", asyn }); }); +test("terminal setup evidence is composite connection/run scoped and survives revisit without run_id", async () => { + await withCredentialKey(TEST_KEY, async () => { + await withServer(async ({ asUrl }) => { + await registerConnector(asUrl, "gmail"); + const cookie = await login(asUrl); + const first = await createDraft(asUrl, cookie, "gmail", { account_email: "first@example.com" }); + const second = await createDraft(asUrl, cookie, "gmail", { account_email: "second@example.com" }); + const third = await createDraft(asUrl, cookie, "gmail", { account_email: "third@example.com" }); + const firstConnectionId = requireString(first.body.connection_id, "first.body.connection_id"); + const secondConnectionId = requireString(second.body.connection_id, "second.body.connection_id"); + const thirdConnectionId = requireString(third.body.connection_id, "third.body.connection_id"); + await capture(asUrl, cookie, firstConnectionId); + await capture(asUrl, cookie, secondConnectionId); + await capture(asUrl, cookie, thirdConnectionId); + + const duplicateRunId = "run_duplicate_connection_scope"; + await emitTerminalRunEvent( + "gmail", + duplicateRunId, + "succeeded", + { records_emitted: 0, reported_records_emitted: 0 }, + firstConnectionId + ); + await emitTerminalRunEvent( + "gmail", + duplicateRunId, + "succeeded", + { ...verifiedEmptyCollectionFacts("gmail"), records_emitted: 0, reported_records_emitted: 0 }, + secondConnectionId + ); + await emitTerminalRunEvent("gmail", duplicateRunId, "succeeded", {}, thirdConnectionId); + + const firstRevisit = await getStatus(asUrl, cookie, firstConnectionId); + const secondRevisit = await getStatus(asUrl, cookie, secondConnectionId); + const thirdRevisit = await getStatus(asUrl, cookie, thirdConnectionId); + assert.equal(firstRevisit.body.terminal_setup_disposition, "unverified_zero"); + assert.equal(firstRevisit.body.setup_state, "first_sync_unverified_zero"); + assert.equal(secondRevisit.body.terminal_setup_disposition, "verified_empty"); + assert.equal(secondRevisit.body.setup_state, "first_sync_verified_empty"); + assert.equal(thirdRevisit.body.terminal_setup_disposition, "unverified_missing_counts"); + assert.equal(thirdRevisit.body.setup_state, "first_sync_unverified_missing_counts"); + + // The explicit run_id override remains fenced by the addressed + // connector_instance_id even when both connections share the run id. + const firstExact = await getStatus(asUrl, cookie, firstConnectionId, duplicateRunId); + const secondExact = await getStatus(asUrl, cookie, secondConnectionId, duplicateRunId); + const thirdExact = await getStatus(asUrl, cookie, thirdConnectionId, duplicateRunId); + assert.equal(firstExact.body.terminal_setup_disposition, "unverified_zero"); + assert.equal(secondExact.body.terminal_setup_disposition, "verified_empty"); + assert.equal(thirdExact.body.terminal_setup_disposition, "unverified_missing_counts"); + + const summaries = await listRefConnectors(asUrl, cookie); + assert.equal(summaries.status, 200, summaries.text); + const firstSummary = dataArrayOf(summaries.body).find((item) => item.connection_id === firstConnectionId); + const secondSummary = dataArrayOf(summaries.body).find((item) => item.connection_id === secondConnectionId); + const thirdSummary = dataArrayOf(summaries.body).find((item) => item.connection_id === thirdConnectionId); + assert.equal(firstSummary?.terminal_setup_disposition, "unverified_zero"); + assert.equal(secondSummary?.terminal_setup_disposition, "verified_empty"); + assert.equal(thirdSummary?.terminal_setup_disposition, "unverified_missing_counts"); + }); + }); +}); + test("a failed first sync is visible with an actionable error and no secret leak", async () => { await withCredentialKey(TEST_KEY, async () => { await withServer(async ({ asUrl }) => { @@ -521,14 +699,14 @@ test("a failed first sync is visible with an actionable error and no secret leak // The run terminated as failed (no active-run row remains). The owner // surface holds the run id; the route resolves its terminal status. const runId = "run_status_failed"; - await emitTerminalRunEvent("gmail", runId, "failed"); + await emitTerminalRunEvent("gmail", runId, "failed", {}, connectionId); const failed = await getStatus(asUrl, cookie, connectionId, runId); assert.equal(failed.status, 200, failed.text); assert.equal(failed.body.status, "draft"); assert.equal(failed.body.setup_state, "first_sync_failed"); assert.equal(failed.body.health_state, "needs_attention"); - assert.equal(failed.body.pending, true); + assert.equal(failed.body.pending, false); assert.equal(failed.body.running, false); assert.ok(failed.body.last_error, "failed first sync must carry last_error"); assert.equal(typeof subObject(failed.body, "last_error").reason, "string"); @@ -565,6 +743,11 @@ test("setup status flips to active once first ingest accepts records", async () assert.equal(active.body.setup_state, "active"); assert.equal(active.body.health_state, "healthy"); assert.equal(active.body.pending, false); + // Promotion (server/index.ts SETUP_BINDING_PROMOTIONS) moved the + // binding from static_secret_draft to static_secret on this same + // ingest; setup_kind must resolve from the promoted binding, not + // fall through to the manifest-only legacy classifier. + assert.equal(active.body.setup_kind, "static_secret"); const rotated = await capture(asUrl, cookie, connectionId); assert.equal(rotated.status, 200, rotated.text); @@ -592,8 +775,8 @@ test("setup status flips to active once first ingest accepts records", async () clearActiveRun(connectionId); const failedRunId = "run_status_credential_rotation_failed"; - await emitStartedRunEvent("gmail", failedRunId, "9999-01-01T00:00:00.000Z"); - await emitTerminalRunEvent("gmail", failedRunId, "failed"); + await emitStartedRunEvent("gmail", failedRunId, "9999-01-01T00:00:00.000Z", connectionId); + await emitTerminalRunEvent("gmail", failedRunId, "failed", {}, connectionId); const failedVerification = await getStatus(asUrl, cookie, connectionId, failedRunId); assert.equal(failedVerification.status, 200, failedVerification.text); assert.equal(failedVerification.body.status, "active"); @@ -692,6 +875,9 @@ test("manual/upload setup status shows committed acquisition-batch counts after assert.equal(active.status, 200, active.text); assert.equal(active.body.status, "active"); assert.equal(active.body.setup_state, "active"); + // Promotion moved the binding from manual_upload_draft to manual_upload + // on this ingest; setup_kind must resolve from the promoted binding. + assert.equal(active.body.setup_kind, "manual_upload"); assert.equal(subObject(active.body, "import_receipt").status, "committed"); assert.equal(subObject(active.body, "import_receipt").parsed_count, 1); assert.equal(subObject(active.body, "import_receipt").accepted_count, 1); @@ -839,11 +1025,12 @@ test("ChatGPT browser-enrollment-shell draft is classified browser_session, not // A failed first sync on a browser-session connection gets a browser-safe // remediation, never "re-enter the provider credential". const runId = "run_browser_status_failed"; - await emitTerminalRunEvent("chatgpt", runId, "failed"); + await emitTerminalRunEvent("chatgpt", runId, "failed", {}, connectionId); const failed = await getStatus(asUrl, cookie, connectionId, runId); assert.equal(failed.status, 200, failed.text); assert.equal(failed.body.setup_kind, "browser_session"); assert.equal(failed.body.setup_state, "first_sync_failed"); + assert.equal(failed.body.pending, false); assert.ok(failed.body.last_error, "failed first sync must carry last_error"); const remediation = String(subObject(failed.body, "last_error").remediation); assert.doesNotMatch(remediation, NO_BROWSER_CREDENTIAL_REMEDIATION); diff --git a/scripts/check-owner-journey-acceptance.test.ts b/scripts/check-owner-journey-acceptance.test.ts index 1e684788f..2f6ce830e 100644 --- a/scripts/check-owner-journey-acceptance.test.ts +++ b/scripts/check-owner-journey-acceptance.test.ts @@ -505,6 +505,41 @@ test("live probe can create an owner session from PDPP_OWNER_PASSWORD and scan a ); }); +test("live Explore render fails when only one sort direction is present", async () => { + const response = ( + status: number, + body: string + ): { headers: { get: () => null }; status: number; text: () => Promise } => ({ + headers: { get: () => null }, + status, + text: () => Promise.resolve(body), + }); + // biome-ignore lint/suspicious/useAwait: fetchImpl models the async fetch contract for the live harness. + const fetchImpl = async (url: string | URL) => { + const href = String(url); + if (href.includes("/_ref/connectors")) { + return response(200, JSON.stringify({ data: [], has_more: false, object: "list" })); + } + if (href.endsWith("/explore")) { + return response( + 200, + "

    Explore

    Filters
    " + ); + } + return response(200, defaultLiveOwnerPageHtml(url)); + }; + + const result = await runLiveAcceptance({ + env: { PDPP_OWNER_SESSION_COOKIE: "sid=secret" }, + fetchImpl, + origin: "https://example.com/", + }); + + assert.equal(result.ok, false, "a one-sided sort control must fail the rendered acceptance gate"); + assert.ok(result.findings.some((finding) => finding.ruleId === "explore-content-rendered")); + assert.equal(result.semanticChecks.find((check) => check.id === "explore-content-rendered")?.status, "fail"); +}); + test("live semantic probe requests connectors at limit=100 (the reference's own page-size ceiling), never the invalid limit=200", async () => { const urlsSeen: string[] = []; const response = (status: number, body: string, setCookie: string | null = null) => ({ @@ -1118,6 +1153,54 @@ test("live semantic probe accepts visible source count claims that match connect assert.equal(result.semanticChecks.find((check) => check.id === "records-counts-match-reality")?.status, "pass"); }); +test("live semantic probe compares the configured stream roster for a fresh draft", async () => { + const response = ( + status: number, + body: string + ): { headers: { get: () => null }; status: number; text: () => Promise } => ({ + status, + headers: { get: () => null }, + text: () => Promise.resolve(body), + }); + // biome-ignore lint/suspicious/useAwait: fetchImpl must satisfy the async FetchImpl contract even though this mock resolves synchronously; the caller awaits it like real fetch. + const fetchImpl = async (url: string | URL) => { + const href = String(url); + if (href.includes("/_ref/connectors")) { + return response( + 200, + JSON.stringify({ + object: "list", + has_more: false, + data: [ + { + connection_id: "cin_chatgpt_draft", + connector_id: "chatgpt", + display_name: "ChatGPT", + status: "draft", + stream_count: 0, + streams: ["conversations", "messages", "attachments"], + total_records: 0, + }, + ], + }) + ); + } + if (href.endsWith("/sources")) { + return response(200, "
    "); + } + return response(200, defaultLiveOwnerPageHtml(url)); + }; + + const result = await runLiveAcceptance({ + origin: "https://example.com", + env: { PDPP_OWNER_SESSION_COOKIE: "sid=secret" }, + fetchImpl, + }); + + assert.equal(result.ok, true); + assert.equal(result.semanticChecks.find((check) => check.id === "records-counts-match-reality")?.status, "pass"); +}); + test("live semantic probe rejects direct browser-session new-source controls", async () => { const response = ( status: number, diff --git a/scripts/dev.ts b/scripts/dev.ts index b86f3f400..a0ab2092d 100644 --- a/scripts/dev.ts +++ b/scripts/dev.ts @@ -59,6 +59,10 @@ const env = { ...process.env, PDPP_WEB_PORT: String(webPort), PDPP_REFERENCE_ORIGIN: process.env.PDPP_REFERENCE_ORIGIN ?? `http://localhost:${webPort}`, + // The local console and reference server run from this checkout, where the + // merged-timeline read contract supports direction=asc. Preserve an explicit + // =0 override for testing an unsupported backend honestly. + PDPP_EXPLORE_TIMELINE_DIRECTION: process.env.PDPP_EXPLORE_TIMELINE_DIRECTION ?? "1", }; console.error(`[pdpp dev] console origin: ${env.PDPP_REFERENCE_ORIGIN}`); diff --git a/scripts/docker-slackdump-core-bundle.test.ts b/scripts/docker-slackdump-core-bundle.test.ts new file mode 100644 index 000000000..14d246a1b --- /dev/null +++ b/scripts/docker-slackdump-core-bundle.test.ts @@ -0,0 +1,191 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +// Integration test: verify core Docker image includes slackdump v4.4.2. +// Accepts an explicitly provided image tag (from CI) or builds locally for dev. +// +// Rationale: Slack connector is declared as "background_safe" only if slackdump +// is present. This test ensures the default deployment image actually includes it. +// Uses structural assertions (Go/toolchain/build-deps absence) rather than +// brittle absolute size bounds, which are host/architecture-sensitive. + +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { test } from "node:test"; + +const DOCKER_CMD = "docker"; +const DOCKERFILE_PATH = "./Dockerfile"; +// Accept image tag from ENV (set by CI) or build locally for dev +const CORE_IMAGE_TAG = process.env.PDPP_CORE_IMAGE_TAG || "pdpp:test-core-slackdump"; +const CI_PROVIDED_IMAGE = !!process.env.PDPP_CORE_IMAGE_TAG; + +// Check Docker availability before running. +function isDockerAvailable(): boolean { + try { + execFileSync(DOCKER_CMD, ["--version"], { timeout: 2000, stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +const skipIfNoDocker = isDockerAvailable() ? test : test.skip; + +skipIfNoDocker("Inspect core image: slackdump bundling, license, and builder bloat", async (t) => { + if (!CI_PROVIDED_IMAGE) { + t.diagnostic("Building core Docker image locally (not in CI)..."); + // Only build locally for dev; CI provides pre-built image via load: true + try { + execFileSync(DOCKER_CMD, [ + "build", + "--target", + "core", + "-t", + CORE_IMAGE_TAG, + "-f", + DOCKERFILE_PATH, + ".", + ]); + t.diagnostic("core image built successfully"); + } catch (err) { + t.diagnostic(`Build failed: ${err}`); + throw err; + } + } else { + t.diagnostic(`Using CI-provided image: ${CORE_IMAGE_TAG}`); + } + + // Sequential assertions: all must complete before parent test finishes + // (avoid early exit that leaves subtests orphaned) + + // Test 1: slackdump binary exists and is executable + await t.test("slackdump binary is executable", () => { + try { + execFileSync(DOCKER_CMD, [ + "run", + "--rm", + CORE_IMAGE_TAG, + "test", + "-x", + "/usr/local/bin/slackdump", + ]); + t.diagnostic("✓ slackdump binary found at /usr/local/bin/slackdump"); + } catch (err) { + throw new Error(`slackdump binary check failed: ${err}`); + } + }); + + // Test 2: slackdump version check passes + await t.test("slackdump version returns success", () => { + try { + const output = execFileSync(DOCKER_CMD, [ + "run", + "--rm", + CORE_IMAGE_TAG, + "slackdump", + "version", + ], { encoding: "utf8" }); + + assert( + output.includes("Slackdump") && output.match(/4\.4\.\d+/), + `slackdump version output should contain Slackdump 4.4.x; got: ${output.trim()}` + ); + const versionMatch = output.match(/Slackdump [\d.]+/); + t.diagnostic(`✓ slackdump version: ${versionMatch ? versionMatch[0] : "unknown"}`); + } catch (err) { + throw new Error(`slackdump version check failed: ${err}`); + } + }); + + // Test 3: AGPL-3.0 license file is present + await t.test("AGPL-3.0 license file present", () => { + try { + const licenseText = execFileSync(DOCKER_CMD, [ + "run", + "--rm", + CORE_IMAGE_TAG, + "head", + "-1", + "/usr/local/share/slackdump/LICENSE.agpl-3.0.txt", + ], { encoding: "utf8" }); + + assert(licenseText.length > 0, "License file should have content"); + assert( + licenseText.includes("GNU AFFERO") || licenseText.includes("AGPL"), + "License should reference AGPL" + ); + t.diagnostic("✓ AGPL-3.0 license file present"); + } catch (err) { + throw new Error(`License file check failed: ${err}`); + } + }); + + // Test 4: Upstream source URL reference is exact and valid + // AGPL section 6(d): Corresponding Source must be resolvable tree/archive URL + await t.test("Upstream source URL reference exact and resolvable", () => { + try { + const sourceUrl = execFileSync(DOCKER_CMD, [ + "run", + "--rm", + CORE_IMAGE_TAG, + "cat", + "/usr/local/share/slackdump/SOURCE_URL", + ], { encoding: "utf8" }).trim(); + + const expectedUrl = "https://github.com/rusq/slackdump/tree/v4.4.2"; + assert( + sourceUrl === expectedUrl, + `SOURCE_URL must be exact versioned tree URL; expected ${expectedUrl}, got ${sourceUrl}` + ); + t.diagnostic(`✓ Upstream source URL: ${sourceUrl}`); + } catch (err) { + throw new Error(`Source URL check failed: ${err}`); + } + }); + + // Test 5: Builder-stage isolation check: Go toolchain absent + // Rationale: slackdump-builder stage includes Go compiler (~600MB). If COPY --from + // is missing or misconfigured, Go would leak into final image. Absence of Go + // toolchain verifies builder-stage isolation is correct. + await t.test("Go toolchain not present (builder-stage isolation verified)", () => { + try { + try { + execFileSync(DOCKER_CMD, [ + "run", + "--rm", + CORE_IMAGE_TAG, + "which", + "go", + ]); + throw new Error("go toolchain found in final image (builder-stage isolation failed)"); + } catch (err) { + if (err.message.includes("builder-stage isolation")) throw err; + // Expected: go not found + } + + t.diagnostic("✓ Go toolchain absent (builder-stage isolation verified)"); + } catch (err) { + throw new Error(`Builder isolation check failed: ${err}`); + } + }); + + // Test 6: Record measured image size for evidence (informational only, not a gate) + await t.test("Record image size for evidence", () => { + try { + const sizeOutput = execFileSync(DOCKER_CMD, [ + "image", + "inspect", + CORE_IMAGE_TAG, + "--format={{.Size}}", + ], { encoding: "utf8" }); + + const sizeBytes = parseInt(sizeOutput.trim()); + const sizeMB = sizeBytes / (1024 * 1024); + + t.diagnostic(`✓ Measured image size: ${sizeMB.toFixed(1)}MB`); + t.diagnostic(" (Structural bloat checks passed; size is informational only)"); + } catch (err) { + throw new Error(`Image size measurement failed: ${err}`); + } + }); +}); diff --git a/scripts/friend-journey-acceptance.ts b/scripts/friend-journey-acceptance.ts index bfde5f1db..bbb6a2b09 100644 --- a/scripts/friend-journey-acceptance.ts +++ b/scripts/friend-journey-acceptance.ts @@ -652,13 +652,19 @@ async function main(argv: string[]): Promise { }); await runCheck(checks, "data.explore", "Collected data appears in the owner UI", async () => { - const body = await requireStatus(context, "/explore", 200); + // Seed timestamps are stable fixtures; search proves owner-UI visibility without depending on unrelated volume history. const expected = ["Deploy Test Quartet", "Restart Survival Band"]; - const missing = expected.filter((value) => !body.text.includes(value)); + const results = await Promise.all( + expected.map(async (value) => ({ + body: await requireStatus(context, `/explore?q=${encodeURIComponent(value)}`, 200), + value, + })) + ); + const missing = results.filter(({ body, value }) => !body.text.includes(value)).map(({ value }) => value); if (missing.length > 0) { - throw new GateBlocker(`/explore did not render seeded data: missing ${missing.join(", ")}`); + throw new GateBlocker(`/explore search did not render seeded data: missing ${missing.join(", ")}`); } - return "Owner /explore rendered both durable seed records"; + return "Owner /explore search rendered both durable seed records"; }); await runCheck(checks, "semantic.search", "Semantic search returns a semantic result", async () => { diff --git a/scripts/owner-journey-acceptance/live.ts b/scripts/owner-journey-acceptance/live.ts index 0c4d3f7a4..801c1c8a6 100644 --- a/scripts/owner-journey-acceptance/live.ts +++ b/scripts/owner-journey-acceptance/live.ts @@ -251,8 +251,12 @@ function connectorLabel(connector: Connector): string { function sourceCountPhrase(connector: Connector): string | null { const records = Number(connector.total_records); - const rawStreamCount = - connector.stream_count ?? (Array.isArray(connector.streams) ? (connector.streams as unknown[]).length : null); + // Sources shows the manifest-declared stream roster. `stream_count` is a + // different protocol fact: streams with retained evidence, which is + // legitimately zero for a fresh draft whose declared streams are visible. + const rawStreamCount = Array.isArray(connector.streams) + ? (connector.streams as unknown[]).length + : connector.stream_count; const streams = Number(rawStreamCount); if (!(Number.isFinite(records) && Number.isFinite(streams))) { return null; diff --git a/scripts/perf/bench.ts b/scripts/perf/bench.ts index a2430d91b..e3da7c165 100644 --- a/scripts/perf/bench.ts +++ b/scripts/perf/bench.ts @@ -138,12 +138,15 @@ export const PAGE_TARGETS = [ { route: "/", marker: 'class="rr-stand"' }, // This source-specific ownership copy is a semantic route marker, unlike // the heading's generated style/class ordering which changes legitimately. - { route: "/sources", marker: "your loading dock · each source pushes into your streams · nothing leaves" }, + { + route: "/sources", + marker: "Sources populate streams in this instance. Connected apps read only what a grant allows.", + }, { route: "/sources/add", marker: '

    Add source

    ' }, { route: "/explore", marker: 'aria-label="Search or filter"' }, { route: "/syncs", marker: '

    Syncs

    ' }, { route: "/grants", marker: '

    Grants

    ' }, - { route: "/connect", marker: '

    Connect AI apps

    ' }, + { route: "/connect", marker: '

    Connect apps

    ' }, { route: "/search", marker: '

    Jump to artifact

    ' }, ] as const; diff --git a/scripts/test-accounting/inventory.test.ts b/scripts/test-accounting/inventory.test.ts index dc1c63507..6e1ea4284 100644 --- a/scripts/test-accounting/inventory.test.ts +++ b/scripts/test-accounting/inventory.test.ts @@ -650,6 +650,26 @@ test("keeps every terminal-LIST PostgreSQL skip title in the exact receipt mappi ["Postgres terminal LIST projection rejects late canonical snapshots"] ); }); +test("keeps the setup-binding promotion PostgreSQL skip title in the exact receipt mapping", () => { + assert.deepEqual( + [ + "Postgres: every setup-binding kind promotes on success, stays hidden on abandon, survives its revoke path", + ].filter((name) => POSTGRES_UNNAMED_SKIP_TEST_NAME_ROWS.includes(name)), + ["Postgres: every setup-binding kind promotes on success, stays hidden on abandon, survives its revoke path"] + ); +}); +test("keeps the Explore upcoming PostgreSQL skip titles in the exact receipt mapping", () => { + assert.deepEqual( + [ + "postgresFetchUpcoming: live Postgres in-flight partition workers never exceed the configured limit", + "sqliteFetchUpcoming & postgresFetchUpcoming: output is bit-identical and deterministic", + ].filter((name) => POSTGRES_UNNAMED_SKIP_TEST_NAME_ROWS.includes(name)), + [ + "postgresFetchUpcoming: live Postgres in-flight partition workers never exceed the configured limit", + "sqliteFetchUpcoming & postgresFetchUpcoming: output is bit-identical and deterministic", + ] + ); +}); // SECOND LIVE CANARY REVISE (2026-07-30): the interrupted-migration // reconciliation test file added two PostgreSQL tests using the same // bare-boolean `skip: !POSTGRES_URL` shape. FOURTH-PASS GATE REVISE @@ -915,15 +935,16 @@ test("the PostgreSQL profile declares its exact live-gate skip baseline", async // FIFTH-PASS GATE FIX (2026-07-30): this hardcoded literal must track // test-accounting.manifest.json's memory-default "PDPP_TEST_POSTGRES_URL // unset" count exactly. The integration branch already carried a verified -// baseline of 135; interrupted-migration reconciliation adds three more -// PostgreSQL-only tests, for a final baseline of 138. +// baseline of 135; the integrated PostgreSQL-only contracts add thirteen more +// canonical skips plus two ordering probes that require a dedicated database. test("the memory-default profile declares the exact current skip baseline", async () => { const root = execFileSync("git", ["rev-parse", "--show-toplevel"], { encoding: "utf8" }).trim(); const manifestValue = await readManifest(join(root, "test-accounting.manifest.json"), { root }); const suite = manifestValue.suites.find((entry) => entry.id === "ri-default"); const memoryDefault = suite?.profiles?.find((entry) => typeof entry !== "string" && entry.id === "memory-default"); assert.deepEqual(typeof memoryDefault === "string" ? undefined : memoryDefault?.skip_reasons, { - "PDPP_TEST_POSTGRES_URL unset": 138, + "PDPP_TEST_POSTGRES_URL unset": 148, + "PDPP_TEST_POSTGRES_URL unset or non-dedicated": 2, "set PDPP_TEST_POSTGRES_URL to the dedicated loopback listener": 13, "dedicated disposable URL not selected": 1, "set PDPP_LIVE_CONNECTOR_HEALTH_GATE=1 to run": 1, diff --git a/scripts/test-accounting/receipt.ts b/scripts/test-accounting/receipt.ts index cfa18832c..71f93dd12 100644 --- a/scripts/test-accounting/receipt.ts +++ b/scripts/test-accounting/receipt.ts @@ -88,6 +88,9 @@ export const POSTGRES_UNNAMED_SKIP_TEST_NAME_ROWS: readonly string[] = [ "Postgres sort repair fences all manifest streams for an instance and blob binding respects the same fence", "Postgres startup does not require pg_search and keeps native FTS as fallback", "Postgres store factory is consistent with the resolver", + "Postgres: every setup-binding kind promotes on success, stays hidden on abandon, survives its revoke path", + "postgresFetchUpcoming: live Postgres in-flight partition workers never exceed the configured limit", + "sqliteFetchUpcoming & postgresFetchUpcoming: output is bit-identical and deterministic", "Postgres terminal LIST projection rejects late canonical snapshots", "an actual PostgreSQL advisory-session disconnect leaks no lock and the same key recovers", "dedicated PostgreSQL manifest generations fence historical facts and undeclared-write provenance", diff --git a/spec-connector-ecosystem.md b/spec-connector-ecosystem.md index 6c0ae92e2..c95a90698 100644 --- a/spec-connector-ecosystem.md +++ b/spec-connector-ecosystem.md @@ -33,7 +33,7 @@ How connectors get data from sources: | Tool | Data | Auth method | License | Wrap difficulty | |---|---|---|---|---| -| **slackdump** (rusq/slackdump) | Slack messages, threads, files, users, emojis | Browser session cookie (`d` cookie) or export token | GPL-3.0 | Easy: already outputs JSON/SQLite | +| **slackdump** (rusq/slackdump) | Slack messages, threads, files, users, emojis | Browser session cookie (`d` cookie) or export token | AGPL-3.0 | Easy: already outputs JSON/SQLite | | **Timelinize** (timelinize/timelinize) | 10+ sources: photos, Facebook, Instagram, Twitter, Google, iCloud, Strava, SMS, email, contacts | Per-source (OAuth, file import, API keys) | Apache-2.0 | Medium: need Go wrapper per data source | ### C# / .NET diff --git a/test-accounting.manifest.json b/test-accounting.manifest.json index 2b184a3b8..3de73191e 100644 --- a/test-accounting.manifest.json +++ b/test-accounting.manifest.json @@ -14,7 +14,8 @@ "required": true, "skip_reasons": { "set PDPP_TEST_LIVE_NEKO_CAP=1 inside the Docker reference service": 1, - "PDPP_TEST_POSTGRES_URL unset": 138, + "PDPP_TEST_POSTGRES_URL unset": 148, + "PDPP_TEST_POSTGRES_URL unset or non-dedicated": 2, "dedicated disposable URL not selected": 1, "set PDPP_TEST_LIVE_NEKO=1 and NEKO_ORIGIN to run": 2, "set PDPP_MULTILINGUAL_MINILM_SMOKE=1 to run the external model-download smoke": 1,

    Sources

    ChatGPT 0 records · 3 streams