diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e4f4ff86..958e6967 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,6 +32,8 @@ jobs: target: aarch64-apple-darwin artifact_key: macos-arm64 expected_arch: arm64 + node_platform: darwin + node_arch: arm64 pkg_target: node22-macos-arm64 binary_name: playwright-runner ps_binary_name: personal-server @@ -40,6 +42,8 @@ jobs: target: x86_64-apple-darwin artifact_key: macos-x64 expected_arch: x86_64 + node_platform: darwin + node_arch: x64 pkg_target: node22-macos-x64 binary_name: playwright-runner ps_binary_name: personal-server @@ -47,6 +51,8 @@ jobs: os_family: linux target: x86_64-unknown-linux-gnu artifact_key: linux-x64 + node_platform: linux + node_arch: x64 pkg_target: node22-linux-x64 binary_name: playwright-runner ps_binary_name: personal-server @@ -54,6 +60,8 @@ jobs: os_family: windows target: x86_64-pc-windows-msvc artifact_key: windows-x64 + node_platform: win32 + node_arch: x64 pkg_target: node22-win-x64 binary_name: playwright-runner.exe ps_binary_name: personal-server.exe @@ -79,7 +87,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: - node-version: "22" + node-version: "22.23.1" cache: npm - name: Setup Rust @@ -88,6 +96,10 @@ jobs: toolchain: stable targets: ${{ matrix.target }} + - name: Stage Node.js runtime sidecar + shell: bash + run: node scripts/stage-pdpp-node.mjs --target "${{ matrix.target }}" --expected-platform "${{ matrix.node_platform }}" --expected-arch "${{ matrix.node_arch }}" + - name: Install Linux bundle dependencies if: matrix.platform == 'ubuntu-22.04' run: | @@ -127,12 +139,16 @@ jobs: npm run test:security npm run build + - name: Install PDPP runtime dependencies + run: node scripts/ensure-pdpp-runtime.js + - name: Optionally sign macOS helper binaries if: matrix.os_family == 'macos' && env.APPLE_SIGNING_AVAILABLE == 'true' env: APPLE_SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }} run: | set -euo pipefail + codesign --force --options runtime --timestamp --sign "$APPLE_SIGNING_IDENTITY" src-tauri/binaries/pdpp-node-${{ matrix.target }} codesign --force --options runtime --timestamp --entitlements playwright-runner/entitlements.plist --sign "$APPLE_SIGNING_IDENTITY" playwright-runner/dist/${{ matrix.binary_name }} codesign --force --options runtime --timestamp --entitlements personal-server/entitlements.plist --sign "$APPLE_SIGNING_IDENTITY" personal-server/dist/${{ matrix.ps_binary_name }} find personal-server/dist/node_modules -name '*.node' -type f -exec codesign --force --options runtime --timestamp --sign "$APPLE_SIGNING_IDENTITY" {} \; @@ -246,11 +262,15 @@ jobs: fi } - require_artifact manual-install-macos-arm64 '*.dmg' - require_artifact manual-install-macos-x64 '*.dmg' - require_artifact manual-install-linux-x64 '*.deb' - require_artifact manual-install-linux-x64 '*.AppImage' - require_artifact manual-install-windows-x64 '*.exe' + require_artifact manual-install-macos-arm64 'dmg/*.dmg' + require_artifact manual-install-macos-x64 'dmg/*.dmg' + require_artifact manual-install-linux-x64 'deb/*.deb' + require_artifact manual-install-linux-x64 'appimage/*.AppImage' + require_artifact manual-install-windows-x64 'nsis/*.exe' mapfile -d '' artifacts < <(find release-artifacts -type f -print0) + if [ "${#artifacts[@]}" -ne 5 ]; then + echo "::error::Expected exactly five verified release artifacts, found ${#artifacts[@]}." + exit 1 + fi gh release upload "$RELEASE_TAG" "${artifacts[@]}" --clobber diff --git a/.gitignore b/.gitignore index 6a5753cb..4abd4050 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ pnpm-debug.log* lerna-debug.log* node_modules +src-tauri/binaries/pdpp-node-* +src-tauri/binaries/pdpp-node-LICENSE dist dist-ssr *.local @@ -39,3 +41,4 @@ src-tauri/auth-page/ .devcontainer/ connectors/ .vercel +pdpp-runtime/.install-stamp diff --git a/README.md b/README.md index 7f4994f8..14386d2d 100644 --- a/README.md +++ b/README.md @@ -1,187 +1,216 @@ # DataConnect -> DataConnect is being made local-first and vendor-neutral. The app runs and exports your data to disk with no sign-in and no external account. Vana is one optional storage and sync provider you can enable in Settings, not a bundled default. Some code paths, URLs, and package names still assume Vana; we are actively generalizing these as the project moves toward a neutral, provider-agnostic architecture. +DataConnect is a local-first desktop application for collecting data from +online services and keeping the resulting data on your computer. -Desktop app for exporting your data from various platforms. -Screenshot 2026-02-24 at 8 10 08 PM +## Status -## Installation - -Download the latest release from [Releases](../../releases). +DataConnect is in active development. Use the +[Releases page](https://github.com/PDP-Connect/data-connect/releases) for +binary downloads when a release is available. Release artifacts are manual +installs. This build does not include an in-app auto-updater. -### macOS - -macOS artifacts may be unsigned. For an unsigned build, run this after installing: +Public code signing is not available yet. Some macOS artifacts do not have a +signature. For an unsigned macOS build, run this command after installation: ```bash xattr -cr /Applications/DataConnect.app ``` -Then open the app normally. - -### Windows +## What DataConnect does now -Run the `.exe` installer and follow the prompts. +DataConnect writes collected data to local disk by default. Local collection +does not require a Vana account or a Vana sign-in. -### Linux +Every installer bundles and runs a local Personal Server. It gives the desktop +app loopback-only access to imported and exported data. It also enables local +PDPP and MCP integrations. Remote registration and tunneling stay disabled +unless you configure service endpoints. -Use the `.deb` or `.AppImage` package. +## Installation -Every installer bundles and runs a local Personal Server. It gives the desktop app loopback-only access to imported and exported data, and it enables local PDPP and MCP integrations. Remote registration and tunneling stay disabled unless you explicitly configure service endpoints. +Download the latest release from [Releases](../../releases). -## Browser Requirements +### macOS -DataConnect uses browser automation to export your data. Release installers include a compatible Chromium build. At runtime: +Install the DMG. If macOS blocks an unsigned build, run the `xattr` command in +the Status section and open the app again. -1. **If you have Chrome/Edge installed:** The app uses your existing browser (recommended) -2. **If no system browser is found:** The app uses the Chromium bundled with the installer -3. **For local builds without bundled Chromium:** The app downloads Chromium automatically when needed +### Windows -The downloaded browser is stored in `~/.dataconnect/browsers/` and persists across app updates. +Run the NSIS installer and follow the prompts. -## Supported Platforms +### Linux -DataConnect currently supports exporting data from ChatGPT, GitHub, Instagram, LinkedIn, Spotify,YouTube, and Shop (Shopify) — covering your conversations, social profiles, listening history, watch history, order history, and more. +Use the `.deb` or `.AppImage` package. -For the latest available connectors, visit the [Data Connectors repository](https://github.com/vana-com/data-connectors). +## Build from source -## Development +Use this path if a binary is not available for your platform. -### Prerequisites +### Requirements -- Node.js 20+ -- Rust (latest stable) -- For Playwright connectors: `cd playwright-runner && npm install` +- Node.js 22 or 23. +- Rust stable and the Tauri build prerequisites for your operating system. +- Internet access during setup to install dependencies and resolve bundled + connectors when they are absent. -### Running locally +### Run the development application ```bash -# Install dependencies +git clone https://github.com/PDP-Connect/data-connect.git +cd data-connect npm install - -# Run in development mode npm run tauri:dev - -# Copy .env file -cp .env.example .env ``` -### Connector management +`npm run tauri:dev` builds required local helper programs when needed. It also +resolves the bundled connectors from the signed connector index when they are +not already present. -Connector scripts live upstream in [`vana-com/data-connectors`](https://github.com/vana-com/data-connectors). This repo consumes them as pinned dependencies. - -#### Updating connectors +### Create a package ```bash -npm run connectors:resolve -# Review the diff in connectors/, commit, push. +npm run tauri:build ``` -This fetches the latest matching versions from the signed data-connectors index, verifies checksums, and writes them to `connectors/`. Version constraints are declared in `connectors/connector-dependencies.json`. +The command builds the frontend, the Personal Server, and the Playwright +runner. It creates Linux and macOS packages from a local checkout. Release CI +also builds the Windows NSIS installer. -If you only want to verify the lockfile and bundled connector tree without mutating them, run `npm run connectors:check`. +Treat local build output as a development build. Do not treat it as a signed +public release. -#### How it works at runtime +## Browser Requirements -- `tauri dev` runs `ensure-connectors.js`, which restores missing bundled connectors from `~/.dataconnect/connectors/` first and then resolves them from the signed connector index if needed. -- The Rust backend loads connectors from active installs in `~/.dataconnect/connectors-store/` via `connectors-active.json`, then legacy `~/.dataconnect/connectors/`, then bundled `connectors/`. -- The `playwright-runner` executes connector scripts with a local Chromium browser. +DataConnect uses browser automation for legacy connectors and for browser-based +PDPP collection profiles. -### Agent config files +1. If you installed Chrome or Edge, DataConnect can use that browser. +2. If no supported system browser is found, DataConnect can use the Chromium + build bundled with the installer. +3. For local builds without bundled Chromium, DataConnect provisions Chromium at + runtime when needed. -This repo keeps both `AGENTS.md` and `CLAUDE.md`: Claude Code auto‑loads `CLAUDE.md` but not `AGENTS.md`, and Cursor does the opposite. Keep them aligned. +DataConnect stores the downloaded browser in `~/.dataconnect/browsers/`. It +persists across app updates. -### Agent skills sync +## Data sources -Skills are stored in `.agents/skills` (source of truth). Cursor reads them via per-skill symlinks in `.cursor/skills`. The sync script rebuilds those symlinks so any manually created skills show up in Cursor. +### GitHub PDPP Collection Profile -```bash -# One-off sync (default is .cursor/skills) -npm run skills:sync +GitHub is a PDPP-native collection path. DataConnect prefers this path when the +GitHub PDPP connector is available. It collects GitHub profile, repositories, +stars, issues, pull requests, and gists through the PDPP Collection Profile +protocol. -# Sync to Claude instead -npm run skills:sync -- --target=claude +To collect GitHub data: -# Auto-sync on changes -npm run skills:watch -``` +1. Select **GitHub** on the Home page. +2. Enter a GitHub personal access token when DataConnect requests one. +3. Give the token the GitHub permissions needed for the data that you want to + collect. +4. Select **Start import**. -### Building for production +DataConnect uses the token for that import only. The application does not save +the token. Use a token that you can revoke in GitHub if you no longer want to +use it. -```bash -# Install the locked dependencies, then build helpers and the native bundle -npm ci -npm run tauri:build -``` +### ChatGPT PDPP Collection Profile -Local installs and builds do not download Chromium as an npm lifecycle step. A local build bundles a compatible Playwright Chromium already present in its cache when available. Otherwise, the app uses a supported system browser or provisions Chromium at runtime. Release CI explicitly provisions Chromium and fails verification if the browser executable is absent from an installer. +ChatGPT is also available as a PDPP Collection Profile. It uses a browser +session to collect conversations, messages, memories, custom GPTs, custom +instructions, and shared conversations. -The built app will be in `src-tauri/target/release/bundle/`. +To collect ChatGPT data: -### Releasing +1. Select **ChatGPT** on the Home page. +2. Enter your ChatGPT sign-in details when DataConnect requests them. +3. Complete any browser sign-in or verification step. +4. Select **Start import**. -Releases are created via the release script, which bumps the version in `tauri.conf.json`, commits, pushes, and creates a GitHub release that triggers CI builds across macOS, Linux, and Windows. +### Legacy browser connectors -```bash -# Check current and suggested versions -npm run release:github -- --show-versions +DataConnect keeps its Playwright-based legacy connectors. They automate export +flows in a browser and write exports to local disk. The bundled legacy +connectors currently cover ChatGPT, Claude, GitHub, H-E-B, Instagram, +Instagram Ads, LinkedIn, Oura, Shop, Spotify, Whole Foods Market, and YouTube. -# Dry run to preview what will happen -npm run release:github -- --version X.Y.Z --dry-run +These connectors depend on the website and export process of each service. A +service can change that process without notice. A connector can therefore fail +or collect less data than expected. -# Create a new release -npm run release:github -- --version X.Y.Z -``` +The [PDP-Connect Data Connectors repository](https://github.com/PDP-Connect/data-connectors) +contains the connector artifacts and their signed index. + +## Use Timeline locally + +Timeline gives a chronological view of records from connected sources. The +current Timeline reads verified GitHub records from your local Personal Server. + +1. Collect GitHub data with the GitHub PDPP Collection Profile. +2. Open **Apps > Timeline**. +3. Select **Review local Timeline access**. +4. Review the requested GitHub streams, access mode, retention, and expiry. +5. Select **Approve local Timeline access**. +6. View the Timeline. -> **Do not** create releases manually via `gh release create` or the GitHub UI — the CI workflow will fail if `tauri.conf.json` version doesn't match the release tag. +The approval applies only to the local Personal Server on that device. +Timeline access expires after its issued lifetime, which is eight hours by +default. -Release artifacts are manual installs: macOS DMGs, Linux `.deb` and AppImage files, and Windows NSIS installers. This build does not include an in-app auto-updater. macOS artifacts are unsigned unless the optional `APPLE_BUILD_CERTIFICATE_BASE64`, `APPLE_BUILD_CERTIFICATE_PASSWORD`, and `APPLE_SIGNING_IDENTITY` GitHub secrets are configured. Signed builds are not notarized. +To stop Timeline access, select **Revoke Timeline access** in Timeline. The +app revokes the local approval and returns you to the consent screen. -The workflow can also be run manually without uploading artifacts (`workflow_dispatch` with `upload: false`) to validate all platform build legs. +## Connector management -## Architecture +Connector versions use two pinned files: `connectors/connector-dependencies.json` +and `connectors/lock.json`. +```bash +npm run connectors:resolve +npm run connectors:check ``` -┌─────────────────────────────────────────────────────────┐ -│ DataConnect App │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────┐ │ -│ │ React UI │ │ Tauri/Rust │ │ Playwright │ │ -│ │ (Frontend) │◄─►│ (Backend) │◄─►│ Runner │ │ -│ └─────────────┘ └─────────────┘ └────────┬────────┘ │ -└────────────────────────────────────────────┬┼───────────┘ - ││ - ┌────────────────────────┘│ - │ │ - ┌─────▼─────┐ ┌───────▼───────┐ - │ System │ │ Downloaded │ - │ Chrome │ OR │ Chromium │ - └───────────┘ └───────────────┘ + +`connectors:resolve` resolves the pinned versions from the signed connector +index and updates the bundled connector tree. `connectors:check` verifies the +lockfile and bundled connector tree without changing them. + +## Releasing + +Use the release script to create releases: + +```bash +npm run release:github -- --show-versions +npm run release:github -- --version X.Y.Z --dry-run +npm run release:github -- --version X.Y.Z ``` -### Browser Selection Priority +If AI helped with the release commit, run `npm run release:github -- --version X.Y.Z --assisted-by-ai`. -1. **System Chrome** - `/Applications/Google Chrome.app` (macOS) -2. **System Edge** - Available on Windows -3. **Downloaded Chromium** - `~/.dataconnect/browsers/` -4. **Auto-download** - If nothing found, downloads Chromium on first run +Do not create releases manually with `gh release create` or the GitHub UI. The +CI workflow checks that `tauri.conf.json` matches the release tag. -## Connectors +Release artifacts are manual installs: macOS DMGs, Linux `.deb` and AppImage +files, and Windows NSIS installers. macOS artifacts do not have signatures +unless CI has the optional Apple signing secrets. Signed builds do not have +notarization. -Connectors are JavaScript files that automate data export. Located in the [Data Connectors repository](https://github.com/vana-com/data-connectors). +The workflow can also run without uploading artifacts to validate all platform +build legs. -### Connector API (Playwright runtime) +## Not supported yet -```javascript -// Available in connector scripts: -page.goto(url) // Navigate to URL -page.evaluate(script) // Run JS in page context -page.sleep(ms) // Wait for milliseconds -page.setData(key, value) // Send data back to app -page.promptUser(message, checkFn) // Wait for user action -``` +- Public code signing or notarization. +- A cloud account or remote service as a requirement for local collection. +- Timeline data from sources other than verified GitHub PDPP records. +- A guarantee that every legacy connector works with every account, browser, or + service change. +- Automatic updates. ## License -This project is licensed under the Apache License 2.0. See the LICENSE file for details. -This software is provided as open-source utility software and is not a managed or hosted service. -See LEGAL.md for additional legal disclaimers and responsibility framing. +DataConnect uses the Apache License 2.0. See [LICENSE](LICENSE) for details. +This software is open-source utility software. It is not a managed or hosted +service. See [LEGAL.md](LEGAL.md) for legal disclaimers and responsibility +information. diff --git a/connectors/connector-dependencies.json b/connectors/connector-dependencies.json index 107926c1..e71d3f92 100644 --- a/connectors/connector-dependencies.json +++ b/connectors/connector-dependencies.json @@ -15,6 +15,7 @@ "wholefoods-playwright": "1.0.0", "youtube-playwright": "1.0.0", "claude-export-playwright": "2.0.1", + "chatgpt-pdpp": "0.1.0", "github-pdpp": "0.5.0" } } diff --git a/connectors/lock.json b/connectors/lock.json index d4a117c4..7b18667f 100644 --- a/connectors/lock.json +++ b/connectors/lock.json @@ -25,9 +25,37 @@ "wholefoods-playwright": "1.0.0", "youtube-playwright": "1.0.0", "claude-export-playwright": "2.0.1", + "chatgpt-pdpp": "0.1.0", "github-pdpp": "0.5.0" }, "connectors": [ + { + "connectorId": "chatgpt-pdpp", + "company": "openai", + "version": "0.1.0", + "resolvedFrom": "0.1.0", + "artifactUrl": "https://github.com/PDP-Connect/data-connectors/releases/download/connectors-48440fead534/chatgpt-pdpp-0.1.0.tgz", + "artifactPath": "artifacts/chatgpt-pdpp/chatgpt-pdpp-0.1.0.tgz", + "artifactSha256": "sha256:0188385dbb782cf23a5331f0b05a4dddbab422cc21113b0ba8b5c1f72b42772f", + "artifactSignature": { + "type": "sigstoreBundle", + "bundlePath": "chatgpt-pdpp-0.1.0.tgz.sigstore.json", + "bundleUrl": "https://github.com/PDP-Connect/data-connectors/releases/download/connectors-48440fead534/chatgpt-pdpp-0.1.0.tgz.sigstore.json" + }, + "manifestSha256": "sha256:e2c66b9d584ff935b31790e3a698a5576168002988ccc8076ded327f189beff0", + "artifactKind": "pdpp-collection-profile", + "manifestPath": "profile/collection-profile.json", + "entrypointPath": "dist/collection-profile.mjs", + "entrypointSha256": "sha256:2c813fb3ae7fb6cc94af327a17ea17347a43b78aa4a401718a660b2edd6a25cc", + "provenancePath": "provenance.json", + "provenanceSha256": "sha256:95635fdd7865401deeecde499621d181397b301cc266a7fdf659c175ec04e4cf", + "sourceTag": "pdpp-cc07e3a896c2c0df7841da4ec6b2c660ffe1e792", + "sourceCommit": "cc07e3a896c2c0df7841da4ec6b2c660ffe1e792", + "releaseId": "github-48440fead5343d568773594b20cfc9306f1d6255", + "publishedAt": "2026-07-31T00:00:00Z", + "name": "ChatGPT (PDPP Collection Profile)", + "description": "Collects ChatGPT conversations, messages, memories, custom GPTs, custom instructions, and shared conversations through the PDPP Collection Profile protocol." + }, { "connectorId": "chatgpt-playwright", "company": "openai", diff --git a/package.json b/package.json index 40ef8958..8d4850e1 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "dev": "vite", "dev:app": "vite --port 5173 --strictPort", "build": "tsc -b && vite build", - "prebuild:all": "node scripts/ensure-connectors.js && node scripts/ensure-playwright-runner.js && node scripts/ensure-personal-server.js", + "prebuild:all": "node scripts/ensure-connectors.js && node scripts/ensure-pdpp-runtime.js && node scripts/ensure-playwright-runner.js && node scripts/ensure-personal-server.js", "connectors:resolve": "node scripts/resolve-connectors.js", "connectors:install": "npm run connectors:resolve", "connectors:upgrade": "npm run connectors:resolve", diff --git a/pdpp-runtime/connector-loader-bootstrap.mjs b/pdpp-runtime/connector-loader-bootstrap.mjs new file mode 100644 index 00000000..afada4ba --- /dev/null +++ b/pdpp-runtime/connector-loader-bootstrap.mjs @@ -0,0 +1,6 @@ +import { register } from "node:module" + +// Node 22/23 loads this module with --import before the connector entrypoint. +// Registering the narrow resolver here is the supported replacement for the +// deprecated --experimental-loader command-line hook. +register(new URL("./connector-loader.mjs", import.meta.url), import.meta.url) diff --git a/pdpp-runtime/connector-loader.mjs b/pdpp-runtime/connector-loader.mjs new file mode 100644 index 00000000..4253fe8d --- /dev/null +++ b/pdpp-runtime/connector-loader.mjs @@ -0,0 +1,42 @@ +import { readFileSync } from "node:fs" +import { createRequire } from "node:module" +import { fileURLToPath, pathToFileURL } from "node:url" + +const runtimeRoot = process.env.DATACONNECT_PDPP_RUNTIME_ROOT +if (!runtimeRoot) throw new Error("DATACONNECT_PDPP_RUNTIME_ROOT is required") + +const resolveFromRuntime = createRequire(`${runtimeRoot}/package.json`) +const patchrightPackageUrl = pathToFileURL( + resolveFromRuntime.resolve("patchright/package.json") +) +const patchrightPackage = JSON.parse( + readFileSync(fileURLToPath(patchrightPackageUrl), "utf8") +) +const patchrightEsmEntry = patchrightPackage.exports?.["."]?.import +if (typeof patchrightEsmEntry !== "string") { + throw new Error("Packaged patchright must declare an ESM import entry") +} + +function resolveExternal(specifier) { + if (specifier === "p-queue") { + return pathToFileURL(resolveFromRuntime.resolve(specifier)).href + } + if (specifier === "patchright") { + // createRequire.resolve() selects the require condition, which turns a + // dynamic ESM import into a CommonJS namespace with chromium only under + // default. The authoritative ChatGPT runtime destructures chromium, so + // preserve patchright's declared import condition exactly. + return new URL(patchrightEsmEntry, patchrightPackageUrl).href + } +} + +export async function resolve(specifier, context, nextResolve) { + const external = resolveExternal(specifier) + if (external) { + return { + url: external, + shortCircuit: true, + } + } + return nextResolve(specifier, context) +} diff --git a/pdpp-runtime/package-lock.json b/pdpp-runtime/package-lock.json new file mode 100644 index 00000000..2411df68 --- /dev/null +++ b/pdpp-runtime/package-lock.json @@ -0,0 +1,92 @@ +{ + "name": "dataconnect-pdpp-runtime", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "dataconnect-pdpp-runtime", + "dependencies": { + "p-queue": "^9.3.3", + "patchright": "^1.61.1" + } + }, + "node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/p-queue": { + "version": "9.3.3", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.3.3.tgz", + "integrity": "sha512-NXAOdnEe5FsZJfT4oK84lE1Y5cFFdWlRuOo5tww8DyNMxyRXwn39fIkUtNLKppcPC+UYU/bXujNCUGDv01y7CA==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^5.0.4", + "p-timeout": "^7.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.1.tgz", + "integrity": "sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/patchright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/patchright/-/patchright-1.61.1.tgz", + "integrity": "sha512-kZWNZ2tunsBMTBWtARFVWaNvB739GOybTlIPLT91eyx0YERjtUjESawomwO0hnOh6jt1L3Sru0PK62ylYOfqQw==", + "license": "Apache-2.0", + "dependencies": { + "patchright-core": "1.61.1" + }, + "bin": { + "patchright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/patchright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/patchright-core/-/patchright-core-1.61.1.tgz", + "integrity": "sha512-d6Ju67DE6OzqlEX3WPvUX2SLEs6iMY1WERit//mpsrT98VHcztaUo0CBw2Lpxq25VdufVEiWEdcc0HnLwdiq6A==", + "license": "Apache-2.0", + "bin": { + "patchright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + } + } +} diff --git a/pdpp-runtime/package.json b/pdpp-runtime/package.json new file mode 100644 index 00000000..18d84fcb --- /dev/null +++ b/pdpp-runtime/package.json @@ -0,0 +1,9 @@ +{ + "name": "dataconnect-pdpp-runtime", + "private": true, + "type": "module", + "dependencies": { + "p-queue": "^9.3.3", + "patchright": "^1.61.1" + } +} diff --git a/personal-server/index.js b/personal-server/index.js index bdf07a86..34f41db3 100644 --- a/personal-server/index.js +++ b/personal-server/index.js @@ -22,13 +22,17 @@ process.env.NODE_ENV = 'production'; import { dirname, join } from 'node:path'; import { readFileSync, writeFileSync } from 'node:fs'; import { homedir } from 'node:os'; -import { randomUUID } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import { execSync, spawn } from 'node:child_process'; import { pathToFileURL } from 'node:url'; import { registerProtectedRoutes } from './protected-routes.js'; -import { mountPdppResourceServer } from './pdpp/resource-server.js'; -import { createGithubAuthorizationAdapter } from './pdpp/github-authorization/index.js'; -import { registerGithubAuthorizationRoutes } from './pdpp/github-authorization/http-routes.js'; +import { + createPdppResourceServer, + mountPdppResourceRoutes, +} from './pdpp/resource-server.js'; +import { createPdppAuthorizationAdapter } from './pdpp/github-authorization/index.js'; +import { registerPdppAuthorizationRoutes } from './pdpp/github-authorization/http-routes.js'; +import { loadInstalledManifest } from './pdpp/installed-manifest.js'; import { createPersonalServerServeOptions } from './listener-options.cjs'; const PACKAGED_RUNTIME_ENTRYPOINTS = { @@ -37,6 +41,170 @@ const PACKAGED_RUNTIME_ENTRYPOINTS = { '@hono/node-server': '@hono/node-server/dist/index.js', }; +const PDPP_SERVING_PROFILES = { + 'github-pdpp': { + connector: { + key: 'github', + id: 'https://registry.pdpp.org/connectors/github', + }, + scopeForStream: stream => ({ + user: 'github.profile', + repositories: 'github.repositories', + starred: 'github.starred', + })[stream], + enableLocalTimeline: true, + }, + 'chatgpt-pdpp': { + connector: { + key: 'chatgpt', + id: 'https://registry.pdpp.org/connectors/chatgpt', + }, + scopeForStream: stream => [ + 'conversations', + 'messages', + 'memories', + 'custom_gpts', + 'custom_instructions', + 'shared_conversations', + ].includes(stream) ? `chatgpt.${stream}` : undefined, + enableLocalTimeline: false, + }, +}; + +function selectedPdppServingProfile() { + const connectorId = process.env.PDPP_SERVING_CONNECTOR_ID || 'github-pdpp'; + const profile = PDPP_SERVING_PROFILES[connectorId]; + if (!profile) { + throw new Error(`PDPP_SERVING_CONNECTOR_ID must select a supported profile, got ${connectorId}`); + } + return { connectorId, ...profile }; +} + +function pdppProfileStorageName(connectorId) { + if (connectorId === 'github-pdpp') return 'github'; + return createHash('sha256').update(connectorId).digest('hex').slice(0, 16); +} + +export function pdppProfileDatabasePath(root, connectorId, kind) { + return join( + root, + `pdpp-${pdppProfileStorageName(connectorId)}-${kind}.sqlite` + ); +} + +export function pdppDefaultStorageRoots({ + configDir, + pdppStorageDir, + homeDir = homedir(), +} = {}) { + const serverRoot = + configDir || join(homeDir, '.data-connect', 'personal-server'); + return { + authorizationRoot: serverRoot, + recordsRoot: + pdppStorageDir || + configDir || + join(homeDir, '.dataconnect', 'personal-server'), + }; +} + +export function createPdppRevocationSink({ + storageRoot, + activeManifestPath, + selectedPdppProfile = selectedPdppServingProfile(), +}) { + return createPdppAuthorizationAdapter({ + databasePath: pdppProfileDatabasePath( + storageRoot, + selectedPdppProfile.connectorId, + 'authorization' + ), + activeManifestPath, + connectorId: selectedPdppProfile.connectorId, + expectedConnector: selectedPdppProfile.connector, + scopeForStream: selectedPdppProfile.scopeForStream, + enableLocalTimeline: selectedPdppProfile.enableLocalTimeline, + }); +} + +export async function registerOptionalPdppSurfaces({ + app, + devToken, + storageRoot, + recordsRoot = storageRoot, + activeManifestPath, + exportRoot, + connectionId = 'default', + selectedPdppProfile = selectedPdppServingProfile(), + singleUseAccessExpiresInSeconds, + externalOrigin, + send = () => {}, +}) { + let adapter; + try { + // Resolve this once, then require both independently mounted surfaces to + // re-verify this exact selected install. The two surfaces must never + // compose grants from one active-manifest path with records from another. + const selectedInstall = loadInstalledManifest({ + activeManifestPath, + connectorId: selectedPdppProfile.connectorId, + expectedConnector: selectedPdppProfile.connector, + }); + adapter = createPdppAuthorizationAdapter({ + databasePath: pdppProfileDatabasePath( + storageRoot, + selectedPdppProfile.connectorId, + 'authorization' + ), + activeManifestPath, + connectorId: selectedPdppProfile.connectorId, + expectedConnector: selectedPdppProfile.connector, + selectedInstall, + scopeForStream: selectedPdppProfile.scopeForStream, + enableLocalTimeline: selectedPdppProfile.enableLocalTimeline, + singleUseAccessExpiresInSeconds, + }); + const resourceServer = await createPdppResourceServer({ + activeManifestPath, + connectorId: selectedPdppProfile.connectorId, + expectedConnector: selectedPdppProfile.connector, + selectedInstall, + databasePath: pdppProfileDatabasePath( + recordsRoot, + selectedPdppProfile.connectorId, + 'records' + ), + exportRoot, + connectionId, + tokenIntrospector: { + introspect: token => adapter.resolveForResourceServer(token), + }, + }); + registerPdppAuthorizationRoutes({ + app, + devToken, + adapter, + enableLocalTimeline: selectedPdppProfile.enableLocalTimeline, + externalOrigin, + }); + mountPdppResourceRoutes(app, resourceServer); + send({ + type: 'log', + message: `[pdpp] mounted selected ${selectedPdppProfile.connector.key} resource routes`, + }); + return adapter; + } catch (error) { + adapter?.close(); + send({ + type: 'log', + message: `[pdpp] routes unavailable: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + return null; + } +} + function send(msg) { let json; try { @@ -407,32 +575,37 @@ async function main() { // Keep as a reference because startBackgroundServices mutates context.tunnelManager / context.tunnelUrl. const context = await createServer(config, { rootPath: configDir }); const { app, devToken, cleanup, gatewayClient, serverSigner } = context; - const pdppAuthorization = createGithubAuthorizationAdapter({ - databasePath: join(configDir || join((await import('node:os')).homedir(), '.data-connect', 'personal-server'), 'pdpp-github-authorization.sqlite'), + const selectedPdppProfile = selectedPdppServingProfile(); + const pdppActiveManifestPath = process.env.DATACONNECT_ACTIVE_CONNECTORS_PATH; + const pdppStorageRoots = pdppDefaultStorageRoots({ + configDir, + pdppStorageDir: process.env.PDPP_STORAGE_DIR, + }); + const pdppRevocationSink = createPdppRevocationSink({ + storageRoot: pdppStorageRoots.authorizationRoot, + activeManifestPath: pdppActiveManifestPath, + selectedPdppProfile, + }); + const pdppAuthorization = await registerOptionalPdppSurfaces({ + app, + devToken, + storageRoot: pdppStorageRoots.authorizationRoot, + recordsRoot: pdppStorageRoots.recordsRoot, + activeManifestPath: pdppActiveManifestPath, + exportRoot: process.env.DATACONNECT_EXPORT_ROOT, + connectionId: + process.env.PDPP_SERVING_CONNECTION_ID || + process.env.PDPP_GITHUB_CONNECTION_ID || + 'default', + selectedPdppProfile, singleUseAccessExpiresInSeconds, + externalOrigin: personalServerExternalOrigin( + context.serverAccount?.address, + tunnelServerAddr + ), + send, }); - // PDPP reads are additive: unavailable or invalid local connector state - // leaves the established Personal Server routes running without a fallback - // manifest or lossy data source. - try { - const pdppStorageRoot = process.env.PDPP_STORAGE_DIR - || configDir - || join(homedir(), '.dataconnect', 'personal-server'); - await mountPdppResourceServer(app, { - activeManifestPath: process.env.DATACONNECT_ACTIVE_CONNECTORS_PATH, - databasePath: join(pdppStorageRoot, 'pdpp-github-records.sqlite'), - exportRoot: process.env.DATACONNECT_EXPORT_ROOT, - connectionId: process.env.PDPP_GITHUB_CONNECTION_ID || 'default', - tokenIntrospector: { - introspect: token => pdppAuthorization.resolveForResourceServer(token), - }, - }); - send({ type: 'log', message: '[pdpp] mounted installed GitHub resource routes' }); - } catch (error) { - send({ type: 'log', message: `[pdpp] resource routes unavailable: ${error instanceof Error ? error.message : String(error)}` }); - } - // --- Request logging --- // Wrap app.fetch to log all incoming requests (including routes registered // by the library before our code runs, like /health). @@ -469,16 +642,8 @@ async function main() { port, send, serverSigner, - onLegacyGrantRevoked: grantId => pdppAuthorization.revokeByLegacyGrantId(grantId), - }); - registerGithubAuthorizationRoutes({ - app, - devToken, - adapter: pdppAuthorization, - externalOrigin: personalServerExternalOrigin( - context.serverAccount?.address, - tunnelServerAddr - ), + onLegacyGrantRevoked: grantId => + pdppRevocationSink.revokeByLegacyGrantId(grantId), }); // Start HTTP server first so the desktop app can connect immediately. @@ -639,7 +804,10 @@ async function main() { function shutdown(signal) { send({ type: 'log', message: `Shutdown signal: ${signal}` }); - pdppAuthorization.close(); + pdppAuthorization?.close(); + if (pdppAuthorization !== pdppRevocationSink) { + pdppRevocationSink.close(); + } if (cleanup) cleanup().catch(() => {}); server.close(() => { process.exit(0); @@ -656,4 +824,6 @@ async function main() { } } -main(); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(); +} diff --git a/personal-server/package-lock.json b/personal-server/package-lock.json index fdd29dac..c11b86ce 100644 --- a/personal-server/package-lock.json +++ b/personal-server/package-lock.json @@ -14,7 +14,8 @@ "@opendatalabs/personal-server-ts-mcp": "0.0.1-canary.92ee4d6", "@opendatalabs/personal-server-ts-server": "0.0.1-canary.92ee4d6", "better-sqlite3": "^12.6.2", - "hono": "^4.7.0" + "hono": "^4.7.0", + "viem": "^2.45.1" }, "devDependencies": { "@yao-pkg/pkg": "^5.12.0", diff --git a/personal-server/pdpp/github-authorization/http-routes.js b/personal-server/pdpp/github-authorization/http-routes.js index ab70a88e..ba80660f 100644 --- a/personal-server/pdpp/github-authorization/http-routes.js +++ b/personal-server/pdpp/github-authorization/http-routes.js @@ -26,6 +26,7 @@ export function registerGithubAuthorizationRoutes({ devToken, adapter, externalOrigin, + enableLocalTimeline = true, }) { const desktopAuth = requireDesktopAuth(devToken) @@ -90,53 +91,62 @@ export function registerGithubAuthorizationRoutes({ } } ) - app.post("/v1/pdpp/local-timeline/consent-requests", desktopAuth, async c => { - try { - const body = await c.req.json() - return c.json( - adapter.createLocalTimelineConsentRequest({ - sessionId: body.session_id, - subjectId: body.subject_id, - }), - 201 - ) - } catch (error) { - return errorResponse(c, error) - } - }) - app.post( - "/v1/pdpp/local-timeline/consent-requests/:requestId/approve", - desktopAuth, - async c => { + if (enableLocalTimeline) { + app.post( + "/v1/pdpp/local-timeline/consent-requests", + desktopAuth, + async c => { + try { + const body = await c.req.json() + return c.json( + adapter.createLocalTimelineConsentRequest({ + sessionId: body.session_id, + subjectId: body.subject_id, + }), + 201 + ) + } catch (error) { + return errorResponse(c, error) + } + } + ) + app.post( + "/v1/pdpp/local-timeline/consent-requests/:requestId/approve", + desktopAuth, + async c => { + try { + const body = await c.req.json() + return c.json( + adapter.issueLocalTimelineGrant({ + requestId: c.req.param("requestId"), + sessionId: body.session_id, + subjectId: body.subject_id, + }), + 201 + ) + } catch (error) { + return errorResponse(c, error) + } + } + ) + app.post("/v1/pdpp/local-timeline/revoke", desktopAuth, async c => { try { const body = await c.req.json() - return c.json( - adapter.issueLocalTimelineGrant({ - requestId: c.req.param("requestId"), + return c.json({ + revoked: adapter.revokeLocalTimelineSession({ sessionId: body.session_id, subjectId: body.subject_id, }), - 201 - ) + }) } catch (error) { return errorResponse(c, error) } - } - ) - app.post("/v1/pdpp/local-timeline/revoke", desktopAuth, async c => { - try { - const body = await c.req.json() - return c.json({ - revoked: adapter.revokeLocalTimelineSession({ - sessionId: body.session_id, - subjectId: body.subject_id, - }), - }) - } catch (error) { - return errorResponse(c, error) - } - }) + }) + } app.post("/v1/pdpp/introspect", c => c.json(adapter.introspectPublicBearer(c.req.header("authorization"))) ) } + +/** Neutral route registration retains the existing endpoint contract. */ +export const registerPdppAuthorizationRoutes = registerGithubAuthorizationRoutes diff --git a/personal-server/pdpp/github-authorization/index.js b/personal-server/pdpp/github-authorization/index.js index a4d980b7..6ddca556 100644 --- a/personal-server/pdpp/github-authorization/index.js +++ b/personal-server/pdpp/github-authorization/index.js @@ -1,10 +1,8 @@ -import { createHash } from "node:crypto" -import { readFileSync } from "node:fs" -import { loadInstalledGithubManifest } from "../installed-manifest.js" +import { resolveSelectedInstalledManifest } from "../installed-manifest.js" import { createLocalTimelineAuthorizationRequest, LOCAL_TIMELINE_CLIENT_ID, - validateGithubAuthorizationDetails, + validateAuthorizationDetails, } from "./policy.js" import { openGithubAuthorizationStore } from "./store.js" @@ -19,21 +17,34 @@ function inactive() { return { active: false } } -function readVerifiedManifest(activeManifestPath) { - const installed = loadInstalledGithubManifest({ activeManifestPath }) +function readVerifiedManifest( + activeManifestPath, + connectorId, + expectedConnector, + selectedInstall +) { + const installed = resolveSelectedInstalledManifest({ + activeManifestPath, + connectorId, + expectedConnector, + selectedInstall, + }) return { version: installed.version, - digest: createHash("sha256") - .update(readFileSync(installed.manifestPath)) - .digest("hex"), + digest: installed.manifestDigest, manifest: installed.manifest, } } -/** A separate UAT authorization composition; it intentionally does not alter query-core pdpp/index.js. */ -export function createGithubAuthorizationAdapter({ +/** Selected-install authorization composition. The route core remains unchanged. */ +export function createPdppAuthorizationAdapter({ databasePath, activeManifestPath, + connectorId = "github-pdpp", + expectedConnector, + selectedInstall, + scopeForStream, + enableLocalTimeline = false, now, random, singleUseAccessExpiresInSeconds = SINGLE_USE_ACCESS_EXPIRES_IN_SECONDS, @@ -47,7 +58,16 @@ export function createGithubAuthorizationAdapter({ credentialHandoffExpiresInSeconds, }) const clock = now ?? (() => new Date()) - const currentManifest = () => readVerifiedManifest(activeManifestPath) + if (typeof scopeForStream !== "function") { + throw new TypeError("scopeForStream must be a function") + } + const currentManifest = () => + readVerifiedManifest( + activeManifestPath, + connectorId, + expectedConnector, + selectedInstall + ) const publicIdentity = token => { try { const found = store.findActiveGrant(token, currentManifest()) @@ -68,19 +88,27 @@ export function createGithubAuthorizationAdapter({ return { createConsentRequest({ sessionId, scopes, authorizationDetails }) { const manifest = currentManifest() - const terms = validateGithubAuthorizationDetails({ + const terms = validateAuthorizationDetails({ authorizationDetails, manifest: manifest.manifest, scopes, + scopeForStream, }) return store.createRequest({ sessionId, scopes, terms, manifest }) }, createLocalTimelineConsentRequest({ sessionId, subjectId }) { + if (!enableLocalTimeline) { + throw new Error( + "The selected connector does not support Timeline consent" + ) + } const manifest = currentManifest() const local = createLocalTimelineAuthorizationRequest(manifest.manifest) - const terms = validateGithubAuthorizationDetails({ + const terms = validateAuthorizationDetails({ ...local, manifest: manifest.manifest, + scopeForStream: stream => `pdpp.local.github.${stream}`, + sourceIds: ["github", "https://registry.pdpp.org/connectors/github"], localTimeline: true, }) return { @@ -132,6 +160,11 @@ export function createGithubAuthorizationAdapter({ }) }, issueLocalTimelineGrant({ requestId, sessionId, subjectId }) { + if (!enableLocalTimeline) { + throw new Error( + "The selected connector does not support Timeline consent" + ) + } return store.issueGrant({ requestId, // The legacy field is retained for the existing schema and legacy @@ -171,6 +204,11 @@ export function createGithubAuthorizationAdapter({ }, revokeByLegacyGrantId: store.revokeByLegacyGrantId, revokeLocalTimelineSession({ sessionId, subjectId }) { + if (!enableLocalTimeline) { + throw new Error( + "The selected connector does not support Timeline consent" + ) + } return store.revokeBoundSession({ sessionId, subjectId, @@ -181,6 +219,21 @@ export function createGithubAuthorizationAdapter({ } } +/** GitHub remains the default authorization composition for deployed clients. */ +export function createGithubAuthorizationAdapter(options = {}) { + const githubScopes = { + user: "github.profile", + repositories: "github.repositories", + starred: "github.starred", + } + return createPdppAuthorizationAdapter({ + ...options, + connectorId: options.connectorId ?? "github-pdpp", + scopeForStream: stream => githubScopes[stream], + enableLocalTimeline: true, + }) +} + function requiredLocalSession(sessionId) { if (typeof sessionId !== "string" || !sessionId.trim()) { throw new Error("session_id is required") diff --git a/personal-server/pdpp/github-authorization/policy.js b/personal-server/pdpp/github-authorization/policy.js index cc3ce0fd..dae832a0 100644 --- a/personal-server/pdpp/github-authorization/policy.js +++ b/personal-server/pdpp/github-authorization/policy.js @@ -157,10 +157,12 @@ function normalizeSelectedFields(stream, selected) { * currently hash-verified installed manifest. The stream-to-scope check binds * this additional authorization to the legacy Session Relay authorization. */ -export function validateGithubAuthorizationDetails({ +export function validateAuthorizationDetails({ authorizationDetails, manifest, scopes, + scopeForStream, + sourceIds = [manifest?.connector_key, manifest?.connector_id], localTimeline = false, }) { if ( @@ -184,11 +186,15 @@ export function validateGithubAuthorizationDetails({ if ( !detail.source || detail.source.kind !== "connector" || - detail.source.id !== "github" || + !sourceIds.includes(detail.source.id) || Object.keys(detail.source).length !== 2 ) { - throw invalid("source must be { kind: 'connector', id: 'github' }") + throw invalid("source must identify the selected connector") } + const canonicalSourceId = string( + manifest.connector_key, + "manifest.connector_key is required" + ) if (!["single_use", "continuous"].includes(detail.access_mode)) { throw invalid('access_mode must be "single_use" or "continuous"') } @@ -263,22 +269,21 @@ export function validateGithubAuthorizationDetails({ return normalized }) - const requestedScopes = normalizedStreams.map(stream => - localTimeline - ? `pdpp.local.github.${stream.name}` - : GITHUB_STREAM_SCOPES[stream.name] - ) + const requestedScopes = normalizedStreams.map(stream => { + if (localTimeline) return `pdpp.local.github.${stream.name}` + return scopeForStream?.(stream.name) + }) if ( requestedScopes.some(scope => !scope) || !sameScopeSet(requestedScopes, scopes) ) { throw invalid( - "GitHub authorization details do not exactly match the claimed session scopes" + "Authorization details do not exactly match the claimed session scopes" ) } return { type: PDPP_DATA_ACCESS_TYPE, - source: { kind: "connector", id: "github" }, + source: { kind: "connector", id: canonicalSourceId }, access_mode: detail.access_mode, purpose_code: detail.purpose_code, purpose_description: detail.purpose_description, @@ -289,3 +294,12 @@ export function validateGithubAuthorizationDetails({ streams: normalizedStreams, } } + +/** GitHub policy is retained as the legacy Session Relay compatibility adapter. */ +export function validateGithubAuthorizationDetails(options) { + return validateAuthorizationDetails({ + ...options, + sourceIds: ["github", "https://registry.pdpp.org/connectors/github"], + scopeForStream: stream => GITHUB_STREAM_SCOPES[stream], + }) +} diff --git a/personal-server/pdpp/github-snapshot.js b/personal-server/pdpp/github-snapshot.js index 8f370b53..30957744 100644 --- a/personal-server/pdpp/github-snapshot.js +++ b/personal-server/pdpp/github-snapshot.js @@ -5,12 +5,7 @@ import { RecordsRepositoryError } from "./grant-scoped-records-repository.js" const DEFAULT_FILE_OPERATIONS = { readdir, readFile, stat } -async function visitJsonFiles( - directory, - files, - directories, - fileOperations -) { +async function visitJsonFiles(directory, files, directories, fileOperations) { directories.set( directory, directoryGeneration(await fileOperations.stat(directory)) @@ -26,10 +21,10 @@ async function visitJsonFiles( } /** - * Return lossless GitHub export candidates newest first. The repository validates + * Return lossless selected-profile export candidates newest first. The repository validates * their record envelopes transactionally before one is accepted for serving. */ -export async function findGithubSnapshotCandidates({ +export async function findSnapshotCandidates({ exportRoot, manifest, manifestDigest, @@ -79,7 +74,7 @@ export async function findGithubSnapshotCandidates({ } continue } - const result = await readGithubSnapshotCandidate({ + const result = await readSnapshotCandidate({ path, generation, metadata, @@ -112,7 +107,7 @@ export async function findGithubSnapshotCandidates({ * record validation. `importSnapshot` is one SQLite transaction, so a rejected * candidate cannot leave partial collection state behind. */ -export async function importLatestGithubSnapshot({ +export async function importLatestSnapshot({ exportRoot, manifest, manifestDigest, @@ -127,7 +122,7 @@ export async function importLatestGithubSnapshot({ }, fileOperations = DEFAULT_FILE_OPERATIONS, }) { - const candidates = await findGithubSnapshotCandidates({ + const candidates = await findSnapshotCandidates({ exportRoot, manifest, manifestDigest, @@ -222,7 +217,7 @@ async function loadCachedCandidate({ if (fileGeneration(candidate.path, metadata) !== candidate.generation) { return null } - const result = await readGithubSnapshotCandidate({ + const result = await readSnapshotCandidate({ path: candidate.path, generation: candidate.generation, metadata, @@ -237,7 +232,7 @@ async function loadCachedCandidate({ } } -async function readGithubSnapshotCandidate({ +async function readSnapshotCandidate({ path, generation, metadata, @@ -260,7 +255,7 @@ async function readGithubSnapshotCandidate({ const content = exportFile?.content const recordsByStream = content?.["pdpp.recordsByStream"] if ( - content?.platform !== "github" || + content?.platform !== manifest.connector_key || content?.version !== manifest.version || !hasVerifiedSnapshotProvenance({ provenance: content?.["pdpp.provenance"], @@ -284,6 +279,10 @@ async function readGithubSnapshotCandidate({ } } +/** GitHub names remain available to avoid an unrelated call-site migration. */ +export const findGithubSnapshotCandidates = findSnapshotCandidates +export const importLatestGithubSnapshot = importLatestSnapshot + function fileGeneration(path, metadata) { return `${path}\0${metadata.mtimeMs}:${metadata.size}` } diff --git a/personal-server/pdpp/grant-scoped-records-repository.js b/personal-server/pdpp/grant-scoped-records-repository.js index 7eb92489..7cc0a294 100644 --- a/personal-server/pdpp/grant-scoped-records-repository.js +++ b/personal-server/pdpp/grant-scoped-records-repository.js @@ -67,21 +67,29 @@ export const GITHUB_STREAMS = Object.freeze({ }) /** - * Turn the hash-verified installed profile into the repository's narrow - * record contract. We deliberately support the profile's declared GitHub + * Turn the hash-verified selected profile into the repository's narrow + * record contract. We deliberately support only the profile's declared * streams, not an independently-maintained stream-name list. */ -export function createGithubStreamMetadata(manifest) { +export function createStreamMetadata(manifest) { if (!isPlainObject(manifest) || !Array.isArray(manifest.streams)) { - throw new TypeError("Installed GitHub manifest must declare streams") + throw new TypeError("Installed connector manifest must declare streams") } const streams = {} for (const stream of manifest.streams) { - if (!isPlainObject(stream) || typeof stream.name !== "string" || !stream.name) { - throw new TypeError("Installed GitHub manifest stream must have a name") + if ( + !isPlainObject(stream) || + typeof stream.name !== "string" || + !stream.name + ) { + throw new TypeError( + "Installed connector manifest stream must have a name" + ) } if (Object.hasOwn(streams, stream.name)) { - throw new TypeError(`Installed GitHub manifest has duplicate stream '${stream.name}'`) + throw new TypeError( + `Installed connector manifest has duplicate stream '${stream.name}'` + ) } const schema = stream.schema const properties = schema?.properties @@ -93,14 +101,17 @@ export function createGithubStreamMetadata(manifest) { !Array.isArray(stream.primary_key) || stream.primary_key.length !== 1 || stream.primary_key[0] !== "id" || - !requiredFields.includes("id") + !requiredFields.includes("id") || + typeof stream.cursor_field !== "string" ) { throw new TypeError( - `Installed GitHub stream '${stream.name}' has an unsupported record contract` + `Installed connector stream '${stream.name}' has an unsupported record contract` ) } const fields = Object.keys(properties) - const timingFields = [stream.cursor_field, stream.consent_time_field] + const timingFields = [stream.cursor_field] + if (stream.consent_time_field !== undefined) + timingFields.push(stream.consent_time_field) if ( timingFields.some( field => @@ -108,11 +119,13 @@ export function createGithubStreamMetadata(manifest) { !fields.includes(field) || !["date", "date-time"].includes(properties[field]?.format) ) || - requiredFields.some(field => typeof field !== "string" || !fields.includes(field)) || + requiredFields.some( + field => typeof field !== "string" || !fields.includes(field) + ) || fields.some(field => !validSchemaProperty(properties[field])) ) { throw new TypeError( - `Installed GitHub stream '${stream.name}' has an unsupported record contract` + `Installed connector stream '${stream.name}' has an unsupported record contract` ) } streams[stream.name] = Object.freeze({ @@ -121,7 +134,10 @@ export function createGithubStreamMetadata(manifest) { consentTimeField: stream.consent_time_field, fields, fieldTypes: Object.fromEntries( - fields.map(field => [field, normalizeSchemaTypes(properties[field].type)]) + fields.map(field => [ + field, + normalizeSchemaTypes(properties[field].type), + ]) ), fieldFormats: Object.fromEntries( fields.map(field => [field, properties[field].format ?? null]) @@ -131,6 +147,9 @@ export function createGithubStreamMetadata(manifest) { return Object.freeze(streams) } +/** GitHub-named export retained for existing callers and tests. */ +export const createGithubStreamMetadata = createStreamMetadata + function validSchemaProperty(property) { return ( isPlainObject(property) && @@ -143,7 +162,17 @@ function validSchemaProperty(property) { function normalizeSchemaTypes(value) { const types = Array.isArray(value) ? value : [value] - return types.every(type => ["string", "integer", "number", "boolean", "array", "object", "null"].includes(type)) + return types.every(type => + [ + "string", + "integer", + "number", + "boolean", + "array", + "object", + "null", + ].includes(type) + ) ? types : [] } @@ -217,18 +246,24 @@ export class GrantScopedRecordsRepository { } upsert({ connectionId, stream, key, data, emittedAt = this.#now() }) { - const identity = validateLiveRecord({ - connectionId, - stream, - key, - data, - emittedAt, - }, this.#streamMetadata) + const identity = validateLiveRecord( + { + connectionId, + stream, + key, + data, + emittedAt, + }, + this.#streamMetadata + ) return this.#mutate(identity, "upsert") } delete({ connectionId, stream, key, emittedAt = this.#now() }) { - const identity = validateDelete({ connectionId, stream, key, emittedAt }, this.#streamMetadata) + const identity = validateDelete( + { connectionId, stream, key, emittedAt }, + this.#streamMetadata + ) return this.#mutate(identity, "delete") } @@ -245,7 +280,11 @@ export class GrantScopedRecordsRepository { "recordsByStream must be an object" ) } - const snapshotMetadata = validateSnapshotMetadata(snapshot, recordsByStream, this.#streamMetadata) + const snapshotMetadata = validateSnapshotMetadata( + snapshot, + recordsByStream, + this.#streamMetadata + ) const apply = this.#db.transaction(() => { const results = [] @@ -269,24 +308,30 @@ export class GrantScopedRecordsRepository { } const op = envelope.op ?? "upsert" if (op === "upsert") { - const record = validateLiveRecord({ - connectionId, - stream, - key: envelope.key, - data: envelope.data, - emittedAt: envelope.emitted_at, - }, this.#streamMetadata) + const record = validateLiveRecord( + { + connectionId, + stream, + key: envelope.key, + data: envelope.data, + emittedAt: envelope.emitted_at, + }, + this.#streamMetadata + ) observedKeysByResetStream.get(stream)?.add(record.key) results.push(this.#mutateInTransaction(record, "upsert")) } else if (op === "delete") { results.push( this.#mutateInTransaction( - validateDelete({ - connectionId, - stream, - key: envelope.key, - emittedAt: envelope.emitted_at, - }, this.#streamMetadata), + validateDelete( + { + connectionId, + stream, + key: envelope.key, + emittedAt: envelope.emitted_at, + }, + this.#streamMetadata + ), "delete" ) ) @@ -315,7 +360,12 @@ export class GrantScopedRecordsRepository { getCurrent({ connectionId, stream, key, grant }) { validateLocation({ connectionId, stream, key }, this.#streamMetadata) - const effectiveGrant = normalizeGrant(stream, grant, undefined, this.#streamMetadata) + const effectiveGrant = normalizeGrant( + stream, + grant, + undefined, + this.#streamMetadata + ) const row = this.#db .prepare( ` @@ -331,7 +381,12 @@ export class GrantScopedRecordsRepository { summarizeCurrent({ connectionId, stream, grant }) { validateLocation({ connectionId, stream }, this.#streamMetadata) - const effectiveGrant = normalizeGrant(stream, grant, undefined, this.#streamMetadata) + const effectiveGrant = normalizeGrant( + stream, + grant, + undefined, + this.#streamMetadata + ) const metadata = this.#streamMetadata[stream] const visible = this.#db .prepare( @@ -343,17 +398,26 @@ export class GrantScopedRecordsRepository { ) .all(connectionId, stream) .map(row => { - const record = discloseLiveRow(row, stream, effectiveGrant, this.#streamMetadata) + const record = discloseLiveRow( + row, + stream, + effectiveGrant, + this.#streamMetadata + ) return record === null ? null - : { cursorValue: JSON.parse(row.payload)[metadata.cursorField], record } + : { + cursorValue: JSON.parse(row.payload)[metadata.cursorField], + record, + } }) .filter(entry => entry !== null) - const updated = visible - .map(entry => entry.cursorValue) - .filter(value => typeof value === "string") - .sort() - .at(-1) ?? null + const updated = + visible + .map(entry => entry.cursorValue) + .filter(value => typeof value === "string") + .sort() + .at(-1) ?? null return { record_count: visible.length, last_updated: updated } } @@ -370,7 +434,12 @@ export class GrantScopedRecordsRepository { validateLocation({ connectionId, stream }, this.#streamMetadata) const normalizedLimit = normalizeLimit(limit) const normalizedOrder = normalizeOrder(order) - const effectiveGrant = normalizeGrant(stream, grant, fields, this.#streamMetadata) + const effectiveGrant = normalizeGrant( + stream, + grant, + fields, + this.#streamMetadata + ) const pageCursor = cursor ? decodeCurrentCursor(cursor, connectionId, stream, normalizedOrder) : null @@ -386,15 +455,21 @@ export class GrantScopedRecordsRepository { const visible = rows .map(row => { - const record = discloseLiveRow(row, stream, effectiveGrant, this.#streamMetadata) + const record = discloseLiveRow( + row, + stream, + effectiveGrant, + this.#streamMetadata + ) return record === null ? null : { record, key: row.record_key, cursorValue: - JSON.parse(row.payload)[this.#streamMetadata[stream].cursorField] ?? - null, + JSON.parse(row.payload)[ + this.#streamMetadata[stream].cursorField + ] ?? null, } }) .filter( @@ -444,7 +519,12 @@ export class GrantScopedRecordsRepository { }) { validateLocation({ connectionId, stream }, this.#streamMetadata) const normalizedLimit = normalizeLimit(limit) - const effectiveGrant = normalizeGrant(stream, grant, undefined, this.#streamMetadata) + const effectiveGrant = normalizeGrant( + stream, + grant, + undefined, + this.#streamMetadata + ) const session = cursor ? decodeChangesCursor(cursor, connectionId, stream) : this.#startChangesSession(connectionId, stream, changesSince) @@ -655,7 +735,10 @@ export class GrantScopedRecordsRepository { .filter(key => !presentKeys.has(key)) .map(key => this.#mutateInTransaction( - validateDelete({ connectionId, stream, key, emittedAt }, this.#streamMetadata), + validateDelete( + { connectionId, stream, key, emittedAt }, + this.#streamMetadata + ), "delete" ) ) @@ -773,7 +856,9 @@ export class GrantScopedRecordsRepository { change.record_key, sinceVersion ) - const beforeRecord = before ? discloseSnapshot(before, stream, grant, this.#streamMetadata) : null + const beforeRecord = before + ? discloseSnapshot(before, stream, grant, this.#streamMetadata) + : null const afterRecord = change.deleted ? null : discloseSnapshot(change, stream, grant, this.#streamMetadata) @@ -807,7 +892,10 @@ export class GrantScopedRecordsRepository { } } -function validateLiveRecord({ connectionId, stream, key, data, emittedAt }, streams) { +function validateLiveRecord( + { connectionId, stream, key, data, emittedAt }, + streams +) { validateLocation({ connectionId, stream, key }, streams) if (!isPlainObject(data)) throw new RecordsRepositoryError( @@ -816,7 +904,12 @@ function validateLiveRecord({ connectionId, stream, key, data, emittedAt }, stre ) const metadata = requireStream(stream, streams) for (const field of metadata.requiredFields) { - if (!matchesSchemaType(data[field], metadata.fieldTypes?.[field] ?? ["string"])) { + if ( + !matchesSchemaType( + data[field], + metadata.fieldTypes?.[field] ?? ["string"] + ) + ) { throw new RecordsRepositoryError( "invalid_record", `Stream '${stream}' requires a valid '${field}'` @@ -829,7 +922,13 @@ function validateLiveRecord({ connectionId, stream, key, data, emittedAt }, stre "Record key must equal data.id" ) } - if (!validTemporalValue(data[metadata.consentTimeField], metadata.fieldFormats?.[metadata.consentTimeField] ?? "date-time")) { + if ( + metadata.consentTimeField && + !validTemporalValue( + data[metadata.consentTimeField], + metadata.fieldFormats?.[metadata.consentTimeField] ?? "date-time" + ) + ) { throw new RecordsRepositoryError( "invalid_record", `Stream '${stream}' requires a valid '${metadata.consentTimeField}' timestamp` @@ -837,7 +936,10 @@ function validateLiveRecord({ connectionId, stream, key, data, emittedAt }, stre } if ( data[metadata.cursorField] != null && - !validTemporalValue(data[metadata.cursorField], metadata.fieldFormats?.[metadata.cursorField] ?? "date-time") + !validTemporalValue( + data[metadata.cursorField], + metadata.fieldFormats?.[metadata.cursorField] ?? "date-time" + ) ) { throw new RecordsRepositoryError( "invalid_record", @@ -990,6 +1092,12 @@ function normalizeGrant(stream, grant = {}, requestedFields, streams) { ? null : normalizeStringList(grant.resources, "grant.resources") const timeRange = grant.timeRange ?? grant.time_range + if (timeRange !== undefined && !metadata.consentTimeField) { + throw new RecordsRepositoryError( + "invalid_request", + `Stream '${stream}' does not support time_range` + ) + } if ( timeRange !== undefined && (!isPlainObject(timeRange) || @@ -1039,6 +1147,7 @@ function discloseSnapshot(row, stream, grant, streams) { const metadata = requireStream(stream, streams) if (grant.resources && !grant.resources.includes(data.id)) return null const time = data[metadata.consentTimeField] + if (grant.timeRange && time == null) return null if ( grant.timeRange?.since && Date.parse(time) < Date.parse(grant.timeRange.since) @@ -1054,7 +1163,9 @@ function discloseSnapshot(row, stream, grant, streams) { ) const projected = grant.fields ? Object.fromEntries( - Object.entries(manifestData).filter(([field]) => grant.fields.includes(field)) + Object.entries(manifestData).filter(([field]) => + grant.fields.includes(field) + ) ) : manifestData return { @@ -1123,9 +1234,14 @@ function compareCurrentRows(left, right, order) { function matchesSchemaType(value, types) { if (value === null) return types.includes("null") - if (typeof value === "string") return types.includes("string") && value.length > 0 + if (typeof value === "string") + return types.includes("string") && value.length > 0 if (typeof value === "number") - return (types.includes("number") || (Number.isInteger(value) && types.includes("integer"))) && Number.isFinite(value) + return ( + (types.includes("number") || + (Number.isInteger(value) && types.includes("integer"))) && + Number.isFinite(value) + ) if (typeof value === "boolean") return types.includes("boolean") if (Array.isArray(value)) return types.includes("array") return isPlainObject(value) && types.includes("object") @@ -1133,7 +1249,12 @@ function matchesSchemaType(value, types) { function validTemporalValue(value, format) { if (value == null) return false - if (format === "date") return typeof value === "string" && /^\d{4}-\d{2}-\d{2}$/.test(value) && !Number.isNaN(Date.parse(`${value}T00:00:00Z`)) + if (format === "date") + return ( + typeof value === "string" && + /^\d{4}-\d{2}-\d{2}$/.test(value) && + !Number.isNaN(Date.parse(`${value}T00:00:00Z`)) + ) return isIsoTimestamp(value) } diff --git a/personal-server/pdpp/grant-scoped-records-repository.test.js b/personal-server/pdpp/grant-scoped-records-repository.test.js index d5497ea9..366f656e 100644 --- a/personal-server/pdpp/grant-scoped-records-repository.test.js +++ b/personal-server/pdpp/grant-scoped-records-repository.test.js @@ -6,6 +6,7 @@ import test from "node:test" import { CursorExpiredError, + createStreamMetadata, GrantScopedRecordsRepository, RecordsRepositoryError, } from "./grant-scoped-records-repository.js" @@ -28,6 +29,52 @@ function withRepository(options = {}) { } } +test("supports streams without consent_time_field but rejects unsupported serving contracts", () => { + const withoutConsentTime = { + streams: [ + { + name: "events", + primary_key: ["id"], + cursor_field: "updated_at", + schema: { + required: ["id"], + properties: { + id: { type: "string" }, + updated_at: { type: ["string", "null"], format: "date-time" }, + }, + }, + }, + ], + } + assert.equal( + createStreamMetadata(withoutConsentTime).events.consentTimeField, + undefined + ) + for (const invalid of [ + { + ...withoutConsentTime, + streams: [ + { ...withoutConsentTime.streams[0], primary_key: ["id", "other"] }, + ], + }, + { + ...withoutConsentTime, + streams: [{ ...withoutConsentTime.streams[0], cursor_field: "id" }], + }, + { + ...withoutConsentTime, + streams: [ + { ...withoutConsentTime.streams[0], schema: { required: ["id"] } }, + ], + }, + ]) { + assert.throws( + () => createStreamMetadata(invalid), + /unsupported record contract/ + ) + } +}) + const TIMES = { created: "2026-01-01T00:00:00.000Z", updated: "2026-02-01T00:00:00.000Z", @@ -331,6 +378,81 @@ test("a malformed authoritative full refresh does not partially alter current re } }) +test("rejects missing, null, and invalid declared consent time atomically", () => { + const harness = withRepository() + const connectionId = "github-account-a" + try { + harness.repository.importSnapshot({ + connectionId, + recordsByStream: { + repositories: [ + { + stream: "repositories", + key: "existing", + data: record("repositories", "existing"), + emitted_at: TIMES.emitted, + }, + ], + }, + }) + + for (const [name, extra] of [ + ["missing", { created_at: undefined }], + ["null", { created_at: null }], + ["invalid", { created_at: "not-a-date" }], + ]) { + assert.throws( + () => + harness.repository.importSnapshot({ + connectionId, + recordsByStream: { + repositories: [ + { + stream: "repositories", + key: "new-but-rolled-back", + data: record("repositories", "new-but-rolled-back"), + emitted_at: TIMES.emitted, + }, + { + stream: "repositories", + key: name, + data: record("repositories", name, extra), + emitted_at: TIMES.emitted, + }, + ], + }, + snapshot: { + collection_mode: "full_refresh", + reset_streams: ["repositories"], + completed_at: "2026-07-30T20:00:00.000Z", + }, + }), + error => + error instanceof RecordsRepositoryError && + error.code === "invalid_record" + ) + assert.notEqual( + harness.repository.getCurrent({ + connectionId, + stream: "repositories", + key: "existing", + }), + null + ) + assert.equal( + harness.repository.getCurrent({ + connectionId, + stream: "repositories", + key: "new-but-rolled-back", + }), + null + ) + } + } finally { + harness.dispose() + } +}) + test("upserts and deletes are atomic, idempotent, and survive reopen", () => { const harness = withRepository() try { @@ -629,7 +751,10 @@ test("manifest fields bound every disclosure while retaining lossless extension emittedAt: TIMES.emitted, }) - const unrestrictedCurrent = harness.repository.listCurrent({ ...base, grant: {} }) + const unrestrictedCurrent = harness.repository.listCurrent({ + ...base, + grant: {}, + }) assert.equal(unrestrictedCurrent.data[0].data.extra, undefined) assert.equal( harness.repository.getCurrent({ ...base, grant: {} }).data.extra, @@ -644,7 +769,8 @@ test("manifest fields bound every disclosure while retaining lossless extension const narrowGrant = { fields: ["id", "full_name"] } assert.deepEqual( - harness.repository.listCurrent({ ...base, grant: narrowGrant }).data[0].data, + harness.repository.listCurrent({ ...base, grant: narrowGrant }).data[0] + .data, { id: "1", full_name: "owner/repo-1" } ) assert.deepEqual( diff --git a/personal-server/pdpp/installed-manifest.js b/personal-server/pdpp/installed-manifest.js index f8f6c91b..11a27452 100644 --- a/personal-server/pdpp/installed-manifest.js +++ b/personal-server/pdpp/installed-manifest.js @@ -1,107 +1,177 @@ -import { createHash } from "node:crypto"; -import { existsSync, readFileSync } from "node:fs"; -import { homedir } from "node:os"; -import { isAbsolute, join, relative, resolve } from "node:path"; +import { createHash } from "node:crypto" +import { existsSync, readFileSync } from "node:fs" +import { homedir } from "node:os" +import { isAbsolute, join, relative, resolve } from "node:path" -const ACTIVE_CONNECTOR_ID = "github-pdpp"; -const CONNECTOR_ID = "https://registry.pdpp.org/connectors/github"; -const CONNECTOR_KEY = "github"; -const ARTIFACT_KIND = "pdpp-collection-profile"; +const ACTIVE_CONNECTOR_ID = "github-pdpp" +const ARTIFACT_KIND = "pdpp-collection-profile" -function fail(message) { - throw new Error(`Invalid installed PDPP GitHub connector: ${message}`); +function fail(label, message) { + throw new Error(`Invalid installed PDPP ${label}: ${message}`) } -function readJson(path, label) { +function readJson(path, label, connectorLabel) { try { - return JSON.parse(readFileSync(path, "utf8")); + return JSON.parse(readFileSync(path, "utf8")) } catch (error) { - fail(`could not read ${label}: ${error instanceof Error ? error.message : String(error)}`); + fail( + connectorLabel, + `could not read ${label}: ${error instanceof Error ? error.message : String(error)}` + ) } } -function requireString(value, field) { - if (typeof value !== "string" || value.length === 0) fail(`${field} is required`); - return value; +function requireString(value, field, connectorLabel) { + if (typeof value !== "string" || value.length === 0) + fail(connectorLabel, `${field} is required`) + return value } -function confinedFile(root, relativePath, label) { - requireString(relativePath, label); - if (isAbsolute(relativePath)) fail(`${label} must be relative`); +function confinedFile(root, relativePath, label, connectorLabel) { + requireString(relativePath, label, connectorLabel) + if (isAbsolute(relativePath)) + fail(connectorLabel, `${label} must be relative`) - const path = resolve(root, relativePath); - const escaped = relative(root, path).startsWith("..") || isAbsolute(relative(root, path)); - if (escaped) fail(`${label} escapes the install root`); - if (!existsSync(path)) fail(`${label} is not accessible`); - return path; + const path = resolve(root, relativePath) + const escaped = + relative(root, path).startsWith("..") || isAbsolute(relative(root, path)) + if (escaped) fail(connectorLabel, `${label} escapes the install root`) + if (!existsSync(path)) fail(connectorLabel, `${label} is not accessible`) + return path } -function verifyHash(path, expected, label) { - requireString(expected, `${label} hash`); - if (!expected.startsWith("sha256:")) fail(`${label} hash must be sha256`); - const actual = `sha256:${createHash("sha256").update(readFileSync(path)).digest("hex")}`; - if (actual !== expected) fail(`${label} hash does not match the active install`); +function verifyHash(path, expected, label, connectorLabel) { + requireString(expected, `${label} hash`, connectorLabel) + if (!expected.startsWith("sha256:")) + fail(connectorLabel, `${label} hash must be sha256`) + const actual = `sha256:${createHash("sha256").update(readFileSync(path)).digest("hex")}` + if (actual !== expected) + fail(connectorLabel, `${label} hash does not match the active install`) } -function validateManifest(install, manifest) { - if (manifest === null || typeof manifest !== "object" || Array.isArray(manifest)) { - fail("manifest must be an object"); +function validateManifest( + install, + manifest, + connectorLabel, + expectedConnector +) { + if ( + manifest === null || + typeof manifest !== "object" || + Array.isArray(manifest) + ) { + fail(connectorLabel, "manifest must be an object") } if ( - manifest.connector_key !== CONNECTOR_KEY || - manifest.connector_id !== CONNECTOR_ID + typeof manifest.connector_key !== "string" || + manifest.connector_key.length === 0 || + typeof manifest.connector_id !== "string" || + manifest.connector_id.length === 0 ) { - fail("manifest identity does not match the active install"); + fail(connectorLabel, "manifest must declare a connector key and ID") } if (manifest.version !== install.version) { - fail(`manifest version ${String(manifest.version)} does not match active version ${install.version}`); + fail( + connectorLabel, + `manifest version ${String(manifest.version)} does not match active version ${install.version}` + ) + } + if ( + expectedConnector && + (manifest.connector_key !== expectedConnector.key || + manifest.connector_id !== expectedConnector.id) + ) { + fail( + connectorLabel, + "manifest identity does not match the selected serving profile" + ) } if (!Array.isArray(manifest.streams) || manifest.streams.length === 0) { - fail("manifest must declare at least one stream"); + fail(connectorLabel, "manifest must declare at least one stream") } - const names = new Set(); + const names = new Set() for (const stream of manifest.streams) { - if (stream === null || typeof stream !== "object" || typeof stream.name !== "string" || stream.name.length === 0) { - fail("manifest stream names must be non-empty strings"); - } - if (!names.add(stream.name)) fail("manifest stream names must be unique"); - } - const network = manifest.runtime_requirements?.bindings?.network; - if (network?.required !== true) fail("manifest must require the network binding"); - for (const [binding, requirement] of Object.entries(manifest.runtime_requirements?.bindings ?? {})) { - if (binding !== "network" && requirement?.required === true) { - fail(`manifest requires unsupported binding ${binding}`); + if ( + stream === null || + typeof stream !== "object" || + typeof stream.name !== "string" || + stream.name.length === 0 + ) { + fail(connectorLabel, "manifest stream names must be non-empty strings") } + if (!names.add(stream.name)) + fail(connectorLabel, "manifest stream names must be unique") } } /** * Load the manifest and provenance from the active, hash-verified install. * Consumers receive the artifact's values; they do not maintain a second - * handwritten GitHub manifest or version constant. + * handwritten connector manifest or version constant. */ -export function loadInstalledGithubManifest({ - activeManifestPath = join(homedir(), ".dataconnect", "connectors-active.json"), - connectorId = ACTIVE_CONNECTOR_ID, +export function loadInstalledManifest({ + activeManifestPath = join( + homedir(), + ".dataconnect", + "connectors-active.json" + ), + connectorId, + connectorLabel = "connector", + expectedConnector, } = {}) { - if (connectorId !== ACTIVE_CONNECTOR_ID) fail(`unsupported connector ${connectorId}`); + requireString(connectorId, "selected connector", connectorLabel) - const active = readJson(activeManifestPath, "active connector manifest"); - const install = active?.connectors?.[connectorId]; - if (install === null || typeof install !== "object") fail("active install is missing"); - if (install.artifactKind !== ARTIFACT_KIND) fail("active install is not a collection profile artifact"); - requireString(install.version, "active install version"); + const active = readJson( + activeManifestPath, + "active connector manifest", + connectorLabel + ) + const install = active?.connectors?.[connectorId] + if (install === null || typeof install !== "object") + fail(connectorLabel, "active install is missing") + if (install.artifactKind !== ARTIFACT_KIND) + fail(connectorLabel, "active install is not a collection profile artifact") + if (install.connectorId !== undefined && install.connectorId !== connectorId) + fail(connectorLabel, "active install ID does not match selected connector") + requireString(install.version, "active install version", connectorLabel) - const root = resolve(requireString(install.rootPath, "active install root")); - const manifestPath = confinedFile(root, install.manifestPath, "manifest path"); - const entrypointPath = confinedFile(root, install.entrypointPath, "entrypoint path"); - const provenancePath = confinedFile(root, install.provenancePath, "provenance path"); - verifyHash(manifestPath, install.manifestSha256, "manifest"); - verifyHash(entrypointPath, install.entrypointSha256, "entrypoint"); - verifyHash(provenancePath, install.provenanceSha256, "provenance"); + const root = resolve( + requireString(install.rootPath, "active install root", connectorLabel) + ) + const manifestPath = confinedFile( + root, + install.manifestPath, + "manifest path", + connectorLabel + ) + const entrypointPath = confinedFile( + root, + install.entrypointPath, + "entrypoint path", + connectorLabel + ) + const provenancePath = confinedFile( + root, + install.provenancePath, + "provenance path", + connectorLabel + ) + verifyHash(manifestPath, install.manifestSha256, "manifest", connectorLabel) + verifyHash( + entrypointPath, + install.entrypointSha256, + "entrypoint", + connectorLabel + ) + verifyHash( + provenancePath, + install.provenanceSha256, + "provenance", + connectorLabel + ) - const manifest = readJson(manifestPath, "manifest"); - validateManifest(install, manifest); - const provenance = readJson(provenancePath, "provenance"); + const manifest = readJson(manifestPath, "manifest", connectorLabel) + validateManifest(install, manifest, connectorLabel, expectedConnector) + const provenance = readJson(provenancePath, "provenance", connectorLabel) return Object.freeze({ connectorId, @@ -112,5 +182,57 @@ export function loadInstalledGithubManifest({ manifestPath, entrypointPath, provenancePath, - }); + }) +} + +/** + * Re-read one explicit active-install path and fail closed if it no longer + * resolves to the selection that was composed at startup. This prevents the + * authorization and resource surfaces from cross-binding same-identity + * artifacts from different active manifests. + */ +export function resolveSelectedInstalledManifest({ + activeManifestPath, + connectorId, + connectorLabel, + expectedConnector, + selectedInstall, +} = {}) { + const installed = loadInstalledManifest({ + activeManifestPath, + connectorId, + connectorLabel, + expectedConnector, + }) + if (!selectedInstall) return installed + if ( + selectedInstall.connectorId !== installed.connectorId || + selectedInstall.version !== installed.version || + selectedInstall.manifestDigest !== installed.manifestDigest || + selectedInstall.manifest.connector_key !== installed.manifest.connector_key || + selectedInstall.manifest.connector_id !== installed.manifest.connector_id + ) { + fail( + connectorLabel ?? "connector", + "active install no longer matches the composed serving selection" + ) + } + return selectedInstall +} + +/** GitHub remains the default serving profile for existing deployments. */ +export function loadInstalledGithubManifest(options = {}) { + const connectorId = options.connectorId ?? ACTIVE_CONNECTOR_ID + if (connectorId !== ACTIVE_CONNECTOR_ID) { + fail("GitHub connector", `unsupported connector ${connectorId}`) + } + return loadInstalledManifest({ + ...options, + connectorId, + connectorLabel: "GitHub connector", + expectedConnector: { + key: "github", + id: "https://registry.pdpp.org/connectors/github", + }, + }) } diff --git a/personal-server/pdpp/installed-manifest.test.js b/personal-server/pdpp/installed-manifest.test.js index 906ba62d..2ab650c8 100644 --- a/personal-server/pdpp/installed-manifest.test.js +++ b/personal-server/pdpp/installed-manifest.test.js @@ -1,22 +1,25 @@ -import { createHash } from "node:crypto"; -import assert from "node:assert/strict"; -import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterEach, describe, it } from "node:test"; -import { loadInstalledGithubManifest } from "./installed-manifest.js"; +import { createHash } from "node:crypto" +import assert from "node:assert/strict" +import { mkdtempSync, mkdirSync, readFileSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { afterEach, describe, it } from "node:test" +import { + loadInstalledGithubManifest, + loadInstalledManifest, +} from "./installed-manifest.js" -const tempRoots = []; +const tempRoots = [] function hash(bytes) { - return `sha256:${createHash("sha256").update(bytes).digest("hex")}`; + return `sha256:${createHash("sha256").update(bytes).digest("hex")}` } function fixture(overrides = {}) { - const root = mkdtempSync(join(tmpdir(), "dataconnect-pdpp-manifest-")); - tempRoots.push(root); - mkdirSync(join(root, "profile")); - mkdirSync(join(root, "dist")); + const root = mkdtempSync(join(tmpdir(), "dataconnect-pdpp-manifest-")) + tempRoots.push(root) + mkdirSync(join(root, "profile")) + mkdirSync(join(root, "dist")) const manifest = { protocol_version: "0.1.0", connector_id: "https://registry.pdpp.org/connectors/github", @@ -25,13 +28,15 @@ function fixture(overrides = {}) { runtime_requirements: { bindings: { network: { required: true } } }, streams: [{ name: "user" }], ...overrides.manifest, - }; - const manifestBytes = Buffer.from(JSON.stringify(manifest)); - const entrypointBytes = Buffer.from("export default {};"); - const provenanceBytes = Buffer.from(JSON.stringify({ source: "fixture", version: "0.5.0" })); - writeFileSync(join(root, "profile/collection-profile.json"), manifestBytes); - writeFileSync(join(root, "dist/collection-profile.mjs"), entrypointBytes); - writeFileSync(join(root, "provenance.json"), provenanceBytes); + } + const manifestBytes = Buffer.from(JSON.stringify(manifest)) + const entrypointBytes = Buffer.from("export default {};") + const provenanceBytes = Buffer.from( + JSON.stringify({ source: "fixture", version: "0.5.0" }) + ) + writeFileSync(join(root, "profile/collection-profile.json"), manifestBytes) + writeFileSync(join(root, "dist/collection-profile.mjs"), entrypointBytes) + writeFileSync(join(root, "provenance.json"), provenanceBytes) const install = { connectorId: "github-pdpp", version: "0.5.0", @@ -44,27 +49,34 @@ function fixture(overrides = {}) { entrypointSha256: hash(entrypointBytes), provenanceSha256: hash(provenanceBytes), ...overrides.install, - }; - const activePath = join(root, "connectors-active.json"); - writeFileSync(activePath, JSON.stringify({ connectors: { "github-pdpp": install } })); - return { activePath, root }; + } + const activePath = join(root, "connectors-active.json") + writeFileSync( + activePath, + JSON.stringify({ connectors: { "github-pdpp": install } }) + ) + return { activePath, root } } afterEach(async () => { - const { rm } = await import("node:fs/promises"); - await Promise.all(tempRoots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); -}); + const { rm } = await import("node:fs/promises") + await Promise.all( + tempRoots.splice(0).map(root => rm(root, { recursive: true, force: true })) + ) +}) describe("loadInstalledGithubManifest", () => { it("returns the actual validated manifest with provenance and version", () => { - const { activePath } = fixture(); - const loaded = loadInstalledGithubManifest({ activeManifestPath: activePath }); - assert.equal(loaded.version, "0.5.0"); - assert.equal(loaded.manifestDigest.startsWith("sha256:"), true); - assert.equal(loaded.manifest.connector_key, "github"); - assert.equal(loaded.manifest.streams[0].name, "user"); - assert.deepEqual(loaded.provenance, { source: "fixture", version: "0.5.0" }); - }); + const { activePath } = fixture() + const loaded = loadInstalledGithubManifest({ + activeManifestPath: activePath, + }) + assert.equal(loaded.version, "0.5.0") + assert.equal(loaded.manifestDigest.startsWith("sha256:"), true) + assert.equal(loaded.manifest.connector_key, "github") + assert.equal(loaded.manifest.streams[0].name, "user") + assert.deepEqual(loaded.provenance, { source: "fixture", version: "0.5.0" }) + }) for (const [label, overrides] of [ ["manifest version", { manifest: { version: "0.6.0" } }], @@ -77,11 +89,38 @@ describe("loadInstalledGithubManifest", () => { ["provenance digest", { install: { provenanceSha256: "sha256:bad" } }], ]) { it(`fails closed on ${label} mismatch`, () => { - const { activePath } = fixture(overrides); + const { activePath } = fixture(overrides) assert.throws( () => loadInstalledGithubManifest({ activeManifestPath: activePath }), - /Invalid installed PDPP GitHub connector/, - ); - }); + /Invalid installed PDPP GitHub connector/ + ) + }) } -}); +}) + +it("loads a selected browser-bound ChatGPT profile after all artifact hashes verify", () => { + const { activePath } = fixture({ + manifest: { + connector_id: "https://registry.pdpp.org/connectors/chatgpt", + connector_key: "chatgpt", + runtime_requirements: { + bindings: { network: { required: true }, browser: { required: true } }, + }, + }, + install: { connectorId: "chatgpt-pdpp" }, + }) + const active = JSON.parse(readFileSync(activePath, "utf8")) + active.connectors["chatgpt-pdpp"] = active.connectors["github-pdpp"] + delete active.connectors["github-pdpp"] + writeFileSync(activePath, JSON.stringify(active)) + + const loaded = loadInstalledManifest({ + activeManifestPath: activePath, + connectorId: "chatgpt-pdpp", + expectedConnector: { + key: "chatgpt", + id: "https://registry.pdpp.org/connectors/chatgpt", + }, + }) + assert.equal(loaded.manifest.connector_key, "chatgpt") +}) diff --git a/personal-server/pdpp/resource-server.js b/personal-server/pdpp/resource-server.js index 1ea16294..66faccec 100644 --- a/personal-server/pdpp/resource-server.js +++ b/personal-server/pdpp/resource-server.js @@ -5,10 +5,10 @@ import { CursorExpiredError, GrantScopedRecordsRepository, RecordsRepositoryError, - createGithubStreamMetadata, + createStreamMetadata, } from "./grant-scoped-records-repository.js" -import { importLatestGithubSnapshot } from "./github-snapshot.js" -import { loadInstalledGithubManifest } from "./installed-manifest.js" +import { importLatestSnapshot } from "./github-snapshot.js" +import { resolveSelectedInstalledManifest } from "./installed-manifest.js" import { createCoreApp } from "./index.js" import { CoreOperationError } from "./operations.js" import { createHttpTokenIntrospector } from "./token-introspector.js" @@ -16,11 +16,14 @@ import { createHttpTokenIntrospector } from "./token-introspector.js" const DEFAULT_CONNECTION_ID = "default" /** - * Compose the validated installed GitHub profile, durable record store, and + * Compose the validated selected profile, durable record store, and * opaque-token boundary into the transport-independent Core route surface. */ export async function createPdppResourceServer({ activeManifestPath, + connectorId = "github-pdpp", + expectedConnector, + selectedInstall, databasePath, exportRoot, connectionId = DEFAULT_CONNECTION_ID, @@ -31,9 +34,15 @@ export async function createPdppResourceServer({ authorization: process.env.PDPP_INTROSPECTION_AUTHORIZATION, }), } = {}) { - const installed = loadInstalledGithubManifest({ activeManifestPath }) - const streamMetadata = createGithubStreamMetadata(installed.manifest) - const repository = recordsRepository ?? createRepository(databasePath, streamMetadata) + const installed = resolveSelectedInstalledManifest({ + activeManifestPath, + connectorId, + expectedConnector, + selectedInstall, + }) + const streamMetadata = createStreamMetadata(installed.manifest) + const repository = + recordsRepository ?? createRepository(databasePath, streamMetadata) const refreshSnapshot = createSnapshotRefresher({ exportRoot, manifest: installed.manifest, @@ -47,7 +56,7 @@ export async function createPdppResourceServer({ requestId, tokenIntrospector: createGrantValidatedIntrospector( tokenIntrospector, - [installed.manifest.connector_id, installed.manifest.connector_key] + installed ), recordsRepository: createCoreRepositoryPort({ repository, @@ -60,11 +69,15 @@ export async function createPdppResourceServer({ /** Add PDPP reads without taking ownership of legacy Personal Server routes. */ export async function mountPdppResourceServer(app, options) { const resourceServer = await createPdppResourceServer(options) + mountPdppResourceRoutes(app, resourceServer) + return resourceServer +} + +export function mountPdppResourceRoutes(app, resourceServer) { const serve = context => resourceServer.fetch(context.req.raw) app.get("/v1/streams", serve) app.get("/v1/streams/:stream/records", serve) app.get("/v1/streams/:stream/records/:recordId", serve) - return resourceServer } function createRepository(databasePath, streamMetadata) { @@ -77,7 +90,7 @@ function createRepository(databasePath, streamMetadata) { return new GrantScopedRecordsRepository({ databasePath, streamMetadata }) } -function createGrantValidatedIntrospector(tokenIntrospector, connectorIds) { +function createGrantValidatedIntrospector(tokenIntrospector, installed) { if (typeof tokenIntrospector?.introspect !== "function") { throw new TypeError("tokenIntrospector.introspect must be a function") } @@ -111,12 +124,17 @@ function createGrantValidatedIntrospector(tokenIntrospector, connectorIds) { } if ( identity.grant.source?.kind !== "connector" || - !connectorIds.includes(identity.grant.source?.id) + ![ + installed.manifest.connector_id, + installed.manifest.connector_key, + ].includes(identity.grant.source?.id) || + identity.grant.manifest_version !== installed.version || + identity.grant.manifest_digest !== installed.manifestDigest ) { throw new CoreOperationError( 403, "grant_invalid", - "The grant is not bound to the installed GitHub connector" + "The grant is not bound to the selected installed manifest" ) } return identity @@ -135,7 +153,7 @@ export function createSnapshotRefresher(dependencies) { } return async () => { if (inFlight !== null) return inFlight - inFlight = importLatestGithubSnapshot({ + inFlight = importLatestSnapshot({ ...dependencies, snapshotCache, }).finally(() => { diff --git a/personal-server/pdpp/resource-server.test.js b/personal-server/pdpp/resource-server.test.js index 61f15fc6..db2ec0fc 100644 --- a/personal-server/pdpp/resource-server.test.js +++ b/personal-server/pdpp/resource-server.test.js @@ -1,6 +1,6 @@ import assert from "node:assert/strict" import { createHash } from "node:crypto" -import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs" +import { mkdtempSync, mkdirSync, readFileSync, writeFileSync } from "node:fs" import { readdir, readFile, rm, stat } from "node:fs/promises" import { tmpdir } from "node:os" import { dirname, join } from "node:path" @@ -9,6 +9,12 @@ import { privateKeyToAccount } from "viem/accounts" import { Hono } from "hono" +import { + createPdppRevocationSink, + pdppDefaultStorageRoots, + pdppProfileDatabasePath, + registerOptionalPdppSurfaces, +} from "../index.js" import { registerProtectedRoutes } from "../protected-routes.js" import { createGithubStreamMetadata, @@ -16,13 +22,19 @@ import { } from "./grant-scoped-records-repository.js" import { createGithubAuthorizationAdapter, + createPdppAuthorizationAdapter, PDPP_DATA_ACCESS_TYPE, } from "./github-authorization/index.js" -import { registerGithubAuthorizationRoutes } from "./github-authorization/http-routes.js" +import { + registerGithubAuthorizationRoutes, + registerPdppAuthorizationRoutes, +} from "./github-authorization/http-routes.js" import { createSnapshotRefresher, mountPdppResourceServer, } from "./resource-server.js" +import { loadInstalledManifest } from "./installed-manifest.js" +import { canonicalChatgptManifestBytes } from "./test/fixtures/chatgpt.collection-profile.js" const tempRoots = [] @@ -193,14 +205,123 @@ function createInstalledGithubFixture({ allStreams = false } = {}) { }) ) return { + root, + installRoot, + activeManifestPath, + exportRoot, + databasePath: join(root, "records.sqlite"), + manifest, + manifestDigest, + } +} + +function createInstalledChatgptFixture() { + const root = mkdtempSync( + join(tmpdir(), "dataconnect-pdpp-chatgpt-resource-server-") + ) + tempRoots.push(root) + const installRoot = join(root, "install") + const exportRoot = join(root, "exports") + mkdirSync(join(installRoot, "profile"), { recursive: true }) + mkdirSync(join(installRoot, "dist"), { recursive: true }) + mkdirSync(exportRoot) + + const manifestBytes = canonicalChatgptManifestBytes + const manifest = JSON.parse(manifestBytes) + const manifestDigest = hash(manifestBytes) + const entrypointBytes = Buffer.from("export default {};") + const provenanceBytes = Buffer.from( + JSON.stringify({ source: "canonical-chatgpt" }) + ) + writeFileSync( + join(installRoot, "profile/collection-profile.json"), + manifestBytes + ) + writeFileSync( + join(installRoot, "dist/collection-profile.mjs"), + entrypointBytes + ) + writeFileSync(join(installRoot, "provenance.json"), provenanceBytes) + const activeManifestPath = join(root, "connectors-active.json") + writeFileSync( + activeManifestPath, + JSON.stringify({ + connectors: { + "chatgpt-pdpp": { + connectorId: "chatgpt-pdpp", + version: manifest.version, + rootPath: installRoot, + artifactKind: "pdpp-collection-profile", + manifestPath: "profile/collection-profile.json", + entrypointPath: "dist/collection-profile.mjs", + provenancePath: "provenance.json", + manifestSha256: manifestDigest, + entrypointSha256: hash(entrypointBytes), + provenanceSha256: hash(provenanceBytes), + }, + }, + }) + ) + return { + root, + installRoot, activeManifestPath, exportRoot, databasePath: join(root, "records.sqlite"), manifest, + manifestBytes, manifestDigest, } } +function chatgptSnapshotProvenance({ + manifestDigest, + connectionId = "chatgpt-account-a", + runId = "chatgpt-run-1", + overrides = {}, +}) { + return { + connector_key: "chatgpt", + connector_id: "https://registry.pdpp.org/connectors/chatgpt", + manifest_version: "0.1.0", + manifest_sha256: manifestDigest, + run_id: runId, + connection_id: connectionId, + ...overrides, + } +} + +function chatgptConversation(id, createTime, updateTime = createTime) { + return { + stream: "conversations", + key: id, + data: { + id, + title: `title-${id}`, + create_time: createTime, + update_time: updateTime, + is_archived: false, + }, + emitted_at: "2026-07-31T00:00:00.000Z", + } +} + +function rewriteInstalledManifest(fixture, mutate) { + const manifest = JSON.parse(JSON.stringify(fixture.manifest)) + mutate(manifest) + const manifestBytes = Buffer.from(JSON.stringify(manifest)) + writeFileSync( + join(fixture.installRoot, "profile/collection-profile.json"), + manifestBytes + ) + const active = JSON.parse(readFileSync(fixture.activeManifestPath)) + active.connectors["github-pdpp"].version = manifest.version + active.connectors["github-pdpp"].manifestSha256 = hash(manifestBytes) + writeFileSync(fixture.activeManifestPath, JSON.stringify(active)) + fixture.manifest = manifest + fixture.manifestDigest = hash(manifestBytes) +} + function githubSnapshotProvenance({ manifestDigest, connectionId = "default", @@ -282,7 +403,11 @@ function record(id, createdAt) { } } -function activeToken(overrides = {}) { +function activeToken({ + manifestDigest, + grant: grantOverrides, + ...overrides +} = {}) { return { active: true, pdpp_token_kind: "client", @@ -292,6 +417,8 @@ function activeToken(overrides = {}) { kind: "connector", id: "https://registry.pdpp.org/connectors/github", }, + manifest_version: "0.5.0", + manifest_digest: manifestDigest, streams: [ { name: "repositories", @@ -300,6 +427,7 @@ function activeToken(overrides = {}) { time_range: { since: "2026-01-01T00:00:00Z" }, }, ], + ...grantOverrides, }, ...overrides, } @@ -421,6 +549,8 @@ test("serves every verified GitHub stream while omitted export streams are empty id: "https://registry.pdpp.org/connectors/github", }, streams: streamNames.map(name => ({ name })), + manifest_version: fixture.manifest.version, + manifest_digest: fixture.manifestDigest, }, }), }, @@ -472,6 +602,7 @@ test("mounts installed GitHub PDPP streams beside legacy routes with opaque gran return { active: false, inactive_reason: "grant_expired" } if (token === "wrong-connector") { return activeToken({ + manifestDigest: fixture.manifestDigest, grant: { source: { kind: "connector", @@ -483,6 +614,7 @@ test("mounts installed GitHub PDPP streams beside legacy routes with opaque gran } if (token === "unrestricted-fields") { return activeToken({ + manifestDigest: fixture.manifestDigest, grant: { source: { kind: "connector", @@ -500,6 +632,7 @@ test("mounts installed GitHub PDPP streams beside legacy routes with opaque gran } if (token === "name-only") { return activeToken({ + manifestDigest: fixture.manifestDigest, grant: { source: { kind: "connector", @@ -516,7 +649,7 @@ test("mounts installed GitHub PDPP streams beside legacy routes with opaque gran }, }) } - return activeToken() + return activeToken({ manifestDigest: fixture.manifestDigest }) }, }, }) @@ -666,7 +799,10 @@ test("a successful authoritative full refresh removes records absent from the ne const app = new Hono() await mountPdppResourceServer(app, { ...fixture, - tokenIntrospector: { introspect: async () => activeToken() }, + tokenIntrospector: { + introspect: async () => + activeToken({ manifestDigest: fixture.manifestDigest }), + }, }) const headers = { authorization: "Bearer opaque" } const before = await app.request( @@ -766,7 +902,10 @@ test("serves only the requested connection's verified installed snapshot", async const defaultApp = new Hono() await mountPdppResourceServer(defaultApp, { ...fixture, - tokenIntrospector: { introspect: async () => activeToken() }, + tokenIntrospector: { + introspect: async () => + activeToken({ manifestDigest: fixture.manifestDigest }), + }, }) const defaultRead = await defaultApp.request( "http://personal.example/v1/streams/repositories/records", @@ -786,11 +925,13 @@ test("serves only the requested connection's verified installed snapshot", async tokenIntrospector: { introspect: async () => activeToken({ + manifestDigest: fixture.manifestDigest, grant: { - ...activeToken().grant, + ...activeToken({ manifestDigest: fixture.manifestDigest }).grant, streams: [ { - ...activeToken().grant.streams[0], + ...activeToken({ manifestDigest: fixture.manifestDigest }).grant + .streams[0], resources: ["other-account"], }, ], @@ -1263,6 +1404,366 @@ test("rejects a local PDPP bearer after owner revoke intent even when Gateway re adapter.close() }) +test("default GitHub serving retains legacy authorization and record database paths on upgrade", async () => { + const fixture = createInstalledGithubFixture() + const desktopToken = "desktop-token" + assert.equal( + pdppProfileDatabasePath(fixture.root, "github-pdpp", "authorization"), + join(fixture.root, "pdpp-github-authorization.sqlite") + ) + assert.equal( + pdppProfileDatabasePath(fixture.root, "github-pdpp", "records"), + join(fixture.root, "pdpp-github-records.sqlite") + ) + + const seededAdapter = createPdppAuthorizationAdapter({ + activeManifestPath: fixture.activeManifestPath, + connectorId: "github-pdpp", + expectedConnector: { + key: "github", + id: "https://registry.pdpp.org/connectors/github", + }, + databasePath: pdppProfileDatabasePath( + fixture.root, + "github-pdpp", + "authorization" + ), + scopeForStream: stream => + ({ + user: "github.profile", + repositories: "github.repositories", + starred: "github.starred", + })[stream], + }) + const consent = seededAdapter.createConsentRequest({ + sessionId: "legacy-upgrade-session", + scopes: ["github.repositories"], + authorizationDetails: [ + { + type: PDPP_DATA_ACCESS_TYPE, + source: { kind: "connector", id: "github" }, + access_mode: "continuous", + purpose_code: "https://example.test/purpose/research", + streams: [{ name: "repositories", resources: ["allowed"] }], + }, + ], + }) + const issued = seededAdapter.issueApprovedGrant({ + requestId: consent.request_id, + legacyGrantId: "legacy-upgrade-grant", + subjectId: "subject-1", + clientId: TEST_BUILDER.address, + }) + seededAdapter.close() + + const app = new Hono() + app.get("/health", context => context.json({ ok: true })) + const adapter = await registerOptionalPdppSurfaces({ + app, + devToken: desktopToken, + storageRoot: fixture.root, + recordsRoot: fixture.root, + activeManifestPath: fixture.activeManifestPath, + exportRoot: fixture.exportRoot, + connectionId: "default", + selectedPdppProfile: { + connectorId: "github-pdpp", + connector: { + key: "github", + id: "https://registry.pdpp.org/connectors/github", + }, + scopeForStream: stream => + ({ + user: "github.profile", + repositories: "github.repositories", + starred: "github.starred", + })[stream], + enableLocalTimeline: true, + }, + }) + assert.notEqual(adapter, null) + registerProtectedRoutes({ + app, + devToken: desktopToken, + gatewayClient: { revokeGrant: async () => {} }, + ownerAddress: "0xowner", + port: 8080, + send: () => {}, + serverSigner: { signGrantRevocation: async () => "signature" }, + onLegacyGrantRevoked: grantId => adapter.revokeByLegacyGrantId(grantId), + }) + + const headers = { authorization: `Bearer ${issued.access_token}` } + const streams = await app.request("http://personal.example/v1/streams", { + headers, + }) + assert.equal(streams.status, 200, await streams.clone().text()) + assert.deepEqual( + (await streams.json()).data.map(stream => stream.name), + ["repositories"] + ) + + const revoke = await app.request( + "http://personal.example/v1/grants/legacy-upgrade-grant", + { + method: "DELETE", + headers: { authorization: `Bearer ${desktopToken}` }, + } + ) + assert.equal(revoke.status, 204) + const revoked = await app.request("http://personal.example/v1/streams", { + headers, + }) + assert.equal(revoked.status, 403) + assert.equal((await revoked.json()).error.code, "grant_revoked") + adapter.close() +}) + +test("invalid selected install leaves legacy routes alive and PDPP routes absent", async () => { + const fixture = createInstalledGithubFixture() + const app = new Hono() + app.get("/health", context => context.json({ ok: true })) + const logs = [] + const adapter = await registerOptionalPdppSurfaces({ + app, + devToken: "desktop-token", + storageRoot: fixture.root, + recordsRoot: fixture.root, + activeManifestPath: join(fixture.root, "missing-active.json"), + exportRoot: fixture.exportRoot, + selectedPdppProfile: { + connectorId: "github-pdpp", + connector: { + key: "github", + id: "https://registry.pdpp.org/connectors/github", + }, + scopeForStream: stream => + ({ + user: "github.profile", + repositories: "github.repositories", + starred: "github.starred", + })[stream], + enableLocalTimeline: true, + }, + send: message => logs.push(message), + }) + + assert.equal(adapter, null) + assert.equal( + (await app.request("http://personal.example/health")).status, + 200 + ) + assert.equal( + (await app.request("http://personal.example/v1/streams")).status, + 404 + ) + assert.equal( + ( + await app.request("http://personal.example/v1/pdpp/consent-requests", { + method: "POST", + }) + ).status, + 404 + ) + assert.match(logs.at(-1).message, /routes unavailable/) +}) + +test("unsupported selected install atomically leaves all PDPP routes absent", async () => { + const fixture = createInstalledGithubFixture() + rewriteInstalledManifest(fixture, manifest => { + manifest.streams[0].primary_key = ["id", "full_name"] + }) + const app = new Hono() + app.get("/health", context => context.json({ ok: true })) + const logs = [] + const adapter = await registerOptionalPdppSurfaces({ + app, + devToken: "desktop-token", + storageRoot: fixture.root, + recordsRoot: fixture.root, + activeManifestPath: fixture.activeManifestPath, + exportRoot: fixture.exportRoot, + selectedPdppProfile: { + connectorId: "github-pdpp", + connector: { + key: "github", + id: "https://registry.pdpp.org/connectors/github", + }, + scopeForStream: stream => + ({ + user: "github.profile", + repositories: "github.repositories", + starred: "github.starred", + })[stream], + enableLocalTimeline: true, + }, + send: message => logs.push(message), + }) + + assert.equal(adapter, null) + assert.equal( + (await app.request("http://personal.example/health")).status, + 200 + ) + for (const [url, options] of [ + ["http://personal.example/v1/streams"], + [ + "http://personal.example/v1/pdpp/consent-requests", + { method: "POST", headers: { authorization: "Bearer desktop-token" } }, + ], + [ + "http://personal.example/v1/pdpp/consent-requests/request/approve", + { method: "POST", headers: { authorization: "Bearer desktop-token" } }, + ], + [ + "http://personal.example/v1/pdpp/credentials/session/redeem", + { method: "POST" }, + ], + [ + "http://personal.example/v1/pdpp/local-timeline/consent-requests", + { method: "POST", headers: { authorization: "Bearer desktop-token" } }, + ], + ["http://personal.example/v1/pdpp/introspect", { method: "POST" }], + ]) { + assert.equal((await app.request(url, options)).status, 404, url) + } + assert.match(logs.at(-1).message, /unsupported record contract/) +}) + +test("revocation during optional PDPP outage persists through recovery and Gateway failure", async () => { + const fixture = createInstalledGithubFixture() + const desktopToken = "desktop-token" + const selectedPdppProfile = { + connectorId: "github-pdpp", + connector: { + key: "github", + id: "https://registry.pdpp.org/connectors/github", + }, + scopeForStream: stream => + ({ + user: "github.profile", + repositories: "github.repositories", + starred: "github.starred", + })[stream], + enableLocalTimeline: true, + } + const seededAdapter = createPdppAuthorizationAdapter({ + activeManifestPath: fixture.activeManifestPath, + connectorId: "github-pdpp", + expectedConnector: selectedPdppProfile.connector, + databasePath: pdppProfileDatabasePath( + fixture.root, + "github-pdpp", + "authorization" + ), + scopeForStream: selectedPdppProfile.scopeForStream, + }) + const consent = seededAdapter.createConsentRequest({ + sessionId: "outage-session", + scopes: ["github.repositories"], + authorizationDetails: [ + { + type: PDPP_DATA_ACCESS_TYPE, + source: { kind: "connector", id: "github" }, + access_mode: "continuous", + purpose_code: "https://example.test/purpose/research", + streams: [{ name: "repositories" }], + }, + ], + }) + const issued = seededAdapter.issueApprovedGrant({ + requestId: consent.request_id, + legacyGrantId: "outage-grant", + subjectId: "subject-1", + clientId: TEST_BUILDER.address, + }) + seededAdapter.close() + + const outageApp = new Hono() + outageApp.get("/health", context => context.json({ ok: true })) + const outageAdapter = await registerOptionalPdppSurfaces({ + app: outageApp, + devToken: desktopToken, + storageRoot: fixture.root, + recordsRoot: fixture.root, + activeManifestPath: join(fixture.root, "missing-active.json"), + exportRoot: fixture.exportRoot, + selectedPdppProfile, + }) + assert.equal(outageAdapter, null) + const revocationSink = createPdppRevocationSink({ + storageRoot: fixture.root, + activeManifestPath: join(fixture.root, "missing-active.json"), + selectedPdppProfile, + }) + registerProtectedRoutes({ + app: outageApp, + devToken: desktopToken, + gatewayClient: { + revokeGrant: async () => { + throw new Error("gateway unavailable") + }, + }, + ownerAddress: "0xowner", + port: 8080, + send: () => {}, + serverSigner: { signGrantRevocation: async () => "signature" }, + onLegacyGrantRevoked: grantId => + revocationSink.revokeByLegacyGrantId(grantId), + }) + const revoke = await outageApp.request( + "http://personal.example/v1/grants/outage-grant", + { + method: "DELETE", + headers: { authorization: `Bearer ${desktopToken}` }, + } + ) + assert.equal(revoke.status, 500) + revocationSink.close() + + const recoveredApp = new Hono() + const recoveredAdapter = await registerOptionalPdppSurfaces({ + app: recoveredApp, + devToken: desktopToken, + storageRoot: fixture.root, + recordsRoot: fixture.root, + activeManifestPath: fixture.activeManifestPath, + exportRoot: fixture.exportRoot, + selectedPdppProfile, + }) + assert.notEqual(recoveredAdapter, null) + const revoked = await recoveredApp.request( + "http://personal.example/v1/streams", + { headers: { authorization: `Bearer ${issued.access_token}` } } + ) + assert.equal(revoked.status, 403) + assert.equal((await revoked.json()).error.code, "grant_revoked") + recoveredAdapter.close() +}) + +test("default records root preserves the legacy standalone fallback path", () => { + const roots = pdppDefaultStorageRoots({ homeDir: "/tmp/fake-home" }) + assert.equal( + roots.authorizationRoot, + "/tmp/fake-home/.data-connect/personal-server" + ) + assert.equal(roots.recordsRoot, "/tmp/fake-home/.dataconnect/personal-server") + assert.equal( + pdppDefaultStorageRoots({ + homeDir: "/tmp/fake-home", + configDir: "/tmp/config", + }).recordsRoot, + "/tmp/config" + ) + assert.equal( + pdppDefaultStorageRoots({ + homeDir: "/tmp/fake-home", + pdppStorageDir: "/tmp/pdpp-storage", + }).recordsRoot, + "/tmp/pdpp-storage" + ) +}) + test("serves Timeline only after local consent and fails closed after its bound session is revoked", async () => { const fixture = createInstalledGithubFixture() const adapter = createGithubAuthorizationAdapter({ @@ -1385,7 +1886,10 @@ test("maps durable cursor errors to stable resource-server responses", async () await mountPdppResourceServer(app, { ...fixture, recordsRepository: repository, - tokenIntrospector: { introspect: async () => activeToken() }, + tokenIntrospector: { + introspect: async () => + activeToken({ manifestDigest: fixture.manifestDigest }), + }, }) const headers = { authorization: "Bearer opaque" } const initial = await app.request( @@ -1421,3 +1925,476 @@ test("maps durable cursor errors to stable resource-server responses", async () assert.equal((await malformed.json()).error.code, "invalid_cursor") repository.close() }) + +test("fails closed when the active manifest diverges from the composed selection", async () => { + const fixture = createInstalledGithubFixture() + const divergent = createInstalledGithubFixture() + const selectedInstall = loadInstalledManifest({ + activeManifestPath: fixture.activeManifestPath, + connectorId: "github-pdpp", + expectedConnector: { + key: "github", + id: "https://registry.pdpp.org/connectors/github", + }, + }) + const divergentManifest = { ...divergent.manifest, version: "0.5.1" } + const divergentBytes = Buffer.from(JSON.stringify(divergentManifest)) + writeFileSync( + join( + dirname(divergent.activeManifestPath), + "install/profile/collection-profile.json" + ), + divergentBytes + ) + const divergentActive = JSON.parse(readFileSync(divergent.activeManifestPath)) + divergentActive.connectors["github-pdpp"].version = "0.5.1" + divergentActive.connectors["github-pdpp"].manifestSha256 = + hash(divergentBytes) + writeFileSync(divergent.activeManifestPath, JSON.stringify(divergentActive)) + + const adapter = createPdppAuthorizationAdapter({ + activeManifestPath: fixture.activeManifestPath, + connectorId: "github-pdpp", + expectedConnector: { + key: "github", + id: "https://registry.pdpp.org/connectors/github", + }, + selectedInstall, + databasePath: join( + dirname(fixture.activeManifestPath), + "authorization.sqlite" + ), + scopeForStream: stream => ({ repositories: "github.repositories" })[stream], + }) + const app = new Hono() + try { + await mountPdppResourceServer(app, { + ...fixture, + selectedInstall, + tokenIntrospector: { + introspect: token => adapter.resolveForResourceServer(token), + }, + }) + const consent = adapter.createConsentRequest({ + sessionId: "selection-bound-session", + scopes: ["github.repositories"], + authorizationDetails: [ + { + type: PDPP_DATA_ACCESS_TYPE, + source: { kind: "connector", id: "github" }, + access_mode: "continuous", + purpose_code: "https://example.test/purpose/research", + streams: [{ name: "repositories" }], + }, + ], + }) + const issued = adapter.issueApprovedGrant({ + requestId: consent.request_id, + legacyGrantId: "selection-bound-grant", + subjectId: "selection-bound-subject", + clientId: "selection-bound-client", + }) + const beforeDivergence = await app.request( + "http://personal.example/v1/streams", + { headers: { authorization: `Bearer ${issued.access_token}` } } + ) + assert.equal(beforeDivergence.status, 200) + + writeFileSync( + fixture.activeManifestPath, + readFileSync(divergent.activeManifestPath) + ) + assert.throws( + () => + adapter.createConsentRequest({ + sessionId: "divergent-selection-session", + scopes: ["github.repositories"], + authorizationDetails: [ + { + type: PDPP_DATA_ACCESS_TYPE, + source: { kind: "connector", id: "github" }, + access_mode: "continuous", + purpose_code: "https://example.test/purpose/research", + streams: [{ name: "repositories" }], + }, + ], + }), + /active install no longer matches the composed serving selection/ + ) + const afterDivergence = await app.request( + "http://personal.example/v1/streams", + { headers: { authorization: `Bearer ${issued.access_token}` } } + ) + assert.equal(afterDivergence.status, 401) + + await assert.rejects( + mountPdppResourceServer(new Hono(), { + ...fixture, + activeManifestPath: divergent.activeManifestPath, + selectedInstall, + tokenIntrospector: { + introspect: token => adapter.resolveForResourceServer(token), + }, + }), + /active install no longer matches the composed serving selection/ + ) + } finally { + adapter.close() + } +}) + +test("serves the selected canonical ChatGPT profile through grant-scoped routes", async () => { + const fixture = createInstalledChatgptFixture() + const desktopToken = "desktop-token" + let now = new Date("2026-07-31T12:00:00.000Z") + const adapter = createPdppAuthorizationAdapter({ + activeManifestPath: fixture.activeManifestPath, + connectorId: "chatgpt-pdpp", + expectedConnector: { + key: "chatgpt", + id: "https://registry.pdpp.org/connectors/chatgpt", + }, + databasePath: join(fixture.root, "authorization.sqlite"), + now: () => now, + scopeForStream: stream => `chatgpt.${stream}`, + singleUseAccessExpiresInSeconds: 60, + }) + const app = new Hono() + registerProtectedRoutes({ + app, + devToken: desktopToken, + gatewayClient: { revokeGrant: async () => {} }, + ownerAddress: "0xowner", + port: 8080, + send: () => {}, + serverSigner: { signGrantRevocation: async () => "signature" }, + onLegacyGrantRevoked: grantId => adapter.revokeByLegacyGrantId(grantId), + }) + registerPdppAuthorizationRoutes({ + app, + devToken: desktopToken, + adapter, + enableLocalTimeline: false, + }) + + const writeExport = (name, timestamp, content) => + writeFileSync( + join(fixture.exportRoot, name), + JSON.stringify({ timestamp, content }) + ) + const exportContent = ({ provenance, recordsByStream, snapshot } = {}) => ({ + platform: "chatgpt", + version: fixture.manifest.version, + "pdpp.provenance": + provenance ?? + chatgptSnapshotProvenance({ + manifestDigest: fixture.manifestDigest, + }), + "pdpp.recordsByStream": recordsByStream ?? { + conversations: [ + chatgptConversation("conversation-a", "2026-02-01T00:00:00.000Z"), + chatgptConversation("conversation-b", "2026-03-01T00:00:00.000Z"), + chatgptConversation("resource-excluded", "2026-04-01T00:00:00.000Z"), + chatgptConversation("time-excluded", "2025-01-01T00:00:00.000Z"), + ], + }, + ...(snapshot ? { "pdpp.snapshot": snapshot } : {}), + }) + writeExport("initial.json", 1785456000000, exportContent()) + await mountPdppResourceServer(app, { + ...fixture, + connectorId: "chatgpt-pdpp", + expectedConnector: { + key: "chatgpt", + id: "https://registry.pdpp.org/connectors/chatgpt", + }, + connectionId: "chatgpt-account-a", + tokenIntrospector: { + introspect: token => adapter.resolveForResourceServer(token), + }, + }) + + const details = ({ source = "chatgpt", accessMode = "continuous" } = {}) => [ + { + type: PDPP_DATA_ACCESS_TYPE, + source: { kind: "connector", id: source }, + access_mode: accessMode, + purpose_code: "https://example.test/purpose/research", + streams: [ + { + name: "conversations", + fields: ["title"], + resources: ["conversation-a", "conversation-b", "time-excluded"], + time_range: { since: "2026-01-01T00:00:00.000Z" }, + }, + ], + }, + ] + const createConsent = async ({ sessionId, scopes, authorizationDetails }) => + app.request("http://personal.example/v1/pdpp/consent-requests", { + method: "POST", + headers: { + authorization: `Bearer ${desktopToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + session_id: sessionId, + scopes, + authorization_details: authorizationDetails, + }), + }) + const issueBearer = async ({ + sessionId, + legacyGrantId, + accessMode = "continuous", + }) => { + const consent = await createConsent({ + sessionId, + scopes: ["chatgpt.conversations"], + authorizationDetails: details({ accessMode }), + }) + assert.equal(consent.status, 201, await consent.clone().text()) + const request = await consent.json() + assert.deepEqual(request.authorization_details.streams[0].fields, [ + "id", + "title", + ]) + const approval = await app.request( + `http://personal.example/v1/pdpp/consent-requests/${request.request_id}/approve`, + { + method: "POST", + headers: { + authorization: `Bearer ${desktopToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + legacy_grant_id: legacyGrantId, + subject_id: "chatgpt-subject", + client_id: TEST_BUILDER.address, + }), + } + ) + assert.equal(approval.status, 201, await approval.clone().text()) + const authorization = await redemptionAuthorization({ + account: TEST_BUILDER, + origin: "http://personal.example", + sessionId, + }) + const credential = await app.request( + `http://personal.example/v1/pdpp/credentials/${sessionId}/redeem`, + { method: "POST", headers: { authorization } } + ) + assert.equal(credential.status, 200, await credential.clone().text()) + return `Bearer ${(await credential.json()).access_token}` + } + + try { + const timeline = await app.request( + "http://personal.example/v1/pdpp/local-timeline/consent-requests", + { + method: "POST", + headers: { authorization: `Bearer ${desktopToken}` }, + } + ) + assert.equal(timeline.status, 404) + + for (const [index, invalid] of [ + { + scopes: ["chatgpt.conversations"], + authorizationDetails: details({ source: "github" }), + }, + { scopes: [], authorizationDetails: details() }, + { + scopes: ["chatgpt.conversations", "chatgpt.messages"], + authorizationDetails: details(), + }, + ].entries()) { + const response = await createConsent({ + sessionId: `invalid-${index}`, + ...invalid, + }) + assert.equal(response.status, 400) + } + + const uriSourceConsent = await createConsent({ + sessionId: "chatgpt-uri-source", + scopes: ["chatgpt.conversations"], + authorizationDetails: details({ + source: "https://registry.pdpp.org/connectors/chatgpt", + }), + }) + assert.equal(uriSourceConsent.status, 201) + assert.equal( + (await uriSourceConsent.json()).authorization_details.source.id, + "chatgpt" + ) + + const bearer = await issueBearer({ + sessionId: "chatgpt-session", + legacyGrantId: "chatgpt-legacy-grant", + }) + const headers = { authorization: bearer } + const streams = await app.request("http://personal.example/v1/streams", { + headers, + }) + assert.equal(streams.status, 200) + assert.deepEqual( + (await streams.json()).data.map(stream => stream.name), + ["conversations"] + ) + + const firstPage = await app.request( + "http://personal.example/v1/streams/conversations/records?limit=1", + { headers } + ) + assert.equal(firstPage.status, 200) + const first = await firstPage.json() + assert.deepEqual( + first.data.map(record => record.id), + ["conversation-a"] + ) + assert.deepEqual(first.data[0].data, { + id: "conversation-a", + title: "title-conversation-a", + }) + assert.equal(typeof first.next_cursor, "string") + const secondPage = await app.request( + `http://personal.example/v1/streams/conversations/records?limit=1&cursor=${encodeURIComponent(first.next_cursor)}`, + { headers } + ) + assert.equal(secondPage.status, 200) + assert.deepEqual( + (await secondPage.json()).data.map(record => record.id), + ["conversation-b"] + ) + const detail = await app.request( + "http://personal.example/v1/streams/conversations/records/conversation-a", + { headers } + ) + assert.equal(detail.status, 200) + assert.deepEqual((await detail.json()).data, { + id: "conversation-a", + title: "title-conversation-a", + }) + + writeExport( + "newer-malformed.json", + 1785542400000, + exportContent({ + recordsByStream: { + conversations: [{ stream: "conversations", key: "bad" }], + }, + }) + ) + writeExport("wrong-source.json", 1785628800000, { + ...exportContent(), + platform: "github", + }) + writeExport("wrong-version.json", 1785672000000, { + ...exportContent(), + version: "0.1.1", + }) + writeExport( + "wrong-provenance.json", + 1785715200000, + exportContent({ + provenance: chatgptSnapshotProvenance({ + manifestDigest: "sha256:wrong", + connectionId: "wrong-connection", + }), + }) + ) + const retained = await app.request( + "http://personal.example/v1/streams/conversations/records?limit=100", + { headers } + ) + assert.equal(retained.status, 200) + assert.deepEqual( + (await retained.json()).data.map(record => record.id), + ["conversation-a", "conversation-b"] + ) + + const revoke = await app.request( + "http://personal.example/v1/grants/chatgpt-legacy-grant", + { method: "DELETE", headers: { authorization: `Bearer ${desktopToken}` } } + ) + assert.equal(revoke.status, 204) + const revoked = await app.request("http://personal.example/v1/streams", { + headers, + }) + assert.equal(revoked.status, 403) + assert.equal((await revoked.json()).error.code, "grant_revoked") + + const expiringBearer = await issueBearer({ + sessionId: "chatgpt-expiring-session", + legacyGrantId: "chatgpt-expiring-grant", + accessMode: "single_use", + }) + now = new Date("2026-07-31T12:02:00.000Z") + const expired = await app.request("http://personal.example/v1/streams", { + headers: { authorization: expiringBearer }, + }) + assert.equal(expired.status, 403) + assert.equal((await expired.json()).error.code, "grant_expired") + + now = new Date("2026-07-31T12:00:00.000Z") + const refreshBearer = await issueBearer({ + sessionId: "chatgpt-refresh-session", + legacyGrantId: "chatgpt-refresh-grant", + }) + writeExport( + "authoritative-empty.json", + 1785801600000, + exportContent({ + recordsByStream: { conversations: [] }, + snapshot: { + collection_mode: "full_refresh", + reset_streams: ["conversations"], + completed_at: "2026-07-31T12:00:00.000Z", + }, + }) + ) + const empty = await app.request( + "http://personal.example/v1/streams/conversations/records", + { headers: { authorization: refreshBearer } } + ) + assert.equal(empty.status, 200) + assert.deepEqual((await empty.json()).data, []) + + const bindingConsent = await createConsent({ + sessionId: "chatgpt-manifest-mismatch", + scopes: ["chatgpt.conversations"], + authorizationDetails: details(), + }) + assert.equal(bindingConsent.status, 201) + const bindingRequest = await bindingConsent.json() + const changedManifest = { ...fixture.manifest, version: "0.1.1" } + const changedBytes = Buffer.from(JSON.stringify(changedManifest)) + writeFileSync( + join(fixture.installRoot, "profile/collection-profile.json"), + changedBytes + ) + const active = JSON.parse(readFileSync(fixture.activeManifestPath)) + active.connectors["chatgpt-pdpp"].version = "0.1.1" + active.connectors["chatgpt-pdpp"].manifestSha256 = hash(changedBytes) + writeFileSync(fixture.activeManifestPath, JSON.stringify(active)) + const mismatchedApproval = await app.request( + `http://personal.example/v1/pdpp/consent-requests/${bindingRequest.request_id}/approve`, + { + method: "POST", + headers: { + authorization: `Bearer ${desktopToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + legacy_grant_id: "chatgpt-mismatched-grant", + subject_id: "chatgpt-subject", + client_id: TEST_BUILDER.address, + }), + } + ) + assert.equal(mismatchedApproval.status, 400) + } finally { + adapter.close() + } +}) diff --git a/personal-server/pdpp/test/fixtures/chatgpt.collection-profile.js b/personal-server/pdpp/test/fixtures/chatgpt.collection-profile.js new file mode 100644 index 00000000..ea4d5def --- /dev/null +++ b/personal-server/pdpp/test/fixtures/chatgpt.collection-profile.js @@ -0,0 +1,61 @@ +import { gunzipSync } from "node:zlib" + +const compressedCanonicalChatgptManifest = ` +H4sICAP0bGoCA2NoYXRncHQtY2Fub25pY2FsLXBpbm5lZC5qc29uAO1bUW/cNhJ+968g9sUJINs5 +3KE4uC8XNME1QHsXxAmKQ2AQXIkr8UyJKknZ3hYB+nQ/4NBf2F9yMyQlUbLW3l3LPvuuD0lakZzh +DIfffBxyfz4gZFFrZVWqJL3k2ghVLU7J4tXxH45fLRJsTlVV8dQqTUWGTYW1tTk9OdE8F8bq9XGd +1fWx0vlJ19OcpAWzeW3HEi74GkUMWktWiRU3ljZa7Cd/et6ZMLVka1qxkmPTNzDkr+8/+kbdVFaU +nGr+YyM0L3llDXT6GdqgdSmqTFR5/wW+VdxeKX0RfUIxfjj6xeqGh4YvSTtoqdWV4XqrQQft3274 +wnDb1P2USpUxKaxzn7HMipQanmruXYBOBolghWCSpqy2jebx7C/AIhzawGzQIbRmxoA92aKbq2RL +LiNHESPy6khUJOOWCWn6nhk3qRa1DT4/40wSW3CyYSBZKQ3twhCjGp3yY/IROneLSUq2JjAvFFES +Vck1ySAUqpyoq4rro5Jnglmekd5CAqMMN7joRPNUQQCsey1BMjR+TUxa8KyRMBpW3EBnVJQ2WoOg +TgS/FCA45YRVGckUqZQlsCnK2jqhvV5z3DvBNMtSWNp57Yxd9h6IpooyYcE0yBLagNJ1lfZSVoLL +DOPscxcffaRg1IXgbdetGzm9ZLwEhw/72HXtJEw0jQJx0BaCa6KFNYgWZS25vWVqsPVSXiiZuehf +rFXzF37NcNQxDB725dUlemDxzbevP4IV9NPZ2w9/e/3928V51+tLcrt/boTztH+mu7Uumm6dx0sh +5o6mdYw98P712dkPf//wJvZA+K/zEU7AZmdLAdAgeIRgRQOwSkVluWZp2KifEWobwIfw5Twgh+Yr +zU1BayVFuo5RA/dWCdiY8YwCADlD0LAS8acP4ribU3kJSsA1qnKh/cevXr3q+paiEmVT3t2PXbt+ +sHUkr2CjRh3//NWfop6RkWCCCcg3tJVKccHlOpoyIAp8xB1seGXAe5cBXBFumj5AF0tlKQCZB5Rx +50LkRdSVpRe5Vk2VUcNWfBQWC1h5SGjgI7aCGVOHbhTcWYw7wuRAF9gdh24IQxNQEcchwqQOGkOe +QaCxgK+v2zUiYW0Jgq9HuyOHgUeIs1+PEVbzmglNCsAsCZoAAxEZNZEqBziHnCthLXJosQWYmRck +pFHiTWOIoS2YHi8Ooo27qJslRBe43FiA9jjIpHPK2AdoSYOLDYmXZ4b6gHbeGidKqzkrewztM3bA +BggbZAjOqZuzWAfdce8I7wOdGKbyDmD+AZmNTCtyqjAROi1RF4KWkythC2KFBYcnmDdgIU6aOoN/ +CLoWPjKdFuKSZyeYRgCIyEqyHL4jHTE1wCx5IVaE4zaotTD8ZdJmuKOlhjVB8fUx+R7Wn+HiMQ0L +BuFQM9wFxPsvLFdMXgykDIwwtwplY9lScuoiLMqCkF9LNnRKC6dq+U/YNbEfIKBqrgdI1W7ibPQl +kgMTxJCJGr8M4Rudt3H453Z8AhHRSNnCXtfvmiLBpFr5/QYuLJleH3mpG5X6laK4RPurBnYB2xS1 +4nofOWEbNfqgeEyNwtA29m7RuFTgOVb1Km+TF0J4FnFd/FOR7eCRzUvqNw2tfKa7t7zSbziaQkaw +FJJHq8DvyltUYErLgTjdrSMXP5Vqf/un+FVMeD7jxjy/gQxhj4TjnO/UNYKVBo56jtk6fhjFbd8J +EBJd4Q5hXdd4U0UwJHlLYSKU6ZjzkHrB7P0pw2w6k4kq9Qc+JsdJ58cGDhJDNYbjFhjDleTXkHkl +7el7QKHzET30CDrRb9rzDLIrdJYA5Tcgcog4nxc54jBEgMW/pf8faUczGILGloPuDtzNgibtYnkO +p/mQF8dmoZIpBo1MqqbLdT/5QSQNHXKTJ3+JYg1SLtoDOwEG5KEEwdOLWsFOo1dwNlZX0ZkMGZOn +nNGA7iTZ9o/pzQ3OEVy4mW502RjzP5KrmDskmOKtBd4LyR4Oou258hCYl+YuZTcpEl1ilTt5j/J9 +zWxxTN4zt2giQ/ZXg1FAuFEgnpNRCtL2yosCpbtRnRv2DVjOWwaTCF3gRB9PkYQpKmAs7tgeG35K +MAsn+A2MtwnBQ4dsaZHjQwmYrCSBzSeRLwnbegz8BVpxX5N3bwxwYNnqCpQHT/X8ulYame6LunPO +0DOaLxshs5d38SFW13DgoUijnxYbiv1J7yOo89AsubUAp4I8EGc2Twm4AYTdRiGBos3D8xjWnm51 +I4bgo9NKr5YGLffnILiBqJFNPssqPjrpXYlKmIKCXjMiAPua0CLG3lGI+EMd/uwtoseqe+2H7Sll +x6p3Jn/JTUS5Jx+cJHlPkw9qLn2wFKLeokgbO2pY3/Nn7M3FiHZjcJFX3SXJyO3DMGY6E1V3I1Aw +A6ko2kddtXAWbutALOlBMRkASzLepneT4FbQA9LghySi92KeIa96tkRZ6vQ68JqXgJZKi1sIKN5X +ZKTt1t1cQCQRtlSNdaQNa/tbE8NWxA3VA27o9a4g/fZKcf+VS1jsoHqtmsQXxryns1ARy3xJzLKy +Bob3BoDArSTx9VlovhSMZGK1Ir/98it5/f4dyRTY5phfKcAkVS6NhY1innG56y5m0uXC+cpcGWV2 +3oR/++TgZACHmW3KYfPPbM8MOYCze1VKnElbJcZhz2dSJ+l89fBJIgTHTqWSO8c8fIHDL+vAiJ2T +zArCnYoK95HS63lTS9oAhpY0r+0ttymuDwFwN+SFK1Kal11G8UmmsYXSAbOxBROBO3vvkHL8tUuv +a8OlSzQZSC1Oiz0laE9CooknxKyN5SURfVnEJIRXmAoyV3gAa1JgPv76staQd3QSbt9AFN6PvYTM +xWWqSt5WQJJgawKmGuHuateJfxdQYGHi04fvCBKpZ5yTDJiHD3jkHEe00cOdRz1/t7qH4TybSSEy +aIiMOUTHsTqHPBflex9J/7/5wvgFSNE+mrv3qgRhO26J227i/IX8LBdx+AYhVyNGsG/4sXyn6Hu8 +G6z/bV429PcAgpPpthgiN3QZw92o2wC6Rm1xwI+aunCLPt/JJh/FhnhCv5PW88mBntq3S/gUie1k +WI4J7sfAZQ9Nyz/jYeQFMtrDHwpmyZVqZOZ4Jz5C62oeVpGLSl31FY9DRwkPv4Vvm4eAlbWqskPi +I/vlXkx5w76LGDNaZ68UAa9CmuPXNujzt5Fn3GLBypDf/vVv8h4iGR+riZ/8oyb8Fgj3u0jPKTmM +LHVHO5CKFgOLXnH3rskbZ/B+c43XgLVsDLkqOPgSn/by9aHu7hHluuXlx+QMJiO5ReW//OpfEKsK +pHBLgOuGItaz5dbObbQZPyPfM7+2PqbOx3NIDMswC5N4KJK2F318WtRiquczKflEEZzciMC760C3 +D989zz6tnLlzBjQt2lH3cqFK+bxZ0BUlMrrlq9m417DA4+W48o4/a0Ayqy52S1hBxhYPa89cKQWf +tOjM/+JkMMilnUtOSpbxMJ1TN11fgnn3JvGTrzKu5Rp/fBIP943Bik8fvsPqjarWpYL8hI9xfTWH +VWuCD8Il/HF3FZDhnvc9w1YPSXZIFc7Zc5WJ/ovPfp9ifaOLyLme6s5YIIh2BcVd8YRfq06d4J/k +Wf/3twnxe9pkIsi2fYs7OfbBju8P/y5hP1ox14PYg7CwmJXAU3zihzn+d8T4E14z8fPTGz+lgaRq +hof98U9+tor6qDxz168Ju9etW4+YJE2bfruXTLmjvQ+b8MjHwp22sR7Q3ZqJisQ/aN7RLdEN3tYm +TlVHdrKwfRcyejTbmfl9+xYFydRtpYrtDe2eojyelVigmjDuLT52xpfe+SlxkZ9072SS+AYzmbI8 +8TTVPOUdsLer54zCOXcpAtnBl4P/AC84SGojQQAA +` + +/** Exact bytes from PDPP cc07e3a, packages/polyfill-connectors/manifests/chatgpt.json. */ +export const canonicalChatgptManifestBytes = gunzipSync( + Buffer.from(compressedCanonicalChatgptManifest, "base64") +) diff --git a/personal-server/protected-routes.js b/personal-server/protected-routes.js index 71e94c07..7da46004 100644 --- a/personal-server/protected-routes.js +++ b/personal-server/protected-routes.js @@ -36,6 +36,9 @@ export function registerProtectedRoutes({ if (!gatewayClient) { return c.json({ error: "Gateway client not initialized" }, 500) } + if (typeof onLegacyGrantRevoked !== "function") { + return c.json({ error: "Local revocation store not initialized" }, 500) + } const grantId = c.req.param("grantId") @@ -45,7 +48,7 @@ export function registerProtectedRoutes({ grantId, }) - await onLegacyGrantRevoked?.(grantId) + await onLegacyGrantRevoked(grantId) await gatewayClient.revokeGrant({ grantId, diff --git a/scripts/build-prod.js b/scripts/build-prod.js index a4acbdc2..9069e12d 100644 --- a/scripts/build-prod.js +++ b/scripts/build-prod.js @@ -20,6 +20,7 @@ import { existsSync, cpSync, readdirSync, mkdirSync, readFileSync } from 'fs'; import { join, dirname } from 'path'; import { fileURLToPath } from 'url'; import { platform, arch } from 'os'; +import { nativeTauriTarget, stagePdppNode } from './stage-pdpp-node.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const ROOT = join(__dirname, '..'); @@ -72,6 +73,10 @@ function findAppBundle() { async function build() { log('Building DataConnect for production...'); + // Keep local production builds on the same fail-closed Node 22 sidecar + // contract as the release workflow. + stagePdppNode({ ...nativeTauriTarget(PLAT, arch()), projectRoot: ROOT }); + // 1. Install playwright-runner dependencies log('Installing playwright-runner dependencies...'); exec('npm install', { cwd: PLAYWRIGHT_RUNNER }); @@ -89,7 +94,11 @@ async function build() { log('Installing personal-server dependencies...'); exec('npm install', { cwd: PERSONAL_SERVER }); - // 4. Build personal-server binary + // 4. Install PDPP runtime dependencies that Tauri packages as resources + log('Installing PDPP runtime dependencies...'); + exec('node scripts/ensure-pdpp-runtime.js'); + + // 5. Build personal-server binary log('Building personal-server binary...'); exec('npm run build', { cwd: PERSONAL_SERVER }); @@ -98,7 +107,7 @@ async function build() { throw new Error('personal-server build failed - dist directory not found'); } - // 5. Build frontend + // 6. Build frontend log('Building frontend...'); exec('npm run build'); @@ -111,14 +120,14 @@ async function build() { return; } - // 6. Build the .app bundle only (no DMG). + // 7. Build the .app bundle only (no DMG). // Tauri's resource glob flattens directory structures, so node_modules/ // can't be included via tauri.conf.json. We build .app first, inject // node_modules, then create the DMG ourselves. log('Building Tauri .app bundle...'); exec('npx tauri build --bundles app'); - // 7. Inject personal-server native addons into the .app bundle. + // 8. Inject personal-server native addons into the .app bundle. const appPath = findAppBundle(); if (!appPath) { throw new Error('.app bundle not found after build'); @@ -126,7 +135,7 @@ async function build() { log(`Injecting native addons into ${appPath}...`); copyNativeModulesIntoApp(appPath); - // 8. Create DMG from the complete .app. + // 9. Create DMG from the complete .app. if (PLAT === 'darwin') { const version = getVersion(); const archName = arch() === 'arm64' ? 'aarch64' : 'x64'; diff --git a/scripts/ensure-pdpp-runtime.js b/scripts/ensure-pdpp-runtime.js new file mode 100644 index 00000000..a86c22e9 --- /dev/null +++ b/scripts/ensure-pdpp-runtime.js @@ -0,0 +1,92 @@ +import { spawnSync } from "node:child_process" +import { createHash } from "node:crypto" +import { existsSync, readFileSync, writeFileSync } from "node:fs" +import { dirname, join } from "node:path" +import { fileURLToPath } from "node:url" +import { isMainModule } from "./is-main-module.js" + +const root = dirname(dirname(fileURLToPath(import.meta.url))) +const runtimeRoot = join(root, "pdpp-runtime") + +function runtimeStamp(root) { + return createHash("sha256") + .update(readFileSync(join(root, "package.json"))) + .update(readFileSync(join(root, "package-lock.json"))) + .digest("hex") +} + +export function npmInstallCommand(options = {}) { + const { + platformName = process.platform, + nodePath = process.execPath, + } = options + const npmCliPath = Object.hasOwn(options, "npmCliPath") + ? options.npmCliPath + : process.env.npm_execpath + return { + command: npmCliPath + ? nodePath + : platformName === "win32" + ? "npm.cmd" + : "npm", + args: [...(npmCliPath ? [npmCliPath] : []), "ci", "--ignore-scripts"], + shell: !npmCliPath && platformName === "win32", + } +} + +export function runEnsurePdppRuntime(options = {}) { + const { + root = runtimeRoot, + spawn = spawnSync, + platformName = process.platform, + nodePath = process.execPath, + } = options + const npmCliPath = Object.hasOwn(options, "npmCliPath") + ? options.npmCliPath + : process.env.npm_execpath + const required = ["p-queue", "patchright"].map(name => + join(root, "node_modules", name, "package.json") + ) + const stampPath = join(root, ".install-stamp") + const stamp = runtimeStamp(root) + + if ( + required.every(existsSync) && + existsSync(stampPath) && + readFileSync(stampPath, "utf8").trim() === stamp + ) { + return 0 + } + + const installCommand = npmInstallCommand({ + platformName, + nodePath, + npmCliPath, + }) + const install = spawn(installCommand.command, installCommand.args, { + cwd: root, + stdio: "inherit", + shell: installCommand.shell, + }) + if (install.error) { + throw new Error( + `Failed to install PDPP runtime dependencies: ${install.error.message}` + ) + } + if (install.status !== 0) { + throw new Error( + `Failed to install PDPP runtime dependencies: npm exited with status ${install.status ?? "unknown"}` + ) + } + writeFileSync(stampPath, `${stamp}\n`) + return 0 +} + +if (isMainModule(import.meta.url, process.argv[1])) { + try { + process.exit(runEnsurePdppRuntime()) + } catch (error) { + console.error(error instanceof Error ? error.message : error) + process.exit(1) + } +} diff --git a/scripts/ensure-pdpp-runtime.test.mjs b/scripts/ensure-pdpp-runtime.test.mjs new file mode 100644 index 00000000..4bdb1115 --- /dev/null +++ b/scripts/ensure-pdpp-runtime.test.mjs @@ -0,0 +1,101 @@ +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { + npmInstallCommand, + runEnsurePdppRuntime, +} from "./ensure-pdpp-runtime.js" + +function createRuntimeFixture() { + const root = mkdtempSync(join(tmpdir(), "pdpp-runtime-install-")) + writeFileSync(join(root, "package.json"), "{}") + writeFileSync(join(root, "package-lock.json"), "{}") + return root +} + +describe("ensure PDPP runtime dependencies", () => { + it("runs npm through Node on Windows when npm_execpath is available", () => { + expect( + npmInstallCommand({ + platformName: "win32", + nodePath: "C:\\hostedtoolcache\\node.exe", + npmCliPath: "C:\\hostedtoolcache\\npm\\bin\\npm-cli.js", + }) + ).toEqual({ + command: "C:\\hostedtoolcache\\node.exe", + args: [ + "C:\\hostedtoolcache\\npm\\bin\\npm-cli.js", + "ci", + "--ignore-scripts", + ], + shell: false, + }) + }) + + it("falls back to npm.cmd on Windows without shell injection inputs", () => { + expect( + npmInstallCommand({ + platformName: "win32", + nodePath: "C:\\hostedtoolcache\\node.exe", + npmCliPath: undefined, + }) + ).toEqual({ + command: "npm.cmd", + args: ["ci", "--ignore-scripts"], + shell: true, + }) + }) + + it("reports spawn errors and non-zero exits with useful messages", () => { + const root = createRuntimeFixture() + try { + expect(() => + runEnsurePdppRuntime({ + root, + spawn: () => ({ error: new Error("spawn npm ENOENT") }), + }) + ).toThrow("spawn npm ENOENT") + + expect(() => + runEnsurePdppRuntime({ + root, + spawn: () => ({ status: 1 }), + }) + ).toThrow("npm exited with status 1") + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) + + it("writes the install stamp after npm restores runtime packages", () => { + const root = createRuntimeFixture() + try { + let invocation + const status = runEnsurePdppRuntime({ + root, + spawn: (...args) => { + invocation = args + for (const dependency of ["p-queue", "patchright"]) { + const packageRoot = join(root, "node_modules", dependency) + mkdirSync(packageRoot, { recursive: true }) + writeFileSync(join(packageRoot, "package.json"), "{}") + } + return { status: 0 } + }, + platformName: "linux", + npmCliPath: undefined, + }) + + expect(status).toBe(0) + expect(invocation).toEqual([ + "npm", + ["ci", "--ignore-scripts"], + { cwd: root, stdio: "inherit", shell: false }, + ]) + expect(readFileSync(join(root, ".install-stamp"), "utf8")).toMatch(/\S/) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) +}) diff --git a/scripts/release-github.mjs b/scripts/release-github.mjs index b3986ee4..d6e13ea0 100644 --- a/scripts/release-github.mjs +++ b/scripts/release-github.mjs @@ -34,7 +34,7 @@ function info(message) { process.stdout.write(`[release] ${message}\n`); } -function parseArgs(argv) { +export function parseArgs(argv) { const args = { version: "", target: "main", @@ -45,6 +45,7 @@ function parseArgs(argv) { checkVersion: false, showVersions: false, suggestVersion: false, + assistedByAi: false, }; for (let i = 0; i < argv.length; i += 1) { @@ -89,6 +90,10 @@ function parseArgs(argv) { args.suggestVersion = true; continue; } + if (token === "--assisted-by-ai") { + args.assistedByAi = true; + continue; + } fail(`Unknown argument: ${token}`); } @@ -193,6 +198,11 @@ function updateTauriVersion(nextVersion) { writeFileSync(TAURI_CONF_PATH, `${JSON.stringify(parsed, null, 2)}\n`, "utf8"); } +export function releaseCommitCommand(tagName, args) { + const assistedByAiTrailer = args.assistedByAi ? ' -m "Assisted-by: AI"' : ""; + return `git commit -s -m "release: ${tagName}"${assistedByAiTrailer}`; +} + function assertVersionOrdering(nextVersion, currentTauriVersion, latestRemoteTagVersion) { if (compareSemver(nextVersion, currentTauriVersion) <= 0) { fail( @@ -265,7 +275,7 @@ function main() { info(`Would run: git pull --ff-only origin ${args.target}`); info(`Would update: src-tauri/tauri.conf.json version -> ${version}`); info(`Would run: git add src-tauri/tauri.conf.json`); - info(`Would run: git commit -m "release: ${tagName}"`); + info(`Would run: ${releaseCommitCommand(tagName, args)}`); if (!args.noPush) { info(`Would run: git push origin ${args.target}`); } @@ -282,7 +292,7 @@ function main() { updateTauriVersion(version); runInherit("git add src-tauri/tauri.conf.json"); - runInherit(`git commit -m "release: ${tagName}"`); + runInherit(releaseCommitCommand(tagName, args)); if (!args.noPush) { runInherit(`git push origin ${args.target}`); @@ -297,4 +307,6 @@ function main() { info(`Release created: ${tagName}`); } -main(); +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + main(); +} diff --git a/scripts/release-github.test.mjs b/scripts/release-github.test.mjs new file mode 100644 index 00000000..196d345f --- /dev/null +++ b/scripts/release-github.test.mjs @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest" + +import { parseArgs, releaseCommitCommand } from "./release-github.mjs" + +describe("release helper", () => { + it("DCO signs release commits by default", () => { + const args = parseArgs(["--version", "1.2.3"]) + + expect(releaseCommitCommand("v1.2.3", args)).toBe( + 'git commit -s -m "release: v1.2.3"' + ) + }) + + it("adds an AI assistance trailer only when requested", () => { + const args = parseArgs(["--version", "1.2.3", "--assisted-by-ai"]) + + expect(args.assistedByAi).toBe(true) + expect(releaseCommitCommand("v1.2.3", args)).toBe( + 'git commit -s -m "release: v1.2.3" -m "Assisted-by: AI"' + ) + }) +}) diff --git a/scripts/release-workflow.test.ts b/scripts/release-workflow.test.ts index a6e4c956..19d189ac 100644 --- a/scripts/release-workflow.test.ts +++ b/scripts/release-workflow.test.ts @@ -8,7 +8,7 @@ import { writeFileSync, } from "node:fs" import { tmpdir } from "node:os" -import { basename, join, resolve } from "node:path" +import { basename, dirname, join, resolve } from "node:path" import { describe, expect, it } from "vitest" const releaseWorkflowPath = resolve( @@ -28,6 +28,19 @@ function readWorkflowStep(workflow: string, name: string) { return workflow.slice(start, next === -1 ? workflow.length : next) } +function readWorkflowRunScript(workflow: string, name: string) { + const step = readWorkflowStep(workflow, name) + const marker = " run: |\n" + const start = step.indexOf(marker) + if (start === -1) + throw new Error(`Missing run script for workflow step: ${name}`) + return step + .slice(start + marker.length) + .split("\n") + .map(line => (line.startsWith(" ") ? line.slice(10) : line)) + .join("\n") +} + describe("release workflow", () => { it("builds manual-install artifacts on demand without an updater", () => { const workflow = readReleaseWorkflow() @@ -50,9 +63,16 @@ describe("release workflow", () => { "verification_args=(--platform macos --expected-arch" ) expect(workflow).toContain("npm ci") + expect(workflow).toContain('node-version: "22.23.1"') + expect(workflow).toContain("Stage Node.js runtime sidecar") + expect(workflow).toContain( + 'node scripts/stage-pdpp-node.mjs --target "${{ matrix.target }}"' + ) expect(workflow).toContain( "npm run build -- --require-browser --target ${{ matrix.pkg_target }}" ) + expect(workflow).toContain("Install PDPP runtime dependencies") + expect(workflow).toContain("node scripts/ensure-pdpp-runtime.js") expect(workflow).toContain("if: github.event_name == 'release'") expect(workflow).not.toMatch( /updater|latest\.json|TAURI_SIGNING_PRIVATE_KEY/i @@ -83,6 +103,9 @@ describe("release workflow", () => { "env.APPLE_SIGNING_AVAILABLE == 'true' && secrets.APPLE_SIGNING_IDENTITY || ''" ) expect(workflow).toContain("verification_args+=(--verify-code-signature)") + expect(workflow).toContain( + 'codesign --force --options runtime --timestamp --sign "$APPLE_SIGNING_IDENTITY" src-tauri/binaries/pdpp-node-${{ matrix.target }}' + ) expect(workflow).toContain("codesign --force --deep --options runtime") expect(workflow).toContain( "node scripts/create-macos-dmg.mjs --volume-name DataConnect" @@ -154,13 +177,63 @@ describe("release workflow", () => { "manual-install-${{ matrix.artifact_key }}" ) expect(downloadArtifacts).toContain("pattern: manual-install-*") - expect(publishArtifacts).toContain("manual-install-macos-arm64") - expect(publishArtifacts).toContain("manual-install-macos-x64") - expect(publishArtifacts).toContain("manual-install-linux-x64 '*.deb'") - expect(publishArtifacts).toContain("manual-install-linux-x64 '*.AppImage'") - expect(publishArtifacts).toContain("manual-install-windows-x64 '*.exe'") + expect(publishArtifacts).toContain("manual-install-macos-arm64 'dmg/*.dmg'") + expect(publishArtifacts).toContain("manual-install-macos-x64 'dmg/*.dmg'") + expect(publishArtifacts).toContain("manual-install-linux-x64 'deb/*.deb'") + expect(publishArtifacts).toContain( + "manual-install-linux-x64 'appimage/*.AppImage'" + ) + expect(publishArtifacts).toContain( + "manual-install-windows-x64 'nsis/*.exe'" + ) + expect(publishArtifacts).toContain('${#artifacts[@]}" -ne 5') expect(publishArtifacts).toContain( 'gh release upload "$RELEASE_TAG" "${artifacts[@]}" --clobber' ) }) + + it("publishes five files from upload-artifact's preserved subdirectories", () => { + const publishScript = readWorkflowRunScript( + readReleaseWorkflow(), + "Publish complete platform set" + ) + const root = mkdtempSync(join(tmpdir(), "data-connect-publish-layout-")) + const expectedAssets = [ + "manual-install-macos-arm64/dmg/DataConnect_0.7.54_arm64.dmg", + "manual-install-macos-x64/dmg/DataConnect_0.7.54_x86_64.dmg", + "manual-install-linux-x64/deb/DataConnect_0.7.54_amd64.deb", + "manual-install-linux-x64/appimage/DataConnect_0.7.54_amd64.AppImage", + "manual-install-windows-x64/nsis/DataConnect_0.7.54_x64-setup.exe", + ] + + try { + for (const asset of expectedAssets) { + const path = join(root, "release-artifacts", asset) + mkdirSync(dirname(path), { recursive: true }) + writeFileSync(path, "verified artifact") + } + + const output = execFileSync( + "bash", + ["-c", `gh() { printf 'GH_ARG=%s\\n' "$@"; }\n${publishScript}`], + { + cwd: root, + encoding: "utf8", + env: { ...process.env, RELEASE_TAG: "v0.7.54" }, + } + ) + const uploadArgs = output + .split("\n") + .filter(line => line.startsWith("GH_ARG=")) + .map(line => line.slice("GH_ARG=".length)) + + expect(uploadArgs.slice(0, 3)).toEqual(["release", "upload", "v0.7.54"]) + expect(uploadArgs.slice(3, -1).sort()).toEqual( + expectedAssets.map(asset => `release-artifacts/${asset}`).sort() + ) + expect(uploadArgs.at(-1)).toBe("--clobber") + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) }) diff --git a/scripts/stage-pdpp-node.mjs b/scripts/stage-pdpp-node.mjs new file mode 100644 index 00000000..a7a4c336 --- /dev/null +++ b/scripts/stage-pdpp-node.mjs @@ -0,0 +1,135 @@ +#!/usr/bin/env node + +import { copyFileSync, existsSync, mkdirSync, rmSync, statSync } from "node:fs" +import { basename, dirname, join, resolve } from "node:path" +import { isMainModule } from "./is-main-module.js" + +const SUPPORTED_TARGETS = new Map([ + ["aarch64-apple-darwin", { platform: "darwin", arch: "arm64" }], + ["x86_64-apple-darwin", { platform: "darwin", arch: "x64" }], + ["x86_64-unknown-linux-gnu", { platform: "linux", arch: "x64" }], + ["x86_64-pc-windows-msvc", { platform: "win32", arch: "x64" }], +]) +export const RELEASE_NODE_VERSION = "22.23.1" + +export function nativeTauriTarget(platform, arch) { + const match = [...SUPPORTED_TARGETS].find( + ([, native]) => native.platform === platform && native.arch === arch + ) + if (!match) fail(`Unsupported native Node.js platform: ${platform}/${arch}`) + return { + target: match[0], + expectedPlatform: platform, + expectedArch: arch, + } +} + +function fail(message) { + throw new Error(message) +} + +export function sidecarFilename(target) { + const expected = SUPPORTED_TARGETS.get(target) + if (!expected) fail(`Unsupported Node.js sidecar target: ${target}`) + return `pdpp-node-${target}${expected.platform === "win32" ? ".exe" : ""}` +} + +export function stagePdppNode({ + target, + expectedPlatform, + expectedArch, + source = process.execPath, + sourceLicense, + actualPlatform = process.platform, + actualArch = process.arch, + nodeVersion = process.versions.node, + projectRoot = process.cwd(), +}) { + const supported = SUPPORTED_TARGETS.get(target) + if (!supported) fail(`Unsupported Node.js sidecar target: ${target}`) + if ( + expectedPlatform !== supported.platform || + expectedArch !== supported.arch + ) { + fail( + `Target ${target} requires Node.js ${supported.platform}/${supported.arch}` + ) + } + if (actualPlatform !== expectedPlatform || actualArch !== expectedArch) { + fail( + `setup-node architecture mismatch: expected ${expectedPlatform}/${expectedArch}, got ${actualPlatform}/${actualArch}` + ) + } + if (nodeVersion !== RELEASE_NODE_VERSION) { + fail( + `Release sidecar must use Node.js ${RELEASE_NODE_VERSION}; setup-node supplied ${nodeVersion}` + ) + } + if (!existsSync(source) || !statSync(source).isFile()) { + fail(`setup-node executable does not exist: ${source}`) + } + const license = + sourceLicense ?? + [ + join(dirname(source), "LICENSE"), + join(dirname(source), "..", "LICENSE"), + ].find(candidate => existsSync(candidate) && statSync(candidate).isFile()) + if (!license) { + fail( + `Node.js LICENSE was not found beside the setup-node executable: ${source}` + ) + } + + const destination = resolve( + projectRoot, + "src-tauri", + "binaries", + sidecarFilename(target) + ) + mkdirSync(dirname(destination), { recursive: true }) + // Debug builds may hard-link these destinations to the developer's Node. + // Unlink first so release staging never mutates that source inode. + rmSync(destination, { force: true }) + copyFileSync(source, destination) + const licenseDestination = resolve( + projectRoot, + "src-tauri", + "binaries", + "pdpp-node-LICENSE" + ) + rmSync(licenseDestination, { force: true }) + copyFileSync(license, licenseDestination) + if (expectedPlatform !== "win32") { + // copyFileSync preserves the executable mode on Unix; fail closed if that + // ever changes instead of shipping a sidecar the OS cannot start. + if ((statSync(destination).mode & 0o111) === 0) { + fail(`Staged Node.js sidecar is not executable: ${destination}`) + } + } + return { executable: destination, license: licenseDestination } +} + +export function parseArgs(argv) { + const args = {} + for (let index = 0; index < argv.length; index += 1) { + const value = argv[index + 1] + if (argv[index] === "--target") args.target = value + else if (argv[index] === "--expected-platform") + args.expectedPlatform = value + else if (argv[index] === "--expected-arch") args.expectedArch = value + else fail(`Unknown argument: ${argv[index]}`) + index += 1 + } + if (!args.target || !args.expectedPlatform || !args.expectedArch) { + fail( + "Usage: --target --expected-platform --expected-arch " + ) + } + return args +} + +if (isMainModule(import.meta.url, process.argv[1])) { + const staged = stagePdppNode(parseArgs(process.argv.slice(2))) + console.log(`Staged ${basename(process.execPath)} as ${staged.executable}`) + console.log(`Staged Node.js notices as ${staged.license}`) +} diff --git a/scripts/stage-pdpp-node.test.mjs b/scripts/stage-pdpp-node.test.mjs new file mode 100644 index 00000000..7c105867 --- /dev/null +++ b/scripts/stage-pdpp-node.test.mjs @@ -0,0 +1,176 @@ +import { + mkdtempSync, + mkdirSync, + linkSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs" +import { tmpdir } from "node:os" +import { join, resolve } from "node:path" +import { describe, expect, it } from "vitest" +import { + parseArgs, + nativeTauriTarget, + sidecarFilename, + stagePdppNode, + RELEASE_NODE_VERSION, +} from "./stage-pdpp-node.mjs" + +describe("stage PDPP Node.js sidecar", () => { + it.each([ + ["darwin", "arm64", "aarch64-apple-darwin"], + ["darwin", "x64", "x86_64-apple-darwin"], + ["linux", "x64", "x86_64-unknown-linux-gnu"], + ["win32", "x64", "x86_64-pc-windows-msvc"], + ])("maps native %s/%s to %s", (platform, arch, target) => { + expect(nativeTauriTarget(platform, arch)).toEqual({ + target, + expectedPlatform: platform, + expectedArch: arch, + }) + }) + + it.each([ + ["aarch64-apple-darwin", "pdpp-node-aarch64-apple-darwin"], + ["x86_64-apple-darwin", "pdpp-node-x86_64-apple-darwin"], + ["x86_64-unknown-linux-gnu", "pdpp-node-x86_64-unknown-linux-gnu"], + ["x86_64-pc-windows-msvc", "pdpp-node-x86_64-pc-windows-msvc.exe"], + ])( + "uses Tauri's target-qualified sidecar name for %s", + (target, filename) => { + expect(sidecarFilename(target)).toBe(filename) + } + ) + + it("copies the setup-node executable only when version and architecture match", () => { + const root = mkdtempSync(join(tmpdir(), "pdpp-node-sidecar-")) + const source = join(root, "node-source") + const sourceLicense = join(root, "LICENSE") + try { + mkdirSync(join(root, "src-tauri"), { recursive: true }) + writeFileSync(source, "fixture", { mode: 0o755 }) + writeFileSync(sourceLicense, "Node.js license fixture") + const staged = stagePdppNode({ + target: "x86_64-unknown-linux-gnu", + expectedPlatform: "linux", + expectedArch: "x64", + source, + sourceLicense, + actualPlatform: "linux", + actualArch: "x64", + nodeVersion: RELEASE_NODE_VERSION, + projectRoot: root, + }) + expect(staged).toEqual({ + executable: join( + root, + "src-tauri/binaries/pdpp-node-x86_64-unknown-linux-gnu" + ), + license: join(root, "src-tauri/binaries/pdpp-node-LICENSE"), + }) + expect(readFileSync(staged.executable, "utf8")).toBe("fixture") + expect(readFileSync(staged.license, "utf8")).toBe( + "Node.js license fixture" + ) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) + + it("rejects the wrong Node major or native architecture", () => { + const common = { + target: "x86_64-unknown-linux-gnu", + expectedPlatform: "linux", + expectedArch: "x64", + source: process.execPath, + sourceLicense: process.execPath, + actualPlatform: "linux", + actualArch: "x64", + projectRoot: tmpdir(), + } + expect(() => stagePdppNode({ ...common, nodeVersion: "22.18.0" })).toThrow( + `must use Node.js ${RELEASE_NODE_VERSION}` + ) + expect(() => + stagePdppNode({ + ...common, + nodeVersion: RELEASE_NODE_VERSION, + actualArch: "arm64", + }) + ).toThrow("architecture mismatch") + }) + + it("replaces debug hard links without modifying their source files", () => { + const root = mkdtempSync(join(tmpdir(), "pdpp-node-hard-link-")) + const source = join(root, "release-node") + const sourceLicense = join(root, "release-LICENSE") + const developmentNode = join(root, "development-node") + const developmentLicense = join(root, "development-LICENSE") + const binaries = join(root, "src-tauri", "binaries") + try { + mkdirSync(binaries, { recursive: true }) + writeFileSync(source, "release", { mode: 0o755 }) + writeFileSync(sourceLicense, "release license") + writeFileSync(developmentNode, "development", { mode: 0o755 }) + writeFileSync(developmentLicense, "development license") + linkSync( + developmentNode, + join(binaries, "pdpp-node-x86_64-unknown-linux-gnu") + ) + linkSync(developmentLicense, join(binaries, "pdpp-node-LICENSE")) + + stagePdppNode({ + target: "x86_64-unknown-linux-gnu", + expectedPlatform: "linux", + expectedArch: "x64", + source, + sourceLicense, + actualPlatform: "linux", + actualArch: "x64", + nodeVersion: RELEASE_NODE_VERSION, + projectRoot: root, + }) + + expect(readFileSync(developmentNode, "utf8")).toBe("development") + expect(readFileSync(developmentLicense, "utf8")).toBe( + "development license" + ) + expect( + readFileSync( + join(binaries, "pdpp-node-x86_64-unknown-linux-gnu"), + "utf8" + ) + ).toBe("release") + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) + + it("parses the release workflow arguments", () => { + expect( + parseArgs([ + "--target", + "aarch64-apple-darwin", + "--expected-platform", + "darwin", + "--expected-arch", + "arm64", + ]) + ).toEqual({ + target: "aarch64-apple-darwin", + expectedPlatform: "darwin", + expectedArch: "arm64", + }) + }) + + it("gates the supported local production build on the same staging contract", () => { + const buildScript = readFileSync( + resolve(process.cwd(), "scripts/build-prod.js"), + "utf8" + ) + expect(buildScript).toContain( + "stagePdppNode({ ...nativeTauriTarget(PLAT, arch()), projectRoot: ROOT })" + ) + }) +}) diff --git a/scripts/verify-bundled-personal-server.mjs b/scripts/verify-bundled-personal-server.mjs index 60ca4ecc..a089bf48 100644 --- a/scripts/verify-bundled-personal-server.mjs +++ b/scripts/verify-bundled-personal-server.mjs @@ -8,9 +8,23 @@ import { createInterface } from "node:readline" import { isMainModule } from "./is-main-module.js" const REQUIRED_PATH_FRAGMENTS = [ + "connectors/lock.json", + "connectors/collection-profiles/github-pdpp/profile/collection-profile.json", + "connectors/collection-profiles/github-pdpp/dist/collection-profile.mjs", + "connectors/collection-profiles/github-pdpp/provenance.json", + "connectors/collection-profiles/chatgpt-pdpp/profile/collection-profile.json", + "connectors/collection-profiles/chatgpt-pdpp/dist/collection-profile.mjs", + "connectors/collection-profiles/chatgpt-pdpp/provenance.json", + "licenses/pdpp-node-license", "personal-server/dist/personal-server", "personal-server/dist/node_modules/better-sqlite3/build/Release/better_sqlite3.node", "playwright-runner/dist/playwright-runner", + "pdpp-runtime/connector-loader.mjs", + "pdpp-runtime/connector-loader-bootstrap.mjs", + "pdpp-runtime/node_modules/p-queue/package.json", + "pdpp-runtime/node_modules/p-queue/dist/index.js", + "pdpp-runtime/node_modules/patchright/package.json", + "pdpp-runtime/node_modules/patchright/index.mjs", ] function fail(message) { @@ -36,10 +50,12 @@ function normalizeEntry(entry) { } export function assertPackagedRuntime(entries, artifactName) { - const normalizedEntries = entries.map(normalizeEntry) - const missing = REQUIRED_PATH_FRAGMENTS.filter( - fragment => - !normalizedEntries.some(entry => entry.includes(fragment.toLowerCase())) + const platform = artifactPlatform(artifactName) + const packagedPaths = new Set( + entries.map(entry => packagedEntryPath(entry, platform)) + ) + const missing = expectedRuntimePaths(platform).filter( + expectedPath => !packagedPaths.has(expectedPath) ) if (missing.length > 0) { @@ -49,7 +65,65 @@ export function assertPackagedRuntime(entries, artifactName) { } } +function artifactPlatform(artifactName) { + const normalizedName = artifactName.toLowerCase() + if (normalizedName.endsWith(".exe")) return "windows" + if (normalizedName.endsWith(".dmg") || normalizedName.endsWith(".app")) + return "macos" + if (normalizedName.endsWith(".deb") || normalizedName.endsWith(".appimage")) + return "linux" + fail(`Cannot determine artifact platform from ${artifactName}`) +} + +function expectedRuntimePaths(platform) { + const root = { + linux: "usr/lib/dataconnect/", + macos: "contents/resources/", + windows: "", + }[platform] + if (root === undefined) fail(`Unsupported runtime platform: ${platform}`) + return REQUIRED_PATH_FRAGMENTS.map(fragment => { + const executableSuffix = + platform === "windows" && + (fragment.endsWith("personal-server/dist/personal-server") || + fragment.endsWith("playwright-runner/dist/playwright-runner")) + ? ".exe" + : "" + return `${root}${fragment}${executableSuffix}`.toLowerCase() + }) +} + +function packagedEntryPath(entry, platform) { + if (platform === "windows") return windowsInstallerEntryPath(entry) + const normalized = normalizeEntry(entry).trim() + if (platform === "macos") return normalized + const path = normalized.includes(" ") + ? normalized.split(/\s+/).at(-1) + : normalized + return path.replace(/^\.\//, "") +} + +export function assertPackagedNode(entries, artifactName, platform) { + const expectedPath = { + linux: "usr/bin/pdpp-node", + macos: "contents/macos/pdpp-node", + windows: "pdpp-node.exe", + }[platform] + if (!expectedPath) fail(`Unsupported Node sidecar platform: ${platform}`) + const hasNode = entries.some( + entry => packagedEntryPath(entry, platform) === expectedPath + ) + if (!hasNode) { + fail(`${artifactName} is missing its packaged Node.js sidecar`) + } +} + export function assertPackagedBrowser(entries, artifactName, platform) { + const expectedPrefix = { + linux: "usr/lib/dataconnect/playwright-runner/dist/browsers/chromium-", + macos: "contents/resources/playwright-runner/dist/browsers/chromium-", + windows: "playwright-runner/dist/browsers/chromium-", + }[platform] const executableNames = { linux: ["/chrome"], macos: [ @@ -58,14 +132,17 @@ export function assertPackagedBrowser(entries, artifactName, platform) { ], windows: ["/chrome.exe"], }[platform] - if (!executableNames) fail(`Unsupported browser platform: ${platform}`) - - const hasBrowserExecutable = entries.map(normalizeEntry).some(entry => { - return ( - entry.includes("playwright-runner/dist/browsers/chromium-") && - executableNames.some(name => entry.endsWith(name)) - ) - }) + if (!expectedPrefix || !executableNames) + fail(`Unsupported browser platform: ${platform}`) + + const hasBrowserExecutable = entries + .map(entry => packagedEntryPath(entry, platform)) + .some(entry => { + return ( + entry.startsWith(expectedPrefix) && + executableNames.some(name => entry.endsWith(name)) + ) + }) if (!hasBrowserExecutable) { fail(`${artifactName} is missing its packaged Chromium executable`) } @@ -119,12 +196,16 @@ function listAppImageEntries(artifact) { const WINDOWS_BROWSER_FRAGMENT = "playwright-runner/dist/browsers/chromium-" const WINDOWS_BROWSER_EXECUTABLE = "/chrome.exe" -const WINDOWS_RELEVANT_FRAGMENTS = [ - ...REQUIRED_PATH_FRAGMENTS, - WINDOWS_BROWSER_FRAGMENT, -].map(normalizeEntry) +const WINDOWS_NODE_FILENAME = "pdpp-node.exe" +const WINDOWS_RUNTIME_PATHS = new Set(expectedRuntimePaths("windows")) const COMMAND_ERROR_OUTPUT_LIMIT = 64 * 1024 +function windowsInstallerEntryPath(entry) { + // `7z l -ba` prefixes metadata, while Tauri's NSIS `/oname` sidecar is a + // bare root entry. Compare that installed path, not the staged source name. + return normalizeEntry(entry).trim().split(/\s+/).at(-1) +} + function appendBoundedOutput(output, chunk) { return `${output}${chunk}`.slice(-COMMAND_ERROR_OUTPUT_LIMIT) } @@ -147,18 +228,13 @@ export async function collectRelevantWindowsInstallerEntries( const lines = createInterface({ input: child.stdout, crlfDelay: Infinity }) lines.on("line", line => { - const normalizedLine = normalizeEntry(line) - for (const fragment of WINDOWS_RELEVANT_FRAGMENTS) { - if (relevantEntries.has(fragment)) continue - if (!normalizedLine.includes(fragment)) continue - if ( - fragment === WINDOWS_BROWSER_FRAGMENT && - !normalizedLine.endsWith(WINDOWS_BROWSER_EXECUTABLE) - ) { - continue - } - relevantEntries.set(fragment, line) - } + const path = windowsInstallerEntryPath(line) + const isRuntime = WINDOWS_RUNTIME_PATHS.has(path) + const isNode = path === WINDOWS_NODE_FILENAME + const isBrowser = + path.startsWith(WINDOWS_BROWSER_FRAGMENT) && + path.endsWith(WINDOWS_BROWSER_EXECUTABLE) + if (isRuntime || isNode || isBrowser) relevantEntries.set(path, line) }) const processExited = new Promise((resolveExit, rejectExit) => { @@ -195,24 +271,74 @@ function findAppBundles(root) { }) } +export function readMacBundleExecutable(infoPlist, runCommand = run) { + const executable = runCommand("plutil", [ + "-extract", + "CFBundleExecutable", + "raw", + "-o", + "-", + infoPlist, + ]).trim() + if (!executable) fail(`${infoPlist} has an empty CFBundleExecutable`) + return executable +} + +export function selectMacAppExecutables( + executableNames, + declaredExecutable, + artifactName +) { + const mainMatches = executableNames.filter( + name => name === declaredExecutable + ) + if (mainMatches.length !== 1) { + fail( + `${artifactName} CFBundleExecutable ${declaredExecutable} must identify exactly one app executable` + ) + } + + const nodeMatches = executableNames.filter(name => name === "pdpp-node") + if (nodeMatches.length !== 1) { + fail(`${artifactName} must contain exactly one pdpp-node sidecar`) + } + if (declaredExecutable === "pdpp-node") { + fail(`${artifactName} app executable must be distinct from pdpp-node`) + } + + return { + appExecutable: declaredExecutable, + nodeSidecar: nodeMatches[0], + } +} + function verifyMacApp(app, expectedArch, artifactName, verifyCodeSignature) { if (verifyCodeSignature) { run("codesign", ["--verify", "--deep", "--strict", app]) } const entries = listDirectoryEntries(app) assertPackagedRuntime(entries, artifactName) + assertPackagedNode(entries, artifactName, "macos") assertPackagedBrowser(entries, artifactName, "macos") const executableDirectory = join(app, "Contents", "MacOS") - const appExecutables = readdirSync(executableDirectory, { + const executableNames = readdirSync(executableDirectory, { withFileTypes: true, - }).filter(entry => entry.isFile()) - if (appExecutables.length !== 1) { - fail(`${artifactName} must contain exactly one app executable`) - } + }) + .filter(entry => entry.isFile()) + .map(entry => entry.name) + const declaredExecutable = readMacBundleExecutable( + join(app, "Contents", "Info.plist") + ) + const { appExecutable, nodeSidecar } = selectMacAppExecutables( + executableNames, + declaredExecutable, + artifactName + ) const binaries = [ - join(executableDirectory, appExecutables[0].name), + join(executableDirectory, appExecutable), + join(executableDirectory, nodeSidecar), join( app, "Contents", @@ -317,9 +443,11 @@ function verifyLinuxArtifacts(bundleRoot) { } const debEntries = listDebEntries(debArtifacts[0]) assertPackagedRuntime(debEntries, basename(debArtifacts[0])) + assertPackagedNode(debEntries, basename(debArtifacts[0]), "linux") assertPackagedBrowser(debEntries, basename(debArtifacts[0]), "linux") const appImageEntries = listAppImageEntries(appImageArtifacts[0]) assertPackagedRuntime(appImageEntries, basename(appImageArtifacts[0])) + assertPackagedNode(appImageEntries, basename(appImageArtifacts[0]), "linux") assertPackagedBrowser( appImageEntries, basename(appImageArtifacts[0]), @@ -334,6 +462,7 @@ async function verifyWindowsArtifacts(bundleRoot) { if (installers.length !== 1) fail("Expected exactly one NSIS installer") const entries = await listWindowsInstallerEntries(installers[0]) assertPackagedRuntime(entries, basename(installers[0])) + assertPackagedNode(entries, basename(installers[0]), "windows") assertPackagedBrowser(entries, basename(installers[0]), "windows") } diff --git a/scripts/verify-bundled-personal-server.test.mjs b/scripts/verify-bundled-personal-server.test.mjs index 2ee23486..f915aed1 100644 --- a/scripts/verify-bundled-personal-server.test.mjs +++ b/scripts/verify-bundled-personal-server.test.mjs @@ -13,20 +13,59 @@ import { describe, expect, it } from "vitest" import { assertBinaryArchitecture, assertPackagedBrowser, + assertPackagedNode, assertPackagedRuntime, collectRelevantWindowsInstallerEntries, listDebEntries, parseArgs, + readMacBundleExecutable, + selectMacAppExecutables, } from "./verify-bundled-personal-server.mjs" const runtimeEntries = [ - "usr/lib/data-connect/resources/personal-server/dist/personal-server", - "usr/lib/data-connect/resources/personal-server/dist/node_modules/better-sqlite3/build/Release/better_sqlite3.node", - "usr/lib/data-connect/resources/playwright-runner/dist/playwright-runner", + "usr/bin/pdpp-node", + "usr/lib/DataConnect/connectors/lock.json", + "usr/lib/DataConnect/connectors/collection-profiles/github-pdpp/profile/collection-profile.json", + "usr/lib/DataConnect/connectors/collection-profiles/github-pdpp/dist/collection-profile.mjs", + "usr/lib/DataConnect/connectors/collection-profiles/github-pdpp/provenance.json", + "usr/lib/DataConnect/connectors/collection-profiles/chatgpt-pdpp/profile/collection-profile.json", + "usr/lib/DataConnect/connectors/collection-profiles/chatgpt-pdpp/dist/collection-profile.mjs", + "usr/lib/DataConnect/connectors/collection-profiles/chatgpt-pdpp/provenance.json", + "usr/lib/DataConnect/licenses/pdpp-node-LICENSE", + "usr/lib/DataConnect/personal-server/dist/personal-server", + "usr/lib/DataConnect/personal-server/dist/node_modules/better-sqlite3/build/Release/better_sqlite3.node", + "usr/lib/DataConnect/playwright-runner/dist/playwright-runner", + "usr/lib/DataConnect/pdpp-runtime/connector-loader.mjs", + "usr/lib/DataConnect/pdpp-runtime/connector-loader-bootstrap.mjs", + "usr/lib/DataConnect/pdpp-runtime/node_modules/p-queue/package.json", + "usr/lib/DataConnect/pdpp-runtime/node_modules/p-queue/dist/index.js", + "usr/lib/DataConnect/pdpp-runtime/node_modules/patchright/package.json", + "usr/lib/DataConnect/pdpp-runtime/node_modules/patchright/index.mjs", ] +const windowsRuntimeEntries = runtimeEntries + .filter(entry => entry !== "usr/bin/pdpp-node") + .map(entry => + entry + .replace("usr/lib/DataConnect/", "") + .replace( + "personal-server/dist/personal-server", + "personal-server/dist/personal-server.exe" + ) + .replace( + "playwright-runner/dist/playwright-runner", + "playwright-runner/dist/playwright-runner.exe" + ) + ) + +const macRuntimeEntries = windowsRuntimeEntries.map(entry => + `Contents/Resources/${entry}` + .replace("personal-server.exe", "personal-server") + .replace("playwright-runner.exe", "playwright-runner") +) + const linuxBrowserEntry = - "usr/lib/data-connect/resources/playwright-runner/dist/browsers/chromium-1228/chrome-linux64/chrome" + "usr/lib/DataConnect/playwright-runner/dist/browsers/chromium-1228/chrome-linux64/chrome" describe("bundled personal-server verifier", () => { it("accepts an artifact with the helper and native dependency", () => { @@ -44,6 +83,106 @@ describe("bundled personal-server verifier", () => { ).toThrow("better-sqlite3") }) + it("rejects an artifact missing a PDPP runtime dependency", () => { + expect(() => + assertPackagedRuntime( + runtimeEntries.filter(entry => !entry.includes("p-queue/package.json")), + "DataConnect.deb" + ) + ).toThrow("p-queue") + }) + + it("rejects an artifact missing a bundled PDPP connector", () => { + expect(() => + assertPackagedRuntime( + runtimeEntries.filter(entry => !entry.includes("chatgpt-pdpp")), + "DataConnect.deb" + ) + ).toThrow("chatgpt-pdpp") + }) + + it("requires packaged runtime paths on exact segment boundaries", () => { + expect(() => + assertPackagedRuntime( + runtimeEntries.map(entry => + entry.endsWith("connectors/lock.json") + ? entry.replace("connectors/lock.json", "not-connectors/lock.json") + : entry + ), + "DataConnect.deb" + ) + ).toThrow("connectors/lock.json") + }) + + it("rejects a complete runtime tree under an unrecognized root", () => { + for (const wrongRoot of ["attacker", "resources-not-really"]) { + expect(() => + assertPackagedRuntime( + windowsRuntimeEntries.map(entry => `${wrongRoot}/${entry}`), + "DataConnect.exe" + ) + ).toThrow("connectors/lock.json") + } + }) + + it("accepts only the macOS app resource root", () => { + expect(() => + assertPackagedRuntime(macRuntimeEntries, "DataConnect.app") + ).not.toThrow() + expect(() => + assertPackagedRuntime( + macRuntimeEntries.map(entry => `attacker/${entry}`), + "DataConnect.dmg" + ) + ).toThrow("connectors/lock.json") + }) + + it("requires the platform-native Node sidecar path", () => { + expect(() => + assertPackagedNode(runtimeEntries, "DataConnect.deb", "linux") + ).not.toThrow() + expect(() => + assertPackagedNode(["pdpp-node.exe"], "DataConnect.exe", "windows") + ).not.toThrow() + expect(() => + assertPackagedNode( + ["Contents/MacOS/pdpp-node"], + "DataConnect.dmg", + "macos" + ) + ).not.toThrow() + expect(() => + assertPackagedNode(runtimeEntries, "DataConnect.exe", "windows") + ).toThrow("packaged Node.js sidecar") + expect(() => + assertPackagedNode( + ["attacker/usr/bin/pdpp-node"], + "DataConnect.deb", + "linux" + ) + ).toThrow("packaged Node.js sidecar") + }) + + it("requires Tauri's exact root-installed Windows x64 Node sidecar", () => { + const tauriNsisListing = [ + "2026-07-31 23:39:00 ....A 124835376 50000000 pdpp-node.exe", + ] + expect(() => + assertPackagedNode(tauriNsisListing, "DataConnect.exe", "windows") + ).not.toThrow() + + for (const wrongEntry of [ + "resources/pdpp-node.exe", + "pdpp-node-x86_64-pc-windows-msvc.exe", + "not-pdpp-node.exe", + "pdpp-node-arm64.exe", + ]) { + expect(() => + assertPackagedNode([wrongEntry], "DataConnect.exe", "windows") + ).toThrow("packaged Node.js sidecar") + } + }) + it("requires a real packaged browser executable", () => { expect(() => assertPackagedBrowser( @@ -55,6 +194,13 @@ describe("bundled personal-server verifier", () => { expect(() => assertPackagedBrowser(runtimeEntries, "DataConnect.deb", "linux") ).toThrow("packaged Chromium executable") + expect(() => + assertPackagedBrowser( + [`attacker/${linuxBrowserEntry}`], + "DataConnect.deb", + "linux" + ) + ).toThrow("packaged Chromium executable") }) it("rejects a helper built for the other macOS architecture", () => { @@ -74,6 +220,72 @@ describe("bundled personal-server verifier", () => { ).toThrow("expected x86_64") }) + it("uses CFBundleExecutable when the macOS app contains its Node sidecar", () => { + const invocation = [] + expect( + readMacBundleExecutable( + "DataConnect.app/Contents/Info.plist", + (command, args) => { + invocation.push(command, args) + return "DataConnect\n" + } + ) + ).toBe("DataConnect") + expect(invocation).toEqual([ + "plutil", + [ + "-extract", + "CFBundleExecutable", + "raw", + "-o", + "-", + "DataConnect.app/Contents/Info.plist", + ], + ]) + expect(() => + readMacBundleExecutable("DataConnect.app/Contents/Info.plist", () => "\n") + ).toThrow("empty CFBundleExecutable") + expect( + selectMacAppExecutables( + ["DataConnect", "pdpp-node"], + "DataConnect", + "DataConnect.app" + ) + ).toEqual({ + appExecutable: "DataConnect", + nodeSidecar: "pdpp-node", + }) + }) + + it("rejects a missing or ambiguous CFBundleExecutable file", () => { + expect(() => + selectMacAppExecutables(["pdpp-node"], "DataConnect", "DataConnect.app") + ).toThrow("must identify exactly one app executable") + expect(() => + selectMacAppExecutables( + ["DataConnect", "DataConnect", "pdpp-node"], + "DataConnect", + "DataConnect.app" + ) + ).toThrow("must identify exactly one app executable") + }) + + it("requires a distinct, exact Node sidecar beside the macOS app executable", () => { + expect(() => + selectMacAppExecutables(["DataConnect"], "DataConnect", "DataConnect.app") + ).toThrow("exactly one pdpp-node sidecar") + expect(() => + selectMacAppExecutables(["pdpp-node"], "pdpp-node", "DataConnect.app") + ).toThrow("must be distinct from pdpp-node") + expect(() => + assertBinaryArchitecture( + "Mach-O 64-bit executable arm64", + "arm64", + "pdpp-node" + ) + ).not.toThrow() + }) + it("enables strict signature checks only when explicitly requested", () => { const commonArgs = [ "--platform", @@ -100,8 +312,10 @@ describe("bundled personal-server verifier", () => { console.log(\`2026-07-31 ..... \${noise}-\${index}\`) } for (const entry of ${JSON.stringify([ - ...runtimeEntries, - "usr/lib/data-connect/resources/playwright-runner/dist/browsers/chromium-1228/chrome-win64/chrome.exe", + ...windowsRuntimeEntries, + "resources/pdpp-node.exe", + "pdpp-node.exe", + "playwright-runner/dist/browsers/chromium-1228/chrome-win64/chrome.exe", ])}) { console.log(\`2026-07-31 ..... \${entry.replaceAll("/", "\\\\")}\`) } @@ -109,10 +323,13 @@ describe("bundled personal-server verifier", () => { ] ) - expect(entries).toHaveLength(4) + expect(entries).toHaveLength(19) expect(() => assertPackagedRuntime(entries, "DataConnect.exe") ).not.toThrow() + expect(() => + assertPackagedNode(entries, "DataConnect.exe", "windows") + ).not.toThrow() expect(() => assertPackagedBrowser(entries, "DataConnect.exe", "windows") ).not.toThrow() @@ -137,17 +354,20 @@ describe("bundled personal-server verifier", () => { () => child ) - child.stdout.write(`${runtimeEntries.join("\n")}\n`) + child.stdout.write(`${windowsRuntimeEntries.join("\n")}\n`) child.stderr.end() child.emit("close", 0) setImmediate(() => { child.stdout.end( - "resources/playwright-runner/dist/browsers/chromium-1228/chrome-win64/chrome.exe" + "pdpp-node.exe\nplaywright-runner/dist/browsers/chromium-1228/chrome-win64/chrome.exe" ) }) const entries = await entriesPromise - expect(entries).toHaveLength(4) + expect(entries).toHaveLength(19) + expect(() => + assertPackagedNode(entries, "DataConnect.exe", "windows") + ).not.toThrow() expect(() => assertPackagedBrowser(entries, "DataConnect.exe", "windows") ).not.toThrow() @@ -160,7 +380,7 @@ describe("bundled personal-server verifier", () => { const temporaryRoot = mkdtempSync(join(process.cwd(), ".verify-deb-")) try { const packageRoot = join(temporaryRoot, "package") - const resources = join(packageRoot, "usr", "lib", "data-connect") + const resources = join(packageRoot, "usr", "lib", "DataConnect") mkdirSync(join(packageRoot, "DEBIAN"), { recursive: true }) writeFileSync( join(packageRoot, "DEBIAN", "control"), diff --git a/src-tauri/build.rs b/src-tauri/build.rs index 795b9b7c..86ee7d37 100644 --- a/src-tauri/build.rs +++ b/src-tauri/build.rs @@ -1,3 +1,50 @@ fn main() { - tauri_build::build() + stage_development_node_sidecar(); + tauri_build::build() +} + +fn stage_development_node_sidecar() { + if std::env::var("PROFILE").as_deref() != Ok("debug") { + return; + } + let Ok(target) = std::env::var("TARGET") else { + return; + }; + let extension = if target.contains("windows") { + ".exe" + } else { + "" + }; + let destination = + std::path::PathBuf::from("binaries").join(format!("pdpp-node-{target}{extension}")); + let license_destination = std::path::PathBuf::from("binaries/pdpp-node-LICENSE"); + let Some(node) = std::env::var_os("PATH").and_then(|path| { + std::env::split_paths(&path) + .map(|directory| directory.join(if cfg!(windows) { "node.exe" } else { "node" })) + .find(|candidate| candidate.is_file()) + }) else { + return; + }; + let Some(license) = [ + node.parent().map(|path| path.join("LICENSE")), + node.parent().map(|path| path.join("../LICENSE")), + ] + .into_iter() + .flatten() + .find(|candidate| candidate.is_file()) else { + return; + }; + std::fs::create_dir_all("binaries").expect("failed to create development sidecar directory"); + stage_development_file(&node, &destination, "Node.js sidecar"); + stage_development_file(&license, &license_destination, "Node.js license"); +} + +fn stage_development_file(source: &std::path::Path, destination: &std::path::Path, label: &str) { + if destination.is_file() { + return; + } + if std::fs::hard_link(source, destination).is_err() { + std::fs::copy(source, destination) + .unwrap_or_else(|error| panic!("failed to stage development {label}: {error}")); + } } diff --git a/src-tauri/src/commands/connector.rs b/src-tauri/src/commands/connector.rs index bcb60dd4..9cb6e3b2 100644 --- a/src-tauri/src/commands/connector.rs +++ b/src-tauri/src/commands/connector.rs @@ -100,6 +100,7 @@ pub struct Platform { pub runtime: Option, /// Scopes this connector can export (just the scope strings, e.g. ["chatgpt.conversations", "chatgpt.memories"]) pub scopes: Option>, + pub setup: Option, } #[derive(Debug, Deserialize)] @@ -108,9 +109,33 @@ struct ActivePdppPlatformManifest { display_name: Option, name: Option, description: Option, + setup: Option, streams: Vec, } +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ActivePdppStaticSecretSetup { + modality: String, + credential_capture: ActivePdppCredentialCapture, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +struct ActivePdppCredentialCapture { + fields: Vec, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +struct ActivePdppCredentialField { + name: String, + label: Option, + #[serde(rename = "type")] + field_type: Option, + required: bool, + secret: bool, + autocomplete: Option, +} + #[derive(Debug, Deserialize)] struct ActivePdppStream { name: String, @@ -397,6 +422,7 @@ fn platform_from_metadata( vectorize_config: metadata.vectorize_config, runtime: runtime_override.or(metadata.runtime), scopes, + setup: None, } } @@ -551,6 +577,7 @@ fn load_active_pdpp_platforms() -> Vec { vectorize_config: None, runtime: Some("pdpp-network".to_string()), scopes: Some(scopes), + setup: manifest.setup, }); } @@ -567,6 +594,12 @@ fn pdpp_streams_to_dataconnect_scopes( ("github", "user") => Some("github.profile"), ("github", "repositories") => Some("github.repositories"), ("github", "starred") => Some("github.starred"), + ("chatgpt", "conversations") => Some("chatgpt.conversations"), + ("chatgpt", "messages") => Some("chatgpt.messages"), + ("chatgpt", "memories") => Some("chatgpt.memories"), + ("chatgpt", "custom_gpts") => Some("chatgpt.custom_gpts"), + ("chatgpt", "custom_instructions") => Some("chatgpt.custom_instructions"), + ("chatgpt", "shared_conversations") => Some("chatgpt.shared_conversations"), _ => None, }; if let Some(scope) = scope { @@ -2209,7 +2242,7 @@ fn resolve_browser_status( } /// Get system browser path (Chrome/Edge) -fn get_system_browser_path() -> Option { +pub(crate) fn get_system_browser_path() -> Option { #[cfg(target_os = "macos")] { let paths = ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"]; @@ -2302,7 +2335,7 @@ pub async fn test_nodejs(app: AppHandle) -> Result { } /// Get downloaded Chromium path from ~/.dataconnect/browsers -fn get_downloaded_chromium_path() -> Option { +pub(crate) fn get_downloaded_chromium_path() -> Option { let home = std::env::var("HOME") .or_else(|_| std::env::var("USERPROFILE")) .ok()?; @@ -2319,7 +2352,10 @@ fn get_bundled_chromium_path(resource_dir: &Path) -> Option { get_bundled_chromium_path_for_platform(resource_dir, current_browser_platform()) } -fn get_bundled_chromium_path_for_platform(resource_dir: &Path, platform: &str) -> Option { +pub(crate) fn get_bundled_chromium_path_for_platform( + resource_dir: &Path, + platform: &str, +) -> Option { let browser_roots = [ resource_dir .join("playwright-runner") @@ -2383,6 +2419,24 @@ fn find_chromium_executable(browsers_dir: &Path, platform: &str) -> Option) -> Option { + resolve_automation_browser_path_from( + get_system_browser_path(), + get_downloaded_chromium_path(), + resource_dir.and_then(get_bundled_chromium_path), + ) +} + +fn resolve_automation_browser_path_from( + system: Option, + downloaded: Option, + bundled: Option, +) -> Option { + system.or(downloaded).or(bundled) +} + /// Get the Chromium download URL for the current platform fn get_chromium_download_info() -> Option<(&'static str, &'static str)> { #[cfg(all(target_os = "macos", target_arch = "aarch64"))] @@ -2736,8 +2790,8 @@ pub async fn download_chromium_rust(app: AppHandle) -> Result { mod tests { use super::{ get_bundled_chromium_path_for_platform, get_downloaded_chromium_path_in_home, - manifest_looks_like_connector, resolve_browser_status, resolve_icon_path, - ConnectorMetadata, + manifest_looks_like_connector, resolve_automation_browser_path_from, + resolve_browser_status, resolve_icon_path, ConnectorMetadata, }; use std::path::{Path, PathBuf}; use tempfile::tempdir; @@ -2870,4 +2924,32 @@ mod tests { Some(executable) ); } + + #[test] + fn bundled_browser_lookup_supports_release_platform_layouts() { + for (platform, relative_path) in [ + ("linux", "playwright-runner/dist/browsers/chromium-1200/chrome-linux64/chrome"), + ("windows", "playwright-runner/dist/browsers/chromium-1200/chrome-win64/chrome.exe"), + ("macos", "playwright-runner/dist/browsers/chromium-1200/chrome-mac-x64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing"), + ("macos", "playwright-runner/dist/browsers/chromium-1200/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing"), + ] { + let temp = tempdir().expect("tempdir"); + let executable = create_file(temp.path(), relative_path); + + assert_eq!( + get_bundled_chromium_path_for_platform(temp.path(), platform), + Some(executable) + ); + } + } + + #[test] + fn pdpp_browser_discovery_uses_bundled_chromium_on_fresh_install() { + let bundled = PathBuf::from("bundled-browser"); + + assert_eq!( + resolve_automation_browser_path_from(None, None, Some(bundled.clone())), + Some(bundled) + ); + } } diff --git a/src-tauri/src/commands/connector_store.rs b/src-tauri/src/commands/connector_store.rs index 20e50ee3..e0c45bee 100644 --- a/src-tauri/src/commands/connector_store.rs +++ b/src-tauri/src/commands/connector_store.rs @@ -1,7 +1,13 @@ +use fs2::FileExt; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use std::fs; +use std::fs::{self, OpenOptions}; +use std::io::Write; use std::path::PathBuf; +use std::sync::Mutex; +use tempfile::NamedTempFile; + +static ACTIVE_CONNECTOR_MANIFEST_LOCK: Mutex<()> = Mutex::new(()); #[derive(Debug, Serialize, Deserialize, Clone, Default)] #[serde(rename_all = "camelCase")] @@ -57,6 +63,12 @@ pub fn get_active_manifest_path() -> Option { pub fn read_active_connector_manifest() -> Option { let manifest_path = get_active_manifest_path()?; + read_active_connector_manifest_from(&manifest_path) +} + +fn read_active_connector_manifest_from( + manifest_path: &std::path::Path, +) -> Option { if !manifest_path.exists() { return None; } @@ -65,20 +77,35 @@ pub fn read_active_connector_manifest() -> Option { serde_json::from_str(&content).ok() } -pub fn write_active_connector_manifest(manifest: &ActiveConnectorManifest) -> Result<(), String> { - let manifest_path = get_active_manifest_path().ok_or("Could not determine active manifest path")?; +fn write_active_connector_manifest_to( + manifest_path: &std::path::Path, + manifest: &ActiveConnectorManifest, +) -> Result<(), String> { if let Some(parent) = manifest_path.parent() { fs::create_dir_all(parent) .map_err(|e| format!("Failed to create connector manifest directory: {}", e))?; } - let temp_path = manifest_path.with_extension("json.tmp"); let content = serde_json::to_string_pretty(manifest) .map_err(|e| format!("Failed to serialize active connector manifest: {}", e))?; - - fs::write(&temp_path, content) + let parent = manifest_path + .parent() + .ok_or("Active connector manifest path has no parent directory")?; + let mut temp_file = NamedTempFile::new_in(parent).map_err(|e| { + format!( + "Failed to create active connector manifest temp file: {}", + e + ) + })?; + temp_file + .write_all(content.as_bytes()) .map_err(|e| format!("Failed to write active connector manifest: {}", e))?; - fs::rename(&temp_path, &manifest_path) + temp_file + .as_file() + .sync_all() + .map_err(|e| format!("Failed to sync active connector manifest: {}", e))?; + temp_file + .persist(manifest_path) .map_err(|e| format!("Failed to activate connector manifest: {}", e))?; Ok(()) @@ -88,3 +115,352 @@ pub fn get_active_connector_install(connector_id: &str) -> Option Result { + let manifest_path = + get_active_manifest_path().ok_or("Could not determine active manifest path")?; + update_active_connector_install_at(&manifest_path, install, policy) +} + +#[derive(Debug, Clone, Copy)] +enum ConnectorInstallUpdatePolicy { + ReplaceExisting, + RefreshBundledPathIfSameArtifact, +} + +fn update_active_connector_install_at( + manifest_path: &std::path::Path, + install: ActiveConnectorInstall, + policy: ConnectorInstallUpdatePolicy, +) -> Result { + let _guard = ACTIVE_CONNECTOR_MANIFEST_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let parent = manifest_path + .parent() + .ok_or("Active connector manifest path has no parent directory")?; + fs::create_dir_all(parent) + .map_err(|e| format!("Failed to create connector manifest directory: {}", e))?; + let lock_path = manifest_path.with_extension("json.lock"); + let lock_file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(&lock_path) + .map_err(|e| format!("Failed to open active connector manifest lock: {}", e))?; + lock_file + .lock_exclusive() + .map_err(|e| format!("Failed to lock active connector manifest: {}", e))?; + let mut manifest = + read_active_connector_manifest_from(manifest_path).unwrap_or(ActiveConnectorManifest { + version: "1.0".to_string(), + updated_at: chrono::Utc::now().to_rfc3339(), + connectors: HashMap::new(), + }); + if let Some(existing) = manifest.connectors.get_mut(&install.connector_id) { + match policy { + ConnectorInstallUpdatePolicy::ReplaceExisting => {} + ConnectorInstallUpdatePolicy::RefreshBundledPathIfSameArtifact => { + if !same_bundled_artifact(existing, &install) { + return Ok(false); + } + existing.root_path = install.root_path; + existing.metadata_relative_path = install.metadata_relative_path; + existing.script_relative_path = install.script_relative_path; + existing.manifest_path = install.manifest_path; + existing.entrypoint_path = install.entrypoint_path; + existing.provenance_path = install.provenance_path; + manifest.updated_at = chrono::Utc::now().to_rfc3339(); + write_active_connector_manifest_to(manifest_path, &manifest)?; + return Ok(true); + } + } + } + manifest.updated_at = chrono::Utc::now().to_rfc3339(); + manifest + .connectors + .insert(install.connector_id.clone(), install); + write_active_connector_manifest_to(manifest_path, &manifest)?; + Ok(true) +} + +fn same_bundled_artifact( + existing: &ActiveConnectorInstall, + install: &ActiveConnectorInstall, +) -> bool { + existing.connector_id == install.connector_id + && existing.company == install.company + && existing.version == install.version + && existing.artifact_kind == install.artifact_kind + && existing.manifest_path == install.manifest_path + && existing.entrypoint_path == install.entrypoint_path + && existing.provenance_path == install.provenance_path + && required_equal(&existing.manifest_sha256, &install.manifest_sha256) + && required_equal(&existing.entrypoint_sha256, &install.entrypoint_sha256) + && required_equal(&existing.provenance_sha256, &install.provenance_sha256) +} + +fn required_equal(existing: &Option, install: &Option) -> bool { + matches!((existing, install), (Some(existing), Some(install)) if existing == install) +} + +pub fn replace_active_connector_install(install: ActiveConnectorInstall) -> Result<(), String> { + update_active_connector_install(install, ConnectorInstallUpdatePolicy::ReplaceExisting) + .map(|_| ()) +} + +pub fn activate_bundled_connector_install(install: ActiveConnectorInstall) -> Result { + update_active_connector_install( + install, + ConnectorInstallUpdatePolicy::RefreshBundledPathIfSameArtifact, + ) +} + +#[cfg(test)] +mod tests { + use super::{ + read_active_connector_manifest_from, update_active_connector_install_at, + ActiveConnectorInstall, ConnectorInstallUpdatePolicy, + }; + use fs2::FileExt; + use std::fs::OpenOptions; + use std::process::Command; + use std::sync::{Arc, Barrier}; + use std::time::{Duration, Instant}; + use tempfile::tempdir; + + fn install(version: &str, root_path: &str) -> ActiveConnectorInstall { + ActiveConnectorInstall { + connector_id: "github-pdpp".to_string(), + company: "github".to_string(), + version: version.to_string(), + root_path: root_path.to_string(), + metadata_relative_path: "profile/collection-profile.json".to_string(), + script_relative_path: "dist/collection-profile.mjs".to_string(), + artifact_kind: Some("pdpp-collection-profile".to_string()), + manifest_path: Some("profile/collection-profile.json".to_string()), + entrypoint_path: Some("dist/collection-profile.mjs".to_string()), + entrypoint_sha256: Some("sha256:entrypoint".to_string()), + manifest_sha256: Some("sha256:manifest".to_string()), + provenance_path: Some("provenance.json".to_string()), + provenance_sha256: Some("sha256:provenance".to_string()), + } + } + + fn selected(manifest_path: &std::path::Path) -> ActiveConnectorInstall { + read_active_connector_manifest_from(manifest_path) + .expect("active connector manifest") + .connectors + .get("github-pdpp") + .expect("selection") + .clone() + } + + #[test] + fn bundled_activation_cannot_replace_a_concurrent_user_selection() { + let temp = tempdir().expect("manifest tempdir"); + + for iteration in 0..32 { + let manifest_path = temp.path().join(format!("active-{iteration}.json")); + let barrier = Arc::new(Barrier::new(3)); + let bundled_path = manifest_path.clone(); + let bundled_barrier = Arc::clone(&barrier); + let bundled = std::thread::spawn(move || { + bundled_barrier.wait(); + update_active_connector_install_at( + &bundled_path, + install("0.1.0", "/bundled/github-pdpp"), + ConnectorInstallUpdatePolicy::RefreshBundledPathIfSameArtifact, + ) + }); + let user_path = manifest_path.clone(); + let user_barrier = Arc::clone(&barrier); + let user = std::thread::spawn(move || { + user_barrier.wait(); + update_active_connector_install_at( + &user_path, + install("9.9.9", "/user/github-pdpp"), + ConnectorInstallUpdatePolicy::ReplaceExisting, + ) + }); + + barrier.wait(); + bundled + .join() + .expect("bundled thread") + .expect("bundled write"); + user.join().expect("user thread").expect("user write"); + + let selected = selected(&manifest_path); + assert_eq!(selected.version, "9.9.9"); + assert_eq!(selected.root_path, "/user/github-pdpp"); + } + } + + #[test] + fn bundled_activation_refreshes_path_for_same_exact_artifact() { + let temp = tempdir().expect("manifest tempdir"); + let manifest_path = temp.path().join("connectors-active.json"); + let mut stale = install("0.1.0", "/tmp/.mount-old/github-pdpp"); + stale.metadata_relative_path = "old-profile/collection-profile.json".to_string(); + stale.script_relative_path = "old-dist/collection-profile.mjs".to_string(); + update_active_connector_install_at( + &manifest_path, + stale, + ConnectorInstallUpdatePolicy::ReplaceExisting, + ) + .expect("stale bundled install"); + + let mut fresh = install("0.1.0", "/tmp/.mount-new/github-pdpp"); + fresh.metadata_relative_path = "profile/collection-profile.json".to_string(); + fresh.script_relative_path = "dist/collection-profile.mjs".to_string(); + assert!(update_active_connector_install_at( + &manifest_path, + fresh, + ConnectorInstallUpdatePolicy::RefreshBundledPathIfSameArtifact, + ) + .expect("fresh bundled activation")); + + let selected = selected(&manifest_path); + assert_eq!(selected.root_path, "/tmp/.mount-new/github-pdpp"); + assert_eq!( + selected.metadata_relative_path, + "profile/collection-profile.json" + ); + assert_eq!(selected.script_relative_path, "dist/collection-profile.mjs"); + assert_eq!(selected.version, "0.1.0"); + } + + #[test] + fn bundled_activation_preserves_distinct_user_install() { + let temp = tempdir().expect("manifest tempdir"); + let manifest_path = temp.path().join("connectors-active.json"); + update_active_connector_install_at( + &manifest_path, + install("9.9.9", "/user/github-pdpp"), + ConnectorInstallUpdatePolicy::ReplaceExisting, + ) + .expect("user install"); + + assert!(!update_active_connector_install_at( + &manifest_path, + install("0.1.0", "/bundled/github-pdpp"), + ConnectorInstallUpdatePolicy::RefreshBundledPathIfSameArtifact, + ) + .expect("bundled activation")); + + let selected = selected(&manifest_path); + assert_eq!(selected.version, "9.9.9"); + assert_eq!(selected.root_path, "/user/github-pdpp"); + } + + #[test] + fn bundled_activation_preserves_same_version_with_changed_or_missing_hash() { + let temp = tempdir().expect("manifest tempdir"); + let changed_hash_path = temp.path().join("changed-hash.json"); + update_active_connector_install_at( + &changed_hash_path, + install("0.1.0", "/user/github-pdpp"), + ConnectorInstallUpdatePolicy::ReplaceExisting, + ) + .expect("user install"); + + let mut changed = install("0.1.0", "/bundled/github-pdpp"); + changed.manifest_sha256 = Some("sha256:changed".to_string()); + assert!(!update_active_connector_install_at( + &changed_hash_path, + changed, + ConnectorInstallUpdatePolicy::RefreshBundledPathIfSameArtifact, + ) + .expect("changed hash bundled activation")); + assert_eq!(selected(&changed_hash_path).root_path, "/user/github-pdpp"); + + let missing_hash_path = temp.path().join("missing-hash.json"); + let mut missing = install("0.1.0", "/user/github-pdpp"); + missing.entrypoint_sha256 = None; + update_active_connector_install_at( + &missing_hash_path, + missing, + ConnectorInstallUpdatePolicy::ReplaceExisting, + ) + .expect("user install without hash"); + + assert!(!update_active_connector_install_at( + &missing_hash_path, + install("0.1.0", "/bundled/github-pdpp"), + ConnectorInstallUpdatePolicy::RefreshBundledPathIfSameArtifact, + ) + .expect("missing hash bundled activation")); + assert_eq!(selected(&missing_hash_path).root_path, "/user/github-pdpp"); + } + + #[test] + fn cross_process_manifest_update_helper() { + let Some(manifest_path) = std::env::var_os("DATACONNECT_TEST_MANIFEST_PATH") else { + return; + }; + let ready_path = + std::env::var_os("DATACONNECT_TEST_READY_PATH").expect("cross-process ready path"); + std::fs::write(&ready_path, b"ready").expect("cross-process ready marker"); + update_active_connector_install_at( + std::path::Path::new(&manifest_path), + install("9.9.9", "/user/github-pdpp"), + ConnectorInstallUpdatePolicy::ReplaceExisting, + ) + .expect("cross-process user update"); + } + + #[test] + fn manifest_updates_take_the_cross_process_file_lock() { + let temp = tempdir().expect("manifest tempdir"); + let manifest_path = temp.path().join("connectors-active.json"); + update_active_connector_install_at( + &manifest_path, + install("0.1.0", "/bundled/github-pdpp"), + ConnectorInstallUpdatePolicy::RefreshBundledPathIfSameArtifact, + ) + .expect("bundled install"); + + let lock_path = manifest_path.with_extension("json.lock"); + let lock_file = OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .expect("active manifest lock"); + lock_file.lock_exclusive().expect("parent file lock"); + + let ready_path = temp.path().join("child-ready"); + let mut child = Command::new(std::env::current_exe().expect("test executable")) + .args([ + "--exact", + "commands::connector_store::tests::cross_process_manifest_update_helper", + "--nocapture", + ]) + .env("DATACONNECT_TEST_MANIFEST_PATH", &manifest_path) + .env("DATACONNECT_TEST_READY_PATH", &ready_path) + .spawn() + .expect("cross-process update child"); + + let deadline = Instant::now() + Duration::from_secs(5); + while !ready_path.exists() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(ready_path.exists(), "child did not reach manifest update"); + std::thread::sleep(Duration::from_millis(100)); + assert!( + child.try_wait().expect("child status").is_none(), + "child update bypassed the held cross-process lock" + ); + + FileExt::unlock(&lock_file).expect("release parent file lock"); + assert!(child.wait().expect("child completion").success()); + let manifest = + read_active_connector_manifest_from(&manifest_path).expect("active connector manifest"); + let selected = manifest.connectors.get("github-pdpp").expect("selection"); + assert_eq!(selected.version, "9.9.9"); + assert_eq!(selected.root_path, "/user/github-pdpp"); + } +} diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 4575b66e..3424a14f 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -2,6 +2,7 @@ pub mod connector; pub mod connector_store; pub mod download; pub mod file_ops; +pub mod pdpp_browser; pub mod pdpp_collection_state; pub mod pdpp_connector; pub mod pdpp_installed_connector; diff --git a/src-tauri/src/commands/pdpp_browser.rs b/src-tauri/src/commands/pdpp_browser.rs new file mode 100644 index 00000000..19e2113f --- /dev/null +++ b/src-tauri/src/commands/pdpp_browser.rs @@ -0,0 +1,697 @@ +//! An exclusive, owner-confined browser lease for PDPP Collection Profiles. +//! +//! This deliberately does not expose the legacy Playwright page API. A PDPP +//! artifact receives a loopback CDP endpoint through its supported environment +//! seam and owns its browser automation protocol. Each lease starts a fresh +//! browser process with +//! the durable profile for exactly one `(connector, connection owner)` pair. +//! Closing a lease removes only its process and lock; an explicit reset removes +//! the authenticated profile. This preserves scheduled collection while +//! preventing any profile or live browser from crossing owners. + +use fs2::FileExt; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::fs::{self, File, OpenOptions}; +#[cfg(unix)] +use std::io; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +const BROWSER_START_TIMEOUT: Duration = Duration::from_secs(10); +const BROWSER_STOP_WAIT: Duration = Duration::from_secs(2); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PdppBrowserInteractionState { + Launching, + Collecting, + WaitingForUser, + Closing, + Closed, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PdppBrowserBinding { + pub backend: &'static str, + pub cdp_http_url: String, + pub lease_id: String, + pub profile_key: String, +} + +pub struct PdppBrowserLease { + binding: PdppBrowserBinding, + profile_dir: PathBuf, + // Keep the OS lock descriptor alive for the full browser lifetime. A + // marker file alone can be deleted by another process and cannot recover + // safely after a crash; an advisory lock is released by the OS when this + // owner dies, while the durable profile remains untouched. + lease_lock: Option, + state: PdppBrowserInteractionState, + child: Option, + termination_failed: bool, +} + +impl PdppBrowserLease { + pub fn launch( + connector_id: &str, + owner_id: &str, + run_id: &str, + resource_dir: Option<&Path>, + ) -> Result { + validate_owner_id(owner_id)?; + let browser = resolve_pdpp_browser_path(resource_dir).ok_or( + "No system, downloaded, or bundled Chromium browser is available for PDPP browser automation", + )?; + let (profile_dir, lease_lock) = + acquire_profile_lease(&profile_root()?, connector_id, owner_id)?; + // A stale port file must never point a new owner lease at an old + // browser. Authentication data remains in the durable profile. + let _ = fs::remove_file(profile_dir.join("DevToolsActivePort")); + let mut command = Command::new(browser); + command + .arg(format!("--user-data-dir={}", profile_dir.display())) + .args([ + "--remote-debugging-address=127.0.0.1", + "--remote-debugging-port=0", + "--no-first-run", + "--no-default-browser-check", + "about:blank", + ]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + command.process_group(0); + } + let child = match command.spawn() { + Ok(child) => child, + Err(error) => { + release_profile_lease(Some(lease_lock)); + return Err(format!("Failed to launch PDPP browser: {error}")); + } + }; + let lease_id = lease_id(connector_id, owner_id, run_id); + let (endpoint, child) = match wait_for_devtools_endpoint(&profile_dir, child) { + Ok(ready) => ready, + Err(failure) => { + if failure.terminated { + release_profile_lease(Some(lease_lock)); + } else { + std::mem::forget(lease_lock); + } + return Err(failure.message); + } + }; + Ok(Self { + binding: PdppBrowserBinding { + backend: "neko", + cdp_http_url: endpoint, + lease_id, + profile_key: profile_key(connector_id, owner_id), + }, + profile_dir, + lease_lock: Some(lease_lock), + state: PdppBrowserInteractionState::Collecting, + child: Some(child), + termination_failed: false, + }) + } + + #[cfg(test)] + fn fixture( + root: &Path, + connector_id: &str, + owner_id: &str, + run_id: &str, + ) -> Result { + validate_owner_id(owner_id)?; + let (profile_dir, lease_lock) = acquire_profile_lease(root, connector_id, owner_id)?; + Ok(Self { + binding: PdppBrowserBinding { + backend: "neko", + cdp_http_url: "http://127.0.0.1:9222".into(), + lease_id: lease_id(connector_id, owner_id, run_id), + profile_key: profile_key(connector_id, owner_id), + }, + profile_dir, + lease_lock: Some(lease_lock), + state: PdppBrowserInteractionState::Launching, + child: None, + termination_failed: false, + }) + } + + pub fn binding(&self) -> &PdppBrowserBinding { + &self.binding + } + + pub fn mark_waiting_for_user(&mut self) { + if self.state == PdppBrowserInteractionState::Collecting { + self.state = PdppBrowserInteractionState::WaitingForUser; + } + } + + #[cfg(test)] + fn state(&self) -> PdppBrowserInteractionState { + self.state.clone() + } + + #[cfg(test)] + fn profile_dir(&self) -> &Path { + &self.profile_dir + } + + pub fn close(&mut self) { + self.close_with_before_release_hook(|_| {}); + } + + fn close_with_before_release_hook(&mut self, before_release: F) + where + F: FnOnce(&Path), + { + self.close_with_terminator(terminate_browser, before_release); + } + + fn close_with_terminator(&mut self, mut terminate: T, before_release: F) + where + T: FnMut(&mut Child) -> bool, + F: FnOnce(&Path), + { + if self.state == PdppBrowserInteractionState::Closed { + return; + } + self.state = PdppBrowserInteractionState::Closing; + if let Some(mut child) = self.child.take() { + if !terminate(&mut child) { + self.child = Some(child); + self.termination_failed = true; + return; + } + } + self.termination_failed = false; + before_release(&self.profile_dir); + release_profile_lease(self.lease_lock.take()); + self.state = PdppBrowserInteractionState::Closed; + } + + /// Removes an authenticated PDPP profile only when no browser lease owns + /// it. Call this from an explicit disconnect/reset action, never from the + /// normal run lifecycle. + pub fn reset_profile(connector_id: &str, owner_id: &str) -> Result<(), String> { + reset_profile_in(&profile_root()?, connector_id, owner_id) + } + + /// A setup marker is insufficient when its owner profile is gone. + pub fn profile_exists(connector_id: &str, owner_id: &str) -> Result { + validate_owner_id(owner_id)?; + Ok(profile_dir(&profile_root()?, connector_id, owner_id).is_dir()) + } +} + +impl Drop for PdppBrowserLease { + fn drop(&mut self) { + if self.termination_failed { + if let Some(lock) = self.lease_lock.take() { + std::mem::forget(lock); + } + return; + } + self.close(); + if self.child.is_some() { + if let Some(lock) = self.lease_lock.take() { + std::mem::forget(lock); + } + } + } +} + +fn validate_owner_id(owner_id: &str) -> Result<(), String> { + if owner_id.is_empty() + || owner_id.len() > 128 + || !owner_id.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '_' | '-' | '.') + }) + { + return Err("PDPP browser binding requires an explicit URL-safe connectionId owner".into()); + } + Ok(()) +} + +fn profile_root() -> Result { + let home = std::env::var("HOME") + .or_else(|_| std::env::var("USERPROFILE")) + .map_err(|_| "Could not determine a home directory for PDPP browser profiles")?; + Ok(PathBuf::from(home) + .join(".dataconnect") + .join("pdpp-browser-leases")) +} + +fn resolve_pdpp_browser_path(resource_dir: Option<&Path>) -> Option { + super::connector::resolve_automation_browser_path(resource_dir) +} + +fn profile_dir(root: &Path, connector_id: &str, owner_id: &str) -> PathBuf { + root.join("profiles") + .join(stable_segment(connector_id)) + .join(stable_segment(owner_id)) +} + +fn lock_path(root: &Path, connector_id: &str, owner_id: &str) -> PathBuf { + root.join("leases") + .join(stable_segment(connector_id)) + .join(format!("{}.lock", stable_segment(owner_id))) +} + +fn acquire_profile_lease( + root: &Path, + connector_id: &str, + owner_id: &str, +) -> Result<(PathBuf, File), String> { + validate_owner_id(owner_id)?; + fs::create_dir_all(root) + .map_err(|error| format!("Failed to create PDPP browser profile root: {error}"))?; + let canonical_root = fs::canonicalize(root) + .map_err(|error| format!("Failed to confine PDPP browser profile root: {error}"))?; + let profile_dir = profile_dir(&canonical_root, connector_id, owner_id); + fs::create_dir_all(&profile_dir) + .map_err(|error| format!("Failed to create PDPP browser profile: {error}"))?; + let canonical_profile = fs::canonicalize(&profile_dir) + .map_err(|error| format!("Failed to confine PDPP browser profile: {error}"))?; + if !canonical_profile.starts_with(&canonical_root) { + return Err("PDPP browser profile escaped its lease root".into()); + } + let lock_path = lock_path(&canonical_root, connector_id, owner_id); + let lock_parent = lock_path + .parent() + .ok_or("PDPP browser lease lock has no parent")?; + fs::create_dir_all(lock_parent) + .map_err(|error| format!("Failed to create PDPP browser lease directory: {error}"))?; + let lock_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(&lock_path) + .map_err(|error| format!("Failed to open PDPP browser lease: {error}"))?; + match lock_file.try_lock_exclusive() { + Ok(()) => Ok((canonical_profile, lock_file)), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => Err( + "PDPP browser profile is already leased by an active run for this connection".into(), + ), + Err(error) => Err(format!("Failed to acquire PDPP browser lease: {error}")), + } +} + +fn release_profile_lease(lock_file: Option) { + if let Some(lock_file) = lock_file { + let _ = FileExt::unlock(&lock_file); + // Do not remove the lock file. Removing an inode while a concurrent + // waiter still holds it permits a second lock file to be created and + // would split ownership between two live leases. + } +} + +fn reset_profile_in(root: &Path, connector_id: &str, owner_id: &str) -> Result<(), String> { + let (profile_dir, lease_lock) = acquire_profile_lease(root, connector_id, owner_id)?; + let removal = fs::remove_dir_all(&profile_dir) + .map_err(|error| format!("Failed to reset PDPP browser profile: {error}")); + release_profile_lease(Some(lease_lock)); + removal +} + +fn stable_segment(value: &str) -> String { + format!("{:x}", Sha256::digest(value.as_bytes())) +} + +fn lease_id(connector_id: &str, owner_id: &str, run_id: &str) -> String { + stable_segment(&format!("{connector_id}:{owner_id}:{run_id}")) +} + +fn profile_key(connector_id: &str, owner_id: &str) -> String { + stable_segment(&format!("{connector_id}:{owner_id}")) +} + +fn wait_for_devtools_endpoint( + profile_dir: &Path, + mut child: Child, +) -> Result<(String, Child), BrowserLaunchFailure> { + let deadline = Instant::now() + BROWSER_START_TIMEOUT; + let active_port = profile_dir.join("DevToolsActivePort"); + while Instant::now() < deadline { + if let Ok(Some(status)) = child.try_wait() { + return Err(BrowserLaunchFailure { + message: format!("PDPP browser exited before becoming ready: {status}"), + terminated: true, + }); + } + if let Ok(contents) = fs::read_to_string(&active_port) { + if let Some(port) = contents + .lines() + .next() + .and_then(|port| port.parse::().ok()) + { + return Ok((format!("http://127.0.0.1:{port}"), child)); + } + } + thread::sleep(Duration::from_millis(25)); + } + Err(BrowserLaunchFailure { + message: "Timed out waiting for PDPP browser CDP endpoint".into(), + terminated: terminate_browser(&mut child), + }) +} + +struct BrowserLaunchFailure { + message: String, + terminated: bool, +} + +fn terminate_browser(child: &mut Child) -> bool { + #[cfg(unix)] + { + let process_group = child.id(); + signal_process_group(process_group, libc::SIGTERM); + let leader_exited = wait_for_child_exit(child, BROWSER_STOP_WAIT); + if leader_exited && wait_for_process_group_exit(process_group, BROWSER_STOP_WAIT) { + return true; + } + signal_process_group(process_group, libc::SIGKILL); + let _ = wait_for_child_exit(child, BROWSER_STOP_WAIT); + return wait_for_process_group_exit(process_group, BROWSER_STOP_WAIT); + } + + #[cfg(not(unix))] + { + #[cfg(windows)] + { + run_windows_taskkill(child.id(), BROWSER_STOP_WAIT); + } + #[cfg(not(windows))] + { + let _ = child.kill(); + } + if wait_for_child_exit(child, BROWSER_STOP_WAIT) { + return true; + } + #[cfg(windows)] + { + run_windows_taskkill(child.id(), BROWSER_STOP_WAIT); + } + #[cfg(not(windows))] + { + let _ = child.kill(); + } + wait_for_child_exit(child, BROWSER_STOP_WAIT) + } +} + +#[cfg(unix)] +fn signal_process_group(process_group: u32, signal: libc::c_int) { + unsafe { + libc::kill(-(process_group as i32), signal); + } +} + +#[cfg(unix)] +fn wait_for_process_group_exit(process_group: u32, timeout: Duration) -> bool { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if !process_group_exists(process_group) { + return true; + } + thread::sleep(Duration::from_millis(20)); + } + !process_group_exists(process_group) +} + +#[cfg(unix)] +fn process_group_exists(process_group: u32) -> bool { + let result = unsafe { libc::kill(-(process_group as i32), 0) }; + if result == 0 { + return true; + } + io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH) +} + +fn wait_for_child_exit(child: &mut Child, timeout: Duration) -> bool { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if child.try_wait().ok().flatten().is_some() { + return true; + } + thread::sleep(Duration::from_millis(20)); + } + false +} + +#[cfg(windows)] +fn run_windows_taskkill(pid: u32, timeout: Duration) { + let (program, args) = windows_taskkill_command(pid); + let Ok(mut taskkill) = Command::new(program) + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + else { + return; + }; + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if taskkill.try_wait().ok().flatten().is_some() { + return; + } + thread::sleep(Duration::from_millis(20)); + } + let _ = taskkill.kill(); + let _ = wait_for_child_exit(&mut taskkill, Duration::from_millis(200)); +} + +fn windows_taskkill_command(pid: u32) -> (&'static str, Vec) { + ( + "taskkill", + vec!["/PID".into(), pid.to_string(), "/T".into(), "/F".into()], + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use fs2::FileExt; + #[cfg(unix)] + use std::os::unix::process::CommandExt; + use std::process::Command; + + const LOCK_HOLDER_ROOT: &str = "PDPP_BROWSER_LOCK_HOLDER_ROOT"; + const LOCK_EXPECT_BLOCKED_ROOT: &str = "PDPP_BROWSER_LOCK_EXPECT_BLOCKED_ROOT"; + + struct ReapedTestChild(Child); + + impl Drop for ReapedTestChild { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + #[test] + fn holds_a_live_profile_lease_until_killed() { + let Ok(root) = std::env::var(LOCK_HOLDER_ROOT) else { + return; + }; + let root = PathBuf::from(root); + let _lease = PdppBrowserLease::fixture(&root, "chatgpt-pdpp", "alice", "holder") + .expect("lock holder must acquire its profile lease"); + fs::write(root.join("lock-holder-ready"), "ready") + .expect("lock holder must signal readiness"); + loop { + thread::sleep(Duration::from_secs(1)); + } + } + + #[test] + fn expects_profile_lease_to_be_blocked() { + let Ok(root) = std::env::var(LOCK_EXPECT_BLOCKED_ROOT) else { + return; + }; + let blocked = + PdppBrowserLease::fixture(Path::new(&root), "chatgpt-pdpp", "alice", "blocked-check"); + assert!(matches!(blocked, Err(error) if error.contains("already leased"))); + } + + #[test] + fn fixture_leases_persist_per_owner_and_clean_only_ephemeral_state() { + let root = tempfile::tempdir().unwrap(); + let mut first = + PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-1").unwrap(); + let profile = first.profile_dir().to_owned(); + assert!(first.profile_dir().starts_with(root.path())); + first.state = PdppBrowserInteractionState::Collecting; + first.mark_waiting_for_user(); + assert_eq!(first.state(), PdppBrowserInteractionState::WaitingForUser); + first.close(); + assert_eq!(first.state(), PdppBrowserInteractionState::Closed); + assert!(profile.exists()); + let second = + PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-2").unwrap(); + assert_eq!(second.profile_dir(), profile); + assert_ne!(second.binding().lease_id, ""); + } + + #[test] + fn fixture_leases_are_owner_confined_and_exclusive() { + let root = tempfile::tempdir().unwrap(); + let first = + PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-1").unwrap(); + let concurrent = PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-2"); + assert!(matches!(concurrent, Err(error) if error.contains("already leased"))); + let second = + PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "bob", "run-1").unwrap(); + assert_ne!(first.profile_dir(), second.profile_dir()); + } + + #[test] + fn live_process_lease_cannot_be_stolen_and_recovers_after_owner_is_killed() { + let root = tempfile::tempdir().unwrap(); + let mut holder = ReapedTestChild( + Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "commands::pdpp_browser::tests::holds_a_live_profile_lease_until_killed", + "--nocapture", + ]) + .env(LOCK_HOLDER_ROOT, root.path()) + .spawn() + .expect("spawn lock holder test process"), + ); + let ready = root.path().join("lock-holder-ready"); + let deadline = Instant::now() + Duration::from_secs(5); + while !ready.exists() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + assert!(ready.exists(), "lock holder did not become ready"); + + let blocked = PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-2"); + assert!(matches!(blocked, Err(error) if error.contains("already leased"))); + + holder.0.kill().expect("kill lock holder"); + holder.0.wait().expect("reap killed lock holder"); + + let recovered = PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-3"); + assert!( + recovered.is_ok(), + "OS lock must be released when its owner dies" + ); + } + + #[test] + fn reset_deletes_only_an_idle_owner_profile() { + let root = tempfile::tempdir().unwrap(); + let lease = + PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-1").unwrap(); + let profile = lease.profile_dir().to_owned(); + assert!(reset_profile_in(root.path(), "chatgpt-pdpp", "alice").is_err()); + drop(lease); + reset_profile_in(root.path(), "chatgpt-pdpp", "alice").unwrap(); + assert!(!profile.exists()); + } + + #[test] + fn refuses_implicit_or_unsafe_browser_owners() { + assert!(validate_owner_id("").is_err()); + assert!(validate_owner_id("../other-owner").is_err()); + assert!(validate_owner_id("account-one").is_ok()); + } + + #[test] + fn windows_tree_cleanup_targets_only_the_browser_process_tree() { + let (program, args) = windows_taskkill_command(4242); + + assert_eq!(program, "taskkill"); + assert_eq!(args, ["/PID", "4242", "/T", "/F"]); + } + + #[cfg(unix)] + #[test] + fn close_releases_profile_lock_after_bounded_browser_termination() { + let root = tempfile::tempdir().unwrap(); + let mut lease = + PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-1").unwrap(); + lease.state = PdppBrowserInteractionState::Collecting; + let mut command = Command::new("sh"); + command.args(["-c", "sleep 30"]); + command.process_group(0); + lease.child = Some(command.spawn().expect("spawn test browser process")); + let canonical_root = fs::canonicalize(root.path()).unwrap(); + let lock_path = lock_path(&canonical_root, "chatgpt-pdpp", "alice"); + let mut observed_still_locked = false; + + lease.close_with_before_release_hook(|_| { + let lock = OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .unwrap(); + let error = lock.try_lock_exclusive().unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::WouldBlock); + observed_still_locked = true; + }); + + assert!(observed_still_locked); + let lock = OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .unwrap(); + lock.try_lock_exclusive() + .expect("profile lock releases only after close completes"); + } + + #[cfg(unix)] + #[test] + fn drop_preserves_profile_lock_when_browser_tree_is_not_reaped() { + let root = tempfile::tempdir().unwrap(); + let mut lease = + PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-1").unwrap(); + lease.state = PdppBrowserInteractionState::Collecting; + let mut command = Command::new("sh"); + command.args(["-c", "sleep 30"]); + command.process_group(0); + let child = command.spawn().expect("spawn test browser process"); + let process_group = child.id(); + lease.child = Some(child); + + lease.close_with_terminator(|_| false, |_| panic!("lock must not release")); + drop(lease); + + let blocked = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "commands::pdpp_browser::tests::expects_profile_lease_to_be_blocked", + "--nocapture", + ]) + .env(LOCK_EXPECT_BLOCKED_ROOT, root.path()) + .status() + .expect("spawn blocked lock oracle"); + assert!(blocked.success()); + crate::commands::server::kill_process_group(process_group, libc::SIGKILL); + } + + #[cfg(unix)] + #[test] + fn terminate_browser_waits_for_descendant_process_group_exit() { + let mut command = Command::new("sh"); + command.args(["-c", "trap '' TERM; (trap '' TERM; sleep 30) & exit 0"]); + command.process_group(0); + let mut child = command.spawn().expect("spawn test browser process tree"); + let process_group = child.id(); + + assert!(terminate_browser(&mut child)); + assert!(!process_group_exists(process_group)); + } +} diff --git a/src-tauri/src/commands/pdpp_collection_state.rs b/src-tauri/src/commands/pdpp_collection_state.rs index b89b35cc..d67c65bd 100644 --- a/src-tauri/src/commands/pdpp_collection_state.rs +++ b/src-tauri/src/commands/pdpp_collection_state.rs @@ -9,7 +9,7 @@ use super::pdpp_connector::{PdppRecord, PdppRunStatus}; use fs2::FileExt; use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::fs::{self, File}; use std::io::Write; use std::path::{Path, PathBuf}; @@ -32,6 +32,10 @@ pub struct PdppCollectionConnectionState { struct PdppCollectionStateFile { version: u8, connectors: HashMap>, + /// A non-secret setup acknowledgement. Browser authentication remains in + /// the owner-confined profile, never in collection state. + #[serde(default)] + setup_complete_connections: HashMap>, } pub fn collection_state_path() -> Result { @@ -63,6 +67,69 @@ pub fn load_connection_state_at( .unwrap_or_default()) } +pub fn is_connection_setup_complete( + connector_id: &str, + connection_id: &str, +) -> Result { + is_connection_setup_complete_at(&collection_state_path()?, connector_id, connection_id) +} + +fn is_connection_setup_complete_at( + path: &Path, + connector_id: &str, + connection_id: &str, +) -> Result { + let _guard = COLLECTION_STATE_LOCK + .lock() + .map_err(|_| "PDPP collection state lock is unavailable")?; + Ok(read_state_file(path)? + .setup_complete_connections + .get(connector_id) + .is_some_and(|connections| connections.contains(connection_id))) +} + +/// Stores only that setup occurred, allowing a later scheduled run to reuse +/// the durable browser profile without another credential prompt. +pub fn mark_connection_setup_complete( + connector_id: &str, + connection_id: &str, +) -> Result<(), String> { + update_connection_setup_complete_at(&collection_state_path()?, connector_id, connection_id, true) +} + +pub fn clear_connection_setup_complete( + connector_id: &str, + connection_id: &str, +) -> Result<(), String> { + update_connection_setup_complete_at(&collection_state_path()?, connector_id, connection_id, false) +} + +fn update_connection_setup_complete_at( + path: &Path, + connector_id: &str, + connection_id: &str, + complete: bool, +) -> Result<(), String> { + let _guard = COLLECTION_STATE_LOCK + .lock() + .map_err(|_| "PDPP collection state lock is unavailable")?; + let _file_lock = lock_state_file(path)?; + let mut file = read_state_file(path)?; + if complete { + file.setup_complete_connections + .entry(connector_id.to_owned()) + .or_default() + .insert(connection_id.to_owned()); + } else if let Some(connections) = file.setup_complete_connections.get_mut(connector_id) { + connections.remove(connection_id); + if connections.is_empty() { + file.setup_complete_connections.remove(connector_id); + } + } + file.version = 1; + write_state_file_atomically(path, &file) +} + /// Commit a terminal run only when the validated protocol result succeeded. /// Failed, cancelled, and timed-out runs return without touching durable state. pub fn commit_terminal_run( @@ -339,6 +406,23 @@ mod tests { ); } + #[test] + fn reset_clears_the_owner_confined_nonsecret_setup_marker() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("state.json"); + update_connection_setup_complete_at(&path, "chatgpt-pdpp", "owner-a", true).unwrap(); + + assert!(is_connection_setup_complete_at(&path, "chatgpt-pdpp", "owner-a").unwrap()); + assert!(!is_connection_setup_complete_at(&path, "chatgpt-pdpp", "owner-b").unwrap()); + let stored = fs::read_to_string(&path).unwrap(); + assert!(stored.contains("setupCompleteConnections")); + assert!(!stored.contains("password")); + assert!(!stored.contains("username")); + + update_connection_setup_complete_at(&path, "chatgpt-pdpp", "owner-a", false).unwrap(); + assert!(!is_connection_setup_complete_at(&path, "chatgpt-pdpp", "owner-a").unwrap()); + } + #[test] fn successful_checkpoint_is_durable_with_its_records_and_snapshot() { let temp = tempfile::tempdir().unwrap(); diff --git a/src-tauri/src/commands/pdpp_connector.rs b/src-tauri/src/commands/pdpp_connector.rs index 02e49cca..da4bdb7e 100644 --- a/src-tauri/src/commands/pdpp_connector.rs +++ b/src-tauri/src/commands/pdpp_connector.rs @@ -13,7 +13,7 @@ use std::path::PathBuf; use std::process::{Command, Stdio}; use std::sync::{ atomic::{AtomicBool, Ordering}, - mpsc, Arc, + mpsc, Arc, Mutex, }; use std::thread; use std::time::{Duration, Instant}; @@ -35,7 +35,6 @@ pub struct PdppStart { pub collection_mode: String, pub scope: Value, pub state: Option, - pub bindings: Value, } impl PdppStart { @@ -44,7 +43,6 @@ impl PdppStart { collection_mode: impl Into, scope: Value, state: Option, - bindings: Value, ) -> Result { let start = Self { message_type: "START", @@ -52,7 +50,6 @@ impl PdppStart { collection_mode: collection_mode.into(), scope, state, - bindings, }; start.validate()?; Ok(start) @@ -68,9 +65,6 @@ impl PdppStart { ) { return Err("PDPP START collection_mode must be full_refresh or incremental".into()); } - if !self.bindings.is_object() { - return Err("PDPP START bindings must be an object".into()); - } Ok(()) } } @@ -84,6 +78,112 @@ pub struct PdppScopeValidators { } pub type PdppEventSink = Arc Result<(), String> + Send + Sync>; +/// Delivers one connector interaction together with the only responder that +/// can answer it. The responder is bound to this kernel run and interaction +/// generation, so a delayed callback cannot answer a later interaction. +pub type PdppInteractionSink = Arc< + dyn for<'interaction> Fn( + &'interaction PdppInteraction, + PdppInteractionResponder, + ) -> Result<(), String> + + Send + + Sync, +>; +/// Notifies a host that a pending interaction can no longer accept a response. +/// Hosts use this to remove any owner-visible responder before a late command +/// can report success after cancellation, timeout, or protocol failure. +pub type PdppInteractionClosedSink = Arc; + +const DEFAULT_INTERACTION_TIMEOUT: Duration = Duration::from_secs(30 * 60); +const MIN_INTERACTION_TIMEOUT: Duration = Duration::from_secs(60); +const MAX_INTERACTION_TIMEOUT: Duration = Duration::from_secs(60 * 60); + +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PdppInteractionResponseStatus { + Success, + Cancelled, + Timeout, +} + +#[derive(Clone, Serialize)] +pub struct PdppInteractionResponse { + #[serde(rename = "type")] + message_type: &'static str, + pub request_id: String, + pub status: PdppInteractionResponseStatus, + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option, +} + +impl PdppInteractionResponse { + fn new( + request_id: impl Into, + status: PdppInteractionResponseStatus, + data: Option, + ) -> Result { + let response = Self { + message_type: "INTERACTION_RESPONSE", + request_id: request_id.into(), + status, + data, + }; + response.validate()?; + Ok(response) + } + + fn validate(&self) -> Result<(), String> { + if self.request_id.is_empty() { + return Err("PDPP INTERACTION_RESPONSE requires a non-empty request_id".into()); + } + if self.data.as_ref().is_some_and(|data| !data.is_object()) { + return Err("PDPP INTERACTION_RESPONSE data must be an object".into()); + } + if !matches!(self.status, PdppInteractionResponseStatus::Success) && self.data.is_some() { + return Err("PDPP INTERACTION_RESPONSE data is only allowed for success".into()); + } + Ok(()) + } +} + +/// A one-shot, run-bound response channel. The JSONL envelope deliberately +/// contains no run_id (the Collection Profile contract keys it by +/// request_id); `run_id()` is exposed only for host-side ownership checks. +#[derive(Clone)] +pub struct PdppInteractionResponder { + run_id: String, + request_id: String, + generation: u64, + sent: Arc, + sender: mpsc::Sender, +} + +impl PdppInteractionResponder { + pub fn run_id(&self) -> &str { + &self.run_id + } + + pub fn request_id(&self) -> &str { + &self.request_id + } + + pub fn respond( + &self, + status: PdppInteractionResponseStatus, + data: Option, + ) -> Result<(), String> { + let response = PdppInteractionResponse::new(self.request_id.clone(), status, data)?; + if self.sent.swap(true, Ordering::SeqCst) { + return Err("PDPP INTERACTION responder was already used".into()); + } + self.sender + .send(PendingInteractionResponse { + generation: self.generation, + response, + }) + .map_err(|_| "PDPP INTERACTION run is no longer active".to_string()) + } +} #[derive(Clone)] pub struct PdppRunOptions { @@ -95,6 +195,11 @@ pub struct PdppRunOptions { pub max_retained_records: usize, pub max_retained_events: usize, pub on_event: Option, + /// Browser-capable hosts can surface an interaction to the user while the + /// connector remains alive. Network-only hosts deliberately leave this + /// unset and retain the prior fail-closed behavior. + pub on_interaction: Option, + pub on_interaction_closed: Option, } impl Default for PdppRunOptions { @@ -108,6 +213,8 @@ impl Default for PdppRunOptions { max_retained_records: 0, max_retained_events: 32, on_event: None, + on_interaction: None, + on_interaction_closed: None, } } } @@ -278,6 +385,69 @@ enum Line { End, } +struct PendingInteractionResponse { + generation: u64, + response: PdppInteractionResponse, +} + +struct PendingInteraction { + generation: u64, + request_id: String, + deadline: Instant, +} + +fn interaction_deadline_elapsed(pending: &PendingInteraction, now: Instant) -> bool { + now >= pending.deadline +} + +fn close_pending_interaction( + options: &PdppRunOptions, + run_id: &str, + pending: PendingInteraction, +) -> PendingInteraction { + if let Some(on_interaction_closed) = &options.on_interaction_closed { + on_interaction_closed(run_id, &pending.request_id); + } + pending +} + +fn interaction_timeout(timeout_seconds: Option) -> Result { + let timeout = timeout_seconds + .map(Duration::from_secs) + .unwrap_or(DEFAULT_INTERACTION_TIMEOUT); + if !(MIN_INTERACTION_TIMEOUT..=MAX_INTERACTION_TIMEOUT).contains(&timeout) { + return Err("PDPP INTERACTION timeout_seconds must be between 60 and 3600".into()); + } + Ok(timeout) +} + +fn write_interaction_response( + writer: &mut W, + response: &PdppInteractionResponse, +) -> Result<(), String> { + response.validate()?; + writeln!( + writer, + "{}", + serde_json::to_string(response) + .map_err(|error| format!("Failed to serialize PDPP INTERACTION_RESPONSE: {error}"))? + ) + .map_err(|error| format!("Failed to send PDPP INTERACTION_RESPONSE: {error}"))?; + writer + .flush() + .map_err(|error| format!("Failed to flush PDPP INTERACTION_RESPONSE: {error}")) +} + +fn send_interaction_response( + stdin: &mut Option, + response: &PdppInteractionResponse, +) -> Result<(), String> { + let stdin = stdin + .as_mut() + .ok_or("PDPP connector stdin closed before INTERACTION_RESPONSE")?; + write_interaction_response(stdin, response) +} + pub fn supervise_pdpp_connector( command: &PdppConnectorCommand, start: &PdppStart, @@ -312,20 +482,25 @@ pub fn supervise_pdpp_connector( let mut child = process .spawn() .map_err(|e| format!("Failed to spawn PDPP connector: {e}"))?; - let mut stdin = child - .stdin - .take() - .ok_or("PDPP connector stdin unavailable")?; + let mut stdin = Some( + child + .stdin + .take() + .ok_or("PDPP connector stdin unavailable")?, + ); writeln!( - stdin, + stdin + .as_mut() + .expect("PDPP connector stdin was initialized"), "{}", serde_json::to_string(start).map_err(|e| e.to_string())? ) .map_err(|e| format!("Failed to send PDPP START: {e}"))?; stdin + .as_mut() + .expect("PDPP connector stdin was initialized") .flush() .map_err(|e| format!("Failed to flush PDPP START: {e}"))?; - drop(stdin); let stdout = child .stdout .take() @@ -335,10 +510,14 @@ pub fn supervise_pdpp_connector( .take() .ok_or("PDPP connector stderr unavailable")?; let (tx, rx) = mpsc::channel(); + let (interaction_tx, interaction_rx) = mpsc::channel::(); let stdout_thread = spawn_reader(stdout, options.max_stdout_line_bytes, true, tx.clone()); let stderr_thread = spawn_reader(stderr, options.max_stderr_bytes, false, tx); - let started = Instant::now(); + let mut normal_started = Instant::now(); + let mut normal_elapsed = Duration::ZERO; + let mut pending_interaction: Option = None; + let mut interaction_generation = 0u64; let mut stdout_closed = false; let mut stderr_closed = false; let mut exit = None; @@ -361,15 +540,74 @@ pub fn supervise_pdpp_connector( .map_err(|e| format!("Failed to poll PDPP connector: {e}"))?; } if termination.is_none() && options.control.is_cancelled() { + if let Some(pending) = pending_interaction.take() { + let pending = close_pending_interaction(&options, &start.run_id, pending); + let response = PdppInteractionResponse::new( + pending.request_id, + PdppInteractionResponseStatus::Cancelled, + None, + )?; + let _ = send_interaction_response(&mut stdin, &response); + } termination = Some(PdppRunStatus::Cancelled); set_failure(&mut failure, "PDPP connector was cancelled by the runtime"); terminate_child(&mut child); } - if termination.is_none() && options.timeout.is_some_and(|t| started.elapsed() >= t) { + if termination.is_none() + && pending_interaction.is_none() + && options + .timeout + .is_some_and(|timeout| normal_elapsed + normal_started.elapsed() >= timeout) + { termination = Some(PdppRunStatus::TimedOut); set_failure(&mut failure, "PDPP connector exceeded its runtime timeout"); terminate_child(&mut child); } + while let Ok(pending_response) = interaction_rx.try_recv() { + let matches_pending = pending_interaction.as_ref().is_some_and(|pending| { + pending.generation == pending_response.generation + && pending.request_id == pending_response.response.request_id + }); + if !matches_pending { + // A responder from an earlier interaction or run cannot affect + // the active connector state. It is intentionally discarded. + continue; + } + if let Err(error) = send_interaction_response(&mut stdin, &pending_response.response) { + if let Some(pending) = pending_interaction.take() { + close_pending_interaction(&options, &start.run_id, pending); + } + set_failure(&mut failure, error); + terminate_child(&mut child); + } else { + let pending = pending_interaction + .take() + .expect("matching response requires a pending interaction"); + close_pending_interaction(&options, &start.run_id, pending); + normal_started = Instant::now(); + } + } + if termination.is_none() + && pending_interaction + .as_ref() + .is_some_and(|pending| interaction_deadline_elapsed(pending, Instant::now())) + { + let pending = pending_interaction + .take() + .expect("pending interaction was checked above"); + let pending = close_pending_interaction(&options, &start.run_id, pending); + let response = PdppInteractionResponse::new( + pending.request_id, + PdppInteractionResponseStatus::Timeout, + None, + )?; + if let Err(error) = send_interaction_response(&mut stdin, &response) { + set_failure(&mut failure, error); + terminate_child(&mut child); + } else { + normal_started = Instant::now(); + } + } match rx.recv_timeout(Duration::from_millis(10)) { Ok(ReaderEvent::Stdout(Ok(line))) => { if done.is_some() { @@ -378,6 +616,17 @@ pub fn supervise_pdpp_connector( continue; } match parse_message(&line, &scope) { + Ok(_) if pending_interaction.is_some() => { + let pending = pending_interaction + .take() + .expect("pending interaction was checked above"); + close_pending_interaction(&options, &start.run_id, pending); + set_failure( + &mut failure, + "PDPP connector emitted output while waiting for INTERACTION_RESPONSE", + ); + terminate_child(&mut child); + } Ok(ConnectorMessage::Record(record)) => { record_count += 1; event_counts.records += 1; @@ -461,15 +710,58 @@ pub fn supervise_pdpp_connector( event_counts.interactions += 1; retain_event( &options, - PdppEvent::Interaction(interaction), + PdppEvent::Interaction(interaction.clone()), &mut events, &mut events_truncated, &mut failure, ); - set_failure(&mut failure, "PDPP INTERACTION requires an INTERACTION_RESPONSE API, which this foundation kernel does not provide"); - terminate_child(&mut child); + if let Some(on_interaction) = &options.on_interaction { + if pending_interaction.is_some() { + set_failure( + &mut failure, + "PDPP connector emitted INTERACTION while already waiting for a response", + ); + terminate_child(&mut child); + continue; + } + let timeout = match interaction_timeout(interaction.timeout_seconds) { + Ok(timeout) => timeout, + Err(error) => { + set_failure(&mut failure, error); + terminate_child(&mut child); + continue; + } + }; + normal_elapsed += normal_started.elapsed(); + interaction_generation += 1; + let responder = PdppInteractionResponder { + run_id: start.run_id.clone(), + request_id: interaction.request_id.clone(), + generation: interaction_generation, + sent: Arc::new(AtomicBool::new(false)), + sender: interaction_tx.clone(), + }; + pending_interaction = Some(PendingInteraction { + generation: interaction_generation, + request_id: interaction.request_id.clone(), + deadline: Instant::now() + timeout, + }); + if let Err(error) = on_interaction(&interaction, responder) { + set_failure(&mut failure, error); + terminate_child(&mut child); + } + } else { + set_failure(&mut failure, "PDPP INTERACTION requires an INTERACTION_RESPONSE API, which this foundation kernel does not provide"); + terminate_child(&mut child); + } + } + Ok(ConnectorMessage::Done(message)) => { + done = Some(message); + // A connector that uses readline keeps its event loop + // alive until its input closes. No further runtime + // message is valid after DONE, so release the writer. + stdin.take(); } - Ok(ConnectorMessage::Done(message)) => done = Some(message), Err(error) => { set_failure(&mut failure, error); terminate_child(&mut child); @@ -682,9 +974,22 @@ fn parse_message(line: &str, scope: &ScopePolicy) -> Result serde_json::from_value(value) - .map(ConnectorMessage::Interaction) - .map_err(|e| format!("Invalid PDPP INTERACTION: {e}")), + "INTERACTION" => { + let interaction: PdppInteraction = serde_json::from_value(value) + .map_err(|e| format!("Invalid PDPP INTERACTION: {e}"))?; + if interaction.request_id.is_empty() + || interaction.kind.is_empty() + || interaction.message.is_empty() + || interaction + .schema + .as_ref() + .is_some_and(|schema| !schema.is_object()) + { + return Err("Invalid PDPP INTERACTION envelope".into()); + } + interaction_timeout(interaction.timeout_seconds)?; + Ok(ConnectorMessage::Interaction(interaction)) + } _ => Err(format!("Unsupported PDPP connector message type: {ty}")), } } @@ -951,14 +1256,7 @@ mod tests { use super::*; use serde_json::json; fn start(scope: Value) -> PdppStart { - PdppStart::new( - "run_test", - "incremental", - scope, - None, - json!({"network": {}}), - ) - .unwrap() + PdppStart::new("run_test", "incremental", scope, None).unwrap() } fn scoped() -> PdppStart { start( @@ -969,6 +1267,15 @@ mod tests { PdppConnectorCommand { program: "node".into(), args: vec![ + "-e".into(), + r#" +const { spawn } = require('node:child_process'); +const [fixturePath, mode] = process.argv.slice(1); +const child = spawn(process.execPath, [fixturePath, mode], { stdio: ['pipe', 'inherit', 'inherit'] }); +process.stdin.once('data', chunk => child.stdin.end(chunk)); +child.on('exit', code => process.exit(code ?? 1)); +"# + .into(), format!( "{}/tests/fixtures/pdpp-connector-fixture.mjs", env!("CARGO_MANIFEST_DIR") @@ -980,6 +1287,102 @@ mod tests { clear_env: false, } } + fn blocking_interaction_fixture(two_interactions: bool) -> PdppConnectorCommand { + let second = if two_interactions { + r#" + if (phase === 0) { + phase = 1; + emit({ type: 'INTERACTION', request_id: 'request-2', kind: 'manual_action', message: 'Continue', timeout_seconds: 60 }); + return; + } +"# + } else { + "" + }; + PdppConnectorCommand { + program: "node".into(), + args: vec![ + "-e".into(), + format!( + r#" +const readline = require('node:readline'); +const emit = message => process.stdout.write(`${{JSON.stringify(message)}}\n`); +let phase = 0; +readline.createInterface({{ input: process.stdin }}).on('line', line => {{ + const message = JSON.parse(line); + if (message.type === 'START') {{ + emit({{ type: 'INTERACTION', request_id: 'request-1', kind: 'manual_action', message: 'Sign in', timeout_seconds: 60 }}); + return; + }} + if (message.type !== 'INTERACTION_RESPONSE') process.exit(70); + const expected = phase === 0 ? 'request-1' : 'request-2'; + if (message.request_id !== expected) process.exit(71); + {second} + emit({{ type: 'RECORD', stream: 'items', key: 'item-1', data: {{ id: 'item-1', source_updated_at: '2026-07-30T00:00:00Z' }}, emitted_at: '2026-07-30T00:00:00Z' }}); + emit({{ type: 'DONE', status: 'succeeded', records_emitted: 1 }}); + setImmediate(() => process.exit(0)); +}}); +"# + ), + ], + cwd: None, + env: HashMap::new(), + clear_env: false, + } + } + + fn delayed_after_interaction_fixture() -> PdppConnectorCommand { + PdppConnectorCommand { + program: "node".into(), + args: vec![ + "-e".into(), + r#" +const readline = require('node:readline'); +const emit = message => process.stdout.write(`${JSON.stringify(message)}\n`); +readline.createInterface({ input: process.stdin }).on('line', line => { + const message = JSON.parse(line); + if (message.type === 'START') { + emit({ type: 'INTERACTION', request_id: 'request-1', kind: 'manual_action', message: 'Sign in', timeout_seconds: 60 }); + return; + } + if (message.type !== 'INTERACTION_RESPONSE' || message.request_id !== 'request-1') process.exit(71); + setTimeout(() => { + emit({ type: 'RECORD', stream: 'items', key: 'item-1', data: { id: 'item-1', source_updated_at: '2026-07-30T00:00:00Z' }, emitted_at: '2026-07-30T00:00:00Z' }); + emit({ type: 'DONE', status: 'succeeded', records_emitted: 1 }); + process.exit(0); + }, 60); +}); +"# + .into(), + ], + cwd: None, + env: HashMap::new(), + clear_env: false, + } + } + + fn output_while_waiting_fixture(output: &str) -> PdppConnectorCommand { + PdppConnectorCommand { + program: "node".into(), + args: vec![ + "-e".into(), + format!( + r#" +const readline = require('node:readline'); +const emit = message => process.stdout.write(`${{JSON.stringify(message)}}\n`); +readline.createInterface({{ input: process.stdin }}).on('line', line => {{ + if (JSON.parse(line).type !== 'START') process.exit(70); + emit({{ type: 'INTERACTION', request_id: 'request-1', kind: 'manual_action', message: 'Sign in', timeout_seconds: 60 }}); + emit({output}); +}}); +"# + ), + ], + cwd: None, + env: HashMap::new(), + clear_env: false, + } + } fn options() -> PdppRunOptions { let mut options = PdppRunOptions { max_retained_records: 4, @@ -995,6 +1398,26 @@ mod tests { ); options } + + fn supervise_with_test_deadline( + command: PdppConnectorCommand, + start: PdppStart, + options: PdppRunOptions, + ) -> PdppRunResult { + let control = options.control.clone(); + let (sender, receiver) = mpsc::sync_channel(1); + thread::spawn(move || { + let _ = sender.send(supervise_pdpp_connector(&command, &start, &options)); + }); + match receiver.recv_timeout(Duration::from_secs(2)) { + Ok(result) => result.expect("interaction fixture supervision should succeed"), + Err(error) => { + control.cancel(); + let _ = receiver.recv_timeout(Duration::from_secs(1)); + panic!("interaction fixture exceeded the 2-second test deadline: {error}"); + } + } + } #[test] fn accepts_scoped_success_and_retains_bounded_output() { let result = supervise_pdpp_connector(&fixture("success"), &scoped(), &options()).unwrap(); @@ -1058,6 +1481,184 @@ mod tests { .contains("INTERACTION_RESPONSE")); } #[test] + fn browser_interaction_hook_keeps_the_protocol_alive_until_done() { + let observed = Arc::new(AtomicBool::new(false)); + let observed_by_hook = observed.clone(); + let result = supervise_pdpp_connector( + &blocking_interaction_fixture(false), + &scoped(), + &PdppRunOptions { + on_interaction: Some(Arc::new(move |interaction, responder| { + assert_eq!(interaction.request_id, "request-1"); + assert_eq!(responder.run_id(), "run_test"); + observed_by_hook.store(true, Ordering::SeqCst); + responder.respond(PdppInteractionResponseStatus::Success, None) + })), + ..options() + }, + ) + .unwrap(); + assert!(observed.load(Ordering::SeqCst)); + assert_eq!(result.status, PdppRunStatus::Succeeded); + } + + #[test] + fn stale_interaction_responder_cannot_answer_the_next_request() { + let first_responder = Arc::new(Mutex::new(None)); + let first_responder_for_hook = first_responder.clone(); + let result = supervise_with_test_deadline( + blocking_interaction_fixture(true), + scoped(), + PdppRunOptions { + on_interaction: Some(Arc::new(move |interaction, responder| { + if interaction.request_id == "request-1" { + *first_responder_for_hook.lock().unwrap() = Some(responder.clone()); + responder.respond(PdppInteractionResponseStatus::Success, None) + } else { + assert!(first_responder_for_hook + .lock() + .unwrap() + .as_ref() + .unwrap() + .respond(PdppInteractionResponseStatus::Cancelled, None) + .is_err()); + responder.respond(PdppInteractionResponseStatus::Success, None) + } + })), + ..options() + }, + ); + assert_eq!( + result.status, + PdppRunStatus::Succeeded, + "{:?}", + result.failure + ); + } + + #[test] + fn rejects_every_connector_output_while_waiting_for_an_interaction_response() { + let outputs = [ + r#"{ type: 'RECORD', stream: 'items', key: 'item-1', data: { id: 'item-1', source_updated_at: '2026-07-30T00:00:00Z' }, emitted_at: '2026-07-30T00:00:00Z' }"#, + r#"{ type: 'STATE', stream: 'items', cursor: { cursor: 'next' } }"#, + r#"{ type: 'PROGRESS', stream: 'items', message: 'working' }"#, + r#"{ type: 'SKIP_RESULT', stream: 'items', reason: 'rate_limited' }"#, + r#"{ type: 'DETAIL_COVERAGE', stream: 'items', state_stream: 'items', required_keys: [], hydrated_keys: [], reference_only: true }"#, + r#"{ type: 'DETAIL_GAP', stream: 'items', record_key: 'item-1', detail_locator: { kind: 'api' }, retryable: true, status: 'pending', reference_only: true }"#, + r#"{ type: 'DETAIL_GAP_RECOVERED', stream: 'items', gap_id: 'gap-1', reference_only: true }"#, + r#"{ type: 'DONE', status: 'succeeded', records_emitted: 0 }"#, + ]; + for output in outputs { + let result = supervise_pdpp_connector( + &output_while_waiting_fixture(output), + &scoped(), + &PdppRunOptions { + on_interaction: Some(Arc::new(|_, _| Ok(()))), + ..options() + }, + ) + .unwrap(); + assert_eq!(result.status, PdppRunStatus::Failed, "{output}"); + assert!(result + .failure + .as_deref() + .is_some_and(|failure| failure.contains("waiting for INTERACTION_RESPONSE"))); + } + } + + #[test] + fn interaction_timeout_contract_is_bounded_and_serializes_the_exact_wire_envelope() { + assert_eq!( + interaction_timeout(None).unwrap(), + DEFAULT_INTERACTION_TIMEOUT + ); + assert_eq!( + interaction_timeout(Some(60)).unwrap(), + MIN_INTERACTION_TIMEOUT + ); + assert!(interaction_timeout(Some(59)).is_err()); + assert!(interaction_timeout(Some(3601)).is_err()); + + let response = + PdppInteractionResponse::new("request-1", PdppInteractionResponseStatus::Timeout, None) + .unwrap(); + let mut wire = Vec::new(); + write_interaction_response(&mut wire, &response).unwrap(); + assert_eq!( + String::from_utf8(wire).unwrap(), + "{\"type\":\"INTERACTION_RESPONSE\",\"request_id\":\"request-1\",\"status\":\"timeout\"}\n" + ); + + let now = Instant::now(); + let expired = PendingInteraction { + generation: 1, + request_id: "request-1".into(), + deadline: now - Duration::from_millis(1), + }; + let active = PendingInteraction { + generation: 1, + request_id: "request-1".into(), + deadline: now + Duration::from_millis(1), + }; + assert!(interaction_deadline_elapsed(&expired, now)); + assert!(!interaction_deadline_elapsed(&active, now)); + } + + #[test] + fn interaction_pauses_the_normal_timeout_then_response_resumes_it() { + let result = supervise_pdpp_connector( + &delayed_after_interaction_fixture(), + &scoped(), + &PdppRunOptions { + timeout: Some(Duration::from_millis(25)), + on_interaction: Some(Arc::new(|_, responder| { + thread::spawn(move || { + thread::sleep(Duration::from_millis(50)); + responder + .respond(PdppInteractionResponseStatus::Success, None) + .unwrap(); + }); + Ok(()) + })), + ..options() + }, + ) + .unwrap(); + + // The 50ms owner interaction exceeds the normal 25ms policy without + // timing out; the connector then exceeds that policy after response. + assert_eq!(result.status, PdppRunStatus::TimedOut); + } + + #[test] + fn cancellation_stops_a_connector_while_its_interaction_is_pending() { + let control = PdppRunControl::default(); + let cancellation = control.clone(); + let interaction_seen = Arc::new(AtomicBool::new(false)); + let interaction_seen_by_hook = interaction_seen.clone(); + let cancellation_worker = thread::spawn(move || { + while !interaction_seen.load(Ordering::SeqCst) { + thread::yield_now(); + } + cancellation.cancel(); + }); + let result = supervise_pdpp_connector( + &blocking_interaction_fixture(false), + &scoped(), + &PdppRunOptions { + control, + on_interaction: Some(Arc::new(move |_, _| { + interaction_seen_by_hook.store(true, Ordering::SeqCst); + Ok(()) + })), + ..options() + }, + ) + .unwrap(); + cancellation_worker.join().unwrap(); + assert_eq!(result.status, PdppRunStatus::Cancelled); + } + #[test] fn requires_a_sink_or_positive_record_retention_before_spawning() { let result = supervise_pdpp_connector( &fixture("success"), diff --git a/src-tauri/src/commands/pdpp_installed_connector.rs b/src-tauri/src/commands/pdpp_installed_connector.rs index 207cebdb..464c6997 100644 --- a/src-tauri/src/commands/pdpp_installed_connector.rs +++ b/src-tauri/src/commands/pdpp_installed_connector.rs @@ -5,36 +5,51 @@ //! capability, then delegates process supervision to the PDPP connector kernel. use super::connector_store::{get_active_connector_install, ActiveConnectorInstall}; +use super::pdpp_browser::{PdppBrowserBinding, PdppBrowserLease}; use super::pdpp_collection_state::{ - commit_terminal_run, load_connection_state, stage_succeeded_run, PdppCollectionConnectionState, - DEFAULT_CONNECTION_ID, + clear_connection_setup_complete, commit_terminal_run, is_connection_setup_complete, + load_connection_state, mark_connection_setup_complete, stage_succeeded_run, + PdppCollectionConnectionState, DEFAULT_CONNECTION_ID, }; use super::pdpp_connector::{ - supervise_pdpp_connector, PdppConnectorCommand, PdppEvent, PdppRecord, PdppRunControl, - PdppRunOptions, PdppRunResult, PdppRunStatus, PdppScopeValidators, PdppStart, + supervise_pdpp_connector, PdppConnectorCommand, PdppEvent, PdppInteractionResponder, + PdppInteractionResponseStatus, PdppRecord, PdppRunControl, PdppRunOptions, PdppRunResult, + PdppRunStatus, PdppScopeValidators, PdppStart, }; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use sha2::{Digest, Sha256}; use std::collections::{HashMap, HashSet}; +use std::ffi::OsStr; use std::fs; use std::path::{Component, Path, PathBuf}; use std::process::Command; use std::sync::{Arc, LazyLock, Mutex}; use std::thread; use std::time::{Duration, Instant}; -use tauri::{AppHandle, Emitter}; +use tauri::{AppHandle, Emitter, Manager}; const PDPP_ARTIFACT_KIND: &str = "pdpp-collection-profile"; const DEFAULT_TIMEOUT_SECONDS: u64 = 120; const MAX_TIMEOUT_SECONDS: u64 = 900; const MAX_RUN_ID_BYTES: usize = 128; const MINIMUM_NODE_MAJOR: u64 = 22; +const BUNDLED_NODE_NAME: &str = if cfg!(windows) { + "pdpp-node.exe" +} else { + "pdpp-node" +}; const CLEANUP_WAIT: Duration = Duration::from_secs(2); const GITHUB_CONNECTOR_KEY: &str = "github"; const GITHUB_CONNECTOR_ID: &str = "https://registry.pdpp.org/connectors/github"; +const CHATGPT_CONNECTOR_KEY: &str = "chatgpt"; +const CHATGPT_CONNECTOR_ID: &str = "https://registry.pdpp.org/connectors/chatgpt"; +const CHATGPT_CONNECTOR_INSTALL_ID: &str = "chatgpt-pdpp"; static ACTIVE_PDPP_RUNS: LazyLock>> = LazyLock::new(|| Mutex::new(HashMap::new())); +static PENDING_PDPP_INTERACTIONS: LazyLock< + Mutex>, +> = LazyLock::new(|| Mutex::new(HashMap::new())); /// A run remains registered until its child-supervision task has returned. /// Keeping connector identity alongside the cancellation control makes the @@ -59,6 +74,11 @@ pub struct StartInstalledPdppConnectorRequest { pub connection_id: Option, #[serde(default)] pub github_token: Option, + /// The pinned ChatGPT profile declares exactly two static-secret fields. + /// They are accepted for this invocation only and never enter run state, + /// export data, or a command response. + #[serde(default)] + pub setup_secrets: Option>, #[serde(default)] pub timeout_seconds: Option, } @@ -119,6 +139,7 @@ struct CommandCustomization { max_retained_records: usize, control: PdppRunControl, on_event: Option, + on_interaction: Option, } #[derive(Debug, Deserialize)] @@ -128,9 +149,36 @@ struct PdppConnectorManifest { display_name: Option, version: Option, runtime_requirements: Option, + setup: Option, streams: Vec, } +#[derive(Debug, Deserialize)] +struct PdppStaticSecretSetup { + modality: String, + credential_capture: PdppCredentialCapture, +} + +#[derive(Debug, Deserialize)] +struct PdppCredentialCapture { + fields: Vec, +} + +#[derive(Debug, Deserialize)] +struct PdppStaticSecretField { + name: String, + required: bool, + secret: bool, + #[serde(default)] + env: Vec, +} + +#[derive(Default)] +struct PdppChildSecrets { + environment: HashMap, + values: Vec, +} + #[derive(Debug, Deserialize)] struct RuntimeRequirements { bindings: Option>, @@ -148,6 +196,23 @@ struct PdppManifestStream { consent_time_field: Option, } +#[derive(Debug, Deserialize)] +struct PdppArtifactProvenance { + #[serde(default)] + external_runtime_packages: Vec, +} + +#[derive(Debug, Deserialize)] +struct PdppRuntimePackageRequirement { + name: String, + version: String, +} + +#[derive(Debug, Deserialize)] +struct NodePackageMetadata { + version: String, +} + /// The kernel validates every record before this accumulator sees it. Keeping /// the complete stream here is intentionally distinct from the kernel's small /// diagnostic retention buffers: the former is the user's export, the latter @@ -202,16 +267,19 @@ fn start_installed_pdpp_connector_run_impl( control: PdppRunControl, ) -> Result { validate_request(&request)?; - let resolved = resolve_active_installed_pdpp_connector(&request.connector_id)?; - let credential = resolve_github_credential(&request, &resolved)?; + let resource_dir = app.path().resource_dir().ok(); + let runtime_root = resolve_pdpp_runtime_root(resource_dir.as_deref())?; + let resolved = resolve_active_installed_pdpp_connector(&request.connector_id, &runtime_root)?; let saved_state = load_connection_state(&resolved.connector_id, request.connection_id())?; + let setup_complete = chatgpt_setup_complete(&resolved, request.connection_id())?; + let secrets = resolve_child_secrets_for_connection(&request, &resolved, setup_complete)?; let start_state = persisted_start_state(&request, &saved_state); let export_accumulator = Arc::new(Mutex::new(PdppExportAccumulator::default())); let sink = event_sink_for_run( - app, + app.clone(), request.run_id.clone(), export_accumulator.clone(), - credential.clone(), + secrets.values.clone(), ); let result = run_resolved_installed_pdpp_connector_with_state( &resolved, @@ -219,10 +287,17 @@ fn start_installed_pdpp_connector_run_impl( CommandCustomization { control, on_event: Some(sink), + on_interaction: Some(interaction_sink_for_run( + app.clone(), + request.run_id.clone(), + secrets.values.clone(), + )), ..Default::default() }, - credential.as_deref(), + &secrets, start_state, + resource_dir, + runtime_root, )?; let export = if result.status == PdppRunStatus::Succeeded { let accumulated = export_accumulator @@ -263,6 +338,12 @@ fn start_installed_pdpp_connector_run_impl( &committed_checkpoints, )? .ok_or("PDPP collection state was not committed after a successful run")?; + // Mark setup only after the credentialed run and its export/state + // commit have succeeded. A failed launch, login, cancellation, or + // timeout must leave the next attempt in owner-attended setup. + if should_mark_chatgpt_setup_complete(&resolved, &request, &result.status) { + mark_connection_setup_complete(&resolved.connector_id, request.connection_id())?; + } Some(export) } else { None @@ -272,7 +353,7 @@ fn start_installed_pdpp_connector_run_impl( request.run_id, resolved.connector_id, result, - credential.as_deref(), + &secrets.values, ), export, }) @@ -283,14 +364,16 @@ fn run_resolved_installed_pdpp_connector( resolved: &ResolvedInstalledPdppConnector, request: &StartInstalledPdppConnectorRequest, customization: CommandCustomization, - github_credential: Option<&str>, + secrets: &PdppChildSecrets, ) -> Result { run_resolved_installed_pdpp_connector_with_state( resolved, request, customization, - github_credential, + secrets, + None, None, + resolve_pdpp_runtime_root(None)?, ) } @@ -298,12 +381,44 @@ fn run_resolved_installed_pdpp_connector_with_state( resolved: &ResolvedInstalledPdppConnector, request: &StartInstalledPdppConnectorRequest, customization: CommandCustomization, - github_credential: Option<&str>, + secrets: &PdppChildSecrets, state: Option, + resource_dir: Option, + runtime_root: PathBuf, ) -> Result { validate_request(request)?; + let browser_lease = if requires_browser(&resolved.manifest) { + let owner_id = request.connection_id.as_deref().filter(|owner| !owner.is_empty()).ok_or( + "PDPP browser connector requires an explicit connectionId owner; the default owner is not permitted", + )?; + Some(Arc::new(Mutex::new(PdppBrowserLease::launch( + &resolved.connector_id, + owner_id, + &request.run_id, + resource_dir.as_deref(), + )?))) + } else { + None + }; + let browser_binding = browser_lease + .as_ref() + .map(|lease| { + lease + .lock() + .map_err(|_| "PDPP browser lease is unavailable") + }) + .transpose()? + .map(|lease| lease.binding().clone()); let start = build_start(request, &resolved.manifest, state)?; - let command = build_command(resolved, github_credential, &customization)?; + let command = build_command( + resolved, + secrets, + &customization, + browser_binding.as_ref(), + &runtime_root, + )?; + let host_interaction = customization.on_interaction.clone(); + let browser_for_interaction = browser_lease.clone(); let options = PdppRunOptions { timeout: Some(Duration::from_secs( request.timeout_seconds.unwrap_or(DEFAULT_TIMEOUT_SECONDS), @@ -317,9 +432,64 @@ fn run_resolved_installed_pdpp_connector_with_state( .unwrap_or_else(|| Arc::new(|_| Ok(()))), ), control: customization.control, + on_interaction: host_interaction.map(|host_interaction| { + Arc::new( + move |interaction: &super::pdpp_connector::PdppInteraction, responder| { + if let Some(lease) = &browser_for_interaction { + lease + .lock() + .map_err(|_| "PDPP browser lease is unavailable")? + .mark_waiting_for_user(); + } + host_interaction(interaction, responder) + }, + ) as super::pdpp_connector::PdppInteractionSink + }), + on_interaction_closed: Some(interaction_closed_sink_for_run(request.run_id.clone())), ..Default::default() }; - supervise_pdpp_connector(&command, &start, &options) + let mut result = supervise_pdpp_connector(&command, &start, &options); + if let (Ok(result), Some(binding)) = (&mut result, browser_binding.as_ref()) { + redact_browser_endpoint(result, &binding.cdp_http_url); + } + if let Some(lease) = browser_lease { + if let Ok(mut lease) = lease.lock() { + lease.close(); + } + } + result +} + +fn redact_browser_endpoint(result: &mut PdppRunResult, endpoint: &str) { + if endpoint.is_empty() { + return; + } + result.stderr = result + .stderr + .replace(endpoint, "[REDACTED_BROWSER_ENDPOINT]"); + if let Some(failure) = &mut result.failure { + *failure = failure.replace(endpoint, "[REDACTED_BROWSER_ENDPOINT]"); + } + for event in &mut result.events { + match event { + PdppEvent::Progress(progress) => { + progress.message = progress + .message + .replace(endpoint, "[REDACTED_BROWSER_ENDPOINT]"); + } + PdppEvent::SkipResult(skip) => { + if let Some(message) = &mut skip.message { + *message = message.replace(endpoint, "[REDACTED_BROWSER_ENDPOINT]"); + } + } + PdppEvent::Interaction(interaction) => { + interaction.message = interaction + .message + .replace(endpoint, "[REDACTED_BROWSER_ENDPOINT]"); + } + _ => {} + } + } } #[cfg(test)] @@ -336,7 +506,7 @@ fn run_resolved_installed_pdpp_connector_for_test( max_retained_records: 256, ..Default::default() }, - request.github_token.as_deref(), + &resolve_child_secrets(request, resolved)?, ) } @@ -375,6 +545,7 @@ fn unregister_run(run_id: &str) { if let Ok(mut runs) = ACTIVE_PDPP_RUNS.lock() { runs.remove(run_id); } + invalidate_pending_interactions_for_run(run_id); } fn cancel_run(run_id: &str) -> Result<(), String> { @@ -384,6 +555,9 @@ fn cancel_run(run_id: &str) -> Result<(), String> { .get(run_id) .map(|run| run.control.clone()) .ok_or_else(|| format!("PDPP runId {run_id} is not active"))?; + // Remove owner-visible responders before signalling the supervisor. A + // late Tauri command must not claim success while cancellation is queued. + invalidate_pending_interactions_for_run(run_id); control.cancel(); Ok(()) } @@ -394,6 +568,71 @@ pub fn stop_installed_pdpp_connector_run(run_id: String) -> Result<(), String> { cancel_run(&run_id) } +/// Sends one authoritative Collection Profile `INTERACTION_RESPONSE` to the +/// still-pending request for this run. OTP values stay only in this command +/// payload and the child stdin; they are never retained in state, events, or +/// logs. +#[tauri::command] +pub fn submit_installed_pdpp_interaction_response( + run_id: String, + request_id: String, + status: String, + data: Option, +) -> Result<(), String> { + validate_run_id(&run_id)?; + validate_interaction_request_id(&request_id)?; + let status = match status.as_str() { + "success" => PdppInteractionResponseStatus::Success, + "cancelled" => PdppInteractionResponseStatus::Cancelled, + "timeout" => PdppInteractionResponseStatus::Timeout, + _ => { + return Err( + "PDPP INTERACTION_RESPONSE status must be success, cancelled, or timeout".into(), + ) + } + }; + if data.as_ref().is_some_and(|value| !value.is_object()) { + return Err("PDPP INTERACTION_RESPONSE data must be an object".into()); + } + let responder = PENDING_PDPP_INTERACTIONS + .lock() + .map_err(|_| "PDPP interaction registry is unavailable")? + .remove(&(run_id.clone(), request_id.clone())) + .ok_or("PDPP interaction is no longer pending for this run")?; + responder.respond(status, data) +} + +/// Explicitly disconnects an installed PDPP browser connector by deleting the +/// durable, owner-confined authenticated profile. It refuses while that owner +/// has a live lease, so a reset can never race a collection run. +#[tauri::command] +pub fn reset_installed_pdpp_browser_profile( + connector_id: String, + connection_id: String, +) -> Result<(), String> { + if connector_id != CHATGPT_CONNECTOR_INSTALL_ID { + return Err("PDPP browser profile reset is only available for ChatGPT".into()); + } + validate_connection_id(&connection_id)?; + PdppBrowserLease::reset_profile(&connector_id, &connection_id)?; + clear_connection_setup_complete(&connector_id, &connection_id) +} + +/// A non-secret marker and its durable owner profile are both required before +/// the UI may omit recovery credentials for a scheduled ChatGPT run. +#[tauri::command] +pub fn is_installed_pdpp_browser_setup_complete( + connector_id: String, + connection_id: String, +) -> Result { + if connector_id != CHATGPT_CONNECTOR_INSTALL_ID { + return Err("PDPP browser setup state is only available for ChatGPT".into()); + } + validate_connection_id(&connection_id)?; + Ok(is_connection_setup_complete(&connector_id, &connection_id)? + && PdppBrowserLease::profile_exists(&connector_id, &connection_id)?) +} + pub fn cleanup_installed_pdpp_connector_runs() { let controls = ACTIVE_PDPP_RUNS .lock() @@ -461,6 +700,20 @@ fn validate_connection_id(connection_id: &str) -> Result<(), String> { Ok(()) } +fn validate_interaction_request_id(request_id: &str) -> Result<(), String> { + if request_id.is_empty() + || request_id.len() > MAX_RUN_ID_BYTES + || !request_id + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.')) + { + return Err( + "PDPP interaction requestId must be 1-128 URL-safe identifier characters".into(), + ); + } + Ok(()) +} + fn persisted_start_state( request: &StartInstalledPdppConnectorRequest, saved_state: &PdppCollectionConnectionState, @@ -481,14 +734,23 @@ impl StartInstalledPdppConnectorRequest { fn resolve_active_installed_pdpp_connector( connector_id: &str, + runtime_root: &Path, ) -> Result { let install = get_active_connector_install(connector_id) .ok_or_else(|| format!("PDPP connector {connector_id} is not installed"))?; - resolve_installed_pdpp_connector(&install) + resolve_installed_pdpp_connector_with_runtime(&install, runtime_root) } fn resolve_installed_pdpp_connector( install: &ActiveConnectorInstall, +) -> Result { + let runtime_root = resolve_pdpp_runtime_root(None)?; + resolve_installed_pdpp_connector_with_runtime(install, &runtime_root) +} + +fn resolve_installed_pdpp_connector_with_runtime( + install: &ActiveConnectorInstall, + runtime_root: &Path, ) -> Result { if install.artifact_kind.as_deref() != Some(PDPP_ARTIFACT_KIND) { return Err(format!( @@ -515,11 +777,7 @@ fn resolve_installed_pdpp_connector( let provenance_path = confined_existing_file(&root, provenance_relative, "PDPP provenance path")?; let manifest_sha256 = required_hash(install.manifest_sha256.as_deref(), "manifestSha256")?; - verify_file_hash( - &manifest_path, - Some(manifest_sha256), - "PDPP manifest", - )?; + verify_file_hash(&manifest_path, Some(manifest_sha256), "PDPP manifest")?; verify_file_hash( &entrypoint_path, Some(required_hash( @@ -542,6 +800,7 @@ fn resolve_installed_pdpp_connector( ) .map_err(|e| format!("Failed to parse PDPP connector manifest: {e}"))?; validate_manifest(&install.connector_id, &install.version, &manifest)?; + validate_chatgpt_runtime_requirements(&provenance_path, &manifest, runtime_root)?; Ok(ResolvedInstalledPdppConnector { connector_id: install.connector_id.clone(), manifest_sha256: manifest_sha256.to_owned(), @@ -602,10 +861,21 @@ fn validate_manifest( if manifest.streams.is_empty() { return Err("PDPP connector manifest must declare at least one stream".into()); } - if connector_id != "github-pdpp" - || manifest.connector_key.as_deref() != Some(GITHUB_CONNECTOR_KEY) - || manifest.connector_id.as_deref() != Some(GITHUB_CONNECTOR_ID) - { + let identity_matches = match manifest.connector_key.as_deref() { + Some(GITHUB_CONNECTOR_KEY) => { + connector_id == "github-pdpp" + && manifest.connector_id.as_deref() == Some(GITHUB_CONNECTOR_ID) + && !requires_browser(manifest) + } + Some(CHATGPT_CONNECTOR_KEY) => { + connector_id == CHATGPT_CONNECTOR_INSTALL_ID + && manifest.connector_id.as_deref() == Some(CHATGPT_CONNECTOR_ID) + && requires_browser(manifest) + && required_chatgpt_static_secret_fields(manifest).is_ok() + } + _ => false, + }; + if !identity_matches { return Err(format!( "PDPP connector manifest does not match active install id {connector_id}" )); @@ -625,7 +895,7 @@ fn validate_manifest( return Err("PDPP connector manifest must require the network binding".into()); } for (binding, requirement) in bindings.into_iter().flat_map(|bindings| bindings.iter()) { - if binding != "network" && requirement.required.unwrap_or(false) { + if binding != "network" && binding != "browser" && requirement.required.unwrap_or(false) { return Err(format!( "PDPP connector requires unsupported binding {binding}" )); @@ -640,6 +910,100 @@ fn validate_manifest( Ok(()) } +fn required_chatgpt_static_secret_fields( + manifest: &PdppConnectorManifest, +) -> Result, String> { + let setup = manifest + .setup + .as_ref() + .ok_or("ChatGPT PDPP connector must declare setup")?; + if setup.modality != "static_secret" { + return Err("ChatGPT PDPP connector must use setup.modality static_secret".into()); + } + let fields = &setup.credential_capture.fields; + if fields.len() != 2 + || fields[0].name != "username" + || !fields[0].required + || !fields[0].secret + || fields[0].env != ["CHATGPT_USERNAME"] + || fields[1].name != "password" + || !fields[1].required + || !fields[1].secret + || fields[1].env != ["CHATGPT_PASSWORD"] + { + return Err( + "ChatGPT PDPP setup must declare only required secret username and password fields" + .into(), + ); + } + Ok(vec![ + ("username", "CHATGPT_USERNAME"), + ("password", "CHATGPT_PASSWORD"), + ]) +} + +fn validate_chatgpt_runtime_requirements( + provenance_path: &Path, + manifest: &PdppConnectorManifest, + runtime_root: &Path, +) -> Result<(), String> { + if manifest.connector_key.as_deref() != Some(CHATGPT_CONNECTOR_KEY) { + return Ok(()); + } + // The distributable tar does not carry build-only artifact.json. The + // artifact builder copies declared external_packages into signed provenance, + // whose hash is verified before this parse. + let provenance: PdppArtifactProvenance = serde_json::from_str( + &fs::read_to_string(provenance_path) + .map_err(|error| format!("Failed to read ChatGPT PDPP provenance: {error}"))?, + ) + .map_err(|error| format!("Failed to parse ChatGPT PDPP provenance: {error}"))?; + for expected in ["p-queue", "patchright"] { + let requirement = provenance + .external_runtime_packages + .iter() + .find(|package| package.name == expected) + .ok_or_else(|| format!("ChatGPT PDPP provenance must declare {expected}"))?; + let metadata: NodePackageMetadata = serde_json::from_str( + &fs::read_to_string(confined_existing_file( + &runtime_root, + &format!("node_modules/{expected}/package.json"), + &format!("PDPP runtime dependency {expected}"), + )?) + .map_err(|error| { + format!("Failed to read PDPP runtime dependency {expected}: {error}") + })?, + ) + .map_err(|error| format!("Failed to parse PDPP runtime dependency {expected}: {error}"))?; + if !satisfies_caret_requirement(&metadata.version, &requirement.version) { + return Err(format!( + "PDPP runtime dependency {expected}@{} does not satisfy artifact requirement {}", + metadata.version, requirement.version + )); + } + } + Ok(()) +} + +fn satisfies_caret_requirement(installed: &str, requirement: &str) -> bool { + let Some(required) = requirement.strip_prefix('^').and_then(parse_semver) else { + return false; + }; + let Some(installed) = parse_semver(installed) else { + return false; + }; + installed.0 == required.0 && installed >= required +} + +fn parse_semver(value: &str) -> Option<(u64, u64, u64)> { + let mut parts = value.split('.'); + Some(( + parts.next()?.parse().ok()?, + parts.next()?.parse().ok()?, + parts.next()?.split(['-', '+']).next()?.parse().ok()?, + )) +} + fn build_start( request: &StartInstalledPdppConnectorRequest, manifest: &PdppConnectorManifest, @@ -661,13 +1025,21 @@ fn build_start( let scope = json!({ "streams": selected.into_iter().map(|name| json!({ "name": name })).collect::>() }); - PdppStart::new( - &request.run_id, - &request.collection_mode, - scope, - state, - json!({ "network": { "enabled": true } }), - ) + PdppStart::new(&request.run_id, &request.collection_mode, scope, state) +} + +fn requires_browser(manifest: &PdppConnectorManifest) -> bool { + manifest + .runtime_requirements + .as_ref() + .and_then(|requirements| requirements.bindings.as_ref()) + .and_then(|bindings| bindings.get("browser")) + .and_then(|binding| binding.required) + .unwrap_or(false) +} + +fn is_chatgpt_connector(manifest: &PdppConnectorManifest) -> bool { + manifest.connector_key.as_deref() == Some(CHATGPT_CONNECTOR_KEY) } fn selected_streams( @@ -742,24 +1114,102 @@ fn validators_from_manifest(manifest: &PdppConnectorManifest) -> PdppScopeValida validators } -fn resolve_github_credential( +fn resolve_child_secrets( + request: &StartInstalledPdppConnectorRequest, + resolved: &ResolvedInstalledPdppConnector, +) -> Result { + resolve_child_secrets_for_connection(request, resolved, false) +} + +fn resolve_child_secrets_for_connection( request: &StartInstalledPdppConnectorRequest, resolved: &ResolvedInstalledPdppConnector, -) -> Result, String> { + setup_complete: bool, +) -> Result { let manifest_key = resolved.manifest.connector_key.as_deref().unwrap_or(""); - if manifest_key != "github" { + if manifest_key == CHATGPT_CONNECTOR_KEY { if request.github_token.is_some() { return Err("githubToken can only be passed to the GitHub PDPP connector".into()); } - return Ok(None); + let expected = required_chatgpt_static_secret_fields(&resolved.manifest)?; + let Some(provided) = request.setup_secrets.as_ref() else { + return if setup_complete { + Ok(PdppChildSecrets::default()) + } else { + Err("ChatGPT PDPP connector requires setupSecrets.username and setupSecrets.password for first setup or explicit recovery".into()) + }; + }; + if provided.len() != expected.len() + || expected.iter().any(|(field, _)| { + provided + .get(*field) + .is_none_or(|value| value.trim().is_empty()) + }) + { + return Err( + "ChatGPT PDPP connector requires only non-empty setupSecrets.username and setupSecrets.password" + .into(), + ); + } + let mut secrets = PdppChildSecrets::default(); + for (field, environment_key) in expected { + let value = provided[field].clone(); + secrets.values.push(value.clone()); + secrets + .environment + .insert(environment_key.to_owned(), value); + } + return Ok(secrets); + } + + if request.setup_secrets.is_some() { + return Err("setupSecrets can only be passed to the ChatGPT PDPP connector".into()); + } + if manifest_key != GITHUB_CONNECTOR_KEY { + return Err("DataConnect does not support this PDPP connector identity".into()); + } + + let token = resolve_github_credential(request)?; + let mut secrets = PdppChildSecrets::default(); + secrets.values.push(token.clone()); + secrets + .environment + .insert("GITHUB_TOKEN".into(), token.clone()); + secrets + .environment + .insert("GITHUB_PERSONAL_ACCESS_TOKEN".into(), token); + Ok(secrets) +} + +fn chatgpt_setup_complete( + resolved: &ResolvedInstalledPdppConnector, + connection_id: &str, +) -> Result { + if !is_chatgpt_connector(&resolved.manifest) { + return Ok(false); } + is_connection_setup_complete(&resolved.connector_id, connection_id) +} + +fn should_mark_chatgpt_setup_complete( + resolved: &ResolvedInstalledPdppConnector, + request: &StartInstalledPdppConnectorRequest, + status: &PdppRunStatus, +) -> bool { + is_chatgpt_connector(&resolved.manifest) + && request.setup_secrets.is_some() + && *status == PdppRunStatus::Succeeded +} +fn resolve_github_credential( + request: &StartInstalledPdppConnectorRequest, +) -> Result { if let Some(token) = request .github_token .as_deref() .filter(|token| !token.is_empty()) { - return Ok(Some(token.to_owned())); + return Ok(token.to_owned()); } // A local desktop UAT may use a shell-provided credential, but release @@ -768,7 +1218,7 @@ fn resolve_github_credential( #[cfg(debug_assertions)] if let Ok(token) = std::env::var("PDPP_E2E_GITHUB_TOKEN") { if !token.is_empty() { - return Ok(Some(token)); + return Ok(token); } } @@ -780,17 +1230,23 @@ fn resolve_github_credential( fn build_command( resolved: &ResolvedInstalledPdppConnector, - github_credential: Option<&str>, + secrets: &PdppChildSecrets, customization: &CommandCustomization, + browser_binding: Option<&PdppBrowserBinding>, + runtime_root: &Path, ) -> Result { - let mut env = HashMap::new(); - if let Some(token) = github_credential { - let manifest_key = resolved.manifest.connector_key.as_deref().unwrap_or(""); - if manifest_key != "github" { - return Err("githubToken can only be passed to the GitHub PDPP connector".into()); + let mut env = secrets.environment.clone(); + if let Some(binding) = browser_binding { + if resolved.manifest.connector_key.as_deref() != Some(CHATGPT_CONNECTOR_KEY) { + return Err("PDPP browser binding is only available to the ChatGPT connector".into()); } - env.insert("GITHUB_TOKEN".into(), token.to_owned()); - env.insert("GITHUB_PERSONAL_ACCESS_TOKEN".into(), token.to_owned()); + // The pinned runtime resolves this documented compatibility seam before it + // considers an isolated local browser. It is a PDPP runtime concern, + // unrelated to DataConnect's legacy Playwright page API. + env.insert( + "PDPP_CHATGPT_REMOTE_CDP_URL".into(), + binding.cdp_http_url.clone(), + ); } env.insert("PDPP_CONNECTOR_NETWORK".into(), "1".into()); env.insert( @@ -805,6 +1261,17 @@ fn build_command( args.push("--import".into()); args.push(import.to_string_lossy().into_owned()); } + args.push("--import".into()); + args.push( + runtime_root + .join("connector-loader-bootstrap.mjs") + .to_string_lossy() + .into_owned(), + ); + env.insert( + "DATACONNECT_PDPP_RUNTIME_ROOT".into(), + runtime_root.to_string_lossy().into_owned(), + ); args.push(resolved.entrypoint_path.to_string_lossy().into_owned()); Ok(PdppConnectorCommand { program: resolve_node_program()?, @@ -815,18 +1282,70 @@ fn build_command( }) } +fn resolve_pdpp_runtime_root(resource_dir: Option<&Path>) -> Result { + if let Some(resource_dir) = resource_dir { + let root = resource_dir.join("pdpp-runtime"); + let root = canonical_existing_dir(&root, "PDPP runtime root")?; + return validate_pdpp_runtime_root(root); + } + if let Some(configured) = std::env::var_os("DATACONNECT_PDPP_RUNTIME_ROOT") { + let root = canonical_existing_dir(Path::new(&configured), "PDPP runtime root")?; + return validate_pdpp_runtime_root(root); + } + let mut candidates = vec![PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("pdpp-runtime")]; + if let Ok(executable) = std::env::current_exe() { + if let Some(macos_resources) = executable + .parent() + .and_then(Path::parent) + .map(|contents| contents.join("Resources").join("pdpp-runtime")) + { + candidates.push(macos_resources); + } + if let Some(executable_dir) = executable.parent() { + candidates.push(executable_dir.join("pdpp-runtime")); + candidates.push(executable_dir.join("resources").join("pdpp-runtime")); + } + } + for candidate in candidates { + if let Ok(root) = canonical_existing_dir(&candidate, "PDPP runtime root") { + if let Ok(root) = validate_pdpp_runtime_root(root) { + return Ok(root); + } + } + } + Err("Packaged PDPP runtime dependencies p-queue and patchright are unavailable".into()) +} + +fn validate_pdpp_runtime_root(root: PathBuf) -> Result { + confined_existing_file(&root, "connector-loader.mjs", "PDPP runtime loader")?; + confined_existing_file( + &root, + "connector-loader-bootstrap.mjs", + "PDPP runtime loader bootstrap", + )?; + for package in ["p-queue", "patchright"] { + confined_existing_file( + &root, + &format!("node_modules/{package}/package.json"), + &format!("PDPP runtime dependency {package}"), + )?; + } + Ok(root) +} + fn event_sink_for_run( app: AppHandle, run_id: String, export_accumulator: Arc>, - secret: Option, + secrets: Vec, ) -> super::pdpp_connector::PdppEventSink { Arc::new(move |event| match event { PdppEvent::Record(record) => { - if secret.as_deref().is_some_and(|credential| { - value_contains_secret(&record.key, credential) - || value_contains_secret(&record.data, credential) - }) { + if value_contains_any_secret(&record.key, &secrets) + || value_contains_any_secret(&record.data, &secrets) + { return Err("PDPP connector attempted to emit its credential as data".into()); } let mut collected = export_accumulator @@ -840,11 +1359,7 @@ fn event_sink_for_run( Ok(()) } PdppEvent::Progress(progress) => { - emit_running_status( - &app, - &run_id, - &redact_secret(&progress.message, secret.as_deref()), - ); + emit_running_status(&app, &run_id, &redact_secrets(&progress.message, &secrets)); Ok(()) } PdppEvent::SkipResult(skip) => { @@ -864,11 +1379,70 @@ fn event_sink_for_run( PdppEvent::State(_) | PdppEvent::DetailCoverage(_) | PdppEvent::DetailGap(_) - | PdppEvent::DetailGapRecovered(_) - | PdppEvent::Interaction(_) => Ok(()), + | PdppEvent::DetailGapRecovered(_) => Ok(()), + PdppEvent::Interaction(_) => { + emit_running_status(&app, &run_id, "Waiting for browser interaction..."); + Ok(()) + } + }) +} + +fn interaction_sink_for_run( + app: AppHandle, + run_id: String, + secrets: Vec, +) -> super::pdpp_connector::PdppInteractionSink { + Arc::new( + move |interaction: &super::pdpp_connector::PdppInteraction, responder| { + if responder.run_id() != run_id || interaction.request_id != responder.request_id() { + return Err("PDPP interaction responder is not bound to this run/request".into()); + } + validate_interaction_request_id(&interaction.request_id)?; + let key = (run_id.clone(), interaction.request_id.clone()); + PENDING_PDPP_INTERACTIONS + .lock() + .map_err(|_| "PDPP interaction registry is unavailable")? + .insert(key, responder); + let message = redact_secrets(&interaction.message, &secrets); + emit_running_status(&app, &run_id, "Waiting for owner interaction..."); + let _ = app.emit( + "pdpp-interaction", + json!({ + "runId": run_id, + "requestId": interaction.request_id, + "kind": interaction.kind, + "message": message, + "schema": interaction.schema, + "timeoutSeconds": interaction.timeout_seconds, + }), + ); + Ok(()) + }, + ) +} + +fn interaction_closed_sink_for_run( + run_id: String, +) -> super::pdpp_connector::PdppInteractionClosedSink { + Arc::new(move |closed_run_id, request_id| { + if closed_run_id == run_id { + invalidate_pending_interaction(closed_run_id, request_id); + } }) } +fn invalidate_pending_interaction(run_id: &str, request_id: &str) { + if let Ok(mut pending) = PENDING_PDPP_INTERACTIONS.lock() { + pending.remove(&(run_id.to_owned(), request_id.to_owned())); + } +} + +fn invalidate_pending_interactions_for_run(run_id: &str) { + if let Ok(mut pending) = PENDING_PDPP_INTERACTIONS.lock() { + pending.retain(|(pending_run_id, _), _| pending_run_id != run_id); + } +} + fn value_contains_secret(value: &Value, secret: &str) -> bool { if secret.is_empty() { return false; @@ -885,6 +1459,12 @@ fn value_contains_secret(value: &Value, secret: &str) -> bool { } } +fn value_contains_any_secret(value: &Value, secrets: &[String]) -> bool { + secrets + .iter() + .any(|secret| value_contains_secret(value, secret)) +} + fn build_export_data( resolved: &ResolvedInstalledPdppConnector, request: &StartInstalledPdppConnectorRequest, @@ -901,7 +1481,7 @@ fn build_export_data( .connector_id .as_deref() .ok_or("PDPP connector manifest is missing connector_id")?; - if connector_key != GITHUB_CONNECTOR_KEY { + if connector_key != GITHUB_CONNECTOR_KEY && connector_key != CHATGPT_CONNECTOR_KEY { return Err( "DataConnect does not yet have a storage projection for this PDPP connector".into(), ); @@ -916,18 +1496,27 @@ fn build_export_data( let records = records_by_stream.get(stream).cloned().unwrap_or_default(); record_count += records.len(); stream_counts.insert(stream.clone(), json!(records.len())); - let projection = match stream.as_str() { + let projection = match (connector_key, stream.as_str()) { // The Personal Server's existing GitHub schemas deliberately use // these shapes. This is a DataConnect storage projection, not a // claim that the PDPP connector only supports three streams. - "user" if !records.is_empty() => { + (GITHUB_CONNECTOR_KEY, "user") if !records.is_empty() => { Some(("github.profile", project_github_profile(&records)?)) } - "repositories" => Some(( + (GITHUB_CONNECTOR_KEY, "repositories") => Some(( "github.repositories", project_github_repositories(&records)?, )), - "starred" => Some(("github.starred", project_github_starred(&records)?)), + (GITHUB_CONNECTOR_KEY, "starred") => { + Some(("github.starred", project_github_starred(&records)?)) + } + // Fixture contract: the ChatGPT Collection Profile emits one + // PDPP record per conversation. Preserve each record's data as + // supplied; schema-specific normalization belongs upstream. + (CHATGPT_CONNECTOR_KEY, "conversations") => Some(( + "chatgpt.conversations", + json!({ "conversations": records.iter().map(|record| record.data.clone()).collect::>() }), + )), _ => None, }; if let Some((scope, value)) = projection { @@ -999,7 +1588,7 @@ fn build_export_data( "count": record_count, "label": format!("{record_count} {connector_key} records exported"), "details": { - "pdppStorageProjection": "github-v1", + "pdppStorageProjection": if connector_key == GITHUB_CONNECTOR_KEY { "github-v1" } else { "chatgpt-fixture-v1" }, "pdppStreamRecords": stream_counts, } }), @@ -1299,7 +1888,29 @@ fn verify_file_hash(path: &Path, expected: Option<&str>, label: &str) -> Result< } fn resolve_node_program() -> Result { - let path = std::env::var_os("PATH").ok_or("PATH is unavailable; cannot resolve node")?; + let executable = std::env::current_exe() + .map_err(|e| format!("Could not locate the DataConnect executable: {e}"))?; + resolve_node_program_from(&executable, std::env::var_os("PATH").as_deref()) +} + +fn resolve_node_program_from(executable: &Path, path: Option<&OsStr>) -> Result { + let executable_dir = executable + .parent() + .ok_or("DataConnect executable has no parent directory")?; + let bundled = executable_dir.join(BUNDLED_NODE_NAME); + if bundled.exists() { + if !bundled.is_file() { + return Err(format!( + "Bundled Node.js path is not a file: {}", + bundled.display() + )); + } + validate_node_program(&bundled)?; + return Ok(bundled.to_string_lossy().into_owned()); + } + + let path = path + .ok_or("Bundled Node.js is unavailable and PATH cannot resolve a development fallback")?; for dir in std::env::split_paths(&path) { let candidate = dir.join(if cfg!(windows) { "node.exe" } else { "node" }); if candidate.is_file() { @@ -1307,7 +1918,7 @@ fn resolve_node_program() -> Result { return Ok(candidate.to_string_lossy().into_owned()); } } - Err("node executable was not found on PATH".into()) + Err("Bundled Node.js is unavailable and node was not found on PATH".into()) } fn validate_node_program(candidate: &Path) -> Result<(), String> { @@ -1356,9 +1967,9 @@ fn to_response( run_id: String, connector_id: String, result: PdppRunResult, - secret: Option<&str>, + secrets: &[String], ) -> InstalledPdppConnectorRunResponse { - let progress = sanitize_retained_events(result.events, secret); + let progress = sanitize_retained_events(result.events, secrets); let event_summary = PdppEventSummary { records: result.event_counts.records, checkpoint_updates: result.event_counts.checkpoint_updates, @@ -1393,7 +2004,7 @@ fn to_response( progress, records_truncated: result.records_truncated, events_truncated: result.events_truncated, - failure: failure.map(|failure| redact_secret(&failure, secret)), + failure: failure.map(|failure| redact_secrets(&failure, secrets)), stderr_bytes: result.stderr.len(), stderr_truncated: result.stderr_truncated, exit_code: result.exit_code, @@ -1402,7 +2013,7 @@ fn to_response( fn sanitize_retained_events( events: Vec, - secret: Option<&str>, + secrets: &[String], ) -> Vec { let mut messages = Vec::new(); for event in events { @@ -1411,21 +2022,23 @@ fn sanitize_retained_events( messages.push(SanitizedConnectorMessage { message_type: "PROGRESS".into(), stream: progress.stream, - message: Some(redact_secret(&progress.message, secret)), + message: Some(redact_secrets(&progress.message, secrets)), }); } PdppEvent::SkipResult(skip) => { messages.push(SanitizedConnectorMessage { message_type: "SKIP_RESULT".into(), stream: skip.stream, - message: skip.message.map(|message| redact_secret(&message, secret)), + message: skip + .message + .map(|message| redact_secrets(&message, secrets)), }); } PdppEvent::Interaction(interaction) => { messages.push(SanitizedConnectorMessage { message_type: "INTERACTION".into(), stream: None, - message: Some(redact_secret(&interaction.message, secret)), + message: Some(redact_secrets(&interaction.message, secrets)), }); } PdppEvent::Record(_) @@ -1438,11 +2051,14 @@ fn sanitize_retained_events( messages } -fn redact_secret(value: &str, secret: Option<&str>) -> String { - match secret { - Some(secret) if !secret.is_empty() => value.replace(secret, "[REDACTED]"), - _ => value.to_string(), - } +fn redact_secrets(value: &str, secrets: &[String]) -> String { + secrets.iter().fold(value.to_owned(), |redacted, secret| { + if secret.is_empty() { + redacted + } else { + redacted.replace(secret, "[REDACTED]") + } + }) } #[cfg(test)] @@ -1465,13 +2081,24 @@ mod tests { ) .unwrap(); fs::write(temp.path().join("dist/profile.cjs"), script).unwrap(); - fs::write(temp.path().join("provenance.json"), "{}").unwrap(); + let provenance = if manifest["connector_key"] == CHATGPT_CONNECTOR_KEY { + json!({ + "external_runtime_packages": [ + { "name": "p-queue", "version": "^9.3.3" }, + { "name": "patchright", "version": "^1.61.1" } + ] + }) + } else { + json!({}) + }; + let provenance_bytes = serde_json::to_vec_pretty(&provenance).unwrap(); + fs::write(temp.path().join("provenance.json"), &provenance_bytes).unwrap(); let manifest_sha = format!( "sha256:{:x}", Sha256::digest(&serde_json::to_vec_pretty(&manifest).unwrap()) ); let entrypoint_sha = format!("sha256:{:x}", Sha256::digest(script.as_bytes())); - let provenance_sha = format!("sha256:{:x}", Sha256::digest(b"{}")); + let provenance_sha = format!("sha256:{:x}", Sha256::digest(&provenance_bytes)); ( temp, ActiveConnectorInstall { @@ -1530,6 +2157,134 @@ mod tests { }) } + fn chatgpt_browser_manifest() -> Value { + serde_json::from_str(include_str!( + "../../tests/fixtures/chatgpt-pdpp-browser.collection-profile.json" + )) + .unwrap() + } + + fn chatgpt_artifact_root() -> Option { + std::env::var_os("PDPP_CHATGPT_ARTIFACT_ROOT").map(PathBuf::from) + } + + fn sha256_for(path: &Path) -> String { + format!("sha256:{:x}", Sha256::digest(fs::read(path).unwrap())) + } + + fn unpacked_actual_chatgpt_install( + artifact_root: &Path, + root: &Path, + ) -> ActiveConnectorInstall { + let artifact_dir = artifact_root.join("artifacts/chatgpt-pdpp"); + let artifact = fs::read_dir(&artifact_dir) + .expect("PDPP_CHATGPT_ARTIFACT_ROOT must contain artifacts/chatgpt-pdpp") + .filter_map(Result::ok) + .map(|entry| entry.path()) + .find(|path| path.extension().is_some_and(|extension| extension == "tgz")) + .expect("PDPP_CHATGPT_ARTIFACT_ROOT must contain one chatgpt-pdpp tarball"); + let status = Command::new("tar") + .args([ + "-xzf", + artifact.to_str().unwrap(), + "-C", + root.to_str().unwrap(), + ]) + .status() + .unwrap(); + assert!(status.success()); + let version = serde_json::from_str::( + &fs::read_to_string(root.join("profile/collection-profile.json")).unwrap(), + ) + .unwrap()["version"] + .as_str() + .unwrap() + .to_owned(); + ActiveConnectorInstall { + connector_id: CHATGPT_CONNECTOR_INSTALL_ID.into(), + company: "OpenAI".into(), + version, + root_path: root.to_string_lossy().into_owned(), + metadata_relative_path: "legacy.json".into(), + script_relative_path: "legacy.js".into(), + artifact_kind: Some(PDPP_ARTIFACT_KIND.into()), + manifest_path: Some("profile/collection-profile.json".into()), + entrypoint_path: Some("dist/collection-profile.mjs".into()), + entrypoint_sha256: Some(sha256_for(&root.join("dist/collection-profile.mjs"))), + manifest_sha256: Some(sha256_for(&root.join("profile/collection-profile.json"))), + provenance_path: Some("provenance.json".into()), + provenance_sha256: Some(sha256_for(&root.join("provenance.json"))), + } + } + + fn node_major(program: &Path) -> Option { + let output = Command::new(program).arg("--version").output().ok()?; + if !output.status.success() { + return None; + } + String::from_utf8(output.stdout) + .ok()? + .trim() + .trim_start_matches('v') + .split('.') + .next()? + .parse() + .ok() + } + + fn optional_node_for_major(major: u64) -> Option { + let environment_name = format!("DATACONNECT_NODE_{major}"); + let configured = std::env::var_os(environment_name).map(PathBuf::from); + let mut candidates = configured + .into_iter() + .chain(std::iter::once(PathBuf::from(format!("node{major}")))); + candidates.find(|candidate| node_major(candidate.as_path()) == Some(major)) + } + + fn assert_actual_patchright_esm_import(root: &Path, node: &Path) { + let runtime_root = resolve_pdpp_runtime_root(None).unwrap(); + let bootstrap = runtime_root.join("connector-loader-bootstrap.mjs"); + let probe = r#" +const { default: PQueue } = await import("p-queue"); +const { chromium } = await import("patchright"); +if (typeof PQueue !== "function") throw new Error("p-queue default export is unavailable"); +if (!chromium || typeof chromium.connectOverCDP !== "function") { + throw new Error("patchright ESM chromium export is unavailable"); +} +process.stdout.write("packaged-externals-ok\n"); +"#; + let positive = Command::new(node) + .args([ + "--import", + bootstrap.to_str().unwrap(), + "--input-type=module", + "-e", + probe, + ]) + .current_dir(root) + .env_clear() + .env("DATACONNECT_PDPP_RUNTIME_ROOT", &runtime_root) + .output() + .unwrap(); + assert!( + positive.status.success(), + "loader import failed: {}", + String::from_utf8_lossy(&positive.stderr) + ); + assert_eq!(positive.stdout, b"packaged-externals-ok\n"); + + let negative = Command::new(node) + .args(["--input-type=module", "-e", "await import('patchright')"]) + .current_dir(root) + .env_clear() + .output() + .unwrap(); + assert!(!negative.status.success()); + assert!( + String::from_utf8_lossy(&negative.stderr).contains("Cannot find package 'patchright'") + ); + } + fn success_script() -> &'static str { r#" const readline = require('node:readline'); @@ -1556,6 +2311,68 @@ rl.on('line', (line) => { "# } + fn pending_interaction_command() -> PdppConnectorCommand { + PdppConnectorCommand { + program: "node".into(), + args: vec![ + "-e".into(), + r#" +const readline = require('node:readline'); +const emit = message => process.stdout.write(`${JSON.stringify(message)}\n`); +readline.createInterface({ input: process.stdin }).on('line', line => { + const message = JSON.parse(line); + if (message.type === 'START') { + emit({ type: 'INTERACTION', request_id: 'pending-request', kind: 'otp', message: 'Enter code', timeout_seconds: 60 }); + } +}); +"# + .into(), + ], + cwd: None, + env: HashMap::new(), + clear_env: false, + } + } + + fn start_pending_interaction_for_test( + run_id: &'static str, + ) -> (PdppRunControl, thread::JoinHandle) { + let control = register_run(run_id, "pending-interaction-test", "owner").unwrap(); + let (ready_sender, ready_receiver) = std::sync::mpsc::channel(); + let run_id_owned = run_id.to_owned(); + let on_interaction: crate::commands::pdpp_connector::PdppInteractionSink = Arc::new( + move |_interaction: &crate::commands::pdpp_connector::PdppInteraction, responder| { + PENDING_PDPP_INTERACTIONS + .lock() + .unwrap() + .insert((run_id_owned.clone(), "pending-request".into()), responder); + ready_sender.send(()).unwrap(); + Ok(()) + }, + ); + let options = PdppRunOptions { + control: control.clone(), + max_retained_records: 1, + on_interaction: Some(on_interaction), + on_interaction_closed: Some(interaction_closed_sink_for_run(run_id.into())), + ..Default::default() + }; + let start = PdppStart::new( + run_id, + "incremental", + json!({"streams":[{"name":"items"}]}), + None, + ) + .unwrap(); + let handle = thread::spawn(move || { + supervise_pdpp_connector(&pending_interaction_command(), &start, &options).unwrap() + }); + ready_receiver + .recv_timeout(Duration::from_secs(2)) + .expect("pending interaction should register before the test continues"); + (control, handle) + } + fn request_with_token(token: &str) -> StartInstalledPdppConnectorRequest { StartInstalledPdppConnectorRequest { run_id: "run-1".into(), @@ -1564,6 +2381,7 @@ rl.on('line', (line) => { streams: vec!["repositories".into()], connection_id: None, github_token: Some(token.into()), + setup_secrets: None, timeout_seconds: Some(5), } } @@ -1637,7 +2455,7 @@ rl.on('line', (line) => { } #[test] - fn rejects_unsupported_browser_binding_for_network_only_host() { + fn rejects_a_browser_binding_for_the_network_only_github_host() { let manifest = json!({ "connector_id": GITHUB_CONNECTOR_ID, "connector_key": GITHUB_CONNECTOR_KEY, @@ -1645,7 +2463,7 @@ rl.on('line', (line) => { "runtime_requirements": { "bindings": { "network": { "required": true }, - "browser_automation": { "required": true } + "browser": { "required": true } } }, "streams": [{ "name": "repositories" }] @@ -1654,7 +2472,435 @@ rl.on('line', (line) => { install.root_path = temp.path().to_string_lossy().into_owned(); assert!(resolve_installed_pdpp_connector(&install) .unwrap_err() - .contains("unsupported binding")); + .contains("does not match")); + } + + #[test] + fn admits_the_actual_chatgpt_browser_capability_without_extending_start() { + let manifest: PdppConnectorManifest = + serde_json::from_value(chatgpt_browser_manifest()).unwrap(); + validate_manifest(CHATGPT_CONNECTOR_INSTALL_ID, "0.1.0", &manifest).unwrap(); + let request = StartInstalledPdppConnectorRequest { + run_id: "chatgpt-browser-fixture".into(), + connector_id: CHATGPT_CONNECTOR_INSTALL_ID.into(), + collection_mode: "incremental".into(), + streams: vec!["conversations".into()], + connection_id: Some("account-one".into()), + github_token: None, + setup_secrets: Some(HashMap::from([ + ("username".into(), "owner@example.com".into()), + ("password".into(), "fixture-password".into()), + ])), + timeout_seconds: Some(5), + }; + let start = build_start(&request, &manifest, None).unwrap(); + let serialized = serde_json::to_value(start).unwrap(); + assert!(serialized.get("bindings").is_none()); + assert!(serialized.get("browser").is_none()); + } + + #[test] + fn chatgpt_static_secrets_are_transient_after_first_owner_setup() { + let (temp, mut install) = install_fixture(chatgpt_browser_manifest(), success_script()); + install.root_path = temp.path().to_string_lossy().into_owned(); + install.connector_id = CHATGPT_CONNECTOR_INSTALL_ID.into(); + install.version = "0.1.0".into(); + let resolved = resolve_installed_pdpp_connector(&install).unwrap(); + let first_setup = StartInstalledPdppConnectorRequest { + run_id: "chatgpt-first-setup".into(), + connector_id: CHATGPT_CONNECTOR_INSTALL_ID.into(), + collection_mode: "incremental".into(), + streams: vec!["conversations".into()], + connection_id: Some("owner-a".into()), + github_token: None, + setup_secrets: Some(HashMap::from([ + ("username".into(), "owner@example.com".into()), + ("password".into(), "not-persisted".into()), + ])), + timeout_seconds: Some(5), + }; + let first = resolve_child_secrets_for_connection(&first_setup, &resolved, false).unwrap(); + assert_eq!(first.environment["CHATGPT_USERNAME"], "owner@example.com"); + assert_eq!(first.environment["CHATGPT_PASSWORD"], "not-persisted"); + + let scheduled = StartInstalledPdppConnectorRequest { + run_id: "chatgpt-scheduled".into(), + setup_secrets: None, + ..first_setup.clone() + }; + let second = resolve_child_secrets_for_connection(&scheduled, &resolved, true).unwrap(); + assert!(second.environment.is_empty()); + assert!(second.values.is_empty()); + assert!(matches!( + resolve_child_secrets_for_connection(&scheduled, &resolved, false), + Err(error) if error.contains("first setup or explicit recovery") + )); + } + + #[test] + fn failed_or_interrupted_chatgpt_setup_keeps_the_next_run_in_setup() { + let (temp, mut install) = install_fixture(chatgpt_browser_manifest(), success_script()); + install.root_path = temp.path().to_string_lossy().into_owned(); + install.connector_id = CHATGPT_CONNECTOR_INSTALL_ID.into(); + install.version = "0.1.0".into(); + let resolved = resolve_installed_pdpp_connector(&install).unwrap(); + let credentialed = StartInstalledPdppConnectorRequest { + run_id: "chatgpt-first-setup".into(), + connector_id: CHATGPT_CONNECTOR_INSTALL_ID.into(), + collection_mode: "incremental".into(), + streams: vec!["conversations".into()], + connection_id: Some("owner-a".into()), + github_token: None, + setup_secrets: Some(HashMap::from([ + ("username".into(), "owner@example.com".into()), + ("password".into(), "not-persisted".into()), + ])), + timeout_seconds: Some(5), + }; + let retry_without_secrets = StartInstalledPdppConnectorRequest { + run_id: "chatgpt-retry".into(), + setup_secrets: None, + ..credentialed.clone() + }; + + for status in [ + PdppRunStatus::Failed, // login/run error or browser launch failure + PdppRunStatus::Cancelled, + PdppRunStatus::TimedOut, + ] { + assert!(!should_mark_chatgpt_setup_complete( + &resolved, + &credentialed, + &status + )); + assert!(matches!( + resolve_child_secrets_for_connection(&retry_without_secrets, &resolved, false), + Err(error) if error.contains("first setup or explicit recovery") + )); + } + assert!(should_mark_chatgpt_setup_complete( + &resolved, + &credentialed, + &PdppRunStatus::Succeeded + )); + } + + #[test] + fn cancelling_a_pending_interaction_rejects_late_tauri_responses() { + let _guard = RUN_REGISTRY_TEST_LOCK.lock().unwrap(); + let run_id = "cancel-pending-interaction"; + let (_control, handle) = start_pending_interaction_for_test(run_id); + + cancel_run(run_id).unwrap(); + assert!(submit_installed_pdpp_interaction_response( + run_id.into(), + "pending-request".into(), + "success".into(), + Some(json!({"code":"late"})), + ) + .unwrap_err() + .contains("no longer pending")); + assert_eq!(handle.join().unwrap().status, PdppRunStatus::Cancelled); + unregister_run(run_id); + } + + #[test] + fn timed_out_interaction_close_callback_rejects_late_tauri_responses() { + let _guard = RUN_REGISTRY_TEST_LOCK.lock().unwrap(); + let run_id = "timeout-pending-interaction"; + let (control, handle) = start_pending_interaction_for_test(run_id); + + interaction_closed_sink_for_run(run_id.into())(run_id, "pending-request"); + assert!(submit_installed_pdpp_interaction_response( + run_id.into(), + "pending-request".into(), + "success".into(), + Some(json!({"code":"late"})), + ) + .unwrap_err() + .contains("no longer pending")); + control.cancel(); + assert_eq!(handle.join().unwrap().status, PdppRunStatus::Cancelled); + unregister_run(run_id); + } + + #[test] + fn chatgpt_fixture_tracks_configured_artifact_manifest_contract() { + let Some(artifact_root) = chatgpt_artifact_root() else { + return; + }; + let actual = fs::read_to_string( + artifact_root.join("connectors/chatgpt-pdpp/collection-profile.json"), + ) + .unwrap(); + let actual: Value = serde_json::from_str(&actual).unwrap(); + let fixture = chatgpt_browser_manifest(); + assert_eq!(fixture["connector_id"], actual["connector_id"]); + assert_eq!( + fixture["runtime_requirements"]["bindings"], + actual["runtime_requirements"]["bindings"] + ); + assert_eq!(fixture["setup"]["modality"], actual["setup"]["modality"]); + assert_eq!( + fixture["setup"]["credential_capture"]["fields"], + actual["setup"]["credential_capture"]["fields"] + ); + assert_eq!( + fixture["streams"] + .as_array() + .unwrap() + .iter() + .map(|stream| &stream["name"]) + .collect::>(), + actual["streams"] + .as_array() + .unwrap() + .iter() + .map(|stream| &stream["name"]) + .collect::>(), + ); + } + + #[test] + fn packaged_runtime_matches_configured_artifact_requirements() { + let Some(artifact_root) = chatgpt_artifact_root() else { + return; + }; + let artifact: Value = serde_json::from_str( + &fs::read_to_string(artifact_root.join("connectors/chatgpt-pdpp/artifact.json")) + .unwrap(), + ) + .unwrap(); + let provenance: Value = serde_json::from_str( + &fs::read_to_string(artifact_root.join("connectors/chatgpt-pdpp/provenance.json")) + .unwrap(), + ) + .unwrap(); + assert_eq!( + artifact["build"]["external_packages"], + provenance["external_runtime_packages"] + ); + let runtime_root = resolve_pdpp_runtime_root(None).unwrap(); + assert_packaged_pdpp_runtime_files(&runtime_root); + let generated_runtime_root = std::env::current_exe() + .unwrap() + .parent() + .unwrap() + .parent() + .unwrap() + .join("pdpp-runtime"); + assert_packaged_pdpp_runtime_files(&generated_runtime_root); + for requirement in artifact["build"]["external_packages"].as_array().unwrap() { + let name = requirement["name"].as_str().unwrap(); + let required = requirement["version"].as_str().unwrap(); + let metadata: NodePackageMetadata = serde_json::from_str( + &fs::read_to_string( + runtime_root + .join("node_modules") + .join(name) + .join("package.json"), + ) + .unwrap(), + ) + .unwrap(); + assert!(satisfies_caret_requirement(&metadata.version, required)); + } + } + + fn assert_packaged_pdpp_runtime_files(runtime_root: &Path) { + for relative_path in [ + "connector-loader.mjs", + "connector-loader-bootstrap.mjs", + "node_modules/p-queue/package.json", + "node_modules/p-queue/dist/index.js", + "node_modules/patchright/package.json", + "node_modules/patchright/index.mjs", + ] { + assert!( + runtime_root.join(relative_path).is_file(), + "packaged PDPP runtime is missing {relative_path} under {}", + runtime_root.display() + ); + } + } + + fn create_minimal_pdpp_runtime_root(root: &Path) { + for relative_path in [ + "connector-loader.mjs", + "connector-loader-bootstrap.mjs", + "node_modules/p-queue/package.json", + "node_modules/patchright/package.json", + ] { + let path = root.join(relative_path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, "{}").unwrap(); + } + } + + #[test] + fn runtime_root_prefers_explicit_tauri_resource_directory() { + let temp = tempfile::tempdir().unwrap(); + let resource_runtime = temp.path().join("Resources").join("pdpp-runtime"); + create_minimal_pdpp_runtime_root(&resource_runtime); + + let resolved = resolve_pdpp_runtime_root(Some(&temp.path().join("Resources"))).unwrap(); + + assert_eq!(resolved, fs::canonicalize(resource_runtime).unwrap()); + } + + #[test] + fn actual_artifact_loader_uses_patchright_esm_chromium_export() { + let Some(artifact_root) = chatgpt_artifact_root() else { + return; + }; + let temp = tempfile::tempdir().unwrap(); + unpacked_actual_chatgpt_install(&artifact_root, temp.path()); + + // The configured Node is always verified. Exercise Node 22 and 23 as + // well when CI or a developer has explicitly provided either binary. + let configured = PathBuf::from(resolve_node_program().unwrap()); + assert_actual_patchright_esm_import(temp.path(), &configured); + for major in [22, 23] { + if let Some(node) = optional_node_for_major(major) { + if node != configured { + assert_actual_patchright_esm_import(temp.path(), &node); + } + } + } + } + + #[test] + fn unpacks_and_starts_the_configured_artifact_with_packaged_externals() { + let Some(artifact_root) = chatgpt_artifact_root() else { + return; + }; + let temp = tempfile::tempdir().unwrap(); + let install = unpacked_actual_chatgpt_install(&artifact_root, temp.path()); + let resolved = resolve_installed_pdpp_connector(&install).unwrap(); + let request = StartInstalledPdppConnectorRequest { + run_id: "actual-chatgpt-artifact".into(), + connector_id: CHATGPT_CONNECTOR_INSTALL_ID.into(), + collection_mode: "incremental".into(), + streams: vec!["conversations".into()], + connection_id: Some("actual-artifact-owner".into()), + github_token: None, + // A scheduled collection reuses its owner profile without replaying + // the initial static-secret handoff. + setup_secrets: None, + timeout_seconds: Some(5), + }; + let binding = PdppBrowserBinding { + backend: "neko", + cdp_http_url: "http://127.0.0.1:9".into(), + lease_id: "actual-artifact-lease".into(), + profile_key: "actual-artifact-profile".into(), + }; + let secrets = resolve_child_secrets_for_connection(&request, &resolved, true).unwrap(); + let runtime_root = resolve_pdpp_runtime_root(None).unwrap(); + let command = build_command( + &resolved, + &secrets, + &CommandCustomization { + max_retained_records: 1, + ..Default::default() + }, + Some(&binding), + &runtime_root, + ) + .unwrap(); + assert!(!command.env.contains_key("CHATGPT_USERNAME")); + assert!(!command.env.contains_key("CHATGPT_PASSWORD")); + assert_eq!( + command.env["PDPP_CHATGPT_REMOTE_CDP_URL"], + "http://127.0.0.1:9" + ); + assert!(!command.env.contains_key("PDPP_BROWSER_SURFACE_REQUIRED")); + assert!(command.clear_env); + let loader_index = command + .args + .iter() + .position(|argument| argument == "--import") + .expect("Node 22 loader hook must be installed with --import"); + assert!(command.args[loader_index + 1].ends_with("connector-loader-bootstrap.mjs")); + + let mut without_loader = command.clone(); + without_loader.args.drain(loader_index..=loader_index + 1); + without_loader.env.remove("DATACONNECT_PDPP_RUNTIME_ROOT"); + let negative = supervise_pdpp_connector( + &without_loader, + &build_start(&request, &resolved.manifest, None).unwrap(), + &PdppRunOptions { + timeout: Some(Duration::from_secs(5)), + max_retained_records: 1, + ..Default::default() + }, + ) + .unwrap(); + assert_ne!(negative.status, PdppRunStatus::Succeeded); + assert!(negative.stderr.contains("Cannot find package 'p-queue'")); + + let mut result = supervise_pdpp_connector( + &command, + &build_start(&request, &resolved.manifest, None).unwrap(), + &PdppRunOptions { + timeout: Some(Duration::from_secs(5)), + max_retained_records: 1, + ..Default::default() + }, + ) + .unwrap(); + assert_ne!(result.status, PdppRunStatus::Succeeded); + assert!(!result.stderr.contains("Cannot find package 'p-queue'")); + assert!(!result.stderr.contains("Cannot find package 'patchright'")); + assert!(result.stderr.contains("remote CDP attach start")); + redact_browser_endpoint(&mut result, "http://127.0.0.1:9"); + assert!(!result.stderr.contains("http://127.0.0.1:9")); + assert!(!result + .failure + .as_deref() + .unwrap_or_default() + .contains("http://127.0.0.1:9")); + } + + #[test] + fn projects_the_provisional_chatgpt_conversation_stream_without_schema_rewrite() { + let (temp, mut install) = install_fixture(chatgpt_browser_manifest(), success_script()); + install.root_path = temp.path().to_string_lossy().into_owned(); + install.connector_id = CHATGPT_CONNECTOR_INSTALL_ID.into(); + install.version = "0.1.0".into(); + let resolved = resolve_installed_pdpp_connector(&install).unwrap(); + let request = StartInstalledPdppConnectorRequest { + run_id: "chatgpt-projection".into(), + connector_id: CHATGPT_CONNECTOR_INSTALL_ID.into(), + collection_mode: "incremental".into(), + streams: vec!["conversations".into()], + connection_id: Some("account-one".into()), + github_token: None, + setup_secrets: Some(HashMap::from([ + ("username".into(), "owner@example.com".into()), + ("password".into(), "fixture-password".into()), + ])), + timeout_seconds: Some(5), + }; + let state = PdppCollectionConnectionState { + snapshot_by_stream: HashMap::from([( + "conversations".into(), + vec![PdppRecord { + stream: "conversations".into(), + key: json!("conversation-1"), + data: json!({ "id": "conversation-1", "upstream_field": "preserved" }), + emitted_at: "2026-07-31T00:00:00Z".into(), + op: None, + }], + )]), + ..Default::default() + }; + let export = build_export_data(&resolved, &request, &state, &[]).unwrap(); + assert_eq!(export["requestedScopes"], json!(["chatgpt.conversations"])); + assert_eq!( + export["chatgpt.conversations"]["conversations"][0]["upstream_field"], + json!("preserved") + ); } #[test] @@ -1666,7 +2912,7 @@ rl.on('line', (line) => { "runtime_requirements": { "bindings": { "network": { "required": true }, - "browser_automation": { "required": false } + "browser": { "required": false } } }, "streams": [{ "name": "repositories" }] @@ -1738,6 +2984,7 @@ rl.on('line', (line) => { streams: vec!["repositories".into()], connection_id: None, github_token: None, + setup_secrets: None, timeout_seconds: None, }; let manifest: PdppConnectorManifest = serde_json::from_value(github_manifest()).unwrap(); @@ -1760,6 +3007,7 @@ rl.on('line', (line) => { streams: vec![], connection_id: None, github_token: None, + setup_secrets: None, timeout_seconds: None, }; let manifest: PdppConnectorManifest = @@ -1790,6 +3038,7 @@ rl.on('line', (line) => { streams: vec!["user_stats".into(), "pull_requests".into()], connection_id: None, github_token: None, + setup_secrets: None, timeout_seconds: None, }; let manifest: PdppConnectorManifest = @@ -1967,6 +3216,7 @@ rl.on('line', (line) => { streams: vec![], connection_id: Some("account-one".into()), github_token: None, + setup_secrets: None, timeout_seconds: None, }; @@ -2052,6 +3302,7 @@ rl.on('line', (line) => { streams: vec!["repositories".into()], connection_id: None, github_token: None, + setup_secrets: None, timeout_seconds: None, }; let removed = PdppRecord { @@ -2097,10 +3348,13 @@ rl.on('line', (line) => { let resolved = resolve_installed_pdpp_connector(&install).unwrap(); let request = request_with_token("test-token"); let start = build_start(&request, &resolved.manifest, None).unwrap(); + let runtime_root = resolve_pdpp_runtime_root(None).unwrap(); let command = build_command( &resolved, - request.github_token.as_deref(), + &resolve_child_secrets(&request, &resolved).unwrap(), &CommandCustomization::default(), + None, + &runtime_root, ) .unwrap(); assert!(command.clear_env); @@ -2149,7 +3403,7 @@ readline.createInterface({ input: process.stdin }).on('line', (line) => { max_retained_records: 4, ..Default::default() }, - request.github_token.as_deref(), + &resolve_child_secrets(&request, &resolved).unwrap(), ) .unwrap(); assert!(result.stderr.contains(token)); @@ -2157,7 +3411,12 @@ readline.createInterface({ input: process.stdin }).on('line', (line) => { .records .iter() .any(|record| record.data.to_string().contains(token))); - let response = to_response("run-1".into(), "github-pdpp".into(), result, Some(token)); + let response = to_response( + "run-1".into(), + "github-pdpp".into(), + result, + &[token.into()], + ); let serialized = serde_json::to_string(&response).unwrap(); assert!(!serialized.contains(token)); assert!(!serialized.contains("repo-1")); @@ -2256,10 +3515,18 @@ setInterval(() => {}, 1000); streams: vec!["repositories".into()], connection_id: None, github_token: None, + setup_secrets: None, timeout_seconds: Some(1), }; let result = supervise_pdpp_connector( - &build_command(&resolved, None, &CommandCustomization::default()).unwrap(), + &build_command( + &resolved, + &PdppChildSecrets::default(), + &CommandCustomization::default(), + None, + &resolve_pdpp_runtime_root(None).unwrap(), + ) + .unwrap(), &build_start(&request, &resolved.manifest, None).unwrap(), &PdppRunOptions { timeout: Some(Duration::from_millis(50)), @@ -2302,6 +3569,7 @@ setInterval(() => {}, 1000); streams: vec!["repositories".into()], connection_id: None, github_token: None, + setup_secrets: None, timeout_seconds: Some(30), }; let handle = thread::spawn(move || { @@ -2313,7 +3581,7 @@ setInterval(() => {}, 1000); control, ..Default::default() }, - None, + &PdppChildSecrets::default(), ) }); @@ -2412,6 +3680,72 @@ setInterval(() => {}, 1000); .contains("invalid version")); } + #[test] + fn uses_path_node_only_when_bundled_node_is_absent() { + let ambient_node = PathBuf::from(resolve_node_program().unwrap()); + let app_dir = tempfile::tempdir().unwrap(); + let fake_app = app_dir.path().join("dataconnect"); + fs::write(&fake_app, b"fixture app").unwrap(); + let path = std::env::join_paths([ambient_node.parent().unwrap()]).unwrap(); + + let fallback = resolve_node_program_from(&fake_app, Some(&path)).unwrap(); + assert_eq!(Path::new(&fallback), ambient_node); + + fs::write(app_dir.path().join(BUNDLED_NODE_NAME), b"not node").unwrap(); + let error = resolve_node_program_from(&fake_app, Some(&path)).unwrap_err(); + assert!(error.contains("Failed to inspect Node.js")); + } + + #[test] + fn starts_connector_with_bundled_node_and_no_node_on_path() { + let ambient_node = PathBuf::from(resolve_node_program().unwrap()); + let app_dir = tempfile::tempdir().unwrap(); + let fake_app = app_dir.path().join(if cfg!(windows) { + "dataconnect.exe" + } else { + "dataconnect" + }); + fs::write(&fake_app, b"fixture app").unwrap(); + let bundled_node = app_dir.path().join(BUNDLED_NODE_NAME); + fs::copy(&ambient_node, &bundled_node).unwrap(); + + let resolved_node = + resolve_node_program_from(&fake_app, Some(OsStr::new(""))).unwrap(); + assert_eq!(Path::new(&resolved_node), bundled_node); + + let (temp, mut install) = install_fixture(github_manifest(), success_script()); + install.root_path = temp.path().to_string_lossy().into_owned(); + let resolved = resolve_installed_pdpp_connector(&install).unwrap(); + let runtime_root = resolve_pdpp_runtime_root(None).unwrap(); + let request = request_with_token("test-token"); + let start = build_start(&request, &resolved.manifest, None).unwrap(); + let mut command = build_command( + &resolved, + &resolve_child_secrets(&request, &resolved).unwrap(), + &CommandCustomization::default(), + None, + &runtime_root, + ) + .unwrap(); + command.program = resolved_node; + assert!(command.clear_env); + assert!(!command.env.contains_key("PATH")); + + let result = supervise_pdpp_connector( + &command, + &start, + &PdppRunOptions { + timeout: Some(Duration::from_secs(5)), + scope_validators: validators_from_manifest(&resolved.manifest), + max_retained_records: 8, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.status, PdppRunStatus::Succeeded); + assert_eq!(result.record_count, 1); + } + #[test] fn event_summary_counts_all_events_when_retained_messages_are_truncated() { let script = r#" @@ -2438,14 +3772,15 @@ readline.createInterface({ input: process.stdin }).on('line', (line) => { max_retained_records: 1, ..Default::default() }, - request.github_token.as_deref(), + &resolve_child_secrets(&request, &resolved).unwrap(), ) .unwrap(); + let secrets = resolve_child_secrets(&request, &resolved).unwrap(); let response = to_response( request.run_id, resolved.connector_id, result, - request.github_token.as_deref(), + &secrets.values, ); assert_eq!(response.event_summary.progress, 80); @@ -2464,7 +3799,9 @@ readline.createInterface({ input: process.stdin }).on('line', (line) => { .map(PathBuf::from) .into_iter() .collect(); - let resolved = resolve_active_installed_pdpp_connector("github-pdpp").unwrap(); + let runtime_root = resolve_pdpp_runtime_root(None).unwrap(); + let resolved = + resolve_active_installed_pdpp_connector("github-pdpp", &runtime_root).unwrap(); let request = StartInstalledPdppConnectorRequest { run_id: "github-pdpp-cross-repo-e2e".into(), connector_id: "github-pdpp".into(), @@ -2472,6 +3809,7 @@ readline.createInterface({ input: process.stdin }).on('line', (line) => { streams: vec!["user".into(), "repositories".into()], connection_id: None, github_token: Some(token.clone()), + setup_secrets: None, timeout_seconds: Some(120), }; let result = @@ -2520,9 +3858,74 @@ readline.createInterface({ input: process.stdin }).on('line', (line) => { result.records_truncated, ); - let response = to_response(request.run_id, resolved.connector_id, result, Some(&token)); + let response = to_response( + request.run_id, + resolved.connector_id, + result, + &[token.clone()], + ); let serialized = serde_json::to_string(&response).unwrap(); assert!(!serialized.contains(&token)); assert!(!serialized.contains("\"data\"")); } + + #[test] + #[ignore = "requires a published ChatGPT PDPP artifact, local browser login, and explicit confirmation"] + fn runs_external_installed_chatgpt_browser_artifact_end_to_end() { + assert_eq!( + std::env::var("PDPP_E2E_CHATGPT_CONFIRM").as_deref(), + Ok("1"), + "set PDPP_E2E_CHATGPT_CONFIRM=1 to allow the credentialed browser E2E" + ); + let runtime_root = resolve_pdpp_runtime_root(None).unwrap(); + let resolved = + resolve_active_installed_pdpp_connector(CHATGPT_CONNECTOR_INSTALL_ID, &runtime_root) + .unwrap(); + let request = StartInstalledPdppConnectorRequest { + run_id: "chatgpt-pdpp-browser-e2e".into(), + connector_id: CHATGPT_CONNECTOR_INSTALL_ID.into(), + collection_mode: "incremental".into(), + streams: vec!["conversations".into()], + connection_id: Some("chatgpt-e2e-owner".into()), + github_token: None, + setup_secrets: Some(HashMap::from([ + ( + "username".into(), + std::env::var("PDPP_E2E_CHATGPT_USERNAME") + .expect("PDPP_E2E_CHATGPT_USERNAME must be set"), + ), + ( + "password".into(), + std::env::var("PDPP_E2E_CHATGPT_PASSWORD") + .expect("PDPP_E2E_CHATGPT_PASSWORD must be set"), + ), + ])), + timeout_seconds: Some(300), + }; + let result = run_resolved_installed_pdpp_connector( + &resolved, + &request, + CommandCustomization { + max_retained_records: 4, + ..Default::default() + }, + &resolve_child_secrets(&request, &resolved).unwrap(), + ) + .unwrap(); + assert_eq!( + result.status, + PdppRunStatus::Succeeded, + "{:?}", + result.failure + ); + assert!(result + .events + .iter() + .any(|event| matches!(event, PdppEvent::Interaction(_)))); + assert!(result + .records + .iter() + .any(|record| record.stream == "conversations")); + assert!(!result.stderr.contains("chatgpt-e2e-owner")); + } } diff --git a/src-tauri/src/commands/server.rs b/src-tauri/src/commands/server.rs index 09131608..02f4c1c0 100644 --- a/src-tauri/src/commands/server.rs +++ b/src-tauri/src/commands/server.rs @@ -3,6 +3,8 @@ use std::path::PathBuf; use std::sync::Mutex; use tauri::{AppHandle, Emitter, Manager, WebviewWindow}; +use super::updates::activate_bundled_pdpp_connectors; + /// Send a signal to an entire process group (negative PID). /// Falls back to single-process signal if pgid lookup fails. #[cfg(unix)] @@ -161,6 +163,13 @@ pub async fn start_personal_server( } }; + // The Personal Server reads the active PDPP install once at startup. + // Seed verified bundled installs before it composes its routes. + if let Err(error) = activate_bundled_pdpp_connectors(&app) { + clear_starting(); + return Err(error); + } + // Kill any stale personal-server from a previous unclean exit #[cfg(unix)] { diff --git a/src-tauri/src/commands/updates.rs b/src-tauri/src/commands/updates.rs index 8720de6e..e1cc28f9 100644 --- a/src-tauri/src/commands/updates.rs +++ b/src-tauri/src/commands/updates.rs @@ -12,9 +12,9 @@ use tauri::{AppHandle, Manager}; use tempfile::tempdir_in; use super::connector_store::{ - get_active_connector_install, get_connectors_store_dir, get_legacy_user_connectors_dir, - read_active_connector_manifest, write_active_connector_manifest, ActiveConnectorInstall, - ActiveConnectorManifest, + activate_bundled_connector_install, get_active_connector_install, get_connectors_store_dir, + get_legacy_user_connectors_dir, read_active_connector_manifest, + replace_active_connector_install, ActiveConnectorInstall, }; const DEFAULT_INDEX_URL: &str = @@ -179,21 +179,46 @@ struct PdppArtifactBundle { provenance: Vec, } +#[derive(Debug, Deserialize)] +struct BundledConnectorLock { + connectors: Vec, +} + fn get_user_connectors_dir() -> Option { get_legacy_user_connectors_dir() } -fn activate_connector_install(install: ActiveConnectorInstall) -> Result<(), String> { - let mut manifest = read_active_connector_manifest().unwrap_or(ActiveConnectorManifest { - version: "1.0".to_string(), - updated_at: chrono::Utc::now().to_rfc3339(), - connectors: HashMap::new(), - }); - manifest.updated_at = chrono::Utc::now().to_rfc3339(); - manifest - .connectors - .insert(install.connector_id.clone(), install); - write_active_connector_manifest(&manifest) +fn bundled_pdpp_connector_installs( + bundled_connectors_dir: &Path, +) -> Result, String> { + let lock_path = bundled_connectors_dir.join("lock.json"); + let lock_bytes = fs::read(&lock_path) + .map_err(|e| format!("Failed to read bundled connector lock: {}", e))?; + let lock: BundledConnectorLock = serde_json::from_slice(&lock_bytes) + .map_err(|e| format!("Failed to parse bundled connector lock: {}", e))?; + + lock.connectors + .iter() + .filter_map(|connector| match connector { + IndexedConnector::PdppCollectionProfile(connector) => Some(connector), + IndexedConnector::Legacy(_) => None, + }) + .map(|connector| { + let connector_id = + safe_store_segment(&connector.common.connector_id, "PDPP connector id")?; + let install_root = bundled_connectors_dir + .join("collection-profiles") + .join(connector_id); + active_pdpp_install_at(connector, &install_root) + }) + .collect() +} + +pub(crate) fn activate_bundled_pdpp_connectors(app: &AppHandle) -> Result<(), String> { + for install in bundled_pdpp_connector_installs(&get_bundled_connectors_dir(&app))? { + activate_bundled_connector_install(install)?; + } + Ok(()) } fn get_bundled_connectors_dir(app: &AppHandle) -> PathBuf { @@ -930,7 +955,7 @@ fn install_verified_connector_artifact( let store_dir = get_connectors_store_dir().ok_or("Could not determine connectors store directory")?; let install = install_verified_connector_artifact_into(connector, artifact_bytes, &store_dir)?; - activate_connector_install(install) + replace_active_connector_install(install) } fn install_verified_connector_artifact_into( @@ -1122,13 +1147,6 @@ fn install_verified_pdpp_connector( let connector_store_dir = install_root .parent() .ok_or("Connector install root is missing its store directory")?; - let manifest_relative = - normalize_nonempty_artifact_path(&connector.manifest_path, "PDPP manifest path")?; - let entrypoint_relative = - normalize_nonempty_artifact_path(&connector.entrypoint_path, "PDPP entrypoint path")?; - let provenance_relative = - normalize_nonempty_artifact_path(&connector.provenance_path, "PDPP provenance path")?; - if !install_root.exists() { log::info!( "Installing PDPP connector artifact for {} to {:?}", @@ -1138,25 +1156,57 @@ fn install_verified_pdpp_connector( promote_staged_install(connector_store_dir, &install_root, |staged| { write_pdpp_artifact_bundle(staged, &bundle) })?; - } else { - verify_installed_file( - &install_root.join(&manifest_relative), - &common.manifest_sha256, - "installed PDPP manifest", - )?; - verify_installed_file( - &install_root.join(&entrypoint_relative), - &connector.entrypoint_sha256, - "installed PDPP entrypoint", - )?; - verify_installed_file( - &install_root.join(&provenance_relative), - &connector.provenance_sha256, - "installed PDPP provenance", - )?; } - let install = ActiveConnectorInstall { + let install = active_pdpp_install_at(connector, &install_root)?; + + log::info!( + "=== Successfully installed PDPP connector: {} ===", + common.connector_id + ); + Ok(install) +} + +fn active_pdpp_install_at( + connector: &PdppIndexedConnector, + install_root: &Path, +) -> Result { + let common = &connector.common; + let manifest_relative = + normalize_nonempty_artifact_path(&connector.manifest_path, "PDPP manifest path")?; + let entrypoint_relative = + normalize_nonempty_artifact_path(&connector.entrypoint_path, "PDPP entrypoint path")?; + let provenance_relative = + normalize_nonempty_artifact_path(&connector.provenance_path, "PDPP provenance path")?; + + verify_installed_file( + &install_root.join(&manifest_relative), + &common.manifest_sha256, + "installed PDPP manifest", + )?; + verify_installed_file( + &install_root.join(&entrypoint_relative), + &connector.entrypoint_sha256, + "installed PDPP entrypoint", + )?; + verify_installed_file( + &install_root.join(&provenance_relative), + &connector.provenance_sha256, + "installed PDPP provenance", + )?; + + let manifest_bytes = fs::read(install_root.join(&manifest_relative)) + .map_err(|e| format!("Failed to read installed PDPP manifest: {}", e))?; + let manifest: PdppManifestIdentity = serde_json::from_slice(&manifest_bytes) + .map_err(|e| format!("Failed to parse installed PDPP manifest: {}", e))?; + if manifest.version != common.version { + return Err(format!( + "Installed PDPP version mismatch. Expected {}, got {}", + common.version, manifest.version + )); + } + + Ok(ActiveConnectorInstall { connector_id: common.connector_id.clone(), company: common.company.clone(), version: common.version.clone(), @@ -1170,13 +1220,7 @@ fn install_verified_pdpp_connector( manifest_sha256: Some(common.manifest_sha256.clone()), provenance_path: Some(provenance_relative.to_string_lossy().into_owned()), provenance_sha256: Some(connector.provenance_sha256.clone()), - }; - - log::info!( - "=== Successfully installed PDPP connector: {} ===", - common.connector_id - ); - Ok(install) + }) } fn verify_named_checksum(bytes: &[u8], expected: &str, label: &str) -> Result<(), String> { @@ -1283,16 +1327,17 @@ fn scan_connectors_dir_no_overwrite(dir: &PathBuf, versions: &mut HashMap>(); + connector_ids.sort_unstable(); + assert_eq!(connector_ids, ["chatgpt-pdpp", "github-pdpp"]); + } + #[test] fn index_rejects_unknown_explicit_artifact_kind() { let manifest = br#"{"version":"0.5.0"}"#; diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index b0fbb218..3012b05a 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -13,7 +13,10 @@ use commands::{ load_source_export_preview_from_path, mark_export_synced, open_folder, open_personal_server_scope_folder, open_platform_export_folder, set_app_config, start_connector_run, start_installed_pdpp_connector_run, start_personal_server, - stop_connector_run, stop_installed_pdpp_connector_run, stop_personal_server, test_nodejs, + is_installed_pdpp_browser_setup_complete, reset_installed_pdpp_browser_profile, + stop_connector_run, stop_installed_pdpp_connector_run, + submit_installed_pdpp_interaction_response, + stop_personal_server, test_nodejs, write_export_data, }; use tauri::{Listener, Manager}; @@ -78,6 +81,9 @@ pub fn run() { start_connector_run, start_installed_pdpp_connector_run, stop_installed_pdpp_connector_run, + reset_installed_pdpp_browser_profile, + is_installed_pdpp_browser_setup_complete, + submit_installed_pdpp_interaction_response, stop_connector_run, check_connected_platforms, check_browser_available, diff --git a/src-tauri/tauri-config.test.ts b/src-tauri/tauri-config.test.ts index 412747eb..526183ae 100644 --- a/src-tauri/tauri-config.test.ts +++ b/src-tauri/tauri-config.test.ts @@ -3,6 +3,21 @@ import { resolve } from "node:path" import { describe, expect, it } from "vitest" describe("tauri manual-install config", () => { + it("bundles the target-native Node runtime as a Tauri sidecar", () => { + const filePath = resolve(process.cwd(), "src-tauri/tauri.conf.json") + const document = JSON.parse(readFileSync(filePath, "utf-8")) as { + bundle?: { + externalBin?: string[] + resources?: Record + } + } + + expect(document.bundle?.externalBin).toEqual(["binaries/pdpp-node"]) + expect(document.bundle?.resources?.["binaries/pdpp-node-LICENSE"]).toBe( + "licenses/pdpp-node-LICENSE" + ) + }) + it("preserves the packaged Playwright browser directory tree", () => { const filePath = resolve(process.cwd(), "src-tauri/tauri.conf.json") const document = JSON.parse(readFileSync(filePath, "utf-8")) as { @@ -17,6 +32,46 @@ describe("tauri manual-install config", () => { ).toBeUndefined() }) + it("preserves the packaged PDPP runtime directory tree", () => { + const filePath = resolve(process.cwd(), "src-tauri/tauri.conf.json") + const document = JSON.parse(readFileSync(filePath, "utf-8")) as { + bundle?: { resources?: Record } + } + + expect(document.bundle?.resources?.["../pdpp-runtime/"]).toBe( + "pdpp-runtime/" + ) + expect(document.bundle?.resources?.["../pdpp-runtime/**/*"]).toBeUndefined() + }) + + it("bundles the pinned GitHub and ChatGPT PDPP profiles", () => { + const filePath = resolve(process.cwd(), "src-tauri/tauri.conf.json") + const document = JSON.parse(readFileSync(filePath, "utf-8")) as { + bundle?: { resources?: Record } + } + + expect( + document.bundle?.resources?.["../connectors/collection-profiles/"] + ).toBe("connectors/collection-profiles/") + expect( + document.bundle?.resources?.["../connectors/collection-profiles/**/*"] + ).toBeUndefined() + expect(document.bundle?.resources?.["../connectors/lock.json"]).toBe( + "connectors/lock.json" + ) + }) + + it("stages PDPP runtime dependencies before production Tauri packaging", () => { + const filePath = resolve(process.cwd(), "src-tauri/tauri.conf.json") + const document = JSON.parse(readFileSync(filePath, "utf-8")) as { + build?: { beforeBuildCommand?: string } + } + + expect(document.build?.beforeBuildCommand).toContain( + "node scripts/ensure-pdpp-runtime.js" + ) + }) + it("disables updater artifacts and ships no updater endpoint", () => { const filePath = resolve(process.cwd(), "src-tauri/tauri.conf.json") const document = JSON.parse(readFileSync(filePath, "utf-8")) as { diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 131d0050..8c93a500 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -7,7 +7,7 @@ "frontendDist": "../dist", "devUrl": "http://localhost:5173", "beforeDevCommand": "node scripts/ensure-connectors.js && npm run dev", - "beforeBuildCommand": "node scripts/ensure-connectors.js && npm run build" + "beforeBuildCommand": "node scripts/ensure-connectors.js && node scripts/ensure-pdpp-runtime.js && npm run build" }, "app": { "withGlobalTauri": true, @@ -40,6 +40,7 @@ "bundle": { "active": true, "createUpdaterArtifacts": false, + "externalBin": ["binaries/pdpp-node"], "targets": ["dmg", "app", "nsis", "appimage", "deb"], "icon": [ "icons/32x32.png", @@ -59,9 +60,13 @@ "../connectors/oura/**/*": "connectors/oura/", "../connectors/shopify/**/*": "connectors/shopify/", "../connectors/wholefoods/**/*": "connectors/wholefoods/", + "../connectors/collection-profiles/": "connectors/collection-profiles/", + "../connectors/lock.json": "connectors/lock.json", "../playwright-runner/dist/": "playwright-runner/dist/", "../personal-server/dist/personal-server*": "personal-server/dist/", - "../personal-server/dist/node_modules/": "personal-server/dist/node_modules/" + "../personal-server/dist/node_modules/": "personal-server/dist/node_modules/", + "../pdpp-runtime/": "pdpp-runtime/", + "binaries/pdpp-node-LICENSE": "licenses/pdpp-node-LICENSE" }, "linux": { "deb": { diff --git a/src-tauri/tests/fixtures/chatgpt-pdpp-browser.collection-profile.json b/src-tauri/tests/fixtures/chatgpt-pdpp-browser.collection-profile.json new file mode 100644 index 00000000..fc8529b9 --- /dev/null +++ b/src-tauri/tests/fixtures/chatgpt-pdpp-browser.collection-profile.json @@ -0,0 +1,73 @@ +{ + "connector_id": "https://registry.pdpp.org/connectors/chatgpt", + "connector_key": "chatgpt", + "display_name": "ChatGPT", + "version": "0.1.0", + "setup": { + "modality": "static_secret", + "credential_capture": { + "fields": [ + { + "name": "username", + "label": "ChatGPT email", + "type": "email", + "required": true, + "secret": true, + "autocomplete": "username", + "placeholder": "you@example.com", + "env": ["CHATGPT_USERNAME"] + }, + { + "name": "password", + "label": "ChatGPT password", + "type": "password", + "required": true, + "secret": true, + "autocomplete": "current-password", + "env": ["CHATGPT_PASSWORD"] + } + ] + } + }, + "runtime_requirements": { + "bindings": { + "network": { "required": true }, + "browser": { "required": true } + } + }, + "streams": [ + { + "name": "conversations", + "schema": { + "type": "object", + "required": ["id"] + } + }, + { + "name": "messages", + "schema": { + "type": "object", + "required": ["id", "conversation_id"] + } + }, + { + "name": "memories", + "schema": { + "type": "object", + "required": ["id", "content"] + } + }, + { + "name": "custom_gpts", + "schema": { "type": "object", "required": ["id"] } + }, + { + "name": "custom_instructions", + "schema": { "type": "object", "required": ["id"] } + }, + { + "name": "shared_conversations", + "schema": { "type": "object", "required": ["id"] } + } + ] +} diff --git a/src-tauri/tests/fixtures/chatgpt-pdpp-browser.fixture.mjs b/src-tauri/tests/fixtures/chatgpt-pdpp-browser.fixture.mjs new file mode 100644 index 00000000..53e9469a --- /dev/null +++ b/src-tauri/tests/fixtures/chatgpt-pdpp-browser.fixture.mjs @@ -0,0 +1,28 @@ +// Local protocol fixture for the published ChatGPT Collection Profile. It +// verifies the environment-based CDP compatibility seam, not a legacy +// Playwright page API or an invented START binding. +import readline from "node:readline" + +readline.createInterface({ input: process.stdin }).on("line", line => { + const start = JSON.parse(line) + if ("bindings" in start || "browser" in start) { + process.exitCode = 41 + return + } + if (!process.env.PDPP_CHATGPT_REMOTE_CDP_URL?.startsWith("http://127.0.0.1:")) { + process.exitCode = 42 + return + } + process.stdout.write( + `${JSON.stringify({ type: "INTERACTION", request_id: "chatgpt-login", kind: "browser", message: "Sign in to ChatGPT in the opened browser" })}\n` + ) + process.stdout.write( + `${JSON.stringify({ type: "PROGRESS", stream: "conversations", message: "Collecting ChatGPT conversations" })}\n` + ) + process.stdout.write( + `${JSON.stringify({ type: "RECORD", stream: "conversations", key: "fixture-conversation", data: { id: "fixture-conversation" }, emitted_at: "2026-07-31T00:00:00Z" })}\n` + ) + process.stdout.write( + `${JSON.stringify({ type: "DONE", status: "succeeded", records_emitted: 1 })}\n` + ) +}) diff --git a/src-tauri/tests/fixtures/pdpp-connector-fixture.mjs b/src-tauri/tests/fixtures/pdpp-connector-fixture.mjs index 11b34c3a..1a5e7791 100644 --- a/src-tauri/tests/fixtures/pdpp-connector-fixture.mjs +++ b/src-tauri/tests/fixtures/pdpp-connector-fixture.mjs @@ -74,6 +74,16 @@ switch (mode) { message: "Log in", }) break + case "interaction-success": + emit({ + type: "INTERACTION", + request_id: "browser-login", + kind: "browser", + message: "Sign in in the opened browser", + }) + emit(record) + emit(done) + break case "oversized-stdout": process.stdout.write(`${"x".repeat(128)}\n`) break diff --git a/src/hooks/useConnector.test.ts b/src/hooks/useConnector.test.ts index 1585d794..109e7fbb 100644 --- a/src/hooks/useConnector.test.ts +++ b/src/hooks/useConnector.test.ts @@ -115,15 +115,20 @@ describe("useConnector.startImport", () => { }) }) - expect(mockInvoke).toHaveBeenCalledWith("start_installed_pdpp_connector_run", { - request: { - runId: "github-pdpp-1700000000000", - connectorId: "github-pdpp", - collectionMode: "incremental", - streams: [], - githubToken: null, - }, - }) + expect(mockInvoke).toHaveBeenCalledWith( + "start_installed_pdpp_connector_run", + { + request: { + runId: "github-pdpp-1700000000000", + connectorId: "github-pdpp", + collectionMode: "incremental", + streams: [], + githubToken: null, + connectionId: null, + setupSecrets: null, + }, + } + ) expect(mockInvoke).not.toHaveBeenCalledWith( "start_connector_run", expect.anything() @@ -149,15 +154,20 @@ describe("useConnector.startImport", () => { ) }) - expect(mockInvoke).toHaveBeenCalledWith("start_installed_pdpp_connector_run", { - request: { - runId: "github-pdpp-1700000000000", - connectorId: "github-pdpp", - collectionMode: "incremental", - streams: [], - githubToken: "ghp_transient", - }, - }) + expect(mockInvoke).toHaveBeenCalledWith( + "start_installed_pdpp_connector_run", + { + request: { + runId: "github-pdpp-1700000000000", + connectorId: "github-pdpp", + collectionMode: "incremental", + streams: [], + githubToken: "ghp_transient", + connectionId: null, + setupSecrets: null, + }, + } + ) expect(startRun).toHaveBeenCalledWith( expect.not.objectContaining({ githubToken: "ghp_transient" }) ) @@ -193,6 +203,46 @@ describe("useConnector.startImport", () => { resolveHost?.() }) + it("hands ChatGPT static secrets only to the installed PDPP host invoke", async () => { + mockInvoke.mockResolvedValue(undefined) + const { useConnector } = await import("./useConnector") + const { result } = renderHook(() => useConnector()) + + await act(async () => { + await result.current.startImport( + { + ...TEST_PLATFORM, + id: "chatgpt-pdpp", + filename: "chatgpt-pdpp", + runtime: "pdpp-network", + }, + { + setupSecrets: { + username: "owner@example.com", + password: "transient-password", + }, + } + ) + }) + + expect(mockInvoke).toHaveBeenCalledWith( + "start_installed_pdpp_connector_run", + { + request: expect.objectContaining({ + connectorId: "chatgpt-pdpp", + connectionId: "chatgpt-pdpp-owner", + setupSecrets: { + username: "owner@example.com", + password: "transient-password", + }, + }), + } + ) + expect(startRun).toHaveBeenCalledWith( + expect.not.objectContaining({ setupSecrets: expect.anything() }) + ) + }) + it("preserves the legacy connector command for non-PDPP platforms", async () => { mockInvoke.mockResolvedValue(undefined) const { useConnector } = await import("./useConnector") @@ -225,9 +275,15 @@ describe("useConnector.startImport", () => { }) expect(stopRun).not.toHaveBeenCalled() - expect(mockInvoke).toHaveBeenCalledWith("stop_installed_pdpp_connector_run", { - runId: "github-pdpp-run", - }) - expect(mockInvoke).not.toHaveBeenCalledWith("stop_connector_run", expect.anything()) + expect(mockInvoke).toHaveBeenCalledWith( + "stop_installed_pdpp_connector_run", + { + runId: "github-pdpp-run", + } + ) + expect(mockInvoke).not.toHaveBeenCalledWith( + "stop_connector_run", + expect.anything() + ) }) }) diff --git a/src/hooks/useConnector.ts b/src/hooks/useConnector.ts index dbcab466..d71f1b2e 100644 --- a/src/hooks/useConnector.ts +++ b/src/hooks/useConnector.ts @@ -1,31 +1,32 @@ -import { useCallback } from 'react'; -import { invoke } from '@tauri-apps/api/core'; -import { useDispatch, useSelector } from 'react-redux'; -import { deleteRun, startRun, updateRunStatus, stopRun } from '../state/store'; -import type { RootState } from '../state/store'; -import type { Platform, Run } from '../types'; -import { getPlatformRegistryEntry } from '@/lib/platform/utils'; +import { useCallback } from "react" +import { invoke } from "@tauri-apps/api/core" +import { useDispatch, useSelector } from "react-redux" +import { deleteRun, startRun, updateRunStatus, stopRun } from "../state/store" +import type { RootState } from "../state/store" +import type { Platform, Run } from "../types" +import { getPlatformRegistryEntry } from "@/lib/platform/utils" import { trackCollectionFailed, trackCollectionStarted, -} from '@/lib/telemetry/events'; -import { durationSince } from '@/lib/telemetry/client'; +} from "@/lib/telemetry/events" +import { durationSince } from "@/lib/telemetry/client" -const DUPLICATE_ACTIVE_RUN_ERROR_CODE = 'DUPLICATE_ACTIVE_RUN'; -const PDPP_NETWORK_RUNTIME = 'pdpp-network'; +const DUPLICATE_ACTIVE_RUN_ERROR_CODE = "DUPLICATE_ACTIVE_RUN" +const PDPP_NETWORK_RUNTIME = "pdpp-network" interface StartImportOptions { - githubToken?: string | null; + githubToken?: string | null + setupSecrets?: { username: string; password: string } | null } function isDuplicateStartError(error: unknown): boolean { const message = - typeof error === 'string' + typeof error === "string" ? error : error instanceof Error ? error.message - : String(error); - return message.includes(DUPLICATE_ACTIVE_RUN_ERROR_CODE); + : String(error) + return message.includes(DUPLICATE_ACTIVE_RUN_ERROR_CODE) } async function startInstalledPdppConnectorRun( @@ -33,26 +34,30 @@ async function startInstalledPdppConnectorRun( platform: Platform, options: StartImportOptions = {} ): Promise { - await invoke('start_installed_pdpp_connector_run', { + await invoke("start_installed_pdpp_connector_run", { request: { runId, connectorId: platform.id, - collectionMode: 'incremental', + collectionMode: "incremental", streams: [], githubToken: - platform.id === 'github-pdpp' ? options.githubToken ?? null : null, + platform.id === "github-pdpp" ? (options.githubToken ?? null) : null, + connectionId: + platform.id === "chatgpt-pdpp" ? "chatgpt-pdpp-owner" : null, + setupSecrets: + platform.id === "chatgpt-pdpp" ? (options.setupSecrets ?? null) : null, }, - }); + }) } export function useConnector() { - const dispatch = useDispatch(); - const runs = useSelector((state: RootState) => state.app.runs); + const dispatch = useDispatch() + const runs = useSelector((state: RootState) => state.app.runs) const startImport = useCallback( async (platform: Platform, options: StartImportOptions = {}) => { - const runId = `${platform.id}-${Date.now()}`; - const source = getPlatformRegistryEntry(platform)?.id ?? platform.id; + const runId = `${platform.id}-${Date.now()}` + const source = getPlatformRegistryEntry(platform)?.id ?? platform.id const newRun: Run = { id: runId, @@ -61,117 +66,124 @@ export function useConnector() { runtime: platform.runtime, isConnected: false, startDate: new Date().toISOString(), - status: 'running', - url: platform.connectURL || '', + status: "running", + url: platform.connectURL || "", company: platform.company, name: platform.name, - logs: '', - }; + logs: "", + } - dispatch(startRun(newRun)); + dispatch(startRun(newRun)) trackCollectionStarted({ collectionRunId: runId, source, - authMode: 'interactive', - }); + authMode: "interactive", + }) // The installed PDPP host streams progress and terminal state through // `connector-status`. Its command response arrives only after the // subprocess finishes, so waiting here would leave /connect without a // run id (and therefore without live busy/progress UI). if (platform.runtime === PDPP_NETWORK_RUNTIME) { - void startInstalledPdppConnectorRun(runId, platform, options).catch((error) => { - if (isDuplicateStartError(error)) { - dispatch(deleteRun(runId)); - return; + void startInstalledPdppConnectorRun(runId, platform, options).catch( + error => { + if (isDuplicateStartError(error)) { + dispatch(deleteRun(runId)) + return + } + + console.error( + "Failed to start installed PDPP connector run:", + error + ) + // A host-side failure normally emits its own terminal event. This + // only closes the UI state when no event has already done so. + dispatch( + updateRunStatus({ + runId, + status: "error", + endDate: new Date().toISOString(), + onlyIfRunning: true, + }) + ) } - - console.error('Failed to start installed PDPP connector run:', error); - // A host-side failure normally emits its own terminal event. This - // only closes the UI state when no event has already done so. - dispatch( - updateRunStatus({ - runId, - status: 'error', - endDate: new Date().toISOString(), - onlyIfRunning: true, - }) - ); - }); - return runId; + ) + return runId } try { const simulateNoChrome = - typeof window !== 'undefined' && window.localStorage?.getItem?.('dataconnect_simulate_no_chrome') === 'true'; + typeof window !== "undefined" && + window.localStorage?.getItem?.("dataconnect_simulate_no_chrome") === + "true" - await invoke('start_connector_run', { + await invoke("start_connector_run", { runId, platformId: platform.id, filename: platform.filename, company: platform.company, name: platform.name, - connectUrl: platform.connectURL || '', + connectUrl: platform.connectURL || "", runtime: platform.runtime || null, simulateNoChrome, - }); + }) } catch (error) { if (isDuplicateStartError(error)) { - dispatch(deleteRun(runId)); - return null; + dispatch(deleteRun(runId)) + return null } - console.error('Failed to start connector run:', error); + console.error("Failed to start connector run:", error) dispatch( updateRunStatus({ runId, - status: 'error', + status: "error", endDate: new Date().toISOString(), }) - ); + ) trackCollectionFailed({ collectionRunId: runId, source, durationMs: durationSince(newRun.startDate), error, - }); + }) } - return runId; + return runId }, [dispatch] - ); + ) const stopExport = useCallback( async (runId: string) => { try { - const run = runs.find(candidate => candidate.id === runId); + const run = runs.find(candidate => candidate.id === runId) if (run?.runtime === PDPP_NETWORK_RUNTIME) { - await invoke('stop_installed_pdpp_connector_run', { runId }); + await invoke("stop_installed_pdpp_connector_run", { runId }) // The PDPP host emits STOPPED after its subprocess is actually // reaped. Keep the source card active until then. } else { - dispatch(stopRun(runId)); - await invoke('stop_connector_run', { runId }); + dispatch(stopRun(runId)) + await invoke("stop_connector_run", { runId }) } } catch (error) { - console.log('Stop connector run (window may be closed):', error); + console.log("Stop connector run (window may be closed):", error) } }, [dispatch, runs] - ); + ) const getRunById = useCallback( (runId: string) => { - return runs.find((r) => r.id === runId); + return runs.find(r => r.id === runId) }, [runs] - ); + ) return { runs, startImport, stopExport, getRunById, - }; + } } diff --git a/src/pages/data-apps/components/registry-app-card.test.tsx b/src/pages/data-apps/components/registry-app-card.test.tsx index b880ffe5..3b43f0e3 100644 --- a/src/pages/data-apps/components/registry-app-card.test.tsx +++ b/src/pages/data-apps/components/registry-app-card.test.tsx @@ -1,4 +1,10 @@ -import { cleanup, render, screen, waitFor } from "@testing-library/react" +import { + cleanup, + fireEvent, + render, + screen, + waitFor, +} from "@testing-library/react" import { createMemoryRouter, RouterProvider } from "react-router-dom" import { afterEach, describe, expect, it, vi } from "vitest" import { RegistryAppCard } from "./registry-app-card" @@ -180,7 +186,7 @@ describe("RegistryAppCard", () => { }) expect(screen.getByText("Uses PDPP")).toBeTruthy() - screen.getByRole("button", { name: "Open Timeline" }).click() + fireEvent.click(screen.getByRole("button", { name: "Open Timeline" })) await waitFor(() => { expect(router.state.location.pathname).toBe("/apps/timeline") diff --git a/src/pages/home/components/connected-sources-list.tsx b/src/pages/home/components/connected-sources-list.tsx index c2cb0ea1..fec7258a 100644 --- a/src/pages/home/components/connected-sources-list.tsx +++ b/src/pages/home/components/connected-sources-list.tsx @@ -25,6 +25,7 @@ interface ConnectedSourcesListProps { headline?: string onOpenRuns?: (platform: Platform) => void onSyncSource?: (platform: Platform) => void + onReconnectSource?: (platform: Platform) => void } type OnboardingMessageState = "empty" | "early" | "mature" @@ -44,6 +45,7 @@ export function ConnectedSourcesList({ headline = "Your sources at the moment.", onOpenRuns, onSyncSource, + onReconnectSource, }: ConnectedSourcesListProps) { const inFlightSyncPlatformIdsRef = useRef>(new Set()) const syncFeedbackTimeoutsRef = useRef< @@ -166,6 +168,10 @@ export function ConnectedSourcesList({ hasBlockingRun || hasActiveRun || isShowingSyncFeedback + const canReconnect = + platform.id === "chatgpt-pdpp" && + Boolean(onReconnectSource) && + !hasActiveRun const syncTooltipCopy = hasActiveRun || syncFeedbackState === "backgrounding" ? "Fetching in background" @@ -184,41 +190,59 @@ export function ConnectedSourcesList({ ariaLabel: `Open ${platform.name}`, }} middleSlot={ - - - triggerSyncFeedback(platform) - : undefined - } - disabled={isSyncDisabled} - aria-label={`Fetch latest data for ${platform.name}`} - > - {syncFeedbackState ? ( - + {canReconnect ? ( + + + onReconnectSource?.(platform)} + aria-label={`Reconnect ${platform.name}`} > - {syncFeedbackState === "running" - ? "Fetching…" - : "Backgrounding…"} - - ) : null} - - - - {syncTooltipCopy} - + + + + + Reset this browser session and reconnect + + + ) : null} + + + triggerSyncFeedback(platform) + : undefined + } + disabled={isSyncDisabled} + aria-label={`Fetch latest data for ${platform.name}`} + > + {syncFeedbackState ? ( + + {syncFeedbackState === "running" + ? "Fetching…" + : "Backgrounding…"} + + ) : null} + + + + {syncTooltipCopy} + + } endSlotClassName="[&_svg:not([class*='size-']):not([data-slot=spinner])]:size-7!" surface="list-item" diff --git a/src/pages/home/index.test.tsx b/src/pages/home/index.test.tsx index b83d019e..801c41ba 100644 --- a/src/pages/home/index.test.tsx +++ b/src/pages/home/index.test.tsx @@ -1,6 +1,17 @@ import { describe, expect, it, vi, beforeEach, afterEach } from "vitest" -import { render, waitFor, cleanup, fireEvent, screen } from "@testing-library/react" -import { createMemoryRouter, MemoryRouter, RouterProvider } from "react-router-dom" +import { + render, + act, + waitFor, + cleanup, + fireEvent, + screen, +} from "@testing-library/react" +import { + createMemoryRouter, + MemoryRouter, + RouterProvider, +} from "react-router-dom" import { ROUTES } from "@/config/routes" import { TooltipProvider } from "@/components/ui/tooltip" import { Home } from "./index" @@ -10,6 +21,8 @@ const mockStartImport = vi.fn() const mockStopExport = vi.fn() const mockNavigate = vi.fn() const mockRefreshConnectedStatus = vi.fn() +const mockInvoke = vi.fn() +const mockListen = vi.fn() let mockConnectedPlatforms: Record = {} let mockRuns: Array<{ id: string @@ -27,9 +40,8 @@ let mockRuns: Array<{ }> = [] vi.mock("react-router-dom", async () => { - const actual = await vi.importActual( - "react-router-dom" - ) + const actual = + await vi.importActual("react-router-dom") return { ...actual, useNavigate: () => mockNavigate, @@ -40,6 +52,14 @@ vi.mock("@/hooks/usePlatforms", () => ({ usePlatforms: () => mockUsePlatforms(), })) +vi.mock("@tauri-apps/api/core", () => ({ + invoke: (...args: unknown[]) => mockInvoke(...args), +})) + +vi.mock("@tauri-apps/api/event", () => ({ + listen: (...args: unknown[]) => mockListen(...args), +})) + vi.mock("@/hooks/useConnector", () => ({ useConnector: () => ({ startImport: mockStartImport, @@ -103,6 +123,10 @@ describe("Home", () => { mockStopExport.mockReset() mockNavigate.mockReset() mockRefreshConnectedStatus.mockReset() + mockInvoke.mockReset() + mockInvoke.mockResolvedValue(false) + mockListen.mockReset() + mockListen.mockResolvedValue(() => undefined) mockConnectedPlatforms = {} mockRuns = [] mockStartImport.mockResolvedValue("run-1") @@ -125,17 +149,13 @@ describe("Home", () => { const { getByRole } = renderHome() expect(getByRole("heading", { level: 1, name: /your data/i })).toBeTruthy() - expect( - getByRole("heading", { name: /your imported data/i }) - ).toBeTruthy() + expect(getByRole("heading", { name: /your imported data/i })).toBeTruthy() }) it("does not render the connected apps tab or surface", () => { const { container } = renderHome() - expect( - screen.queryByRole("tab", { name: /connected apps/i }) - ).toBeNull() + expect(screen.queryByRole("tab", { name: /connected apps/i })).toBeNull() expect( container.querySelector('[data-component="connected-apps-list"]') ).toBeNull() @@ -241,6 +261,225 @@ describe("Home", () => { expect(screen.queryByLabelText(/personal access token/i)).toBeNull() }) + it("captures the manifest-declared ChatGPT static secrets before launching", async () => { + mockUsePlatforms.mockReturnValue({ + platforms: [ + { + id: "chatgpt-pdpp", + company: "OpenAI", + name: "ChatGPT", + filename: "chatgpt-pdpp", + description: "ChatGPT PDPP export", + isUpdated: false, + logoURL: "", + needsConnection: true, + connectURL: null, + connectSelector: null, + exportFrequency: null, + vectorize_config: null, + runtime: "pdpp-network", + setup: { + modality: "static_secret", + credentialCapture: { fields: [] }, + }, + }, + ], + connectedPlatforms: {}, + loadPlatforms: vi.fn(), + refreshConnectedStatus: vi.fn(), + getPlatformById: vi.fn(), + isPlatformConnected: vi.fn(() => false), + }) + + renderHome() + fireEvent.click(screen.getByRole("button", { name: /connect chatgpt/i })) + await waitFor(() => { + expect(screen.getByLabelText(/chatgpt email/i)).toBeTruthy() + }) + expect(mockStartImport).not.toHaveBeenCalled() + fireEvent.change(screen.getByLabelText(/chatgpt email/i), { + target: { value: "owner@example.com" }, + }) + fireEvent.change(screen.getByLabelText(/chatgpt password/i), { + target: { value: "transient-password" }, + }) + fireEvent.click( + screen.getByRole("button", { + name: /start owner-attended sync/i, + }) + ) + + await waitFor(() => { + expect(mockStartImport).toHaveBeenCalledWith( + expect.objectContaining({ id: "chatgpt-pdpp" }), + { + setupSecrets: { + username: "owner@example.com", + password: "transient-password", + }, + } + ) + }) + expect(screen.queryByLabelText(/chatgpt password/i)).toBeNull() + }) + + it("reuses a completed owner profile without asking for static secrets again", async () => { + mockInvoke.mockResolvedValue(true) + mockUsePlatforms.mockReturnValue({ + platforms: [ + { + id: "chatgpt-pdpp", + company: "OpenAI", + name: "ChatGPT", + filename: "chatgpt-pdpp", + description: "ChatGPT PDPP export", + isUpdated: false, + logoURL: "", + needsConnection: true, + connectURL: null, + connectSelector: null, + exportFrequency: null, + vectorize_config: null, + runtime: "pdpp-network", + setup: { + modality: "static_secret", + credentialCapture: { fields: [] }, + }, + }, + ], + connectedPlatforms: {}, + loadPlatforms: vi.fn(), + refreshConnectedStatus: vi.fn(), + getPlatformById: vi.fn(), + isPlatformConnected: vi.fn(() => false), + }) + + renderHome() + fireEvent.click(screen.getByRole("button", { name: /connect chatgpt/i })) + + await waitFor(() => { + expect(mockInvoke).toHaveBeenCalledWith( + "is_installed_pdpp_browser_setup_complete", + { connectorId: "chatgpt-pdpp", connectionId: "chatgpt-pdpp-owner" } + ) + expect(mockStartImport).toHaveBeenCalledWith( + expect.objectContaining({ id: "chatgpt-pdpp" }) + ) + }) + expect(screen.queryByLabelText(/chatgpt password/i)).toBeNull() + expect(mockStartImport).not.toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ setupSecrets: expect.anything() }) + ) + }) + + it("resets an expired ChatGPT session and returns the owner to setup", async () => { + mockInvoke.mockImplementation(command => + command === "reset_installed_pdpp_browser_profile" + ? Promise.resolve(undefined) + : Promise.resolve(false) + ) + mockUsePlatforms.mockReturnValue({ + platforms: [ + { + id: "chatgpt-pdpp", + company: "OpenAI", + name: "ChatGPT", + filename: "chatgpt-pdpp", + description: "ChatGPT PDPP export", + isUpdated: false, + logoURL: "", + needsConnection: true, + connectURL: null, + connectSelector: null, + exportFrequency: null, + vectorize_config: null, + runtime: "pdpp-network", + setup: { + modality: "static_secret", + credentialCapture: { fields: [] }, + }, + }, + ], + connectedPlatforms: { "chatgpt-pdpp": true }, + loadPlatforms: vi.fn(), + refreshConnectedStatus: vi.fn(), + getPlatformById: vi.fn(), + isPlatformConnected: vi.fn(() => true), + }) + + renderHome() + fireEvent.click(screen.getByRole("button", { name: "Reconnect ChatGPT" })) + + await waitFor(() => { + expect(mockInvoke).toHaveBeenCalledWith( + "reset_installed_pdpp_browser_profile", + { connectorId: "chatgpt-pdpp", connectionId: "chatgpt-pdpp-owner" } + ) + expect(mockInvoke).toHaveBeenCalledWith( + "is_installed_pdpp_browser_setup_complete", + { connectorId: "chatgpt-pdpp", connectionId: "chatgpt-pdpp-owner" } + ) + expect(screen.getByLabelText(/chatgpt email/i)).toBeTruthy() + }) + expect(mockStartImport).not.toHaveBeenCalled() + }) + + it("submits a transient code response for the exact no-schema ChatGPT OTP interaction", async () => { + let deliverInteraction: ((event: { payload: unknown }) => void) | undefined + mockListen.mockImplementation((_event, callback) => { + deliverInteraction = callback as (event: { payload: unknown }) => void + return Promise.resolve(() => undefined) + }) + mockInvoke.mockResolvedValue(undefined) + mockRuns = [ + { + id: "chatgpt-pdpp-run-1", + platformId: "chatgpt-pdpp", + filename: "chatgpt-pdpp", + isConnected: false, + startDate: new Date().toISOString(), + status: "running", + url: "", + company: "OpenAI", + name: "ChatGPT", + logs: "", + }, + ] + renderHome() + await waitFor(() => expect(deliverInteraction).toBeTypeOf("function")) + + await act(async () => { + deliverInteraction?.({ + payload: { + runId: "chatgpt-pdpp-run-1", + requestId: "otp-1", + kind: "otp", + message: "Enter your verification code", + }, + }) + }) + + fireEvent.change(screen.getByLabelText(/verification code/i), { + target: { value: "123456" }, + }) + fireEvent.click(screen.getByRole("button", { name: "Continue" })) + await waitFor(() => { + expect(mockInvoke).toHaveBeenCalledWith( + "submit_installed_pdpp_interaction_response", + { + runId: "chatgpt-pdpp-run-1", + requestId: "otp-1", + status: "success", + data: { code: "123456" }, + } + ) + }) + await waitFor(() => { + expect(screen.queryByLabelText(/verification code/i)).toBeNull() + }) + }) + it("cancels the GitHub PAT prompt without starting import", () => { mockUsePlatforms.mockReturnValue({ platforms: [ @@ -351,7 +590,9 @@ describe("Home", () => { expect( screen.queryByRole("button", { name: /connect chatgpt/i }) ).toBeNull() - expect(screen.getAllByRole("button", { name: /open chatgpt/i }).length).toBeGreaterThan(0) + expect( + screen.getAllByRole("button", { name: /open chatgpt/i }).length + ).toBeGreaterThan(0) }) it("shows connected source from persisted run even when connected status map is stale", async () => { @@ -400,7 +641,9 @@ describe("Home", () => { expect( screen.queryByRole("button", { name: /connect chatgpt/i }) ).toBeNull() - expect(screen.getAllByRole("button", { name: /open chatgpt/i }).length).toBeGreaterThan(0) + expect( + screen.getAllByRole("button", { name: /open chatgpt/i }).length + ).toBeGreaterThan(0) }) it.each([ @@ -553,7 +796,8 @@ describe("Home", () => { company: "OpenAI", name: "ChatGPT", logs: "", - exportPath: "/tmp/dataconnect/exported_data/OpenAI/ChatGPT/run-chatgpt-1", + exportPath: + "/tmp/dataconnect/exported_data/OpenAI/ChatGPT/run-chatgpt-1", }, ] @@ -629,9 +873,11 @@ describe("Home", () => { renderHome() expect( - screen.getByRole("button", { - name: /fetch latest data for chatgpt/i, - }).hasAttribute("disabled") + screen + .getByRole("button", { + name: /fetch latest data for chatgpt/i, + }) + .hasAttribute("disabled") ).toBe(true) }) @@ -828,7 +1074,9 @@ describe("Home", () => { renderHome() - const spotifyButton = screen.getByRole("button", { name: /connect spotify/i }) + const spotifyButton = screen.getByRole("button", { + name: /connect spotify/i, + }) expect(spotifyButton.hasAttribute("disabled")).toBe(false) fireEvent.click(spotifyButton) diff --git a/src/pages/home/index.tsx b/src/pages/home/index.tsx index 737be3e1..5c227747 100644 --- a/src/pages/home/index.tsx +++ b/src/pages/home/index.tsx @@ -7,6 +7,8 @@ import { useState, } from "react" import { useLocation, useNavigate } from "react-router-dom" +import { invoke } from "@tauri-apps/api/core" +import { listen } from "@tauri-apps/api/event" import { useSelector } from "react-redux" import { usePlatforms } from "@/hooks/usePlatforms" import { useConnector } from "@/hooks/useConnector" @@ -46,6 +48,14 @@ import { resolveHomeImportSourcesUiDebugState, } from "./home-import-sources-ui-debug" +type PendingPdppInteraction = { + runId: string + requestId: string + kind: string + message: string + schema?: { properties?: Record } | null +} + export function Home() { const homeDebugScenarioLabel: Record = { "blocking-waiting": "blocking-waiting", @@ -67,6 +77,21 @@ export function Home() { const [githubTokenDialogPlatform, setGithubTokenDialogPlatform] = useState(null) const [githubTokenInput, setGithubTokenInput] = useState("") + const [chatgptSetupDialogPlatform, setChatgptSetupDialogPlatform] = + useState(null) + const [chatgptUsernameInput, setChatgptUsernameInput] = useState("") + const [chatgptPasswordInput, setChatgptPasswordInput] = useState("") + const [pendingInteraction, setPendingInteraction] = + useState(null) + const [interactionInput, setInteractionInput] = useState("") + const chatgptSetupFields = + chatgptSetupDialogPlatform?.setup?.credentialCapture.fields ?? [] + const chatgptUsernameField = chatgptSetupFields.find( + field => field.name === "username" + ) + const chatgptPasswordField = chatgptSetupFields.find( + field => field.name === "password" + ) const knownSuccessfulRunIdsRef = useRef | null>(null) const homeUiDebugEnabled = useMemo( () => isHomeImportSourcesDebugEnabled(location.search), @@ -108,13 +133,42 @@ export function Home() { } }, [refreshConnectedStatus, runs]) + useEffect(() => { + let unlisten: (() => void) | undefined + void listen("pdpp-interaction", event => { + setInteractionInput("") + setPendingInteraction(event.payload) + }).then(listener => { + unlisten = listener + }) + return () => unlisten?.() + }, []) + + useEffect(() => { + if ( + pendingInteraction && + !runs.some( + run => run.id === pendingInteraction.runId && run.status === "running" + ) + ) { + setInteractionInput("") + setPendingInteraction(null) + } + }, [pendingInteraction, runs]) + const runImportSource = useCallback( - async (platform: Platform, githubToken?: string) => { + async ( + platform: Platform, + options?: { + githubToken?: string + setupSecrets?: { username: string; password: string } + } + ) => { try { - if (githubToken === undefined) { + if (options === undefined) { await startImport(platform) } else { - await startImport(platform, { githubToken }) + await startImport(platform, options) } } catch (error) { console.error("Import failed:", error) @@ -130,6 +184,32 @@ export function Home() { setGithubTokenDialogPlatform(platform) return } + if ( + platform.id === "chatgpt-pdpp" && + platform.setup?.modality === "static_secret" + ) { + void invoke("is_installed_pdpp_browser_setup_complete", { + connectorId: platform.id, + connectionId: "chatgpt-pdpp-owner", + }) + .then(setupComplete => { + if (setupComplete) { + void runImportSource(platform) + return + } + setChatgptUsernameInput("") + setChatgptPasswordInput("") + setChatgptSetupDialogPlatform(platform) + }) + // A missing marker is the safe fallback for a failed or older host: + // show first-setup recovery rather than accidentally sending no auth. + .catch(() => { + setChatgptUsernameInput("") + setChatgptPasswordInput("") + setChatgptSetupDialogPlatform(platform) + }) + return + } void runImportSource(platform) }, @@ -149,7 +229,7 @@ export function Home() { if (!platform || !githubToken) return closeGithubTokenDialog() - void runImportSource(platform, githubToken) + void runImportSource(platform, { githubToken }) }, [ closeGithubTokenDialog, @@ -159,6 +239,34 @@ export function Home() { ] ) + const closeChatgptSetupDialog = useCallback(() => { + setChatgptSetupDialogPlatform(null) + setChatgptUsernameInput("") + setChatgptPasswordInput("") + }, []) + + const submitChatgptSetup = useCallback( + (event: FormEvent) => { + event.preventDefault() + const platform = chatgptSetupDialogPlatform + const username = chatgptUsernameInput.trim() + const password = chatgptPasswordInput + if (!platform || !username || !password) return + + closeChatgptSetupDialog() + void runImportSource(platform, { + setupSecrets: { username, password }, + }) + }, + [ + chatgptPasswordInput, + chatgptSetupDialogPlatform, + chatgptUsernameInput, + closeChatgptSetupDialog, + runImportSource, + ] + ) + const handleStopImport = useCallback( async (runId: string) => { try { @@ -170,6 +278,54 @@ export function Home() { [stopExport] ) + const handleReconnectSource = useCallback( + async (platform: Platform) => { + if (platform.id !== "chatgpt-pdpp") return + try { + await invoke("reset_installed_pdpp_browser_profile", { + connectorId: platform.id, + connectionId: "chatgpt-pdpp-owner", + }) + // The reset clears the non-secret setup marker. Re-enter the normal + // setup gate so an expired session gets owner-attended recovery. + handleImportSource(platform) + } catch (error) { + console.error("Failed to reset ChatGPT browser session:", error) + } + }, + [handleImportSource] + ) + + const respondToPendingInteraction = useCallback( + async (status: "success" | "cancelled") => { + const interaction = pendingInteraction + if (!interaction) return + const fields = Object.keys(interaction.schema?.properties ?? {}) + const requiresVerificationCode = interaction.kind === "otp" + const data = + status === "success" && requiresVerificationCode + ? { code: interactionInput } + : status === "success" && fields.length > 0 + ? { [fields[0]]: interactionInput } + : undefined + try { + await invoke("submit_installed_pdpp_interaction_response", { + runId: interaction.runId, + requestId: interaction.requestId, + status, + data, + }) + } catch (error) { + console.error("Failed to submit connector interaction:", error) + } finally { + // OTP/recovery input is never promoted into Redux, logs, or storage. + setInteractionInput("") + setPendingInteraction(null) + } + }, + [interactionInput, pendingInteraction] + ) + const handleTestDeepLink = useCallback(() => { const trimmed = deepLinkInput.trim() if (!trimmed) return @@ -303,6 +459,7 @@ export function Home() { headline="Your imported data" onOpenRuns={handleOpenRuns} onSyncSource={handleImportSource} + onReconnectSource={handleReconnectSource} /> + { + if (!open) closeChatgptSetupDialog() + }} + > + +
+ + + Connect ChatGPT + + + Use these only for initial setup or owner-mediated recovery. + DataConnect passes them only to this run and does not save them. + + +
+ + setChatgptUsernameInput(event.target.value)} + autoFocus + /> +
+
+ + setChatgptPasswordInput(event.target.value)} + /> +
+ + + Cancel + + + +
+
+
+ + + + + + ChatGPT needs your attention + + + {pendingInteraction?.message} + + + {pendingInteraction?.kind === "otp" || + Object.keys(pendingInteraction?.schema?.properties ?? {}).length > 0 ? ( +
+ + setInteractionInput(event.target.value)} + autoFocus + /> +
+ ) : null} + + void respondToPendingInteraction("cancelled")} + > + Cancel run + + + +
+
+ {import.meta.env.DEV && (
diff --git a/src/types/index.ts b/src/types/index.ts index 903f821c..1124a7a8 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,177 +1,193 @@ export interface Platform { - id: string; - company: string; - name: string; - filename: string; - description: string; - isUpdated: boolean; - logoURL: string; - needsConnection: boolean; - connectURL: string | null; - connectSelector: string | null; - exportFrequency: string | null; - vectorize_config: Record | null; + id: string + company: string + name: string + filename: string + description: string + isUpdated: boolean + logoURL: string + needsConnection: boolean + connectURL: string | null + connectSelector: string | null + exportFrequency: string | null + vectorize_config: Record | null /** Runtime type: "vanilla" (default) or "network-capture" (uses network interception) */ - runtime?: string | null; + runtime?: string | null /** Scopes this connector can export (e.g. ["chatgpt.conversations", "chatgpt.memories"]) */ - scopes?: string[] | null; + scopes?: string[] | null + /** Exact static-secret setup shape currently supported for chatgpt-pdpp. */ + setup?: PdppStaticSecretSetup | null +} + +export interface PdppStaticSecretSetup { + modality: "static_secret" + credentialCapture: { + fields: Array<{ + name: "username" | "password" + label?: string | null + type?: "email" | "password" | string | null + required: true + secret: true + autocomplete?: string | null + }> + } } export interface ProgressPhase { - step: number; - total: number; - label: string; + step: number + total: number + label: string } export interface Run { - id: string; - platformId: string; - filename: string; - runtime?: string | null; - isConnected: boolean; - startDate: string; - endDate?: string; - status: 'pending' | 'running' | 'success' | 'partial' | 'error' | 'stopped'; - url: string; - exportSize?: number; - exportPath?: string; - company: string; - name: string; - currentStep?: string; - logs?: string; - statusMessage?: string; - itemsExported?: number; - itemLabel?: string; // e.g., "posts", "conversations" - exportData?: ExportedData; + id: string + platformId: string + filename: string + runtime?: string | null + isConnected: boolean + startDate: string + endDate?: string + status: "pending" | "running" | "success" | "partial" | "error" | "stopped" + url: string + exportSize?: number + exportPath?: string + company: string + name: string + currentStep?: string + logs?: string + statusMessage?: string + itemsExported?: number + itemLabel?: string // e.g., "posts", "conversations" + exportData?: ExportedData // Progress tracking - phase?: ProgressPhase; - itemCount?: number; // Real-time count during collection + phase?: ProgressPhase + itemCount?: number // Real-time count during collection // Sync status - syncedToPersonalServer?: boolean; - scope?: string; + syncedToPersonalServer?: boolean + scope?: string } export interface ExportedData { - platform: string; - company: string; - exportedAt: string; - userInfo?: { name?: string; email?: string }; + platform: string + company: string + exportedAt: string + userInfo?: { name?: string; email?: string } conversations?: Array<{ - id: string; - title: string; - url: string; - scrapedAt: string; - }>; - totalConversations?: number; + id: string + title: string + url: string + scrapedAt: string + }> + totalConversations?: number } export interface AppState { - route: string; - activeRunIndex: number; - isFullScreen: boolean; - isMac: boolean; - isRunLayerVisible: boolean; - breadcrumb: { text: string; link: string }[]; - runs: Run[]; - platforms: Platform[]; - connectedPlatforms: Record; - connectorUpdates: ConnectorUpdateInfo[]; - lastUpdateCheck: string | null; - isCheckingUpdates: boolean; - auth: AuthState; - connectedApps: ConnectedApp[]; - appConfig: AppConfig; + route: string + activeRunIndex: number + isFullScreen: boolean + isMac: boolean + isRunLayerVisible: boolean + breadcrumb: { text: string; link: string }[] + runs: Run[] + platforms: Platform[] + connectedPlatforms: Record + connectorUpdates: ConnectorUpdateInfo[] + lastUpdateCheck: string | null + isCheckingUpdates: boolean + auth: AuthState + connectedApps: ConnectedApp[] + appConfig: AppConfig } export interface RootState { - app: AppState; + app: AppState } export interface ConnectorLogEvent { - runId: string; - message: string; - timestamp: number; + runId: string + message: string + timestamp: number } export interface ConnectorStatusPayload { - type: string; - message: string; - phase?: ProgressPhase; - count?: number; - data?: unknown; - outcome?: 'success' | 'partial' | 'failure' | 'cancelled'; - errorClass?: string; - recordCount?: number; + type: string + message: string + phase?: ProgressPhase + count?: number + data?: unknown + outcome?: "success" | "partial" | "failure" | "cancelled" + errorClass?: string + recordCount?: number scopeSummary?: { - requested: number; - produced: number; - degraded: number; - omitted: number; - }; + requested: number + produced: number + degraded: number + omitted: number + } } export interface ConnectorStatusEvent { - runId: string; - status: ConnectorStatusPayload; - timestamp: number; + runId: string + status: ConnectorStatusPayload + timestamp: number } export interface DownloadProgressEvent { - run_id: string; - filename: string; - percent: number; - bytes_downloaded: number; - total_bytes: number | null; + run_id: string + filename: string + percent: number + bytes_downloaded: number + total_bytes: number | null } export interface ExportCompleteEvent { - company: string; - name: string; - run_id: string; - export_path: string; - export_size: number; + company: string + name: string + run_id: string + export_path: string + export_size: number } export interface ConnectorUpdateInfo { - id: string; - name: string; - description: string; - company: string; - currentVersion: string | null; - latestVersion: string; - hasUpdate: boolean; - isNew: boolean; + id: string + name: string + description: string + company: string + currentVersion: string | null + latestVersion: string + hasUpdate: boolean + isNew: boolean } export interface AuthState { - isAuthenticated: boolean; - isLoading: boolean; - user: AuthUser | null; - walletAddress: string | null; - masterKeySignature: string | null; - accountRole: 'standard' | 'debug'; + isAuthenticated: boolean + isLoading: boolean + user: AuthUser | null + walletAddress: string | null + masterKeySignature: string | null + accountRole: "standard" | "debug" } export interface AuthUser { - id: string; - email?: string; + id: string + email?: string wallet?: { - address: string; - walletClientType: string; - }; + address: string + walletClientType: string + } } export interface ConnectedApp { - id: string; - name: string; - icon?: string; - permissions: string[]; - connectedAt: string; - externalUrl?: string; + id: string + name: string + icon?: string + permissions: string[] + connectedAt: string + externalUrl?: string } export interface AppConfig { - storageProvider: 'local' | 'vana' | 'gdrive' | 'dropbox'; + storageProvider: "local" | "vana" | "gdrive" | "dropbox" /** * - `local-only` (default): no backend provider is configured. Exports * land on local disk only (`write_export_data`, always unconditional) @@ -186,18 +202,18 @@ export interface AppConfig { * user obtains the access token via the "Connect with Vana" Hydra * device-code flow. */ - serverMode: 'local-only' | 'local' | 'remote'; + serverMode: "local-only" | "local" | "remote" /** Required when `serverMode === 'remote'`. e.g. `https://0xabc….myvana.app`. */ - remoteServerUrl?: string; + remoteServerUrl?: string /** * Vana session access token for the remote PS. Stored in plaintext for * dev simplicity; production should move to Tauri Stronghold or OS * Keychain. Refreshed on demand via the refresh token at * `vanaRefreshToken`. */ - vanaAccessToken?: string; + vanaAccessToken?: string /** Vana session refresh token. */ - vanaRefreshToken?: string; + vanaRefreshToken?: string /** Unix epoch (seconds) when `vanaAccessToken` expires. */ - vanaAccessTokenExpiresAt?: number; + vanaAccessTokenExpiresAt?: number } diff --git a/vite.config.ts b/vite.config.ts index b5da2f7b..418880da 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -22,7 +22,10 @@ export default defineConfig({ "src-tauri/**/*.test.ts", "scripts/is-main-module.test.mjs", "scripts/create-macos-dmg.test.mjs", + "scripts/ensure-pdpp-runtime.test.mjs", + "scripts/release-github.test.mjs", "scripts/release-workflow.test.ts", + "scripts/stage-pdpp-node.test.mjs", "scripts/resolve-connectors.test.mjs", "scripts/verify-release-ref.test.mjs", "scripts/verify-bundled-personal-server.test.mjs",