From 95c73de4c4ceb8d9fd6350038753139828b526d7 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 31 Jul 2026 12:54:42 -0500 Subject: [PATCH 01/14] feat(pdpp): host ChatGPT browser artifact Signed-off-by: Tim Nunamaker Assisted-by: AI --- .gitignore | 1 + package.json | 2 +- pdpp-runtime/connector-loader-bootstrap.mjs | 6 + pdpp-runtime/connector-loader.mjs | 42 + pdpp-runtime/package-lock.json | 92 ++ pdpp-runtime/package.json | 9 + scripts/ensure-pdpp-runtime.js | 30 + src-tauri/src/commands/connector.rs | 43 +- src-tauri/src/commands/mod.rs | 1 + src-tauri/src/commands/pdpp_browser.rs | 455 ++++++ .../src/commands/pdpp_collection_state.rs | 86 +- src-tauri/src/commands/pdpp_connector.rs | 661 +++++++- .../src/commands/pdpp_installed_connector.rs | 1326 ++++++++++++++++- src-tauri/src/lib.rs | 8 +- src-tauri/tauri.conf.json | 3 +- ...atgpt-pdpp-browser.collection-profile.json | 73 + .../fixtures/chatgpt-pdpp-browser.fixture.mjs | 28 + .../tests/fixtures/pdpp-connector-fixture.mjs | 10 + src/hooks/useConnector.test.ts | 100 +- src/hooks/useConnector.ts | 150 +- .../components/registry-app-card.test.tsx | 10 +- .../components/connected-sources-list.tsx | 92 +- src/pages/home/index.test.tsx | 284 +++- src/pages/home/index.tsx | 290 +++- src/types/index.ts | 268 ++-- 25 files changed, 3680 insertions(+), 390 deletions(-) create mode 100644 pdpp-runtime/connector-loader-bootstrap.mjs create mode 100644 pdpp-runtime/connector-loader.mjs create mode 100644 pdpp-runtime/package-lock.json create mode 100644 pdpp-runtime/package.json create mode 100644 scripts/ensure-pdpp-runtime.js create mode 100644 src-tauri/src/commands/pdpp_browser.rs create mode 100644 src-tauri/tests/fixtures/chatgpt-pdpp-browser.collection-profile.json create mode 100644 src-tauri/tests/fixtures/chatgpt-pdpp-browser.fixture.mjs diff --git a/.gitignore b/.gitignore index 6a5753cb..c2a26cef 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ src-tauri/auth-page/ .devcontainer/ connectors/ .vercel +pdpp-runtime/.install-stamp diff --git a/package.json b/package.json index 40ef8958..8d4850e1 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "dev": "vite", "dev:app": "vite --port 5173 --strictPort", "build": "tsc -b && vite build", - "prebuild:all": "node scripts/ensure-connectors.js && node scripts/ensure-playwright-runner.js && node scripts/ensure-personal-server.js", + "prebuild:all": "node scripts/ensure-connectors.js && node scripts/ensure-pdpp-runtime.js && node scripts/ensure-playwright-runner.js && node scripts/ensure-personal-server.js", "connectors:resolve": "node scripts/resolve-connectors.js", "connectors:install": "npm run connectors:resolve", "connectors:upgrade": "npm run connectors:resolve", diff --git a/pdpp-runtime/connector-loader-bootstrap.mjs b/pdpp-runtime/connector-loader-bootstrap.mjs new file mode 100644 index 00000000..afada4ba --- /dev/null +++ b/pdpp-runtime/connector-loader-bootstrap.mjs @@ -0,0 +1,6 @@ +import { register } from "node:module" + +// Node 22/23 loads this module with --import before the connector entrypoint. +// Registering the narrow resolver here is the supported replacement for the +// deprecated --experimental-loader command-line hook. +register(new URL("./connector-loader.mjs", import.meta.url), import.meta.url) diff --git a/pdpp-runtime/connector-loader.mjs b/pdpp-runtime/connector-loader.mjs new file mode 100644 index 00000000..4253fe8d --- /dev/null +++ b/pdpp-runtime/connector-loader.mjs @@ -0,0 +1,42 @@ +import { readFileSync } from "node:fs" +import { createRequire } from "node:module" +import { fileURLToPath, pathToFileURL } from "node:url" + +const runtimeRoot = process.env.DATACONNECT_PDPP_RUNTIME_ROOT +if (!runtimeRoot) throw new Error("DATACONNECT_PDPP_RUNTIME_ROOT is required") + +const resolveFromRuntime = createRequire(`${runtimeRoot}/package.json`) +const patchrightPackageUrl = pathToFileURL( + resolveFromRuntime.resolve("patchright/package.json") +) +const patchrightPackage = JSON.parse( + readFileSync(fileURLToPath(patchrightPackageUrl), "utf8") +) +const patchrightEsmEntry = patchrightPackage.exports?.["."]?.import +if (typeof patchrightEsmEntry !== "string") { + throw new Error("Packaged patchright must declare an ESM import entry") +} + +function resolveExternal(specifier) { + if (specifier === "p-queue") { + return pathToFileURL(resolveFromRuntime.resolve(specifier)).href + } + if (specifier === "patchright") { + // createRequire.resolve() selects the require condition, which turns a + // dynamic ESM import into a CommonJS namespace with chromium only under + // default. The authoritative ChatGPT runtime destructures chromium, so + // preserve patchright's declared import condition exactly. + return new URL(patchrightEsmEntry, patchrightPackageUrl).href + } +} + +export async function resolve(specifier, context, nextResolve) { + const external = resolveExternal(specifier) + if (external) { + return { + url: external, + shortCircuit: true, + } + } + return nextResolve(specifier, context) +} diff --git a/pdpp-runtime/package-lock.json b/pdpp-runtime/package-lock.json new file mode 100644 index 00000000..2411df68 --- /dev/null +++ b/pdpp-runtime/package-lock.json @@ -0,0 +1,92 @@ +{ + "name": "dataconnect-pdpp-runtime", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "dataconnect-pdpp-runtime", + "dependencies": { + "p-queue": "^9.3.3", + "patchright": "^1.61.1" + } + }, + "node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/p-queue": { + "version": "9.3.3", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.3.3.tgz", + "integrity": "sha512-NXAOdnEe5FsZJfT4oK84lE1Y5cFFdWlRuOo5tww8DyNMxyRXwn39fIkUtNLKppcPC+UYU/bXujNCUGDv01y7CA==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^5.0.4", + "p-timeout": "^7.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.1.tgz", + "integrity": "sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/patchright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/patchright/-/patchright-1.61.1.tgz", + "integrity": "sha512-kZWNZ2tunsBMTBWtARFVWaNvB739GOybTlIPLT91eyx0YERjtUjESawomwO0hnOh6jt1L3Sru0PK62ylYOfqQw==", + "license": "Apache-2.0", + "dependencies": { + "patchright-core": "1.61.1" + }, + "bin": { + "patchright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/patchright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/patchright-core/-/patchright-core-1.61.1.tgz", + "integrity": "sha512-d6Ju67DE6OzqlEX3WPvUX2SLEs6iMY1WERit//mpsrT98VHcztaUo0CBw2Lpxq25VdufVEiWEdcc0HnLwdiq6A==", + "license": "Apache-2.0", + "bin": { + "patchright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + } + } +} diff --git a/pdpp-runtime/package.json b/pdpp-runtime/package.json new file mode 100644 index 00000000..18d84fcb --- /dev/null +++ b/pdpp-runtime/package.json @@ -0,0 +1,9 @@ +{ + "name": "dataconnect-pdpp-runtime", + "private": true, + "type": "module", + "dependencies": { + "p-queue": "^9.3.3", + "patchright": "^1.61.1" + } +} diff --git a/scripts/ensure-pdpp-runtime.js b/scripts/ensure-pdpp-runtime.js new file mode 100644 index 00000000..53e048f0 --- /dev/null +++ b/scripts/ensure-pdpp-runtime.js @@ -0,0 +1,30 @@ +import { existsSync, readFileSync, writeFileSync } from "node:fs" +import { spawnSync } from "node:child_process" +import { createHash } from "node:crypto" +import { dirname, join } from "node:path" +import { fileURLToPath } from "node:url" + +const root = dirname(dirname(fileURLToPath(import.meta.url))) +const runtimeRoot = join(root, "pdpp-runtime") +const required = ["p-queue", "patchright"].map(name => + join(runtimeRoot, "node_modules", name, "package.json") +) +const stampPath = join(runtimeRoot, ".install-stamp") +const stamp = createHash("sha256") + .update(readFileSync(join(runtimeRoot, "package.json"))) + .update(readFileSync(join(runtimeRoot, "package-lock.json"))) + .digest("hex") + +if ( + required.every(existsSync) && + existsSync(stampPath) && + readFileSync(stampPath, "utf8").trim() === stamp +) + process.exit(0) + +const install = spawnSync("npm", ["ci", "--ignore-scripts"], { + cwd: runtimeRoot, + stdio: "inherit", +}) +if (install.status === 0) writeFileSync(stampPath, `${stamp}\n`) +process.exit(install.status ?? 1) diff --git a/src-tauri/src/commands/connector.rs b/src-tauri/src/commands/connector.rs index bcb60dd4..347d3abc 100644 --- a/src-tauri/src/commands/connector.rs +++ b/src-tauri/src/commands/connector.rs @@ -100,6 +100,7 @@ pub struct Platform { pub runtime: Option, /// Scopes this connector can export (just the scope strings, e.g. ["chatgpt.conversations", "chatgpt.memories"]) pub scopes: Option>, + pub setup: Option, } #[derive(Debug, Deserialize)] @@ -108,9 +109,33 @@ struct ActivePdppPlatformManifest { display_name: Option, name: Option, description: Option, + setup: Option, streams: Vec, } +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ActivePdppStaticSecretSetup { + modality: String, + credential_capture: ActivePdppCredentialCapture, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +struct ActivePdppCredentialCapture { + fields: Vec, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +struct ActivePdppCredentialField { + name: String, + label: Option, + #[serde(rename = "type")] + field_type: Option, + required: bool, + secret: bool, + autocomplete: Option, +} + #[derive(Debug, Deserialize)] struct ActivePdppStream { name: String, @@ -397,6 +422,7 @@ fn platform_from_metadata( vectorize_config: metadata.vectorize_config, runtime: runtime_override.or(metadata.runtime), scopes, + setup: None, } } @@ -551,6 +577,7 @@ fn load_active_pdpp_platforms() -> Vec { vectorize_config: None, runtime: Some("pdpp-network".to_string()), scopes: Some(scopes), + setup: manifest.setup, }); } @@ -567,6 +594,12 @@ fn pdpp_streams_to_dataconnect_scopes( ("github", "user") => Some("github.profile"), ("github", "repositories") => Some("github.repositories"), ("github", "starred") => Some("github.starred"), + ("chatgpt", "conversations") => Some("chatgpt.conversations"), + ("chatgpt", "messages") => Some("chatgpt.messages"), + ("chatgpt", "memories") => Some("chatgpt.memories"), + ("chatgpt", "custom_gpts") => Some("chatgpt.custom_gpts"), + ("chatgpt", "custom_instructions") => Some("chatgpt.custom_instructions"), + ("chatgpt", "shared_conversations") => Some("chatgpt.shared_conversations"), _ => None, }; if let Some(scope) = scope { @@ -2209,7 +2242,7 @@ fn resolve_browser_status( } /// Get system browser path (Chrome/Edge) -fn get_system_browser_path() -> Option { +pub(crate) fn get_system_browser_path() -> Option { #[cfg(target_os = "macos")] { let paths = ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"]; @@ -2302,7 +2335,7 @@ pub async fn test_nodejs(app: AppHandle) -> Result { } /// Get downloaded Chromium path from ~/.dataconnect/browsers -fn get_downloaded_chromium_path() -> Option { +pub(crate) fn get_downloaded_chromium_path() -> Option { let home = std::env::var("HOME") .or_else(|_| std::env::var("USERPROFILE")) .ok()?; @@ -2383,6 +2416,12 @@ fn find_chromium_executable(browsers_dir: &Path, platform: &str) -> Option Option { + get_system_browser_path().or_else(get_downloaded_chromium_path) +} + /// Get the Chromium download URL for the current platform fn get_chromium_download_info() -> Option<(&'static str, &'static str)> { #[cfg(all(target_os = "macos", target_arch = "aarch64"))] diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 4575b66e..3424a14f 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -2,6 +2,7 @@ pub mod connector; pub mod connector_store; pub mod download; pub mod file_ops; +pub mod pdpp_browser; pub mod pdpp_collection_state; pub mod pdpp_connector; pub mod pdpp_installed_connector; diff --git a/src-tauri/src/commands/pdpp_browser.rs b/src-tauri/src/commands/pdpp_browser.rs new file mode 100644 index 00000000..e198e0cb --- /dev/null +++ b/src-tauri/src/commands/pdpp_browser.rs @@ -0,0 +1,455 @@ +//! An exclusive, owner-confined browser lease for PDPP Collection Profiles. +//! +//! This deliberately does not expose the legacy Playwright page API. A PDPP +//! artifact receives a loopback CDP endpoint through its supported environment +//! seam and owns its browser automation protocol. Each lease starts a fresh +//! browser process with +//! the durable profile for exactly one `(connector, connection owner)` pair. +//! Closing a lease removes only its process and lock; an explicit reset removes +//! the authenticated profile. This preserves scheduled collection while +//! preventing any profile or live browser from crossing owners. + +use fs2::FileExt; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::fs::{self, File, OpenOptions}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +const BROWSER_START_TIMEOUT: Duration = Duration::from_secs(10); +const BROWSER_STOP_WAIT: Duration = Duration::from_secs(2); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PdppBrowserInteractionState { + Launching, + Collecting, + WaitingForUser, + Closing, + Closed, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PdppBrowserBinding { + pub backend: &'static str, + pub cdp_http_url: String, + pub lease_id: String, + pub profile_key: String, +} + +pub struct PdppBrowserLease { + binding: PdppBrowserBinding, + profile_dir: PathBuf, + // Keep the OS lock descriptor alive for the full browser lifetime. A + // marker file alone can be deleted by another process and cannot recover + // safely after a crash; an advisory lock is released by the OS when this + // owner dies, while the durable profile remains untouched. + lease_lock: Option, + state: PdppBrowserInteractionState, + child: Option, +} + +impl PdppBrowserLease { + pub fn launch(connector_id: &str, owner_id: &str, run_id: &str) -> Result { + validate_owner_id(owner_id)?; + let browser = super::connector::resolve_automation_browser_path().ok_or( + "No system or downloaded Chromium browser is available for PDPP browser automation", + )?; + let (profile_dir, lease_lock) = + acquire_profile_lease(&profile_root()?, connector_id, owner_id)?; + // A stale port file must never point a new owner lease at an old + // browser. Authentication data remains in the durable profile. + let _ = fs::remove_file(profile_dir.join("DevToolsActivePort")); + let mut command = Command::new(browser); + command + .arg(format!("--user-data-dir={}", profile_dir.display())) + .args([ + "--remote-debugging-address=127.0.0.1", + "--remote-debugging-port=0", + "--no-first-run", + "--no-default-browser-check", + "about:blank", + ]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + command.process_group(0); + } + let child = match command.spawn() { + Ok(child) => child, + Err(error) => { + release_profile_lease(Some(lease_lock)); + return Err(format!("Failed to launch PDPP browser: {error}")); + } + }; + let lease_id = lease_id(connector_id, owner_id, run_id); + let (endpoint, child) = match wait_for_devtools_endpoint(&profile_dir, child) { + Ok(ready) => ready, + Err(error) => { + release_profile_lease(Some(lease_lock)); + return Err(error); + } + }; + Ok(Self { + binding: PdppBrowserBinding { + backend: "neko", + cdp_http_url: endpoint, + lease_id, + profile_key: profile_key(connector_id, owner_id), + }, + profile_dir, + lease_lock: Some(lease_lock), + state: PdppBrowserInteractionState::Collecting, + child: Some(child), + }) + } + + #[cfg(test)] + fn fixture( + root: &Path, + connector_id: &str, + owner_id: &str, + run_id: &str, + ) -> Result { + validate_owner_id(owner_id)?; + let (profile_dir, lease_lock) = acquire_profile_lease(root, connector_id, owner_id)?; + Ok(Self { + binding: PdppBrowserBinding { + backend: "neko", + cdp_http_url: "http://127.0.0.1:9222".into(), + lease_id: lease_id(connector_id, owner_id, run_id), + profile_key: profile_key(connector_id, owner_id), + }, + profile_dir, + lease_lock: Some(lease_lock), + state: PdppBrowserInteractionState::Launching, + child: None, + }) + } + + pub fn binding(&self) -> &PdppBrowserBinding { + &self.binding + } + + pub fn mark_waiting_for_user(&mut self) { + if self.state == PdppBrowserInteractionState::Collecting { + self.state = PdppBrowserInteractionState::WaitingForUser; + } + } + + #[cfg(test)] + fn state(&self) -> PdppBrowserInteractionState { + self.state.clone() + } + + #[cfg(test)] + fn profile_dir(&self) -> &Path { + &self.profile_dir + } + + pub fn close(&mut self) { + if self.state == PdppBrowserInteractionState::Closed { + return; + } + self.state = PdppBrowserInteractionState::Closing; + if let Some(mut child) = self.child.take() { + terminate_browser(&mut child); + } + release_profile_lease(self.lease_lock.take()); + self.state = PdppBrowserInteractionState::Closed; + } + + /// Removes an authenticated PDPP profile only when no browser lease owns + /// it. Call this from an explicit disconnect/reset action, never from the + /// normal run lifecycle. + pub fn reset_profile(connector_id: &str, owner_id: &str) -> Result<(), String> { + reset_profile_in(&profile_root()?, connector_id, owner_id) + } + + /// A setup marker is insufficient when its owner profile is gone. + pub fn profile_exists(connector_id: &str, owner_id: &str) -> Result { + validate_owner_id(owner_id)?; + Ok(profile_dir(&profile_root()?, connector_id, owner_id).is_dir()) + } +} + +impl Drop for PdppBrowserLease { + fn drop(&mut self) { + self.close(); + } +} + +fn validate_owner_id(owner_id: &str) -> Result<(), String> { + if owner_id.is_empty() + || owner_id.len() > 128 + || !owner_id.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '_' | '-' | '.') + }) + { + return Err("PDPP browser binding requires an explicit URL-safe connectionId owner".into()); + } + Ok(()) +} + +fn profile_root() -> Result { + let home = std::env::var("HOME") + .or_else(|_| std::env::var("USERPROFILE")) + .map_err(|_| "Could not determine a home directory for PDPP browser profiles")?; + Ok(PathBuf::from(home) + .join(".dataconnect") + .join("pdpp-browser-leases")) +} + +fn profile_dir(root: &Path, connector_id: &str, owner_id: &str) -> PathBuf { + root.join("profiles") + .join(stable_segment(connector_id)) + .join(stable_segment(owner_id)) +} + +fn lock_path(root: &Path, connector_id: &str, owner_id: &str) -> PathBuf { + root.join("leases") + .join(stable_segment(connector_id)) + .join(format!("{}.lock", stable_segment(owner_id))) +} + +fn acquire_profile_lease( + root: &Path, + connector_id: &str, + owner_id: &str, +) -> Result<(PathBuf, File), String> { + validate_owner_id(owner_id)?; + fs::create_dir_all(root) + .map_err(|error| format!("Failed to create PDPP browser profile root: {error}"))?; + let canonical_root = fs::canonicalize(root) + .map_err(|error| format!("Failed to confine PDPP browser profile root: {error}"))?; + let profile_dir = profile_dir(&canonical_root, connector_id, owner_id); + fs::create_dir_all(&profile_dir) + .map_err(|error| format!("Failed to create PDPP browser profile: {error}"))?; + let canonical_profile = fs::canonicalize(&profile_dir) + .map_err(|error| format!("Failed to confine PDPP browser profile: {error}"))?; + if !canonical_profile.starts_with(&canonical_root) { + return Err("PDPP browser profile escaped its lease root".into()); + } + let lock_path = lock_path(&canonical_root, connector_id, owner_id); + let lock_parent = lock_path + .parent() + .ok_or("PDPP browser lease lock has no parent")?; + fs::create_dir_all(lock_parent) + .map_err(|error| format!("Failed to create PDPP browser lease directory: {error}"))?; + let lock_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(&lock_path) + .map_err(|error| format!("Failed to open PDPP browser lease: {error}"))?; + match lock_file.try_lock_exclusive() { + Ok(()) => Ok((canonical_profile, lock_file)), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => Err( + "PDPP browser profile is already leased by an active run for this connection".into(), + ), + Err(error) => Err(format!("Failed to acquire PDPP browser lease: {error}")), + } +} + +fn release_profile_lease(lock_file: Option) { + if let Some(lock_file) = lock_file { + let _ = FileExt::unlock(&lock_file); + // Do not remove the lock file. Removing an inode while a concurrent + // waiter still holds it permits a second lock file to be created and + // would split ownership between two live leases. + } +} + +fn reset_profile_in(root: &Path, connector_id: &str, owner_id: &str) -> Result<(), String> { + let (profile_dir, lease_lock) = acquire_profile_lease(root, connector_id, owner_id)?; + let removal = fs::remove_dir_all(&profile_dir) + .map_err(|error| format!("Failed to reset PDPP browser profile: {error}")); + release_profile_lease(Some(lease_lock)); + removal +} + +fn stable_segment(value: &str) -> String { + format!("{:x}", Sha256::digest(value.as_bytes())) +} + +fn lease_id(connector_id: &str, owner_id: &str, run_id: &str) -> String { + stable_segment(&format!("{connector_id}:{owner_id}:{run_id}")) +} + +fn profile_key(connector_id: &str, owner_id: &str) -> String { + stable_segment(&format!("{connector_id}:{owner_id}")) +} + +fn wait_for_devtools_endpoint( + profile_dir: &Path, + mut child: Child, +) -> Result<(String, Child), String> { + let deadline = Instant::now() + BROWSER_START_TIMEOUT; + let active_port = profile_dir.join("DevToolsActivePort"); + while Instant::now() < deadline { + if let Ok(Some(status)) = child.try_wait() { + return Err(format!( + "PDPP browser exited before becoming ready: {status}" + )); + } + if let Ok(contents) = fs::read_to_string(&active_port) { + if let Some(port) = contents + .lines() + .next() + .and_then(|port| port.parse::().ok()) + { + return Ok((format!("http://127.0.0.1:{port}"), child)); + } + } + thread::sleep(Duration::from_millis(25)); + } + terminate_browser(&mut child); + Err("Timed out waiting for PDPP browser CDP endpoint".into()) +} + +fn terminate_browser(child: &mut Child) { + #[cfg(unix)] + { + crate::commands::server::kill_process_group(child.id(), libc::SIGTERM); + } + #[cfg(not(unix))] + { + let _ = child.kill(); + } + let deadline = Instant::now() + BROWSER_STOP_WAIT; + while Instant::now() < deadline { + if child.try_wait().ok().flatten().is_some() { + return; + } + thread::sleep(Duration::from_millis(20)); + } + #[cfg(unix)] + crate::commands::server::kill_process_group(child.id(), libc::SIGKILL); + #[cfg(not(unix))] + { + let _ = child.kill(); + } + let _ = child.wait(); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::process::Command; + + const LOCK_HOLDER_ROOT: &str = "PDPP_BROWSER_LOCK_HOLDER_ROOT"; + + struct ReapedTestChild(Child); + + impl Drop for ReapedTestChild { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + #[test] + fn holds_a_live_profile_lease_until_killed() { + let Ok(root) = std::env::var(LOCK_HOLDER_ROOT) else { + return; + }; + let root = PathBuf::from(root); + let _lease = PdppBrowserLease::fixture(&root, "chatgpt-pdpp", "alice", "holder") + .expect("lock holder must acquire its profile lease"); + fs::write(root.join("lock-holder-ready"), "ready") + .expect("lock holder must signal readiness"); + loop { + thread::sleep(Duration::from_secs(1)); + } + } + + #[test] + fn fixture_leases_persist_per_owner_and_clean_only_ephemeral_state() { + let root = tempfile::tempdir().unwrap(); + let mut first = + PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-1").unwrap(); + let profile = first.profile_dir().to_owned(); + assert!(first.profile_dir().starts_with(root.path())); + first.state = PdppBrowserInteractionState::Collecting; + first.mark_waiting_for_user(); + assert_eq!(first.state(), PdppBrowserInteractionState::WaitingForUser); + first.close(); + assert_eq!(first.state(), PdppBrowserInteractionState::Closed); + assert!(profile.exists()); + let second = + PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-2").unwrap(); + assert_eq!(second.profile_dir(), profile); + assert_ne!(second.binding().lease_id, ""); + } + + #[test] + fn fixture_leases_are_owner_confined_and_exclusive() { + let root = tempfile::tempdir().unwrap(); + let first = + PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-1").unwrap(); + let concurrent = PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-2"); + assert!(matches!(concurrent, Err(error) if error.contains("already leased"))); + let second = + PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "bob", "run-1").unwrap(); + assert_ne!(first.profile_dir(), second.profile_dir()); + } + + #[test] + fn live_process_lease_cannot_be_stolen_and_recovers_after_owner_is_killed() { + let root = tempfile::tempdir().unwrap(); + let mut holder = ReapedTestChild( + Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "commands::pdpp_browser::tests::holds_a_live_profile_lease_until_killed", + "--nocapture", + ]) + .env(LOCK_HOLDER_ROOT, root.path()) + .spawn() + .expect("spawn lock holder test process"), + ); + let ready = root.path().join("lock-holder-ready"); + let deadline = Instant::now() + Duration::from_secs(5); + while !ready.exists() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + assert!(ready.exists(), "lock holder did not become ready"); + + let blocked = PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-2"); + assert!(matches!(blocked, Err(error) if error.contains("already leased"))); + + holder.0.kill().expect("kill lock holder"); + holder.0.wait().expect("reap killed lock holder"); + + let recovered = PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-3"); + assert!( + recovered.is_ok(), + "OS lock must be released when its owner dies" + ); + } + + #[test] + fn reset_deletes_only_an_idle_owner_profile() { + let root = tempfile::tempdir().unwrap(); + let lease = + PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-1").unwrap(); + let profile = lease.profile_dir().to_owned(); + assert!(reset_profile_in(root.path(), "chatgpt-pdpp", "alice").is_err()); + drop(lease); + reset_profile_in(root.path(), "chatgpt-pdpp", "alice").unwrap(); + assert!(!profile.exists()); + } + + #[test] + fn refuses_implicit_or_unsafe_browser_owners() { + assert!(validate_owner_id("").is_err()); + assert!(validate_owner_id("../other-owner").is_err()); + assert!(validate_owner_id("account-one").is_ok()); + } +} diff --git a/src-tauri/src/commands/pdpp_collection_state.rs b/src-tauri/src/commands/pdpp_collection_state.rs index b89b35cc..d67c65bd 100644 --- a/src-tauri/src/commands/pdpp_collection_state.rs +++ b/src-tauri/src/commands/pdpp_collection_state.rs @@ -9,7 +9,7 @@ use super::pdpp_connector::{PdppRecord, PdppRunStatus}; use fs2::FileExt; use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::fs::{self, File}; use std::io::Write; use std::path::{Path, PathBuf}; @@ -32,6 +32,10 @@ pub struct PdppCollectionConnectionState { struct PdppCollectionStateFile { version: u8, connectors: HashMap>, + /// A non-secret setup acknowledgement. Browser authentication remains in + /// the owner-confined profile, never in collection state. + #[serde(default)] + setup_complete_connections: HashMap>, } pub fn collection_state_path() -> Result { @@ -63,6 +67,69 @@ pub fn load_connection_state_at( .unwrap_or_default()) } +pub fn is_connection_setup_complete( + connector_id: &str, + connection_id: &str, +) -> Result { + is_connection_setup_complete_at(&collection_state_path()?, connector_id, connection_id) +} + +fn is_connection_setup_complete_at( + path: &Path, + connector_id: &str, + connection_id: &str, +) -> Result { + let _guard = COLLECTION_STATE_LOCK + .lock() + .map_err(|_| "PDPP collection state lock is unavailable")?; + Ok(read_state_file(path)? + .setup_complete_connections + .get(connector_id) + .is_some_and(|connections| connections.contains(connection_id))) +} + +/// Stores only that setup occurred, allowing a later scheduled run to reuse +/// the durable browser profile without another credential prompt. +pub fn mark_connection_setup_complete( + connector_id: &str, + connection_id: &str, +) -> Result<(), String> { + update_connection_setup_complete_at(&collection_state_path()?, connector_id, connection_id, true) +} + +pub fn clear_connection_setup_complete( + connector_id: &str, + connection_id: &str, +) -> Result<(), String> { + update_connection_setup_complete_at(&collection_state_path()?, connector_id, connection_id, false) +} + +fn update_connection_setup_complete_at( + path: &Path, + connector_id: &str, + connection_id: &str, + complete: bool, +) -> Result<(), String> { + let _guard = COLLECTION_STATE_LOCK + .lock() + .map_err(|_| "PDPP collection state lock is unavailable")?; + let _file_lock = lock_state_file(path)?; + let mut file = read_state_file(path)?; + if complete { + file.setup_complete_connections + .entry(connector_id.to_owned()) + .or_default() + .insert(connection_id.to_owned()); + } else if let Some(connections) = file.setup_complete_connections.get_mut(connector_id) { + connections.remove(connection_id); + if connections.is_empty() { + file.setup_complete_connections.remove(connector_id); + } + } + file.version = 1; + write_state_file_atomically(path, &file) +} + /// Commit a terminal run only when the validated protocol result succeeded. /// Failed, cancelled, and timed-out runs return without touching durable state. pub fn commit_terminal_run( @@ -339,6 +406,23 @@ mod tests { ); } + #[test] + fn reset_clears_the_owner_confined_nonsecret_setup_marker() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("state.json"); + update_connection_setup_complete_at(&path, "chatgpt-pdpp", "owner-a", true).unwrap(); + + assert!(is_connection_setup_complete_at(&path, "chatgpt-pdpp", "owner-a").unwrap()); + assert!(!is_connection_setup_complete_at(&path, "chatgpt-pdpp", "owner-b").unwrap()); + let stored = fs::read_to_string(&path).unwrap(); + assert!(stored.contains("setupCompleteConnections")); + assert!(!stored.contains("password")); + assert!(!stored.contains("username")); + + update_connection_setup_complete_at(&path, "chatgpt-pdpp", "owner-a", false).unwrap(); + assert!(!is_connection_setup_complete_at(&path, "chatgpt-pdpp", "owner-a").unwrap()); + } + #[test] fn successful_checkpoint_is_durable_with_its_records_and_snapshot() { let temp = tempfile::tempdir().unwrap(); diff --git a/src-tauri/src/commands/pdpp_connector.rs b/src-tauri/src/commands/pdpp_connector.rs index 02e49cca..da4bdb7e 100644 --- a/src-tauri/src/commands/pdpp_connector.rs +++ b/src-tauri/src/commands/pdpp_connector.rs @@ -13,7 +13,7 @@ use std::path::PathBuf; use std::process::{Command, Stdio}; use std::sync::{ atomic::{AtomicBool, Ordering}, - mpsc, Arc, + mpsc, Arc, Mutex, }; use std::thread; use std::time::{Duration, Instant}; @@ -35,7 +35,6 @@ pub struct PdppStart { pub collection_mode: String, pub scope: Value, pub state: Option, - pub bindings: Value, } impl PdppStart { @@ -44,7 +43,6 @@ impl PdppStart { collection_mode: impl Into, scope: Value, state: Option, - bindings: Value, ) -> Result { let start = Self { message_type: "START", @@ -52,7 +50,6 @@ impl PdppStart { collection_mode: collection_mode.into(), scope, state, - bindings, }; start.validate()?; Ok(start) @@ -68,9 +65,6 @@ impl PdppStart { ) { return Err("PDPP START collection_mode must be full_refresh or incremental".into()); } - if !self.bindings.is_object() { - return Err("PDPP START bindings must be an object".into()); - } Ok(()) } } @@ -84,6 +78,112 @@ pub struct PdppScopeValidators { } pub type PdppEventSink = Arc Result<(), String> + Send + Sync>; +/// Delivers one connector interaction together with the only responder that +/// can answer it. The responder is bound to this kernel run and interaction +/// generation, so a delayed callback cannot answer a later interaction. +pub type PdppInteractionSink = Arc< + dyn for<'interaction> Fn( + &'interaction PdppInteraction, + PdppInteractionResponder, + ) -> Result<(), String> + + Send + + Sync, +>; +/// Notifies a host that a pending interaction can no longer accept a response. +/// Hosts use this to remove any owner-visible responder before a late command +/// can report success after cancellation, timeout, or protocol failure. +pub type PdppInteractionClosedSink = Arc; + +const DEFAULT_INTERACTION_TIMEOUT: Duration = Duration::from_secs(30 * 60); +const MIN_INTERACTION_TIMEOUT: Duration = Duration::from_secs(60); +const MAX_INTERACTION_TIMEOUT: Duration = Duration::from_secs(60 * 60); + +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PdppInteractionResponseStatus { + Success, + Cancelled, + Timeout, +} + +#[derive(Clone, Serialize)] +pub struct PdppInteractionResponse { + #[serde(rename = "type")] + message_type: &'static str, + pub request_id: String, + pub status: PdppInteractionResponseStatus, + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option, +} + +impl PdppInteractionResponse { + fn new( + request_id: impl Into, + status: PdppInteractionResponseStatus, + data: Option, + ) -> Result { + let response = Self { + message_type: "INTERACTION_RESPONSE", + request_id: request_id.into(), + status, + data, + }; + response.validate()?; + Ok(response) + } + + fn validate(&self) -> Result<(), String> { + if self.request_id.is_empty() { + return Err("PDPP INTERACTION_RESPONSE requires a non-empty request_id".into()); + } + if self.data.as_ref().is_some_and(|data| !data.is_object()) { + return Err("PDPP INTERACTION_RESPONSE data must be an object".into()); + } + if !matches!(self.status, PdppInteractionResponseStatus::Success) && self.data.is_some() { + return Err("PDPP INTERACTION_RESPONSE data is only allowed for success".into()); + } + Ok(()) + } +} + +/// A one-shot, run-bound response channel. The JSONL envelope deliberately +/// contains no run_id (the Collection Profile contract keys it by +/// request_id); `run_id()` is exposed only for host-side ownership checks. +#[derive(Clone)] +pub struct PdppInteractionResponder { + run_id: String, + request_id: String, + generation: u64, + sent: Arc, + sender: mpsc::Sender, +} + +impl PdppInteractionResponder { + pub fn run_id(&self) -> &str { + &self.run_id + } + + pub fn request_id(&self) -> &str { + &self.request_id + } + + pub fn respond( + &self, + status: PdppInteractionResponseStatus, + data: Option, + ) -> Result<(), String> { + let response = PdppInteractionResponse::new(self.request_id.clone(), status, data)?; + if self.sent.swap(true, Ordering::SeqCst) { + return Err("PDPP INTERACTION responder was already used".into()); + } + self.sender + .send(PendingInteractionResponse { + generation: self.generation, + response, + }) + .map_err(|_| "PDPP INTERACTION run is no longer active".to_string()) + } +} #[derive(Clone)] pub struct PdppRunOptions { @@ -95,6 +195,11 @@ pub struct PdppRunOptions { pub max_retained_records: usize, pub max_retained_events: usize, pub on_event: Option, + /// Browser-capable hosts can surface an interaction to the user while the + /// connector remains alive. Network-only hosts deliberately leave this + /// unset and retain the prior fail-closed behavior. + pub on_interaction: Option, + pub on_interaction_closed: Option, } impl Default for PdppRunOptions { @@ -108,6 +213,8 @@ impl Default for PdppRunOptions { max_retained_records: 0, max_retained_events: 32, on_event: None, + on_interaction: None, + on_interaction_closed: None, } } } @@ -278,6 +385,69 @@ enum Line { End, } +struct PendingInteractionResponse { + generation: u64, + response: PdppInteractionResponse, +} + +struct PendingInteraction { + generation: u64, + request_id: String, + deadline: Instant, +} + +fn interaction_deadline_elapsed(pending: &PendingInteraction, now: Instant) -> bool { + now >= pending.deadline +} + +fn close_pending_interaction( + options: &PdppRunOptions, + run_id: &str, + pending: PendingInteraction, +) -> PendingInteraction { + if let Some(on_interaction_closed) = &options.on_interaction_closed { + on_interaction_closed(run_id, &pending.request_id); + } + pending +} + +fn interaction_timeout(timeout_seconds: Option) -> Result { + let timeout = timeout_seconds + .map(Duration::from_secs) + .unwrap_or(DEFAULT_INTERACTION_TIMEOUT); + if !(MIN_INTERACTION_TIMEOUT..=MAX_INTERACTION_TIMEOUT).contains(&timeout) { + return Err("PDPP INTERACTION timeout_seconds must be between 60 and 3600".into()); + } + Ok(timeout) +} + +fn write_interaction_response( + writer: &mut W, + response: &PdppInteractionResponse, +) -> Result<(), String> { + response.validate()?; + writeln!( + writer, + "{}", + serde_json::to_string(response) + .map_err(|error| format!("Failed to serialize PDPP INTERACTION_RESPONSE: {error}"))? + ) + .map_err(|error| format!("Failed to send PDPP INTERACTION_RESPONSE: {error}"))?; + writer + .flush() + .map_err(|error| format!("Failed to flush PDPP INTERACTION_RESPONSE: {error}")) +} + +fn send_interaction_response( + stdin: &mut Option, + response: &PdppInteractionResponse, +) -> Result<(), String> { + let stdin = stdin + .as_mut() + .ok_or("PDPP connector stdin closed before INTERACTION_RESPONSE")?; + write_interaction_response(stdin, response) +} + pub fn supervise_pdpp_connector( command: &PdppConnectorCommand, start: &PdppStart, @@ -312,20 +482,25 @@ pub fn supervise_pdpp_connector( let mut child = process .spawn() .map_err(|e| format!("Failed to spawn PDPP connector: {e}"))?; - let mut stdin = child - .stdin - .take() - .ok_or("PDPP connector stdin unavailable")?; + let mut stdin = Some( + child + .stdin + .take() + .ok_or("PDPP connector stdin unavailable")?, + ); writeln!( - stdin, + stdin + .as_mut() + .expect("PDPP connector stdin was initialized"), "{}", serde_json::to_string(start).map_err(|e| e.to_string())? ) .map_err(|e| format!("Failed to send PDPP START: {e}"))?; stdin + .as_mut() + .expect("PDPP connector stdin was initialized") .flush() .map_err(|e| format!("Failed to flush PDPP START: {e}"))?; - drop(stdin); let stdout = child .stdout .take() @@ -335,10 +510,14 @@ pub fn supervise_pdpp_connector( .take() .ok_or("PDPP connector stderr unavailable")?; let (tx, rx) = mpsc::channel(); + let (interaction_tx, interaction_rx) = mpsc::channel::(); let stdout_thread = spawn_reader(stdout, options.max_stdout_line_bytes, true, tx.clone()); let stderr_thread = spawn_reader(stderr, options.max_stderr_bytes, false, tx); - let started = Instant::now(); + let mut normal_started = Instant::now(); + let mut normal_elapsed = Duration::ZERO; + let mut pending_interaction: Option = None; + let mut interaction_generation = 0u64; let mut stdout_closed = false; let mut stderr_closed = false; let mut exit = None; @@ -361,15 +540,74 @@ pub fn supervise_pdpp_connector( .map_err(|e| format!("Failed to poll PDPP connector: {e}"))?; } if termination.is_none() && options.control.is_cancelled() { + if let Some(pending) = pending_interaction.take() { + let pending = close_pending_interaction(&options, &start.run_id, pending); + let response = PdppInteractionResponse::new( + pending.request_id, + PdppInteractionResponseStatus::Cancelled, + None, + )?; + let _ = send_interaction_response(&mut stdin, &response); + } termination = Some(PdppRunStatus::Cancelled); set_failure(&mut failure, "PDPP connector was cancelled by the runtime"); terminate_child(&mut child); } - if termination.is_none() && options.timeout.is_some_and(|t| started.elapsed() >= t) { + if termination.is_none() + && pending_interaction.is_none() + && options + .timeout + .is_some_and(|timeout| normal_elapsed + normal_started.elapsed() >= timeout) + { termination = Some(PdppRunStatus::TimedOut); set_failure(&mut failure, "PDPP connector exceeded its runtime timeout"); terminate_child(&mut child); } + while let Ok(pending_response) = interaction_rx.try_recv() { + let matches_pending = pending_interaction.as_ref().is_some_and(|pending| { + pending.generation == pending_response.generation + && pending.request_id == pending_response.response.request_id + }); + if !matches_pending { + // A responder from an earlier interaction or run cannot affect + // the active connector state. It is intentionally discarded. + continue; + } + if let Err(error) = send_interaction_response(&mut stdin, &pending_response.response) { + if let Some(pending) = pending_interaction.take() { + close_pending_interaction(&options, &start.run_id, pending); + } + set_failure(&mut failure, error); + terminate_child(&mut child); + } else { + let pending = pending_interaction + .take() + .expect("matching response requires a pending interaction"); + close_pending_interaction(&options, &start.run_id, pending); + normal_started = Instant::now(); + } + } + if termination.is_none() + && pending_interaction + .as_ref() + .is_some_and(|pending| interaction_deadline_elapsed(pending, Instant::now())) + { + let pending = pending_interaction + .take() + .expect("pending interaction was checked above"); + let pending = close_pending_interaction(&options, &start.run_id, pending); + let response = PdppInteractionResponse::new( + pending.request_id, + PdppInteractionResponseStatus::Timeout, + None, + )?; + if let Err(error) = send_interaction_response(&mut stdin, &response) { + set_failure(&mut failure, error); + terminate_child(&mut child); + } else { + normal_started = Instant::now(); + } + } match rx.recv_timeout(Duration::from_millis(10)) { Ok(ReaderEvent::Stdout(Ok(line))) => { if done.is_some() { @@ -378,6 +616,17 @@ pub fn supervise_pdpp_connector( continue; } match parse_message(&line, &scope) { + Ok(_) if pending_interaction.is_some() => { + let pending = pending_interaction + .take() + .expect("pending interaction was checked above"); + close_pending_interaction(&options, &start.run_id, pending); + set_failure( + &mut failure, + "PDPP connector emitted output while waiting for INTERACTION_RESPONSE", + ); + terminate_child(&mut child); + } Ok(ConnectorMessage::Record(record)) => { record_count += 1; event_counts.records += 1; @@ -461,15 +710,58 @@ pub fn supervise_pdpp_connector( event_counts.interactions += 1; retain_event( &options, - PdppEvent::Interaction(interaction), + PdppEvent::Interaction(interaction.clone()), &mut events, &mut events_truncated, &mut failure, ); - set_failure(&mut failure, "PDPP INTERACTION requires an INTERACTION_RESPONSE API, which this foundation kernel does not provide"); - terminate_child(&mut child); + if let Some(on_interaction) = &options.on_interaction { + if pending_interaction.is_some() { + set_failure( + &mut failure, + "PDPP connector emitted INTERACTION while already waiting for a response", + ); + terminate_child(&mut child); + continue; + } + let timeout = match interaction_timeout(interaction.timeout_seconds) { + Ok(timeout) => timeout, + Err(error) => { + set_failure(&mut failure, error); + terminate_child(&mut child); + continue; + } + }; + normal_elapsed += normal_started.elapsed(); + interaction_generation += 1; + let responder = PdppInteractionResponder { + run_id: start.run_id.clone(), + request_id: interaction.request_id.clone(), + generation: interaction_generation, + sent: Arc::new(AtomicBool::new(false)), + sender: interaction_tx.clone(), + }; + pending_interaction = Some(PendingInteraction { + generation: interaction_generation, + request_id: interaction.request_id.clone(), + deadline: Instant::now() + timeout, + }); + if let Err(error) = on_interaction(&interaction, responder) { + set_failure(&mut failure, error); + terminate_child(&mut child); + } + } else { + set_failure(&mut failure, "PDPP INTERACTION requires an INTERACTION_RESPONSE API, which this foundation kernel does not provide"); + terminate_child(&mut child); + } + } + Ok(ConnectorMessage::Done(message)) => { + done = Some(message); + // A connector that uses readline keeps its event loop + // alive until its input closes. No further runtime + // message is valid after DONE, so release the writer. + stdin.take(); } - Ok(ConnectorMessage::Done(message)) => done = Some(message), Err(error) => { set_failure(&mut failure, error); terminate_child(&mut child); @@ -682,9 +974,22 @@ fn parse_message(line: &str, scope: &ScopePolicy) -> Result serde_json::from_value(value) - .map(ConnectorMessage::Interaction) - .map_err(|e| format!("Invalid PDPP INTERACTION: {e}")), + "INTERACTION" => { + let interaction: PdppInteraction = serde_json::from_value(value) + .map_err(|e| format!("Invalid PDPP INTERACTION: {e}"))?; + if interaction.request_id.is_empty() + || interaction.kind.is_empty() + || interaction.message.is_empty() + || interaction + .schema + .as_ref() + .is_some_and(|schema| !schema.is_object()) + { + return Err("Invalid PDPP INTERACTION envelope".into()); + } + interaction_timeout(interaction.timeout_seconds)?; + Ok(ConnectorMessage::Interaction(interaction)) + } _ => Err(format!("Unsupported PDPP connector message type: {ty}")), } } @@ -951,14 +1256,7 @@ mod tests { use super::*; use serde_json::json; fn start(scope: Value) -> PdppStart { - PdppStart::new( - "run_test", - "incremental", - scope, - None, - json!({"network": {}}), - ) - .unwrap() + PdppStart::new("run_test", "incremental", scope, None).unwrap() } fn scoped() -> PdppStart { start( @@ -969,6 +1267,15 @@ mod tests { PdppConnectorCommand { program: "node".into(), args: vec![ + "-e".into(), + r#" +const { spawn } = require('node:child_process'); +const [fixturePath, mode] = process.argv.slice(1); +const child = spawn(process.execPath, [fixturePath, mode], { stdio: ['pipe', 'inherit', 'inherit'] }); +process.stdin.once('data', chunk => child.stdin.end(chunk)); +child.on('exit', code => process.exit(code ?? 1)); +"# + .into(), format!( "{}/tests/fixtures/pdpp-connector-fixture.mjs", env!("CARGO_MANIFEST_DIR") @@ -980,6 +1287,102 @@ mod tests { clear_env: false, } } + fn blocking_interaction_fixture(two_interactions: bool) -> PdppConnectorCommand { + let second = if two_interactions { + r#" + if (phase === 0) { + phase = 1; + emit({ type: 'INTERACTION', request_id: 'request-2', kind: 'manual_action', message: 'Continue', timeout_seconds: 60 }); + return; + } +"# + } else { + "" + }; + PdppConnectorCommand { + program: "node".into(), + args: vec![ + "-e".into(), + format!( + r#" +const readline = require('node:readline'); +const emit = message => process.stdout.write(`${{JSON.stringify(message)}}\n`); +let phase = 0; +readline.createInterface({{ input: process.stdin }}).on('line', line => {{ + const message = JSON.parse(line); + if (message.type === 'START') {{ + emit({{ type: 'INTERACTION', request_id: 'request-1', kind: 'manual_action', message: 'Sign in', timeout_seconds: 60 }}); + return; + }} + if (message.type !== 'INTERACTION_RESPONSE') process.exit(70); + const expected = phase === 0 ? 'request-1' : 'request-2'; + if (message.request_id !== expected) process.exit(71); + {second} + emit({{ type: 'RECORD', stream: 'items', key: 'item-1', data: {{ id: 'item-1', source_updated_at: '2026-07-30T00:00:00Z' }}, emitted_at: '2026-07-30T00:00:00Z' }}); + emit({{ type: 'DONE', status: 'succeeded', records_emitted: 1 }}); + setImmediate(() => process.exit(0)); +}}); +"# + ), + ], + cwd: None, + env: HashMap::new(), + clear_env: false, + } + } + + fn delayed_after_interaction_fixture() -> PdppConnectorCommand { + PdppConnectorCommand { + program: "node".into(), + args: vec![ + "-e".into(), + r#" +const readline = require('node:readline'); +const emit = message => process.stdout.write(`${JSON.stringify(message)}\n`); +readline.createInterface({ input: process.stdin }).on('line', line => { + const message = JSON.parse(line); + if (message.type === 'START') { + emit({ type: 'INTERACTION', request_id: 'request-1', kind: 'manual_action', message: 'Sign in', timeout_seconds: 60 }); + return; + } + if (message.type !== 'INTERACTION_RESPONSE' || message.request_id !== 'request-1') process.exit(71); + setTimeout(() => { + emit({ type: 'RECORD', stream: 'items', key: 'item-1', data: { id: 'item-1', source_updated_at: '2026-07-30T00:00:00Z' }, emitted_at: '2026-07-30T00:00:00Z' }); + emit({ type: 'DONE', status: 'succeeded', records_emitted: 1 }); + process.exit(0); + }, 60); +}); +"# + .into(), + ], + cwd: None, + env: HashMap::new(), + clear_env: false, + } + } + + fn output_while_waiting_fixture(output: &str) -> PdppConnectorCommand { + PdppConnectorCommand { + program: "node".into(), + args: vec![ + "-e".into(), + format!( + r#" +const readline = require('node:readline'); +const emit = message => process.stdout.write(`${{JSON.stringify(message)}}\n`); +readline.createInterface({{ input: process.stdin }}).on('line', line => {{ + if (JSON.parse(line).type !== 'START') process.exit(70); + emit({{ type: 'INTERACTION', request_id: 'request-1', kind: 'manual_action', message: 'Sign in', timeout_seconds: 60 }}); + emit({output}); +}}); +"# + ), + ], + cwd: None, + env: HashMap::new(), + clear_env: false, + } + } fn options() -> PdppRunOptions { let mut options = PdppRunOptions { max_retained_records: 4, @@ -995,6 +1398,26 @@ mod tests { ); options } + + fn supervise_with_test_deadline( + command: PdppConnectorCommand, + start: PdppStart, + options: PdppRunOptions, + ) -> PdppRunResult { + let control = options.control.clone(); + let (sender, receiver) = mpsc::sync_channel(1); + thread::spawn(move || { + let _ = sender.send(supervise_pdpp_connector(&command, &start, &options)); + }); + match receiver.recv_timeout(Duration::from_secs(2)) { + Ok(result) => result.expect("interaction fixture supervision should succeed"), + Err(error) => { + control.cancel(); + let _ = receiver.recv_timeout(Duration::from_secs(1)); + panic!("interaction fixture exceeded the 2-second test deadline: {error}"); + } + } + } #[test] fn accepts_scoped_success_and_retains_bounded_output() { let result = supervise_pdpp_connector(&fixture("success"), &scoped(), &options()).unwrap(); @@ -1058,6 +1481,184 @@ mod tests { .contains("INTERACTION_RESPONSE")); } #[test] + fn browser_interaction_hook_keeps_the_protocol_alive_until_done() { + let observed = Arc::new(AtomicBool::new(false)); + let observed_by_hook = observed.clone(); + let result = supervise_pdpp_connector( + &blocking_interaction_fixture(false), + &scoped(), + &PdppRunOptions { + on_interaction: Some(Arc::new(move |interaction, responder| { + assert_eq!(interaction.request_id, "request-1"); + assert_eq!(responder.run_id(), "run_test"); + observed_by_hook.store(true, Ordering::SeqCst); + responder.respond(PdppInteractionResponseStatus::Success, None) + })), + ..options() + }, + ) + .unwrap(); + assert!(observed.load(Ordering::SeqCst)); + assert_eq!(result.status, PdppRunStatus::Succeeded); + } + + #[test] + fn stale_interaction_responder_cannot_answer_the_next_request() { + let first_responder = Arc::new(Mutex::new(None)); + let first_responder_for_hook = first_responder.clone(); + let result = supervise_with_test_deadline( + blocking_interaction_fixture(true), + scoped(), + PdppRunOptions { + on_interaction: Some(Arc::new(move |interaction, responder| { + if interaction.request_id == "request-1" { + *first_responder_for_hook.lock().unwrap() = Some(responder.clone()); + responder.respond(PdppInteractionResponseStatus::Success, None) + } else { + assert!(first_responder_for_hook + .lock() + .unwrap() + .as_ref() + .unwrap() + .respond(PdppInteractionResponseStatus::Cancelled, None) + .is_err()); + responder.respond(PdppInteractionResponseStatus::Success, None) + } + })), + ..options() + }, + ); + assert_eq!( + result.status, + PdppRunStatus::Succeeded, + "{:?}", + result.failure + ); + } + + #[test] + fn rejects_every_connector_output_while_waiting_for_an_interaction_response() { + let outputs = [ + r#"{ type: 'RECORD', stream: 'items', key: 'item-1', data: { id: 'item-1', source_updated_at: '2026-07-30T00:00:00Z' }, emitted_at: '2026-07-30T00:00:00Z' }"#, + r#"{ type: 'STATE', stream: 'items', cursor: { cursor: 'next' } }"#, + r#"{ type: 'PROGRESS', stream: 'items', message: 'working' }"#, + r#"{ type: 'SKIP_RESULT', stream: 'items', reason: 'rate_limited' }"#, + r#"{ type: 'DETAIL_COVERAGE', stream: 'items', state_stream: 'items', required_keys: [], hydrated_keys: [], reference_only: true }"#, + r#"{ type: 'DETAIL_GAP', stream: 'items', record_key: 'item-1', detail_locator: { kind: 'api' }, retryable: true, status: 'pending', reference_only: true }"#, + r#"{ type: 'DETAIL_GAP_RECOVERED', stream: 'items', gap_id: 'gap-1', reference_only: true }"#, + r#"{ type: 'DONE', status: 'succeeded', records_emitted: 0 }"#, + ]; + for output in outputs { + let result = supervise_pdpp_connector( + &output_while_waiting_fixture(output), + &scoped(), + &PdppRunOptions { + on_interaction: Some(Arc::new(|_, _| Ok(()))), + ..options() + }, + ) + .unwrap(); + assert_eq!(result.status, PdppRunStatus::Failed, "{output}"); + assert!(result + .failure + .as_deref() + .is_some_and(|failure| failure.contains("waiting for INTERACTION_RESPONSE"))); + } + } + + #[test] + fn interaction_timeout_contract_is_bounded_and_serializes_the_exact_wire_envelope() { + assert_eq!( + interaction_timeout(None).unwrap(), + DEFAULT_INTERACTION_TIMEOUT + ); + assert_eq!( + interaction_timeout(Some(60)).unwrap(), + MIN_INTERACTION_TIMEOUT + ); + assert!(interaction_timeout(Some(59)).is_err()); + assert!(interaction_timeout(Some(3601)).is_err()); + + let response = + PdppInteractionResponse::new("request-1", PdppInteractionResponseStatus::Timeout, None) + .unwrap(); + let mut wire = Vec::new(); + write_interaction_response(&mut wire, &response).unwrap(); + assert_eq!( + String::from_utf8(wire).unwrap(), + "{\"type\":\"INTERACTION_RESPONSE\",\"request_id\":\"request-1\",\"status\":\"timeout\"}\n" + ); + + let now = Instant::now(); + let expired = PendingInteraction { + generation: 1, + request_id: "request-1".into(), + deadline: now - Duration::from_millis(1), + }; + let active = PendingInteraction { + generation: 1, + request_id: "request-1".into(), + deadline: now + Duration::from_millis(1), + }; + assert!(interaction_deadline_elapsed(&expired, now)); + assert!(!interaction_deadline_elapsed(&active, now)); + } + + #[test] + fn interaction_pauses_the_normal_timeout_then_response_resumes_it() { + let result = supervise_pdpp_connector( + &delayed_after_interaction_fixture(), + &scoped(), + &PdppRunOptions { + timeout: Some(Duration::from_millis(25)), + on_interaction: Some(Arc::new(|_, responder| { + thread::spawn(move || { + thread::sleep(Duration::from_millis(50)); + responder + .respond(PdppInteractionResponseStatus::Success, None) + .unwrap(); + }); + Ok(()) + })), + ..options() + }, + ) + .unwrap(); + + // The 50ms owner interaction exceeds the normal 25ms policy without + // timing out; the connector then exceeds that policy after response. + assert_eq!(result.status, PdppRunStatus::TimedOut); + } + + #[test] + fn cancellation_stops_a_connector_while_its_interaction_is_pending() { + let control = PdppRunControl::default(); + let cancellation = control.clone(); + let interaction_seen = Arc::new(AtomicBool::new(false)); + let interaction_seen_by_hook = interaction_seen.clone(); + let cancellation_worker = thread::spawn(move || { + while !interaction_seen.load(Ordering::SeqCst) { + thread::yield_now(); + } + cancellation.cancel(); + }); + let result = supervise_pdpp_connector( + &blocking_interaction_fixture(false), + &scoped(), + &PdppRunOptions { + control, + on_interaction: Some(Arc::new(move |_, _| { + interaction_seen_by_hook.store(true, Ordering::SeqCst); + Ok(()) + })), + ..options() + }, + ) + .unwrap(); + cancellation_worker.join().unwrap(); + assert_eq!(result.status, PdppRunStatus::Cancelled); + } + #[test] fn requires_a_sink_or_positive_record_retention_before_spawning() { let result = supervise_pdpp_connector( &fixture("success"), diff --git a/src-tauri/src/commands/pdpp_installed_connector.rs b/src-tauri/src/commands/pdpp_installed_connector.rs index 207cebdb..aea8a770 100644 --- a/src-tauri/src/commands/pdpp_installed_connector.rs +++ b/src-tauri/src/commands/pdpp_installed_connector.rs @@ -5,13 +5,16 @@ //! capability, then delegates process supervision to the PDPP connector kernel. use super::connector_store::{get_active_connector_install, ActiveConnectorInstall}; +use super::pdpp_browser::{PdppBrowserBinding, PdppBrowserLease}; use super::pdpp_collection_state::{ - commit_terminal_run, load_connection_state, stage_succeeded_run, PdppCollectionConnectionState, - DEFAULT_CONNECTION_ID, + clear_connection_setup_complete, commit_terminal_run, is_connection_setup_complete, + load_connection_state, mark_connection_setup_complete, stage_succeeded_run, + PdppCollectionConnectionState, DEFAULT_CONNECTION_ID, }; use super::pdpp_connector::{ supervise_pdpp_connector, PdppConnectorCommand, PdppEvent, PdppRecord, PdppRunControl, PdppRunOptions, PdppRunResult, PdppRunStatus, PdppScopeValidators, PdppStart, + PdppInteractionResponder, PdppInteractionResponseStatus, }; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; @@ -33,8 +36,13 @@ const MINIMUM_NODE_MAJOR: u64 = 22; const CLEANUP_WAIT: Duration = Duration::from_secs(2); const GITHUB_CONNECTOR_KEY: &str = "github"; const GITHUB_CONNECTOR_ID: &str = "https://registry.pdpp.org/connectors/github"; +const CHATGPT_CONNECTOR_KEY: &str = "chatgpt"; +const CHATGPT_CONNECTOR_ID: &str = "https://registry.pdpp.org/connectors/chatgpt"; +const CHATGPT_CONNECTOR_INSTALL_ID: &str = "chatgpt-pdpp"; static ACTIVE_PDPP_RUNS: LazyLock>> = LazyLock::new(|| Mutex::new(HashMap::new())); +static PENDING_PDPP_INTERACTIONS: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); /// A run remains registered until its child-supervision task has returned. /// Keeping connector identity alongside the cancellation control makes the @@ -59,6 +67,11 @@ pub struct StartInstalledPdppConnectorRequest { pub connection_id: Option, #[serde(default)] pub github_token: Option, + /// The pinned ChatGPT profile declares exactly two static-secret fields. + /// They are accepted for this invocation only and never enter run state, + /// export data, or a command response. + #[serde(default)] + pub setup_secrets: Option>, #[serde(default)] pub timeout_seconds: Option, } @@ -119,6 +132,7 @@ struct CommandCustomization { max_retained_records: usize, control: PdppRunControl, on_event: Option, + on_interaction: Option, } #[derive(Debug, Deserialize)] @@ -128,9 +142,36 @@ struct PdppConnectorManifest { display_name: Option, version: Option, runtime_requirements: Option, + setup: Option, streams: Vec, } +#[derive(Debug, Deserialize)] +struct PdppStaticSecretSetup { + modality: String, + credential_capture: PdppCredentialCapture, +} + +#[derive(Debug, Deserialize)] +struct PdppCredentialCapture { + fields: Vec, +} + +#[derive(Debug, Deserialize)] +struct PdppStaticSecretField { + name: String, + required: bool, + secret: bool, + #[serde(default)] + env: Vec, +} + +#[derive(Default)] +struct PdppChildSecrets { + environment: HashMap, + values: Vec, +} + #[derive(Debug, Deserialize)] struct RuntimeRequirements { bindings: Option>, @@ -148,6 +189,23 @@ struct PdppManifestStream { consent_time_field: Option, } +#[derive(Debug, Deserialize)] +struct PdppArtifactProvenance { + #[serde(default)] + external_runtime_packages: Vec, +} + +#[derive(Debug, Deserialize)] +struct PdppRuntimePackageRequirement { + name: String, + version: String, +} + +#[derive(Debug, Deserialize)] +struct NodePackageMetadata { + version: String, +} + /// The kernel validates every record before this accumulator sees it. Keeping /// the complete stream here is intentionally distinct from the kernel's small /// diagnostic retention buffers: the former is the user's export, the latter @@ -203,15 +261,16 @@ fn start_installed_pdpp_connector_run_impl( ) -> Result { validate_request(&request)?; let resolved = resolve_active_installed_pdpp_connector(&request.connector_id)?; - let credential = resolve_github_credential(&request, &resolved)?; let saved_state = load_connection_state(&resolved.connector_id, request.connection_id())?; + let setup_complete = chatgpt_setup_complete(&resolved, request.connection_id())?; + let secrets = resolve_child_secrets_for_connection(&request, &resolved, setup_complete)?; let start_state = persisted_start_state(&request, &saved_state); let export_accumulator = Arc::new(Mutex::new(PdppExportAccumulator::default())); let sink = event_sink_for_run( - app, + app.clone(), request.run_id.clone(), export_accumulator.clone(), - credential.clone(), + secrets.values.clone(), ); let result = run_resolved_installed_pdpp_connector_with_state( &resolved, @@ -219,9 +278,14 @@ fn start_installed_pdpp_connector_run_impl( CommandCustomization { control, on_event: Some(sink), + on_interaction: Some(interaction_sink_for_run( + app.clone(), + request.run_id.clone(), + secrets.values.clone(), + )), ..Default::default() }, - credential.as_deref(), + &secrets, start_state, )?; let export = if result.status == PdppRunStatus::Succeeded { @@ -263,6 +327,12 @@ fn start_installed_pdpp_connector_run_impl( &committed_checkpoints, )? .ok_or("PDPP collection state was not committed after a successful run")?; + // Mark setup only after the credentialed run and its export/state + // commit have succeeded. A failed launch, login, cancellation, or + // timeout must leave the next attempt in owner-attended setup. + if should_mark_chatgpt_setup_complete(&resolved, &request, &result.status) { + mark_connection_setup_complete(&resolved.connector_id, request.connection_id())?; + } Some(export) } else { None @@ -272,7 +342,7 @@ fn start_installed_pdpp_connector_run_impl( request.run_id, resolved.connector_id, result, - credential.as_deref(), + &secrets.values, ), export, }) @@ -283,13 +353,13 @@ fn run_resolved_installed_pdpp_connector( resolved: &ResolvedInstalledPdppConnector, request: &StartInstalledPdppConnectorRequest, customization: CommandCustomization, - github_credential: Option<&str>, + secrets: &PdppChildSecrets, ) -> Result { run_resolved_installed_pdpp_connector_with_state( resolved, request, customization, - github_credential, + secrets, None, ) } @@ -298,12 +368,35 @@ fn run_resolved_installed_pdpp_connector_with_state( resolved: &ResolvedInstalledPdppConnector, request: &StartInstalledPdppConnectorRequest, customization: CommandCustomization, - github_credential: Option<&str>, + secrets: &PdppChildSecrets, state: Option, ) -> Result { validate_request(request)?; + let browser_lease = if requires_browser(&resolved.manifest) { + let owner_id = request.connection_id.as_deref().filter(|owner| !owner.is_empty()).ok_or( + "PDPP browser connector requires an explicit connectionId owner; the default owner is not permitted", + )?; + Some(Arc::new(Mutex::new(PdppBrowserLease::launch( + &resolved.connector_id, + owner_id, + &request.run_id, + )?))) + } else { + None + }; + let browser_binding = browser_lease + .as_ref() + .map(|lease| { + lease + .lock() + .map_err(|_| "PDPP browser lease is unavailable") + }) + .transpose()? + .map(|lease| lease.binding().clone()); let start = build_start(request, &resolved.manifest, state)?; - let command = build_command(resolved, github_credential, &customization)?; + let command = build_command(resolved, secrets, &customization, browser_binding.as_ref())?; + let host_interaction = customization.on_interaction.clone(); + let browser_for_interaction = browser_lease.clone(); let options = PdppRunOptions { timeout: Some(Duration::from_secs( request.timeout_seconds.unwrap_or(DEFAULT_TIMEOUT_SECONDS), @@ -317,9 +410,58 @@ fn run_resolved_installed_pdpp_connector_with_state( .unwrap_or_else(|| Arc::new(|_| Ok(()))), ), control: customization.control, + on_interaction: host_interaction.map(|host_interaction| { + Arc::new(move |interaction: &super::pdpp_connector::PdppInteraction, responder| { + if let Some(lease) = &browser_for_interaction { + lease + .lock() + .map_err(|_| "PDPP browser lease is unavailable")? + .mark_waiting_for_user(); + } + host_interaction(interaction, responder) + }) as super::pdpp_connector::PdppInteractionSink + }), + on_interaction_closed: Some(interaction_closed_sink_for_run(request.run_id.clone())), ..Default::default() }; - supervise_pdpp_connector(&command, &start, &options) + let mut result = supervise_pdpp_connector(&command, &start, &options); + if let (Ok(result), Some(binding)) = (&mut result, browser_binding.as_ref()) { + redact_browser_endpoint(result, &binding.cdp_http_url); + } + if let Some(lease) = browser_lease { + if let Ok(mut lease) = lease.lock() { + lease.close(); + } + } + result +} + +fn redact_browser_endpoint(result: &mut PdppRunResult, endpoint: &str) { + if endpoint.is_empty() { + return; + } + result.stderr = result.stderr.replace(endpoint, "[REDACTED_BROWSER_ENDPOINT]"); + if let Some(failure) = &mut result.failure { + *failure = failure.replace(endpoint, "[REDACTED_BROWSER_ENDPOINT]"); + } + for event in &mut result.events { + match event { + PdppEvent::Progress(progress) => { + progress.message = progress.message.replace(endpoint, "[REDACTED_BROWSER_ENDPOINT]"); + } + PdppEvent::SkipResult(skip) => { + if let Some(message) = &mut skip.message { + *message = message.replace(endpoint, "[REDACTED_BROWSER_ENDPOINT]"); + } + } + PdppEvent::Interaction(interaction) => { + interaction.message = interaction + .message + .replace(endpoint, "[REDACTED_BROWSER_ENDPOINT]"); + } + _ => {} + } + } } #[cfg(test)] @@ -336,7 +478,7 @@ fn run_resolved_installed_pdpp_connector_for_test( max_retained_records: 256, ..Default::default() }, - request.github_token.as_deref(), + &resolve_child_secrets(request, resolved)?, ) } @@ -375,6 +517,7 @@ fn unregister_run(run_id: &str) { if let Ok(mut runs) = ACTIVE_PDPP_RUNS.lock() { runs.remove(run_id); } + invalidate_pending_interactions_for_run(run_id); } fn cancel_run(run_id: &str) -> Result<(), String> { @@ -384,6 +527,9 @@ fn cancel_run(run_id: &str) -> Result<(), String> { .get(run_id) .map(|run| run.control.clone()) .ok_or_else(|| format!("PDPP runId {run_id} is not active"))?; + // Remove owner-visible responders before signalling the supervisor. A + // late Tauri command must not claim success while cancellation is queued. + invalidate_pending_interactions_for_run(run_id); control.cancel(); Ok(()) } @@ -394,6 +540,67 @@ pub fn stop_installed_pdpp_connector_run(run_id: String) -> Result<(), String> { cancel_run(&run_id) } +/// Sends one authoritative Collection Profile `INTERACTION_RESPONSE` to the +/// still-pending request for this run. OTP values stay only in this command +/// payload and the child stdin; they are never retained in state, events, or +/// logs. +#[tauri::command] +pub fn submit_installed_pdpp_interaction_response( + run_id: String, + request_id: String, + status: String, + data: Option, +) -> Result<(), String> { + validate_run_id(&run_id)?; + validate_interaction_request_id(&request_id)?; + let status = match status.as_str() { + "success" => PdppInteractionResponseStatus::Success, + "cancelled" => PdppInteractionResponseStatus::Cancelled, + "timeout" => PdppInteractionResponseStatus::Timeout, + _ => return Err("PDPP INTERACTION_RESPONSE status must be success, cancelled, or timeout".into()), + }; + if data.as_ref().is_some_and(|value| !value.is_object()) { + return Err("PDPP INTERACTION_RESPONSE data must be an object".into()); + } + let responder = PENDING_PDPP_INTERACTIONS + .lock() + .map_err(|_| "PDPP interaction registry is unavailable")? + .remove(&(run_id.clone(), request_id.clone())) + .ok_or("PDPP interaction is no longer pending for this run")?; + responder.respond(status, data) +} + +/// Explicitly disconnects an installed PDPP browser connector by deleting the +/// durable, owner-confined authenticated profile. It refuses while that owner +/// has a live lease, so a reset can never race a collection run. +#[tauri::command] +pub fn reset_installed_pdpp_browser_profile( + connector_id: String, + connection_id: String, +) -> Result<(), String> { + if connector_id != CHATGPT_CONNECTOR_INSTALL_ID { + return Err("PDPP browser profile reset is only available for ChatGPT".into()); + } + validate_connection_id(&connection_id)?; + PdppBrowserLease::reset_profile(&connector_id, &connection_id)?; + clear_connection_setup_complete(&connector_id, &connection_id) +} + +/// A non-secret marker and its durable owner profile are both required before +/// the UI may omit recovery credentials for a scheduled ChatGPT run. +#[tauri::command] +pub fn is_installed_pdpp_browser_setup_complete( + connector_id: String, + connection_id: String, +) -> Result { + if connector_id != CHATGPT_CONNECTOR_INSTALL_ID { + return Err("PDPP browser setup state is only available for ChatGPT".into()); + } + validate_connection_id(&connection_id)?; + Ok(is_connection_setup_complete(&connector_id, &connection_id)? + && PdppBrowserLease::profile_exists(&connector_id, &connection_id)?) +} + pub fn cleanup_installed_pdpp_connector_runs() { let controls = ACTIVE_PDPP_RUNS .lock() @@ -461,6 +668,18 @@ fn validate_connection_id(connection_id: &str) -> Result<(), String> { Ok(()) } +fn validate_interaction_request_id(request_id: &str) -> Result<(), String> { + if request_id.is_empty() + || request_id.len() > MAX_RUN_ID_BYTES + || !request_id + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.')) + { + return Err("PDPP interaction requestId must be 1-128 URL-safe identifier characters".into()); + } + Ok(()) +} + fn persisted_start_state( request: &StartInstalledPdppConnectorRequest, saved_state: &PdppCollectionConnectionState, @@ -542,6 +761,7 @@ fn resolve_installed_pdpp_connector( ) .map_err(|e| format!("Failed to parse PDPP connector manifest: {e}"))?; validate_manifest(&install.connector_id, &install.version, &manifest)?; + validate_chatgpt_runtime_requirements(&provenance_path, &manifest)?; Ok(ResolvedInstalledPdppConnector { connector_id: install.connector_id.clone(), manifest_sha256: manifest_sha256.to_owned(), @@ -602,10 +822,21 @@ fn validate_manifest( if manifest.streams.is_empty() { return Err("PDPP connector manifest must declare at least one stream".into()); } - if connector_id != "github-pdpp" - || manifest.connector_key.as_deref() != Some(GITHUB_CONNECTOR_KEY) - || manifest.connector_id.as_deref() != Some(GITHUB_CONNECTOR_ID) - { + let identity_matches = match manifest.connector_key.as_deref() { + Some(GITHUB_CONNECTOR_KEY) => { + connector_id == "github-pdpp" + && manifest.connector_id.as_deref() == Some(GITHUB_CONNECTOR_ID) + && !requires_browser(manifest) + } + Some(CHATGPT_CONNECTOR_KEY) => { + connector_id == CHATGPT_CONNECTOR_INSTALL_ID + && manifest.connector_id.as_deref() == Some(CHATGPT_CONNECTOR_ID) + && requires_browser(manifest) + && required_chatgpt_static_secret_fields(manifest).is_ok() + } + _ => false, + }; + if !identity_matches { return Err(format!( "PDPP connector manifest does not match active install id {connector_id}" )); @@ -625,7 +856,10 @@ fn validate_manifest( return Err("PDPP connector manifest must require the network binding".into()); } for (binding, requirement) in bindings.into_iter().flat_map(|bindings| bindings.iter()) { - if binding != "network" && requirement.required.unwrap_or(false) { + if binding != "network" + && binding != "browser" + && requirement.required.unwrap_or(false) + { return Err(format!( "PDPP connector requires unsupported binding {binding}" )); @@ -640,6 +874,98 @@ fn validate_manifest( Ok(()) } +fn required_chatgpt_static_secret_fields( + manifest: &PdppConnectorManifest, +) -> Result, String> { + let setup = manifest + .setup + .as_ref() + .ok_or("ChatGPT PDPP connector must declare setup")?; + if setup.modality != "static_secret" { + return Err("ChatGPT PDPP connector must use setup.modality static_secret".into()); + } + let fields = &setup.credential_capture.fields; + if fields.len() != 2 + || fields[0].name != "username" + || !fields[0].required + || !fields[0].secret + || fields[0].env != ["CHATGPT_USERNAME"] + || fields[1].name != "password" + || !fields[1].required + || !fields[1].secret + || fields[1].env != ["CHATGPT_PASSWORD"] + { + return Err( + "ChatGPT PDPP setup must declare only required secret username and password fields" + .into(), + ); + } + Ok(vec![ + ("username", "CHATGPT_USERNAME"), + ("password", "CHATGPT_PASSWORD"), + ]) +} + +fn validate_chatgpt_runtime_requirements( + provenance_path: &Path, + manifest: &PdppConnectorManifest, +) -> Result<(), String> { + if manifest.connector_key.as_deref() != Some(CHATGPT_CONNECTOR_KEY) { + return Ok(()); + } + // The distributable tar does not carry build-only artifact.json. The + // artifact builder copies declared external_packages into signed provenance, + // whose hash is verified before this parse. + let provenance: PdppArtifactProvenance = serde_json::from_str( + &fs::read_to_string(provenance_path) + .map_err(|error| format!("Failed to read ChatGPT PDPP provenance: {error}"))?, + ) + .map_err(|error| format!("Failed to parse ChatGPT PDPP provenance: {error}"))?; + let runtime_root = resolve_pdpp_runtime_root()?; + for expected in ["p-queue", "patchright"] { + let requirement = provenance + .external_runtime_packages + .iter() + .find(|package| package.name == expected) + .ok_or_else(|| format!("ChatGPT PDPP provenance must declare {expected}"))?; + let metadata: NodePackageMetadata = serde_json::from_str( + &fs::read_to_string(confined_existing_file( + &runtime_root, + &format!("node_modules/{expected}/package.json"), + &format!("PDPP runtime dependency {expected}"), + )?) + .map_err(|error| format!("Failed to read PDPP runtime dependency {expected}: {error}"))?, + ) + .map_err(|error| format!("Failed to parse PDPP runtime dependency {expected}: {error}"))?; + if !satisfies_caret_requirement(&metadata.version, &requirement.version) { + return Err(format!( + "PDPP runtime dependency {expected}@{} does not satisfy artifact requirement {}", + metadata.version, requirement.version + )); + } + } + Ok(()) +} + +fn satisfies_caret_requirement(installed: &str, requirement: &str) -> bool { + let Some(required) = requirement.strip_prefix('^').and_then(parse_semver) else { + return false; + }; + let Some(installed) = parse_semver(installed) else { + return false; + }; + installed.0 == required.0 && installed >= required +} + +fn parse_semver(value: &str) -> Option<(u64, u64, u64)> { + let mut parts = value.split('.'); + Some(( + parts.next()?.parse().ok()?, + parts.next()?.parse().ok()?, + parts.next()?.split(['-', '+']).next()?.parse().ok()?, + )) +} + fn build_start( request: &StartInstalledPdppConnectorRequest, manifest: &PdppConnectorManifest, @@ -661,13 +987,21 @@ fn build_start( let scope = json!({ "streams": selected.into_iter().map(|name| json!({ "name": name })).collect::>() }); - PdppStart::new( - &request.run_id, - &request.collection_mode, - scope, - state, - json!({ "network": { "enabled": true } }), - ) + PdppStart::new(&request.run_id, &request.collection_mode, scope, state) +} + +fn requires_browser(manifest: &PdppConnectorManifest) -> bool { + manifest + .runtime_requirements + .as_ref() + .and_then(|requirements| requirements.bindings.as_ref()) + .and_then(|bindings| bindings.get("browser")) + .and_then(|binding| binding.required) + .unwrap_or(false) +} + +fn is_chatgpt_connector(manifest: &PdppConnectorManifest) -> bool { + manifest.connector_key.as_deref() == Some(CHATGPT_CONNECTOR_KEY) } fn selected_streams( @@ -742,24 +1076,102 @@ fn validators_from_manifest(manifest: &PdppConnectorManifest) -> PdppScopeValida validators } -fn resolve_github_credential( +fn resolve_child_secrets( + request: &StartInstalledPdppConnectorRequest, + resolved: &ResolvedInstalledPdppConnector, +) -> Result { + resolve_child_secrets_for_connection(request, resolved, false) +} + +fn resolve_child_secrets_for_connection( request: &StartInstalledPdppConnectorRequest, resolved: &ResolvedInstalledPdppConnector, -) -> Result, String> { + setup_complete: bool, +) -> Result { let manifest_key = resolved.manifest.connector_key.as_deref().unwrap_or(""); - if manifest_key != "github" { + if manifest_key == CHATGPT_CONNECTOR_KEY { if request.github_token.is_some() { return Err("githubToken can only be passed to the GitHub PDPP connector".into()); } - return Ok(None); + let expected = required_chatgpt_static_secret_fields(&resolved.manifest)?; + let Some(provided) = request.setup_secrets.as_ref() else { + return if setup_complete { + Ok(PdppChildSecrets::default()) + } else { + Err("ChatGPT PDPP connector requires setupSecrets.username and setupSecrets.password for first setup or explicit recovery".into()) + }; + }; + if provided.len() != expected.len() + || expected.iter().any(|(field, _)| { + provided + .get(*field) + .is_none_or(|value| value.trim().is_empty()) + }) + { + return Err( + "ChatGPT PDPP connector requires only non-empty setupSecrets.username and setupSecrets.password" + .into(), + ); + } + let mut secrets = PdppChildSecrets::default(); + for (field, environment_key) in expected { + let value = provided[field].clone(); + secrets.values.push(value.clone()); + secrets.environment.insert( + environment_key.to_owned(), + value, + ); + } + return Ok(secrets); + } + + if request.setup_secrets.is_some() { + return Err("setupSecrets can only be passed to the ChatGPT PDPP connector".into()); + } + if manifest_key != GITHUB_CONNECTOR_KEY { + return Err("DataConnect does not support this PDPP connector identity".into()); + } + + let token = resolve_github_credential(request)?; + let mut secrets = PdppChildSecrets::default(); + secrets.values.push(token.clone()); + secrets.environment.insert("GITHUB_TOKEN".into(), token.clone()); + secrets + .environment + .insert("GITHUB_PERSONAL_ACCESS_TOKEN".into(), token); + Ok(secrets) +} + +fn chatgpt_setup_complete( + resolved: &ResolvedInstalledPdppConnector, + connection_id: &str, +) -> Result { + if !is_chatgpt_connector(&resolved.manifest) { + return Ok(false); } + is_connection_setup_complete(&resolved.connector_id, connection_id) +} + +fn should_mark_chatgpt_setup_complete( + resolved: &ResolvedInstalledPdppConnector, + request: &StartInstalledPdppConnectorRequest, + status: &PdppRunStatus, +) -> bool { + is_chatgpt_connector(&resolved.manifest) + && request.setup_secrets.is_some() + && *status == PdppRunStatus::Succeeded +} + +fn resolve_github_credential( + request: &StartInstalledPdppConnectorRequest, +) -> Result { if let Some(token) = request .github_token .as_deref() .filter(|token| !token.is_empty()) { - return Ok(Some(token.to_owned())); + return Ok(token.to_owned()); } // A local desktop UAT may use a shell-provided credential, but release @@ -768,7 +1180,7 @@ fn resolve_github_credential( #[cfg(debug_assertions)] if let Ok(token) = std::env::var("PDPP_E2E_GITHUB_TOKEN") { if !token.is_empty() { - return Ok(Some(token)); + return Ok(token); } } @@ -780,17 +1192,22 @@ fn resolve_github_credential( fn build_command( resolved: &ResolvedInstalledPdppConnector, - github_credential: Option<&str>, + secrets: &PdppChildSecrets, customization: &CommandCustomization, + browser_binding: Option<&PdppBrowserBinding>, ) -> Result { - let mut env = HashMap::new(); - if let Some(token) = github_credential { - let manifest_key = resolved.manifest.connector_key.as_deref().unwrap_or(""); - if manifest_key != "github" { - return Err("githubToken can only be passed to the GitHub PDPP connector".into()); + let mut env = secrets.environment.clone(); + if let Some(binding) = browser_binding { + if resolved.manifest.connector_key.as_deref() != Some(CHATGPT_CONNECTOR_KEY) { + return Err("PDPP browser binding is only available to the ChatGPT connector".into()); } - env.insert("GITHUB_TOKEN".into(), token.to_owned()); - env.insert("GITHUB_PERSONAL_ACCESS_TOKEN".into(), token.to_owned()); + // The pinned runtime resolves this documented compatibility seam before it + // considers an isolated local browser. It is a PDPP runtime concern, + // unrelated to DataConnect's legacy Playwright page API. + env.insert( + "PDPP_CHATGPT_REMOTE_CDP_URL".into(), + binding.cdp_http_url.clone(), + ); } env.insert("PDPP_CONNECTOR_NETWORK".into(), "1".into()); env.insert( @@ -805,6 +1222,18 @@ fn build_command( args.push("--import".into()); args.push(import.to_string_lossy().into_owned()); } + let runtime_root = resolve_pdpp_runtime_root()?; + args.push("--import".into()); + args.push( + runtime_root + .join("connector-loader-bootstrap.mjs") + .to_string_lossy() + .into_owned(), + ); + env.insert( + "DATACONNECT_PDPP_RUNTIME_ROOT".into(), + runtime_root.to_string_lossy().into_owned(), + ); args.push(resolved.entrypoint_path.to_string_lossy().into_owned()); Ok(PdppConnectorCommand { program: resolve_node_program()?, @@ -815,18 +1244,65 @@ fn build_command( }) } +fn resolve_pdpp_runtime_root() -> Result { + if let Some(configured) = std::env::var_os("DATACONNECT_PDPP_RUNTIME_ROOT") { + let root = canonical_existing_dir(Path::new(&configured), "PDPP runtime root")?; + return validate_pdpp_runtime_root(root); + } + let mut candidates = vec![PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("pdpp-runtime")]; + if let Ok(executable) = std::env::current_exe() { + if let Some(macos_resources) = executable + .parent() + .and_then(Path::parent) + .map(|contents| contents.join("Resources").join("pdpp-runtime")) + { + candidates.push(macos_resources); + } + if let Some(executable_dir) = executable.parent() { + candidates.push(executable_dir.join("pdpp-runtime")); + candidates.push(executable_dir.join("resources").join("pdpp-runtime")); + } + } + for candidate in candidates { + if let Ok(root) = canonical_existing_dir(&candidate, "PDPP runtime root") { + if let Ok(root) = validate_pdpp_runtime_root(root) { + return Ok(root); + } + } + } + Err("Packaged PDPP runtime dependencies p-queue and patchright are unavailable".into()) +} + +fn validate_pdpp_runtime_root(root: PathBuf) -> Result { + confined_existing_file(&root, "connector-loader.mjs", "PDPP runtime loader")?; + confined_existing_file( + &root, + "connector-loader-bootstrap.mjs", + "PDPP runtime loader bootstrap", + )?; + for package in ["p-queue", "patchright"] { + confined_existing_file( + &root, + &format!("node_modules/{package}/package.json"), + &format!("PDPP runtime dependency {package}"), + )?; + } + Ok(root) +} + fn event_sink_for_run( app: AppHandle, run_id: String, export_accumulator: Arc>, - secret: Option, + secrets: Vec, ) -> super::pdpp_connector::PdppEventSink { Arc::new(move |event| match event { PdppEvent::Record(record) => { - if secret.as_deref().is_some_and(|credential| { - value_contains_secret(&record.key, credential) - || value_contains_secret(&record.data, credential) - }) { + if value_contains_any_secret(&record.key, &secrets) + || value_contains_any_secret(&record.data, &secrets) + { return Err("PDPP connector attempted to emit its credential as data".into()); } let mut collected = export_accumulator @@ -843,7 +1319,7 @@ fn event_sink_for_run( emit_running_status( &app, &run_id, - &redact_secret(&progress.message, secret.as_deref()), + &redact_secrets(&progress.message, &secrets), ); Ok(()) } @@ -864,11 +1340,66 @@ fn event_sink_for_run( PdppEvent::State(_) | PdppEvent::DetailCoverage(_) | PdppEvent::DetailGap(_) - | PdppEvent::DetailGapRecovered(_) - | PdppEvent::Interaction(_) => Ok(()), + | PdppEvent::DetailGapRecovered(_) => Ok(()), + PdppEvent::Interaction(_) => { + emit_running_status(&app, &run_id, "Waiting for browser interaction..."); + Ok(()) + } + }) +} + +fn interaction_sink_for_run( + app: AppHandle, + run_id: String, + secrets: Vec, +) -> super::pdpp_connector::PdppInteractionSink { + Arc::new(move |interaction: &super::pdpp_connector::PdppInteraction, responder| { + if responder.run_id() != run_id || interaction.request_id != responder.request_id() { + return Err("PDPP interaction responder is not bound to this run/request".into()); + } + validate_interaction_request_id(&interaction.request_id)?; + let key = (run_id.clone(), interaction.request_id.clone()); + PENDING_PDPP_INTERACTIONS + .lock() + .map_err(|_| "PDPP interaction registry is unavailable")? + .insert(key, responder); + let message = redact_secrets(&interaction.message, &secrets); + emit_running_status(&app, &run_id, "Waiting for owner interaction..."); + let _ = app.emit( + "pdpp-interaction", + json!({ + "runId": run_id, + "requestId": interaction.request_id, + "kind": interaction.kind, + "message": message, + "schema": interaction.schema, + "timeoutSeconds": interaction.timeout_seconds, + }), + ); + Ok(()) }) } +fn interaction_closed_sink_for_run(run_id: String) -> super::pdpp_connector::PdppInteractionClosedSink { + Arc::new(move |closed_run_id, request_id| { + if closed_run_id == run_id { + invalidate_pending_interaction(closed_run_id, request_id); + } + }) +} + +fn invalidate_pending_interaction(run_id: &str, request_id: &str) { + if let Ok(mut pending) = PENDING_PDPP_INTERACTIONS.lock() { + pending.remove(&(run_id.to_owned(), request_id.to_owned())); + } +} + +fn invalidate_pending_interactions_for_run(run_id: &str) { + if let Ok(mut pending) = PENDING_PDPP_INTERACTIONS.lock() { + pending.retain(|(pending_run_id, _), _| pending_run_id != run_id); + } +} + fn value_contains_secret(value: &Value, secret: &str) -> bool { if secret.is_empty() { return false; @@ -885,6 +1416,12 @@ fn value_contains_secret(value: &Value, secret: &str) -> bool { } } +fn value_contains_any_secret(value: &Value, secrets: &[String]) -> bool { + secrets + .iter() + .any(|secret| value_contains_secret(value, secret)) +} + fn build_export_data( resolved: &ResolvedInstalledPdppConnector, request: &StartInstalledPdppConnectorRequest, @@ -901,7 +1438,7 @@ fn build_export_data( .connector_id .as_deref() .ok_or("PDPP connector manifest is missing connector_id")?; - if connector_key != GITHUB_CONNECTOR_KEY { + if connector_key != GITHUB_CONNECTOR_KEY && connector_key != CHATGPT_CONNECTOR_KEY { return Err( "DataConnect does not yet have a storage projection for this PDPP connector".into(), ); @@ -916,18 +1453,27 @@ fn build_export_data( let records = records_by_stream.get(stream).cloned().unwrap_or_default(); record_count += records.len(); stream_counts.insert(stream.clone(), json!(records.len())); - let projection = match stream.as_str() { + let projection = match (connector_key, stream.as_str()) { // The Personal Server's existing GitHub schemas deliberately use // these shapes. This is a DataConnect storage projection, not a // claim that the PDPP connector only supports three streams. - "user" if !records.is_empty() => { + (GITHUB_CONNECTOR_KEY, "user") if !records.is_empty() => { Some(("github.profile", project_github_profile(&records)?)) } - "repositories" => Some(( + (GITHUB_CONNECTOR_KEY, "repositories") => Some(( "github.repositories", project_github_repositories(&records)?, )), - "starred" => Some(("github.starred", project_github_starred(&records)?)), + (GITHUB_CONNECTOR_KEY, "starred") => { + Some(("github.starred", project_github_starred(&records)?)) + } + // Fixture contract: the ChatGPT Collection Profile emits one + // PDPP record per conversation. Preserve each record's data as + // supplied; schema-specific normalization belongs upstream. + (CHATGPT_CONNECTOR_KEY, "conversations") => Some(( + "chatgpt.conversations", + json!({ "conversations": records.iter().map(|record| record.data.clone()).collect::>() }), + )), _ => None, }; if let Some((scope, value)) = projection { @@ -999,7 +1545,7 @@ fn build_export_data( "count": record_count, "label": format!("{record_count} {connector_key} records exported"), "details": { - "pdppStorageProjection": "github-v1", + "pdppStorageProjection": if connector_key == GITHUB_CONNECTOR_KEY { "github-v1" } else { "chatgpt-fixture-v1" }, "pdppStreamRecords": stream_counts, } }), @@ -1356,9 +1902,9 @@ fn to_response( run_id: String, connector_id: String, result: PdppRunResult, - secret: Option<&str>, + secrets: &[String], ) -> InstalledPdppConnectorRunResponse { - let progress = sanitize_retained_events(result.events, secret); + let progress = sanitize_retained_events(result.events, secrets); let event_summary = PdppEventSummary { records: result.event_counts.records, checkpoint_updates: result.event_counts.checkpoint_updates, @@ -1393,7 +1939,7 @@ fn to_response( progress, records_truncated: result.records_truncated, events_truncated: result.events_truncated, - failure: failure.map(|failure| redact_secret(&failure, secret)), + failure: failure.map(|failure| redact_secrets(&failure, secrets)), stderr_bytes: result.stderr.len(), stderr_truncated: result.stderr_truncated, exit_code: result.exit_code, @@ -1402,7 +1948,7 @@ fn to_response( fn sanitize_retained_events( events: Vec, - secret: Option<&str>, + secrets: &[String], ) -> Vec { let mut messages = Vec::new(); for event in events { @@ -1411,21 +1957,21 @@ fn sanitize_retained_events( messages.push(SanitizedConnectorMessage { message_type: "PROGRESS".into(), stream: progress.stream, - message: Some(redact_secret(&progress.message, secret)), + message: Some(redact_secrets(&progress.message, secrets)), }); } PdppEvent::SkipResult(skip) => { messages.push(SanitizedConnectorMessage { message_type: "SKIP_RESULT".into(), stream: skip.stream, - message: skip.message.map(|message| redact_secret(&message, secret)), + message: skip.message.map(|message| redact_secrets(&message, secrets)), }); } PdppEvent::Interaction(interaction) => { messages.push(SanitizedConnectorMessage { message_type: "INTERACTION".into(), stream: None, - message: Some(redact_secret(&interaction.message, secret)), + message: Some(redact_secrets(&interaction.message, secrets)), }); } PdppEvent::Record(_) @@ -1438,11 +1984,14 @@ fn sanitize_retained_events( messages } -fn redact_secret(value: &str, secret: Option<&str>) -> String { - match secret { - Some(secret) if !secret.is_empty() => value.replace(secret, "[REDACTED]"), - _ => value.to_string(), - } +fn redact_secrets(value: &str, secrets: &[String]) -> String { + secrets.iter().fold(value.to_owned(), |redacted, secret| { + if secret.is_empty() { + redacted + } else { + redacted.replace(secret, "[REDACTED]") + } + }) } #[cfg(test)] @@ -1465,13 +2014,24 @@ mod tests { ) .unwrap(); fs::write(temp.path().join("dist/profile.cjs"), script).unwrap(); - fs::write(temp.path().join("provenance.json"), "{}").unwrap(); + let provenance = if manifest["connector_key"] == CHATGPT_CONNECTOR_KEY { + json!({ + "external_runtime_packages": [ + { "name": "p-queue", "version": "^9.3.3" }, + { "name": "patchright", "version": "^1.61.1" } + ] + }) + } else { + json!({}) + }; + let provenance_bytes = serde_json::to_vec_pretty(&provenance).unwrap(); + fs::write(temp.path().join("provenance.json"), &provenance_bytes).unwrap(); let manifest_sha = format!( "sha256:{:x}", Sha256::digest(&serde_json::to_vec_pretty(&manifest).unwrap()) ); let entrypoint_sha = format!("sha256:{:x}", Sha256::digest(script.as_bytes())); - let provenance_sha = format!("sha256:{:x}", Sha256::digest(b"{}")); + let provenance_sha = format!("sha256:{:x}", Sha256::digest(&provenance_bytes)); ( temp, ActiveConnectorInstall { @@ -1530,6 +2090,121 @@ mod tests { }) } + fn chatgpt_browser_manifest() -> Value { + serde_json::from_str(include_str!( + "../../tests/fixtures/chatgpt-pdpp-browser.collection-profile.json" + )) + .unwrap() + } + + fn chatgpt_artifact_root() -> Option { + std::env::var_os("PDPP_CHATGPT_ARTIFACT_ROOT").map(PathBuf::from) + } + + fn sha256_for(path: &Path) -> String { + format!("sha256:{:x}", Sha256::digest(fs::read(path).unwrap())) + } + + fn unpacked_actual_chatgpt_install( + artifact_root: &Path, + root: &Path, + ) -> ActiveConnectorInstall { + let artifact_dir = artifact_root.join("artifacts/chatgpt-pdpp"); + let artifact = fs::read_dir(&artifact_dir) + .expect("PDPP_CHATGPT_ARTIFACT_ROOT must contain artifacts/chatgpt-pdpp") + .filter_map(Result::ok) + .map(|entry| entry.path()) + .find(|path| path.extension().is_some_and(|extension| extension == "tgz")) + .expect("PDPP_CHATGPT_ARTIFACT_ROOT must contain one chatgpt-pdpp tarball"); + let status = Command::new("tar") + .args(["-xzf", artifact.to_str().unwrap(), "-C", root.to_str().unwrap()]) + .status() + .unwrap(); + assert!(status.success()); + let version = serde_json::from_str::( + &fs::read_to_string(root.join("profile/collection-profile.json")).unwrap(), + ) + .unwrap()["version"] + .as_str() + .unwrap() + .to_owned(); + ActiveConnectorInstall { + connector_id: CHATGPT_CONNECTOR_INSTALL_ID.into(), + company: "OpenAI".into(), + version, + root_path: root.to_string_lossy().into_owned(), + metadata_relative_path: "legacy.json".into(), + script_relative_path: "legacy.js".into(), + artifact_kind: Some(PDPP_ARTIFACT_KIND.into()), + manifest_path: Some("profile/collection-profile.json".into()), + entrypoint_path: Some("dist/collection-profile.mjs".into()), + entrypoint_sha256: Some(sha256_for(&root.join("dist/collection-profile.mjs"))), + manifest_sha256: Some(sha256_for(&root.join("profile/collection-profile.json"))), + provenance_path: Some("provenance.json".into()), + provenance_sha256: Some(sha256_for(&root.join("provenance.json"))), + } + } + + fn node_major(program: &Path) -> Option { + let output = Command::new(program).arg("--version").output().ok()?; + if !output.status.success() { + return None; + } + String::from_utf8(output.stdout) + .ok()? + .trim() + .trim_start_matches('v') + .split('.') + .next()? + .parse() + .ok() + } + + fn optional_node_for_major(major: u64) -> Option { + let environment_name = format!("DATACONNECT_NODE_{major}"); + let configured = std::env::var_os(environment_name).map(PathBuf::from); + let mut candidates = configured + .into_iter() + .chain(std::iter::once(PathBuf::from(format!("node{major}")))); + candidates.find(|candidate| node_major(candidate.as_path()) == Some(major)) + } + + fn assert_actual_patchright_esm_import(root: &Path, node: &Path) { + let runtime_root = resolve_pdpp_runtime_root().unwrap(); + let bootstrap = runtime_root.join("connector-loader-bootstrap.mjs"); + let probe = r#" +const { default: PQueue } = await import("p-queue"); +const { chromium } = await import("patchright"); +if (typeof PQueue !== "function") throw new Error("p-queue default export is unavailable"); +if (!chromium || typeof chromium.connectOverCDP !== "function") { + throw new Error("patchright ESM chromium export is unavailable"); +} +process.stdout.write("packaged-externals-ok\n"); +"#; + let positive = Command::new(node) + .args(["--import", bootstrap.to_str().unwrap(), "--input-type=module", "-e", probe]) + .current_dir(root) + .env_clear() + .env("DATACONNECT_PDPP_RUNTIME_ROOT", &runtime_root) + .output() + .unwrap(); + assert!( + positive.status.success(), + "loader import failed: {}", + String::from_utf8_lossy(&positive.stderr) + ); + assert_eq!(positive.stdout, b"packaged-externals-ok\n"); + + let negative = Command::new(node) + .args(["--input-type=module", "-e", "await import('patchright')"]) + .current_dir(root) + .env_clear() + .output() + .unwrap(); + assert!(!negative.status.success()); + assert!(String::from_utf8_lossy(&negative.stderr).contains("Cannot find package 'patchright'")); + } + fn success_script() -> &'static str { r#" const readline = require('node:readline'); @@ -1556,6 +2231,68 @@ rl.on('line', (line) => { "# } + fn pending_interaction_command() -> PdppConnectorCommand { + PdppConnectorCommand { + program: "node".into(), + args: vec![ + "-e".into(), + r#" +const readline = require('node:readline'); +const emit = message => process.stdout.write(`${JSON.stringify(message)}\n`); +readline.createInterface({ input: process.stdin }).on('line', line => { + const message = JSON.parse(line); + if (message.type === 'START') { + emit({ type: 'INTERACTION', request_id: 'pending-request', kind: 'otp', message: 'Enter code', timeout_seconds: 60 }); + } +}); +"# + .into(), + ], + cwd: None, + env: HashMap::new(), + clear_env: false, + } + } + + fn start_pending_interaction_for_test( + run_id: &'static str, + ) -> (PdppRunControl, thread::JoinHandle) { + let control = register_run(run_id, "pending-interaction-test", "owner").unwrap(); + let (ready_sender, ready_receiver) = std::sync::mpsc::channel(); + let run_id_owned = run_id.to_owned(); + let on_interaction: crate::commands::pdpp_connector::PdppInteractionSink = Arc::new( + move |_interaction: &crate::commands::pdpp_connector::PdppInteraction, responder| { + PENDING_PDPP_INTERACTIONS + .lock() + .unwrap() + .insert((run_id_owned.clone(), "pending-request".into()), responder); + ready_sender.send(()).unwrap(); + Ok(()) + }, + ); + let options = PdppRunOptions { + control: control.clone(), + max_retained_records: 1, + on_interaction: Some(on_interaction), + on_interaction_closed: Some(interaction_closed_sink_for_run(run_id.into())), + ..Default::default() + }; + let start = PdppStart::new( + run_id, + "incremental", + json!({"streams":[{"name":"items"}]}), + None, + ) + .unwrap(); + let handle = thread::spawn(move || { + supervise_pdpp_connector(&pending_interaction_command(), &start, &options).unwrap() + }); + ready_receiver + .recv_timeout(Duration::from_secs(2)) + .expect("pending interaction should register before the test continues"); + (control, handle) + } + fn request_with_token(token: &str) -> StartInstalledPdppConnectorRequest { StartInstalledPdppConnectorRequest { run_id: "run-1".into(), @@ -1564,6 +2301,7 @@ rl.on('line', (line) => { streams: vec!["repositories".into()], connection_id: None, github_token: Some(token.into()), + setup_secrets: None, timeout_seconds: Some(5), } } @@ -1637,7 +2375,7 @@ rl.on('line', (line) => { } #[test] - fn rejects_unsupported_browser_binding_for_network_only_host() { + fn rejects_a_browser_binding_for_the_network_only_github_host() { let manifest = json!({ "connector_id": GITHUB_CONNECTOR_ID, "connector_key": GITHUB_CONNECTOR_KEY, @@ -1645,7 +2383,7 @@ rl.on('line', (line) => { "runtime_requirements": { "bindings": { "network": { "required": true }, - "browser_automation": { "required": true } + "browser": { "required": true } } }, "streams": [{ "name": "repositories" }] @@ -1654,7 +2392,366 @@ rl.on('line', (line) => { install.root_path = temp.path().to_string_lossy().into_owned(); assert!(resolve_installed_pdpp_connector(&install) .unwrap_err() - .contains("unsupported binding")); + .contains("does not match")); + } + + #[test] + fn admits_the_actual_chatgpt_browser_capability_without_extending_start() { + let manifest: PdppConnectorManifest = + serde_json::from_value(chatgpt_browser_manifest()).unwrap(); + validate_manifest(CHATGPT_CONNECTOR_INSTALL_ID, "0.1.0", &manifest).unwrap(); + let request = StartInstalledPdppConnectorRequest { + run_id: "chatgpt-browser-fixture".into(), + connector_id: CHATGPT_CONNECTOR_INSTALL_ID.into(), + collection_mode: "incremental".into(), + streams: vec!["conversations".into()], + connection_id: Some("account-one".into()), + github_token: None, + setup_secrets: Some(HashMap::from([ + ("username".into(), "owner@example.com".into()), + ("password".into(), "fixture-password".into()), + ])), + timeout_seconds: Some(5), + }; + let start = build_start(&request, &manifest, None).unwrap(); + let serialized = serde_json::to_value(start).unwrap(); + assert!(serialized.get("bindings").is_none()); + assert!(serialized.get("browser").is_none()); + } + + #[test] + fn chatgpt_static_secrets_are_transient_after_first_owner_setup() { + let (temp, mut install) = install_fixture(chatgpt_browser_manifest(), success_script()); + install.root_path = temp.path().to_string_lossy().into_owned(); + install.connector_id = CHATGPT_CONNECTOR_INSTALL_ID.into(); + install.version = "0.1.0".into(); + let resolved = resolve_installed_pdpp_connector(&install).unwrap(); + let first_setup = StartInstalledPdppConnectorRequest { + run_id: "chatgpt-first-setup".into(), + connector_id: CHATGPT_CONNECTOR_INSTALL_ID.into(), + collection_mode: "incremental".into(), + streams: vec!["conversations".into()], + connection_id: Some("owner-a".into()), + github_token: None, + setup_secrets: Some(HashMap::from([ + ("username".into(), "owner@example.com".into()), + ("password".into(), "not-persisted".into()), + ])), + timeout_seconds: Some(5), + }; + let first = resolve_child_secrets_for_connection(&first_setup, &resolved, false).unwrap(); + assert_eq!(first.environment["CHATGPT_USERNAME"], "owner@example.com"); + assert_eq!(first.environment["CHATGPT_PASSWORD"], "not-persisted"); + + let scheduled = StartInstalledPdppConnectorRequest { + run_id: "chatgpt-scheduled".into(), + setup_secrets: None, + ..first_setup.clone() + }; + let second = resolve_child_secrets_for_connection(&scheduled, &resolved, true).unwrap(); + assert!(second.environment.is_empty()); + assert!(second.values.is_empty()); + assert!(matches!( + resolve_child_secrets_for_connection(&scheduled, &resolved, false), + Err(error) if error.contains("first setup or explicit recovery") + )); + } + + #[test] + fn failed_or_interrupted_chatgpt_setup_keeps_the_next_run_in_setup() { + let (temp, mut install) = install_fixture(chatgpt_browser_manifest(), success_script()); + install.root_path = temp.path().to_string_lossy().into_owned(); + install.connector_id = CHATGPT_CONNECTOR_INSTALL_ID.into(); + install.version = "0.1.0".into(); + let resolved = resolve_installed_pdpp_connector(&install).unwrap(); + let credentialed = StartInstalledPdppConnectorRequest { + run_id: "chatgpt-first-setup".into(), + connector_id: CHATGPT_CONNECTOR_INSTALL_ID.into(), + collection_mode: "incremental".into(), + streams: vec!["conversations".into()], + connection_id: Some("owner-a".into()), + github_token: None, + setup_secrets: Some(HashMap::from([ + ("username".into(), "owner@example.com".into()), + ("password".into(), "not-persisted".into()), + ])), + timeout_seconds: Some(5), + }; + let retry_without_secrets = StartInstalledPdppConnectorRequest { + run_id: "chatgpt-retry".into(), + setup_secrets: None, + ..credentialed.clone() + }; + + for status in [ + PdppRunStatus::Failed, // login/run error or browser launch failure + PdppRunStatus::Cancelled, + PdppRunStatus::TimedOut, + ] { + assert!(!should_mark_chatgpt_setup_complete( + &resolved, + &credentialed, + &status + )); + assert!(matches!( + resolve_child_secrets_for_connection(&retry_without_secrets, &resolved, false), + Err(error) if error.contains("first setup or explicit recovery") + )); + } + assert!(should_mark_chatgpt_setup_complete( + &resolved, + &credentialed, + &PdppRunStatus::Succeeded + )); + } + + #[test] + fn cancelling_a_pending_interaction_rejects_late_tauri_responses() { + let _guard = RUN_REGISTRY_TEST_LOCK.lock().unwrap(); + let run_id = "cancel-pending-interaction"; + let (_control, handle) = start_pending_interaction_for_test(run_id); + + cancel_run(run_id).unwrap(); + assert!(submit_installed_pdpp_interaction_response( + run_id.into(), + "pending-request".into(), + "success".into(), + Some(json!({"code":"late"})), + ) + .unwrap_err() + .contains("no longer pending")); + assert_eq!(handle.join().unwrap().status, PdppRunStatus::Cancelled); + unregister_run(run_id); + } + + #[test] + fn timed_out_interaction_close_callback_rejects_late_tauri_responses() { + let _guard = RUN_REGISTRY_TEST_LOCK.lock().unwrap(); + let run_id = "timeout-pending-interaction"; + let (control, handle) = start_pending_interaction_for_test(run_id); + + interaction_closed_sink_for_run(run_id.into())(run_id, "pending-request"); + assert!(submit_installed_pdpp_interaction_response( + run_id.into(), + "pending-request".into(), + "success".into(), + Some(json!({"code":"late"})), + ) + .unwrap_err() + .contains("no longer pending")); + control.cancel(); + assert_eq!(handle.join().unwrap().status, PdppRunStatus::Cancelled); + unregister_run(run_id); + } + + #[test] + fn chatgpt_fixture_tracks_configured_artifact_manifest_contract() { + let Some(artifact_root) = chatgpt_artifact_root() else { + return; + }; + let actual = fs::read_to_string( + artifact_root.join("connectors/chatgpt-pdpp/collection-profile.json"), + ) + .unwrap(); + let actual: Value = serde_json::from_str(&actual).unwrap(); + let fixture = chatgpt_browser_manifest(); + assert_eq!(fixture["connector_id"], actual["connector_id"]); + assert_eq!(fixture["runtime_requirements"]["bindings"], actual["runtime_requirements"]["bindings"]); + assert_eq!(fixture["setup"]["modality"], actual["setup"]["modality"]); + assert_eq!( + fixture["setup"]["credential_capture"]["fields"], + actual["setup"]["credential_capture"]["fields"] + ); + assert_eq!( + fixture["streams"].as_array().unwrap().iter().map(|stream| &stream["name"]).collect::>(), + actual["streams"].as_array().unwrap().iter().map(|stream| &stream["name"]).collect::>(), + ); + } + + #[test] + fn packaged_runtime_matches_configured_artifact_requirements() { + let Some(artifact_root) = chatgpt_artifact_root() else { + return; + }; + let artifact: Value = serde_json::from_str( + &fs::read_to_string( + artifact_root.join("connectors/chatgpt-pdpp/artifact.json"), + ) + .unwrap(), + ) + .unwrap(); + let provenance: Value = serde_json::from_str( + &fs::read_to_string( + artifact_root.join("connectors/chatgpt-pdpp/provenance.json"), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!( + artifact["build"]["external_packages"], + provenance["external_runtime_packages"] + ); + let runtime_root = resolve_pdpp_runtime_root().unwrap(); + for requirement in artifact["build"]["external_packages"].as_array().unwrap() { + let name = requirement["name"].as_str().unwrap(); + let required = requirement["version"].as_str().unwrap(); + let metadata: NodePackageMetadata = serde_json::from_str( + &fs::read_to_string(runtime_root.join("node_modules").join(name).join("package.json")) + .unwrap(), + ) + .unwrap(); + assert!(satisfies_caret_requirement(&metadata.version, required)); + } + } + + #[test] + fn actual_artifact_loader_uses_patchright_esm_chromium_export() { + let Some(artifact_root) = chatgpt_artifact_root() else { + return; + }; + let temp = tempfile::tempdir().unwrap(); + unpacked_actual_chatgpt_install(&artifact_root, temp.path()); + + // The configured Node is always verified. Exercise Node 22 and 23 as + // well when CI or a developer has explicitly provided either binary. + let configured = PathBuf::from(resolve_node_program().unwrap()); + assert_actual_patchright_esm_import(temp.path(), &configured); + for major in [22, 23] { + if let Some(node) = optional_node_for_major(major) { + if node != configured { + assert_actual_patchright_esm_import(temp.path(), &node); + } + } + } + } + + #[test] + fn unpacks_and_starts_the_configured_artifact_with_packaged_externals() { + let Some(artifact_root) = chatgpt_artifact_root() else { + return; + }; + let temp = tempfile::tempdir().unwrap(); + let install = unpacked_actual_chatgpt_install(&artifact_root, temp.path()); + let resolved = resolve_installed_pdpp_connector(&install).unwrap(); + let request = StartInstalledPdppConnectorRequest { + run_id: "actual-chatgpt-artifact".into(), + connector_id: CHATGPT_CONNECTOR_INSTALL_ID.into(), + collection_mode: "incremental".into(), + streams: vec!["conversations".into()], + connection_id: Some("actual-artifact-owner".into()), + github_token: None, + // A scheduled collection reuses its owner profile without replaying + // the initial static-secret handoff. + setup_secrets: None, + timeout_seconds: Some(5), + }; + let binding = PdppBrowserBinding { + backend: "neko", + cdp_http_url: "http://127.0.0.1:9".into(), + lease_id: "actual-artifact-lease".into(), + profile_key: "actual-artifact-profile".into(), + }; + let secrets = resolve_child_secrets_for_connection(&request, &resolved, true).unwrap(); + let command = build_command( + &resolved, + &secrets, + &CommandCustomization { + max_retained_records: 1, + ..Default::default() + }, + Some(&binding), + ) + .unwrap(); + assert!(!command.env.contains_key("CHATGPT_USERNAME")); + assert!(!command.env.contains_key("CHATGPT_PASSWORD")); + assert_eq!(command.env["PDPP_CHATGPT_REMOTE_CDP_URL"], "http://127.0.0.1:9"); + assert!(!command.env.contains_key("PDPP_BROWSER_SURFACE_REQUIRED")); + assert!(command.clear_env); + let loader_index = command + .args + .iter() + .position(|argument| argument == "--import") + .expect("Node 22 loader hook must be installed with --import"); + assert!(command.args[loader_index + 1].ends_with("connector-loader-bootstrap.mjs")); + + let mut without_loader = command.clone(); + without_loader.args.drain(loader_index..=loader_index + 1); + without_loader.env.remove("DATACONNECT_PDPP_RUNTIME_ROOT"); + let negative = supervise_pdpp_connector( + &without_loader, + &build_start(&request, &resolved.manifest, None).unwrap(), + &PdppRunOptions { + timeout: Some(Duration::from_secs(5)), + max_retained_records: 1, + ..Default::default() + }, + ) + .unwrap(); + assert_ne!(negative.status, PdppRunStatus::Succeeded); + assert!(negative.stderr.contains("Cannot find package 'p-queue'")); + + let mut result = supervise_pdpp_connector( + &command, + &build_start(&request, &resolved.manifest, None).unwrap(), + &PdppRunOptions { + timeout: Some(Duration::from_secs(5)), + max_retained_records: 1, + ..Default::default() + }, + ) + .unwrap(); + assert_ne!(result.status, PdppRunStatus::Succeeded); + assert!(!result.stderr.contains("Cannot find package 'p-queue'")); + assert!(!result.stderr.contains("Cannot find package 'patchright'")); + assert!(result.stderr.contains("remote CDP attach start")); + redact_browser_endpoint(&mut result, "http://127.0.0.1:9"); + assert!(!result.stderr.contains("http://127.0.0.1:9")); + assert!(!result + .failure + .as_deref() + .unwrap_or_default() + .contains("http://127.0.0.1:9")); + } + + #[test] + fn projects_the_provisional_chatgpt_conversation_stream_without_schema_rewrite() { + let (temp, mut install) = install_fixture(chatgpt_browser_manifest(), success_script()); + install.root_path = temp.path().to_string_lossy().into_owned(); + install.connector_id = CHATGPT_CONNECTOR_INSTALL_ID.into(); + install.version = "0.1.0".into(); + let resolved = resolve_installed_pdpp_connector(&install).unwrap(); + let request = StartInstalledPdppConnectorRequest { + run_id: "chatgpt-projection".into(), + connector_id: CHATGPT_CONNECTOR_INSTALL_ID.into(), + collection_mode: "incremental".into(), + streams: vec!["conversations".into()], + connection_id: Some("account-one".into()), + github_token: None, + setup_secrets: Some(HashMap::from([ + ("username".into(), "owner@example.com".into()), + ("password".into(), "fixture-password".into()), + ])), + timeout_seconds: Some(5), + }; + let state = PdppCollectionConnectionState { + snapshot_by_stream: HashMap::from([( + "conversations".into(), + vec![PdppRecord { + stream: "conversations".into(), + key: json!("conversation-1"), + data: json!({ "id": "conversation-1", "upstream_field": "preserved" }), + emitted_at: "2026-07-31T00:00:00Z".into(), + op: None, + }], + )]), + ..Default::default() + }; + let export = build_export_data(&resolved, &request, &state, &[]).unwrap(); + assert_eq!(export["requestedScopes"], json!(["chatgpt.conversations"])); + assert_eq!( + export["chatgpt.conversations"]["conversations"][0]["upstream_field"], + json!("preserved") + ); } #[test] @@ -1666,7 +2763,7 @@ rl.on('line', (line) => { "runtime_requirements": { "bindings": { "network": { "required": true }, - "browser_automation": { "required": false } + "browser": { "required": false } } }, "streams": [{ "name": "repositories" }] @@ -1738,6 +2835,7 @@ rl.on('line', (line) => { streams: vec!["repositories".into()], connection_id: None, github_token: None, + setup_secrets: None, timeout_seconds: None, }; let manifest: PdppConnectorManifest = serde_json::from_value(github_manifest()).unwrap(); @@ -1760,6 +2858,7 @@ rl.on('line', (line) => { streams: vec![], connection_id: None, github_token: None, + setup_secrets: None, timeout_seconds: None, }; let manifest: PdppConnectorManifest = @@ -1790,6 +2889,7 @@ rl.on('line', (line) => { streams: vec!["user_stats".into(), "pull_requests".into()], connection_id: None, github_token: None, + setup_secrets: None, timeout_seconds: None, }; let manifest: PdppConnectorManifest = @@ -1967,6 +3067,7 @@ rl.on('line', (line) => { streams: vec![], connection_id: Some("account-one".into()), github_token: None, + setup_secrets: None, timeout_seconds: None, }; @@ -2052,6 +3153,7 @@ rl.on('line', (line) => { streams: vec!["repositories".into()], connection_id: None, github_token: None, + setup_secrets: None, timeout_seconds: None, }; let removed = PdppRecord { @@ -2099,8 +3201,9 @@ rl.on('line', (line) => { let start = build_start(&request, &resolved.manifest, None).unwrap(); let command = build_command( &resolved, - request.github_token.as_deref(), + &resolve_child_secrets(&request, &resolved).unwrap(), &CommandCustomization::default(), + None, ) .unwrap(); assert!(command.clear_env); @@ -2149,7 +3252,7 @@ readline.createInterface({ input: process.stdin }).on('line', (line) => { max_retained_records: 4, ..Default::default() }, - request.github_token.as_deref(), + &resolve_child_secrets(&request, &resolved).unwrap(), ) .unwrap(); assert!(result.stderr.contains(token)); @@ -2157,7 +3260,12 @@ readline.createInterface({ input: process.stdin }).on('line', (line) => { .records .iter() .any(|record| record.data.to_string().contains(token))); - let response = to_response("run-1".into(), "github-pdpp".into(), result, Some(token)); + let response = to_response( + "run-1".into(), + "github-pdpp".into(), + result, + &[token.into()], + ); let serialized = serde_json::to_string(&response).unwrap(); assert!(!serialized.contains(token)); assert!(!serialized.contains("repo-1")); @@ -2256,10 +3364,17 @@ setInterval(() => {}, 1000); streams: vec!["repositories".into()], connection_id: None, github_token: None, + setup_secrets: None, timeout_seconds: Some(1), }; let result = supervise_pdpp_connector( - &build_command(&resolved, None, &CommandCustomization::default()).unwrap(), + &build_command( + &resolved, + &PdppChildSecrets::default(), + &CommandCustomization::default(), + None, + ) + .unwrap(), &build_start(&request, &resolved.manifest, None).unwrap(), &PdppRunOptions { timeout: Some(Duration::from_millis(50)), @@ -2302,6 +3417,7 @@ setInterval(() => {}, 1000); streams: vec!["repositories".into()], connection_id: None, github_token: None, + setup_secrets: None, timeout_seconds: Some(30), }; let handle = thread::spawn(move || { @@ -2313,7 +3429,7 @@ setInterval(() => {}, 1000); control, ..Default::default() }, - None, + &PdppChildSecrets::default(), ) }); @@ -2438,14 +3554,15 @@ readline.createInterface({ input: process.stdin }).on('line', (line) => { max_retained_records: 1, ..Default::default() }, - request.github_token.as_deref(), + &resolve_child_secrets(&request, &resolved).unwrap(), ) .unwrap(); + let secrets = resolve_child_secrets(&request, &resolved).unwrap(); let response = to_response( request.run_id, resolved.connector_id, result, - request.github_token.as_deref(), + &secrets.values, ); assert_eq!(response.event_summary.progress, 80); @@ -2472,6 +3589,7 @@ readline.createInterface({ input: process.stdin }).on('line', (line) => { streams: vec!["user".into(), "repositories".into()], connection_id: None, github_token: Some(token.clone()), + setup_secrets: None, timeout_seconds: Some(120), }; let result = @@ -2520,9 +3638,59 @@ readline.createInterface({ input: process.stdin }).on('line', (line) => { result.records_truncated, ); - let response = to_response(request.run_id, resolved.connector_id, result, Some(&token)); + let response = to_response(request.run_id, resolved.connector_id, result, &[token.clone()]); let serialized = serde_json::to_string(&response).unwrap(); assert!(!serialized.contains(&token)); assert!(!serialized.contains("\"data\"")); } + + #[test] + #[ignore = "requires a published ChatGPT PDPP artifact, local browser login, and explicit confirmation"] + fn runs_external_installed_chatgpt_browser_artifact_end_to_end() { + assert_eq!( + std::env::var("PDPP_E2E_CHATGPT_CONFIRM").as_deref(), + Ok("1"), + "set PDPP_E2E_CHATGPT_CONFIRM=1 to allow the credentialed browser E2E" + ); + let resolved = + resolve_active_installed_pdpp_connector(CHATGPT_CONNECTOR_INSTALL_ID).unwrap(); + let request = StartInstalledPdppConnectorRequest { + run_id: "chatgpt-pdpp-browser-e2e".into(), + connector_id: CHATGPT_CONNECTOR_INSTALL_ID.into(), + collection_mode: "incremental".into(), + streams: vec!["conversations".into()], + connection_id: Some("chatgpt-e2e-owner".into()), + github_token: None, + setup_secrets: Some(HashMap::from([ + ("username".into(), std::env::var("PDPP_E2E_CHATGPT_USERNAME").expect("PDPP_E2E_CHATGPT_USERNAME must be set")), + ("password".into(), std::env::var("PDPP_E2E_CHATGPT_PASSWORD").expect("PDPP_E2E_CHATGPT_PASSWORD must be set")), + ])), + timeout_seconds: Some(300), + }; + let result = run_resolved_installed_pdpp_connector( + &resolved, + &request, + CommandCustomization { + max_retained_records: 4, + ..Default::default() + }, + &resolve_child_secrets(&request, &resolved).unwrap(), + ) + .unwrap(); + assert_eq!( + result.status, + PdppRunStatus::Succeeded, + "{:?}", + result.failure + ); + assert!(result + .events + .iter() + .any(|event| matches!(event, PdppEvent::Interaction(_)))); + assert!(result + .records + .iter() + .any(|record| record.stream == "conversations")); + assert!(!result.stderr.contains("chatgpt-e2e-owner")); + } } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index b0fbb218..3012b05a 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -13,7 +13,10 @@ use commands::{ load_source_export_preview_from_path, mark_export_synced, open_folder, open_personal_server_scope_folder, open_platform_export_folder, set_app_config, start_connector_run, start_installed_pdpp_connector_run, start_personal_server, - stop_connector_run, stop_installed_pdpp_connector_run, stop_personal_server, test_nodejs, + is_installed_pdpp_browser_setup_complete, reset_installed_pdpp_browser_profile, + stop_connector_run, stop_installed_pdpp_connector_run, + submit_installed_pdpp_interaction_response, + stop_personal_server, test_nodejs, write_export_data, }; use tauri::{Listener, Manager}; @@ -78,6 +81,9 @@ pub fn run() { start_connector_run, start_installed_pdpp_connector_run, stop_installed_pdpp_connector_run, + reset_installed_pdpp_browser_profile, + is_installed_pdpp_browser_setup_complete, + submit_installed_pdpp_interaction_response, stop_connector_run, check_connected_platforms, check_browser_available, diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 131d0050..0cb48689 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -61,7 +61,8 @@ "../connectors/wholefoods/**/*": "connectors/wholefoods/", "../playwright-runner/dist/": "playwright-runner/dist/", "../personal-server/dist/personal-server*": "personal-server/dist/", - "../personal-server/dist/node_modules/": "personal-server/dist/node_modules/" + "../personal-server/dist/node_modules/": "personal-server/dist/node_modules/", + "../pdpp-runtime/**/*": "pdpp-runtime/" }, "linux": { "deb": { diff --git a/src-tauri/tests/fixtures/chatgpt-pdpp-browser.collection-profile.json b/src-tauri/tests/fixtures/chatgpt-pdpp-browser.collection-profile.json new file mode 100644 index 00000000..fc8529b9 --- /dev/null +++ b/src-tauri/tests/fixtures/chatgpt-pdpp-browser.collection-profile.json @@ -0,0 +1,73 @@ +{ + "connector_id": "https://registry.pdpp.org/connectors/chatgpt", + "connector_key": "chatgpt", + "display_name": "ChatGPT", + "version": "0.1.0", + "setup": { + "modality": "static_secret", + "credential_capture": { + "fields": [ + { + "name": "username", + "label": "ChatGPT email", + "type": "email", + "required": true, + "secret": true, + "autocomplete": "username", + "placeholder": "you@example.com", + "env": ["CHATGPT_USERNAME"] + }, + { + "name": "password", + "label": "ChatGPT password", + "type": "password", + "required": true, + "secret": true, + "autocomplete": "current-password", + "env": ["CHATGPT_PASSWORD"] + } + ] + } + }, + "runtime_requirements": { + "bindings": { + "network": { "required": true }, + "browser": { "required": true } + } + }, + "streams": [ + { + "name": "conversations", + "schema": { + "type": "object", + "required": ["id"] + } + }, + { + "name": "messages", + "schema": { + "type": "object", + "required": ["id", "conversation_id"] + } + }, + { + "name": "memories", + "schema": { + "type": "object", + "required": ["id", "content"] + } + }, + { + "name": "custom_gpts", + "schema": { "type": "object", "required": ["id"] } + }, + { + "name": "custom_instructions", + "schema": { "type": "object", "required": ["id"] } + }, + { + "name": "shared_conversations", + "schema": { "type": "object", "required": ["id"] } + } + ] +} diff --git a/src-tauri/tests/fixtures/chatgpt-pdpp-browser.fixture.mjs b/src-tauri/tests/fixtures/chatgpt-pdpp-browser.fixture.mjs new file mode 100644 index 00000000..53e9469a --- /dev/null +++ b/src-tauri/tests/fixtures/chatgpt-pdpp-browser.fixture.mjs @@ -0,0 +1,28 @@ +// Local protocol fixture for the published ChatGPT Collection Profile. It +// verifies the environment-based CDP compatibility seam, not a legacy +// Playwright page API or an invented START binding. +import readline from "node:readline" + +readline.createInterface({ input: process.stdin }).on("line", line => { + const start = JSON.parse(line) + if ("bindings" in start || "browser" in start) { + process.exitCode = 41 + return + } + if (!process.env.PDPP_CHATGPT_REMOTE_CDP_URL?.startsWith("http://127.0.0.1:")) { + process.exitCode = 42 + return + } + process.stdout.write( + `${JSON.stringify({ type: "INTERACTION", request_id: "chatgpt-login", kind: "browser", message: "Sign in to ChatGPT in the opened browser" })}\n` + ) + process.stdout.write( + `${JSON.stringify({ type: "PROGRESS", stream: "conversations", message: "Collecting ChatGPT conversations" })}\n` + ) + process.stdout.write( + `${JSON.stringify({ type: "RECORD", stream: "conversations", key: "fixture-conversation", data: { id: "fixture-conversation" }, emitted_at: "2026-07-31T00:00:00Z" })}\n` + ) + process.stdout.write( + `${JSON.stringify({ type: "DONE", status: "succeeded", records_emitted: 1 })}\n` + ) +}) diff --git a/src-tauri/tests/fixtures/pdpp-connector-fixture.mjs b/src-tauri/tests/fixtures/pdpp-connector-fixture.mjs index 11b34c3a..1a5e7791 100644 --- a/src-tauri/tests/fixtures/pdpp-connector-fixture.mjs +++ b/src-tauri/tests/fixtures/pdpp-connector-fixture.mjs @@ -74,6 +74,16 @@ switch (mode) { message: "Log in", }) break + case "interaction-success": + emit({ + type: "INTERACTION", + request_id: "browser-login", + kind: "browser", + message: "Sign in in the opened browser", + }) + emit(record) + emit(done) + break case "oversized-stdout": process.stdout.write(`${"x".repeat(128)}\n`) break diff --git a/src/hooks/useConnector.test.ts b/src/hooks/useConnector.test.ts index 1585d794..109e7fbb 100644 --- a/src/hooks/useConnector.test.ts +++ b/src/hooks/useConnector.test.ts @@ -115,15 +115,20 @@ describe("useConnector.startImport", () => { }) }) - expect(mockInvoke).toHaveBeenCalledWith("start_installed_pdpp_connector_run", { - request: { - runId: "github-pdpp-1700000000000", - connectorId: "github-pdpp", - collectionMode: "incremental", - streams: [], - githubToken: null, - }, - }) + expect(mockInvoke).toHaveBeenCalledWith( + "start_installed_pdpp_connector_run", + { + request: { + runId: "github-pdpp-1700000000000", + connectorId: "github-pdpp", + collectionMode: "incremental", + streams: [], + githubToken: null, + connectionId: null, + setupSecrets: null, + }, + } + ) expect(mockInvoke).not.toHaveBeenCalledWith( "start_connector_run", expect.anything() @@ -149,15 +154,20 @@ describe("useConnector.startImport", () => { ) }) - expect(mockInvoke).toHaveBeenCalledWith("start_installed_pdpp_connector_run", { - request: { - runId: "github-pdpp-1700000000000", - connectorId: "github-pdpp", - collectionMode: "incremental", - streams: [], - githubToken: "ghp_transient", - }, - }) + expect(mockInvoke).toHaveBeenCalledWith( + "start_installed_pdpp_connector_run", + { + request: { + runId: "github-pdpp-1700000000000", + connectorId: "github-pdpp", + collectionMode: "incremental", + streams: [], + githubToken: "ghp_transient", + connectionId: null, + setupSecrets: null, + }, + } + ) expect(startRun).toHaveBeenCalledWith( expect.not.objectContaining({ githubToken: "ghp_transient" }) ) @@ -193,6 +203,46 @@ describe("useConnector.startImport", () => { resolveHost?.() }) + it("hands ChatGPT static secrets only to the installed PDPP host invoke", async () => { + mockInvoke.mockResolvedValue(undefined) + const { useConnector } = await import("./useConnector") + const { result } = renderHook(() => useConnector()) + + await act(async () => { + await result.current.startImport( + { + ...TEST_PLATFORM, + id: "chatgpt-pdpp", + filename: "chatgpt-pdpp", + runtime: "pdpp-network", + }, + { + setupSecrets: { + username: "owner@example.com", + password: "transient-password", + }, + } + ) + }) + + expect(mockInvoke).toHaveBeenCalledWith( + "start_installed_pdpp_connector_run", + { + request: expect.objectContaining({ + connectorId: "chatgpt-pdpp", + connectionId: "chatgpt-pdpp-owner", + setupSecrets: { + username: "owner@example.com", + password: "transient-password", + }, + }), + } + ) + expect(startRun).toHaveBeenCalledWith( + expect.not.objectContaining({ setupSecrets: expect.anything() }) + ) + }) + it("preserves the legacy connector command for non-PDPP platforms", async () => { mockInvoke.mockResolvedValue(undefined) const { useConnector } = await import("./useConnector") @@ -225,9 +275,15 @@ describe("useConnector.startImport", () => { }) expect(stopRun).not.toHaveBeenCalled() - expect(mockInvoke).toHaveBeenCalledWith("stop_installed_pdpp_connector_run", { - runId: "github-pdpp-run", - }) - expect(mockInvoke).not.toHaveBeenCalledWith("stop_connector_run", expect.anything()) + expect(mockInvoke).toHaveBeenCalledWith( + "stop_installed_pdpp_connector_run", + { + runId: "github-pdpp-run", + } + ) + expect(mockInvoke).not.toHaveBeenCalledWith( + "stop_connector_run", + expect.anything() + ) }) }) diff --git a/src/hooks/useConnector.ts b/src/hooks/useConnector.ts index dbcab466..d71f1b2e 100644 --- a/src/hooks/useConnector.ts +++ b/src/hooks/useConnector.ts @@ -1,31 +1,32 @@ -import { useCallback } from 'react'; -import { invoke } from '@tauri-apps/api/core'; -import { useDispatch, useSelector } from 'react-redux'; -import { deleteRun, startRun, updateRunStatus, stopRun } from '../state/store'; -import type { RootState } from '../state/store'; -import type { Platform, Run } from '../types'; -import { getPlatformRegistryEntry } from '@/lib/platform/utils'; +import { useCallback } from "react" +import { invoke } from "@tauri-apps/api/core" +import { useDispatch, useSelector } from "react-redux" +import { deleteRun, startRun, updateRunStatus, stopRun } from "../state/store" +import type { RootState } from "../state/store" +import type { Platform, Run } from "../types" +import { getPlatformRegistryEntry } from "@/lib/platform/utils" import { trackCollectionFailed, trackCollectionStarted, -} from '@/lib/telemetry/events'; -import { durationSince } from '@/lib/telemetry/client'; +} from "@/lib/telemetry/events" +import { durationSince } from "@/lib/telemetry/client" -const DUPLICATE_ACTIVE_RUN_ERROR_CODE = 'DUPLICATE_ACTIVE_RUN'; -const PDPP_NETWORK_RUNTIME = 'pdpp-network'; +const DUPLICATE_ACTIVE_RUN_ERROR_CODE = "DUPLICATE_ACTIVE_RUN" +const PDPP_NETWORK_RUNTIME = "pdpp-network" interface StartImportOptions { - githubToken?: string | null; + githubToken?: string | null + setupSecrets?: { username: string; password: string } | null } function isDuplicateStartError(error: unknown): boolean { const message = - typeof error === 'string' + typeof error === "string" ? error : error instanceof Error ? error.message - : String(error); - return message.includes(DUPLICATE_ACTIVE_RUN_ERROR_CODE); + : String(error) + return message.includes(DUPLICATE_ACTIVE_RUN_ERROR_CODE) } async function startInstalledPdppConnectorRun( @@ -33,26 +34,30 @@ async function startInstalledPdppConnectorRun( platform: Platform, options: StartImportOptions = {} ): Promise { - await invoke('start_installed_pdpp_connector_run', { + await invoke("start_installed_pdpp_connector_run", { request: { runId, connectorId: platform.id, - collectionMode: 'incremental', + collectionMode: "incremental", streams: [], githubToken: - platform.id === 'github-pdpp' ? options.githubToken ?? null : null, + platform.id === "github-pdpp" ? (options.githubToken ?? null) : null, + connectionId: + platform.id === "chatgpt-pdpp" ? "chatgpt-pdpp-owner" : null, + setupSecrets: + platform.id === "chatgpt-pdpp" ? (options.setupSecrets ?? null) : null, }, - }); + }) } export function useConnector() { - const dispatch = useDispatch(); - const runs = useSelector((state: RootState) => state.app.runs); + const dispatch = useDispatch() + const runs = useSelector((state: RootState) => state.app.runs) const startImport = useCallback( async (platform: Platform, options: StartImportOptions = {}) => { - const runId = `${platform.id}-${Date.now()}`; - const source = getPlatformRegistryEntry(platform)?.id ?? platform.id; + const runId = `${platform.id}-${Date.now()}` + const source = getPlatformRegistryEntry(platform)?.id ?? platform.id const newRun: Run = { id: runId, @@ -61,117 +66,124 @@ export function useConnector() { runtime: platform.runtime, isConnected: false, startDate: new Date().toISOString(), - status: 'running', - url: platform.connectURL || '', + status: "running", + url: platform.connectURL || "", company: platform.company, name: platform.name, - logs: '', - }; + logs: "", + } - dispatch(startRun(newRun)); + dispatch(startRun(newRun)) trackCollectionStarted({ collectionRunId: runId, source, - authMode: 'interactive', - }); + authMode: "interactive", + }) // The installed PDPP host streams progress and terminal state through // `connector-status`. Its command response arrives only after the // subprocess finishes, so waiting here would leave /connect without a // run id (and therefore without live busy/progress UI). if (platform.runtime === PDPP_NETWORK_RUNTIME) { - void startInstalledPdppConnectorRun(runId, platform, options).catch((error) => { - if (isDuplicateStartError(error)) { - dispatch(deleteRun(runId)); - return; + void startInstalledPdppConnectorRun(runId, platform, options).catch( + error => { + if (isDuplicateStartError(error)) { + dispatch(deleteRun(runId)) + return + } + + console.error( + "Failed to start installed PDPP connector run:", + error + ) + // A host-side failure normally emits its own terminal event. This + // only closes the UI state when no event has already done so. + dispatch( + updateRunStatus({ + runId, + status: "error", + endDate: new Date().toISOString(), + onlyIfRunning: true, + }) + ) } - - console.error('Failed to start installed PDPP connector run:', error); - // A host-side failure normally emits its own terminal event. This - // only closes the UI state when no event has already done so. - dispatch( - updateRunStatus({ - runId, - status: 'error', - endDate: new Date().toISOString(), - onlyIfRunning: true, - }) - ); - }); - return runId; + ) + return runId } try { const simulateNoChrome = - typeof window !== 'undefined' && window.localStorage?.getItem?.('dataconnect_simulate_no_chrome') === 'true'; + typeof window !== "undefined" && + window.localStorage?.getItem?.("dataconnect_simulate_no_chrome") === + "true" - await invoke('start_connector_run', { + await invoke("start_connector_run", { runId, platformId: platform.id, filename: platform.filename, company: platform.company, name: platform.name, - connectUrl: platform.connectURL || '', + connectUrl: platform.connectURL || "", runtime: platform.runtime || null, simulateNoChrome, - }); + }) } catch (error) { if (isDuplicateStartError(error)) { - dispatch(deleteRun(runId)); - return null; + dispatch(deleteRun(runId)) + return null } - console.error('Failed to start connector run:', error); + console.error("Failed to start connector run:", error) dispatch( updateRunStatus({ runId, - status: 'error', + status: "error", endDate: new Date().toISOString(), }) - ); + ) trackCollectionFailed({ collectionRunId: runId, source, durationMs: durationSince(newRun.startDate), error, - }); + }) } - return runId; + return runId }, [dispatch] - ); + ) const stopExport = useCallback( async (runId: string) => { try { - const run = runs.find(candidate => candidate.id === runId); + const run = runs.find(candidate => candidate.id === runId) if (run?.runtime === PDPP_NETWORK_RUNTIME) { - await invoke('stop_installed_pdpp_connector_run', { runId }); + await invoke("stop_installed_pdpp_connector_run", { runId }) // The PDPP host emits STOPPED after its subprocess is actually // reaped. Keep the source card active until then. } else { - dispatch(stopRun(runId)); - await invoke('stop_connector_run', { runId }); + dispatch(stopRun(runId)) + await invoke("stop_connector_run", { runId }) } } catch (error) { - console.log('Stop connector run (window may be closed):', error); + console.log("Stop connector run (window may be closed):", error) } }, [dispatch, runs] - ); + ) const getRunById = useCallback( (runId: string) => { - return runs.find((r) => r.id === runId); + return runs.find(r => r.id === runId) }, [runs] - ); + ) return { runs, startImport, stopExport, getRunById, - }; + } } diff --git a/src/pages/data-apps/components/registry-app-card.test.tsx b/src/pages/data-apps/components/registry-app-card.test.tsx index b880ffe5..3b43f0e3 100644 --- a/src/pages/data-apps/components/registry-app-card.test.tsx +++ b/src/pages/data-apps/components/registry-app-card.test.tsx @@ -1,4 +1,10 @@ -import { cleanup, render, screen, waitFor } from "@testing-library/react" +import { + cleanup, + fireEvent, + render, + screen, + waitFor, +} from "@testing-library/react" import { createMemoryRouter, RouterProvider } from "react-router-dom" import { afterEach, describe, expect, it, vi } from "vitest" import { RegistryAppCard } from "./registry-app-card" @@ -180,7 +186,7 @@ describe("RegistryAppCard", () => { }) expect(screen.getByText("Uses PDPP")).toBeTruthy() - screen.getByRole("button", { name: "Open Timeline" }).click() + fireEvent.click(screen.getByRole("button", { name: "Open Timeline" })) await waitFor(() => { expect(router.state.location.pathname).toBe("/apps/timeline") diff --git a/src/pages/home/components/connected-sources-list.tsx b/src/pages/home/components/connected-sources-list.tsx index c2cb0ea1..fec7258a 100644 --- a/src/pages/home/components/connected-sources-list.tsx +++ b/src/pages/home/components/connected-sources-list.tsx @@ -25,6 +25,7 @@ interface ConnectedSourcesListProps { headline?: string onOpenRuns?: (platform: Platform) => void onSyncSource?: (platform: Platform) => void + onReconnectSource?: (platform: Platform) => void } type OnboardingMessageState = "empty" | "early" | "mature" @@ -44,6 +45,7 @@ export function ConnectedSourcesList({ headline = "Your sources at the moment.", onOpenRuns, onSyncSource, + onReconnectSource, }: ConnectedSourcesListProps) { const inFlightSyncPlatformIdsRef = useRef>(new Set()) const syncFeedbackTimeoutsRef = useRef< @@ -166,6 +168,10 @@ export function ConnectedSourcesList({ hasBlockingRun || hasActiveRun || isShowingSyncFeedback + const canReconnect = + platform.id === "chatgpt-pdpp" && + Boolean(onReconnectSource) && + !hasActiveRun const syncTooltipCopy = hasActiveRun || syncFeedbackState === "backgrounding" ? "Fetching in background" @@ -184,41 +190,59 @@ export function ConnectedSourcesList({ ariaLabel: `Open ${platform.name}`, }} middleSlot={ - - - triggerSyncFeedback(platform) - : undefined - } - disabled={isSyncDisabled} - aria-label={`Fetch latest data for ${platform.name}`} - > - {syncFeedbackState ? ( - + {canReconnect ? ( + + + onReconnectSource?.(platform)} + aria-label={`Reconnect ${platform.name}`} > - {syncFeedbackState === "running" - ? "Fetching…" - : "Backgrounding…"} - - ) : null} - - - - {syncTooltipCopy} - + + + + + Reset this browser session and reconnect + + + ) : null} + + + triggerSyncFeedback(platform) + : undefined + } + disabled={isSyncDisabled} + aria-label={`Fetch latest data for ${platform.name}`} + > + {syncFeedbackState ? ( + + {syncFeedbackState === "running" + ? "Fetching…" + : "Backgrounding…"} + + ) : null} + + + + {syncTooltipCopy} + + } endSlotClassName="[&_svg:not([class*='size-']):not([data-slot=spinner])]:size-7!" surface="list-item" diff --git a/src/pages/home/index.test.tsx b/src/pages/home/index.test.tsx index b83d019e..801c41ba 100644 --- a/src/pages/home/index.test.tsx +++ b/src/pages/home/index.test.tsx @@ -1,6 +1,17 @@ import { describe, expect, it, vi, beforeEach, afterEach } from "vitest" -import { render, waitFor, cleanup, fireEvent, screen } from "@testing-library/react" -import { createMemoryRouter, MemoryRouter, RouterProvider } from "react-router-dom" +import { + render, + act, + waitFor, + cleanup, + fireEvent, + screen, +} from "@testing-library/react" +import { + createMemoryRouter, + MemoryRouter, + RouterProvider, +} from "react-router-dom" import { ROUTES } from "@/config/routes" import { TooltipProvider } from "@/components/ui/tooltip" import { Home } from "./index" @@ -10,6 +21,8 @@ const mockStartImport = vi.fn() const mockStopExport = vi.fn() const mockNavigate = vi.fn() const mockRefreshConnectedStatus = vi.fn() +const mockInvoke = vi.fn() +const mockListen = vi.fn() let mockConnectedPlatforms: Record = {} let mockRuns: Array<{ id: string @@ -27,9 +40,8 @@ let mockRuns: Array<{ }> = [] vi.mock("react-router-dom", async () => { - const actual = await vi.importActual( - "react-router-dom" - ) + const actual = + await vi.importActual("react-router-dom") return { ...actual, useNavigate: () => mockNavigate, @@ -40,6 +52,14 @@ vi.mock("@/hooks/usePlatforms", () => ({ usePlatforms: () => mockUsePlatforms(), })) +vi.mock("@tauri-apps/api/core", () => ({ + invoke: (...args: unknown[]) => mockInvoke(...args), +})) + +vi.mock("@tauri-apps/api/event", () => ({ + listen: (...args: unknown[]) => mockListen(...args), +})) + vi.mock("@/hooks/useConnector", () => ({ useConnector: () => ({ startImport: mockStartImport, @@ -103,6 +123,10 @@ describe("Home", () => { mockStopExport.mockReset() mockNavigate.mockReset() mockRefreshConnectedStatus.mockReset() + mockInvoke.mockReset() + mockInvoke.mockResolvedValue(false) + mockListen.mockReset() + mockListen.mockResolvedValue(() => undefined) mockConnectedPlatforms = {} mockRuns = [] mockStartImport.mockResolvedValue("run-1") @@ -125,17 +149,13 @@ describe("Home", () => { const { getByRole } = renderHome() expect(getByRole("heading", { level: 1, name: /your data/i })).toBeTruthy() - expect( - getByRole("heading", { name: /your imported data/i }) - ).toBeTruthy() + expect(getByRole("heading", { name: /your imported data/i })).toBeTruthy() }) it("does not render the connected apps tab or surface", () => { const { container } = renderHome() - expect( - screen.queryByRole("tab", { name: /connected apps/i }) - ).toBeNull() + expect(screen.queryByRole("tab", { name: /connected apps/i })).toBeNull() expect( container.querySelector('[data-component="connected-apps-list"]') ).toBeNull() @@ -241,6 +261,225 @@ describe("Home", () => { expect(screen.queryByLabelText(/personal access token/i)).toBeNull() }) + it("captures the manifest-declared ChatGPT static secrets before launching", async () => { + mockUsePlatforms.mockReturnValue({ + platforms: [ + { + id: "chatgpt-pdpp", + company: "OpenAI", + name: "ChatGPT", + filename: "chatgpt-pdpp", + description: "ChatGPT PDPP export", + isUpdated: false, + logoURL: "", + needsConnection: true, + connectURL: null, + connectSelector: null, + exportFrequency: null, + vectorize_config: null, + runtime: "pdpp-network", + setup: { + modality: "static_secret", + credentialCapture: { fields: [] }, + }, + }, + ], + connectedPlatforms: {}, + loadPlatforms: vi.fn(), + refreshConnectedStatus: vi.fn(), + getPlatformById: vi.fn(), + isPlatformConnected: vi.fn(() => false), + }) + + renderHome() + fireEvent.click(screen.getByRole("button", { name: /connect chatgpt/i })) + await waitFor(() => { + expect(screen.getByLabelText(/chatgpt email/i)).toBeTruthy() + }) + expect(mockStartImport).not.toHaveBeenCalled() + fireEvent.change(screen.getByLabelText(/chatgpt email/i), { + target: { value: "owner@example.com" }, + }) + fireEvent.change(screen.getByLabelText(/chatgpt password/i), { + target: { value: "transient-password" }, + }) + fireEvent.click( + screen.getByRole("button", { + name: /start owner-attended sync/i, + }) + ) + + await waitFor(() => { + expect(mockStartImport).toHaveBeenCalledWith( + expect.objectContaining({ id: "chatgpt-pdpp" }), + { + setupSecrets: { + username: "owner@example.com", + password: "transient-password", + }, + } + ) + }) + expect(screen.queryByLabelText(/chatgpt password/i)).toBeNull() + }) + + it("reuses a completed owner profile without asking for static secrets again", async () => { + mockInvoke.mockResolvedValue(true) + mockUsePlatforms.mockReturnValue({ + platforms: [ + { + id: "chatgpt-pdpp", + company: "OpenAI", + name: "ChatGPT", + filename: "chatgpt-pdpp", + description: "ChatGPT PDPP export", + isUpdated: false, + logoURL: "", + needsConnection: true, + connectURL: null, + connectSelector: null, + exportFrequency: null, + vectorize_config: null, + runtime: "pdpp-network", + setup: { + modality: "static_secret", + credentialCapture: { fields: [] }, + }, + }, + ], + connectedPlatforms: {}, + loadPlatforms: vi.fn(), + refreshConnectedStatus: vi.fn(), + getPlatformById: vi.fn(), + isPlatformConnected: vi.fn(() => false), + }) + + renderHome() + fireEvent.click(screen.getByRole("button", { name: /connect chatgpt/i })) + + await waitFor(() => { + expect(mockInvoke).toHaveBeenCalledWith( + "is_installed_pdpp_browser_setup_complete", + { connectorId: "chatgpt-pdpp", connectionId: "chatgpt-pdpp-owner" } + ) + expect(mockStartImport).toHaveBeenCalledWith( + expect.objectContaining({ id: "chatgpt-pdpp" }) + ) + }) + expect(screen.queryByLabelText(/chatgpt password/i)).toBeNull() + expect(mockStartImport).not.toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ setupSecrets: expect.anything() }) + ) + }) + + it("resets an expired ChatGPT session and returns the owner to setup", async () => { + mockInvoke.mockImplementation(command => + command === "reset_installed_pdpp_browser_profile" + ? Promise.resolve(undefined) + : Promise.resolve(false) + ) + mockUsePlatforms.mockReturnValue({ + platforms: [ + { + id: "chatgpt-pdpp", + company: "OpenAI", + name: "ChatGPT", + filename: "chatgpt-pdpp", + description: "ChatGPT PDPP export", + isUpdated: false, + logoURL: "", + needsConnection: true, + connectURL: null, + connectSelector: null, + exportFrequency: null, + vectorize_config: null, + runtime: "pdpp-network", + setup: { + modality: "static_secret", + credentialCapture: { fields: [] }, + }, + }, + ], + connectedPlatforms: { "chatgpt-pdpp": true }, + loadPlatforms: vi.fn(), + refreshConnectedStatus: vi.fn(), + getPlatformById: vi.fn(), + isPlatformConnected: vi.fn(() => true), + }) + + renderHome() + fireEvent.click(screen.getByRole("button", { name: "Reconnect ChatGPT" })) + + await waitFor(() => { + expect(mockInvoke).toHaveBeenCalledWith( + "reset_installed_pdpp_browser_profile", + { connectorId: "chatgpt-pdpp", connectionId: "chatgpt-pdpp-owner" } + ) + expect(mockInvoke).toHaveBeenCalledWith( + "is_installed_pdpp_browser_setup_complete", + { connectorId: "chatgpt-pdpp", connectionId: "chatgpt-pdpp-owner" } + ) + expect(screen.getByLabelText(/chatgpt email/i)).toBeTruthy() + }) + expect(mockStartImport).not.toHaveBeenCalled() + }) + + it("submits a transient code response for the exact no-schema ChatGPT OTP interaction", async () => { + let deliverInteraction: ((event: { payload: unknown }) => void) | undefined + mockListen.mockImplementation((_event, callback) => { + deliverInteraction = callback as (event: { payload: unknown }) => void + return Promise.resolve(() => undefined) + }) + mockInvoke.mockResolvedValue(undefined) + mockRuns = [ + { + id: "chatgpt-pdpp-run-1", + platformId: "chatgpt-pdpp", + filename: "chatgpt-pdpp", + isConnected: false, + startDate: new Date().toISOString(), + status: "running", + url: "", + company: "OpenAI", + name: "ChatGPT", + logs: "", + }, + ] + renderHome() + await waitFor(() => expect(deliverInteraction).toBeTypeOf("function")) + + await act(async () => { + deliverInteraction?.({ + payload: { + runId: "chatgpt-pdpp-run-1", + requestId: "otp-1", + kind: "otp", + message: "Enter your verification code", + }, + }) + }) + + fireEvent.change(screen.getByLabelText(/verification code/i), { + target: { value: "123456" }, + }) + fireEvent.click(screen.getByRole("button", { name: "Continue" })) + await waitFor(() => { + expect(mockInvoke).toHaveBeenCalledWith( + "submit_installed_pdpp_interaction_response", + { + runId: "chatgpt-pdpp-run-1", + requestId: "otp-1", + status: "success", + data: { code: "123456" }, + } + ) + }) + await waitFor(() => { + expect(screen.queryByLabelText(/verification code/i)).toBeNull() + }) + }) + it("cancels the GitHub PAT prompt without starting import", () => { mockUsePlatforms.mockReturnValue({ platforms: [ @@ -351,7 +590,9 @@ describe("Home", () => { expect( screen.queryByRole("button", { name: /connect chatgpt/i }) ).toBeNull() - expect(screen.getAllByRole("button", { name: /open chatgpt/i }).length).toBeGreaterThan(0) + expect( + screen.getAllByRole("button", { name: /open chatgpt/i }).length + ).toBeGreaterThan(0) }) it("shows connected source from persisted run even when connected status map is stale", async () => { @@ -400,7 +641,9 @@ describe("Home", () => { expect( screen.queryByRole("button", { name: /connect chatgpt/i }) ).toBeNull() - expect(screen.getAllByRole("button", { name: /open chatgpt/i }).length).toBeGreaterThan(0) + expect( + screen.getAllByRole("button", { name: /open chatgpt/i }).length + ).toBeGreaterThan(0) }) it.each([ @@ -553,7 +796,8 @@ describe("Home", () => { company: "OpenAI", name: "ChatGPT", logs: "", - exportPath: "/tmp/dataconnect/exported_data/OpenAI/ChatGPT/run-chatgpt-1", + exportPath: + "/tmp/dataconnect/exported_data/OpenAI/ChatGPT/run-chatgpt-1", }, ] @@ -629,9 +873,11 @@ describe("Home", () => { renderHome() expect( - screen.getByRole("button", { - name: /fetch latest data for chatgpt/i, - }).hasAttribute("disabled") + screen + .getByRole("button", { + name: /fetch latest data for chatgpt/i, + }) + .hasAttribute("disabled") ).toBe(true) }) @@ -828,7 +1074,9 @@ describe("Home", () => { renderHome() - const spotifyButton = screen.getByRole("button", { name: /connect spotify/i }) + const spotifyButton = screen.getByRole("button", { + name: /connect spotify/i, + }) expect(spotifyButton.hasAttribute("disabled")).toBe(false) fireEvent.click(spotifyButton) diff --git a/src/pages/home/index.tsx b/src/pages/home/index.tsx index 737be3e1..5c227747 100644 --- a/src/pages/home/index.tsx +++ b/src/pages/home/index.tsx @@ -7,6 +7,8 @@ import { useState, } from "react" import { useLocation, useNavigate } from "react-router-dom" +import { invoke } from "@tauri-apps/api/core" +import { listen } from "@tauri-apps/api/event" import { useSelector } from "react-redux" import { usePlatforms } from "@/hooks/usePlatforms" import { useConnector } from "@/hooks/useConnector" @@ -46,6 +48,14 @@ import { resolveHomeImportSourcesUiDebugState, } from "./home-import-sources-ui-debug" +type PendingPdppInteraction = { + runId: string + requestId: string + kind: string + message: string + schema?: { properties?: Record } | null +} + export function Home() { const homeDebugScenarioLabel: Record = { "blocking-waiting": "blocking-waiting", @@ -67,6 +77,21 @@ export function Home() { const [githubTokenDialogPlatform, setGithubTokenDialogPlatform] = useState(null) const [githubTokenInput, setGithubTokenInput] = useState("") + const [chatgptSetupDialogPlatform, setChatgptSetupDialogPlatform] = + useState(null) + const [chatgptUsernameInput, setChatgptUsernameInput] = useState("") + const [chatgptPasswordInput, setChatgptPasswordInput] = useState("") + const [pendingInteraction, setPendingInteraction] = + useState(null) + const [interactionInput, setInteractionInput] = useState("") + const chatgptSetupFields = + chatgptSetupDialogPlatform?.setup?.credentialCapture.fields ?? [] + const chatgptUsernameField = chatgptSetupFields.find( + field => field.name === "username" + ) + const chatgptPasswordField = chatgptSetupFields.find( + field => field.name === "password" + ) const knownSuccessfulRunIdsRef = useRef | null>(null) const homeUiDebugEnabled = useMemo( () => isHomeImportSourcesDebugEnabled(location.search), @@ -108,13 +133,42 @@ export function Home() { } }, [refreshConnectedStatus, runs]) + useEffect(() => { + let unlisten: (() => void) | undefined + void listen("pdpp-interaction", event => { + setInteractionInput("") + setPendingInteraction(event.payload) + }).then(listener => { + unlisten = listener + }) + return () => unlisten?.() + }, []) + + useEffect(() => { + if ( + pendingInteraction && + !runs.some( + run => run.id === pendingInteraction.runId && run.status === "running" + ) + ) { + setInteractionInput("") + setPendingInteraction(null) + } + }, [pendingInteraction, runs]) + const runImportSource = useCallback( - async (platform: Platform, githubToken?: string) => { + async ( + platform: Platform, + options?: { + githubToken?: string + setupSecrets?: { username: string; password: string } + } + ) => { try { - if (githubToken === undefined) { + if (options === undefined) { await startImport(platform) } else { - await startImport(platform, { githubToken }) + await startImport(platform, options) } } catch (error) { console.error("Import failed:", error) @@ -130,6 +184,32 @@ export function Home() { setGithubTokenDialogPlatform(platform) return } + if ( + platform.id === "chatgpt-pdpp" && + platform.setup?.modality === "static_secret" + ) { + void invoke("is_installed_pdpp_browser_setup_complete", { + connectorId: platform.id, + connectionId: "chatgpt-pdpp-owner", + }) + .then(setupComplete => { + if (setupComplete) { + void runImportSource(platform) + return + } + setChatgptUsernameInput("") + setChatgptPasswordInput("") + setChatgptSetupDialogPlatform(platform) + }) + // A missing marker is the safe fallback for a failed or older host: + // show first-setup recovery rather than accidentally sending no auth. + .catch(() => { + setChatgptUsernameInput("") + setChatgptPasswordInput("") + setChatgptSetupDialogPlatform(platform) + }) + return + } void runImportSource(platform) }, @@ -149,7 +229,7 @@ export function Home() { if (!platform || !githubToken) return closeGithubTokenDialog() - void runImportSource(platform, githubToken) + void runImportSource(platform, { githubToken }) }, [ closeGithubTokenDialog, @@ -159,6 +239,34 @@ export function Home() { ] ) + const closeChatgptSetupDialog = useCallback(() => { + setChatgptSetupDialogPlatform(null) + setChatgptUsernameInput("") + setChatgptPasswordInput("") + }, []) + + const submitChatgptSetup = useCallback( + (event: FormEvent) => { + event.preventDefault() + const platform = chatgptSetupDialogPlatform + const username = chatgptUsernameInput.trim() + const password = chatgptPasswordInput + if (!platform || !username || !password) return + + closeChatgptSetupDialog() + void runImportSource(platform, { + setupSecrets: { username, password }, + }) + }, + [ + chatgptPasswordInput, + chatgptSetupDialogPlatform, + chatgptUsernameInput, + closeChatgptSetupDialog, + runImportSource, + ] + ) + const handleStopImport = useCallback( async (runId: string) => { try { @@ -170,6 +278,54 @@ export function Home() { [stopExport] ) + const handleReconnectSource = useCallback( + async (platform: Platform) => { + if (platform.id !== "chatgpt-pdpp") return + try { + await invoke("reset_installed_pdpp_browser_profile", { + connectorId: platform.id, + connectionId: "chatgpt-pdpp-owner", + }) + // The reset clears the non-secret setup marker. Re-enter the normal + // setup gate so an expired session gets owner-attended recovery. + handleImportSource(platform) + } catch (error) { + console.error("Failed to reset ChatGPT browser session:", error) + } + }, + [handleImportSource] + ) + + const respondToPendingInteraction = useCallback( + async (status: "success" | "cancelled") => { + const interaction = pendingInteraction + if (!interaction) return + const fields = Object.keys(interaction.schema?.properties ?? {}) + const requiresVerificationCode = interaction.kind === "otp" + const data = + status === "success" && requiresVerificationCode + ? { code: interactionInput } + : status === "success" && fields.length > 0 + ? { [fields[0]]: interactionInput } + : undefined + try { + await invoke("submit_installed_pdpp_interaction_response", { + runId: interaction.runId, + requestId: interaction.requestId, + status, + data, + }) + } catch (error) { + console.error("Failed to submit connector interaction:", error) + } finally { + // OTP/recovery input is never promoted into Redux, logs, or storage. + setInteractionInput("") + setPendingInteraction(null) + } + }, + [interactionInput, pendingInteraction] + ) + const handleTestDeepLink = useCallback(() => { const trimmed = deepLinkInput.trim() if (!trimmed) return @@ -303,6 +459,7 @@ export function Home() { headline="Your imported data" onOpenRuns={handleOpenRuns} onSyncSource={handleImportSource} + onReconnectSource={handleReconnectSource} /> + { + if (!open) closeChatgptSetupDialog() + }} + > + +
+ + + Connect ChatGPT + + + Use these only for initial setup or owner-mediated recovery. + DataConnect passes them only to this run and does not save them. + + +
+ + setChatgptUsernameInput(event.target.value)} + autoFocus + /> +
+
+ + setChatgptPasswordInput(event.target.value)} + /> +
+ + + Cancel + + + +
+ + + + + + + + ChatGPT needs your attention + + + {pendingInteraction?.message} + + + {pendingInteraction?.kind === "otp" || + Object.keys(pendingInteraction?.schema?.properties ?? {}).length > 0 ? ( +
+ + setInteractionInput(event.target.value)} + autoFocus + /> +
+ ) : null} + + void respondToPendingInteraction("cancelled")} + > + Cancel run + + + + + + {import.meta.env.DEV && (
diff --git a/src/types/index.ts b/src/types/index.ts index 903f821c..1124a7a8 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,177 +1,193 @@ export interface Platform { - id: string; - company: string; - name: string; - filename: string; - description: string; - isUpdated: boolean; - logoURL: string; - needsConnection: boolean; - connectURL: string | null; - connectSelector: string | null; - exportFrequency: string | null; - vectorize_config: Record | null; + id: string + company: string + name: string + filename: string + description: string + isUpdated: boolean + logoURL: string + needsConnection: boolean + connectURL: string | null + connectSelector: string | null + exportFrequency: string | null + vectorize_config: Record | null /** Runtime type: "vanilla" (default) or "network-capture" (uses network interception) */ - runtime?: string | null; + runtime?: string | null /** Scopes this connector can export (e.g. ["chatgpt.conversations", "chatgpt.memories"]) */ - scopes?: string[] | null; + scopes?: string[] | null + /** Exact static-secret setup shape currently supported for chatgpt-pdpp. */ + setup?: PdppStaticSecretSetup | null +} + +export interface PdppStaticSecretSetup { + modality: "static_secret" + credentialCapture: { + fields: Array<{ + name: "username" | "password" + label?: string | null + type?: "email" | "password" | string | null + required: true + secret: true + autocomplete?: string | null + }> + } } export interface ProgressPhase { - step: number; - total: number; - label: string; + step: number + total: number + label: string } export interface Run { - id: string; - platformId: string; - filename: string; - runtime?: string | null; - isConnected: boolean; - startDate: string; - endDate?: string; - status: 'pending' | 'running' | 'success' | 'partial' | 'error' | 'stopped'; - url: string; - exportSize?: number; - exportPath?: string; - company: string; - name: string; - currentStep?: string; - logs?: string; - statusMessage?: string; - itemsExported?: number; - itemLabel?: string; // e.g., "posts", "conversations" - exportData?: ExportedData; + id: string + platformId: string + filename: string + runtime?: string | null + isConnected: boolean + startDate: string + endDate?: string + status: "pending" | "running" | "success" | "partial" | "error" | "stopped" + url: string + exportSize?: number + exportPath?: string + company: string + name: string + currentStep?: string + logs?: string + statusMessage?: string + itemsExported?: number + itemLabel?: string // e.g., "posts", "conversations" + exportData?: ExportedData // Progress tracking - phase?: ProgressPhase; - itemCount?: number; // Real-time count during collection + phase?: ProgressPhase + itemCount?: number // Real-time count during collection // Sync status - syncedToPersonalServer?: boolean; - scope?: string; + syncedToPersonalServer?: boolean + scope?: string } export interface ExportedData { - platform: string; - company: string; - exportedAt: string; - userInfo?: { name?: string; email?: string }; + platform: string + company: string + exportedAt: string + userInfo?: { name?: string; email?: string } conversations?: Array<{ - id: string; - title: string; - url: string; - scrapedAt: string; - }>; - totalConversations?: number; + id: string + title: string + url: string + scrapedAt: string + }> + totalConversations?: number } export interface AppState { - route: string; - activeRunIndex: number; - isFullScreen: boolean; - isMac: boolean; - isRunLayerVisible: boolean; - breadcrumb: { text: string; link: string }[]; - runs: Run[]; - platforms: Platform[]; - connectedPlatforms: Record; - connectorUpdates: ConnectorUpdateInfo[]; - lastUpdateCheck: string | null; - isCheckingUpdates: boolean; - auth: AuthState; - connectedApps: ConnectedApp[]; - appConfig: AppConfig; + route: string + activeRunIndex: number + isFullScreen: boolean + isMac: boolean + isRunLayerVisible: boolean + breadcrumb: { text: string; link: string }[] + runs: Run[] + platforms: Platform[] + connectedPlatforms: Record + connectorUpdates: ConnectorUpdateInfo[] + lastUpdateCheck: string | null + isCheckingUpdates: boolean + auth: AuthState + connectedApps: ConnectedApp[] + appConfig: AppConfig } export interface RootState { - app: AppState; + app: AppState } export interface ConnectorLogEvent { - runId: string; - message: string; - timestamp: number; + runId: string + message: string + timestamp: number } export interface ConnectorStatusPayload { - type: string; - message: string; - phase?: ProgressPhase; - count?: number; - data?: unknown; - outcome?: 'success' | 'partial' | 'failure' | 'cancelled'; - errorClass?: string; - recordCount?: number; + type: string + message: string + phase?: ProgressPhase + count?: number + data?: unknown + outcome?: "success" | "partial" | "failure" | "cancelled" + errorClass?: string + recordCount?: number scopeSummary?: { - requested: number; - produced: number; - degraded: number; - omitted: number; - }; + requested: number + produced: number + degraded: number + omitted: number + } } export interface ConnectorStatusEvent { - runId: string; - status: ConnectorStatusPayload; - timestamp: number; + runId: string + status: ConnectorStatusPayload + timestamp: number } export interface DownloadProgressEvent { - run_id: string; - filename: string; - percent: number; - bytes_downloaded: number; - total_bytes: number | null; + run_id: string + filename: string + percent: number + bytes_downloaded: number + total_bytes: number | null } export interface ExportCompleteEvent { - company: string; - name: string; - run_id: string; - export_path: string; - export_size: number; + company: string + name: string + run_id: string + export_path: string + export_size: number } export interface ConnectorUpdateInfo { - id: string; - name: string; - description: string; - company: string; - currentVersion: string | null; - latestVersion: string; - hasUpdate: boolean; - isNew: boolean; + id: string + name: string + description: string + company: string + currentVersion: string | null + latestVersion: string + hasUpdate: boolean + isNew: boolean } export interface AuthState { - isAuthenticated: boolean; - isLoading: boolean; - user: AuthUser | null; - walletAddress: string | null; - masterKeySignature: string | null; - accountRole: 'standard' | 'debug'; + isAuthenticated: boolean + isLoading: boolean + user: AuthUser | null + walletAddress: string | null + masterKeySignature: string | null + accountRole: "standard" | "debug" } export interface AuthUser { - id: string; - email?: string; + id: string + email?: string wallet?: { - address: string; - walletClientType: string; - }; + address: string + walletClientType: string + } } export interface ConnectedApp { - id: string; - name: string; - icon?: string; - permissions: string[]; - connectedAt: string; - externalUrl?: string; + id: string + name: string + icon?: string + permissions: string[] + connectedAt: string + externalUrl?: string } export interface AppConfig { - storageProvider: 'local' | 'vana' | 'gdrive' | 'dropbox'; + storageProvider: "local" | "vana" | "gdrive" | "dropbox" /** * - `local-only` (default): no backend provider is configured. Exports * land on local disk only (`write_export_data`, always unconditional) @@ -186,18 +202,18 @@ export interface AppConfig { * user obtains the access token via the "Connect with Vana" Hydra * device-code flow. */ - serverMode: 'local-only' | 'local' | 'remote'; + serverMode: "local-only" | "local" | "remote" /** Required when `serverMode === 'remote'`. e.g. `https://0xabc….myvana.app`. */ - remoteServerUrl?: string; + remoteServerUrl?: string /** * Vana session access token for the remote PS. Stored in plaintext for * dev simplicity; production should move to Tauri Stronghold or OS * Keychain. Refreshed on demand via the refresh token at * `vanaRefreshToken`. */ - vanaAccessToken?: string; + vanaAccessToken?: string /** Vana session refresh token. */ - vanaRefreshToken?: string; + vanaRefreshToken?: string /** Unix epoch (seconds) when `vanaAccessToken` expires. */ - vanaAccessTokenExpiresAt?: number; + vanaAccessTokenExpiresAt?: number } From c542ff164145583022a89be54393a3f6998a199d Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 31 Jul 2026 14:28:22 -0500 Subject: [PATCH 02/14] feat(pdpp): serve selected ChatGPT profile Generalize verified manifest selection, snapshot import, authorization, and stream serving while retaining the GitHub compatibility path. Bind authorization and resource composition to one verified active install so divergent manifests fail closed. Signed-off-by: Tim Nunamaker Assisted-by: AI --- personal-server/index.js | 85 ++- .../pdpp/github-authorization/http-routes.js | 86 +-- .../pdpp/github-authorization/index.js | 81 ++- .../pdpp/github-authorization/policy.js | 30 +- personal-server/pdpp/github-snapshot.js | 27 +- .../pdpp/grant-scoped-records-repository.js | 242 +++++-- .../grant-scoped-records-repository.test.js | 55 +- personal-server/pdpp/installed-manifest.js | 260 ++++++-- .../pdpp/installed-manifest.test.js | 119 ++-- personal-server/pdpp/resource-server.js | 38 +- personal-server/pdpp/resource-server.test.js | 589 +++++++++++++++++- .../fixtures/chatgpt.collection-profile.js | 61 ++ 12 files changed, 1395 insertions(+), 278 deletions(-) create mode 100644 personal-server/pdpp/test/fixtures/chatgpt.collection-profile.js diff --git a/personal-server/index.js b/personal-server/index.js index bdf07a86..9a00ebf2 100644 --- a/personal-server/index.js +++ b/personal-server/index.js @@ -22,13 +22,14 @@ process.env.NODE_ENV = 'production'; import { dirname, join } from 'node:path'; import { readFileSync, writeFileSync } from 'node:fs'; import { homedir } from 'node:os'; -import { randomUUID } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import { execSync, spawn } from 'node:child_process'; import { pathToFileURL } from 'node:url'; import { registerProtectedRoutes } from './protected-routes.js'; import { mountPdppResourceServer } from './pdpp/resource-server.js'; -import { createGithubAuthorizationAdapter } from './pdpp/github-authorization/index.js'; -import { registerGithubAuthorizationRoutes } from './pdpp/github-authorization/http-routes.js'; +import { createPdppAuthorizationAdapter } from './pdpp/github-authorization/index.js'; +import { registerPdppAuthorizationRoutes } from './pdpp/github-authorization/http-routes.js'; +import { loadInstalledManifest } from './pdpp/installed-manifest.js'; import { createPersonalServerServeOptions } from './listener-options.cjs'; const PACKAGED_RUNTIME_ENTRYPOINTS = { @@ -37,6 +38,49 @@ const PACKAGED_RUNTIME_ENTRYPOINTS = { '@hono/node-server': '@hono/node-server/dist/index.js', }; +const PDPP_SERVING_PROFILES = { + 'github-pdpp': { + connector: { + key: 'github', + id: 'https://registry.pdpp.org/connectors/github', + }, + scopeForStream: stream => ({ + user: 'github.profile', + repositories: 'github.repositories', + starred: 'github.starred', + })[stream], + enableLocalTimeline: true, + }, + 'chatgpt-pdpp': { + connector: { + key: 'chatgpt', + id: 'https://registry.pdpp.org/connectors/chatgpt', + }, + scopeForStream: stream => [ + 'conversations', + 'messages', + 'memories', + 'custom_gpts', + 'custom_instructions', + 'shared_conversations', + ].includes(stream) ? `chatgpt.${stream}` : undefined, + enableLocalTimeline: false, + }, +}; + +function selectedPdppServingProfile() { + const connectorId = process.env.PDPP_SERVING_CONNECTOR_ID || 'github-pdpp'; + const profile = PDPP_SERVING_PROFILES[connectorId]; + if (!profile) { + throw new Error(`PDPP_SERVING_CONNECTOR_ID must select a supported profile, got ${connectorId}`); + } + return { connectorId, ...profile }; +} + +function pdppProfileStorageName(connectorId) { + return createHash('sha256').update(connectorId).digest('hex').slice(0, 16); +} + function send(msg) { let json; try { @@ -407,8 +451,25 @@ async function main() { // Keep as a reference because startBackgroundServices mutates context.tunnelManager / context.tunnelUrl. const context = await createServer(config, { rootPath: configDir }); const { app, devToken, cleanup, gatewayClient, serverSigner } = context; - const pdppAuthorization = createGithubAuthorizationAdapter({ - databasePath: join(configDir || join((await import('node:os')).homedir(), '.data-connect', 'personal-server'), 'pdpp-github-authorization.sqlite'), + const selectedPdppProfile = selectedPdppServingProfile(); + const pdppStorageName = pdppProfileStorageName(selectedPdppProfile.connectorId); + const pdppActiveManifestPath = process.env.DATACONNECT_ACTIVE_CONNECTORS_PATH; + // Resolve this once, then require both independently mounted surfaces to + // re-verify this exact selected install. The two surfaces must never + // compose grants from one active-manifest path with records from another. + const pdppSelectedInstall = loadInstalledManifest({ + activeManifestPath: pdppActiveManifestPath, + connectorId: selectedPdppProfile.connectorId, + expectedConnector: selectedPdppProfile.connector, + }); + const pdppAuthorization = createPdppAuthorizationAdapter({ + databasePath: join(configDir || join((await import('node:os')).homedir(), '.data-connect', 'personal-server'), `pdpp-${pdppStorageName}-authorization.sqlite`), + activeManifestPath: pdppActiveManifestPath, + connectorId: selectedPdppProfile.connectorId, + expectedConnector: selectedPdppProfile.connector, + selectedInstall: pdppSelectedInstall, + scopeForStream: selectedPdppProfile.scopeForStream, + enableLocalTimeline: selectedPdppProfile.enableLocalTimeline, singleUseAccessExpiresInSeconds, }); @@ -420,15 +481,18 @@ async function main() { || configDir || join(homedir(), '.dataconnect', 'personal-server'); await mountPdppResourceServer(app, { - activeManifestPath: process.env.DATACONNECT_ACTIVE_CONNECTORS_PATH, - databasePath: join(pdppStorageRoot, 'pdpp-github-records.sqlite'), + activeManifestPath: pdppActiveManifestPath, + connectorId: selectedPdppProfile.connectorId, + expectedConnector: selectedPdppProfile.connector, + selectedInstall: pdppSelectedInstall, + databasePath: join(pdppStorageRoot, `pdpp-${pdppStorageName}-records.sqlite`), exportRoot: process.env.DATACONNECT_EXPORT_ROOT, - connectionId: process.env.PDPP_GITHUB_CONNECTION_ID || 'default', + connectionId: process.env.PDPP_SERVING_CONNECTION_ID || process.env.PDPP_GITHUB_CONNECTION_ID || 'default', tokenIntrospector: { introspect: token => pdppAuthorization.resolveForResourceServer(token), }, }); - send({ type: 'log', message: '[pdpp] mounted installed GitHub resource routes' }); + send({ type: 'log', message: `[pdpp] mounted selected ${selectedPdppProfile.connector.key} resource routes` }); } catch (error) { send({ type: 'log', message: `[pdpp] resource routes unavailable: ${error instanceof Error ? error.message : String(error)}` }); } @@ -471,10 +535,11 @@ async function main() { serverSigner, onLegacyGrantRevoked: grantId => pdppAuthorization.revokeByLegacyGrantId(grantId), }); - registerGithubAuthorizationRoutes({ + registerPdppAuthorizationRoutes({ app, devToken, adapter: pdppAuthorization, + enableLocalTimeline: selectedPdppProfile.enableLocalTimeline, externalOrigin: personalServerExternalOrigin( context.serverAccount?.address, tunnelServerAddr diff --git a/personal-server/pdpp/github-authorization/http-routes.js b/personal-server/pdpp/github-authorization/http-routes.js index ab70a88e..ba80660f 100644 --- a/personal-server/pdpp/github-authorization/http-routes.js +++ b/personal-server/pdpp/github-authorization/http-routes.js @@ -26,6 +26,7 @@ export function registerGithubAuthorizationRoutes({ devToken, adapter, externalOrigin, + enableLocalTimeline = true, }) { const desktopAuth = requireDesktopAuth(devToken) @@ -90,53 +91,62 @@ export function registerGithubAuthorizationRoutes({ } } ) - app.post("/v1/pdpp/local-timeline/consent-requests", desktopAuth, async c => { - try { - const body = await c.req.json() - return c.json( - adapter.createLocalTimelineConsentRequest({ - sessionId: body.session_id, - subjectId: body.subject_id, - }), - 201 - ) - } catch (error) { - return errorResponse(c, error) - } - }) - app.post( - "/v1/pdpp/local-timeline/consent-requests/:requestId/approve", - desktopAuth, - async c => { + if (enableLocalTimeline) { + app.post( + "/v1/pdpp/local-timeline/consent-requests", + desktopAuth, + async c => { + try { + const body = await c.req.json() + return c.json( + adapter.createLocalTimelineConsentRequest({ + sessionId: body.session_id, + subjectId: body.subject_id, + }), + 201 + ) + } catch (error) { + return errorResponse(c, error) + } + } + ) + app.post( + "/v1/pdpp/local-timeline/consent-requests/:requestId/approve", + desktopAuth, + async c => { + try { + const body = await c.req.json() + return c.json( + adapter.issueLocalTimelineGrant({ + requestId: c.req.param("requestId"), + sessionId: body.session_id, + subjectId: body.subject_id, + }), + 201 + ) + } catch (error) { + return errorResponse(c, error) + } + } + ) + app.post("/v1/pdpp/local-timeline/revoke", desktopAuth, async c => { try { const body = await c.req.json() - return c.json( - adapter.issueLocalTimelineGrant({ - requestId: c.req.param("requestId"), + return c.json({ + revoked: adapter.revokeLocalTimelineSession({ sessionId: body.session_id, subjectId: body.subject_id, }), - 201 - ) + }) } catch (error) { return errorResponse(c, error) } - } - ) - app.post("/v1/pdpp/local-timeline/revoke", desktopAuth, async c => { - try { - const body = await c.req.json() - return c.json({ - revoked: adapter.revokeLocalTimelineSession({ - sessionId: body.session_id, - subjectId: body.subject_id, - }), - }) - } catch (error) { - return errorResponse(c, error) - } - }) + }) + } app.post("/v1/pdpp/introspect", c => c.json(adapter.introspectPublicBearer(c.req.header("authorization"))) ) } + +/** Neutral route registration retains the existing endpoint contract. */ +export const registerPdppAuthorizationRoutes = registerGithubAuthorizationRoutes diff --git a/personal-server/pdpp/github-authorization/index.js b/personal-server/pdpp/github-authorization/index.js index a4d980b7..6ddca556 100644 --- a/personal-server/pdpp/github-authorization/index.js +++ b/personal-server/pdpp/github-authorization/index.js @@ -1,10 +1,8 @@ -import { createHash } from "node:crypto" -import { readFileSync } from "node:fs" -import { loadInstalledGithubManifest } from "../installed-manifest.js" +import { resolveSelectedInstalledManifest } from "../installed-manifest.js" import { createLocalTimelineAuthorizationRequest, LOCAL_TIMELINE_CLIENT_ID, - validateGithubAuthorizationDetails, + validateAuthorizationDetails, } from "./policy.js" import { openGithubAuthorizationStore } from "./store.js" @@ -19,21 +17,34 @@ function inactive() { return { active: false } } -function readVerifiedManifest(activeManifestPath) { - const installed = loadInstalledGithubManifest({ activeManifestPath }) +function readVerifiedManifest( + activeManifestPath, + connectorId, + expectedConnector, + selectedInstall +) { + const installed = resolveSelectedInstalledManifest({ + activeManifestPath, + connectorId, + expectedConnector, + selectedInstall, + }) return { version: installed.version, - digest: createHash("sha256") - .update(readFileSync(installed.manifestPath)) - .digest("hex"), + digest: installed.manifestDigest, manifest: installed.manifest, } } -/** A separate UAT authorization composition; it intentionally does not alter query-core pdpp/index.js. */ -export function createGithubAuthorizationAdapter({ +/** Selected-install authorization composition. The route core remains unchanged. */ +export function createPdppAuthorizationAdapter({ databasePath, activeManifestPath, + connectorId = "github-pdpp", + expectedConnector, + selectedInstall, + scopeForStream, + enableLocalTimeline = false, now, random, singleUseAccessExpiresInSeconds = SINGLE_USE_ACCESS_EXPIRES_IN_SECONDS, @@ -47,7 +58,16 @@ export function createGithubAuthorizationAdapter({ credentialHandoffExpiresInSeconds, }) const clock = now ?? (() => new Date()) - const currentManifest = () => readVerifiedManifest(activeManifestPath) + if (typeof scopeForStream !== "function") { + throw new TypeError("scopeForStream must be a function") + } + const currentManifest = () => + readVerifiedManifest( + activeManifestPath, + connectorId, + expectedConnector, + selectedInstall + ) const publicIdentity = token => { try { const found = store.findActiveGrant(token, currentManifest()) @@ -68,19 +88,27 @@ export function createGithubAuthorizationAdapter({ return { createConsentRequest({ sessionId, scopes, authorizationDetails }) { const manifest = currentManifest() - const terms = validateGithubAuthorizationDetails({ + const terms = validateAuthorizationDetails({ authorizationDetails, manifest: manifest.manifest, scopes, + scopeForStream, }) return store.createRequest({ sessionId, scopes, terms, manifest }) }, createLocalTimelineConsentRequest({ sessionId, subjectId }) { + if (!enableLocalTimeline) { + throw new Error( + "The selected connector does not support Timeline consent" + ) + } const manifest = currentManifest() const local = createLocalTimelineAuthorizationRequest(manifest.manifest) - const terms = validateGithubAuthorizationDetails({ + const terms = validateAuthorizationDetails({ ...local, manifest: manifest.manifest, + scopeForStream: stream => `pdpp.local.github.${stream}`, + sourceIds: ["github", "https://registry.pdpp.org/connectors/github"], localTimeline: true, }) return { @@ -132,6 +160,11 @@ export function createGithubAuthorizationAdapter({ }) }, issueLocalTimelineGrant({ requestId, sessionId, subjectId }) { + if (!enableLocalTimeline) { + throw new Error( + "The selected connector does not support Timeline consent" + ) + } return store.issueGrant({ requestId, // The legacy field is retained for the existing schema and legacy @@ -171,6 +204,11 @@ export function createGithubAuthorizationAdapter({ }, revokeByLegacyGrantId: store.revokeByLegacyGrantId, revokeLocalTimelineSession({ sessionId, subjectId }) { + if (!enableLocalTimeline) { + throw new Error( + "The selected connector does not support Timeline consent" + ) + } return store.revokeBoundSession({ sessionId, subjectId, @@ -181,6 +219,21 @@ export function createGithubAuthorizationAdapter({ } } +/** GitHub remains the default authorization composition for deployed clients. */ +export function createGithubAuthorizationAdapter(options = {}) { + const githubScopes = { + user: "github.profile", + repositories: "github.repositories", + starred: "github.starred", + } + return createPdppAuthorizationAdapter({ + ...options, + connectorId: options.connectorId ?? "github-pdpp", + scopeForStream: stream => githubScopes[stream], + enableLocalTimeline: true, + }) +} + function requiredLocalSession(sessionId) { if (typeof sessionId !== "string" || !sessionId.trim()) { throw new Error("session_id is required") diff --git a/personal-server/pdpp/github-authorization/policy.js b/personal-server/pdpp/github-authorization/policy.js index cc3ce0fd..89fa260a 100644 --- a/personal-server/pdpp/github-authorization/policy.js +++ b/personal-server/pdpp/github-authorization/policy.js @@ -157,10 +157,12 @@ function normalizeSelectedFields(stream, selected) { * currently hash-verified installed manifest. The stream-to-scope check binds * this additional authorization to the legacy Session Relay authorization. */ -export function validateGithubAuthorizationDetails({ +export function validateAuthorizationDetails({ authorizationDetails, manifest, scopes, + scopeForStream, + sourceIds = [manifest?.connector_key, manifest?.connector_id], localTimeline = false, }) { if ( @@ -184,10 +186,10 @@ export function validateGithubAuthorizationDetails({ if ( !detail.source || detail.source.kind !== "connector" || - detail.source.id !== "github" || + !sourceIds.includes(detail.source.id) || Object.keys(detail.source).length !== 2 ) { - throw invalid("source must be { kind: 'connector', id: 'github' }") + throw invalid("source must identify the selected connector") } if (!["single_use", "continuous"].includes(detail.access_mode)) { throw invalid('access_mode must be "single_use" or "continuous"') @@ -263,22 +265,21 @@ export function validateGithubAuthorizationDetails({ return normalized }) - const requestedScopes = normalizedStreams.map(stream => - localTimeline - ? `pdpp.local.github.${stream.name}` - : GITHUB_STREAM_SCOPES[stream.name] - ) + const requestedScopes = normalizedStreams.map(stream => { + if (localTimeline) return `pdpp.local.github.${stream.name}` + return scopeForStream?.(stream.name) + }) if ( requestedScopes.some(scope => !scope) || !sameScopeSet(requestedScopes, scopes) ) { throw invalid( - "GitHub authorization details do not exactly match the claimed session scopes" + "Authorization details do not exactly match the claimed session scopes" ) } return { type: PDPP_DATA_ACCESS_TYPE, - source: { kind: "connector", id: "github" }, + source: { kind: "connector", id: detail.source.id }, access_mode: detail.access_mode, purpose_code: detail.purpose_code, purpose_description: detail.purpose_description, @@ -289,3 +290,12 @@ export function validateGithubAuthorizationDetails({ streams: normalizedStreams, } } + +/** GitHub policy is retained as the legacy Session Relay compatibility adapter. */ +export function validateGithubAuthorizationDetails(options) { + return validateAuthorizationDetails({ + ...options, + sourceIds: ["github", "https://registry.pdpp.org/connectors/github"], + scopeForStream: stream => GITHUB_STREAM_SCOPES[stream], + }) +} diff --git a/personal-server/pdpp/github-snapshot.js b/personal-server/pdpp/github-snapshot.js index 8f370b53..30957744 100644 --- a/personal-server/pdpp/github-snapshot.js +++ b/personal-server/pdpp/github-snapshot.js @@ -5,12 +5,7 @@ import { RecordsRepositoryError } from "./grant-scoped-records-repository.js" const DEFAULT_FILE_OPERATIONS = { readdir, readFile, stat } -async function visitJsonFiles( - directory, - files, - directories, - fileOperations -) { +async function visitJsonFiles(directory, files, directories, fileOperations) { directories.set( directory, directoryGeneration(await fileOperations.stat(directory)) @@ -26,10 +21,10 @@ async function visitJsonFiles( } /** - * Return lossless GitHub export candidates newest first. The repository validates + * Return lossless selected-profile export candidates newest first. The repository validates * their record envelopes transactionally before one is accepted for serving. */ -export async function findGithubSnapshotCandidates({ +export async function findSnapshotCandidates({ exportRoot, manifest, manifestDigest, @@ -79,7 +74,7 @@ export async function findGithubSnapshotCandidates({ } continue } - const result = await readGithubSnapshotCandidate({ + const result = await readSnapshotCandidate({ path, generation, metadata, @@ -112,7 +107,7 @@ export async function findGithubSnapshotCandidates({ * record validation. `importSnapshot` is one SQLite transaction, so a rejected * candidate cannot leave partial collection state behind. */ -export async function importLatestGithubSnapshot({ +export async function importLatestSnapshot({ exportRoot, manifest, manifestDigest, @@ -127,7 +122,7 @@ export async function importLatestGithubSnapshot({ }, fileOperations = DEFAULT_FILE_OPERATIONS, }) { - const candidates = await findGithubSnapshotCandidates({ + const candidates = await findSnapshotCandidates({ exportRoot, manifest, manifestDigest, @@ -222,7 +217,7 @@ async function loadCachedCandidate({ if (fileGeneration(candidate.path, metadata) !== candidate.generation) { return null } - const result = await readGithubSnapshotCandidate({ + const result = await readSnapshotCandidate({ path: candidate.path, generation: candidate.generation, metadata, @@ -237,7 +232,7 @@ async function loadCachedCandidate({ } } -async function readGithubSnapshotCandidate({ +async function readSnapshotCandidate({ path, generation, metadata, @@ -260,7 +255,7 @@ async function readGithubSnapshotCandidate({ const content = exportFile?.content const recordsByStream = content?.["pdpp.recordsByStream"] if ( - content?.platform !== "github" || + content?.platform !== manifest.connector_key || content?.version !== manifest.version || !hasVerifiedSnapshotProvenance({ provenance: content?.["pdpp.provenance"], @@ -284,6 +279,10 @@ async function readGithubSnapshotCandidate({ } } +/** GitHub names remain available to avoid an unrelated call-site migration. */ +export const findGithubSnapshotCandidates = findSnapshotCandidates +export const importLatestGithubSnapshot = importLatestSnapshot + function fileGeneration(path, metadata) { return `${path}\0${metadata.mtimeMs}:${metadata.size}` } diff --git a/personal-server/pdpp/grant-scoped-records-repository.js b/personal-server/pdpp/grant-scoped-records-repository.js index 7eb92489..f9253314 100644 --- a/personal-server/pdpp/grant-scoped-records-repository.js +++ b/personal-server/pdpp/grant-scoped-records-repository.js @@ -67,21 +67,29 @@ export const GITHUB_STREAMS = Object.freeze({ }) /** - * Turn the hash-verified installed profile into the repository's narrow - * record contract. We deliberately support the profile's declared GitHub + * Turn the hash-verified selected profile into the repository's narrow + * record contract. We deliberately support only the profile's declared * streams, not an independently-maintained stream-name list. */ -export function createGithubStreamMetadata(manifest) { +export function createStreamMetadata(manifest) { if (!isPlainObject(manifest) || !Array.isArray(manifest.streams)) { - throw new TypeError("Installed GitHub manifest must declare streams") + throw new TypeError("Installed connector manifest must declare streams") } const streams = {} for (const stream of manifest.streams) { - if (!isPlainObject(stream) || typeof stream.name !== "string" || !stream.name) { - throw new TypeError("Installed GitHub manifest stream must have a name") + if ( + !isPlainObject(stream) || + typeof stream.name !== "string" || + !stream.name + ) { + throw new TypeError( + "Installed connector manifest stream must have a name" + ) } if (Object.hasOwn(streams, stream.name)) { - throw new TypeError(`Installed GitHub manifest has duplicate stream '${stream.name}'`) + throw new TypeError( + `Installed connector manifest has duplicate stream '${stream.name}'` + ) } const schema = stream.schema const properties = schema?.properties @@ -93,14 +101,17 @@ export function createGithubStreamMetadata(manifest) { !Array.isArray(stream.primary_key) || stream.primary_key.length !== 1 || stream.primary_key[0] !== "id" || - !requiredFields.includes("id") + !requiredFields.includes("id") || + typeof stream.cursor_field !== "string" ) { throw new TypeError( - `Installed GitHub stream '${stream.name}' has an unsupported record contract` + `Installed connector stream '${stream.name}' has an unsupported record contract` ) } const fields = Object.keys(properties) - const timingFields = [stream.cursor_field, stream.consent_time_field] + const timingFields = [stream.cursor_field] + if (stream.consent_time_field !== undefined) + timingFields.push(stream.consent_time_field) if ( timingFields.some( field => @@ -108,11 +119,13 @@ export function createGithubStreamMetadata(manifest) { !fields.includes(field) || !["date", "date-time"].includes(properties[field]?.format) ) || - requiredFields.some(field => typeof field !== "string" || !fields.includes(field)) || + requiredFields.some( + field => typeof field !== "string" || !fields.includes(field) + ) || fields.some(field => !validSchemaProperty(properties[field])) ) { throw new TypeError( - `Installed GitHub stream '${stream.name}' has an unsupported record contract` + `Installed connector stream '${stream.name}' has an unsupported record contract` ) } streams[stream.name] = Object.freeze({ @@ -121,7 +134,10 @@ export function createGithubStreamMetadata(manifest) { consentTimeField: stream.consent_time_field, fields, fieldTypes: Object.fromEntries( - fields.map(field => [field, normalizeSchemaTypes(properties[field].type)]) + fields.map(field => [ + field, + normalizeSchemaTypes(properties[field].type), + ]) ), fieldFormats: Object.fromEntries( fields.map(field => [field, properties[field].format ?? null]) @@ -131,6 +147,9 @@ export function createGithubStreamMetadata(manifest) { return Object.freeze(streams) } +/** GitHub-named export retained for existing callers and tests. */ +export const createGithubStreamMetadata = createStreamMetadata + function validSchemaProperty(property) { return ( isPlainObject(property) && @@ -143,7 +162,17 @@ function validSchemaProperty(property) { function normalizeSchemaTypes(value) { const types = Array.isArray(value) ? value : [value] - return types.every(type => ["string", "integer", "number", "boolean", "array", "object", "null"].includes(type)) + return types.every(type => + [ + "string", + "integer", + "number", + "boolean", + "array", + "object", + "null", + ].includes(type) + ) ? types : [] } @@ -217,18 +246,24 @@ export class GrantScopedRecordsRepository { } upsert({ connectionId, stream, key, data, emittedAt = this.#now() }) { - const identity = validateLiveRecord({ - connectionId, - stream, - key, - data, - emittedAt, - }, this.#streamMetadata) + const identity = validateLiveRecord( + { + connectionId, + stream, + key, + data, + emittedAt, + }, + this.#streamMetadata + ) return this.#mutate(identity, "upsert") } delete({ connectionId, stream, key, emittedAt = this.#now() }) { - const identity = validateDelete({ connectionId, stream, key, emittedAt }, this.#streamMetadata) + const identity = validateDelete( + { connectionId, stream, key, emittedAt }, + this.#streamMetadata + ) return this.#mutate(identity, "delete") } @@ -245,7 +280,11 @@ export class GrantScopedRecordsRepository { "recordsByStream must be an object" ) } - const snapshotMetadata = validateSnapshotMetadata(snapshot, recordsByStream, this.#streamMetadata) + const snapshotMetadata = validateSnapshotMetadata( + snapshot, + recordsByStream, + this.#streamMetadata + ) const apply = this.#db.transaction(() => { const results = [] @@ -269,24 +308,30 @@ export class GrantScopedRecordsRepository { } const op = envelope.op ?? "upsert" if (op === "upsert") { - const record = validateLiveRecord({ - connectionId, - stream, - key: envelope.key, - data: envelope.data, - emittedAt: envelope.emitted_at, - }, this.#streamMetadata) + const record = validateLiveRecord( + { + connectionId, + stream, + key: envelope.key, + data: envelope.data, + emittedAt: envelope.emitted_at, + }, + this.#streamMetadata + ) observedKeysByResetStream.get(stream)?.add(record.key) results.push(this.#mutateInTransaction(record, "upsert")) } else if (op === "delete") { results.push( this.#mutateInTransaction( - validateDelete({ - connectionId, - stream, - key: envelope.key, - emittedAt: envelope.emitted_at, - }, this.#streamMetadata), + validateDelete( + { + connectionId, + stream, + key: envelope.key, + emittedAt: envelope.emitted_at, + }, + this.#streamMetadata + ), "delete" ) ) @@ -315,7 +360,12 @@ export class GrantScopedRecordsRepository { getCurrent({ connectionId, stream, key, grant }) { validateLocation({ connectionId, stream, key }, this.#streamMetadata) - const effectiveGrant = normalizeGrant(stream, grant, undefined, this.#streamMetadata) + const effectiveGrant = normalizeGrant( + stream, + grant, + undefined, + this.#streamMetadata + ) const row = this.#db .prepare( ` @@ -331,7 +381,12 @@ export class GrantScopedRecordsRepository { summarizeCurrent({ connectionId, stream, grant }) { validateLocation({ connectionId, stream }, this.#streamMetadata) - const effectiveGrant = normalizeGrant(stream, grant, undefined, this.#streamMetadata) + const effectiveGrant = normalizeGrant( + stream, + grant, + undefined, + this.#streamMetadata + ) const metadata = this.#streamMetadata[stream] const visible = this.#db .prepare( @@ -343,17 +398,26 @@ export class GrantScopedRecordsRepository { ) .all(connectionId, stream) .map(row => { - const record = discloseLiveRow(row, stream, effectiveGrant, this.#streamMetadata) + const record = discloseLiveRow( + row, + stream, + effectiveGrant, + this.#streamMetadata + ) return record === null ? null - : { cursorValue: JSON.parse(row.payload)[metadata.cursorField], record } + : { + cursorValue: JSON.parse(row.payload)[metadata.cursorField], + record, + } }) .filter(entry => entry !== null) - const updated = visible - .map(entry => entry.cursorValue) - .filter(value => typeof value === "string") - .sort() - .at(-1) ?? null + const updated = + visible + .map(entry => entry.cursorValue) + .filter(value => typeof value === "string") + .sort() + .at(-1) ?? null return { record_count: visible.length, last_updated: updated } } @@ -370,7 +434,12 @@ export class GrantScopedRecordsRepository { validateLocation({ connectionId, stream }, this.#streamMetadata) const normalizedLimit = normalizeLimit(limit) const normalizedOrder = normalizeOrder(order) - const effectiveGrant = normalizeGrant(stream, grant, fields, this.#streamMetadata) + const effectiveGrant = normalizeGrant( + stream, + grant, + fields, + this.#streamMetadata + ) const pageCursor = cursor ? decodeCurrentCursor(cursor, connectionId, stream, normalizedOrder) : null @@ -386,15 +455,21 @@ export class GrantScopedRecordsRepository { const visible = rows .map(row => { - const record = discloseLiveRow(row, stream, effectiveGrant, this.#streamMetadata) + const record = discloseLiveRow( + row, + stream, + effectiveGrant, + this.#streamMetadata + ) return record === null ? null : { record, key: row.record_key, cursorValue: - JSON.parse(row.payload)[this.#streamMetadata[stream].cursorField] ?? - null, + JSON.parse(row.payload)[ + this.#streamMetadata[stream].cursorField + ] ?? null, } }) .filter( @@ -444,7 +519,12 @@ export class GrantScopedRecordsRepository { }) { validateLocation({ connectionId, stream }, this.#streamMetadata) const normalizedLimit = normalizeLimit(limit) - const effectiveGrant = normalizeGrant(stream, grant, undefined, this.#streamMetadata) + const effectiveGrant = normalizeGrant( + stream, + grant, + undefined, + this.#streamMetadata + ) const session = cursor ? decodeChangesCursor(cursor, connectionId, stream) : this.#startChangesSession(connectionId, stream, changesSince) @@ -655,7 +735,10 @@ export class GrantScopedRecordsRepository { .filter(key => !presentKeys.has(key)) .map(key => this.#mutateInTransaction( - validateDelete({ connectionId, stream, key, emittedAt }, this.#streamMetadata), + validateDelete( + { connectionId, stream, key, emittedAt }, + this.#streamMetadata + ), "delete" ) ) @@ -773,7 +856,9 @@ export class GrantScopedRecordsRepository { change.record_key, sinceVersion ) - const beforeRecord = before ? discloseSnapshot(before, stream, grant, this.#streamMetadata) : null + const beforeRecord = before + ? discloseSnapshot(before, stream, grant, this.#streamMetadata) + : null const afterRecord = change.deleted ? null : discloseSnapshot(change, stream, grant, this.#streamMetadata) @@ -807,7 +892,10 @@ export class GrantScopedRecordsRepository { } } -function validateLiveRecord({ connectionId, stream, key, data, emittedAt }, streams) { +function validateLiveRecord( + { connectionId, stream, key, data, emittedAt }, + streams +) { validateLocation({ connectionId, stream, key }, streams) if (!isPlainObject(data)) throw new RecordsRepositoryError( @@ -816,7 +904,12 @@ function validateLiveRecord({ connectionId, stream, key, data, emittedAt }, stre ) const metadata = requireStream(stream, streams) for (const field of metadata.requiredFields) { - if (!matchesSchemaType(data[field], metadata.fieldTypes?.[field] ?? ["string"])) { + if ( + !matchesSchemaType( + data[field], + metadata.fieldTypes?.[field] ?? ["string"] + ) + ) { throw new RecordsRepositoryError( "invalid_record", `Stream '${stream}' requires a valid '${field}'` @@ -829,7 +922,14 @@ function validateLiveRecord({ connectionId, stream, key, data, emittedAt }, stre "Record key must equal data.id" ) } - if (!validTemporalValue(data[metadata.consentTimeField], metadata.fieldFormats?.[metadata.consentTimeField] ?? "date-time")) { + if ( + metadata.consentTimeField && + data[metadata.consentTimeField] != null && + !validTemporalValue( + data[metadata.consentTimeField], + metadata.fieldFormats?.[metadata.consentTimeField] ?? "date-time" + ) + ) { throw new RecordsRepositoryError( "invalid_record", `Stream '${stream}' requires a valid '${metadata.consentTimeField}' timestamp` @@ -837,7 +937,10 @@ function validateLiveRecord({ connectionId, stream, key, data, emittedAt }, stre } if ( data[metadata.cursorField] != null && - !validTemporalValue(data[metadata.cursorField], metadata.fieldFormats?.[metadata.cursorField] ?? "date-time") + !validTemporalValue( + data[metadata.cursorField], + metadata.fieldFormats?.[metadata.cursorField] ?? "date-time" + ) ) { throw new RecordsRepositoryError( "invalid_record", @@ -990,6 +1093,12 @@ function normalizeGrant(stream, grant = {}, requestedFields, streams) { ? null : normalizeStringList(grant.resources, "grant.resources") const timeRange = grant.timeRange ?? grant.time_range + if (timeRange !== undefined && !metadata.consentTimeField) { + throw new RecordsRepositoryError( + "invalid_request", + `Stream '${stream}' does not support time_range` + ) + } if ( timeRange !== undefined && (!isPlainObject(timeRange) || @@ -1039,6 +1148,7 @@ function discloseSnapshot(row, stream, grant, streams) { const metadata = requireStream(stream, streams) if (grant.resources && !grant.resources.includes(data.id)) return null const time = data[metadata.consentTimeField] + if (grant.timeRange && time == null) return null if ( grant.timeRange?.since && Date.parse(time) < Date.parse(grant.timeRange.since) @@ -1054,7 +1164,9 @@ function discloseSnapshot(row, stream, grant, streams) { ) const projected = grant.fields ? Object.fromEntries( - Object.entries(manifestData).filter(([field]) => grant.fields.includes(field)) + Object.entries(manifestData).filter(([field]) => + grant.fields.includes(field) + ) ) : manifestData return { @@ -1123,9 +1235,14 @@ function compareCurrentRows(left, right, order) { function matchesSchemaType(value, types) { if (value === null) return types.includes("null") - if (typeof value === "string") return types.includes("string") && value.length > 0 + if (typeof value === "string") + return types.includes("string") && value.length > 0 if (typeof value === "number") - return (types.includes("number") || (Number.isInteger(value) && types.includes("integer"))) && Number.isFinite(value) + return ( + (types.includes("number") || + (Number.isInteger(value) && types.includes("integer"))) && + Number.isFinite(value) + ) if (typeof value === "boolean") return types.includes("boolean") if (Array.isArray(value)) return types.includes("array") return isPlainObject(value) && types.includes("object") @@ -1133,7 +1250,12 @@ function matchesSchemaType(value, types) { function validTemporalValue(value, format) { if (value == null) return false - if (format === "date") return typeof value === "string" && /^\d{4}-\d{2}-\d{2}$/.test(value) && !Number.isNaN(Date.parse(`${value}T00:00:00Z`)) + if (format === "date") + return ( + typeof value === "string" && + /^\d{4}-\d{2}-\d{2}$/.test(value) && + !Number.isNaN(Date.parse(`${value}T00:00:00Z`)) + ) return isIsoTimestamp(value) } diff --git a/personal-server/pdpp/grant-scoped-records-repository.test.js b/personal-server/pdpp/grant-scoped-records-repository.test.js index d5497ea9..78f5072e 100644 --- a/personal-server/pdpp/grant-scoped-records-repository.test.js +++ b/personal-server/pdpp/grant-scoped-records-repository.test.js @@ -6,6 +6,7 @@ import test from "node:test" import { CursorExpiredError, + createStreamMetadata, GrantScopedRecordsRepository, RecordsRepositoryError, } from "./grant-scoped-records-repository.js" @@ -28,6 +29,52 @@ function withRepository(options = {}) { } } +test("supports streams without consent_time_field but rejects unsupported serving contracts", () => { + const withoutConsentTime = { + streams: [ + { + name: "events", + primary_key: ["id"], + cursor_field: "updated_at", + schema: { + required: ["id"], + properties: { + id: { type: "string" }, + updated_at: { type: ["string", "null"], format: "date-time" }, + }, + }, + }, + ], + } + assert.equal( + createStreamMetadata(withoutConsentTime).events.consentTimeField, + undefined + ) + for (const invalid of [ + { + ...withoutConsentTime, + streams: [ + { ...withoutConsentTime.streams[0], primary_key: ["id", "other"] }, + ], + }, + { + ...withoutConsentTime, + streams: [{ ...withoutConsentTime.streams[0], cursor_field: "id" }], + }, + { + ...withoutConsentTime, + streams: [ + { ...withoutConsentTime.streams[0], schema: { required: ["id"] } }, + ], + }, + ]) { + assert.throws( + () => createStreamMetadata(invalid), + /unsupported record contract/ + ) + } +}) + const TIMES = { created: "2026-01-01T00:00:00.000Z", updated: "2026-02-01T00:00:00.000Z", @@ -629,7 +676,10 @@ test("manifest fields bound every disclosure while retaining lossless extension emittedAt: TIMES.emitted, }) - const unrestrictedCurrent = harness.repository.listCurrent({ ...base, grant: {} }) + const unrestrictedCurrent = harness.repository.listCurrent({ + ...base, + grant: {}, + }) assert.equal(unrestrictedCurrent.data[0].data.extra, undefined) assert.equal( harness.repository.getCurrent({ ...base, grant: {} }).data.extra, @@ -644,7 +694,8 @@ test("manifest fields bound every disclosure while retaining lossless extension const narrowGrant = { fields: ["id", "full_name"] } assert.deepEqual( - harness.repository.listCurrent({ ...base, grant: narrowGrant }).data[0].data, + harness.repository.listCurrent({ ...base, grant: narrowGrant }).data[0] + .data, { id: "1", full_name: "owner/repo-1" } ) assert.deepEqual( diff --git a/personal-server/pdpp/installed-manifest.js b/personal-server/pdpp/installed-manifest.js index f8f6c91b..11a27452 100644 --- a/personal-server/pdpp/installed-manifest.js +++ b/personal-server/pdpp/installed-manifest.js @@ -1,107 +1,177 @@ -import { createHash } from "node:crypto"; -import { existsSync, readFileSync } from "node:fs"; -import { homedir } from "node:os"; -import { isAbsolute, join, relative, resolve } from "node:path"; +import { createHash } from "node:crypto" +import { existsSync, readFileSync } from "node:fs" +import { homedir } from "node:os" +import { isAbsolute, join, relative, resolve } from "node:path" -const ACTIVE_CONNECTOR_ID = "github-pdpp"; -const CONNECTOR_ID = "https://registry.pdpp.org/connectors/github"; -const CONNECTOR_KEY = "github"; -const ARTIFACT_KIND = "pdpp-collection-profile"; +const ACTIVE_CONNECTOR_ID = "github-pdpp" +const ARTIFACT_KIND = "pdpp-collection-profile" -function fail(message) { - throw new Error(`Invalid installed PDPP GitHub connector: ${message}`); +function fail(label, message) { + throw new Error(`Invalid installed PDPP ${label}: ${message}`) } -function readJson(path, label) { +function readJson(path, label, connectorLabel) { try { - return JSON.parse(readFileSync(path, "utf8")); + return JSON.parse(readFileSync(path, "utf8")) } catch (error) { - fail(`could not read ${label}: ${error instanceof Error ? error.message : String(error)}`); + fail( + connectorLabel, + `could not read ${label}: ${error instanceof Error ? error.message : String(error)}` + ) } } -function requireString(value, field) { - if (typeof value !== "string" || value.length === 0) fail(`${field} is required`); - return value; +function requireString(value, field, connectorLabel) { + if (typeof value !== "string" || value.length === 0) + fail(connectorLabel, `${field} is required`) + return value } -function confinedFile(root, relativePath, label) { - requireString(relativePath, label); - if (isAbsolute(relativePath)) fail(`${label} must be relative`); +function confinedFile(root, relativePath, label, connectorLabel) { + requireString(relativePath, label, connectorLabel) + if (isAbsolute(relativePath)) + fail(connectorLabel, `${label} must be relative`) - const path = resolve(root, relativePath); - const escaped = relative(root, path).startsWith("..") || isAbsolute(relative(root, path)); - if (escaped) fail(`${label} escapes the install root`); - if (!existsSync(path)) fail(`${label} is not accessible`); - return path; + const path = resolve(root, relativePath) + const escaped = + relative(root, path).startsWith("..") || isAbsolute(relative(root, path)) + if (escaped) fail(connectorLabel, `${label} escapes the install root`) + if (!existsSync(path)) fail(connectorLabel, `${label} is not accessible`) + return path } -function verifyHash(path, expected, label) { - requireString(expected, `${label} hash`); - if (!expected.startsWith("sha256:")) fail(`${label} hash must be sha256`); - const actual = `sha256:${createHash("sha256").update(readFileSync(path)).digest("hex")}`; - if (actual !== expected) fail(`${label} hash does not match the active install`); +function verifyHash(path, expected, label, connectorLabel) { + requireString(expected, `${label} hash`, connectorLabel) + if (!expected.startsWith("sha256:")) + fail(connectorLabel, `${label} hash must be sha256`) + const actual = `sha256:${createHash("sha256").update(readFileSync(path)).digest("hex")}` + if (actual !== expected) + fail(connectorLabel, `${label} hash does not match the active install`) } -function validateManifest(install, manifest) { - if (manifest === null || typeof manifest !== "object" || Array.isArray(manifest)) { - fail("manifest must be an object"); +function validateManifest( + install, + manifest, + connectorLabel, + expectedConnector +) { + if ( + manifest === null || + typeof manifest !== "object" || + Array.isArray(manifest) + ) { + fail(connectorLabel, "manifest must be an object") } if ( - manifest.connector_key !== CONNECTOR_KEY || - manifest.connector_id !== CONNECTOR_ID + typeof manifest.connector_key !== "string" || + manifest.connector_key.length === 0 || + typeof manifest.connector_id !== "string" || + manifest.connector_id.length === 0 ) { - fail("manifest identity does not match the active install"); + fail(connectorLabel, "manifest must declare a connector key and ID") } if (manifest.version !== install.version) { - fail(`manifest version ${String(manifest.version)} does not match active version ${install.version}`); + fail( + connectorLabel, + `manifest version ${String(manifest.version)} does not match active version ${install.version}` + ) + } + if ( + expectedConnector && + (manifest.connector_key !== expectedConnector.key || + manifest.connector_id !== expectedConnector.id) + ) { + fail( + connectorLabel, + "manifest identity does not match the selected serving profile" + ) } if (!Array.isArray(manifest.streams) || manifest.streams.length === 0) { - fail("manifest must declare at least one stream"); + fail(connectorLabel, "manifest must declare at least one stream") } - const names = new Set(); + const names = new Set() for (const stream of manifest.streams) { - if (stream === null || typeof stream !== "object" || typeof stream.name !== "string" || stream.name.length === 0) { - fail("manifest stream names must be non-empty strings"); - } - if (!names.add(stream.name)) fail("manifest stream names must be unique"); - } - const network = manifest.runtime_requirements?.bindings?.network; - if (network?.required !== true) fail("manifest must require the network binding"); - for (const [binding, requirement] of Object.entries(manifest.runtime_requirements?.bindings ?? {})) { - if (binding !== "network" && requirement?.required === true) { - fail(`manifest requires unsupported binding ${binding}`); + if ( + stream === null || + typeof stream !== "object" || + typeof stream.name !== "string" || + stream.name.length === 0 + ) { + fail(connectorLabel, "manifest stream names must be non-empty strings") } + if (!names.add(stream.name)) + fail(connectorLabel, "manifest stream names must be unique") } } /** * Load the manifest and provenance from the active, hash-verified install. * Consumers receive the artifact's values; they do not maintain a second - * handwritten GitHub manifest or version constant. + * handwritten connector manifest or version constant. */ -export function loadInstalledGithubManifest({ - activeManifestPath = join(homedir(), ".dataconnect", "connectors-active.json"), - connectorId = ACTIVE_CONNECTOR_ID, +export function loadInstalledManifest({ + activeManifestPath = join( + homedir(), + ".dataconnect", + "connectors-active.json" + ), + connectorId, + connectorLabel = "connector", + expectedConnector, } = {}) { - if (connectorId !== ACTIVE_CONNECTOR_ID) fail(`unsupported connector ${connectorId}`); + requireString(connectorId, "selected connector", connectorLabel) - const active = readJson(activeManifestPath, "active connector manifest"); - const install = active?.connectors?.[connectorId]; - if (install === null || typeof install !== "object") fail("active install is missing"); - if (install.artifactKind !== ARTIFACT_KIND) fail("active install is not a collection profile artifact"); - requireString(install.version, "active install version"); + const active = readJson( + activeManifestPath, + "active connector manifest", + connectorLabel + ) + const install = active?.connectors?.[connectorId] + if (install === null || typeof install !== "object") + fail(connectorLabel, "active install is missing") + if (install.artifactKind !== ARTIFACT_KIND) + fail(connectorLabel, "active install is not a collection profile artifact") + if (install.connectorId !== undefined && install.connectorId !== connectorId) + fail(connectorLabel, "active install ID does not match selected connector") + requireString(install.version, "active install version", connectorLabel) - const root = resolve(requireString(install.rootPath, "active install root")); - const manifestPath = confinedFile(root, install.manifestPath, "manifest path"); - const entrypointPath = confinedFile(root, install.entrypointPath, "entrypoint path"); - const provenancePath = confinedFile(root, install.provenancePath, "provenance path"); - verifyHash(manifestPath, install.manifestSha256, "manifest"); - verifyHash(entrypointPath, install.entrypointSha256, "entrypoint"); - verifyHash(provenancePath, install.provenanceSha256, "provenance"); + const root = resolve( + requireString(install.rootPath, "active install root", connectorLabel) + ) + const manifestPath = confinedFile( + root, + install.manifestPath, + "manifest path", + connectorLabel + ) + const entrypointPath = confinedFile( + root, + install.entrypointPath, + "entrypoint path", + connectorLabel + ) + const provenancePath = confinedFile( + root, + install.provenancePath, + "provenance path", + connectorLabel + ) + verifyHash(manifestPath, install.manifestSha256, "manifest", connectorLabel) + verifyHash( + entrypointPath, + install.entrypointSha256, + "entrypoint", + connectorLabel + ) + verifyHash( + provenancePath, + install.provenanceSha256, + "provenance", + connectorLabel + ) - const manifest = readJson(manifestPath, "manifest"); - validateManifest(install, manifest); - const provenance = readJson(provenancePath, "provenance"); + const manifest = readJson(manifestPath, "manifest", connectorLabel) + validateManifest(install, manifest, connectorLabel, expectedConnector) + const provenance = readJson(provenancePath, "provenance", connectorLabel) return Object.freeze({ connectorId, @@ -112,5 +182,57 @@ export function loadInstalledGithubManifest({ manifestPath, entrypointPath, provenancePath, - }); + }) +} + +/** + * Re-read one explicit active-install path and fail closed if it no longer + * resolves to the selection that was composed at startup. This prevents the + * authorization and resource surfaces from cross-binding same-identity + * artifacts from different active manifests. + */ +export function resolveSelectedInstalledManifest({ + activeManifestPath, + connectorId, + connectorLabel, + expectedConnector, + selectedInstall, +} = {}) { + const installed = loadInstalledManifest({ + activeManifestPath, + connectorId, + connectorLabel, + expectedConnector, + }) + if (!selectedInstall) return installed + if ( + selectedInstall.connectorId !== installed.connectorId || + selectedInstall.version !== installed.version || + selectedInstall.manifestDigest !== installed.manifestDigest || + selectedInstall.manifest.connector_key !== installed.manifest.connector_key || + selectedInstall.manifest.connector_id !== installed.manifest.connector_id + ) { + fail( + connectorLabel ?? "connector", + "active install no longer matches the composed serving selection" + ) + } + return selectedInstall +} + +/** GitHub remains the default serving profile for existing deployments. */ +export function loadInstalledGithubManifest(options = {}) { + const connectorId = options.connectorId ?? ACTIVE_CONNECTOR_ID + if (connectorId !== ACTIVE_CONNECTOR_ID) { + fail("GitHub connector", `unsupported connector ${connectorId}`) + } + return loadInstalledManifest({ + ...options, + connectorId, + connectorLabel: "GitHub connector", + expectedConnector: { + key: "github", + id: "https://registry.pdpp.org/connectors/github", + }, + }) } diff --git a/personal-server/pdpp/installed-manifest.test.js b/personal-server/pdpp/installed-manifest.test.js index 906ba62d..2ab650c8 100644 --- a/personal-server/pdpp/installed-manifest.test.js +++ b/personal-server/pdpp/installed-manifest.test.js @@ -1,22 +1,25 @@ -import { createHash } from "node:crypto"; -import assert from "node:assert/strict"; -import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterEach, describe, it } from "node:test"; -import { loadInstalledGithubManifest } from "./installed-manifest.js"; +import { createHash } from "node:crypto" +import assert from "node:assert/strict" +import { mkdtempSync, mkdirSync, readFileSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { afterEach, describe, it } from "node:test" +import { + loadInstalledGithubManifest, + loadInstalledManifest, +} from "./installed-manifest.js" -const tempRoots = []; +const tempRoots = [] function hash(bytes) { - return `sha256:${createHash("sha256").update(bytes).digest("hex")}`; + return `sha256:${createHash("sha256").update(bytes).digest("hex")}` } function fixture(overrides = {}) { - const root = mkdtempSync(join(tmpdir(), "dataconnect-pdpp-manifest-")); - tempRoots.push(root); - mkdirSync(join(root, "profile")); - mkdirSync(join(root, "dist")); + const root = mkdtempSync(join(tmpdir(), "dataconnect-pdpp-manifest-")) + tempRoots.push(root) + mkdirSync(join(root, "profile")) + mkdirSync(join(root, "dist")) const manifest = { protocol_version: "0.1.0", connector_id: "https://registry.pdpp.org/connectors/github", @@ -25,13 +28,15 @@ function fixture(overrides = {}) { runtime_requirements: { bindings: { network: { required: true } } }, streams: [{ name: "user" }], ...overrides.manifest, - }; - const manifestBytes = Buffer.from(JSON.stringify(manifest)); - const entrypointBytes = Buffer.from("export default {};"); - const provenanceBytes = Buffer.from(JSON.stringify({ source: "fixture", version: "0.5.0" })); - writeFileSync(join(root, "profile/collection-profile.json"), manifestBytes); - writeFileSync(join(root, "dist/collection-profile.mjs"), entrypointBytes); - writeFileSync(join(root, "provenance.json"), provenanceBytes); + } + const manifestBytes = Buffer.from(JSON.stringify(manifest)) + const entrypointBytes = Buffer.from("export default {};") + const provenanceBytes = Buffer.from( + JSON.stringify({ source: "fixture", version: "0.5.0" }) + ) + writeFileSync(join(root, "profile/collection-profile.json"), manifestBytes) + writeFileSync(join(root, "dist/collection-profile.mjs"), entrypointBytes) + writeFileSync(join(root, "provenance.json"), provenanceBytes) const install = { connectorId: "github-pdpp", version: "0.5.0", @@ -44,27 +49,34 @@ function fixture(overrides = {}) { entrypointSha256: hash(entrypointBytes), provenanceSha256: hash(provenanceBytes), ...overrides.install, - }; - const activePath = join(root, "connectors-active.json"); - writeFileSync(activePath, JSON.stringify({ connectors: { "github-pdpp": install } })); - return { activePath, root }; + } + const activePath = join(root, "connectors-active.json") + writeFileSync( + activePath, + JSON.stringify({ connectors: { "github-pdpp": install } }) + ) + return { activePath, root } } afterEach(async () => { - const { rm } = await import("node:fs/promises"); - await Promise.all(tempRoots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); -}); + const { rm } = await import("node:fs/promises") + await Promise.all( + tempRoots.splice(0).map(root => rm(root, { recursive: true, force: true })) + ) +}) describe("loadInstalledGithubManifest", () => { it("returns the actual validated manifest with provenance and version", () => { - const { activePath } = fixture(); - const loaded = loadInstalledGithubManifest({ activeManifestPath: activePath }); - assert.equal(loaded.version, "0.5.0"); - assert.equal(loaded.manifestDigest.startsWith("sha256:"), true); - assert.equal(loaded.manifest.connector_key, "github"); - assert.equal(loaded.manifest.streams[0].name, "user"); - assert.deepEqual(loaded.provenance, { source: "fixture", version: "0.5.0" }); - }); + const { activePath } = fixture() + const loaded = loadInstalledGithubManifest({ + activeManifestPath: activePath, + }) + assert.equal(loaded.version, "0.5.0") + assert.equal(loaded.manifestDigest.startsWith("sha256:"), true) + assert.equal(loaded.manifest.connector_key, "github") + assert.equal(loaded.manifest.streams[0].name, "user") + assert.deepEqual(loaded.provenance, { source: "fixture", version: "0.5.0" }) + }) for (const [label, overrides] of [ ["manifest version", { manifest: { version: "0.6.0" } }], @@ -77,11 +89,38 @@ describe("loadInstalledGithubManifest", () => { ["provenance digest", { install: { provenanceSha256: "sha256:bad" } }], ]) { it(`fails closed on ${label} mismatch`, () => { - const { activePath } = fixture(overrides); + const { activePath } = fixture(overrides) assert.throws( () => loadInstalledGithubManifest({ activeManifestPath: activePath }), - /Invalid installed PDPP GitHub connector/, - ); - }); + /Invalid installed PDPP GitHub connector/ + ) + }) } -}); +}) + +it("loads a selected browser-bound ChatGPT profile after all artifact hashes verify", () => { + const { activePath } = fixture({ + manifest: { + connector_id: "https://registry.pdpp.org/connectors/chatgpt", + connector_key: "chatgpt", + runtime_requirements: { + bindings: { network: { required: true }, browser: { required: true } }, + }, + }, + install: { connectorId: "chatgpt-pdpp" }, + }) + const active = JSON.parse(readFileSync(activePath, "utf8")) + active.connectors["chatgpt-pdpp"] = active.connectors["github-pdpp"] + delete active.connectors["github-pdpp"] + writeFileSync(activePath, JSON.stringify(active)) + + const loaded = loadInstalledManifest({ + activeManifestPath: activePath, + connectorId: "chatgpt-pdpp", + expectedConnector: { + key: "chatgpt", + id: "https://registry.pdpp.org/connectors/chatgpt", + }, + }) + assert.equal(loaded.manifest.connector_key, "chatgpt") +}) diff --git a/personal-server/pdpp/resource-server.js b/personal-server/pdpp/resource-server.js index 1ea16294..898b0b04 100644 --- a/personal-server/pdpp/resource-server.js +++ b/personal-server/pdpp/resource-server.js @@ -5,10 +5,10 @@ import { CursorExpiredError, GrantScopedRecordsRepository, RecordsRepositoryError, - createGithubStreamMetadata, + createStreamMetadata, } from "./grant-scoped-records-repository.js" -import { importLatestGithubSnapshot } from "./github-snapshot.js" -import { loadInstalledGithubManifest } from "./installed-manifest.js" +import { importLatestSnapshot } from "./github-snapshot.js" +import { resolveSelectedInstalledManifest } from "./installed-manifest.js" import { createCoreApp } from "./index.js" import { CoreOperationError } from "./operations.js" import { createHttpTokenIntrospector } from "./token-introspector.js" @@ -16,11 +16,14 @@ import { createHttpTokenIntrospector } from "./token-introspector.js" const DEFAULT_CONNECTION_ID = "default" /** - * Compose the validated installed GitHub profile, durable record store, and + * Compose the validated selected profile, durable record store, and * opaque-token boundary into the transport-independent Core route surface. */ export async function createPdppResourceServer({ activeManifestPath, + connectorId = "github-pdpp", + expectedConnector, + selectedInstall, databasePath, exportRoot, connectionId = DEFAULT_CONNECTION_ID, @@ -31,9 +34,15 @@ export async function createPdppResourceServer({ authorization: process.env.PDPP_INTROSPECTION_AUTHORIZATION, }), } = {}) { - const installed = loadInstalledGithubManifest({ activeManifestPath }) - const streamMetadata = createGithubStreamMetadata(installed.manifest) - const repository = recordsRepository ?? createRepository(databasePath, streamMetadata) + const installed = resolveSelectedInstalledManifest({ + activeManifestPath, + connectorId, + expectedConnector, + selectedInstall, + }) + const streamMetadata = createStreamMetadata(installed.manifest) + const repository = + recordsRepository ?? createRepository(databasePath, streamMetadata) const refreshSnapshot = createSnapshotRefresher({ exportRoot, manifest: installed.manifest, @@ -47,7 +56,7 @@ export async function createPdppResourceServer({ requestId, tokenIntrospector: createGrantValidatedIntrospector( tokenIntrospector, - [installed.manifest.connector_id, installed.manifest.connector_key] + installed ), recordsRepository: createCoreRepositoryPort({ repository, @@ -77,7 +86,7 @@ function createRepository(databasePath, streamMetadata) { return new GrantScopedRecordsRepository({ databasePath, streamMetadata }) } -function createGrantValidatedIntrospector(tokenIntrospector, connectorIds) { +function createGrantValidatedIntrospector(tokenIntrospector, installed) { if (typeof tokenIntrospector?.introspect !== "function") { throw new TypeError("tokenIntrospector.introspect must be a function") } @@ -111,12 +120,17 @@ function createGrantValidatedIntrospector(tokenIntrospector, connectorIds) { } if ( identity.grant.source?.kind !== "connector" || - !connectorIds.includes(identity.grant.source?.id) + ![ + installed.manifest.connector_id, + installed.manifest.connector_key, + ].includes(identity.grant.source?.id) || + identity.grant.manifest_version !== installed.version || + identity.grant.manifest_digest !== installed.manifestDigest ) { throw new CoreOperationError( 403, "grant_invalid", - "The grant is not bound to the installed GitHub connector" + "The grant is not bound to the selected installed manifest" ) } return identity @@ -135,7 +149,7 @@ export function createSnapshotRefresher(dependencies) { } return async () => { if (inFlight !== null) return inFlight - inFlight = importLatestGithubSnapshot({ + inFlight = importLatestSnapshot({ ...dependencies, snapshotCache, }).finally(() => { diff --git a/personal-server/pdpp/resource-server.test.js b/personal-server/pdpp/resource-server.test.js index 61f15fc6..cfa3c119 100644 --- a/personal-server/pdpp/resource-server.test.js +++ b/personal-server/pdpp/resource-server.test.js @@ -1,6 +1,6 @@ import assert from "node:assert/strict" import { createHash } from "node:crypto" -import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs" +import { mkdtempSync, mkdirSync, readFileSync, writeFileSync } from "node:fs" import { readdir, readFile, rm, stat } from "node:fs/promises" import { tmpdir } from "node:os" import { dirname, join } from "node:path" @@ -16,13 +16,19 @@ import { } from "./grant-scoped-records-repository.js" import { createGithubAuthorizationAdapter, + createPdppAuthorizationAdapter, PDPP_DATA_ACCESS_TYPE, } from "./github-authorization/index.js" -import { registerGithubAuthorizationRoutes } from "./github-authorization/http-routes.js" +import { + registerGithubAuthorizationRoutes, + registerPdppAuthorizationRoutes, +} from "./github-authorization/http-routes.js" import { createSnapshotRefresher, mountPdppResourceServer, } from "./resource-server.js" +import { loadInstalledManifest } from "./installed-manifest.js" +import { canonicalChatgptManifestBytes } from "./test/fixtures/chatgpt.collection-profile.js" const tempRoots = [] @@ -201,6 +207,97 @@ function createInstalledGithubFixture({ allStreams = false } = {}) { } } +function createInstalledChatgptFixture() { + const root = mkdtempSync( + join(tmpdir(), "dataconnect-pdpp-chatgpt-resource-server-") + ) + tempRoots.push(root) + const installRoot = join(root, "install") + const exportRoot = join(root, "exports") + mkdirSync(join(installRoot, "profile"), { recursive: true }) + mkdirSync(join(installRoot, "dist"), { recursive: true }) + mkdirSync(exportRoot) + + const manifestBytes = canonicalChatgptManifestBytes + const manifest = JSON.parse(manifestBytes) + const manifestDigest = hash(manifestBytes) + const entrypointBytes = Buffer.from("export default {};") + const provenanceBytes = Buffer.from( + JSON.stringify({ source: "canonical-chatgpt" }) + ) + writeFileSync( + join(installRoot, "profile/collection-profile.json"), + manifestBytes + ) + writeFileSync( + join(installRoot, "dist/collection-profile.mjs"), + entrypointBytes + ) + writeFileSync(join(installRoot, "provenance.json"), provenanceBytes) + const activeManifestPath = join(root, "connectors-active.json") + writeFileSync( + activeManifestPath, + JSON.stringify({ + connectors: { + "chatgpt-pdpp": { + connectorId: "chatgpt-pdpp", + version: manifest.version, + rootPath: installRoot, + artifactKind: "pdpp-collection-profile", + manifestPath: "profile/collection-profile.json", + entrypointPath: "dist/collection-profile.mjs", + provenancePath: "provenance.json", + manifestSha256: manifestDigest, + entrypointSha256: hash(entrypointBytes), + provenanceSha256: hash(provenanceBytes), + }, + }, + }) + ) + return { + root, + installRoot, + activeManifestPath, + exportRoot, + databasePath: join(root, "records.sqlite"), + manifest, + manifestBytes, + manifestDigest, + } +} + +function chatgptSnapshotProvenance({ + manifestDigest, + connectionId = "chatgpt-account-a", + runId = "chatgpt-run-1", + overrides = {}, +}) { + return { + connector_key: "chatgpt", + connector_id: "https://registry.pdpp.org/connectors/chatgpt", + manifest_version: "0.1.0", + manifest_sha256: manifestDigest, + run_id: runId, + connection_id: connectionId, + ...overrides, + } +} + +function chatgptConversation(id, createTime, updateTime = createTime) { + return { + stream: "conversations", + key: id, + data: { + id, + title: `title-${id}`, + create_time: createTime, + update_time: updateTime, + is_archived: false, + }, + emitted_at: "2026-07-31T00:00:00.000Z", + } +} + function githubSnapshotProvenance({ manifestDigest, connectionId = "default", @@ -282,7 +379,7 @@ function record(id, createdAt) { } } -function activeToken(overrides = {}) { +function activeToken({ manifestDigest, grant: grantOverrides, ...overrides } = {}) { return { active: true, pdpp_token_kind: "client", @@ -292,6 +389,8 @@ function activeToken(overrides = {}) { kind: "connector", id: "https://registry.pdpp.org/connectors/github", }, + manifest_version: "0.5.0", + manifest_digest: manifestDigest, streams: [ { name: "repositories", @@ -300,6 +399,7 @@ function activeToken(overrides = {}) { time_range: { since: "2026-01-01T00:00:00Z" }, }, ], + ...grantOverrides, }, ...overrides, } @@ -421,6 +521,8 @@ test("serves every verified GitHub stream while omitted export streams are empty id: "https://registry.pdpp.org/connectors/github", }, streams: streamNames.map(name => ({ name })), + manifest_version: fixture.manifest.version, + manifest_digest: fixture.manifestDigest, }, }), }, @@ -472,6 +574,7 @@ test("mounts installed GitHub PDPP streams beside legacy routes with opaque gran return { active: false, inactive_reason: "grant_expired" } if (token === "wrong-connector") { return activeToken({ + manifestDigest: fixture.manifestDigest, grant: { source: { kind: "connector", @@ -483,6 +586,7 @@ test("mounts installed GitHub PDPP streams beside legacy routes with opaque gran } if (token === "unrestricted-fields") { return activeToken({ + manifestDigest: fixture.manifestDigest, grant: { source: { kind: "connector", @@ -500,6 +604,7 @@ test("mounts installed GitHub PDPP streams beside legacy routes with opaque gran } if (token === "name-only") { return activeToken({ + manifestDigest: fixture.manifestDigest, grant: { source: { kind: "connector", @@ -516,7 +621,7 @@ test("mounts installed GitHub PDPP streams beside legacy routes with opaque gran }, }) } - return activeToken() + return activeToken({ manifestDigest: fixture.manifestDigest }) }, }, }) @@ -666,7 +771,9 @@ test("a successful authoritative full refresh removes records absent from the ne const app = new Hono() await mountPdppResourceServer(app, { ...fixture, - tokenIntrospector: { introspect: async () => activeToken() }, + tokenIntrospector: { + introspect: async () => activeToken({ manifestDigest: fixture.manifestDigest }), + }, }) const headers = { authorization: "Bearer opaque" } const before = await app.request( @@ -766,7 +873,9 @@ test("serves only the requested connection's verified installed snapshot", async const defaultApp = new Hono() await mountPdppResourceServer(defaultApp, { ...fixture, - tokenIntrospector: { introspect: async () => activeToken() }, + tokenIntrospector: { + introspect: async () => activeToken({ manifestDigest: fixture.manifestDigest }), + }, }) const defaultRead = await defaultApp.request( "http://personal.example/v1/streams/repositories/records", @@ -786,11 +895,13 @@ test("serves only the requested connection's verified installed snapshot", async tokenIntrospector: { introspect: async () => activeToken({ + manifestDigest: fixture.manifestDigest, grant: { - ...activeToken().grant, + ...activeToken({ manifestDigest: fixture.manifestDigest }).grant, streams: [ { - ...activeToken().grant.streams[0], + ...activeToken({ manifestDigest: fixture.manifestDigest }).grant + .streams[0], resources: ["other-account"], }, ], @@ -1385,7 +1496,9 @@ test("maps durable cursor errors to stable resource-server responses", async () await mountPdppResourceServer(app, { ...fixture, recordsRepository: repository, - tokenIntrospector: { introspect: async () => activeToken() }, + tokenIntrospector: { + introspect: async () => activeToken({ manifestDigest: fixture.manifestDigest }), + }, }) const headers = { authorization: "Bearer opaque" } const initial = await app.request( @@ -1421,3 +1534,461 @@ test("maps durable cursor errors to stable resource-server responses", async () assert.equal((await malformed.json()).error.code, "invalid_cursor") repository.close() }) + +test("fails closed when the active manifest diverges from the composed selection", async () => { + const fixture = createInstalledGithubFixture() + const divergent = createInstalledGithubFixture() + const selectedInstall = loadInstalledManifest({ + activeManifestPath: fixture.activeManifestPath, + connectorId: "github-pdpp", + expectedConnector: { + key: "github", + id: "https://registry.pdpp.org/connectors/github", + }, + }) + const divergentManifest = { ...divergent.manifest, version: "0.5.1" } + const divergentBytes = Buffer.from(JSON.stringify(divergentManifest)) + writeFileSync( + join( + dirname(divergent.activeManifestPath), + "install/profile/collection-profile.json" + ), + divergentBytes + ) + const divergentActive = JSON.parse(readFileSync(divergent.activeManifestPath)) + divergentActive.connectors["github-pdpp"].version = "0.5.1" + divergentActive.connectors["github-pdpp"].manifestSha256 = hash( + divergentBytes + ) + writeFileSync(divergent.activeManifestPath, JSON.stringify(divergentActive)) + + const adapter = createPdppAuthorizationAdapter({ + activeManifestPath: fixture.activeManifestPath, + connectorId: "github-pdpp", + expectedConnector: { + key: "github", + id: "https://registry.pdpp.org/connectors/github", + }, + selectedInstall, + databasePath: join(dirname(fixture.activeManifestPath), "authorization.sqlite"), + scopeForStream: stream => ({ repositories: "github.repositories" })[stream], + }) + const app = new Hono() + try { + await mountPdppResourceServer(app, { + ...fixture, + selectedInstall, + tokenIntrospector: { + introspect: token => adapter.resolveForResourceServer(token), + }, + }) + const consent = adapter.createConsentRequest({ + sessionId: "selection-bound-session", + scopes: ["github.repositories"], + authorizationDetails: [ + { + type: PDPP_DATA_ACCESS_TYPE, + source: { kind: "connector", id: "github" }, + access_mode: "continuous", + purpose_code: "https://example.test/purpose/research", + streams: [{ name: "repositories" }], + }, + ], + }) + const issued = adapter.issueApprovedGrant({ + requestId: consent.request_id, + legacyGrantId: "selection-bound-grant", + subjectId: "selection-bound-subject", + clientId: "selection-bound-client", + }) + const beforeDivergence = await app.request( + "http://personal.example/v1/streams", + { headers: { authorization: `Bearer ${issued.access_token}` } } + ) + assert.equal(beforeDivergence.status, 200) + + writeFileSync( + fixture.activeManifestPath, + readFileSync(divergent.activeManifestPath) + ) + assert.throws( + () => + adapter.createConsentRequest({ + sessionId: "divergent-selection-session", + scopes: ["github.repositories"], + authorizationDetails: [ + { + type: PDPP_DATA_ACCESS_TYPE, + source: { kind: "connector", id: "github" }, + access_mode: "continuous", + purpose_code: "https://example.test/purpose/research", + streams: [{ name: "repositories" }], + }, + ], + }), + /active install no longer matches the composed serving selection/ + ) + const afterDivergence = await app.request( + "http://personal.example/v1/streams", + { headers: { authorization: `Bearer ${issued.access_token}` } } + ) + assert.equal(afterDivergence.status, 401) + + await assert.rejects( + mountPdppResourceServer(new Hono(), { + ...fixture, + activeManifestPath: divergent.activeManifestPath, + selectedInstall, + tokenIntrospector: { + introspect: token => adapter.resolveForResourceServer(token), + }, + }), + /active install no longer matches the composed serving selection/ + ) + } finally { + adapter.close() + } +}) + +test("serves the selected canonical ChatGPT profile through grant-scoped routes", async () => { + const fixture = createInstalledChatgptFixture() + const desktopToken = "desktop-token" + let now = new Date("2026-07-31T12:00:00.000Z") + const adapter = createPdppAuthorizationAdapter({ + activeManifestPath: fixture.activeManifestPath, + connectorId: "chatgpt-pdpp", + expectedConnector: { + key: "chatgpt", + id: "https://registry.pdpp.org/connectors/chatgpt", + }, + databasePath: join(fixture.root, "authorization.sqlite"), + now: () => now, + scopeForStream: stream => `chatgpt.${stream}`, + singleUseAccessExpiresInSeconds: 60, + }) + const app = new Hono() + registerProtectedRoutes({ + app, + devToken: desktopToken, + gatewayClient: { revokeGrant: async () => {} }, + ownerAddress: "0xowner", + port: 8080, + send: () => {}, + serverSigner: { signGrantRevocation: async () => "signature" }, + onLegacyGrantRevoked: grantId => adapter.revokeByLegacyGrantId(grantId), + }) + registerPdppAuthorizationRoutes({ + app, + devToken: desktopToken, + adapter, + enableLocalTimeline: false, + }) + + const writeExport = (name, timestamp, content) => + writeFileSync( + join(fixture.exportRoot, name), + JSON.stringify({ timestamp, content }) + ) + const exportContent = ({ provenance, recordsByStream, snapshot } = {}) => ({ + platform: "chatgpt", + version: fixture.manifest.version, + "pdpp.provenance": + provenance ?? + chatgptSnapshotProvenance({ + manifestDigest: fixture.manifestDigest, + }), + "pdpp.recordsByStream": recordsByStream ?? { + conversations: [ + chatgptConversation("conversation-a", "2026-02-01T00:00:00.000Z"), + chatgptConversation("conversation-b", "2026-03-01T00:00:00.000Z"), + chatgptConversation("resource-excluded", "2026-04-01T00:00:00.000Z"), + chatgptConversation("time-excluded", "2025-01-01T00:00:00.000Z"), + ], + }, + ...(snapshot ? { "pdpp.snapshot": snapshot } : {}), + }) + writeExport("initial.json", 1785456000000, exportContent()) + await mountPdppResourceServer(app, { + ...fixture, + connectorId: "chatgpt-pdpp", + expectedConnector: { + key: "chatgpt", + id: "https://registry.pdpp.org/connectors/chatgpt", + }, + connectionId: "chatgpt-account-a", + tokenIntrospector: { + introspect: token => adapter.resolveForResourceServer(token), + }, + }) + + const details = ({ source = "chatgpt", accessMode = "continuous" } = {}) => [ + { + type: PDPP_DATA_ACCESS_TYPE, + source: { kind: "connector", id: source }, + access_mode: accessMode, + purpose_code: "https://example.test/purpose/research", + streams: [ + { + name: "conversations", + fields: ["title"], + resources: ["conversation-a", "conversation-b", "time-excluded"], + time_range: { since: "2026-01-01T00:00:00.000Z" }, + }, + ], + }, + ] + const createConsent = async ({ sessionId, scopes, authorizationDetails }) => + app.request("http://personal.example/v1/pdpp/consent-requests", { + method: "POST", + headers: { + authorization: `Bearer ${desktopToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + session_id: sessionId, + scopes, + authorization_details: authorizationDetails, + }), + }) + const issueBearer = async ({ + sessionId, + legacyGrantId, + accessMode = "continuous", + }) => { + const consent = await createConsent({ + sessionId, + scopes: ["chatgpt.conversations"], + authorizationDetails: details({ accessMode }), + }) + assert.equal(consent.status, 201, await consent.clone().text()) + const request = await consent.json() + assert.deepEqual(request.authorization_details.streams[0].fields, [ + "id", + "title", + ]) + const approval = await app.request( + `http://personal.example/v1/pdpp/consent-requests/${request.request_id}/approve`, + { + method: "POST", + headers: { + authorization: `Bearer ${desktopToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + legacy_grant_id: legacyGrantId, + subject_id: "chatgpt-subject", + client_id: TEST_BUILDER.address, + }), + } + ) + assert.equal(approval.status, 201, await approval.clone().text()) + const authorization = await redemptionAuthorization({ + account: TEST_BUILDER, + origin: "http://personal.example", + sessionId, + }) + const credential = await app.request( + `http://personal.example/v1/pdpp/credentials/${sessionId}/redeem`, + { method: "POST", headers: { authorization } } + ) + assert.equal(credential.status, 200, await credential.clone().text()) + return `Bearer ${(await credential.json()).access_token}` + } + + try { + const timeline = await app.request( + "http://personal.example/v1/pdpp/local-timeline/consent-requests", + { + method: "POST", + headers: { authorization: `Bearer ${desktopToken}` }, + } + ) + assert.equal(timeline.status, 404) + + for (const [index, invalid] of [ + { + scopes: ["chatgpt.conversations"], + authorizationDetails: details({ source: "github" }), + }, + { scopes: [], authorizationDetails: details() }, + { + scopes: ["chatgpt.conversations", "chatgpt.messages"], + authorizationDetails: details(), + }, + ].entries()) { + const response = await createConsent({ + sessionId: `invalid-${index}`, + ...invalid, + }) + assert.equal(response.status, 400) + } + + const bearer = await issueBearer({ + sessionId: "chatgpt-session", + legacyGrantId: "chatgpt-legacy-grant", + }) + const headers = { authorization: bearer } + const streams = await app.request("http://personal.example/v1/streams", { + headers, + }) + assert.equal(streams.status, 200) + assert.deepEqual( + (await streams.json()).data.map(stream => stream.name), + ["conversations"] + ) + + const firstPage = await app.request( + "http://personal.example/v1/streams/conversations/records?limit=1", + { headers } + ) + assert.equal(firstPage.status, 200) + const first = await firstPage.json() + assert.deepEqual( + first.data.map(record => record.id), + ["conversation-a"] + ) + assert.deepEqual(first.data[0].data, { + id: "conversation-a", + title: "title-conversation-a", + }) + assert.equal(typeof first.next_cursor, "string") + const secondPage = await app.request( + `http://personal.example/v1/streams/conversations/records?limit=1&cursor=${encodeURIComponent(first.next_cursor)}`, + { headers } + ) + assert.equal(secondPage.status, 200) + assert.deepEqual( + (await secondPage.json()).data.map(record => record.id), + ["conversation-b"] + ) + const detail = await app.request( + "http://personal.example/v1/streams/conversations/records/conversation-a", + { headers } + ) + assert.equal(detail.status, 200) + assert.deepEqual((await detail.json()).data, { + id: "conversation-a", + title: "title-conversation-a", + }) + + writeExport( + "newer-malformed.json", + 1785542400000, + exportContent({ + recordsByStream: { + conversations: [{ stream: "conversations", key: "bad" }], + }, + }) + ) + writeExport("wrong-source.json", 1785628800000, { + ...exportContent(), + platform: "github", + }) + writeExport("wrong-version.json", 1785672000000, { + ...exportContent(), + version: "0.1.1", + }) + writeExport( + "wrong-provenance.json", + 1785715200000, + exportContent({ + provenance: chatgptSnapshotProvenance({ + manifestDigest: "sha256:wrong", + connectionId: "wrong-connection", + }), + }) + ) + const retained = await app.request( + "http://personal.example/v1/streams/conversations/records?limit=100", + { headers } + ) + assert.equal(retained.status, 200) + assert.deepEqual( + (await retained.json()).data.map(record => record.id), + ["conversation-a", "conversation-b"] + ) + + const revoke = await app.request( + "http://personal.example/v1/grants/chatgpt-legacy-grant", + { method: "DELETE", headers: { authorization: `Bearer ${desktopToken}` } } + ) + assert.equal(revoke.status, 204) + const revoked = await app.request("http://personal.example/v1/streams", { + headers, + }) + assert.equal(revoked.status, 403) + assert.equal((await revoked.json()).error.code, "grant_revoked") + + const expiringBearer = await issueBearer({ + sessionId: "chatgpt-expiring-session", + legacyGrantId: "chatgpt-expiring-grant", + accessMode: "single_use", + }) + now = new Date("2026-07-31T12:02:00.000Z") + const expired = await app.request("http://personal.example/v1/streams", { + headers: { authorization: expiringBearer }, + }) + assert.equal(expired.status, 403) + assert.equal((await expired.json()).error.code, "grant_expired") + + now = new Date("2026-07-31T12:00:00.000Z") + const refreshBearer = await issueBearer({ + sessionId: "chatgpt-refresh-session", + legacyGrantId: "chatgpt-refresh-grant", + }) + writeExport( + "authoritative-empty.json", + 1785801600000, + exportContent({ + recordsByStream: { conversations: [] }, + snapshot: { + collection_mode: "full_refresh", + reset_streams: ["conversations"], + completed_at: "2026-07-31T12:00:00.000Z", + }, + }) + ) + const empty = await app.request( + "http://personal.example/v1/streams/conversations/records", + { headers: { authorization: refreshBearer } } + ) + assert.equal(empty.status, 200) + assert.deepEqual((await empty.json()).data, []) + + const bindingConsent = await createConsent({ + sessionId: "chatgpt-manifest-mismatch", + scopes: ["chatgpt.conversations"], + authorizationDetails: details(), + }) + assert.equal(bindingConsent.status, 201) + const bindingRequest = await bindingConsent.json() + const changedManifest = { ...fixture.manifest, version: "0.1.1" } + const changedBytes = Buffer.from(JSON.stringify(changedManifest)) + writeFileSync( + join(fixture.installRoot, "profile/collection-profile.json"), + changedBytes + ) + const active = JSON.parse(readFileSync(fixture.activeManifestPath)) + active.connectors["chatgpt-pdpp"].version = "0.1.1" + active.connectors["chatgpt-pdpp"].manifestSha256 = hash(changedBytes) + writeFileSync(fixture.activeManifestPath, JSON.stringify(active)) + const mismatchedApproval = await app.request( + `http://personal.example/v1/pdpp/consent-requests/${bindingRequest.request_id}/approve`, + { + method: "POST", + headers: { + authorization: `Bearer ${desktopToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + legacy_grant_id: "chatgpt-mismatched-grant", + subject_id: "chatgpt-subject", + client_id: TEST_BUILDER.address, + }), + } + ) + assert.equal(mismatchedApproval.status, 400) + } finally { + adapter.close() + } +}) diff --git a/personal-server/pdpp/test/fixtures/chatgpt.collection-profile.js b/personal-server/pdpp/test/fixtures/chatgpt.collection-profile.js new file mode 100644 index 00000000..ea4d5def --- /dev/null +++ b/personal-server/pdpp/test/fixtures/chatgpt.collection-profile.js @@ -0,0 +1,61 @@ +import { gunzipSync } from "node:zlib" + +const compressedCanonicalChatgptManifest = ` +H4sICAP0bGoCA2NoYXRncHQtY2Fub25pY2FsLXBpbm5lZC5qc29uAO1bUW/cNhJ+968g9sUJINs5 +3KE4uC8XNME1QHsXxAmKQ2AQXIkr8UyJKknZ3hYB+nQ/4NBf2F9yMyQlUbLW3l3LPvuuD0lakZzh +DIfffBxyfz4gZFFrZVWqJL3k2ghVLU7J4tXxH45fLRJsTlVV8dQqTUWGTYW1tTk9OdE8F8bq9XGd +1fWx0vlJ19OcpAWzeW3HEi74GkUMWktWiRU3ljZa7Cd/et6ZMLVka1qxkmPTNzDkr+8/+kbdVFaU +nGr+YyM0L3llDXT6GdqgdSmqTFR5/wW+VdxeKX0RfUIxfjj6xeqGh4YvSTtoqdWV4XqrQQft3274 +wnDb1P2USpUxKaxzn7HMipQanmruXYBOBolghWCSpqy2jebx7C/AIhzawGzQIbRmxoA92aKbq2RL +LiNHESPy6khUJOOWCWn6nhk3qRa1DT4/40wSW3CyYSBZKQ3twhCjGp3yY/IROneLSUq2JjAvFFES +Vck1ySAUqpyoq4rro5Jnglmekd5CAqMMN7joRPNUQQCsey1BMjR+TUxa8KyRMBpW3EBnVJQ2WoOg +TgS/FCA45YRVGckUqZQlsCnK2jqhvV5z3DvBNMtSWNp57Yxd9h6IpooyYcE0yBLagNJ1lfZSVoLL +DOPscxcffaRg1IXgbdetGzm9ZLwEhw/72HXtJEw0jQJx0BaCa6KFNYgWZS25vWVqsPVSXiiZuehf +rFXzF37NcNQxDB725dUlemDxzbevP4IV9NPZ2w9/e/3928V51+tLcrt/boTztH+mu7Uumm6dx0sh +5o6mdYw98P712dkPf//wJvZA+K/zEU7AZmdLAdAgeIRgRQOwSkVluWZp2KifEWobwIfw5Twgh+Yr +zU1BayVFuo5RA/dWCdiY8YwCADlD0LAS8acP4ribU3kJSsA1qnKh/cevXr3q+paiEmVT3t2PXbt+ +sHUkr2CjRh3//NWfop6RkWCCCcg3tJVKccHlOpoyIAp8xB1seGXAe5cBXBFumj5AF0tlKQCZB5Rx +50LkRdSVpRe5Vk2VUcNWfBQWC1h5SGjgI7aCGVOHbhTcWYw7wuRAF9gdh24IQxNQEcchwqQOGkOe +QaCxgK+v2zUiYW0Jgq9HuyOHgUeIs1+PEVbzmglNCsAsCZoAAxEZNZEqBziHnCthLXJosQWYmRck +pFHiTWOIoS2YHi8Ooo27qJslRBe43FiA9jjIpHPK2AdoSYOLDYmXZ4b6gHbeGidKqzkrewztM3bA +BggbZAjOqZuzWAfdce8I7wOdGKbyDmD+AZmNTCtyqjAROi1RF4KWkythC2KFBYcnmDdgIU6aOoN/ +CLoWPjKdFuKSZyeYRgCIyEqyHL4jHTE1wCx5IVaE4zaotTD8ZdJmuKOlhjVB8fUx+R7Wn+HiMQ0L +BuFQM9wFxPsvLFdMXgykDIwwtwplY9lScuoiLMqCkF9LNnRKC6dq+U/YNbEfIKBqrgdI1W7ibPQl +kgMTxJCJGr8M4Rudt3H453Z8AhHRSNnCXtfvmiLBpFr5/QYuLJleH3mpG5X6laK4RPurBnYB2xS1 +4nofOWEbNfqgeEyNwtA29m7RuFTgOVb1Km+TF0J4FnFd/FOR7eCRzUvqNw2tfKa7t7zSbziaQkaw +FJJHq8DvyltUYErLgTjdrSMXP5Vqf/un+FVMeD7jxjy/gQxhj4TjnO/UNYKVBo56jtk6fhjFbd8J +EBJd4Q5hXdd4U0UwJHlLYSKU6ZjzkHrB7P0pw2w6k4kq9Qc+JsdJ58cGDhJDNYbjFhjDleTXkHkl +7el7QKHzET30CDrRb9rzDLIrdJYA5Tcgcog4nxc54jBEgMW/pf8faUczGILGloPuDtzNgibtYnkO +p/mQF8dmoZIpBo1MqqbLdT/5QSQNHXKTJ3+JYg1SLtoDOwEG5KEEwdOLWsFOo1dwNlZX0ZkMGZOn +nNGA7iTZ9o/pzQ3OEVy4mW502RjzP5KrmDskmOKtBd4LyR4Oou258hCYl+YuZTcpEl1ilTt5j/J9 +zWxxTN4zt2giQ/ZXg1FAuFEgnpNRCtL2yosCpbtRnRv2DVjOWwaTCF3gRB9PkYQpKmAs7tgeG35K +MAsn+A2MtwnBQ4dsaZHjQwmYrCSBzSeRLwnbegz8BVpxX5N3bwxwYNnqCpQHT/X8ulYame6LunPO +0DOaLxshs5d38SFW13DgoUijnxYbiv1J7yOo89AsubUAp4I8EGc2Twm4AYTdRiGBos3D8xjWnm51 +I4bgo9NKr5YGLffnILiBqJFNPssqPjrpXYlKmIKCXjMiAPua0CLG3lGI+EMd/uwtoseqe+2H7Sll +x6p3Jn/JTUS5Jx+cJHlPkw9qLn2wFKLeokgbO2pY3/Nn7M3FiHZjcJFX3SXJyO3DMGY6E1V3I1Aw +A6ko2kddtXAWbutALOlBMRkASzLepneT4FbQA9LghySi92KeIa96tkRZ6vQ68JqXgJZKi1sIKN5X +ZKTt1t1cQCQRtlSNdaQNa/tbE8NWxA3VA27o9a4g/fZKcf+VS1jsoHqtmsQXxryns1ARy3xJzLKy +Bob3BoDArSTx9VlovhSMZGK1Ir/98it5/f4dyRTY5phfKcAkVS6NhY1innG56y5m0uXC+cpcGWV2 +3oR/++TgZACHmW3KYfPPbM8MOYCze1VKnElbJcZhz2dSJ+l89fBJIgTHTqWSO8c8fIHDL+vAiJ2T +zArCnYoK95HS63lTS9oAhpY0r+0ttymuDwFwN+SFK1Kal11G8UmmsYXSAbOxBROBO3vvkHL8tUuv +a8OlSzQZSC1Oiz0laE9CooknxKyN5SURfVnEJIRXmAoyV3gAa1JgPv76staQd3QSbt9AFN6PvYTM +xWWqSt5WQJJgawKmGuHuateJfxdQYGHi04fvCBKpZ5yTDJiHD3jkHEe00cOdRz1/t7qH4TybSSEy +aIiMOUTHsTqHPBflex9J/7/5wvgFSNE+mrv3qgRhO26J227i/IX8LBdx+AYhVyNGsG/4sXyn6Hu8 +G6z/bV429PcAgpPpthgiN3QZw92o2wC6Rm1xwI+aunCLPt/JJh/FhnhCv5PW88mBntq3S/gUie1k +WI4J7sfAZQ9Nyz/jYeQFMtrDHwpmyZVqZOZ4Jz5C62oeVpGLSl31FY9DRwkPv4Vvm4eAlbWqskPi +I/vlXkx5w76LGDNaZ68UAa9CmuPXNujzt5Fn3GLBypDf/vVv8h4iGR+riZ/8oyb8Fgj3u0jPKTmM +LHVHO5CKFgOLXnH3rskbZ/B+c43XgLVsDLkqOPgSn/by9aHu7hHluuXlx+QMJiO5ReW//OpfEKsK +pHBLgOuGItaz5dbObbQZPyPfM7+2PqbOx3NIDMswC5N4KJK2F318WtRiquczKflEEZzciMC760C3 +D989zz6tnLlzBjQt2lH3cqFK+bxZ0BUlMrrlq9m417DA4+W48o4/a0Ayqy52S1hBxhYPa89cKQWf +tOjM/+JkMMilnUtOSpbxMJ1TN11fgnn3JvGTrzKu5Rp/fBIP943Bik8fvsPqjarWpYL8hI9xfTWH +VWuCD8Il/HF3FZDhnvc9w1YPSXZIFc7Zc5WJ/ovPfp9ifaOLyLme6s5YIIh2BcVd8YRfq06d4J/k +Wf/3twnxe9pkIsi2fYs7OfbBju8P/y5hP1ox14PYg7CwmJXAU3zihzn+d8T4E14z8fPTGz+lgaRq +hof98U9+tor6qDxz168Ju9etW4+YJE2bfruXTLmjvQ+b8MjHwp22sR7Q3ZqJisQ/aN7RLdEN3tYm +TlVHdrKwfRcyejTbmfl9+xYFydRtpYrtDe2eojyelVigmjDuLT52xpfe+SlxkZ9072SS+AYzmbI8 +8TTVPOUdsLer54zCOXcpAtnBl4P/AC84SGojQQAA +` + +/** Exact bytes from PDPP cc07e3a, packages/polyfill-connectors/manifests/chatgpt.json. */ +export const canonicalChatgptManifestBytes = gunzipSync( + Buffer.from(compressedCanonicalChatgptManifest, "base64") +) From 512a64e4107cb15cb814da63fca631dc20aada33 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 31 Jul 2026 14:59:59 -0500 Subject: [PATCH 03/14] fix(personal-server): close generic serving compatibility gaps Retain the legacy GitHub PDPP database paths, keep optional PDPP route mounting inside a fail-closed startup boundary, reject malformed declared consent-time values atomically, and normalize accepted connector URI sources to the canonical connector key before persistence. Signed-off-by: Tim Nunamaker Assisted-by: AI --- personal-server/index.js | 209 ++++++--- .../pdpp/github-authorization/policy.js | 6 +- .../pdpp/grant-scoped-records-repository.js | 1 - .../grant-scoped-records-repository.test.js | 75 ++++ personal-server/pdpp/resource-server.js | 6 +- personal-server/pdpp/resource-server.test.js | 425 +++++++++++++++++- personal-server/protected-routes.js | 5 +- 7 files changed, 663 insertions(+), 64 deletions(-) diff --git a/personal-server/index.js b/personal-server/index.js index 9a00ebf2..34f41db3 100644 --- a/personal-server/index.js +++ b/personal-server/index.js @@ -26,7 +26,10 @@ import { createHash, randomUUID } from 'node:crypto'; import { execSync, spawn } from 'node:child_process'; import { pathToFileURL } from 'node:url'; import { registerProtectedRoutes } from './protected-routes.js'; -import { mountPdppResourceServer } from './pdpp/resource-server.js'; +import { + createPdppResourceServer, + mountPdppResourceRoutes, +} from './pdpp/resource-server.js'; import { createPdppAuthorizationAdapter } from './pdpp/github-authorization/index.js'; import { registerPdppAuthorizationRoutes } from './pdpp/github-authorization/http-routes.js'; import { loadInstalledManifest } from './pdpp/installed-manifest.js'; @@ -78,9 +81,130 @@ function selectedPdppServingProfile() { } function pdppProfileStorageName(connectorId) { + if (connectorId === 'github-pdpp') return 'github'; return createHash('sha256').update(connectorId).digest('hex').slice(0, 16); } +export function pdppProfileDatabasePath(root, connectorId, kind) { + return join( + root, + `pdpp-${pdppProfileStorageName(connectorId)}-${kind}.sqlite` + ); +} + +export function pdppDefaultStorageRoots({ + configDir, + pdppStorageDir, + homeDir = homedir(), +} = {}) { + const serverRoot = + configDir || join(homeDir, '.data-connect', 'personal-server'); + return { + authorizationRoot: serverRoot, + recordsRoot: + pdppStorageDir || + configDir || + join(homeDir, '.dataconnect', 'personal-server'), + }; +} + +export function createPdppRevocationSink({ + storageRoot, + activeManifestPath, + selectedPdppProfile = selectedPdppServingProfile(), +}) { + return createPdppAuthorizationAdapter({ + databasePath: pdppProfileDatabasePath( + storageRoot, + selectedPdppProfile.connectorId, + 'authorization' + ), + activeManifestPath, + connectorId: selectedPdppProfile.connectorId, + expectedConnector: selectedPdppProfile.connector, + scopeForStream: selectedPdppProfile.scopeForStream, + enableLocalTimeline: selectedPdppProfile.enableLocalTimeline, + }); +} + +export async function registerOptionalPdppSurfaces({ + app, + devToken, + storageRoot, + recordsRoot = storageRoot, + activeManifestPath, + exportRoot, + connectionId = 'default', + selectedPdppProfile = selectedPdppServingProfile(), + singleUseAccessExpiresInSeconds, + externalOrigin, + send = () => {}, +}) { + let adapter; + try { + // Resolve this once, then require both independently mounted surfaces to + // re-verify this exact selected install. The two surfaces must never + // compose grants from one active-manifest path with records from another. + const selectedInstall = loadInstalledManifest({ + activeManifestPath, + connectorId: selectedPdppProfile.connectorId, + expectedConnector: selectedPdppProfile.connector, + }); + adapter = createPdppAuthorizationAdapter({ + databasePath: pdppProfileDatabasePath( + storageRoot, + selectedPdppProfile.connectorId, + 'authorization' + ), + activeManifestPath, + connectorId: selectedPdppProfile.connectorId, + expectedConnector: selectedPdppProfile.connector, + selectedInstall, + scopeForStream: selectedPdppProfile.scopeForStream, + enableLocalTimeline: selectedPdppProfile.enableLocalTimeline, + singleUseAccessExpiresInSeconds, + }); + const resourceServer = await createPdppResourceServer({ + activeManifestPath, + connectorId: selectedPdppProfile.connectorId, + expectedConnector: selectedPdppProfile.connector, + selectedInstall, + databasePath: pdppProfileDatabasePath( + recordsRoot, + selectedPdppProfile.connectorId, + 'records' + ), + exportRoot, + connectionId, + tokenIntrospector: { + introspect: token => adapter.resolveForResourceServer(token), + }, + }); + registerPdppAuthorizationRoutes({ + app, + devToken, + adapter, + enableLocalTimeline: selectedPdppProfile.enableLocalTimeline, + externalOrigin, + }); + mountPdppResourceRoutes(app, resourceServer); + send({ + type: 'log', + message: `[pdpp] mounted selected ${selectedPdppProfile.connector.key} resource routes`, + }); + return adapter; + } catch (error) { + adapter?.close(); + send({ + type: 'log', + message: `[pdpp] routes unavailable: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + return null; + } +} + function send(msg) { let json; try { @@ -452,51 +576,36 @@ async function main() { const context = await createServer(config, { rootPath: configDir }); const { app, devToken, cleanup, gatewayClient, serverSigner } = context; const selectedPdppProfile = selectedPdppServingProfile(); - const pdppStorageName = pdppProfileStorageName(selectedPdppProfile.connectorId); const pdppActiveManifestPath = process.env.DATACONNECT_ACTIVE_CONNECTORS_PATH; - // Resolve this once, then require both independently mounted surfaces to - // re-verify this exact selected install. The two surfaces must never - // compose grants from one active-manifest path with records from another. - const pdppSelectedInstall = loadInstalledManifest({ + const pdppStorageRoots = pdppDefaultStorageRoots({ + configDir, + pdppStorageDir: process.env.PDPP_STORAGE_DIR, + }); + const pdppRevocationSink = createPdppRevocationSink({ + storageRoot: pdppStorageRoots.authorizationRoot, activeManifestPath: pdppActiveManifestPath, - connectorId: selectedPdppProfile.connectorId, - expectedConnector: selectedPdppProfile.connector, + selectedPdppProfile, }); - const pdppAuthorization = createPdppAuthorizationAdapter({ - databasePath: join(configDir || join((await import('node:os')).homedir(), '.data-connect', 'personal-server'), `pdpp-${pdppStorageName}-authorization.sqlite`), + const pdppAuthorization = await registerOptionalPdppSurfaces({ + app, + devToken, + storageRoot: pdppStorageRoots.authorizationRoot, + recordsRoot: pdppStorageRoots.recordsRoot, activeManifestPath: pdppActiveManifestPath, - connectorId: selectedPdppProfile.connectorId, - expectedConnector: selectedPdppProfile.connector, - selectedInstall: pdppSelectedInstall, - scopeForStream: selectedPdppProfile.scopeForStream, - enableLocalTimeline: selectedPdppProfile.enableLocalTimeline, + exportRoot: process.env.DATACONNECT_EXPORT_ROOT, + connectionId: + process.env.PDPP_SERVING_CONNECTION_ID || + process.env.PDPP_GITHUB_CONNECTION_ID || + 'default', + selectedPdppProfile, singleUseAccessExpiresInSeconds, + externalOrigin: personalServerExternalOrigin( + context.serverAccount?.address, + tunnelServerAddr + ), + send, }); - // PDPP reads are additive: unavailable or invalid local connector state - // leaves the established Personal Server routes running without a fallback - // manifest or lossy data source. - try { - const pdppStorageRoot = process.env.PDPP_STORAGE_DIR - || configDir - || join(homedir(), '.dataconnect', 'personal-server'); - await mountPdppResourceServer(app, { - activeManifestPath: pdppActiveManifestPath, - connectorId: selectedPdppProfile.connectorId, - expectedConnector: selectedPdppProfile.connector, - selectedInstall: pdppSelectedInstall, - databasePath: join(pdppStorageRoot, `pdpp-${pdppStorageName}-records.sqlite`), - exportRoot: process.env.DATACONNECT_EXPORT_ROOT, - connectionId: process.env.PDPP_SERVING_CONNECTION_ID || process.env.PDPP_GITHUB_CONNECTION_ID || 'default', - tokenIntrospector: { - introspect: token => pdppAuthorization.resolveForResourceServer(token), - }, - }); - send({ type: 'log', message: `[pdpp] mounted selected ${selectedPdppProfile.connector.key} resource routes` }); - } catch (error) { - send({ type: 'log', message: `[pdpp] resource routes unavailable: ${error instanceof Error ? error.message : String(error)}` }); - } - // --- Request logging --- // Wrap app.fetch to log all incoming requests (including routes registered // by the library before our code runs, like /health). @@ -533,17 +642,8 @@ async function main() { port, send, serverSigner, - onLegacyGrantRevoked: grantId => pdppAuthorization.revokeByLegacyGrantId(grantId), - }); - registerPdppAuthorizationRoutes({ - app, - devToken, - adapter: pdppAuthorization, - enableLocalTimeline: selectedPdppProfile.enableLocalTimeline, - externalOrigin: personalServerExternalOrigin( - context.serverAccount?.address, - tunnelServerAddr - ), + onLegacyGrantRevoked: grantId => + pdppRevocationSink.revokeByLegacyGrantId(grantId), }); // Start HTTP server first so the desktop app can connect immediately. @@ -704,7 +804,10 @@ async function main() { function shutdown(signal) { send({ type: 'log', message: `Shutdown signal: ${signal}` }); - pdppAuthorization.close(); + pdppAuthorization?.close(); + if (pdppAuthorization !== pdppRevocationSink) { + pdppRevocationSink.close(); + } if (cleanup) cleanup().catch(() => {}); server.close(() => { process.exit(0); @@ -721,4 +824,6 @@ async function main() { } } -main(); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(); +} diff --git a/personal-server/pdpp/github-authorization/policy.js b/personal-server/pdpp/github-authorization/policy.js index 89fa260a..dae832a0 100644 --- a/personal-server/pdpp/github-authorization/policy.js +++ b/personal-server/pdpp/github-authorization/policy.js @@ -191,6 +191,10 @@ export function validateAuthorizationDetails({ ) { throw invalid("source must identify the selected connector") } + const canonicalSourceId = string( + manifest.connector_key, + "manifest.connector_key is required" + ) if (!["single_use", "continuous"].includes(detail.access_mode)) { throw invalid('access_mode must be "single_use" or "continuous"') } @@ -279,7 +283,7 @@ export function validateAuthorizationDetails({ } return { type: PDPP_DATA_ACCESS_TYPE, - source: { kind: "connector", id: detail.source.id }, + source: { kind: "connector", id: canonicalSourceId }, access_mode: detail.access_mode, purpose_code: detail.purpose_code, purpose_description: detail.purpose_description, diff --git a/personal-server/pdpp/grant-scoped-records-repository.js b/personal-server/pdpp/grant-scoped-records-repository.js index f9253314..7cc0a294 100644 --- a/personal-server/pdpp/grant-scoped-records-repository.js +++ b/personal-server/pdpp/grant-scoped-records-repository.js @@ -924,7 +924,6 @@ function validateLiveRecord( } if ( metadata.consentTimeField && - data[metadata.consentTimeField] != null && !validTemporalValue( data[metadata.consentTimeField], metadata.fieldFormats?.[metadata.consentTimeField] ?? "date-time" diff --git a/personal-server/pdpp/grant-scoped-records-repository.test.js b/personal-server/pdpp/grant-scoped-records-repository.test.js index 78f5072e..366f656e 100644 --- a/personal-server/pdpp/grant-scoped-records-repository.test.js +++ b/personal-server/pdpp/grant-scoped-records-repository.test.js @@ -378,6 +378,81 @@ test("a malformed authoritative full refresh does not partially alter current re } }) +test("rejects missing, null, and invalid declared consent time atomically", () => { + const harness = withRepository() + const connectionId = "github-account-a" + try { + harness.repository.importSnapshot({ + connectionId, + recordsByStream: { + repositories: [ + { + stream: "repositories", + key: "existing", + data: record("repositories", "existing"), + emitted_at: TIMES.emitted, + }, + ], + }, + }) + + for (const [name, extra] of [ + ["missing", { created_at: undefined }], + ["null", { created_at: null }], + ["invalid", { created_at: "not-a-date" }], + ]) { + assert.throws( + () => + harness.repository.importSnapshot({ + connectionId, + recordsByStream: { + repositories: [ + { + stream: "repositories", + key: "new-but-rolled-back", + data: record("repositories", "new-but-rolled-back"), + emitted_at: TIMES.emitted, + }, + { + stream: "repositories", + key: name, + data: record("repositories", name, extra), + emitted_at: TIMES.emitted, + }, + ], + }, + snapshot: { + collection_mode: "full_refresh", + reset_streams: ["repositories"], + completed_at: "2026-07-30T20:00:00.000Z", + }, + }), + error => + error instanceof RecordsRepositoryError && + error.code === "invalid_record" + ) + assert.notEqual( + harness.repository.getCurrent({ + connectionId, + stream: "repositories", + key: "existing", + }), + null + ) + assert.equal( + harness.repository.getCurrent({ + connectionId, + stream: "repositories", + key: "new-but-rolled-back", + }), + null + ) + } + } finally { + harness.dispose() + } +}) + test("upserts and deletes are atomic, idempotent, and survive reopen", () => { const harness = withRepository() try { diff --git a/personal-server/pdpp/resource-server.js b/personal-server/pdpp/resource-server.js index 898b0b04..66faccec 100644 --- a/personal-server/pdpp/resource-server.js +++ b/personal-server/pdpp/resource-server.js @@ -69,11 +69,15 @@ export async function createPdppResourceServer({ /** Add PDPP reads without taking ownership of legacy Personal Server routes. */ export async function mountPdppResourceServer(app, options) { const resourceServer = await createPdppResourceServer(options) + mountPdppResourceRoutes(app, resourceServer) + return resourceServer +} + +export function mountPdppResourceRoutes(app, resourceServer) { const serve = context => resourceServer.fetch(context.req.raw) app.get("/v1/streams", serve) app.get("/v1/streams/:stream/records", serve) app.get("/v1/streams/:stream/records/:recordId", serve) - return resourceServer } function createRepository(databasePath, streamMetadata) { diff --git a/personal-server/pdpp/resource-server.test.js b/personal-server/pdpp/resource-server.test.js index cfa3c119..860aceec 100644 --- a/personal-server/pdpp/resource-server.test.js +++ b/personal-server/pdpp/resource-server.test.js @@ -9,6 +9,12 @@ import { privateKeyToAccount } from "viem/accounts" import { Hono } from "hono" +import { + createPdppRevocationSink, + pdppDefaultStorageRoots, + pdppProfileDatabasePath, + registerOptionalPdppSurfaces, +} from "../index.js" import { registerProtectedRoutes } from "../protected-routes.js" import { createGithubStreamMetadata, @@ -199,6 +205,8 @@ function createInstalledGithubFixture({ allStreams = false } = {}) { }) ) return { + root, + installRoot, activeManifestPath, exportRoot, databasePath: join(root, "records.sqlite"), @@ -298,6 +306,22 @@ function chatgptConversation(id, createTime, updateTime = createTime) { } } +function rewriteInstalledManifest(fixture, mutate) { + const manifest = JSON.parse(JSON.stringify(fixture.manifest)) + mutate(manifest) + const manifestBytes = Buffer.from(JSON.stringify(manifest)) + writeFileSync( + join(fixture.installRoot, "profile/collection-profile.json"), + manifestBytes + ) + const active = JSON.parse(readFileSync(fixture.activeManifestPath)) + active.connectors["github-pdpp"].version = manifest.version + active.connectors["github-pdpp"].manifestSha256 = hash(manifestBytes) + writeFileSync(fixture.activeManifestPath, JSON.stringify(active)) + fixture.manifest = manifest + fixture.manifestDigest = hash(manifestBytes) +} + function githubSnapshotProvenance({ manifestDigest, connectionId = "default", @@ -379,7 +403,11 @@ function record(id, createdAt) { } } -function activeToken({ manifestDigest, grant: grantOverrides, ...overrides } = {}) { +function activeToken({ + manifestDigest, + grant: grantOverrides, + ...overrides +} = {}) { return { active: true, pdpp_token_kind: "client", @@ -772,7 +800,8 @@ test("a successful authoritative full refresh removes records absent from the ne await mountPdppResourceServer(app, { ...fixture, tokenIntrospector: { - introspect: async () => activeToken({ manifestDigest: fixture.manifestDigest }), + introspect: async () => + activeToken({ manifestDigest: fixture.manifestDigest }), }, }) const headers = { authorization: "Bearer opaque" } @@ -874,7 +903,8 @@ test("serves only the requested connection's verified installed snapshot", async await mountPdppResourceServer(defaultApp, { ...fixture, tokenIntrospector: { - introspect: async () => activeToken({ manifestDigest: fixture.manifestDigest }), + introspect: async () => + activeToken({ manifestDigest: fixture.manifestDigest }), }, }) const defaultRead = await defaultApp.request( @@ -1374,6 +1404,369 @@ test("rejects a local PDPP bearer after owner revoke intent even when Gateway re adapter.close() }) +test("default GitHub serving retains legacy authorization and record database paths on upgrade", async () => { + const fixture = createInstalledGithubFixture() + const desktopToken = "desktop-token" + assert.equal( + pdppProfileDatabasePath(fixture.root, "github-pdpp", "authorization"), + join(fixture.root, "pdpp-github-authorization.sqlite") + ) + assert.equal( + pdppProfileDatabasePath(fixture.root, "github-pdpp", "records"), + join(fixture.root, "pdpp-github-records.sqlite") + ) + + const seededAdapter = createPdppAuthorizationAdapter({ + activeManifestPath: fixture.activeManifestPath, + connectorId: "github-pdpp", + expectedConnector: { + key: "github", + id: "https://registry.pdpp.org/connectors/github", + }, + databasePath: pdppProfileDatabasePath( + fixture.root, + "github-pdpp", + "authorization" + ), + scopeForStream: stream => + ({ + user: "github.profile", + repositories: "github.repositories", + starred: "github.starred", + })[stream], + }) + const consent = seededAdapter.createConsentRequest({ + sessionId: "legacy-upgrade-session", + scopes: ["github.repositories"], + authorizationDetails: [ + { + type: PDPP_DATA_ACCESS_TYPE, + source: { kind: "connector", id: "github" }, + access_mode: "continuous", + purpose_code: "https://example.test/purpose/research", + streams: [{ name: "repositories", resources: ["allowed"] }], + }, + ], + }) + const issued = seededAdapter.issueApprovedGrant({ + requestId: consent.request_id, + legacyGrantId: "legacy-upgrade-grant", + subjectId: "subject-1", + clientId: TEST_BUILDER.address, + }) + seededAdapter.close() + + const app = new Hono() + app.get("/health", context => context.json({ ok: true })) + const adapter = await registerOptionalPdppSurfaces({ + app, + devToken: desktopToken, + storageRoot: fixture.root, + recordsRoot: fixture.root, + activeManifestPath: fixture.activeManifestPath, + exportRoot: fixture.exportRoot, + connectionId: "default", + selectedPdppProfile: { + connectorId: "github-pdpp", + connector: { + key: "github", + id: "https://registry.pdpp.org/connectors/github", + }, + scopeForStream: stream => + ({ + user: "github.profile", + repositories: "github.repositories", + starred: "github.starred", + })[stream], + enableLocalTimeline: true, + }, + }) + assert.notEqual(adapter, null) + registerProtectedRoutes({ + app, + devToken: desktopToken, + gatewayClient: { revokeGrant: async () => {} }, + ownerAddress: "0xowner", + port: 8080, + send: () => {}, + serverSigner: { signGrantRevocation: async () => "signature" }, + onLegacyGrantRevoked: grantId => adapter.revokeByLegacyGrantId(grantId), + }) + + const headers = { authorization: `Bearer ${issued.access_token}` } + const streams = await app.request("http://personal.example/v1/streams", { + headers, + }) + assert.equal(streams.status, 200, await streams.clone().text()) + assert.deepEqual( + (await streams.json()).data.map(stream => stream.name), + ["repositories"] + ) + + const revoke = await app.request( + "http://personal.example/v1/grants/legacy-upgrade-grant", + { + method: "DELETE", + headers: { authorization: `Bearer ${desktopToken}` }, + } + ) + assert.equal(revoke.status, 204) + const revoked = await app.request("http://personal.example/v1/streams", { + headers, + }) + assert.equal(revoked.status, 403) + assert.equal((await revoked.json()).error.code, "grant_revoked") + adapter.close() +}) + +test("invalid selected install leaves legacy routes alive and PDPP routes absent", async () => { + const fixture = createInstalledGithubFixture() + const app = new Hono() + app.get("/health", context => context.json({ ok: true })) + const logs = [] + const adapter = await registerOptionalPdppSurfaces({ + app, + devToken: "desktop-token", + storageRoot: fixture.root, + recordsRoot: fixture.root, + activeManifestPath: join(fixture.root, "missing-active.json"), + exportRoot: fixture.exportRoot, + selectedPdppProfile: { + connectorId: "github-pdpp", + connector: { + key: "github", + id: "https://registry.pdpp.org/connectors/github", + }, + scopeForStream: stream => + ({ + user: "github.profile", + repositories: "github.repositories", + starred: "github.starred", + })[stream], + enableLocalTimeline: true, + }, + send: message => logs.push(message), + }) + + assert.equal(adapter, null) + assert.equal( + (await app.request("http://personal.example/health")).status, + 200 + ) + assert.equal( + (await app.request("http://personal.example/v1/streams")).status, + 404 + ) + assert.equal( + ( + await app.request("http://personal.example/v1/pdpp/consent-requests", { + method: "POST", + }) + ).status, + 404 + ) + assert.match(logs.at(-1).message, /routes unavailable/) +}) + +test("unsupported selected install atomically leaves all PDPP routes absent", async () => { + const fixture = createInstalledGithubFixture() + rewriteInstalledManifest(fixture, manifest => { + manifest.streams[0].primary_key = ["id", "full_name"] + }) + const app = new Hono() + app.get("/health", context => context.json({ ok: true })) + const logs = [] + const adapter = await registerOptionalPdppSurfaces({ + app, + devToken: "desktop-token", + storageRoot: fixture.root, + recordsRoot: fixture.root, + activeManifestPath: fixture.activeManifestPath, + exportRoot: fixture.exportRoot, + selectedPdppProfile: { + connectorId: "github-pdpp", + connector: { + key: "github", + id: "https://registry.pdpp.org/connectors/github", + }, + scopeForStream: stream => + ({ + user: "github.profile", + repositories: "github.repositories", + starred: "github.starred", + })[stream], + enableLocalTimeline: true, + }, + send: message => logs.push(message), + }) + + assert.equal(adapter, null) + assert.equal( + (await app.request("http://personal.example/health")).status, + 200 + ) + for (const [url, options] of [ + ["http://personal.example/v1/streams"], + [ + "http://personal.example/v1/pdpp/consent-requests", + { method: "POST", headers: { authorization: "Bearer desktop-token" } }, + ], + [ + "http://personal.example/v1/pdpp/consent-requests/request/approve", + { method: "POST", headers: { authorization: "Bearer desktop-token" } }, + ], + [ + "http://personal.example/v1/pdpp/credentials/session/redeem", + { method: "POST" }, + ], + [ + "http://personal.example/v1/pdpp/local-timeline/consent-requests", + { method: "POST", headers: { authorization: "Bearer desktop-token" } }, + ], + ["http://personal.example/v1/pdpp/introspect", { method: "POST" }], + ]) { + assert.equal((await app.request(url, options)).status, 404, url) + } + assert.match(logs.at(-1).message, /unsupported record contract/) +}) + +test("revocation during optional PDPP outage persists through recovery and Gateway failure", async () => { + const fixture = createInstalledGithubFixture() + const desktopToken = "desktop-token" + const selectedPdppProfile = { + connectorId: "github-pdpp", + connector: { + key: "github", + id: "https://registry.pdpp.org/connectors/github", + }, + scopeForStream: stream => + ({ + user: "github.profile", + repositories: "github.repositories", + starred: "github.starred", + })[stream], + enableLocalTimeline: true, + } + const seededAdapter = createPdppAuthorizationAdapter({ + activeManifestPath: fixture.activeManifestPath, + connectorId: "github-pdpp", + expectedConnector: selectedPdppProfile.connector, + databasePath: pdppProfileDatabasePath( + fixture.root, + "github-pdpp", + "authorization" + ), + scopeForStream: selectedPdppProfile.scopeForStream, + }) + const consent = seededAdapter.createConsentRequest({ + sessionId: "outage-session", + scopes: ["github.repositories"], + authorizationDetails: [ + { + type: PDPP_DATA_ACCESS_TYPE, + source: { kind: "connector", id: "github" }, + access_mode: "continuous", + purpose_code: "https://example.test/purpose/research", + streams: [{ name: "repositories" }], + }, + ], + }) + const issued = seededAdapter.issueApprovedGrant({ + requestId: consent.request_id, + legacyGrantId: "outage-grant", + subjectId: "subject-1", + clientId: TEST_BUILDER.address, + }) + seededAdapter.close() + + const outageApp = new Hono() + outageApp.get("/health", context => context.json({ ok: true })) + const outageAdapter = await registerOptionalPdppSurfaces({ + app: outageApp, + devToken: desktopToken, + storageRoot: fixture.root, + recordsRoot: fixture.root, + activeManifestPath: join(fixture.root, "missing-active.json"), + exportRoot: fixture.exportRoot, + selectedPdppProfile, + }) + assert.equal(outageAdapter, null) + const revocationSink = createPdppRevocationSink({ + storageRoot: fixture.root, + activeManifestPath: join(fixture.root, "missing-active.json"), + selectedPdppProfile, + }) + registerProtectedRoutes({ + app: outageApp, + devToken: desktopToken, + gatewayClient: { + revokeGrant: async () => { + throw new Error("gateway unavailable") + }, + }, + ownerAddress: "0xowner", + port: 8080, + send: () => {}, + serverSigner: { signGrantRevocation: async () => "signature" }, + onLegacyGrantRevoked: grantId => + revocationSink.revokeByLegacyGrantId(grantId), + }) + const revoke = await outageApp.request( + "http://personal.example/v1/grants/outage-grant", + { + method: "DELETE", + headers: { authorization: `Bearer ${desktopToken}` }, + } + ) + assert.equal(revoke.status, 500) + revocationSink.close() + + const recoveredApp = new Hono() + const recoveredAdapter = await registerOptionalPdppSurfaces({ + app: recoveredApp, + devToken: desktopToken, + storageRoot: fixture.root, + recordsRoot: fixture.root, + activeManifestPath: fixture.activeManifestPath, + exportRoot: fixture.exportRoot, + selectedPdppProfile, + }) + assert.notEqual(recoveredAdapter, null) + const revoked = await recoveredApp.request( + "http://personal.example/v1/streams", + { headers: { authorization: `Bearer ${issued.access_token}` } } + ) + assert.equal(revoked.status, 403) + assert.equal((await revoked.json()).error.code, "grant_revoked") + recoveredAdapter.close() +}) + +test("default records root preserves the legacy standalone fallback path", () => { + const roots = pdppDefaultStorageRoots({ homeDir: "/tmp/fake-home" }) + assert.equal( + roots.authorizationRoot, + "/tmp/fake-home/.data-connect/personal-server" + ) + assert.equal( + roots.recordsRoot, + "/tmp/fake-home/.dataconnect/personal-server" + ) + assert.equal( + pdppDefaultStorageRoots({ + homeDir: "/tmp/fake-home", + configDir: "/tmp/config", + }).recordsRoot, + "/tmp/config" + ) + assert.equal( + pdppDefaultStorageRoots({ + homeDir: "/tmp/fake-home", + pdppStorageDir: "/tmp/pdpp-storage", + }).recordsRoot, + "/tmp/pdpp-storage" + ) +}) + test("serves Timeline only after local consent and fails closed after its bound session is revoked", async () => { const fixture = createInstalledGithubFixture() const adapter = createGithubAuthorizationAdapter({ @@ -1497,7 +1890,8 @@ test("maps durable cursor errors to stable resource-server responses", async () ...fixture, recordsRepository: repository, tokenIntrospector: { - introspect: async () => activeToken({ manifestDigest: fixture.manifestDigest }), + introspect: async () => + activeToken({ manifestDigest: fixture.manifestDigest }), }, }) const headers = { authorization: "Bearer opaque" } @@ -1557,9 +1951,8 @@ test("fails closed when the active manifest diverges from the composed selection ) const divergentActive = JSON.parse(readFileSync(divergent.activeManifestPath)) divergentActive.connectors["github-pdpp"].version = "0.5.1" - divergentActive.connectors["github-pdpp"].manifestSha256 = hash( - divergentBytes - ) + divergentActive.connectors["github-pdpp"].manifestSha256 = + hash(divergentBytes) writeFileSync(divergent.activeManifestPath, JSON.stringify(divergentActive)) const adapter = createPdppAuthorizationAdapter({ @@ -1570,7 +1963,10 @@ test("fails closed when the active manifest diverges from the composed selection id: "https://registry.pdpp.org/connectors/github", }, selectedInstall, - databasePath: join(dirname(fixture.activeManifestPath), "authorization.sqlite"), + databasePath: join( + dirname(fixture.activeManifestPath), + "authorization.sqlite" + ), scopeForStream: stream => ({ repositories: "github.repositories" })[stream], }) const app = new Hono() @@ -1823,6 +2219,19 @@ test("serves the selected canonical ChatGPT profile through grant-scoped routes" assert.equal(response.status, 400) } + const uriSourceConsent = await createConsent({ + sessionId: "chatgpt-uri-source", + scopes: ["chatgpt.conversations"], + authorizationDetails: details({ + source: "https://registry.pdpp.org/connectors/chatgpt", + }), + }) + assert.equal(uriSourceConsent.status, 201) + assert.equal( + (await uriSourceConsent.json()).authorization_details.source.id, + "chatgpt" + ) + const bearer = await issueBearer({ sessionId: "chatgpt-session", legacyGrantId: "chatgpt-legacy-grant", diff --git a/personal-server/protected-routes.js b/personal-server/protected-routes.js index 71e94c07..7da46004 100644 --- a/personal-server/protected-routes.js +++ b/personal-server/protected-routes.js @@ -36,6 +36,9 @@ export function registerProtectedRoutes({ if (!gatewayClient) { return c.json({ error: "Gateway client not initialized" }, 500) } + if (typeof onLegacyGrantRevoked !== "function") { + return c.json({ error: "Local revocation store not initialized" }, 500) + } const grantId = c.req.param("grantId") @@ -45,7 +48,7 @@ export function registerProtectedRoutes({ grantId, }) - await onLegacyGrantRevoked?.(grantId) + await onLegacyGrantRevoked(grantId) await gatewayClient.revokeGrant({ grantId, From d9dbc07e2ff822d7bb45f6202ea2eb7b0f055650 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 31 Jul 2026 15:26:01 -0500 Subject: [PATCH 04/14] chore(integration): pin ChatGPT PDPP artifact Add the signed ChatGPT PDPP 0.1.0 artifact from the immutable connectors-48440fead534 release alongside GitHub, keep the personal-server dependency lock aligned, and apply the focused resource-server test formatting fix. Signed-off-by: Tim Nunamaker Assisted-by: AI --- connectors/connector-dependencies.json | 1 + connectors/lock.json | 28 ++++++++++++++++++++ personal-server/package-lock.json | 3 ++- personal-server/pdpp/resource-server.test.js | 5 +--- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/connectors/connector-dependencies.json b/connectors/connector-dependencies.json index 107926c1..e71d3f92 100644 --- a/connectors/connector-dependencies.json +++ b/connectors/connector-dependencies.json @@ -15,6 +15,7 @@ "wholefoods-playwright": "1.0.0", "youtube-playwright": "1.0.0", "claude-export-playwright": "2.0.1", + "chatgpt-pdpp": "0.1.0", "github-pdpp": "0.5.0" } } diff --git a/connectors/lock.json b/connectors/lock.json index d4a117c4..7b18667f 100644 --- a/connectors/lock.json +++ b/connectors/lock.json @@ -25,9 +25,37 @@ "wholefoods-playwright": "1.0.0", "youtube-playwright": "1.0.0", "claude-export-playwright": "2.0.1", + "chatgpt-pdpp": "0.1.0", "github-pdpp": "0.5.0" }, "connectors": [ + { + "connectorId": "chatgpt-pdpp", + "company": "openai", + "version": "0.1.0", + "resolvedFrom": "0.1.0", + "artifactUrl": "https://github.com/PDP-Connect/data-connectors/releases/download/connectors-48440fead534/chatgpt-pdpp-0.1.0.tgz", + "artifactPath": "artifacts/chatgpt-pdpp/chatgpt-pdpp-0.1.0.tgz", + "artifactSha256": "sha256:0188385dbb782cf23a5331f0b05a4dddbab422cc21113b0ba8b5c1f72b42772f", + "artifactSignature": { + "type": "sigstoreBundle", + "bundlePath": "chatgpt-pdpp-0.1.0.tgz.sigstore.json", + "bundleUrl": "https://github.com/PDP-Connect/data-connectors/releases/download/connectors-48440fead534/chatgpt-pdpp-0.1.0.tgz.sigstore.json" + }, + "manifestSha256": "sha256:e2c66b9d584ff935b31790e3a698a5576168002988ccc8076ded327f189beff0", + "artifactKind": "pdpp-collection-profile", + "manifestPath": "profile/collection-profile.json", + "entrypointPath": "dist/collection-profile.mjs", + "entrypointSha256": "sha256:2c813fb3ae7fb6cc94af327a17ea17347a43b78aa4a401718a660b2edd6a25cc", + "provenancePath": "provenance.json", + "provenanceSha256": "sha256:95635fdd7865401deeecde499621d181397b301cc266a7fdf659c175ec04e4cf", + "sourceTag": "pdpp-cc07e3a896c2c0df7841da4ec6b2c660ffe1e792", + "sourceCommit": "cc07e3a896c2c0df7841da4ec6b2c660ffe1e792", + "releaseId": "github-48440fead5343d568773594b20cfc9306f1d6255", + "publishedAt": "2026-07-31T00:00:00Z", + "name": "ChatGPT (PDPP Collection Profile)", + "description": "Collects ChatGPT conversations, messages, memories, custom GPTs, custom instructions, and shared conversations through the PDPP Collection Profile protocol." + }, { "connectorId": "chatgpt-playwright", "company": "openai", diff --git a/personal-server/package-lock.json b/personal-server/package-lock.json index fdd29dac..c11b86ce 100644 --- a/personal-server/package-lock.json +++ b/personal-server/package-lock.json @@ -14,7 +14,8 @@ "@opendatalabs/personal-server-ts-mcp": "0.0.1-canary.92ee4d6", "@opendatalabs/personal-server-ts-server": "0.0.1-canary.92ee4d6", "better-sqlite3": "^12.6.2", - "hono": "^4.7.0" + "hono": "^4.7.0", + "viem": "^2.45.1" }, "devDependencies": { "@yao-pkg/pkg": "^5.12.0", diff --git a/personal-server/pdpp/resource-server.test.js b/personal-server/pdpp/resource-server.test.js index 860aceec..db2ec0fc 100644 --- a/personal-server/pdpp/resource-server.test.js +++ b/personal-server/pdpp/resource-server.test.js @@ -1747,10 +1747,7 @@ test("default records root preserves the legacy standalone fallback path", () => roots.authorizationRoot, "/tmp/fake-home/.data-connect/personal-server" ) - assert.equal( - roots.recordsRoot, - "/tmp/fake-home/.dataconnect/personal-server" - ) + assert.equal(roots.recordsRoot, "/tmp/fake-home/.dataconnect/personal-server") assert.equal( pdppDefaultStorageRoots({ homeDir: "/tmp/fake-home", From d42ddb24f9e78b4bf322900623279bcef9460e6f Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 31 Jul 2026 12:14:19 -0500 Subject: [PATCH 05/14] docs: clarify local-first user setup Signed-off-by: Tim Nunamaker Assisted-by: AI --- README.md | 261 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 144 insertions(+), 117 deletions(-) diff --git a/README.md b/README.md index 7f4994f8..d2f3f396 100644 --- a/README.md +++ b/README.md @@ -1,187 +1,214 @@ # DataConnect -> DataConnect is being made local-first and vendor-neutral. The app runs and exports your data to disk with no sign-in and no external account. Vana is one optional storage and sync provider you can enable in Settings, not a bundled default. Some code paths, URLs, and package names still assume Vana; we are actively generalizing these as the project moves toward a neutral, provider-agnostic architecture. +DataConnect is a local-first desktop application for collecting data from +online services and keeping the resulting data on your computer. -Desktop app for exporting your data from various platforms. -Screenshot 2026-02-24 at 8 10 08 PM +## Status -## Installation - -Download the latest release from [Releases](../../releases). +DataConnect is in active development. Use the +[Releases page](https://github.com/PDP-Connect/data-connect/releases) for +binary downloads when a release is available. Release artifacts are manual +installs. This build does not include an in-app auto-updater. -### macOS - -macOS artifacts may be unsigned. For an unsigned build, run this after installing: +Public code signing is not available yet. Some macOS artifacts do not have a +signature. For an unsigned macOS build, run this command after installation: ```bash xattr -cr /Applications/DataConnect.app ``` -Then open the app normally. - -### Windows +## What DataConnect does now -Run the `.exe` installer and follow the prompts. +DataConnect writes collected data to local disk by default. Local collection +does not require a Vana account or a Vana sign-in. -### Linux +Every installer bundles and runs a local Personal Server. It gives the desktop +app loopback-only access to imported and exported data. It also enables local +PDPP and MCP integrations. Remote registration and tunneling stay disabled +unless you configure service endpoints. -Use the `.deb` or `.AppImage` package. +## Installation -Every installer bundles and runs a local Personal Server. It gives the desktop app loopback-only access to imported and exported data, and it enables local PDPP and MCP integrations. Remote registration and tunneling stay disabled unless you explicitly configure service endpoints. +Download the latest release from [Releases](../../releases). -## Browser Requirements +### macOS -DataConnect uses browser automation to export your data. Release installers include a compatible Chromium build. At runtime: +Install the DMG. If macOS blocks an unsigned build, run the `xattr` command in +the Status section and open the app again. -1. **If you have Chrome/Edge installed:** The app uses your existing browser (recommended) -2. **If no system browser is found:** The app uses the Chromium bundled with the installer -3. **For local builds without bundled Chromium:** The app downloads Chromium automatically when needed +### Windows -The downloaded browser is stored in `~/.dataconnect/browsers/` and persists across app updates. +Run the NSIS installer and follow the prompts. -## Supported Platforms +### Linux -DataConnect currently supports exporting data from ChatGPT, GitHub, Instagram, LinkedIn, Spotify,YouTube, and Shop (Shopify) — covering your conversations, social profiles, listening history, watch history, order history, and more. +Use the `.deb` or `.AppImage` package. -For the latest available connectors, visit the [Data Connectors repository](https://github.com/vana-com/data-connectors). +## Build from source -## Development +Use this path if a binary is not available for your platform. -### Prerequisites +### Requirements -- Node.js 20+ -- Rust (latest stable) -- For Playwright connectors: `cd playwright-runner && npm install` +- Node.js 22 or 23. +- Rust stable and the Tauri build prerequisites for your operating system. +- Internet access during setup to install dependencies and resolve bundled + connectors when they are absent. -### Running locally +### Run the development application ```bash -# Install dependencies +git clone https://github.com/PDP-Connect/data-connect.git +cd data-connect npm install - -# Run in development mode npm run tauri:dev - -# Copy .env file -cp .env.example .env ``` -### Connector management +`npm run tauri:dev` builds required local helper programs when needed. It also +resolves the bundled connectors from the signed connector index when they are +not already present. -Connector scripts live upstream in [`vana-com/data-connectors`](https://github.com/vana-com/data-connectors). This repo consumes them as pinned dependencies. - -#### Updating connectors +### Create a package ```bash -npm run connectors:resolve -# Review the diff in connectors/, commit, push. +npm run tauri:build ``` -This fetches the latest matching versions from the signed data-connectors index, verifies checksums, and writes them to `connectors/`. Version constraints are declared in `connectors/connector-dependencies.json`. +The command builds the frontend, the Personal Server, and the Playwright +runner. It creates Linux and macOS packages from a local checkout. Release CI +also builds the Windows NSIS installer. -If you only want to verify the lockfile and bundled connector tree without mutating them, run `npm run connectors:check`. +Treat local build output as a development build. Do not treat it as a signed +public release. -#### How it works at runtime +## Browser Requirements -- `tauri dev` runs `ensure-connectors.js`, which restores missing bundled connectors from `~/.dataconnect/connectors/` first and then resolves them from the signed connector index if needed. -- The Rust backend loads connectors from active installs in `~/.dataconnect/connectors-store/` via `connectors-active.json`, then legacy `~/.dataconnect/connectors/`, then bundled `connectors/`. -- The `playwright-runner` executes connector scripts with a local Chromium browser. +DataConnect uses browser automation for legacy connectors and for browser-based +PDPP collection profiles. -### Agent config files +1. If you installed Chrome or Edge, DataConnect can use that browser. +2. If no supported system browser is found, DataConnect can use the Chromium + build bundled with the installer. +3. For local builds without bundled Chromium, DataConnect provisions Chromium at + runtime when needed. -This repo keeps both `AGENTS.md` and `CLAUDE.md`: Claude Code auto‑loads `CLAUDE.md` but not `AGENTS.md`, and Cursor does the opposite. Keep them aligned. +DataConnect stores the downloaded browser in `~/.dataconnect/browsers/`. It +persists across app updates. -### Agent skills sync +## Data sources -Skills are stored in `.agents/skills` (source of truth). Cursor reads them via per-skill symlinks in `.cursor/skills`. The sync script rebuilds those symlinks so any manually created skills show up in Cursor. +### GitHub PDPP Collection Profile -```bash -# One-off sync (default is .cursor/skills) -npm run skills:sync +GitHub is a PDPP-native collection path. DataConnect prefers this path when the +GitHub PDPP connector is available. It collects GitHub profile, repositories, +stars, issues, pull requests, and gists through the PDPP Collection Profile +protocol. -# Sync to Claude instead -npm run skills:sync -- --target=claude +To collect GitHub data: -# Auto-sync on changes -npm run skills:watch -``` +1. Select **GitHub** on the Home page. +2. Enter a GitHub personal access token when DataConnect requests one. +3. Give the token the GitHub permissions needed for the data that you want to + collect. +4. Select **Start import**. -### Building for production +DataConnect uses the token for that import only. The application does not save +the token. Use a token that you can revoke in GitHub if you no longer want to +use it. -```bash -# Install the locked dependencies, then build helpers and the native bundle -npm ci -npm run tauri:build -``` +### ChatGPT PDPP Collection Profile -Local installs and builds do not download Chromium as an npm lifecycle step. A local build bundles a compatible Playwright Chromium already present in its cache when available. Otherwise, the app uses a supported system browser or provisions Chromium at runtime. Release CI explicitly provisions Chromium and fails verification if the browser executable is absent from an installer. +ChatGPT is also available as a PDPP Collection Profile. It uses a browser +session to collect conversations, messages, memories, custom GPTs, custom +instructions, and shared conversations. -The built app will be in `src-tauri/target/release/bundle/`. +To collect ChatGPT data: -### Releasing +1. Select **ChatGPT** on the Home page. +2. Enter your ChatGPT sign-in details when DataConnect requests them. +3. Complete any browser sign-in or verification step. +4. Select **Start import**. -Releases are created via the release script, which bumps the version in `tauri.conf.json`, commits, pushes, and creates a GitHub release that triggers CI builds across macOS, Linux, and Windows. +### Legacy browser connectors -```bash -# Check current and suggested versions -npm run release:github -- --show-versions +DataConnect keeps its Playwright-based legacy connectors. They automate export +flows in a browser and write exports to local disk. The bundled legacy +connectors currently cover ChatGPT, Claude, GitHub, H-E-B, Instagram, +Instagram Ads, LinkedIn, Oura, Shop, Spotify, Whole Foods Market, and YouTube. -# Dry run to preview what will happen -npm run release:github -- --version X.Y.Z --dry-run +These connectors depend on the website and export process of each service. A +service can change that process without notice. A connector can therefore fail +or collect less data than expected. -# Create a new release -npm run release:github -- --version X.Y.Z -``` +The [PDP-Connect Data Connectors repository](https://github.com/PDP-Connect/data-connectors) +contains the connector artifacts and their signed index. -> **Do not** create releases manually via `gh release create` or the GitHub UI — the CI workflow will fail if `tauri.conf.json` version doesn't match the release tag. +## Use Timeline locally -Release artifacts are manual installs: macOS DMGs, Linux `.deb` and AppImage files, and Windows NSIS installers. This build does not include an in-app auto-updater. macOS artifacts are unsigned unless the optional `APPLE_BUILD_CERTIFICATE_BASE64`, `APPLE_BUILD_CERTIFICATE_PASSWORD`, and `APPLE_SIGNING_IDENTITY` GitHub secrets are configured. Signed builds are not notarized. +Timeline gives a chronological view of records from connected sources. The +current Timeline reads verified GitHub records from your local Personal Server. -The workflow can also be run manually without uploading artifacts (`workflow_dispatch` with `upload: false`) to validate all platform build legs. +1. Collect GitHub data with the GitHub PDPP Collection Profile. +2. Open **Apps > Timeline**. +3. Select **Review local Timeline access**. +4. Review the requested GitHub streams, access mode, retention, and expiry. +5. Select **Approve local Timeline access**. +6. View the Timeline. -## Architecture +The approval applies only to the local Personal Server on that device. +Timeline access expires after its issued lifetime, which is eight hours by +default. -``` -┌─────────────────────────────────────────────────────────┐ -│ DataConnect App │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────┐ │ -│ │ React UI │ │ Tauri/Rust │ │ Playwright │ │ -│ │ (Frontend) │◄─►│ (Backend) │◄─►│ Runner │ │ -│ └─────────────┘ └─────────────┘ └────────┬────────┘ │ -└────────────────────────────────────────────┬┼───────────┘ - ││ - ┌────────────────────────┘│ - │ │ - ┌─────▼─────┐ ┌───────▼───────┐ - │ System │ │ Downloaded │ - │ Chrome │ OR │ Chromium │ - └───────────┘ └───────────────┘ -``` +To stop Timeline access, select **Revoke Timeline access** in Timeline. The +app revokes the local approval and returns you to the consent screen. -### Browser Selection Priority +## Connector management -1. **System Chrome** - `/Applications/Google Chrome.app` (macOS) -2. **System Edge** - Available on Windows -3. **Downloaded Chromium** - `~/.dataconnect/browsers/` -4. **Auto-download** - If nothing found, downloads Chromium on first run +Connector versions use two pinned files: `connectors/connector-dependencies.json` +and `connectors/lock.json`. -## Connectors +```bash +npm run connectors:resolve +npm run connectors:check +``` + +`connectors:resolve` resolves the pinned versions from the signed connector +index and updates the bundled connector tree. `connectors:check` verifies the +lockfile and bundled connector tree without changing them. -Connectors are JavaScript files that automate data export. Located in the [Data Connectors repository](https://github.com/vana-com/data-connectors). +## Releasing -### Connector API (Playwright runtime) +Use the release script to create releases: -```javascript -// Available in connector scripts: -page.goto(url) // Navigate to URL -page.evaluate(script) // Run JS in page context -page.sleep(ms) // Wait for milliseconds -page.setData(key, value) // Send data back to app -page.promptUser(message, checkFn) // Wait for user action +```bash +npm run release:github -- --show-versions +npm run release:github -- --version X.Y.Z --dry-run +npm run release:github -- --version X.Y.Z ``` +Do not create releases manually with `gh release create` or the GitHub UI. The +CI workflow checks that `tauri.conf.json` matches the release tag. + +Release artifacts are manual installs: macOS DMGs, Linux `.deb` and AppImage +files, and Windows NSIS installers. macOS artifacts do not have signatures +unless CI has the optional Apple signing secrets. Signed builds do not have +notarization. + +The workflow can also run without uploading artifacts to validate all platform +build legs. + +## Not supported yet + +- Public code signing or notarization. +- A cloud account or remote service as a requirement for local collection. +- Timeline data from sources other than verified GitHub PDPP records. +- A guarantee that every legacy connector works with every account, browser, or + service change. +- Automatic updates. + ## License -This project is licensed under the Apache License 2.0. See the LICENSE file for details. -This software is provided as open-source utility software and is not a managed or hosted service. -See LEGAL.md for additional legal disclaimers and responsibility framing. +DataConnect uses the Apache License 2.0. See [LICENSE](LICENSE) for details. +This software is open-source utility software. It is not a managed or hosted +service. See [LEGAL.md](LEGAL.md) for legal disclaimers and responsibility +information. From 5f472afffd28d3949c816ae866d6c097b751380c Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 31 Jul 2026 17:18:44 -0500 Subject: [PATCH 06/14] fix(release): sign helper commits Assisted-by: AI Signed-off-by: Tim Nunamaker --- README.md | 2 ++ scripts/release-github.mjs | 20 ++++++++++++++++---- scripts/release-github.test.mjs | 22 ++++++++++++++++++++++ vite.config.ts | 1 + 4 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 scripts/release-github.test.mjs diff --git a/README.md b/README.md index d2f3f396..14386d2d 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,8 @@ npm run release:github -- --version X.Y.Z --dry-run npm run release:github -- --version X.Y.Z ``` +If AI helped with the release commit, run `npm run release:github -- --version X.Y.Z --assisted-by-ai`. + Do not create releases manually with `gh release create` or the GitHub UI. The CI workflow checks that `tauri.conf.json` matches the release tag. diff --git a/scripts/release-github.mjs b/scripts/release-github.mjs index b3986ee4..d6e13ea0 100644 --- a/scripts/release-github.mjs +++ b/scripts/release-github.mjs @@ -34,7 +34,7 @@ function info(message) { process.stdout.write(`[release] ${message}\n`); } -function parseArgs(argv) { +export function parseArgs(argv) { const args = { version: "", target: "main", @@ -45,6 +45,7 @@ function parseArgs(argv) { checkVersion: false, showVersions: false, suggestVersion: false, + assistedByAi: false, }; for (let i = 0; i < argv.length; i += 1) { @@ -89,6 +90,10 @@ function parseArgs(argv) { args.suggestVersion = true; continue; } + if (token === "--assisted-by-ai") { + args.assistedByAi = true; + continue; + } fail(`Unknown argument: ${token}`); } @@ -193,6 +198,11 @@ function updateTauriVersion(nextVersion) { writeFileSync(TAURI_CONF_PATH, `${JSON.stringify(parsed, null, 2)}\n`, "utf8"); } +export function releaseCommitCommand(tagName, args) { + const assistedByAiTrailer = args.assistedByAi ? ' -m "Assisted-by: AI"' : ""; + return `git commit -s -m "release: ${tagName}"${assistedByAiTrailer}`; +} + function assertVersionOrdering(nextVersion, currentTauriVersion, latestRemoteTagVersion) { if (compareSemver(nextVersion, currentTauriVersion) <= 0) { fail( @@ -265,7 +275,7 @@ function main() { info(`Would run: git pull --ff-only origin ${args.target}`); info(`Would update: src-tauri/tauri.conf.json version -> ${version}`); info(`Would run: git add src-tauri/tauri.conf.json`); - info(`Would run: git commit -m "release: ${tagName}"`); + info(`Would run: ${releaseCommitCommand(tagName, args)}`); if (!args.noPush) { info(`Would run: git push origin ${args.target}`); } @@ -282,7 +292,7 @@ function main() { updateTauriVersion(version); runInherit("git add src-tauri/tauri.conf.json"); - runInherit(`git commit -m "release: ${tagName}"`); + runInherit(releaseCommitCommand(tagName, args)); if (!args.noPush) { runInherit(`git push origin ${args.target}`); @@ -297,4 +307,6 @@ function main() { info(`Release created: ${tagName}`); } -main(); +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + main(); +} diff --git a/scripts/release-github.test.mjs b/scripts/release-github.test.mjs new file mode 100644 index 00000000..196d345f --- /dev/null +++ b/scripts/release-github.test.mjs @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest" + +import { parseArgs, releaseCommitCommand } from "./release-github.mjs" + +describe("release helper", () => { + it("DCO signs release commits by default", () => { + const args = parseArgs(["--version", "1.2.3"]) + + expect(releaseCommitCommand("v1.2.3", args)).toBe( + 'git commit -s -m "release: v1.2.3"' + ) + }) + + it("adds an AI assistance trailer only when requested", () => { + const args = parseArgs(["--version", "1.2.3", "--assisted-by-ai"]) + + expect(args.assistedByAi).toBe(true) + expect(releaseCommitCommand("v1.2.3", args)).toBe( + 'git commit -s -m "release: v1.2.3" -m "Assisted-by: AI"' + ) + }) +}) diff --git a/vite.config.ts b/vite.config.ts index b5da2f7b..c98b16b5 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -22,6 +22,7 @@ export default defineConfig({ "src-tauri/**/*.test.ts", "scripts/is-main-module.test.mjs", "scripts/create-macos-dmg.test.mjs", + "scripts/release-github.test.mjs", "scripts/release-workflow.test.ts", "scripts/resolve-connectors.test.mjs", "scripts/verify-release-ref.test.mjs", From 3606c2d972d374a0b1505997251dfc3d02aa6a22 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 31 Jul 2026 17:36:13 -0500 Subject: [PATCH 07/14] fix(release): publish nested installer artifacts Assisted-by: AI Signed-off-by: Tim Nunamaker --- .github/workflows/release.yml | 14 +++--- scripts/release-workflow.test.ts | 75 +++++++++++++++++++++++++++++--- 2 files changed, 78 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e4f4ff86..62e65af5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -246,11 +246,15 @@ jobs: fi } - require_artifact manual-install-macos-arm64 '*.dmg' - require_artifact manual-install-macos-x64 '*.dmg' - require_artifact manual-install-linux-x64 '*.deb' - require_artifact manual-install-linux-x64 '*.AppImage' - require_artifact manual-install-windows-x64 '*.exe' + require_artifact manual-install-macos-arm64 'dmg/*.dmg' + require_artifact manual-install-macos-x64 'dmg/*.dmg' + require_artifact manual-install-linux-x64 'deb/*.deb' + require_artifact manual-install-linux-x64 'appimage/*.AppImage' + require_artifact manual-install-windows-x64 'nsis/*.exe' mapfile -d '' artifacts < <(find release-artifacts -type f -print0) + if [ "${#artifacts[@]}" -ne 5 ]; then + echo "::error::Expected exactly five verified release artifacts, found ${#artifacts[@]}." + exit 1 + fi gh release upload "$RELEASE_TAG" "${artifacts[@]}" --clobber diff --git a/scripts/release-workflow.test.ts b/scripts/release-workflow.test.ts index a6e4c956..11a8637a 100644 --- a/scripts/release-workflow.test.ts +++ b/scripts/release-workflow.test.ts @@ -8,7 +8,7 @@ import { writeFileSync, } from "node:fs" import { tmpdir } from "node:os" -import { basename, join, resolve } from "node:path" +import { basename, dirname, join, resolve } from "node:path" import { describe, expect, it } from "vitest" const releaseWorkflowPath = resolve( @@ -28,6 +28,19 @@ function readWorkflowStep(workflow: string, name: string) { return workflow.slice(start, next === -1 ? workflow.length : next) } +function readWorkflowRunScript(workflow: string, name: string) { + const step = readWorkflowStep(workflow, name) + const marker = " run: |\n" + const start = step.indexOf(marker) + if (start === -1) + throw new Error(`Missing run script for workflow step: ${name}`) + return step + .slice(start + marker.length) + .split("\n") + .map(line => (line.startsWith(" ") ? line.slice(10) : line)) + .join("\n") +} + describe("release workflow", () => { it("builds manual-install artifacts on demand without an updater", () => { const workflow = readReleaseWorkflow() @@ -154,13 +167,63 @@ describe("release workflow", () => { "manual-install-${{ matrix.artifact_key }}" ) expect(downloadArtifacts).toContain("pattern: manual-install-*") - expect(publishArtifacts).toContain("manual-install-macos-arm64") - expect(publishArtifacts).toContain("manual-install-macos-x64") - expect(publishArtifacts).toContain("manual-install-linux-x64 '*.deb'") - expect(publishArtifacts).toContain("manual-install-linux-x64 '*.AppImage'") - expect(publishArtifacts).toContain("manual-install-windows-x64 '*.exe'") + expect(publishArtifacts).toContain("manual-install-macos-arm64 'dmg/*.dmg'") + expect(publishArtifacts).toContain("manual-install-macos-x64 'dmg/*.dmg'") + expect(publishArtifacts).toContain("manual-install-linux-x64 'deb/*.deb'") + expect(publishArtifacts).toContain( + "manual-install-linux-x64 'appimage/*.AppImage'" + ) + expect(publishArtifacts).toContain( + "manual-install-windows-x64 'nsis/*.exe'" + ) + expect(publishArtifacts).toContain('${#artifacts[@]}" -ne 5') expect(publishArtifacts).toContain( 'gh release upload "$RELEASE_TAG" "${artifacts[@]}" --clobber' ) }) + + it("publishes five files from upload-artifact's preserved subdirectories", () => { + const publishScript = readWorkflowRunScript( + readReleaseWorkflow(), + "Publish complete platform set" + ) + const root = mkdtempSync(join(tmpdir(), "data-connect-publish-layout-")) + const expectedAssets = [ + "manual-install-macos-arm64/dmg/DataConnect_0.7.54_arm64.dmg", + "manual-install-macos-x64/dmg/DataConnect_0.7.54_x86_64.dmg", + "manual-install-linux-x64/deb/DataConnect_0.7.54_amd64.deb", + "manual-install-linux-x64/appimage/DataConnect_0.7.54_amd64.AppImage", + "manual-install-windows-x64/nsis/DataConnect_0.7.54_x64-setup.exe", + ] + + try { + for (const asset of expectedAssets) { + const path = join(root, "release-artifacts", asset) + mkdirSync(dirname(path), { recursive: true }) + writeFileSync(path, "verified artifact") + } + + const output = execFileSync( + "bash", + ["-c", `gh() { printf 'GH_ARG=%s\\n' "$@"; }\n${publishScript}`], + { + cwd: root, + encoding: "utf8", + env: { ...process.env, RELEASE_TAG: "v0.7.54" }, + } + ) + const uploadArgs = output + .split("\n") + .filter(line => line.startsWith("GH_ARG=")) + .map(line => line.slice("GH_ARG=".length)) + + expect(uploadArgs.slice(0, 3)).toEqual(["release", "upload", "v0.7.54"]) + expect(uploadArgs.slice(3, -1).sort()).toEqual( + expectedAssets.map(asset => `release-artifacts/${asset}`).sort() + ) + expect(uploadArgs.at(-1)).toBe("--clobber") + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) }) From 08c7b62dd3349e0cf32199bd1b55c32507a043f7 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 31 Jul 2026 18:10:34 -0500 Subject: [PATCH 08/14] fix(pdpp): preserve packaged runtime resources Assisted-by: AI Signed-off-by: Tim Nunamaker --- .github/workflows/release.yml | 3 + scripts/build-prod.js | 14 +- scripts/release-workflow.test.ts | 2 + scripts/verify-bundled-personal-server.mjs | 6 + .../verify-bundled-personal-server.test.mjs | 19 +- src-tauri/src/commands/connector.rs | 53 ++- src-tauri/src/commands/pdpp_browser.rs | 290 +++++++++++++-- .../src/commands/pdpp_installed_connector.rs | 337 +++++++++++++----- src-tauri/tauri-config.test.ts | 23 ++ src-tauri/tauri.conf.json | 4 +- 10 files changed, 615 insertions(+), 136 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 62e65af5..683a5c47 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -127,6 +127,9 @@ jobs: npm run test:security npm run build + - name: Install PDPP runtime dependencies + run: node scripts/ensure-pdpp-runtime.js + - name: Optionally sign macOS helper binaries if: matrix.os_family == 'macos' && env.APPLE_SIGNING_AVAILABLE == 'true' env: diff --git a/scripts/build-prod.js b/scripts/build-prod.js index a4acbdc2..c4d5fd5d 100644 --- a/scripts/build-prod.js +++ b/scripts/build-prod.js @@ -89,7 +89,11 @@ async function build() { log('Installing personal-server dependencies...'); exec('npm install', { cwd: PERSONAL_SERVER }); - // 4. Build personal-server binary + // 4. Install PDPP runtime dependencies that Tauri packages as resources + log('Installing PDPP runtime dependencies...'); + exec('node scripts/ensure-pdpp-runtime.js'); + + // 5. Build personal-server binary log('Building personal-server binary...'); exec('npm run build', { cwd: PERSONAL_SERVER }); @@ -98,7 +102,7 @@ async function build() { throw new Error('personal-server build failed - dist directory not found'); } - // 5. Build frontend + // 6. Build frontend log('Building frontend...'); exec('npm run build'); @@ -111,14 +115,14 @@ async function build() { return; } - // 6. Build the .app bundle only (no DMG). + // 7. Build the .app bundle only (no DMG). // Tauri's resource glob flattens directory structures, so node_modules/ // can't be included via tauri.conf.json. We build .app first, inject // node_modules, then create the DMG ourselves. log('Building Tauri .app bundle...'); exec('npx tauri build --bundles app'); - // 7. Inject personal-server native addons into the .app bundle. + // 8. Inject personal-server native addons into the .app bundle. const appPath = findAppBundle(); if (!appPath) { throw new Error('.app bundle not found after build'); @@ -126,7 +130,7 @@ async function build() { log(`Injecting native addons into ${appPath}...`); copyNativeModulesIntoApp(appPath); - // 8. Create DMG from the complete .app. + // 9. Create DMG from the complete .app. if (PLAT === 'darwin') { const version = getVersion(); const archName = arch() === 'arm64' ? 'aarch64' : 'x64'; diff --git a/scripts/release-workflow.test.ts b/scripts/release-workflow.test.ts index 11a8637a..6c7ed858 100644 --- a/scripts/release-workflow.test.ts +++ b/scripts/release-workflow.test.ts @@ -66,6 +66,8 @@ describe("release workflow", () => { expect(workflow).toContain( "npm run build -- --require-browser --target ${{ matrix.pkg_target }}" ) + expect(workflow).toContain("Install PDPP runtime dependencies") + expect(workflow).toContain("node scripts/ensure-pdpp-runtime.js") expect(workflow).toContain("if: github.event_name == 'release'") expect(workflow).not.toMatch( /updater|latest\.json|TAURI_SIGNING_PRIVATE_KEY/i diff --git a/scripts/verify-bundled-personal-server.mjs b/scripts/verify-bundled-personal-server.mjs index 60ca4ecc..3b9061c3 100644 --- a/scripts/verify-bundled-personal-server.mjs +++ b/scripts/verify-bundled-personal-server.mjs @@ -11,6 +11,12 @@ const REQUIRED_PATH_FRAGMENTS = [ "personal-server/dist/personal-server", "personal-server/dist/node_modules/better-sqlite3/build/Release/better_sqlite3.node", "playwright-runner/dist/playwright-runner", + "pdpp-runtime/connector-loader.mjs", + "pdpp-runtime/connector-loader-bootstrap.mjs", + "pdpp-runtime/node_modules/p-queue/package.json", + "pdpp-runtime/node_modules/p-queue/dist/index.js", + "pdpp-runtime/node_modules/patchright/package.json", + "pdpp-runtime/node_modules/patchright/index.mjs", ] function fail(message) { diff --git a/scripts/verify-bundled-personal-server.test.mjs b/scripts/verify-bundled-personal-server.test.mjs index 2ee23486..c95b67b6 100644 --- a/scripts/verify-bundled-personal-server.test.mjs +++ b/scripts/verify-bundled-personal-server.test.mjs @@ -23,6 +23,12 @@ const runtimeEntries = [ "usr/lib/data-connect/resources/personal-server/dist/personal-server", "usr/lib/data-connect/resources/personal-server/dist/node_modules/better-sqlite3/build/Release/better_sqlite3.node", "usr/lib/data-connect/resources/playwright-runner/dist/playwright-runner", + "usr/lib/data-connect/resources/pdpp-runtime/connector-loader.mjs", + "usr/lib/data-connect/resources/pdpp-runtime/connector-loader-bootstrap.mjs", + "usr/lib/data-connect/resources/pdpp-runtime/node_modules/p-queue/package.json", + "usr/lib/data-connect/resources/pdpp-runtime/node_modules/p-queue/dist/index.js", + "usr/lib/data-connect/resources/pdpp-runtime/node_modules/patchright/package.json", + "usr/lib/data-connect/resources/pdpp-runtime/node_modules/patchright/index.mjs", ] const linuxBrowserEntry = @@ -44,6 +50,15 @@ describe("bundled personal-server verifier", () => { ).toThrow("better-sqlite3") }) + it("rejects an artifact missing a PDPP runtime dependency", () => { + expect(() => + assertPackagedRuntime( + runtimeEntries.filter(entry => !entry.includes("p-queue/package.json")), + "DataConnect.deb" + ) + ).toThrow("p-queue") + }) + it("requires a real packaged browser executable", () => { expect(() => assertPackagedBrowser( @@ -109,7 +124,7 @@ describe("bundled personal-server verifier", () => { ] ) - expect(entries).toHaveLength(4) + expect(entries).toHaveLength(10) expect(() => assertPackagedRuntime(entries, "DataConnect.exe") ).not.toThrow() @@ -147,7 +162,7 @@ describe("bundled personal-server verifier", () => { }) const entries = await entriesPromise - expect(entries).toHaveLength(4) + expect(entries).toHaveLength(10) expect(() => assertPackagedBrowser(entries, "DataConnect.exe", "windows") ).not.toThrow() diff --git a/src-tauri/src/commands/connector.rs b/src-tauri/src/commands/connector.rs index 347d3abc..9cb6e3b2 100644 --- a/src-tauri/src/commands/connector.rs +++ b/src-tauri/src/commands/connector.rs @@ -2352,7 +2352,10 @@ fn get_bundled_chromium_path(resource_dir: &Path) -> Option { get_bundled_chromium_path_for_platform(resource_dir, current_browser_platform()) } -fn get_bundled_chromium_path_for_platform(resource_dir: &Path, platform: &str) -> Option { +pub(crate) fn get_bundled_chromium_path_for_platform( + resource_dir: &Path, + platform: &str, +) -> Option { let browser_roots = [ resource_dir .join("playwright-runner") @@ -2418,8 +2421,20 @@ fn find_chromium_executable(browsers_dir: &Path, platform: &str) -> Option Option { - get_system_browser_path().or_else(get_downloaded_chromium_path) +pub(crate) fn resolve_automation_browser_path(resource_dir: Option<&Path>) -> Option { + resolve_automation_browser_path_from( + get_system_browser_path(), + get_downloaded_chromium_path(), + resource_dir.and_then(get_bundled_chromium_path), + ) +} + +fn resolve_automation_browser_path_from( + system: Option, + downloaded: Option, + bundled: Option, +) -> Option { + system.or(downloaded).or(bundled) } /// Get the Chromium download URL for the current platform @@ -2775,8 +2790,8 @@ pub async fn download_chromium_rust(app: AppHandle) -> Result { mod tests { use super::{ get_bundled_chromium_path_for_platform, get_downloaded_chromium_path_in_home, - manifest_looks_like_connector, resolve_browser_status, resolve_icon_path, - ConnectorMetadata, + manifest_looks_like_connector, resolve_automation_browser_path_from, + resolve_browser_status, resolve_icon_path, ConnectorMetadata, }; use std::path::{Path, PathBuf}; use tempfile::tempdir; @@ -2909,4 +2924,32 @@ mod tests { Some(executable) ); } + + #[test] + fn bundled_browser_lookup_supports_release_platform_layouts() { + for (platform, relative_path) in [ + ("linux", "playwright-runner/dist/browsers/chromium-1200/chrome-linux64/chrome"), + ("windows", "playwright-runner/dist/browsers/chromium-1200/chrome-win64/chrome.exe"), + ("macos", "playwright-runner/dist/browsers/chromium-1200/chrome-mac-x64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing"), + ("macos", "playwright-runner/dist/browsers/chromium-1200/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing"), + ] { + let temp = tempdir().expect("tempdir"); + let executable = create_file(temp.path(), relative_path); + + assert_eq!( + get_bundled_chromium_path_for_platform(temp.path(), platform), + Some(executable) + ); + } + } + + #[test] + fn pdpp_browser_discovery_uses_bundled_chromium_on_fresh_install() { + let bundled = PathBuf::from("bundled-browser"); + + assert_eq!( + resolve_automation_browser_path_from(None, None, Some(bundled.clone())), + Some(bundled) + ); + } } diff --git a/src-tauri/src/commands/pdpp_browser.rs b/src-tauri/src/commands/pdpp_browser.rs index e198e0cb..19e2113f 100644 --- a/src-tauri/src/commands/pdpp_browser.rs +++ b/src-tauri/src/commands/pdpp_browser.rs @@ -13,6 +13,8 @@ use fs2::FileExt; use serde::Serialize; use sha2::{Digest, Sha256}; use std::fs::{self, File, OpenOptions}; +#[cfg(unix)] +use std::io; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; use std::thread; @@ -50,13 +52,19 @@ pub struct PdppBrowserLease { lease_lock: Option, state: PdppBrowserInteractionState, child: Option, + termination_failed: bool, } impl PdppBrowserLease { - pub fn launch(connector_id: &str, owner_id: &str, run_id: &str) -> Result { + pub fn launch( + connector_id: &str, + owner_id: &str, + run_id: &str, + resource_dir: Option<&Path>, + ) -> Result { validate_owner_id(owner_id)?; - let browser = super::connector::resolve_automation_browser_path().ok_or( - "No system or downloaded Chromium browser is available for PDPP browser automation", + let browser = resolve_pdpp_browser_path(resource_dir).ok_or( + "No system, downloaded, or bundled Chromium browser is available for PDPP browser automation", )?; let (profile_dir, lease_lock) = acquire_profile_lease(&profile_root()?, connector_id, owner_id)?; @@ -91,9 +99,13 @@ impl PdppBrowserLease { let lease_id = lease_id(connector_id, owner_id, run_id); let (endpoint, child) = match wait_for_devtools_endpoint(&profile_dir, child) { Ok(ready) => ready, - Err(error) => { - release_profile_lease(Some(lease_lock)); - return Err(error); + Err(failure) => { + if failure.terminated { + release_profile_lease(Some(lease_lock)); + } else { + std::mem::forget(lease_lock); + } + return Err(failure.message); } }; Ok(Self { @@ -107,6 +119,7 @@ impl PdppBrowserLease { lease_lock: Some(lease_lock), state: PdppBrowserInteractionState::Collecting, child: Some(child), + termination_failed: false, }) } @@ -130,6 +143,7 @@ impl PdppBrowserLease { lease_lock: Some(lease_lock), state: PdppBrowserInteractionState::Launching, child: None, + termination_failed: false, }) } @@ -154,13 +168,34 @@ impl PdppBrowserLease { } pub fn close(&mut self) { + self.close_with_before_release_hook(|_| {}); + } + + fn close_with_before_release_hook(&mut self, before_release: F) + where + F: FnOnce(&Path), + { + self.close_with_terminator(terminate_browser, before_release); + } + + fn close_with_terminator(&mut self, mut terminate: T, before_release: F) + where + T: FnMut(&mut Child) -> bool, + F: FnOnce(&Path), + { if self.state == PdppBrowserInteractionState::Closed { return; } self.state = PdppBrowserInteractionState::Closing; if let Some(mut child) = self.child.take() { - terminate_browser(&mut child); + if !terminate(&mut child) { + self.child = Some(child); + self.termination_failed = true; + return; + } } + self.termination_failed = false; + before_release(&self.profile_dir); release_profile_lease(self.lease_lock.take()); self.state = PdppBrowserInteractionState::Closed; } @@ -181,7 +216,18 @@ impl PdppBrowserLease { impl Drop for PdppBrowserLease { fn drop(&mut self) { + if self.termination_failed { + if let Some(lock) = self.lease_lock.take() { + std::mem::forget(lock); + } + return; + } self.close(); + if self.child.is_some() { + if let Some(lock) = self.lease_lock.take() { + std::mem::forget(lock); + } + } } } @@ -206,6 +252,10 @@ fn profile_root() -> Result { .join("pdpp-browser-leases")) } +fn resolve_pdpp_browser_path(resource_dir: Option<&Path>) -> Option { + super::connector::resolve_automation_browser_path(resource_dir) +} + fn profile_dir(root: &Path, connector_id: &str, owner_id: &str) -> PathBuf { root.join("profiles") .join(stable_segment(connector_id)) @@ -289,14 +339,15 @@ fn profile_key(connector_id: &str, owner_id: &str) -> String { fn wait_for_devtools_endpoint( profile_dir: &Path, mut child: Child, -) -> Result<(String, Child), String> { +) -> Result<(String, Child), BrowserLaunchFailure> { let deadline = Instant::now() + BROWSER_START_TIMEOUT; let active_port = profile_dir.join("DevToolsActivePort"); while Instant::now() < deadline { if let Ok(Some(status)) = child.try_wait() { - return Err(format!( - "PDPP browser exited before becoming ready: {status}" - )); + return Err(BrowserLaunchFailure { + message: format!("PDPP browser exited before becoming ready: {status}"), + terminated: true, + }); } if let Ok(contents) = fs::read_to_string(&active_port) { if let Some(port) = contents @@ -309,41 +360,135 @@ fn wait_for_devtools_endpoint( } thread::sleep(Duration::from_millis(25)); } - terminate_browser(&mut child); - Err("Timed out waiting for PDPP browser CDP endpoint".into()) + Err(BrowserLaunchFailure { + message: "Timed out waiting for PDPP browser CDP endpoint".into(), + terminated: terminate_browser(&mut child), + }) } -fn terminate_browser(child: &mut Child) { +struct BrowserLaunchFailure { + message: String, + terminated: bool, +} + +fn terminate_browser(child: &mut Child) -> bool { #[cfg(unix)] { - crate::commands::server::kill_process_group(child.id(), libc::SIGTERM); + let process_group = child.id(); + signal_process_group(process_group, libc::SIGTERM); + let leader_exited = wait_for_child_exit(child, BROWSER_STOP_WAIT); + if leader_exited && wait_for_process_group_exit(process_group, BROWSER_STOP_WAIT) { + return true; + } + signal_process_group(process_group, libc::SIGKILL); + let _ = wait_for_child_exit(child, BROWSER_STOP_WAIT); + return wait_for_process_group_exit(process_group, BROWSER_STOP_WAIT); } + #[cfg(not(unix))] { - let _ = child.kill(); + #[cfg(windows)] + { + run_windows_taskkill(child.id(), BROWSER_STOP_WAIT); + } + #[cfg(not(windows))] + { + let _ = child.kill(); + } + if wait_for_child_exit(child, BROWSER_STOP_WAIT) { + return true; + } + #[cfg(windows)] + { + run_windows_taskkill(child.id(), BROWSER_STOP_WAIT); + } + #[cfg(not(windows))] + { + let _ = child.kill(); + } + wait_for_child_exit(child, BROWSER_STOP_WAIT) + } +} + +#[cfg(unix)] +fn signal_process_group(process_group: u32, signal: libc::c_int) { + unsafe { + libc::kill(-(process_group as i32), signal); + } +} + +#[cfg(unix)] +fn wait_for_process_group_exit(process_group: u32, timeout: Duration) -> bool { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if !process_group_exists(process_group) { + return true; + } + thread::sleep(Duration::from_millis(20)); + } + !process_group_exists(process_group) +} + +#[cfg(unix)] +fn process_group_exists(process_group: u32) -> bool { + let result = unsafe { libc::kill(-(process_group as i32), 0) }; + if result == 0 { + return true; } - let deadline = Instant::now() + BROWSER_STOP_WAIT; + io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH) +} + +fn wait_for_child_exit(child: &mut Child, timeout: Duration) -> bool { + let deadline = Instant::now() + timeout; while Instant::now() < deadline { if child.try_wait().ok().flatten().is_some() { - return; + return true; } thread::sleep(Duration::from_millis(20)); } - #[cfg(unix)] - crate::commands::server::kill_process_group(child.id(), libc::SIGKILL); - #[cfg(not(unix))] - { - let _ = child.kill(); + false +} + +#[cfg(windows)] +fn run_windows_taskkill(pid: u32, timeout: Duration) { + let (program, args) = windows_taskkill_command(pid); + let Ok(mut taskkill) = Command::new(program) + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + else { + return; + }; + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if taskkill.try_wait().ok().flatten().is_some() { + return; + } + thread::sleep(Duration::from_millis(20)); } - let _ = child.wait(); + let _ = taskkill.kill(); + let _ = wait_for_child_exit(&mut taskkill, Duration::from_millis(200)); +} + +fn windows_taskkill_command(pid: u32) -> (&'static str, Vec) { + ( + "taskkill", + vec!["/PID".into(), pid.to_string(), "/T".into(), "/F".into()], + ) } #[cfg(test)] mod tests { use super::*; + use fs2::FileExt; + #[cfg(unix)] + use std::os::unix::process::CommandExt; use std::process::Command; const LOCK_HOLDER_ROOT: &str = "PDPP_BROWSER_LOCK_HOLDER_ROOT"; + const LOCK_EXPECT_BLOCKED_ROOT: &str = "PDPP_BROWSER_LOCK_EXPECT_BLOCKED_ROOT"; struct ReapedTestChild(Child); @@ -369,6 +514,16 @@ mod tests { } } + #[test] + fn expects_profile_lease_to_be_blocked() { + let Ok(root) = std::env::var(LOCK_EXPECT_BLOCKED_ROOT) else { + return; + }; + let blocked = + PdppBrowserLease::fixture(Path::new(&root), "chatgpt-pdpp", "alice", "blocked-check"); + assert!(matches!(blocked, Err(error) if error.contains("already leased"))); + } + #[test] fn fixture_leases_persist_per_owner_and_clean_only_ephemeral_state() { let root = tempfile::tempdir().unwrap(); @@ -452,4 +607,91 @@ mod tests { assert!(validate_owner_id("../other-owner").is_err()); assert!(validate_owner_id("account-one").is_ok()); } + + #[test] + fn windows_tree_cleanup_targets_only_the_browser_process_tree() { + let (program, args) = windows_taskkill_command(4242); + + assert_eq!(program, "taskkill"); + assert_eq!(args, ["/PID", "4242", "/T", "/F"]); + } + + #[cfg(unix)] + #[test] + fn close_releases_profile_lock_after_bounded_browser_termination() { + let root = tempfile::tempdir().unwrap(); + let mut lease = + PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-1").unwrap(); + lease.state = PdppBrowserInteractionState::Collecting; + let mut command = Command::new("sh"); + command.args(["-c", "sleep 30"]); + command.process_group(0); + lease.child = Some(command.spawn().expect("spawn test browser process")); + let canonical_root = fs::canonicalize(root.path()).unwrap(); + let lock_path = lock_path(&canonical_root, "chatgpt-pdpp", "alice"); + let mut observed_still_locked = false; + + lease.close_with_before_release_hook(|_| { + let lock = OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .unwrap(); + let error = lock.try_lock_exclusive().unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::WouldBlock); + observed_still_locked = true; + }); + + assert!(observed_still_locked); + let lock = OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .unwrap(); + lock.try_lock_exclusive() + .expect("profile lock releases only after close completes"); + } + + #[cfg(unix)] + #[test] + fn drop_preserves_profile_lock_when_browser_tree_is_not_reaped() { + let root = tempfile::tempdir().unwrap(); + let mut lease = + PdppBrowserLease::fixture(root.path(), "chatgpt-pdpp", "alice", "run-1").unwrap(); + lease.state = PdppBrowserInteractionState::Collecting; + let mut command = Command::new("sh"); + command.args(["-c", "sleep 30"]); + command.process_group(0); + let child = command.spawn().expect("spawn test browser process"); + let process_group = child.id(); + lease.child = Some(child); + + lease.close_with_terminator(|_| false, |_| panic!("lock must not release")); + drop(lease); + + let blocked = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "commands::pdpp_browser::tests::expects_profile_lease_to_be_blocked", + "--nocapture", + ]) + .env(LOCK_EXPECT_BLOCKED_ROOT, root.path()) + .status() + .expect("spawn blocked lock oracle"); + assert!(blocked.success()); + crate::commands::server::kill_process_group(process_group, libc::SIGKILL); + } + + #[cfg(unix)] + #[test] + fn terminate_browser_waits_for_descendant_process_group_exit() { + let mut command = Command::new("sh"); + command.args(["-c", "trap '' TERM; (trap '' TERM; sleep 30) & exit 0"]); + command.process_group(0); + let mut child = command.spawn().expect("spawn test browser process tree"); + let process_group = child.id(); + + assert!(terminate_browser(&mut child)); + assert!(!process_group_exists(process_group)); + } } diff --git a/src-tauri/src/commands/pdpp_installed_connector.rs b/src-tauri/src/commands/pdpp_installed_connector.rs index aea8a770..0b3f0d84 100644 --- a/src-tauri/src/commands/pdpp_installed_connector.rs +++ b/src-tauri/src/commands/pdpp_installed_connector.rs @@ -12,9 +12,9 @@ use super::pdpp_collection_state::{ PdppCollectionConnectionState, DEFAULT_CONNECTION_ID, }; use super::pdpp_connector::{ - supervise_pdpp_connector, PdppConnectorCommand, PdppEvent, PdppRecord, PdppRunControl, - PdppRunOptions, PdppRunResult, PdppRunStatus, PdppScopeValidators, PdppStart, - PdppInteractionResponder, PdppInteractionResponseStatus, + supervise_pdpp_connector, PdppConnectorCommand, PdppEvent, PdppInteractionResponder, + PdppInteractionResponseStatus, PdppRecord, PdppRunControl, PdppRunOptions, PdppRunResult, + PdppRunStatus, PdppScopeValidators, PdppStart, }; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; @@ -26,7 +26,7 @@ use std::process::Command; use std::sync::{Arc, LazyLock, Mutex}; use std::thread; use std::time::{Duration, Instant}; -use tauri::{AppHandle, Emitter}; +use tauri::{AppHandle, Emitter, Manager}; const PDPP_ARTIFACT_KIND: &str = "pdpp-collection-profile"; const DEFAULT_TIMEOUT_SECONDS: u64 = 120; @@ -41,8 +41,9 @@ const CHATGPT_CONNECTOR_ID: &str = "https://registry.pdpp.org/connectors/chatgpt const CHATGPT_CONNECTOR_INSTALL_ID: &str = "chatgpt-pdpp"; static ACTIVE_PDPP_RUNS: LazyLock>> = LazyLock::new(|| Mutex::new(HashMap::new())); -static PENDING_PDPP_INTERACTIONS: LazyLock>> = - LazyLock::new(|| Mutex::new(HashMap::new())); +static PENDING_PDPP_INTERACTIONS: LazyLock< + Mutex>, +> = LazyLock::new(|| Mutex::new(HashMap::new())); /// A run remains registered until its child-supervision task has returned. /// Keeping connector identity alongside the cancellation control makes the @@ -260,7 +261,9 @@ fn start_installed_pdpp_connector_run_impl( control: PdppRunControl, ) -> Result { validate_request(&request)?; - let resolved = resolve_active_installed_pdpp_connector(&request.connector_id)?; + let resource_dir = app.path().resource_dir().ok(); + let runtime_root = resolve_pdpp_runtime_root(resource_dir.as_deref())?; + let resolved = resolve_active_installed_pdpp_connector(&request.connector_id, &runtime_root)?; let saved_state = load_connection_state(&resolved.connector_id, request.connection_id())?; let setup_complete = chatgpt_setup_complete(&resolved, request.connection_id())?; let secrets = resolve_child_secrets_for_connection(&request, &resolved, setup_complete)?; @@ -287,6 +290,8 @@ fn start_installed_pdpp_connector_run_impl( }, &secrets, start_state, + resource_dir, + runtime_root, )?; let export = if result.status == PdppRunStatus::Succeeded { let accumulated = export_accumulator @@ -361,6 +366,8 @@ fn run_resolved_installed_pdpp_connector( customization, secrets, None, + None, + resolve_pdpp_runtime_root(None)?, ) } @@ -370,6 +377,8 @@ fn run_resolved_installed_pdpp_connector_with_state( customization: CommandCustomization, secrets: &PdppChildSecrets, state: Option, + resource_dir: Option, + runtime_root: PathBuf, ) -> Result { validate_request(request)?; let browser_lease = if requires_browser(&resolved.manifest) { @@ -380,6 +389,7 @@ fn run_resolved_installed_pdpp_connector_with_state( &resolved.connector_id, owner_id, &request.run_id, + resource_dir.as_deref(), )?))) } else { None @@ -394,7 +404,13 @@ fn run_resolved_installed_pdpp_connector_with_state( .transpose()? .map(|lease| lease.binding().clone()); let start = build_start(request, &resolved.manifest, state)?; - let command = build_command(resolved, secrets, &customization, browser_binding.as_ref())?; + let command = build_command( + resolved, + secrets, + &customization, + browser_binding.as_ref(), + &runtime_root, + )?; let host_interaction = customization.on_interaction.clone(); let browser_for_interaction = browser_lease.clone(); let options = PdppRunOptions { @@ -411,15 +427,17 @@ fn run_resolved_installed_pdpp_connector_with_state( ), control: customization.control, on_interaction: host_interaction.map(|host_interaction| { - Arc::new(move |interaction: &super::pdpp_connector::PdppInteraction, responder| { - if let Some(lease) = &browser_for_interaction { - lease - .lock() - .map_err(|_| "PDPP browser lease is unavailable")? - .mark_waiting_for_user(); - } - host_interaction(interaction, responder) - }) as super::pdpp_connector::PdppInteractionSink + Arc::new( + move |interaction: &super::pdpp_connector::PdppInteraction, responder| { + if let Some(lease) = &browser_for_interaction { + lease + .lock() + .map_err(|_| "PDPP browser lease is unavailable")? + .mark_waiting_for_user(); + } + host_interaction(interaction, responder) + }, + ) as super::pdpp_connector::PdppInteractionSink }), on_interaction_closed: Some(interaction_closed_sink_for_run(request.run_id.clone())), ..Default::default() @@ -440,14 +458,18 @@ fn redact_browser_endpoint(result: &mut PdppRunResult, endpoint: &str) { if endpoint.is_empty() { return; } - result.stderr = result.stderr.replace(endpoint, "[REDACTED_BROWSER_ENDPOINT]"); + result.stderr = result + .stderr + .replace(endpoint, "[REDACTED_BROWSER_ENDPOINT]"); if let Some(failure) = &mut result.failure { *failure = failure.replace(endpoint, "[REDACTED_BROWSER_ENDPOINT]"); } for event in &mut result.events { match event { PdppEvent::Progress(progress) => { - progress.message = progress.message.replace(endpoint, "[REDACTED_BROWSER_ENDPOINT]"); + progress.message = progress + .message + .replace(endpoint, "[REDACTED_BROWSER_ENDPOINT]"); } PdppEvent::SkipResult(skip) => { if let Some(message) = &mut skip.message { @@ -557,7 +579,11 @@ pub fn submit_installed_pdpp_interaction_response( "success" => PdppInteractionResponseStatus::Success, "cancelled" => PdppInteractionResponseStatus::Cancelled, "timeout" => PdppInteractionResponseStatus::Timeout, - _ => return Err("PDPP INTERACTION_RESPONSE status must be success, cancelled, or timeout".into()), + _ => { + return Err( + "PDPP INTERACTION_RESPONSE status must be success, cancelled, or timeout".into(), + ) + } }; if data.as_ref().is_some_and(|value| !value.is_object()) { return Err("PDPP INTERACTION_RESPONSE data must be an object".into()); @@ -675,7 +701,9 @@ fn validate_interaction_request_id(request_id: &str) -> Result<(), String> { .chars() .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.')) { - return Err("PDPP interaction requestId must be 1-128 URL-safe identifier characters".into()); + return Err( + "PDPP interaction requestId must be 1-128 URL-safe identifier characters".into(), + ); } Ok(()) } @@ -700,14 +728,23 @@ impl StartInstalledPdppConnectorRequest { fn resolve_active_installed_pdpp_connector( connector_id: &str, + runtime_root: &Path, ) -> Result { let install = get_active_connector_install(connector_id) .ok_or_else(|| format!("PDPP connector {connector_id} is not installed"))?; - resolve_installed_pdpp_connector(&install) + resolve_installed_pdpp_connector_with_runtime(&install, runtime_root) } fn resolve_installed_pdpp_connector( install: &ActiveConnectorInstall, +) -> Result { + let runtime_root = resolve_pdpp_runtime_root(None)?; + resolve_installed_pdpp_connector_with_runtime(install, &runtime_root) +} + +fn resolve_installed_pdpp_connector_with_runtime( + install: &ActiveConnectorInstall, + runtime_root: &Path, ) -> Result { if install.artifact_kind.as_deref() != Some(PDPP_ARTIFACT_KIND) { return Err(format!( @@ -734,11 +771,7 @@ fn resolve_installed_pdpp_connector( let provenance_path = confined_existing_file(&root, provenance_relative, "PDPP provenance path")?; let manifest_sha256 = required_hash(install.manifest_sha256.as_deref(), "manifestSha256")?; - verify_file_hash( - &manifest_path, - Some(manifest_sha256), - "PDPP manifest", - )?; + verify_file_hash(&manifest_path, Some(manifest_sha256), "PDPP manifest")?; verify_file_hash( &entrypoint_path, Some(required_hash( @@ -761,7 +794,7 @@ fn resolve_installed_pdpp_connector( ) .map_err(|e| format!("Failed to parse PDPP connector manifest: {e}"))?; validate_manifest(&install.connector_id, &install.version, &manifest)?; - validate_chatgpt_runtime_requirements(&provenance_path, &manifest)?; + validate_chatgpt_runtime_requirements(&provenance_path, &manifest, runtime_root)?; Ok(ResolvedInstalledPdppConnector { connector_id: install.connector_id.clone(), manifest_sha256: manifest_sha256.to_owned(), @@ -856,10 +889,7 @@ fn validate_manifest( return Err("PDPP connector manifest must require the network binding".into()); } for (binding, requirement) in bindings.into_iter().flat_map(|bindings| bindings.iter()) { - if binding != "network" - && binding != "browser" - && requirement.required.unwrap_or(false) - { + if binding != "network" && binding != "browser" && requirement.required.unwrap_or(false) { return Err(format!( "PDPP connector requires unsupported binding {binding}" )); @@ -909,6 +939,7 @@ fn required_chatgpt_static_secret_fields( fn validate_chatgpt_runtime_requirements( provenance_path: &Path, manifest: &PdppConnectorManifest, + runtime_root: &Path, ) -> Result<(), String> { if manifest.connector_key.as_deref() != Some(CHATGPT_CONNECTOR_KEY) { return Ok(()); @@ -921,7 +952,6 @@ fn validate_chatgpt_runtime_requirements( .map_err(|error| format!("Failed to read ChatGPT PDPP provenance: {error}"))?, ) .map_err(|error| format!("Failed to parse ChatGPT PDPP provenance: {error}"))?; - let runtime_root = resolve_pdpp_runtime_root()?; for expected in ["p-queue", "patchright"] { let requirement = provenance .external_runtime_packages @@ -934,7 +964,9 @@ fn validate_chatgpt_runtime_requirements( &format!("node_modules/{expected}/package.json"), &format!("PDPP runtime dependency {expected}"), )?) - .map_err(|error| format!("Failed to read PDPP runtime dependency {expected}: {error}"))?, + .map_err(|error| { + format!("Failed to read PDPP runtime dependency {expected}: {error}") + })?, ) .map_err(|error| format!("Failed to parse PDPP runtime dependency {expected}: {error}"))?; if !satisfies_caret_requirement(&metadata.version, &requirement.version) { @@ -1117,10 +1149,9 @@ fn resolve_child_secrets_for_connection( for (field, environment_key) in expected { let value = provided[field].clone(); secrets.values.push(value.clone()); - secrets.environment.insert( - environment_key.to_owned(), - value, - ); + secrets + .environment + .insert(environment_key.to_owned(), value); } return Ok(secrets); } @@ -1135,7 +1166,9 @@ fn resolve_child_secrets_for_connection( let token = resolve_github_credential(request)?; let mut secrets = PdppChildSecrets::default(); secrets.values.push(token.clone()); - secrets.environment.insert("GITHUB_TOKEN".into(), token.clone()); + secrets + .environment + .insert("GITHUB_TOKEN".into(), token.clone()); secrets .environment .insert("GITHUB_PERSONAL_ACCESS_TOKEN".into(), token); @@ -1165,7 +1198,6 @@ fn should_mark_chatgpt_setup_complete( fn resolve_github_credential( request: &StartInstalledPdppConnectorRequest, ) -> Result { - if let Some(token) = request .github_token .as_deref() @@ -1195,6 +1227,7 @@ fn build_command( secrets: &PdppChildSecrets, customization: &CommandCustomization, browser_binding: Option<&PdppBrowserBinding>, + runtime_root: &Path, ) -> Result { let mut env = secrets.environment.clone(); if let Some(binding) = browser_binding { @@ -1222,7 +1255,6 @@ fn build_command( args.push("--import".into()); args.push(import.to_string_lossy().into_owned()); } - let runtime_root = resolve_pdpp_runtime_root()?; args.push("--import".into()); args.push( runtime_root @@ -1244,7 +1276,12 @@ fn build_command( }) } -fn resolve_pdpp_runtime_root() -> Result { +fn resolve_pdpp_runtime_root(resource_dir: Option<&Path>) -> Result { + if let Some(resource_dir) = resource_dir { + let root = resource_dir.join("pdpp-runtime"); + let root = canonical_existing_dir(&root, "PDPP runtime root")?; + return validate_pdpp_runtime_root(root); + } if let Some(configured) = std::env::var_os("DATACONNECT_PDPP_RUNTIME_ROOT") { let root = canonical_existing_dir(Path::new(&configured), "PDPP runtime root")?; return validate_pdpp_runtime_root(root); @@ -1316,11 +1353,7 @@ fn event_sink_for_run( Ok(()) } PdppEvent::Progress(progress) => { - emit_running_status( - &app, - &run_id, - &redact_secrets(&progress.message, &secrets), - ); + emit_running_status(&app, &run_id, &redact_secrets(&progress.message, &secrets)); Ok(()) } PdppEvent::SkipResult(skip) => { @@ -1353,34 +1386,38 @@ fn interaction_sink_for_run( run_id: String, secrets: Vec, ) -> super::pdpp_connector::PdppInteractionSink { - Arc::new(move |interaction: &super::pdpp_connector::PdppInteraction, responder| { - if responder.run_id() != run_id || interaction.request_id != responder.request_id() { - return Err("PDPP interaction responder is not bound to this run/request".into()); - } - validate_interaction_request_id(&interaction.request_id)?; - let key = (run_id.clone(), interaction.request_id.clone()); - PENDING_PDPP_INTERACTIONS - .lock() - .map_err(|_| "PDPP interaction registry is unavailable")? - .insert(key, responder); - let message = redact_secrets(&interaction.message, &secrets); - emit_running_status(&app, &run_id, "Waiting for owner interaction..."); - let _ = app.emit( - "pdpp-interaction", - json!({ - "runId": run_id, - "requestId": interaction.request_id, - "kind": interaction.kind, - "message": message, - "schema": interaction.schema, - "timeoutSeconds": interaction.timeout_seconds, - }), - ); - Ok(()) - }) + Arc::new( + move |interaction: &super::pdpp_connector::PdppInteraction, responder| { + if responder.run_id() != run_id || interaction.request_id != responder.request_id() { + return Err("PDPP interaction responder is not bound to this run/request".into()); + } + validate_interaction_request_id(&interaction.request_id)?; + let key = (run_id.clone(), interaction.request_id.clone()); + PENDING_PDPP_INTERACTIONS + .lock() + .map_err(|_| "PDPP interaction registry is unavailable")? + .insert(key, responder); + let message = redact_secrets(&interaction.message, &secrets); + emit_running_status(&app, &run_id, "Waiting for owner interaction..."); + let _ = app.emit( + "pdpp-interaction", + json!({ + "runId": run_id, + "requestId": interaction.request_id, + "kind": interaction.kind, + "message": message, + "schema": interaction.schema, + "timeoutSeconds": interaction.timeout_seconds, + }), + ); + Ok(()) + }, + ) } -fn interaction_closed_sink_for_run(run_id: String) -> super::pdpp_connector::PdppInteractionClosedSink { +fn interaction_closed_sink_for_run( + run_id: String, +) -> super::pdpp_connector::PdppInteractionClosedSink { Arc::new(move |closed_run_id, request_id| { if closed_run_id == run_id { invalidate_pending_interaction(closed_run_id, request_id); @@ -1964,7 +2001,9 @@ fn sanitize_retained_events( messages.push(SanitizedConnectorMessage { message_type: "SKIP_RESULT".into(), stream: skip.stream, - message: skip.message.map(|message| redact_secrets(&message, secrets)), + message: skip + .message + .map(|message| redact_secrets(&message, secrets)), }); } PdppEvent::Interaction(interaction) => { @@ -2117,7 +2156,12 @@ mod tests { .find(|path| path.extension().is_some_and(|extension| extension == "tgz")) .expect("PDPP_CHATGPT_ARTIFACT_ROOT must contain one chatgpt-pdpp tarball"); let status = Command::new("tar") - .args(["-xzf", artifact.to_str().unwrap(), "-C", root.to_str().unwrap()]) + .args([ + "-xzf", + artifact.to_str().unwrap(), + "-C", + root.to_str().unwrap(), + ]) .status() .unwrap(); assert!(status.success()); @@ -2170,7 +2214,7 @@ mod tests { } fn assert_actual_patchright_esm_import(root: &Path, node: &Path) { - let runtime_root = resolve_pdpp_runtime_root().unwrap(); + let runtime_root = resolve_pdpp_runtime_root(None).unwrap(); let bootstrap = runtime_root.join("connector-loader-bootstrap.mjs"); let probe = r#" const { default: PQueue } = await import("p-queue"); @@ -2182,7 +2226,13 @@ if (!chromium || typeof chromium.connectOverCDP !== "function") { process.stdout.write("packaged-externals-ok\n"); "#; let positive = Command::new(node) - .args(["--import", bootstrap.to_str().unwrap(), "--input-type=module", "-e", probe]) + .args([ + "--import", + bootstrap.to_str().unwrap(), + "--input-type=module", + "-e", + probe, + ]) .current_dir(root) .env_clear() .env("DATACONNECT_PDPP_RUNTIME_ROOT", &runtime_root) @@ -2202,7 +2252,9 @@ process.stdout.write("packaged-externals-ok\n"); .output() .unwrap(); assert!(!negative.status.success()); - assert!(String::from_utf8_lossy(&negative.stderr).contains("Cannot find package 'patchright'")); + assert!( + String::from_utf8_lossy(&negative.stderr).contains("Cannot find package 'patchright'") + ); } fn success_script() -> &'static str { @@ -2556,15 +2608,28 @@ readline.createInterface({ input: process.stdin }).on('line', line => { let actual: Value = serde_json::from_str(&actual).unwrap(); let fixture = chatgpt_browser_manifest(); assert_eq!(fixture["connector_id"], actual["connector_id"]); - assert_eq!(fixture["runtime_requirements"]["bindings"], actual["runtime_requirements"]["bindings"]); + assert_eq!( + fixture["runtime_requirements"]["bindings"], + actual["runtime_requirements"]["bindings"] + ); assert_eq!(fixture["setup"]["modality"], actual["setup"]["modality"]); assert_eq!( fixture["setup"]["credential_capture"]["fields"], actual["setup"]["credential_capture"]["fields"] ); assert_eq!( - fixture["streams"].as_array().unwrap().iter().map(|stream| &stream["name"]).collect::>(), - actual["streams"].as_array().unwrap().iter().map(|stream| &stream["name"]).collect::>(), + fixture["streams"] + .as_array() + .unwrap() + .iter() + .map(|stream| &stream["name"]) + .collect::>(), + actual["streams"] + .as_array() + .unwrap() + .iter() + .map(|stream| &stream["name"]) + .collect::>(), ); } @@ -2574,36 +2639,87 @@ readline.createInterface({ input: process.stdin }).on('line', line => { return; }; let artifact: Value = serde_json::from_str( - &fs::read_to_string( - artifact_root.join("connectors/chatgpt-pdpp/artifact.json"), - ) - .unwrap(), + &fs::read_to_string(artifact_root.join("connectors/chatgpt-pdpp/artifact.json")) + .unwrap(), ) .unwrap(); let provenance: Value = serde_json::from_str( - &fs::read_to_string( - artifact_root.join("connectors/chatgpt-pdpp/provenance.json"), - ) - .unwrap(), + &fs::read_to_string(artifact_root.join("connectors/chatgpt-pdpp/provenance.json")) + .unwrap(), ) .unwrap(); assert_eq!( artifact["build"]["external_packages"], provenance["external_runtime_packages"] ); - let runtime_root = resolve_pdpp_runtime_root().unwrap(); + let runtime_root = resolve_pdpp_runtime_root(None).unwrap(); + assert_packaged_pdpp_runtime_files(&runtime_root); + let generated_runtime_root = std::env::current_exe() + .unwrap() + .parent() + .unwrap() + .parent() + .unwrap() + .join("pdpp-runtime"); + assert_packaged_pdpp_runtime_files(&generated_runtime_root); for requirement in artifact["build"]["external_packages"].as_array().unwrap() { let name = requirement["name"].as_str().unwrap(); let required = requirement["version"].as_str().unwrap(); let metadata: NodePackageMetadata = serde_json::from_str( - &fs::read_to_string(runtime_root.join("node_modules").join(name).join("package.json")) - .unwrap(), + &fs::read_to_string( + runtime_root + .join("node_modules") + .join(name) + .join("package.json"), + ) + .unwrap(), ) .unwrap(); assert!(satisfies_caret_requirement(&metadata.version, required)); } } + fn assert_packaged_pdpp_runtime_files(runtime_root: &Path) { + for relative_path in [ + "connector-loader.mjs", + "connector-loader-bootstrap.mjs", + "node_modules/p-queue/package.json", + "node_modules/p-queue/dist/index.js", + "node_modules/patchright/package.json", + "node_modules/patchright/index.mjs", + ] { + assert!( + runtime_root.join(relative_path).is_file(), + "packaged PDPP runtime is missing {relative_path} under {}", + runtime_root.display() + ); + } + } + + fn create_minimal_pdpp_runtime_root(root: &Path) { + for relative_path in [ + "connector-loader.mjs", + "connector-loader-bootstrap.mjs", + "node_modules/p-queue/package.json", + "node_modules/patchright/package.json", + ] { + let path = root.join(relative_path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, "{}").unwrap(); + } + } + + #[test] + fn runtime_root_prefers_explicit_tauri_resource_directory() { + let temp = tempfile::tempdir().unwrap(); + let resource_runtime = temp.path().join("Resources").join("pdpp-runtime"); + create_minimal_pdpp_runtime_root(&resource_runtime); + + let resolved = resolve_pdpp_runtime_root(Some(&temp.path().join("Resources"))).unwrap(); + + assert_eq!(resolved, fs::canonicalize(resource_runtime).unwrap()); + } + #[test] fn actual_artifact_loader_uses_patchright_esm_chromium_export() { let Some(artifact_root) = chatgpt_artifact_root() else { @@ -2652,6 +2768,7 @@ readline.createInterface({ input: process.stdin }).on('line', line => { profile_key: "actual-artifact-profile".into(), }; let secrets = resolve_child_secrets_for_connection(&request, &resolved, true).unwrap(); + let runtime_root = resolve_pdpp_runtime_root(None).unwrap(); let command = build_command( &resolved, &secrets, @@ -2660,11 +2777,15 @@ readline.createInterface({ input: process.stdin }).on('line', line => { ..Default::default() }, Some(&binding), + &runtime_root, ) .unwrap(); assert!(!command.env.contains_key("CHATGPT_USERNAME")); assert!(!command.env.contains_key("CHATGPT_PASSWORD")); - assert_eq!(command.env["PDPP_CHATGPT_REMOTE_CDP_URL"], "http://127.0.0.1:9"); + assert_eq!( + command.env["PDPP_CHATGPT_REMOTE_CDP_URL"], + "http://127.0.0.1:9" + ); assert!(!command.env.contains_key("PDPP_BROWSER_SURFACE_REQUIRED")); assert!(command.clear_env); let loader_index = command @@ -3199,11 +3320,13 @@ readline.createInterface({ input: process.stdin }).on('line', line => { let resolved = resolve_installed_pdpp_connector(&install).unwrap(); let request = request_with_token("test-token"); let start = build_start(&request, &resolved.manifest, None).unwrap(); + let runtime_root = resolve_pdpp_runtime_root(None).unwrap(); let command = build_command( &resolved, &resolve_child_secrets(&request, &resolved).unwrap(), &CommandCustomization::default(), None, + &runtime_root, ) .unwrap(); assert!(command.clear_env); @@ -3373,6 +3496,7 @@ setInterval(() => {}, 1000); &PdppChildSecrets::default(), &CommandCustomization::default(), None, + &resolve_pdpp_runtime_root(None).unwrap(), ) .unwrap(), &build_start(&request, &resolved.manifest, None).unwrap(), @@ -3581,7 +3705,9 @@ readline.createInterface({ input: process.stdin }).on('line', (line) => { .map(PathBuf::from) .into_iter() .collect(); - let resolved = resolve_active_installed_pdpp_connector("github-pdpp").unwrap(); + let runtime_root = resolve_pdpp_runtime_root(None).unwrap(); + let resolved = + resolve_active_installed_pdpp_connector("github-pdpp", &runtime_root).unwrap(); let request = StartInstalledPdppConnectorRequest { run_id: "github-pdpp-cross-repo-e2e".into(), connector_id: "github-pdpp".into(), @@ -3638,7 +3764,12 @@ readline.createInterface({ input: process.stdin }).on('line', (line) => { result.records_truncated, ); - let response = to_response(request.run_id, resolved.connector_id, result, &[token.clone()]); + let response = to_response( + request.run_id, + resolved.connector_id, + result, + &[token.clone()], + ); let serialized = serde_json::to_string(&response).unwrap(); assert!(!serialized.contains(&token)); assert!(!serialized.contains("\"data\"")); @@ -3652,8 +3783,10 @@ readline.createInterface({ input: process.stdin }).on('line', (line) => { Ok("1"), "set PDPP_E2E_CHATGPT_CONFIRM=1 to allow the credentialed browser E2E" ); + let runtime_root = resolve_pdpp_runtime_root(None).unwrap(); let resolved = - resolve_active_installed_pdpp_connector(CHATGPT_CONNECTOR_INSTALL_ID).unwrap(); + resolve_active_installed_pdpp_connector(CHATGPT_CONNECTOR_INSTALL_ID, &runtime_root) + .unwrap(); let request = StartInstalledPdppConnectorRequest { run_id: "chatgpt-pdpp-browser-e2e".into(), connector_id: CHATGPT_CONNECTOR_INSTALL_ID.into(), @@ -3662,8 +3795,16 @@ readline.createInterface({ input: process.stdin }).on('line', (line) => { connection_id: Some("chatgpt-e2e-owner".into()), github_token: None, setup_secrets: Some(HashMap::from([ - ("username".into(), std::env::var("PDPP_E2E_CHATGPT_USERNAME").expect("PDPP_E2E_CHATGPT_USERNAME must be set")), - ("password".into(), std::env::var("PDPP_E2E_CHATGPT_PASSWORD").expect("PDPP_E2E_CHATGPT_PASSWORD must be set")), + ( + "username".into(), + std::env::var("PDPP_E2E_CHATGPT_USERNAME") + .expect("PDPP_E2E_CHATGPT_USERNAME must be set"), + ), + ( + "password".into(), + std::env::var("PDPP_E2E_CHATGPT_PASSWORD") + .expect("PDPP_E2E_CHATGPT_PASSWORD must be set"), + ), ])), timeout_seconds: Some(300), }; diff --git a/src-tauri/tauri-config.test.ts b/src-tauri/tauri-config.test.ts index 412747eb..89338278 100644 --- a/src-tauri/tauri-config.test.ts +++ b/src-tauri/tauri-config.test.ts @@ -17,6 +17,29 @@ describe("tauri manual-install config", () => { ).toBeUndefined() }) + it("preserves the packaged PDPP runtime directory tree", () => { + const filePath = resolve(process.cwd(), "src-tauri/tauri.conf.json") + const document = JSON.parse(readFileSync(filePath, "utf-8")) as { + bundle?: { resources?: Record } + } + + expect(document.bundle?.resources?.["../pdpp-runtime/"]).toBe( + "pdpp-runtime/" + ) + expect(document.bundle?.resources?.["../pdpp-runtime/**/*"]).toBeUndefined() + }) + + it("stages PDPP runtime dependencies before production Tauri packaging", () => { + const filePath = resolve(process.cwd(), "src-tauri/tauri.conf.json") + const document = JSON.parse(readFileSync(filePath, "utf-8")) as { + build?: { beforeBuildCommand?: string } + } + + expect(document.build?.beforeBuildCommand).toContain( + "node scripts/ensure-pdpp-runtime.js" + ) + }) + it("disables updater artifacts and ships no updater endpoint", () => { const filePath = resolve(process.cwd(), "src-tauri/tauri.conf.json") const document = JSON.parse(readFileSync(filePath, "utf-8")) as { diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 0cb48689..240fe0d6 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -7,7 +7,7 @@ "frontendDist": "../dist", "devUrl": "http://localhost:5173", "beforeDevCommand": "node scripts/ensure-connectors.js && npm run dev", - "beforeBuildCommand": "node scripts/ensure-connectors.js && npm run build" + "beforeBuildCommand": "node scripts/ensure-connectors.js && node scripts/ensure-pdpp-runtime.js && npm run build" }, "app": { "withGlobalTauri": true, @@ -62,7 +62,7 @@ "../playwright-runner/dist/": "playwright-runner/dist/", "../personal-server/dist/personal-server*": "personal-server/dist/", "../personal-server/dist/node_modules/": "personal-server/dist/node_modules/", - "../pdpp-runtime/**/*": "pdpp-runtime/" + "../pdpp-runtime/": "pdpp-runtime/" }, "linux": { "deb": { From 61f6a2c8aeb9148c55233173a60bc114d5378f52 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 31 Jul 2026 18:07:58 -0500 Subject: [PATCH 09/14] fix(release): bundle Node for PDPP connectors Package the setup-node Node 22 executable as a signed Tauri sidecar so fresh installs do not depend on PATH. Verify native targets, license notices, bundle layout, and PATH-less connector startup. Assisted-by: AI Signed-off-by: Tim Nunamaker --- .github/workflows/release.yml | 15 +- .gitignore | 2 + scripts/build-prod.js | 5 + scripts/release-workflow.test.ts | 8 + scripts/stage-pdpp-node.mjs | 135 ++++++++++++++ scripts/stage-pdpp-node.test.mjs | 176 ++++++++++++++++++ scripts/verify-bundled-personal-server.mjs | 22 +++ .../verify-bundled-personal-server.test.mjs | 33 +++- src-tauri/build.rs | 49 ++++- .../src/commands/pdpp_installed_connector.rs | 96 +++++++++- src-tauri/tauri-config.test.ts | 15 ++ src-tauri/tauri.conf.json | 4 +- vite.config.ts | 1 + 13 files changed, 553 insertions(+), 8 deletions(-) create mode 100644 scripts/stage-pdpp-node.mjs create mode 100644 scripts/stage-pdpp-node.test.mjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 683a5c47..958e6967 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,6 +32,8 @@ jobs: target: aarch64-apple-darwin artifact_key: macos-arm64 expected_arch: arm64 + node_platform: darwin + node_arch: arm64 pkg_target: node22-macos-arm64 binary_name: playwright-runner ps_binary_name: personal-server @@ -40,6 +42,8 @@ jobs: target: x86_64-apple-darwin artifact_key: macos-x64 expected_arch: x86_64 + node_platform: darwin + node_arch: x64 pkg_target: node22-macos-x64 binary_name: playwright-runner ps_binary_name: personal-server @@ -47,6 +51,8 @@ jobs: os_family: linux target: x86_64-unknown-linux-gnu artifact_key: linux-x64 + node_platform: linux + node_arch: x64 pkg_target: node22-linux-x64 binary_name: playwright-runner ps_binary_name: personal-server @@ -54,6 +60,8 @@ jobs: os_family: windows target: x86_64-pc-windows-msvc artifact_key: windows-x64 + node_platform: win32 + node_arch: x64 pkg_target: node22-win-x64 binary_name: playwright-runner.exe ps_binary_name: personal-server.exe @@ -79,7 +87,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: - node-version: "22" + node-version: "22.23.1" cache: npm - name: Setup Rust @@ -88,6 +96,10 @@ jobs: toolchain: stable targets: ${{ matrix.target }} + - name: Stage Node.js runtime sidecar + shell: bash + run: node scripts/stage-pdpp-node.mjs --target "${{ matrix.target }}" --expected-platform "${{ matrix.node_platform }}" --expected-arch "${{ matrix.node_arch }}" + - name: Install Linux bundle dependencies if: matrix.platform == 'ubuntu-22.04' run: | @@ -136,6 +148,7 @@ jobs: APPLE_SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }} run: | set -euo pipefail + codesign --force --options runtime --timestamp --sign "$APPLE_SIGNING_IDENTITY" src-tauri/binaries/pdpp-node-${{ matrix.target }} codesign --force --options runtime --timestamp --entitlements playwright-runner/entitlements.plist --sign "$APPLE_SIGNING_IDENTITY" playwright-runner/dist/${{ matrix.binary_name }} codesign --force --options runtime --timestamp --entitlements personal-server/entitlements.plist --sign "$APPLE_SIGNING_IDENTITY" personal-server/dist/${{ matrix.ps_binary_name }} find personal-server/dist/node_modules -name '*.node' -type f -exec codesign --force --options runtime --timestamp --sign "$APPLE_SIGNING_IDENTITY" {} \; diff --git a/.gitignore b/.gitignore index c2a26cef..4abd4050 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ pnpm-debug.log* lerna-debug.log* node_modules +src-tauri/binaries/pdpp-node-* +src-tauri/binaries/pdpp-node-LICENSE dist dist-ssr *.local diff --git a/scripts/build-prod.js b/scripts/build-prod.js index c4d5fd5d..9069e12d 100644 --- a/scripts/build-prod.js +++ b/scripts/build-prod.js @@ -20,6 +20,7 @@ import { existsSync, cpSync, readdirSync, mkdirSync, readFileSync } from 'fs'; import { join, dirname } from 'path'; import { fileURLToPath } from 'url'; import { platform, arch } from 'os'; +import { nativeTauriTarget, stagePdppNode } from './stage-pdpp-node.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const ROOT = join(__dirname, '..'); @@ -72,6 +73,10 @@ function findAppBundle() { async function build() { log('Building DataConnect for production...'); + // Keep local production builds on the same fail-closed Node 22 sidecar + // contract as the release workflow. + stagePdppNode({ ...nativeTauriTarget(PLAT, arch()), projectRoot: ROOT }); + // 1. Install playwright-runner dependencies log('Installing playwright-runner dependencies...'); exec('npm install', { cwd: PLAYWRIGHT_RUNNER }); diff --git a/scripts/release-workflow.test.ts b/scripts/release-workflow.test.ts index 6c7ed858..19d189ac 100644 --- a/scripts/release-workflow.test.ts +++ b/scripts/release-workflow.test.ts @@ -63,6 +63,11 @@ describe("release workflow", () => { "verification_args=(--platform macos --expected-arch" ) expect(workflow).toContain("npm ci") + expect(workflow).toContain('node-version: "22.23.1"') + expect(workflow).toContain("Stage Node.js runtime sidecar") + expect(workflow).toContain( + 'node scripts/stage-pdpp-node.mjs --target "${{ matrix.target }}"' + ) expect(workflow).toContain( "npm run build -- --require-browser --target ${{ matrix.pkg_target }}" ) @@ -98,6 +103,9 @@ describe("release workflow", () => { "env.APPLE_SIGNING_AVAILABLE == 'true' && secrets.APPLE_SIGNING_IDENTITY || ''" ) expect(workflow).toContain("verification_args+=(--verify-code-signature)") + expect(workflow).toContain( + 'codesign --force --options runtime --timestamp --sign "$APPLE_SIGNING_IDENTITY" src-tauri/binaries/pdpp-node-${{ matrix.target }}' + ) expect(workflow).toContain("codesign --force --deep --options runtime") expect(workflow).toContain( "node scripts/create-macos-dmg.mjs --volume-name DataConnect" diff --git a/scripts/stage-pdpp-node.mjs b/scripts/stage-pdpp-node.mjs new file mode 100644 index 00000000..a7a4c336 --- /dev/null +++ b/scripts/stage-pdpp-node.mjs @@ -0,0 +1,135 @@ +#!/usr/bin/env node + +import { copyFileSync, existsSync, mkdirSync, rmSync, statSync } from "node:fs" +import { basename, dirname, join, resolve } from "node:path" +import { isMainModule } from "./is-main-module.js" + +const SUPPORTED_TARGETS = new Map([ + ["aarch64-apple-darwin", { platform: "darwin", arch: "arm64" }], + ["x86_64-apple-darwin", { platform: "darwin", arch: "x64" }], + ["x86_64-unknown-linux-gnu", { platform: "linux", arch: "x64" }], + ["x86_64-pc-windows-msvc", { platform: "win32", arch: "x64" }], +]) +export const RELEASE_NODE_VERSION = "22.23.1" + +export function nativeTauriTarget(platform, arch) { + const match = [...SUPPORTED_TARGETS].find( + ([, native]) => native.platform === platform && native.arch === arch + ) + if (!match) fail(`Unsupported native Node.js platform: ${platform}/${arch}`) + return { + target: match[0], + expectedPlatform: platform, + expectedArch: arch, + } +} + +function fail(message) { + throw new Error(message) +} + +export function sidecarFilename(target) { + const expected = SUPPORTED_TARGETS.get(target) + if (!expected) fail(`Unsupported Node.js sidecar target: ${target}`) + return `pdpp-node-${target}${expected.platform === "win32" ? ".exe" : ""}` +} + +export function stagePdppNode({ + target, + expectedPlatform, + expectedArch, + source = process.execPath, + sourceLicense, + actualPlatform = process.platform, + actualArch = process.arch, + nodeVersion = process.versions.node, + projectRoot = process.cwd(), +}) { + const supported = SUPPORTED_TARGETS.get(target) + if (!supported) fail(`Unsupported Node.js sidecar target: ${target}`) + if ( + expectedPlatform !== supported.platform || + expectedArch !== supported.arch + ) { + fail( + `Target ${target} requires Node.js ${supported.platform}/${supported.arch}` + ) + } + if (actualPlatform !== expectedPlatform || actualArch !== expectedArch) { + fail( + `setup-node architecture mismatch: expected ${expectedPlatform}/${expectedArch}, got ${actualPlatform}/${actualArch}` + ) + } + if (nodeVersion !== RELEASE_NODE_VERSION) { + fail( + `Release sidecar must use Node.js ${RELEASE_NODE_VERSION}; setup-node supplied ${nodeVersion}` + ) + } + if (!existsSync(source) || !statSync(source).isFile()) { + fail(`setup-node executable does not exist: ${source}`) + } + const license = + sourceLicense ?? + [ + join(dirname(source), "LICENSE"), + join(dirname(source), "..", "LICENSE"), + ].find(candidate => existsSync(candidate) && statSync(candidate).isFile()) + if (!license) { + fail( + `Node.js LICENSE was not found beside the setup-node executable: ${source}` + ) + } + + const destination = resolve( + projectRoot, + "src-tauri", + "binaries", + sidecarFilename(target) + ) + mkdirSync(dirname(destination), { recursive: true }) + // Debug builds may hard-link these destinations to the developer's Node. + // Unlink first so release staging never mutates that source inode. + rmSync(destination, { force: true }) + copyFileSync(source, destination) + const licenseDestination = resolve( + projectRoot, + "src-tauri", + "binaries", + "pdpp-node-LICENSE" + ) + rmSync(licenseDestination, { force: true }) + copyFileSync(license, licenseDestination) + if (expectedPlatform !== "win32") { + // copyFileSync preserves the executable mode on Unix; fail closed if that + // ever changes instead of shipping a sidecar the OS cannot start. + if ((statSync(destination).mode & 0o111) === 0) { + fail(`Staged Node.js sidecar is not executable: ${destination}`) + } + } + return { executable: destination, license: licenseDestination } +} + +export function parseArgs(argv) { + const args = {} + for (let index = 0; index < argv.length; index += 1) { + const value = argv[index + 1] + if (argv[index] === "--target") args.target = value + else if (argv[index] === "--expected-platform") + args.expectedPlatform = value + else if (argv[index] === "--expected-arch") args.expectedArch = value + else fail(`Unknown argument: ${argv[index]}`) + index += 1 + } + if (!args.target || !args.expectedPlatform || !args.expectedArch) { + fail( + "Usage: --target --expected-platform --expected-arch " + ) + } + return args +} + +if (isMainModule(import.meta.url, process.argv[1])) { + const staged = stagePdppNode(parseArgs(process.argv.slice(2))) + console.log(`Staged ${basename(process.execPath)} as ${staged.executable}`) + console.log(`Staged Node.js notices as ${staged.license}`) +} diff --git a/scripts/stage-pdpp-node.test.mjs b/scripts/stage-pdpp-node.test.mjs new file mode 100644 index 00000000..7c105867 --- /dev/null +++ b/scripts/stage-pdpp-node.test.mjs @@ -0,0 +1,176 @@ +import { + mkdtempSync, + mkdirSync, + linkSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs" +import { tmpdir } from "node:os" +import { join, resolve } from "node:path" +import { describe, expect, it } from "vitest" +import { + parseArgs, + nativeTauriTarget, + sidecarFilename, + stagePdppNode, + RELEASE_NODE_VERSION, +} from "./stage-pdpp-node.mjs" + +describe("stage PDPP Node.js sidecar", () => { + it.each([ + ["darwin", "arm64", "aarch64-apple-darwin"], + ["darwin", "x64", "x86_64-apple-darwin"], + ["linux", "x64", "x86_64-unknown-linux-gnu"], + ["win32", "x64", "x86_64-pc-windows-msvc"], + ])("maps native %s/%s to %s", (platform, arch, target) => { + expect(nativeTauriTarget(platform, arch)).toEqual({ + target, + expectedPlatform: platform, + expectedArch: arch, + }) + }) + + it.each([ + ["aarch64-apple-darwin", "pdpp-node-aarch64-apple-darwin"], + ["x86_64-apple-darwin", "pdpp-node-x86_64-apple-darwin"], + ["x86_64-unknown-linux-gnu", "pdpp-node-x86_64-unknown-linux-gnu"], + ["x86_64-pc-windows-msvc", "pdpp-node-x86_64-pc-windows-msvc.exe"], + ])( + "uses Tauri's target-qualified sidecar name for %s", + (target, filename) => { + expect(sidecarFilename(target)).toBe(filename) + } + ) + + it("copies the setup-node executable only when version and architecture match", () => { + const root = mkdtempSync(join(tmpdir(), "pdpp-node-sidecar-")) + const source = join(root, "node-source") + const sourceLicense = join(root, "LICENSE") + try { + mkdirSync(join(root, "src-tauri"), { recursive: true }) + writeFileSync(source, "fixture", { mode: 0o755 }) + writeFileSync(sourceLicense, "Node.js license fixture") + const staged = stagePdppNode({ + target: "x86_64-unknown-linux-gnu", + expectedPlatform: "linux", + expectedArch: "x64", + source, + sourceLicense, + actualPlatform: "linux", + actualArch: "x64", + nodeVersion: RELEASE_NODE_VERSION, + projectRoot: root, + }) + expect(staged).toEqual({ + executable: join( + root, + "src-tauri/binaries/pdpp-node-x86_64-unknown-linux-gnu" + ), + license: join(root, "src-tauri/binaries/pdpp-node-LICENSE"), + }) + expect(readFileSync(staged.executable, "utf8")).toBe("fixture") + expect(readFileSync(staged.license, "utf8")).toBe( + "Node.js license fixture" + ) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) + + it("rejects the wrong Node major or native architecture", () => { + const common = { + target: "x86_64-unknown-linux-gnu", + expectedPlatform: "linux", + expectedArch: "x64", + source: process.execPath, + sourceLicense: process.execPath, + actualPlatform: "linux", + actualArch: "x64", + projectRoot: tmpdir(), + } + expect(() => stagePdppNode({ ...common, nodeVersion: "22.18.0" })).toThrow( + `must use Node.js ${RELEASE_NODE_VERSION}` + ) + expect(() => + stagePdppNode({ + ...common, + nodeVersion: RELEASE_NODE_VERSION, + actualArch: "arm64", + }) + ).toThrow("architecture mismatch") + }) + + it("replaces debug hard links without modifying their source files", () => { + const root = mkdtempSync(join(tmpdir(), "pdpp-node-hard-link-")) + const source = join(root, "release-node") + const sourceLicense = join(root, "release-LICENSE") + const developmentNode = join(root, "development-node") + const developmentLicense = join(root, "development-LICENSE") + const binaries = join(root, "src-tauri", "binaries") + try { + mkdirSync(binaries, { recursive: true }) + writeFileSync(source, "release", { mode: 0o755 }) + writeFileSync(sourceLicense, "release license") + writeFileSync(developmentNode, "development", { mode: 0o755 }) + writeFileSync(developmentLicense, "development license") + linkSync( + developmentNode, + join(binaries, "pdpp-node-x86_64-unknown-linux-gnu") + ) + linkSync(developmentLicense, join(binaries, "pdpp-node-LICENSE")) + + stagePdppNode({ + target: "x86_64-unknown-linux-gnu", + expectedPlatform: "linux", + expectedArch: "x64", + source, + sourceLicense, + actualPlatform: "linux", + actualArch: "x64", + nodeVersion: RELEASE_NODE_VERSION, + projectRoot: root, + }) + + expect(readFileSync(developmentNode, "utf8")).toBe("development") + expect(readFileSync(developmentLicense, "utf8")).toBe( + "development license" + ) + expect( + readFileSync( + join(binaries, "pdpp-node-x86_64-unknown-linux-gnu"), + "utf8" + ) + ).toBe("release") + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) + + it("parses the release workflow arguments", () => { + expect( + parseArgs([ + "--target", + "aarch64-apple-darwin", + "--expected-platform", + "darwin", + "--expected-arch", + "arm64", + ]) + ).toEqual({ + target: "aarch64-apple-darwin", + expectedPlatform: "darwin", + expectedArch: "arm64", + }) + }) + + it("gates the supported local production build on the same staging contract", () => { + const buildScript = readFileSync( + resolve(process.cwd(), "scripts/build-prod.js"), + "utf8" + ) + expect(buildScript).toContain( + "stagePdppNode({ ...nativeTauriTarget(PLAT, arch()), projectRoot: ROOT })" + ) + }) +}) diff --git a/scripts/verify-bundled-personal-server.mjs b/scripts/verify-bundled-personal-server.mjs index 3b9061c3..5702c318 100644 --- a/scripts/verify-bundled-personal-server.mjs +++ b/scripts/verify-bundled-personal-server.mjs @@ -8,6 +8,7 @@ import { createInterface } from "node:readline" import { isMainModule } from "./is-main-module.js" const REQUIRED_PATH_FRAGMENTS = [ + "licenses/pdpp-node-license", "personal-server/dist/personal-server", "personal-server/dist/node_modules/better-sqlite3/build/Release/better_sqlite3.node", "playwright-runner/dist/playwright-runner", @@ -55,6 +56,20 @@ export function assertPackagedRuntime(entries, artifactName) { } } +export function assertPackagedNode(entries, artifactName, platform) { + const expectedSuffix = { + linux: "usr/bin/pdpp-node", + macos: "contents/macos/pdpp-node", + windows: "pdpp-node.exe", + }[platform] + if (!expectedSuffix) fail(`Unsupported Node sidecar platform: ${platform}`) + if ( + !entries.map(normalizeEntry).some(entry => entry.endsWith(expectedSuffix)) + ) { + fail(`${artifactName} is missing its packaged Node.js sidecar`) + } +} + export function assertPackagedBrowser(entries, artifactName, platform) { const executableNames = { linux: ["/chrome"], @@ -125,8 +140,10 @@ function listAppImageEntries(artifact) { const WINDOWS_BROWSER_FRAGMENT = "playwright-runner/dist/browsers/chromium-" const WINDOWS_BROWSER_EXECUTABLE = "/chrome.exe" +const WINDOWS_NODE_FRAGMENT = "/pdpp-node.exe" const WINDOWS_RELEVANT_FRAGMENTS = [ ...REQUIRED_PATH_FRAGMENTS, + WINDOWS_NODE_FRAGMENT, WINDOWS_BROWSER_FRAGMENT, ].map(normalizeEntry) const COMMAND_ERROR_OUTPUT_LIMIT = 64 * 1024 @@ -207,6 +224,7 @@ function verifyMacApp(app, expectedArch, artifactName, verifyCodeSignature) { } const entries = listDirectoryEntries(app) assertPackagedRuntime(entries, artifactName) + assertPackagedNode(entries, artifactName, "macos") assertPackagedBrowser(entries, artifactName, "macos") const executableDirectory = join(app, "Contents", "MacOS") @@ -219,6 +237,7 @@ function verifyMacApp(app, expectedArch, artifactName, verifyCodeSignature) { const binaries = [ join(executableDirectory, appExecutables[0].name), + join(executableDirectory, "pdpp-node"), join( app, "Contents", @@ -323,9 +342,11 @@ function verifyLinuxArtifacts(bundleRoot) { } const debEntries = listDebEntries(debArtifacts[0]) assertPackagedRuntime(debEntries, basename(debArtifacts[0])) + assertPackagedNode(debEntries, basename(debArtifacts[0]), "linux") assertPackagedBrowser(debEntries, basename(debArtifacts[0]), "linux") const appImageEntries = listAppImageEntries(appImageArtifacts[0]) assertPackagedRuntime(appImageEntries, basename(appImageArtifacts[0])) + assertPackagedNode(appImageEntries, basename(appImageArtifacts[0]), "linux") assertPackagedBrowser( appImageEntries, basename(appImageArtifacts[0]), @@ -340,6 +361,7 @@ async function verifyWindowsArtifacts(bundleRoot) { if (installers.length !== 1) fail("Expected exactly one NSIS installer") const entries = await listWindowsInstallerEntries(installers[0]) assertPackagedRuntime(entries, basename(installers[0])) + assertPackagedNode(entries, basename(installers[0]), "windows") assertPackagedBrowser(entries, basename(installers[0]), "windows") } diff --git a/scripts/verify-bundled-personal-server.test.mjs b/scripts/verify-bundled-personal-server.test.mjs index c95b67b6..0097235c 100644 --- a/scripts/verify-bundled-personal-server.test.mjs +++ b/scripts/verify-bundled-personal-server.test.mjs @@ -13,6 +13,7 @@ import { describe, expect, it } from "vitest" import { assertBinaryArchitecture, assertPackagedBrowser, + assertPackagedNode, assertPackagedRuntime, collectRelevantWindowsInstallerEntries, listDebEntries, @@ -20,6 +21,8 @@ import { } from "./verify-bundled-personal-server.mjs" const runtimeEntries = [ + "usr/bin/pdpp-node", + "usr/lib/data-connect/licenses/pdpp-node-LICENSE", "usr/lib/data-connect/resources/personal-server/dist/personal-server", "usr/lib/data-connect/resources/personal-server/dist/node_modules/better-sqlite3/build/Release/better_sqlite3.node", "usr/lib/data-connect/resources/playwright-runner/dist/playwright-runner", @@ -59,6 +62,29 @@ describe("bundled personal-server verifier", () => { ).toThrow("p-queue") }) + it("requires the platform-native Node sidecar path", () => { + expect(() => + assertPackagedNode(runtimeEntries, "DataConnect.deb", "linux") + ).not.toThrow() + expect(() => + assertPackagedNode( + ["resources/pdpp-node.exe"], + "DataConnect.exe", + "windows" + ) + ).not.toThrow() + expect(() => + assertPackagedNode( + ["Contents/MacOS/pdpp-node"], + "DataConnect.dmg", + "macos" + ) + ).not.toThrow() + expect(() => + assertPackagedNode(runtimeEntries, "DataConnect.exe", "windows") + ).toThrow("packaged Node.js sidecar") + }) + it("requires a real packaged browser executable", () => { expect(() => assertPackagedBrowser( @@ -116,6 +142,7 @@ describe("bundled personal-server verifier", () => { } for (const entry of ${JSON.stringify([ ...runtimeEntries, + "resources/pdpp-node.exe", "usr/lib/data-connect/resources/playwright-runner/dist/browsers/chromium-1228/chrome-win64/chrome.exe", ])}) { console.log(\`2026-07-31 ..... \${entry.replaceAll("/", "\\\\")}\`) @@ -124,7 +151,7 @@ describe("bundled personal-server verifier", () => { ] ) - expect(entries).toHaveLength(10) + expect(entries).toHaveLength(12) expect(() => assertPackagedRuntime(entries, "DataConnect.exe") ).not.toThrow() @@ -157,12 +184,12 @@ describe("bundled personal-server verifier", () => { child.emit("close", 0) setImmediate(() => { child.stdout.end( - "resources/playwright-runner/dist/browsers/chromium-1228/chrome-win64/chrome.exe" + "resources/pdpp-node.exe\nresources/playwright-runner/dist/browsers/chromium-1228/chrome-win64/chrome.exe" ) }) const entries = await entriesPromise - expect(entries).toHaveLength(10) + expect(entries).toHaveLength(12) expect(() => assertPackagedBrowser(entries, "DataConnect.exe", "windows") ).not.toThrow() diff --git a/src-tauri/build.rs b/src-tauri/build.rs index 795b9b7c..86ee7d37 100644 --- a/src-tauri/build.rs +++ b/src-tauri/build.rs @@ -1,3 +1,50 @@ fn main() { - tauri_build::build() + stage_development_node_sidecar(); + tauri_build::build() +} + +fn stage_development_node_sidecar() { + if std::env::var("PROFILE").as_deref() != Ok("debug") { + return; + } + let Ok(target) = std::env::var("TARGET") else { + return; + }; + let extension = if target.contains("windows") { + ".exe" + } else { + "" + }; + let destination = + std::path::PathBuf::from("binaries").join(format!("pdpp-node-{target}{extension}")); + let license_destination = std::path::PathBuf::from("binaries/pdpp-node-LICENSE"); + let Some(node) = std::env::var_os("PATH").and_then(|path| { + std::env::split_paths(&path) + .map(|directory| directory.join(if cfg!(windows) { "node.exe" } else { "node" })) + .find(|candidate| candidate.is_file()) + }) else { + return; + }; + let Some(license) = [ + node.parent().map(|path| path.join("LICENSE")), + node.parent().map(|path| path.join("../LICENSE")), + ] + .into_iter() + .flatten() + .find(|candidate| candidate.is_file()) else { + return; + }; + std::fs::create_dir_all("binaries").expect("failed to create development sidecar directory"); + stage_development_file(&node, &destination, "Node.js sidecar"); + stage_development_file(&license, &license_destination, "Node.js license"); +} + +fn stage_development_file(source: &std::path::Path, destination: &std::path::Path, label: &str) { + if destination.is_file() { + return; + } + if std::fs::hard_link(source, destination).is_err() { + std::fs::copy(source, destination) + .unwrap_or_else(|error| panic!("failed to stage development {label}: {error}")); + } } diff --git a/src-tauri/src/commands/pdpp_installed_connector.rs b/src-tauri/src/commands/pdpp_installed_connector.rs index 0b3f0d84..37dbd021 100644 --- a/src-tauri/src/commands/pdpp_installed_connector.rs +++ b/src-tauri/src/commands/pdpp_installed_connector.rs @@ -20,6 +20,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use sha2::{Digest, Sha256}; use std::collections::{HashMap, HashSet}; +use std::ffi::OsStr; use std::fs; use std::path::{Component, Path, PathBuf}; use std::process::Command; @@ -33,6 +34,11 @@ const DEFAULT_TIMEOUT_SECONDS: u64 = 120; const MAX_TIMEOUT_SECONDS: u64 = 900; const MAX_RUN_ID_BYTES: usize = 128; const MINIMUM_NODE_MAJOR: u64 = 22; +const BUNDLED_NODE_NAME: &str = if cfg!(windows) { + "pdpp-node.exe" +} else { + "pdpp-node" +}; const CLEANUP_WAIT: Duration = Duration::from_secs(2); const GITHUB_CONNECTOR_KEY: &str = "github"; const GITHUB_CONNECTOR_ID: &str = "https://registry.pdpp.org/connectors/github"; @@ -1882,7 +1888,29 @@ fn verify_file_hash(path: &Path, expected: Option<&str>, label: &str) -> Result< } fn resolve_node_program() -> Result { - let path = std::env::var_os("PATH").ok_or("PATH is unavailable; cannot resolve node")?; + let executable = std::env::current_exe() + .map_err(|e| format!("Could not locate the DataConnect executable: {e}"))?; + resolve_node_program_from(&executable, std::env::var_os("PATH").as_deref()) +} + +fn resolve_node_program_from(executable: &Path, path: Option<&OsStr>) -> Result { + let executable_dir = executable + .parent() + .ok_or("DataConnect executable has no parent directory")?; + let bundled = executable_dir.join(BUNDLED_NODE_NAME); + if bundled.exists() { + if !bundled.is_file() { + return Err(format!( + "Bundled Node.js path is not a file: {}", + bundled.display() + )); + } + validate_node_program(&bundled)?; + return Ok(bundled.to_string_lossy().into_owned()); + } + + let path = path + .ok_or("Bundled Node.js is unavailable and PATH cannot resolve a development fallback")?; for dir in std::env::split_paths(&path) { let candidate = dir.join(if cfg!(windows) { "node.exe" } else { "node" }); if candidate.is_file() { @@ -1890,7 +1918,7 @@ fn resolve_node_program() -> Result { return Ok(candidate.to_string_lossy().into_owned()); } } - Err("node executable was not found on PATH".into()) + Err("Bundled Node.js is unavailable and node was not found on PATH".into()) } fn validate_node_program(candidate: &Path) -> Result<(), String> { @@ -3652,6 +3680,70 @@ setInterval(() => {}, 1000); .contains("invalid version")); } + #[test] + fn uses_path_node_only_when_bundled_node_is_absent() { + let ambient_node = PathBuf::from(resolve_node_program().unwrap()); + let app_dir = tempfile::tempdir().unwrap(); + let fake_app = app_dir.path().join("dataconnect"); + fs::write(&fake_app, b"fixture app").unwrap(); + let path = std::env::join_paths([ambient_node.parent().unwrap()]).unwrap(); + + let fallback = resolve_node_program_from(&fake_app, Some(&path)).unwrap(); + assert_eq!(Path::new(&fallback), ambient_node); + + fs::write(app_dir.path().join(BUNDLED_NODE_NAME), b"not node").unwrap(); + let error = resolve_node_program_from(&fake_app, Some(&path)).unwrap_err(); + assert!(error.contains("Failed to inspect Node.js")); + } + + #[test] + fn starts_connector_with_bundled_node_and_no_node_on_path() { + let ambient_node = PathBuf::from(resolve_node_program().unwrap()); + let app_dir = tempfile::tempdir().unwrap(); + let fake_app = app_dir.path().join(if cfg!(windows) { + "dataconnect.exe" + } else { + "dataconnect" + }); + fs::write(&fake_app, b"fixture app").unwrap(); + let bundled_node = app_dir.path().join(BUNDLED_NODE_NAME); + fs::copy(&ambient_node, &bundled_node).unwrap(); + + let resolved_node = + resolve_node_program_from(&fake_app, Some(OsStr::new(""))).unwrap(); + assert_eq!(Path::new(&resolved_node), bundled_node); + + let (temp, mut install) = install_fixture(github_manifest(), success_script()); + install.root_path = temp.path().to_string_lossy().into_owned(); + let resolved = resolve_installed_pdpp_connector(&install).unwrap(); + let request = request_with_token("test-token"); + let start = build_start(&request, &resolved.manifest, None).unwrap(); + let mut command = build_command( + &resolved, + &resolve_child_secrets(&request, &resolved).unwrap(), + &CommandCustomization::default(), + None, + ) + .unwrap(); + command.program = resolved_node; + assert!(command.clear_env); + assert!(!command.env.contains_key("PATH")); + + let result = supervise_pdpp_connector( + &command, + &start, + &PdppRunOptions { + timeout: Some(Duration::from_secs(5)), + scope_validators: validators_from_manifest(&resolved.manifest), + max_retained_records: 8, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.status, PdppRunStatus::Succeeded); + assert_eq!(result.record_count, 1); + } + #[test] fn event_summary_counts_all_events_when_retained_messages_are_truncated() { let script = r#" diff --git a/src-tauri/tauri-config.test.ts b/src-tauri/tauri-config.test.ts index 89338278..5c9720cd 100644 --- a/src-tauri/tauri-config.test.ts +++ b/src-tauri/tauri-config.test.ts @@ -3,6 +3,21 @@ import { resolve } from "node:path" import { describe, expect, it } from "vitest" describe("tauri manual-install config", () => { + it("bundles the target-native Node runtime as a Tauri sidecar", () => { + const filePath = resolve(process.cwd(), "src-tauri/tauri.conf.json") + const document = JSON.parse(readFileSync(filePath, "utf-8")) as { + bundle?: { + externalBin?: string[] + resources?: Record + } + } + + expect(document.bundle?.externalBin).toEqual(["binaries/pdpp-node"]) + expect(document.bundle?.resources?.["binaries/pdpp-node-LICENSE"]).toBe( + "licenses/pdpp-node-LICENSE" + ) + }) + it("preserves the packaged Playwright browser directory tree", () => { const filePath = resolve(process.cwd(), "src-tauri/tauri.conf.json") const document = JSON.parse(readFileSync(filePath, "utf-8")) as { diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 240fe0d6..5401bff9 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -40,6 +40,7 @@ "bundle": { "active": true, "createUpdaterArtifacts": false, + "externalBin": ["binaries/pdpp-node"], "targets": ["dmg", "app", "nsis", "appimage", "deb"], "icon": [ "icons/32x32.png", @@ -62,7 +63,8 @@ "../playwright-runner/dist/": "playwright-runner/dist/", "../personal-server/dist/personal-server*": "personal-server/dist/", "../personal-server/dist/node_modules/": "personal-server/dist/node_modules/", - "../pdpp-runtime/": "pdpp-runtime/" + "../pdpp-runtime/": "pdpp-runtime/", + "binaries/pdpp-node-LICENSE": "licenses/pdpp-node-LICENSE" }, "linux": { "deb": { diff --git a/vite.config.ts b/vite.config.ts index c98b16b5..599e7fed 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -24,6 +24,7 @@ export default defineConfig({ "scripts/create-macos-dmg.test.mjs", "scripts/release-github.test.mjs", "scripts/release-workflow.test.ts", + "scripts/stage-pdpp-node.test.mjs", "scripts/resolve-connectors.test.mjs", "scripts/verify-release-ref.test.mjs", "scripts/verify-bundled-personal-server.test.mjs", From 2dcf8a1de7dc943f0097c489e01d9f6d040f3fc3 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 31 Jul 2026 18:15:48 -0500 Subject: [PATCH 10/14] fix(pdpp): pass runtime root to bundled node test Assisted-by: AI Signed-off-by: Tim Nunamaker --- src-tauri/src/commands/pdpp_installed_connector.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src-tauri/src/commands/pdpp_installed_connector.rs b/src-tauri/src/commands/pdpp_installed_connector.rs index 37dbd021..464c6997 100644 --- a/src-tauri/src/commands/pdpp_installed_connector.rs +++ b/src-tauri/src/commands/pdpp_installed_connector.rs @@ -3716,6 +3716,7 @@ setInterval(() => {}, 1000); let (temp, mut install) = install_fixture(github_manifest(), success_script()); install.root_path = temp.path().to_string_lossy().into_owned(); let resolved = resolve_installed_pdpp_connector(&install).unwrap(); + let runtime_root = resolve_pdpp_runtime_root(None).unwrap(); let request = request_with_token("test-token"); let start = build_start(&request, &resolved.manifest, None).unwrap(); let mut command = build_command( @@ -3723,6 +3724,7 @@ setInterval(() => {}, 1000); &resolve_child_secrets(&request, &resolved).unwrap(), &CommandCustomization::default(), None, + &runtime_root, ) .unwrap(); command.program = resolved_node; From 43b411c728e6f47b584f6133c3b38cf7d2be74ec Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 31 Jul 2026 18:21:51 -0500 Subject: [PATCH 11/14] fix(release): launch PDPP runtime npm portably Assisted-by: AI Signed-off-by: Tim Nunamaker --- scripts/ensure-pdpp-runtime.js | 104 +++++++++++++++++++++------ scripts/ensure-pdpp-runtime.test.mjs | 101 ++++++++++++++++++++++++++ vite.config.ts | 1 + 3 files changed, 185 insertions(+), 21 deletions(-) create mode 100644 scripts/ensure-pdpp-runtime.test.mjs diff --git a/scripts/ensure-pdpp-runtime.js b/scripts/ensure-pdpp-runtime.js index 53e048f0..a86c22e9 100644 --- a/scripts/ensure-pdpp-runtime.js +++ b/scripts/ensure-pdpp-runtime.js @@ -1,30 +1,92 @@ -import { existsSync, readFileSync, writeFileSync } from "node:fs" import { spawnSync } from "node:child_process" import { createHash } from "node:crypto" +import { existsSync, readFileSync, writeFileSync } from "node:fs" import { dirname, join } from "node:path" import { fileURLToPath } from "node:url" +import { isMainModule } from "./is-main-module.js" const root = dirname(dirname(fileURLToPath(import.meta.url))) const runtimeRoot = join(root, "pdpp-runtime") -const required = ["p-queue", "patchright"].map(name => - join(runtimeRoot, "node_modules", name, "package.json") -) -const stampPath = join(runtimeRoot, ".install-stamp") -const stamp = createHash("sha256") - .update(readFileSync(join(runtimeRoot, "package.json"))) - .update(readFileSync(join(runtimeRoot, "package-lock.json"))) - .digest("hex") -if ( - required.every(existsSync) && - existsSync(stampPath) && - readFileSync(stampPath, "utf8").trim() === stamp -) - process.exit(0) +function runtimeStamp(root) { + return createHash("sha256") + .update(readFileSync(join(root, "package.json"))) + .update(readFileSync(join(root, "package-lock.json"))) + .digest("hex") +} + +export function npmInstallCommand(options = {}) { + const { + platformName = process.platform, + nodePath = process.execPath, + } = options + const npmCliPath = Object.hasOwn(options, "npmCliPath") + ? options.npmCliPath + : process.env.npm_execpath + return { + command: npmCliPath + ? nodePath + : platformName === "win32" + ? "npm.cmd" + : "npm", + args: [...(npmCliPath ? [npmCliPath] : []), "ci", "--ignore-scripts"], + shell: !npmCliPath && platformName === "win32", + } +} + +export function runEnsurePdppRuntime(options = {}) { + const { + root = runtimeRoot, + spawn = spawnSync, + platformName = process.platform, + nodePath = process.execPath, + } = options + const npmCliPath = Object.hasOwn(options, "npmCliPath") + ? options.npmCliPath + : process.env.npm_execpath + const required = ["p-queue", "patchright"].map(name => + join(root, "node_modules", name, "package.json") + ) + const stampPath = join(root, ".install-stamp") + const stamp = runtimeStamp(root) + + if ( + required.every(existsSync) && + existsSync(stampPath) && + readFileSync(stampPath, "utf8").trim() === stamp + ) { + return 0 + } + + const installCommand = npmInstallCommand({ + platformName, + nodePath, + npmCliPath, + }) + const install = spawn(installCommand.command, installCommand.args, { + cwd: root, + stdio: "inherit", + shell: installCommand.shell, + }) + if (install.error) { + throw new Error( + `Failed to install PDPP runtime dependencies: ${install.error.message}` + ) + } + if (install.status !== 0) { + throw new Error( + `Failed to install PDPP runtime dependencies: npm exited with status ${install.status ?? "unknown"}` + ) + } + writeFileSync(stampPath, `${stamp}\n`) + return 0 +} -const install = spawnSync("npm", ["ci", "--ignore-scripts"], { - cwd: runtimeRoot, - stdio: "inherit", -}) -if (install.status === 0) writeFileSync(stampPath, `${stamp}\n`) -process.exit(install.status ?? 1) +if (isMainModule(import.meta.url, process.argv[1])) { + try { + process.exit(runEnsurePdppRuntime()) + } catch (error) { + console.error(error instanceof Error ? error.message : error) + process.exit(1) + } +} diff --git a/scripts/ensure-pdpp-runtime.test.mjs b/scripts/ensure-pdpp-runtime.test.mjs new file mode 100644 index 00000000..4bdb1115 --- /dev/null +++ b/scripts/ensure-pdpp-runtime.test.mjs @@ -0,0 +1,101 @@ +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { + npmInstallCommand, + runEnsurePdppRuntime, +} from "./ensure-pdpp-runtime.js" + +function createRuntimeFixture() { + const root = mkdtempSync(join(tmpdir(), "pdpp-runtime-install-")) + writeFileSync(join(root, "package.json"), "{}") + writeFileSync(join(root, "package-lock.json"), "{}") + return root +} + +describe("ensure PDPP runtime dependencies", () => { + it("runs npm through Node on Windows when npm_execpath is available", () => { + expect( + npmInstallCommand({ + platformName: "win32", + nodePath: "C:\\hostedtoolcache\\node.exe", + npmCliPath: "C:\\hostedtoolcache\\npm\\bin\\npm-cli.js", + }) + ).toEqual({ + command: "C:\\hostedtoolcache\\node.exe", + args: [ + "C:\\hostedtoolcache\\npm\\bin\\npm-cli.js", + "ci", + "--ignore-scripts", + ], + shell: false, + }) + }) + + it("falls back to npm.cmd on Windows without shell injection inputs", () => { + expect( + npmInstallCommand({ + platformName: "win32", + nodePath: "C:\\hostedtoolcache\\node.exe", + npmCliPath: undefined, + }) + ).toEqual({ + command: "npm.cmd", + args: ["ci", "--ignore-scripts"], + shell: true, + }) + }) + + it("reports spawn errors and non-zero exits with useful messages", () => { + const root = createRuntimeFixture() + try { + expect(() => + runEnsurePdppRuntime({ + root, + spawn: () => ({ error: new Error("spawn npm ENOENT") }), + }) + ).toThrow("spawn npm ENOENT") + + expect(() => + runEnsurePdppRuntime({ + root, + spawn: () => ({ status: 1 }), + }) + ).toThrow("npm exited with status 1") + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) + + it("writes the install stamp after npm restores runtime packages", () => { + const root = createRuntimeFixture() + try { + let invocation + const status = runEnsurePdppRuntime({ + root, + spawn: (...args) => { + invocation = args + for (const dependency of ["p-queue", "patchright"]) { + const packageRoot = join(root, "node_modules", dependency) + mkdirSync(packageRoot, { recursive: true }) + writeFileSync(join(packageRoot, "package.json"), "{}") + } + return { status: 0 } + }, + platformName: "linux", + npmCliPath: undefined, + }) + + expect(status).toBe(0) + expect(invocation).toEqual([ + "npm", + ["ci", "--ignore-scripts"], + { cwd: root, stdio: "inherit", shell: false }, + ]) + expect(readFileSync(join(root, ".install-stamp"), "utf8")).toMatch(/\S/) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) +}) diff --git a/vite.config.ts b/vite.config.ts index 599e7fed..418880da 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -22,6 +22,7 @@ export default defineConfig({ "src-tauri/**/*.test.ts", "scripts/is-main-module.test.mjs", "scripts/create-macos-dmg.test.mjs", + "scripts/ensure-pdpp-runtime.test.mjs", "scripts/release-github.test.mjs", "scripts/release-workflow.test.ts", "scripts/stage-pdpp-node.test.mjs", From a4d812324c1a932b6c45dc5a3fbce57f75c8d8b5 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 31 Jul 2026 18:43:17 -0500 Subject: [PATCH 12/14] fix(release): identify macOS app executable Read CFBundleExecutable from Info.plist so the verifier distinguishes the application binary from the bundled Node sidecar. Keep exact sidecar selection and architecture verification for both source apps and DMG contents. Assisted-by: AI Signed-off-by: Tim Nunamaker --- scripts/verify-bundled-personal-server.mjs | 62 +++++++++++++++-- .../verify-bundled-personal-server.test.mjs | 68 +++++++++++++++++++ 2 files changed, 123 insertions(+), 7 deletions(-) diff --git a/scripts/verify-bundled-personal-server.mjs b/scripts/verify-bundled-personal-server.mjs index 5702c318..c2f0dd88 100644 --- a/scripts/verify-bundled-personal-server.mjs +++ b/scripts/verify-bundled-personal-server.mjs @@ -218,6 +218,47 @@ function findAppBundles(root) { }) } +export function readMacBundleExecutable(infoPlist, runCommand = run) { + const executable = runCommand("plutil", [ + "-extract", + "CFBundleExecutable", + "raw", + "-o", + "-", + infoPlist, + ]).trim() + if (!executable) fail(`${infoPlist} has an empty CFBundleExecutable`) + return executable +} + +export function selectMacAppExecutables( + executableNames, + declaredExecutable, + artifactName +) { + const mainMatches = executableNames.filter( + name => name === declaredExecutable + ) + if (mainMatches.length !== 1) { + fail( + `${artifactName} CFBundleExecutable ${declaredExecutable} must identify exactly one app executable` + ) + } + + const nodeMatches = executableNames.filter(name => name === "pdpp-node") + if (nodeMatches.length !== 1) { + fail(`${artifactName} must contain exactly one pdpp-node sidecar`) + } + if (declaredExecutable === "pdpp-node") { + fail(`${artifactName} app executable must be distinct from pdpp-node`) + } + + return { + appExecutable: declaredExecutable, + nodeSidecar: nodeMatches[0], + } +} + function verifyMacApp(app, expectedArch, artifactName, verifyCodeSignature) { if (verifyCodeSignature) { run("codesign", ["--verify", "--deep", "--strict", app]) @@ -228,16 +269,23 @@ function verifyMacApp(app, expectedArch, artifactName, verifyCodeSignature) { assertPackagedBrowser(entries, artifactName, "macos") const executableDirectory = join(app, "Contents", "MacOS") - const appExecutables = readdirSync(executableDirectory, { + const executableNames = readdirSync(executableDirectory, { withFileTypes: true, - }).filter(entry => entry.isFile()) - if (appExecutables.length !== 1) { - fail(`${artifactName} must contain exactly one app executable`) - } + }) + .filter(entry => entry.isFile()) + .map(entry => entry.name) + const declaredExecutable = readMacBundleExecutable( + join(app, "Contents", "Info.plist") + ) + const { appExecutable, nodeSidecar } = selectMacAppExecutables( + executableNames, + declaredExecutable, + artifactName + ) const binaries = [ - join(executableDirectory, appExecutables[0].name), - join(executableDirectory, "pdpp-node"), + join(executableDirectory, appExecutable), + join(executableDirectory, nodeSidecar), join( app, "Contents", diff --git a/scripts/verify-bundled-personal-server.test.mjs b/scripts/verify-bundled-personal-server.test.mjs index 0097235c..f403e1f3 100644 --- a/scripts/verify-bundled-personal-server.test.mjs +++ b/scripts/verify-bundled-personal-server.test.mjs @@ -18,6 +18,8 @@ import { collectRelevantWindowsInstallerEntries, listDebEntries, parseArgs, + readMacBundleExecutable, + selectMacAppExecutables, } from "./verify-bundled-personal-server.mjs" const runtimeEntries = [ @@ -115,6 +117,72 @@ describe("bundled personal-server verifier", () => { ).toThrow("expected x86_64") }) + it("uses CFBundleExecutable when the macOS app contains its Node sidecar", () => { + const invocation = [] + expect( + readMacBundleExecutable( + "DataConnect.app/Contents/Info.plist", + (command, args) => { + invocation.push(command, args) + return "DataConnect\n" + } + ) + ).toBe("DataConnect") + expect(invocation).toEqual([ + "plutil", + [ + "-extract", + "CFBundleExecutable", + "raw", + "-o", + "-", + "DataConnect.app/Contents/Info.plist", + ], + ]) + expect(() => + readMacBundleExecutable("DataConnect.app/Contents/Info.plist", () => "\n") + ).toThrow("empty CFBundleExecutable") + expect( + selectMacAppExecutables( + ["DataConnect", "pdpp-node"], + "DataConnect", + "DataConnect.app" + ) + ).toEqual({ + appExecutable: "DataConnect", + nodeSidecar: "pdpp-node", + }) + }) + + it("rejects a missing or ambiguous CFBundleExecutable file", () => { + expect(() => + selectMacAppExecutables(["pdpp-node"], "DataConnect", "DataConnect.app") + ).toThrow("must identify exactly one app executable") + expect(() => + selectMacAppExecutables( + ["DataConnect", "DataConnect", "pdpp-node"], + "DataConnect", + "DataConnect.app" + ) + ).toThrow("must identify exactly one app executable") + }) + + it("requires a distinct, exact Node sidecar beside the macOS app executable", () => { + expect(() => + selectMacAppExecutables(["DataConnect"], "DataConnect", "DataConnect.app") + ).toThrow("exactly one pdpp-node sidecar") + expect(() => + selectMacAppExecutables(["pdpp-node"], "pdpp-node", "DataConnect.app") + ).toThrow("must be distinct from pdpp-node") + expect(() => + assertBinaryArchitecture( + "Mach-O 64-bit executable arm64", + "arm64", + "pdpp-node" + ) + ).not.toThrow() + }) + it("enables strict signature checks only when explicitly requested", () => { const commonArgs = [ "--platform", From 0ceb01bf8fc01e2e1e37f57d8e7265300e1db86d Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 31 Jul 2026 18:51:18 -0500 Subject: [PATCH 13/14] fix(release): recognize Windows Node sidecar Match Tauri's NSIS root-installed pdpp-node.exe entry while rejecting nested, target-suffixed, and lookalike filenames. Assisted-by: AI Signed-off-by: Tim Nunamaker --- scripts/verify-bundled-personal-server.mjs | 26 +++++++++++--- .../verify-bundled-personal-server.test.mjs | 35 +++++++++++++++---- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/scripts/verify-bundled-personal-server.mjs b/scripts/verify-bundled-personal-server.mjs index c2f0dd88..e1007d24 100644 --- a/scripts/verify-bundled-personal-server.mjs +++ b/scripts/verify-bundled-personal-server.mjs @@ -63,9 +63,13 @@ export function assertPackagedNode(entries, artifactName, platform) { windows: "pdpp-node.exe", }[platform] if (!expectedSuffix) fail(`Unsupported Node sidecar platform: ${platform}`) - if ( - !entries.map(normalizeEntry).some(entry => entry.endsWith(expectedSuffix)) - ) { + const hasNode = entries.some(entry => { + if (platform === "windows") { + return windowsInstallerEntryPath(entry) === expectedSuffix + } + return normalizeEntry(entry).endsWith(expectedSuffix) + }) + if (!hasNode) { fail(`${artifactName} is missing its packaged Node.js sidecar`) } } @@ -140,14 +144,20 @@ function listAppImageEntries(artifact) { const WINDOWS_BROWSER_FRAGMENT = "playwright-runner/dist/browsers/chromium-" const WINDOWS_BROWSER_EXECUTABLE = "/chrome.exe" -const WINDOWS_NODE_FRAGMENT = "/pdpp-node.exe" +const WINDOWS_NODE_FILENAME = "pdpp-node.exe" const WINDOWS_RELEVANT_FRAGMENTS = [ ...REQUIRED_PATH_FRAGMENTS, - WINDOWS_NODE_FRAGMENT, + WINDOWS_NODE_FILENAME, WINDOWS_BROWSER_FRAGMENT, ].map(normalizeEntry) const COMMAND_ERROR_OUTPUT_LIMIT = 64 * 1024 +function windowsInstallerEntryPath(entry) { + // `7z l -ba` prefixes metadata, while Tauri's NSIS `/oname` sidecar is a + // bare root entry. Compare that installed path, not the staged source name. + return normalizeEntry(entry).trim().split(/\s+/).at(-1) +} + function appendBoundedOutput(output, chunk) { return `${output}${chunk}`.slice(-COMMAND_ERROR_OUTPUT_LIMIT) } @@ -174,6 +184,12 @@ export async function collectRelevantWindowsInstallerEntries( for (const fragment of WINDOWS_RELEVANT_FRAGMENTS) { if (relevantEntries.has(fragment)) continue if (!normalizedLine.includes(fragment)) continue + if ( + fragment === WINDOWS_NODE_FILENAME && + windowsInstallerEntryPath(line) !== WINDOWS_NODE_FILENAME + ) { + continue + } if ( fragment === WINDOWS_BROWSER_FRAGMENT && !normalizedLine.endsWith(WINDOWS_BROWSER_EXECUTABLE) diff --git a/scripts/verify-bundled-personal-server.test.mjs b/scripts/verify-bundled-personal-server.test.mjs index f403e1f3..8c53cc1f 100644 --- a/scripts/verify-bundled-personal-server.test.mjs +++ b/scripts/verify-bundled-personal-server.test.mjs @@ -69,11 +69,7 @@ describe("bundled personal-server verifier", () => { assertPackagedNode(runtimeEntries, "DataConnect.deb", "linux") ).not.toThrow() expect(() => - assertPackagedNode( - ["resources/pdpp-node.exe"], - "DataConnect.exe", - "windows" - ) + assertPackagedNode(["pdpp-node.exe"], "DataConnect.exe", "windows") ).not.toThrow() expect(() => assertPackagedNode( @@ -87,6 +83,26 @@ describe("bundled personal-server verifier", () => { ).toThrow("packaged Node.js sidecar") }) + it("requires Tauri's exact root-installed Windows x64 Node sidecar", () => { + const tauriNsisListing = [ + "2026-07-31 23:39:00 ....A 124835376 50000000 pdpp-node.exe", + ] + expect(() => + assertPackagedNode(tauriNsisListing, "DataConnect.exe", "windows") + ).not.toThrow() + + for (const wrongEntry of [ + "resources/pdpp-node.exe", + "pdpp-node-x86_64-pc-windows-msvc.exe", + "not-pdpp-node.exe", + "pdpp-node-arm64.exe", + ]) { + expect(() => + assertPackagedNode([wrongEntry], "DataConnect.exe", "windows") + ).toThrow("packaged Node.js sidecar") + } + }) + it("requires a real packaged browser executable", () => { expect(() => assertPackagedBrowser( @@ -211,6 +227,7 @@ describe("bundled personal-server verifier", () => { for (const entry of ${JSON.stringify([ ...runtimeEntries, "resources/pdpp-node.exe", + "pdpp-node.exe", "usr/lib/data-connect/resources/playwright-runner/dist/browsers/chromium-1228/chrome-win64/chrome.exe", ])}) { console.log(\`2026-07-31 ..... \${entry.replaceAll("/", "\\\\")}\`) @@ -223,6 +240,9 @@ describe("bundled personal-server verifier", () => { expect(() => assertPackagedRuntime(entries, "DataConnect.exe") ).not.toThrow() + expect(() => + assertPackagedNode(entries, "DataConnect.exe", "windows") + ).not.toThrow() expect(() => assertPackagedBrowser(entries, "DataConnect.exe", "windows") ).not.toThrow() @@ -252,12 +272,15 @@ describe("bundled personal-server verifier", () => { child.emit("close", 0) setImmediate(() => { child.stdout.end( - "resources/pdpp-node.exe\nresources/playwright-runner/dist/browsers/chromium-1228/chrome-win64/chrome.exe" + "pdpp-node.exe\nresources/playwright-runner/dist/browsers/chromium-1228/chrome-win64/chrome.exe" ) }) const entries = await entriesPromise expect(entries).toHaveLength(12) + expect(() => + assertPackagedNode(entries, "DataConnect.exe", "windows") + ).not.toThrow() expect(() => assertPackagedBrowser(entries, "DataConnect.exe", "windows") ).not.toThrow() From ba8f7bf988829173446d3ecfdc6ca651812fe5ab Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 31 Jul 2026 18:54:28 -0500 Subject: [PATCH 14/14] fix(pdpp): activate bundled profiles before serving Package the pinned GitHub and ChatGPT profiles, verify their lock hashes, and activate missing or relocated identical bundled installs before the Personal Server composes routes. Assisted-by: AI Signed-off-by: Tim Nunamaker --- scripts/verify-bundled-personal-server.mjs | 123 ++++-- .../verify-bundled-personal-server.test.mjs | 123 +++++- src-tauri/src/commands/connector_store.rs | 390 +++++++++++++++++- src-tauri/src/commands/server.rs | 9 + src-tauri/src/commands/updates.rs | 214 +++++++--- src-tauri/tauri-config.test.ts | 17 + src-tauri/tauri.conf.json | 2 + 7 files changed, 758 insertions(+), 120 deletions(-) diff --git a/scripts/verify-bundled-personal-server.mjs b/scripts/verify-bundled-personal-server.mjs index e1007d24..a089bf48 100644 --- a/scripts/verify-bundled-personal-server.mjs +++ b/scripts/verify-bundled-personal-server.mjs @@ -8,6 +8,13 @@ import { createInterface } from "node:readline" import { isMainModule } from "./is-main-module.js" const REQUIRED_PATH_FRAGMENTS = [ + "connectors/lock.json", + "connectors/collection-profiles/github-pdpp/profile/collection-profile.json", + "connectors/collection-profiles/github-pdpp/dist/collection-profile.mjs", + "connectors/collection-profiles/github-pdpp/provenance.json", + "connectors/collection-profiles/chatgpt-pdpp/profile/collection-profile.json", + "connectors/collection-profiles/chatgpt-pdpp/dist/collection-profile.mjs", + "connectors/collection-profiles/chatgpt-pdpp/provenance.json", "licenses/pdpp-node-license", "personal-server/dist/personal-server", "personal-server/dist/node_modules/better-sqlite3/build/Release/better_sqlite3.node", @@ -43,10 +50,12 @@ function normalizeEntry(entry) { } export function assertPackagedRuntime(entries, artifactName) { - const normalizedEntries = entries.map(normalizeEntry) - const missing = REQUIRED_PATH_FRAGMENTS.filter( - fragment => - !normalizedEntries.some(entry => entry.includes(fragment.toLowerCase())) + const platform = artifactPlatform(artifactName) + const packagedPaths = new Set( + entries.map(entry => packagedEntryPath(entry, platform)) + ) + const missing = expectedRuntimePaths(platform).filter( + expectedPath => !packagedPaths.has(expectedPath) ) if (missing.length > 0) { @@ -56,25 +65,65 @@ export function assertPackagedRuntime(entries, artifactName) { } } +function artifactPlatform(artifactName) { + const normalizedName = artifactName.toLowerCase() + if (normalizedName.endsWith(".exe")) return "windows" + if (normalizedName.endsWith(".dmg") || normalizedName.endsWith(".app")) + return "macos" + if (normalizedName.endsWith(".deb") || normalizedName.endsWith(".appimage")) + return "linux" + fail(`Cannot determine artifact platform from ${artifactName}`) +} + +function expectedRuntimePaths(platform) { + const root = { + linux: "usr/lib/dataconnect/", + macos: "contents/resources/", + windows: "", + }[platform] + if (root === undefined) fail(`Unsupported runtime platform: ${platform}`) + return REQUIRED_PATH_FRAGMENTS.map(fragment => { + const executableSuffix = + platform === "windows" && + (fragment.endsWith("personal-server/dist/personal-server") || + fragment.endsWith("playwright-runner/dist/playwright-runner")) + ? ".exe" + : "" + return `${root}${fragment}${executableSuffix}`.toLowerCase() + }) +} + +function packagedEntryPath(entry, platform) { + if (platform === "windows") return windowsInstallerEntryPath(entry) + const normalized = normalizeEntry(entry).trim() + if (platform === "macos") return normalized + const path = normalized.includes(" ") + ? normalized.split(/\s+/).at(-1) + : normalized + return path.replace(/^\.\//, "") +} + export function assertPackagedNode(entries, artifactName, platform) { - const expectedSuffix = { + const expectedPath = { linux: "usr/bin/pdpp-node", macos: "contents/macos/pdpp-node", windows: "pdpp-node.exe", }[platform] - if (!expectedSuffix) fail(`Unsupported Node sidecar platform: ${platform}`) - const hasNode = entries.some(entry => { - if (platform === "windows") { - return windowsInstallerEntryPath(entry) === expectedSuffix - } - return normalizeEntry(entry).endsWith(expectedSuffix) - }) + if (!expectedPath) fail(`Unsupported Node sidecar platform: ${platform}`) + const hasNode = entries.some( + entry => packagedEntryPath(entry, platform) === expectedPath + ) if (!hasNode) { fail(`${artifactName} is missing its packaged Node.js sidecar`) } } export function assertPackagedBrowser(entries, artifactName, platform) { + const expectedPrefix = { + linux: "usr/lib/dataconnect/playwright-runner/dist/browsers/chromium-", + macos: "contents/resources/playwright-runner/dist/browsers/chromium-", + windows: "playwright-runner/dist/browsers/chromium-", + }[platform] const executableNames = { linux: ["/chrome"], macos: [ @@ -83,14 +132,17 @@ export function assertPackagedBrowser(entries, artifactName, platform) { ], windows: ["/chrome.exe"], }[platform] - if (!executableNames) fail(`Unsupported browser platform: ${platform}`) - - const hasBrowserExecutable = entries.map(normalizeEntry).some(entry => { - return ( - entry.includes("playwright-runner/dist/browsers/chromium-") && - executableNames.some(name => entry.endsWith(name)) - ) - }) + if (!expectedPrefix || !executableNames) + fail(`Unsupported browser platform: ${platform}`) + + const hasBrowserExecutable = entries + .map(entry => packagedEntryPath(entry, platform)) + .some(entry => { + return ( + entry.startsWith(expectedPrefix) && + executableNames.some(name => entry.endsWith(name)) + ) + }) if (!hasBrowserExecutable) { fail(`${artifactName} is missing its packaged Chromium executable`) } @@ -145,11 +197,7 @@ function listAppImageEntries(artifact) { const WINDOWS_BROWSER_FRAGMENT = "playwright-runner/dist/browsers/chromium-" const WINDOWS_BROWSER_EXECUTABLE = "/chrome.exe" const WINDOWS_NODE_FILENAME = "pdpp-node.exe" -const WINDOWS_RELEVANT_FRAGMENTS = [ - ...REQUIRED_PATH_FRAGMENTS, - WINDOWS_NODE_FILENAME, - WINDOWS_BROWSER_FRAGMENT, -].map(normalizeEntry) +const WINDOWS_RUNTIME_PATHS = new Set(expectedRuntimePaths("windows")) const COMMAND_ERROR_OUTPUT_LIMIT = 64 * 1024 function windowsInstallerEntryPath(entry) { @@ -180,24 +228,13 @@ export async function collectRelevantWindowsInstallerEntries( const lines = createInterface({ input: child.stdout, crlfDelay: Infinity }) lines.on("line", line => { - const normalizedLine = normalizeEntry(line) - for (const fragment of WINDOWS_RELEVANT_FRAGMENTS) { - if (relevantEntries.has(fragment)) continue - if (!normalizedLine.includes(fragment)) continue - if ( - fragment === WINDOWS_NODE_FILENAME && - windowsInstallerEntryPath(line) !== WINDOWS_NODE_FILENAME - ) { - continue - } - if ( - fragment === WINDOWS_BROWSER_FRAGMENT && - !normalizedLine.endsWith(WINDOWS_BROWSER_EXECUTABLE) - ) { - continue - } - relevantEntries.set(fragment, line) - } + const path = windowsInstallerEntryPath(line) + const isRuntime = WINDOWS_RUNTIME_PATHS.has(path) + const isNode = path === WINDOWS_NODE_FILENAME + const isBrowser = + path.startsWith(WINDOWS_BROWSER_FRAGMENT) && + path.endsWith(WINDOWS_BROWSER_EXECUTABLE) + if (isRuntime || isNode || isBrowser) relevantEntries.set(path, line) }) const processExited = new Promise((resolveExit, rejectExit) => { diff --git a/scripts/verify-bundled-personal-server.test.mjs b/scripts/verify-bundled-personal-server.test.mjs index 8c53cc1f..f915aed1 100644 --- a/scripts/verify-bundled-personal-server.test.mjs +++ b/scripts/verify-bundled-personal-server.test.mjs @@ -24,20 +24,48 @@ import { const runtimeEntries = [ "usr/bin/pdpp-node", - "usr/lib/data-connect/licenses/pdpp-node-LICENSE", - "usr/lib/data-connect/resources/personal-server/dist/personal-server", - "usr/lib/data-connect/resources/personal-server/dist/node_modules/better-sqlite3/build/Release/better_sqlite3.node", - "usr/lib/data-connect/resources/playwright-runner/dist/playwright-runner", - "usr/lib/data-connect/resources/pdpp-runtime/connector-loader.mjs", - "usr/lib/data-connect/resources/pdpp-runtime/connector-loader-bootstrap.mjs", - "usr/lib/data-connect/resources/pdpp-runtime/node_modules/p-queue/package.json", - "usr/lib/data-connect/resources/pdpp-runtime/node_modules/p-queue/dist/index.js", - "usr/lib/data-connect/resources/pdpp-runtime/node_modules/patchright/package.json", - "usr/lib/data-connect/resources/pdpp-runtime/node_modules/patchright/index.mjs", + "usr/lib/DataConnect/connectors/lock.json", + "usr/lib/DataConnect/connectors/collection-profiles/github-pdpp/profile/collection-profile.json", + "usr/lib/DataConnect/connectors/collection-profiles/github-pdpp/dist/collection-profile.mjs", + "usr/lib/DataConnect/connectors/collection-profiles/github-pdpp/provenance.json", + "usr/lib/DataConnect/connectors/collection-profiles/chatgpt-pdpp/profile/collection-profile.json", + "usr/lib/DataConnect/connectors/collection-profiles/chatgpt-pdpp/dist/collection-profile.mjs", + "usr/lib/DataConnect/connectors/collection-profiles/chatgpt-pdpp/provenance.json", + "usr/lib/DataConnect/licenses/pdpp-node-LICENSE", + "usr/lib/DataConnect/personal-server/dist/personal-server", + "usr/lib/DataConnect/personal-server/dist/node_modules/better-sqlite3/build/Release/better_sqlite3.node", + "usr/lib/DataConnect/playwright-runner/dist/playwright-runner", + "usr/lib/DataConnect/pdpp-runtime/connector-loader.mjs", + "usr/lib/DataConnect/pdpp-runtime/connector-loader-bootstrap.mjs", + "usr/lib/DataConnect/pdpp-runtime/node_modules/p-queue/package.json", + "usr/lib/DataConnect/pdpp-runtime/node_modules/p-queue/dist/index.js", + "usr/lib/DataConnect/pdpp-runtime/node_modules/patchright/package.json", + "usr/lib/DataConnect/pdpp-runtime/node_modules/patchright/index.mjs", ] +const windowsRuntimeEntries = runtimeEntries + .filter(entry => entry !== "usr/bin/pdpp-node") + .map(entry => + entry + .replace("usr/lib/DataConnect/", "") + .replace( + "personal-server/dist/personal-server", + "personal-server/dist/personal-server.exe" + ) + .replace( + "playwright-runner/dist/playwright-runner", + "playwright-runner/dist/playwright-runner.exe" + ) + ) + +const macRuntimeEntries = windowsRuntimeEntries.map(entry => + `Contents/Resources/${entry}` + .replace("personal-server.exe", "personal-server") + .replace("playwright-runner.exe", "playwright-runner") +) + const linuxBrowserEntry = - "usr/lib/data-connect/resources/playwright-runner/dist/browsers/chromium-1228/chrome-linux64/chrome" + "usr/lib/DataConnect/playwright-runner/dist/browsers/chromium-1228/chrome-linux64/chrome" describe("bundled personal-server verifier", () => { it("accepts an artifact with the helper and native dependency", () => { @@ -64,6 +92,51 @@ describe("bundled personal-server verifier", () => { ).toThrow("p-queue") }) + it("rejects an artifact missing a bundled PDPP connector", () => { + expect(() => + assertPackagedRuntime( + runtimeEntries.filter(entry => !entry.includes("chatgpt-pdpp")), + "DataConnect.deb" + ) + ).toThrow("chatgpt-pdpp") + }) + + it("requires packaged runtime paths on exact segment boundaries", () => { + expect(() => + assertPackagedRuntime( + runtimeEntries.map(entry => + entry.endsWith("connectors/lock.json") + ? entry.replace("connectors/lock.json", "not-connectors/lock.json") + : entry + ), + "DataConnect.deb" + ) + ).toThrow("connectors/lock.json") + }) + + it("rejects a complete runtime tree under an unrecognized root", () => { + for (const wrongRoot of ["attacker", "resources-not-really"]) { + expect(() => + assertPackagedRuntime( + windowsRuntimeEntries.map(entry => `${wrongRoot}/${entry}`), + "DataConnect.exe" + ) + ).toThrow("connectors/lock.json") + } + }) + + it("accepts only the macOS app resource root", () => { + expect(() => + assertPackagedRuntime(macRuntimeEntries, "DataConnect.app") + ).not.toThrow() + expect(() => + assertPackagedRuntime( + macRuntimeEntries.map(entry => `attacker/${entry}`), + "DataConnect.dmg" + ) + ).toThrow("connectors/lock.json") + }) + it("requires the platform-native Node sidecar path", () => { expect(() => assertPackagedNode(runtimeEntries, "DataConnect.deb", "linux") @@ -81,6 +154,13 @@ describe("bundled personal-server verifier", () => { expect(() => assertPackagedNode(runtimeEntries, "DataConnect.exe", "windows") ).toThrow("packaged Node.js sidecar") + expect(() => + assertPackagedNode( + ["attacker/usr/bin/pdpp-node"], + "DataConnect.deb", + "linux" + ) + ).toThrow("packaged Node.js sidecar") }) it("requires Tauri's exact root-installed Windows x64 Node sidecar", () => { @@ -114,6 +194,13 @@ describe("bundled personal-server verifier", () => { expect(() => assertPackagedBrowser(runtimeEntries, "DataConnect.deb", "linux") ).toThrow("packaged Chromium executable") + expect(() => + assertPackagedBrowser( + [`attacker/${linuxBrowserEntry}`], + "DataConnect.deb", + "linux" + ) + ).toThrow("packaged Chromium executable") }) it("rejects a helper built for the other macOS architecture", () => { @@ -225,10 +312,10 @@ describe("bundled personal-server verifier", () => { console.log(\`2026-07-31 ..... \${noise}-\${index}\`) } for (const entry of ${JSON.stringify([ - ...runtimeEntries, + ...windowsRuntimeEntries, "resources/pdpp-node.exe", "pdpp-node.exe", - "usr/lib/data-connect/resources/playwright-runner/dist/browsers/chromium-1228/chrome-win64/chrome.exe", + "playwright-runner/dist/browsers/chromium-1228/chrome-win64/chrome.exe", ])}) { console.log(\`2026-07-31 ..... \${entry.replaceAll("/", "\\\\")}\`) } @@ -236,7 +323,7 @@ describe("bundled personal-server verifier", () => { ] ) - expect(entries).toHaveLength(12) + expect(entries).toHaveLength(19) expect(() => assertPackagedRuntime(entries, "DataConnect.exe") ).not.toThrow() @@ -267,17 +354,17 @@ describe("bundled personal-server verifier", () => { () => child ) - child.stdout.write(`${runtimeEntries.join("\n")}\n`) + child.stdout.write(`${windowsRuntimeEntries.join("\n")}\n`) child.stderr.end() child.emit("close", 0) setImmediate(() => { child.stdout.end( - "pdpp-node.exe\nresources/playwright-runner/dist/browsers/chromium-1228/chrome-win64/chrome.exe" + "pdpp-node.exe\nplaywright-runner/dist/browsers/chromium-1228/chrome-win64/chrome.exe" ) }) const entries = await entriesPromise - expect(entries).toHaveLength(12) + expect(entries).toHaveLength(19) expect(() => assertPackagedNode(entries, "DataConnect.exe", "windows") ).not.toThrow() @@ -293,7 +380,7 @@ describe("bundled personal-server verifier", () => { const temporaryRoot = mkdtempSync(join(process.cwd(), ".verify-deb-")) try { const packageRoot = join(temporaryRoot, "package") - const resources = join(packageRoot, "usr", "lib", "data-connect") + const resources = join(packageRoot, "usr", "lib", "DataConnect") mkdirSync(join(packageRoot, "DEBIAN"), { recursive: true }) writeFileSync( join(packageRoot, "DEBIAN", "control"), diff --git a/src-tauri/src/commands/connector_store.rs b/src-tauri/src/commands/connector_store.rs index 20e50ee3..e0c45bee 100644 --- a/src-tauri/src/commands/connector_store.rs +++ b/src-tauri/src/commands/connector_store.rs @@ -1,7 +1,13 @@ +use fs2::FileExt; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use std::fs; +use std::fs::{self, OpenOptions}; +use std::io::Write; use std::path::PathBuf; +use std::sync::Mutex; +use tempfile::NamedTempFile; + +static ACTIVE_CONNECTOR_MANIFEST_LOCK: Mutex<()> = Mutex::new(()); #[derive(Debug, Serialize, Deserialize, Clone, Default)] #[serde(rename_all = "camelCase")] @@ -57,6 +63,12 @@ pub fn get_active_manifest_path() -> Option { pub fn read_active_connector_manifest() -> Option { let manifest_path = get_active_manifest_path()?; + read_active_connector_manifest_from(&manifest_path) +} + +fn read_active_connector_manifest_from( + manifest_path: &std::path::Path, +) -> Option { if !manifest_path.exists() { return None; } @@ -65,20 +77,35 @@ pub fn read_active_connector_manifest() -> Option { serde_json::from_str(&content).ok() } -pub fn write_active_connector_manifest(manifest: &ActiveConnectorManifest) -> Result<(), String> { - let manifest_path = get_active_manifest_path().ok_or("Could not determine active manifest path")?; +fn write_active_connector_manifest_to( + manifest_path: &std::path::Path, + manifest: &ActiveConnectorManifest, +) -> Result<(), String> { if let Some(parent) = manifest_path.parent() { fs::create_dir_all(parent) .map_err(|e| format!("Failed to create connector manifest directory: {}", e))?; } - let temp_path = manifest_path.with_extension("json.tmp"); let content = serde_json::to_string_pretty(manifest) .map_err(|e| format!("Failed to serialize active connector manifest: {}", e))?; - - fs::write(&temp_path, content) + let parent = manifest_path + .parent() + .ok_or("Active connector manifest path has no parent directory")?; + let mut temp_file = NamedTempFile::new_in(parent).map_err(|e| { + format!( + "Failed to create active connector manifest temp file: {}", + e + ) + })?; + temp_file + .write_all(content.as_bytes()) .map_err(|e| format!("Failed to write active connector manifest: {}", e))?; - fs::rename(&temp_path, &manifest_path) + temp_file + .as_file() + .sync_all() + .map_err(|e| format!("Failed to sync active connector manifest: {}", e))?; + temp_file + .persist(manifest_path) .map_err(|e| format!("Failed to activate connector manifest: {}", e))?; Ok(()) @@ -88,3 +115,352 @@ pub fn get_active_connector_install(connector_id: &str) -> Option Result { + let manifest_path = + get_active_manifest_path().ok_or("Could not determine active manifest path")?; + update_active_connector_install_at(&manifest_path, install, policy) +} + +#[derive(Debug, Clone, Copy)] +enum ConnectorInstallUpdatePolicy { + ReplaceExisting, + RefreshBundledPathIfSameArtifact, +} + +fn update_active_connector_install_at( + manifest_path: &std::path::Path, + install: ActiveConnectorInstall, + policy: ConnectorInstallUpdatePolicy, +) -> Result { + let _guard = ACTIVE_CONNECTOR_MANIFEST_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let parent = manifest_path + .parent() + .ok_or("Active connector manifest path has no parent directory")?; + fs::create_dir_all(parent) + .map_err(|e| format!("Failed to create connector manifest directory: {}", e))?; + let lock_path = manifest_path.with_extension("json.lock"); + let lock_file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(&lock_path) + .map_err(|e| format!("Failed to open active connector manifest lock: {}", e))?; + lock_file + .lock_exclusive() + .map_err(|e| format!("Failed to lock active connector manifest: {}", e))?; + let mut manifest = + read_active_connector_manifest_from(manifest_path).unwrap_or(ActiveConnectorManifest { + version: "1.0".to_string(), + updated_at: chrono::Utc::now().to_rfc3339(), + connectors: HashMap::new(), + }); + if let Some(existing) = manifest.connectors.get_mut(&install.connector_id) { + match policy { + ConnectorInstallUpdatePolicy::ReplaceExisting => {} + ConnectorInstallUpdatePolicy::RefreshBundledPathIfSameArtifact => { + if !same_bundled_artifact(existing, &install) { + return Ok(false); + } + existing.root_path = install.root_path; + existing.metadata_relative_path = install.metadata_relative_path; + existing.script_relative_path = install.script_relative_path; + existing.manifest_path = install.manifest_path; + existing.entrypoint_path = install.entrypoint_path; + existing.provenance_path = install.provenance_path; + manifest.updated_at = chrono::Utc::now().to_rfc3339(); + write_active_connector_manifest_to(manifest_path, &manifest)?; + return Ok(true); + } + } + } + manifest.updated_at = chrono::Utc::now().to_rfc3339(); + manifest + .connectors + .insert(install.connector_id.clone(), install); + write_active_connector_manifest_to(manifest_path, &manifest)?; + Ok(true) +} + +fn same_bundled_artifact( + existing: &ActiveConnectorInstall, + install: &ActiveConnectorInstall, +) -> bool { + existing.connector_id == install.connector_id + && existing.company == install.company + && existing.version == install.version + && existing.artifact_kind == install.artifact_kind + && existing.manifest_path == install.manifest_path + && existing.entrypoint_path == install.entrypoint_path + && existing.provenance_path == install.provenance_path + && required_equal(&existing.manifest_sha256, &install.manifest_sha256) + && required_equal(&existing.entrypoint_sha256, &install.entrypoint_sha256) + && required_equal(&existing.provenance_sha256, &install.provenance_sha256) +} + +fn required_equal(existing: &Option, install: &Option) -> bool { + matches!((existing, install), (Some(existing), Some(install)) if existing == install) +} + +pub fn replace_active_connector_install(install: ActiveConnectorInstall) -> Result<(), String> { + update_active_connector_install(install, ConnectorInstallUpdatePolicy::ReplaceExisting) + .map(|_| ()) +} + +pub fn activate_bundled_connector_install(install: ActiveConnectorInstall) -> Result { + update_active_connector_install( + install, + ConnectorInstallUpdatePolicy::RefreshBundledPathIfSameArtifact, + ) +} + +#[cfg(test)] +mod tests { + use super::{ + read_active_connector_manifest_from, update_active_connector_install_at, + ActiveConnectorInstall, ConnectorInstallUpdatePolicy, + }; + use fs2::FileExt; + use std::fs::OpenOptions; + use std::process::Command; + use std::sync::{Arc, Barrier}; + use std::time::{Duration, Instant}; + use tempfile::tempdir; + + fn install(version: &str, root_path: &str) -> ActiveConnectorInstall { + ActiveConnectorInstall { + connector_id: "github-pdpp".to_string(), + company: "github".to_string(), + version: version.to_string(), + root_path: root_path.to_string(), + metadata_relative_path: "profile/collection-profile.json".to_string(), + script_relative_path: "dist/collection-profile.mjs".to_string(), + artifact_kind: Some("pdpp-collection-profile".to_string()), + manifest_path: Some("profile/collection-profile.json".to_string()), + entrypoint_path: Some("dist/collection-profile.mjs".to_string()), + entrypoint_sha256: Some("sha256:entrypoint".to_string()), + manifest_sha256: Some("sha256:manifest".to_string()), + provenance_path: Some("provenance.json".to_string()), + provenance_sha256: Some("sha256:provenance".to_string()), + } + } + + fn selected(manifest_path: &std::path::Path) -> ActiveConnectorInstall { + read_active_connector_manifest_from(manifest_path) + .expect("active connector manifest") + .connectors + .get("github-pdpp") + .expect("selection") + .clone() + } + + #[test] + fn bundled_activation_cannot_replace_a_concurrent_user_selection() { + let temp = tempdir().expect("manifest tempdir"); + + for iteration in 0..32 { + let manifest_path = temp.path().join(format!("active-{iteration}.json")); + let barrier = Arc::new(Barrier::new(3)); + let bundled_path = manifest_path.clone(); + let bundled_barrier = Arc::clone(&barrier); + let bundled = std::thread::spawn(move || { + bundled_barrier.wait(); + update_active_connector_install_at( + &bundled_path, + install("0.1.0", "/bundled/github-pdpp"), + ConnectorInstallUpdatePolicy::RefreshBundledPathIfSameArtifact, + ) + }); + let user_path = manifest_path.clone(); + let user_barrier = Arc::clone(&barrier); + let user = std::thread::spawn(move || { + user_barrier.wait(); + update_active_connector_install_at( + &user_path, + install("9.9.9", "/user/github-pdpp"), + ConnectorInstallUpdatePolicy::ReplaceExisting, + ) + }); + + barrier.wait(); + bundled + .join() + .expect("bundled thread") + .expect("bundled write"); + user.join().expect("user thread").expect("user write"); + + let selected = selected(&manifest_path); + assert_eq!(selected.version, "9.9.9"); + assert_eq!(selected.root_path, "/user/github-pdpp"); + } + } + + #[test] + fn bundled_activation_refreshes_path_for_same_exact_artifact() { + let temp = tempdir().expect("manifest tempdir"); + let manifest_path = temp.path().join("connectors-active.json"); + let mut stale = install("0.1.0", "/tmp/.mount-old/github-pdpp"); + stale.metadata_relative_path = "old-profile/collection-profile.json".to_string(); + stale.script_relative_path = "old-dist/collection-profile.mjs".to_string(); + update_active_connector_install_at( + &manifest_path, + stale, + ConnectorInstallUpdatePolicy::ReplaceExisting, + ) + .expect("stale bundled install"); + + let mut fresh = install("0.1.0", "/tmp/.mount-new/github-pdpp"); + fresh.metadata_relative_path = "profile/collection-profile.json".to_string(); + fresh.script_relative_path = "dist/collection-profile.mjs".to_string(); + assert!(update_active_connector_install_at( + &manifest_path, + fresh, + ConnectorInstallUpdatePolicy::RefreshBundledPathIfSameArtifact, + ) + .expect("fresh bundled activation")); + + let selected = selected(&manifest_path); + assert_eq!(selected.root_path, "/tmp/.mount-new/github-pdpp"); + assert_eq!( + selected.metadata_relative_path, + "profile/collection-profile.json" + ); + assert_eq!(selected.script_relative_path, "dist/collection-profile.mjs"); + assert_eq!(selected.version, "0.1.0"); + } + + #[test] + fn bundled_activation_preserves_distinct_user_install() { + let temp = tempdir().expect("manifest tempdir"); + let manifest_path = temp.path().join("connectors-active.json"); + update_active_connector_install_at( + &manifest_path, + install("9.9.9", "/user/github-pdpp"), + ConnectorInstallUpdatePolicy::ReplaceExisting, + ) + .expect("user install"); + + assert!(!update_active_connector_install_at( + &manifest_path, + install("0.1.0", "/bundled/github-pdpp"), + ConnectorInstallUpdatePolicy::RefreshBundledPathIfSameArtifact, + ) + .expect("bundled activation")); + + let selected = selected(&manifest_path); + assert_eq!(selected.version, "9.9.9"); + assert_eq!(selected.root_path, "/user/github-pdpp"); + } + + #[test] + fn bundled_activation_preserves_same_version_with_changed_or_missing_hash() { + let temp = tempdir().expect("manifest tempdir"); + let changed_hash_path = temp.path().join("changed-hash.json"); + update_active_connector_install_at( + &changed_hash_path, + install("0.1.0", "/user/github-pdpp"), + ConnectorInstallUpdatePolicy::ReplaceExisting, + ) + .expect("user install"); + + let mut changed = install("0.1.0", "/bundled/github-pdpp"); + changed.manifest_sha256 = Some("sha256:changed".to_string()); + assert!(!update_active_connector_install_at( + &changed_hash_path, + changed, + ConnectorInstallUpdatePolicy::RefreshBundledPathIfSameArtifact, + ) + .expect("changed hash bundled activation")); + assert_eq!(selected(&changed_hash_path).root_path, "/user/github-pdpp"); + + let missing_hash_path = temp.path().join("missing-hash.json"); + let mut missing = install("0.1.0", "/user/github-pdpp"); + missing.entrypoint_sha256 = None; + update_active_connector_install_at( + &missing_hash_path, + missing, + ConnectorInstallUpdatePolicy::ReplaceExisting, + ) + .expect("user install without hash"); + + assert!(!update_active_connector_install_at( + &missing_hash_path, + install("0.1.0", "/bundled/github-pdpp"), + ConnectorInstallUpdatePolicy::RefreshBundledPathIfSameArtifact, + ) + .expect("missing hash bundled activation")); + assert_eq!(selected(&missing_hash_path).root_path, "/user/github-pdpp"); + } + + #[test] + fn cross_process_manifest_update_helper() { + let Some(manifest_path) = std::env::var_os("DATACONNECT_TEST_MANIFEST_PATH") else { + return; + }; + let ready_path = + std::env::var_os("DATACONNECT_TEST_READY_PATH").expect("cross-process ready path"); + std::fs::write(&ready_path, b"ready").expect("cross-process ready marker"); + update_active_connector_install_at( + std::path::Path::new(&manifest_path), + install("9.9.9", "/user/github-pdpp"), + ConnectorInstallUpdatePolicy::ReplaceExisting, + ) + .expect("cross-process user update"); + } + + #[test] + fn manifest_updates_take_the_cross_process_file_lock() { + let temp = tempdir().expect("manifest tempdir"); + let manifest_path = temp.path().join("connectors-active.json"); + update_active_connector_install_at( + &manifest_path, + install("0.1.0", "/bundled/github-pdpp"), + ConnectorInstallUpdatePolicy::RefreshBundledPathIfSameArtifact, + ) + .expect("bundled install"); + + let lock_path = manifest_path.with_extension("json.lock"); + let lock_file = OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .expect("active manifest lock"); + lock_file.lock_exclusive().expect("parent file lock"); + + let ready_path = temp.path().join("child-ready"); + let mut child = Command::new(std::env::current_exe().expect("test executable")) + .args([ + "--exact", + "commands::connector_store::tests::cross_process_manifest_update_helper", + "--nocapture", + ]) + .env("DATACONNECT_TEST_MANIFEST_PATH", &manifest_path) + .env("DATACONNECT_TEST_READY_PATH", &ready_path) + .spawn() + .expect("cross-process update child"); + + let deadline = Instant::now() + Duration::from_secs(5); + while !ready_path.exists() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(ready_path.exists(), "child did not reach manifest update"); + std::thread::sleep(Duration::from_millis(100)); + assert!( + child.try_wait().expect("child status").is_none(), + "child update bypassed the held cross-process lock" + ); + + FileExt::unlock(&lock_file).expect("release parent file lock"); + assert!(child.wait().expect("child completion").success()); + let manifest = + read_active_connector_manifest_from(&manifest_path).expect("active connector manifest"); + let selected = manifest.connectors.get("github-pdpp").expect("selection"); + assert_eq!(selected.version, "9.9.9"); + assert_eq!(selected.root_path, "/user/github-pdpp"); + } +} diff --git a/src-tauri/src/commands/server.rs b/src-tauri/src/commands/server.rs index 09131608..02f4c1c0 100644 --- a/src-tauri/src/commands/server.rs +++ b/src-tauri/src/commands/server.rs @@ -3,6 +3,8 @@ use std::path::PathBuf; use std::sync::Mutex; use tauri::{AppHandle, Emitter, Manager, WebviewWindow}; +use super::updates::activate_bundled_pdpp_connectors; + /// Send a signal to an entire process group (negative PID). /// Falls back to single-process signal if pgid lookup fails. #[cfg(unix)] @@ -161,6 +163,13 @@ pub async fn start_personal_server( } }; + // The Personal Server reads the active PDPP install once at startup. + // Seed verified bundled installs before it composes its routes. + if let Err(error) = activate_bundled_pdpp_connectors(&app) { + clear_starting(); + return Err(error); + } + // Kill any stale personal-server from a previous unclean exit #[cfg(unix)] { diff --git a/src-tauri/src/commands/updates.rs b/src-tauri/src/commands/updates.rs index 8720de6e..e1cc28f9 100644 --- a/src-tauri/src/commands/updates.rs +++ b/src-tauri/src/commands/updates.rs @@ -12,9 +12,9 @@ use tauri::{AppHandle, Manager}; use tempfile::tempdir_in; use super::connector_store::{ - get_active_connector_install, get_connectors_store_dir, get_legacy_user_connectors_dir, - read_active_connector_manifest, write_active_connector_manifest, ActiveConnectorInstall, - ActiveConnectorManifest, + activate_bundled_connector_install, get_active_connector_install, get_connectors_store_dir, + get_legacy_user_connectors_dir, read_active_connector_manifest, + replace_active_connector_install, ActiveConnectorInstall, }; const DEFAULT_INDEX_URL: &str = @@ -179,21 +179,46 @@ struct PdppArtifactBundle { provenance: Vec, } +#[derive(Debug, Deserialize)] +struct BundledConnectorLock { + connectors: Vec, +} + fn get_user_connectors_dir() -> Option { get_legacy_user_connectors_dir() } -fn activate_connector_install(install: ActiveConnectorInstall) -> Result<(), String> { - let mut manifest = read_active_connector_manifest().unwrap_or(ActiveConnectorManifest { - version: "1.0".to_string(), - updated_at: chrono::Utc::now().to_rfc3339(), - connectors: HashMap::new(), - }); - manifest.updated_at = chrono::Utc::now().to_rfc3339(); - manifest - .connectors - .insert(install.connector_id.clone(), install); - write_active_connector_manifest(&manifest) +fn bundled_pdpp_connector_installs( + bundled_connectors_dir: &Path, +) -> Result, String> { + let lock_path = bundled_connectors_dir.join("lock.json"); + let lock_bytes = fs::read(&lock_path) + .map_err(|e| format!("Failed to read bundled connector lock: {}", e))?; + let lock: BundledConnectorLock = serde_json::from_slice(&lock_bytes) + .map_err(|e| format!("Failed to parse bundled connector lock: {}", e))?; + + lock.connectors + .iter() + .filter_map(|connector| match connector { + IndexedConnector::PdppCollectionProfile(connector) => Some(connector), + IndexedConnector::Legacy(_) => None, + }) + .map(|connector| { + let connector_id = + safe_store_segment(&connector.common.connector_id, "PDPP connector id")?; + let install_root = bundled_connectors_dir + .join("collection-profiles") + .join(connector_id); + active_pdpp_install_at(connector, &install_root) + }) + .collect() +} + +pub(crate) fn activate_bundled_pdpp_connectors(app: &AppHandle) -> Result<(), String> { + for install in bundled_pdpp_connector_installs(&get_bundled_connectors_dir(&app))? { + activate_bundled_connector_install(install)?; + } + Ok(()) } fn get_bundled_connectors_dir(app: &AppHandle) -> PathBuf { @@ -930,7 +955,7 @@ fn install_verified_connector_artifact( let store_dir = get_connectors_store_dir().ok_or("Could not determine connectors store directory")?; let install = install_verified_connector_artifact_into(connector, artifact_bytes, &store_dir)?; - activate_connector_install(install) + replace_active_connector_install(install) } fn install_verified_connector_artifact_into( @@ -1122,13 +1147,6 @@ fn install_verified_pdpp_connector( let connector_store_dir = install_root .parent() .ok_or("Connector install root is missing its store directory")?; - let manifest_relative = - normalize_nonempty_artifact_path(&connector.manifest_path, "PDPP manifest path")?; - let entrypoint_relative = - normalize_nonempty_artifact_path(&connector.entrypoint_path, "PDPP entrypoint path")?; - let provenance_relative = - normalize_nonempty_artifact_path(&connector.provenance_path, "PDPP provenance path")?; - if !install_root.exists() { log::info!( "Installing PDPP connector artifact for {} to {:?}", @@ -1138,25 +1156,57 @@ fn install_verified_pdpp_connector( promote_staged_install(connector_store_dir, &install_root, |staged| { write_pdpp_artifact_bundle(staged, &bundle) })?; - } else { - verify_installed_file( - &install_root.join(&manifest_relative), - &common.manifest_sha256, - "installed PDPP manifest", - )?; - verify_installed_file( - &install_root.join(&entrypoint_relative), - &connector.entrypoint_sha256, - "installed PDPP entrypoint", - )?; - verify_installed_file( - &install_root.join(&provenance_relative), - &connector.provenance_sha256, - "installed PDPP provenance", - )?; } - let install = ActiveConnectorInstall { + let install = active_pdpp_install_at(connector, &install_root)?; + + log::info!( + "=== Successfully installed PDPP connector: {} ===", + common.connector_id + ); + Ok(install) +} + +fn active_pdpp_install_at( + connector: &PdppIndexedConnector, + install_root: &Path, +) -> Result { + let common = &connector.common; + let manifest_relative = + normalize_nonempty_artifact_path(&connector.manifest_path, "PDPP manifest path")?; + let entrypoint_relative = + normalize_nonempty_artifact_path(&connector.entrypoint_path, "PDPP entrypoint path")?; + let provenance_relative = + normalize_nonempty_artifact_path(&connector.provenance_path, "PDPP provenance path")?; + + verify_installed_file( + &install_root.join(&manifest_relative), + &common.manifest_sha256, + "installed PDPP manifest", + )?; + verify_installed_file( + &install_root.join(&entrypoint_relative), + &connector.entrypoint_sha256, + "installed PDPP entrypoint", + )?; + verify_installed_file( + &install_root.join(&provenance_relative), + &connector.provenance_sha256, + "installed PDPP provenance", + )?; + + let manifest_bytes = fs::read(install_root.join(&manifest_relative)) + .map_err(|e| format!("Failed to read installed PDPP manifest: {}", e))?; + let manifest: PdppManifestIdentity = serde_json::from_slice(&manifest_bytes) + .map_err(|e| format!("Failed to parse installed PDPP manifest: {}", e))?; + if manifest.version != common.version { + return Err(format!( + "Installed PDPP version mismatch. Expected {}, got {}", + common.version, manifest.version + )); + } + + Ok(ActiveConnectorInstall { connector_id: common.connector_id.clone(), company: common.company.clone(), version: common.version.clone(), @@ -1170,13 +1220,7 @@ fn install_verified_pdpp_connector( manifest_sha256: Some(common.manifest_sha256.clone()), provenance_path: Some(provenance_relative.to_string_lossy().into_owned()), provenance_sha256: Some(connector.provenance_sha256.clone()), - }; - - log::info!( - "=== Successfully installed PDPP connector: {} ===", - common.connector_id - ); - Ok(install) + }) } fn verify_named_checksum(bytes: &[u8], expected: &str, label: &str) -> Result<(), String> { @@ -1283,16 +1327,17 @@ fn scan_connectors_dir_no_overwrite(dir: &PathBuf, versions: &mut HashMap>(); + connector_ids.sort_unstable(); + assert_eq!(connector_ids, ["chatgpt-pdpp", "github-pdpp"]); + } + #[test] fn index_rejects_unknown_explicit_artifact_kind() { let manifest = br#"{"version":"0.5.0"}"#; diff --git a/src-tauri/tauri-config.test.ts b/src-tauri/tauri-config.test.ts index 5c9720cd..526183ae 100644 --- a/src-tauri/tauri-config.test.ts +++ b/src-tauri/tauri-config.test.ts @@ -44,6 +44,23 @@ describe("tauri manual-install config", () => { expect(document.bundle?.resources?.["../pdpp-runtime/**/*"]).toBeUndefined() }) + it("bundles the pinned GitHub and ChatGPT PDPP profiles", () => { + const filePath = resolve(process.cwd(), "src-tauri/tauri.conf.json") + const document = JSON.parse(readFileSync(filePath, "utf-8")) as { + bundle?: { resources?: Record } + } + + expect( + document.bundle?.resources?.["../connectors/collection-profiles/"] + ).toBe("connectors/collection-profiles/") + expect( + document.bundle?.resources?.["../connectors/collection-profiles/**/*"] + ).toBeUndefined() + expect(document.bundle?.resources?.["../connectors/lock.json"]).toBe( + "connectors/lock.json" + ) + }) + it("stages PDPP runtime dependencies before production Tauri packaging", () => { const filePath = resolve(process.cwd(), "src-tauri/tauri.conf.json") const document = JSON.parse(readFileSync(filePath, "utf-8")) as { diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 5401bff9..8c93a500 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -60,6 +60,8 @@ "../connectors/oura/**/*": "connectors/oura/", "../connectors/shopify/**/*": "connectors/shopify/", "../connectors/wholefoods/**/*": "connectors/wholefoods/", + "../connectors/collection-profiles/": "connectors/collection-profiles/", + "../connectors/lock.json": "connectors/lock.json", "../playwright-runner/dist/": "playwright-runner/dist/", "../personal-server/dist/personal-server*": "personal-server/dist/", "../personal-server/dist/node_modules/": "personal-server/dist/node_modules/",