diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d79dd31d2..f97742eb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,6 +108,47 @@ jobs: test -f apps/desktop/dist-electron/preload.js grep -nE "desktopBridge|getWsUrl|PICK_FOLDER_CHANNEL|wsUrl" apps/desktop/dist-electron/preload.js + html_preview_platform: + name: HTML Preview Platform Smoke (${{ matrix.label }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - label: Linux + runner: ubuntu-24.04 + - label: Windows + runner: windows-2022 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version-file: package.json + + - name: Setup Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: package.json + + - name: Install Linux listener tools + if: runner.os == 'Linux' + shell: bash + run: | + command -v lsof >/dev/null 2>&1 || { + sudo apt-get update + sudo apt-get install -y lsof + } + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Exercise native localhost preview and discovery + run: bun run --cwd apps/server test:html-preview-platform + windows_process: name: Windows Process Regression runs-on: windows-2022 diff --git a/apps/desktop/src/artifactPreviewPolicy.test.ts b/apps/desktop/src/artifactPreviewPolicy.test.ts new file mode 100644 index 000000000..e38bd878e --- /dev/null +++ b/apps/desktop/src/artifactPreviewPolicy.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest"; + +import { + artifactPreviewNavigationAllowed, + artifactPreviewRequestAllowed, +} from "./artifactPreviewPolicy"; + +const ORIGIN = "http://g-123.preview.localhost:5000"; + +describe("artifactPreviewPolicy", () => { + it("allows only same-capability resources and inert embedded data", () => { + expect( + artifactPreviewRequestAllowed({ + url: `${ORIGIN}/assets/app.js`, + allowedOrigin: ORIGIN, + resourceType: "script", + }), + ).toBe(true); + expect( + artifactPreviewRequestAllowed({ + url: "https://example.com/tracker.js", + allowedOrigin: ORIGIN, + resourceType: "script", + }), + ).toBe(false); + expect( + artifactPreviewRequestAllowed({ + url: `${ORIGIN}/socket`, + allowedOrigin: ORIGIN, + resourceType: "webSocket", + }), + ).toBe(false); + for (const resourceType of ["worker", "sharedWorker", "serviceWorker"]) { + expect( + artifactPreviewRequestAllowed({ + url: `${ORIGIN}/worker.js`, + allowedOrigin: ORIGIN, + resourceType, + }), + ).toBe(false); + } + }); + + it("denies subframes and cross-origin top-level navigation", () => { + expect( + artifactPreviewNavigationAllowed({ + url: `${ORIGIN}/`, + allowedOrigin: ORIGIN, + isMainFrame: true, + }), + ).toBe(true); + expect( + artifactPreviewNavigationAllowed({ + url: "https://example.com/", + allowedOrigin: ORIGIN, + isMainFrame: true, + }), + ).toBe(false); + expect( + artifactPreviewNavigationAllowed({ + url: `${ORIGIN}/frame.html`, + allowedOrigin: ORIGIN, + isMainFrame: false, + }), + ).toBe(false); + }); +}); diff --git a/apps/desktop/src/artifactPreviewPolicy.ts b/apps/desktop/src/artifactPreviewPolicy.ts new file mode 100644 index 000000000..5fd508f25 --- /dev/null +++ b/apps/desktop/src/artifactPreviewPolicy.ts @@ -0,0 +1,47 @@ +// FILE: artifactPreviewPolicy.ts +// Purpose: Pure Electron-level network and navigation policy for untrusted artifact tabs. + +const DENIED_RESOURCE_TYPES = new Set([ + "subFrame", + "webSocket", + "object", + "ping", + "worker", + "sharedWorker", + "serviceWorker", +]); + +export function artifactPreviewRequestAllowed(input: { + url: string; + allowedOrigin: string; + resourceType: string; +}): boolean { + if (DENIED_RESOURCE_TYPES.has(input.resourceType)) { + return false; + } + try { + const requestUrl = new URL(input.url); + return ( + requestUrl.protocol === "data:" || + requestUrl.protocol === "blob:" || + requestUrl.origin === input.allowedOrigin + ); + } catch { + return false; + } +} + +export function artifactPreviewNavigationAllowed(input: { + url: string; + allowedOrigin: string; + isMainFrame: boolean; +}): boolean { + if (!input.isMainFrame) { + return false; + } + try { + return new URL(input.url).origin === input.allowedOrigin; + } catch { + return false; + } +} diff --git a/apps/desktop/src/browserManager.ts b/apps/desktop/src/browserManager.ts index 8d33c2e28..fb79fb306 100644 --- a/apps/desktop/src/browserManager.ts +++ b/apps/desktop/src/browserManager.ts @@ -16,6 +16,10 @@ import { WebContentsView, } from "electron"; import type { WebContents } from "electron"; +import { + artifactPreviewNavigationAllowed, + artifactPreviewRequestAllowed, +} from "./artifactPreviewPolicy"; import type { BrowserAttachWebviewInput, BrowserCaptureScreenshotResult, @@ -28,6 +32,7 @@ import type { BrowserPanelBounds, BrowserSetPanelBoundsInput, BrowserTabInput, + BrowserTabKind, BrowserTabState, BrowserThreadInput, ThreadBrowserState, @@ -36,6 +41,8 @@ import type { import { isBrowserCopyLinkChord } from "@synara/shared/browserShortcuts"; import { BROWSER_BLANK_URL as ABOUT_BLANK_URL, + BROWSER_WEB_SESSION_PARTITION, + browserSessionPartition, buildAcceptLanguageHeader, buildChromeClientHints, classifyBrowserWindowOpen, @@ -45,7 +52,7 @@ import { resolveCopyableBrowserTabUrl, } from "@synara/shared/browserSession"; -const BROWSER_SESSION_PARTITION = "persist:scient-browser"; +const BROWSER_SESSION_PARTITION = BROWSER_WEB_SESSION_PARTITION; const BROWSER_INACTIVE_TAB_SUSPEND_DELAY_MS = 1_500; const BROWSER_INACTIVE_TAB_SUSPEND_DELAY_PRESSURED_MS = 400; const BROWSER_MAX_WARM_INACTIVE_RUNTIMES_PER_THREAD = 1; @@ -88,6 +95,17 @@ interface PendingRuntimeSync { const LIVE_TAB_STATUS: BrowserTabState["status"] = "live"; const SUSPENDED_TAB_STATUS: BrowserTabState["status"] = "suspended"; +function safeUrlOrigin(value: string | null | undefined): string | null { + if (!value) { + return null; + } + try { + return new URL(value).origin; + } catch { + return null; + } +} + interface BrowserPerformanceSnapshot { counters: { setPanelBoundsCalls: number; @@ -116,10 +134,16 @@ export interface BrowserUseCdpEvent { params?: unknown; } -function createBrowserTab(url = ABOUT_BLANK_URL): BrowserTabState { +function createBrowserTab( + url = ABOUT_BLANK_URL, + kind: BrowserTabKind = "web", + displayUrl?: string, +): BrowserTabState { return { id: Crypto.randomUUID(), + kind, url, + displayUrl: displayUrl?.trim() || null, title: defaultTitleForUrl(url), status: SUSPENDED_TAB_STATUS, isLoading: false, @@ -260,6 +284,7 @@ export class DesktopBrowserManager { private readonly popupRuntimes = new Map(); private spoofedUserAgent: string | null = null; private sessionConfigured = false; + private readonly previewSessionsConfigured = new Set(); private readonly tabSuspendTimers = new Map>(); private readonly suspendTimers = new Map>(); private runtimeSyncFlushScheduled = false; @@ -353,6 +378,55 @@ export class DesktopBrowserManager { } } + private ensurePreviewSessionConfigured(partition: string, artifactOrigin?: string): void { + if (this.previewSessionsConfigured.has(partition)) { + return; + } + const partitionSession = session.fromPartition(partition); + partitionSession.setUserAgent(this.resolveSpoofedUserAgent()); + partitionSession.setPermissionCheckHandler(() => false); + partitionSession.setPermissionRequestHandler((_webContents, _permission, callback) => { + callback(false); + }); + if (artifactOrigin) { + partitionSession.webRequest.onBeforeRequest((details, callback) => { + callback({ + cancel: !artifactPreviewRequestAllowed({ + url: details.url, + allowedOrigin: artifactOrigin, + resourceType: details.resourceType, + }), + }); + }); + partitionSession.on("will-download", (event) => { + event.preventDefault(); + }); + } + this.previewSessionsConfigured.add(partition); + } + + private configureTabSession(threadId: ThreadId, tab: BrowserTabState): void { + if (tab.kind === "web") { + return; + } + const partition = browserSessionPartition(tab.kind, threadId, tab.id); + const artifactOrigin = + tab.kind === "artifact" ? (safeUrlOrigin(tab.url) ?? undefined) : undefined; + this.ensurePreviewSessionConfigured(partition, artifactOrigin); + } + + private clearArtifactSession(threadId: ThreadId, tab: BrowserTabState): void { + if (tab.kind !== "artifact") { + return; + } + const partition = browserSessionPartition("artifact", threadId, tab.id); + const artifactSession = session.fromPartition(partition); + this.previewSessionsConfigured.delete(partition); + void Promise.all([artifactSession.clearStorageData(), artifactSession.clearCache()]).catch( + () => undefined, + ); + } + // Options for an OAuth/sign-in popup. Stays on the shared persistent partition and keeps the // hardened sandbox; `window.opener` is preserved by Electron because we allow (not deny) the // open, which is what lets the auth callback `postMessage`/`window.close()` back to the page. @@ -601,12 +675,33 @@ export class DesktopBrowserManager { } open(input: BrowserOpenInput): ThreadBrowserState { - const state = this.ensureWorkspace(input.threadId, input.initialUrl); + const requestedKind = input.kind ?? "web"; + const state = this.ensureWorkspace( + input.threadId, + input.initialUrl, + requestedKind, + input.displayUrl, + ); const didChange = !state.open; state.open = true; const nextInitialUrl = input.initialUrl ? normalizeUrlInput(input.initialUrl) : null; const activeTab = nextInitialUrl ? this.getActiveTab(state) : null; + if ( + nextInitialUrl && + activeTab && + (activeTab.kind !== requestedKind || + (requestedKind === "artifact" && activeTab.url !== nextInitialUrl)) + ) { + return this.newTab({ + threadId: input.threadId, + url: nextInitialUrl, + kind: requestedKind, + ...(input.displayUrl ? { displayUrl: input.displayUrl } : {}), + activate: true, + }); + } if (nextInitialUrl && activeTab && activeTab.url !== nextInitialUrl) { + activeTab.displayUrl = input.displayUrl?.trim() || null; return this.navigate({ threadId: input.threadId, tabId: activeTab.id, @@ -647,6 +742,7 @@ export class DesktopBrowserManager { this.destroyThreadRuntimes(input.threadId); const state = this.getOrCreateState(input.threadId); + const closedArtifactTabs = state.tabs.filter((tab) => tab.kind === "artifact"); state.open = false; state.activeTabId = null; state.tabs = []; @@ -654,6 +750,9 @@ export class DesktopBrowserManager { this.markThreadStateChanged(input.threadId); this.lastEmittedVersionByThreadId.delete(input.threadId); this.emitState(input.threadId); + for (const tab of closedArtifactTabs) { + this.clearArtifactSession(input.threadId, tab); + } return this.snapshotThreadState(input.threadId, state); } @@ -823,6 +922,9 @@ export class DesktopBrowserManager { const state = this.ensureWorkspace(input.threadId); const tab = this.resolveTab(state, input.tabId); const nextUrl = normalizeUrlInput(input.url); + if (tab.kind === "artifact" && safeUrlOrigin(nextUrl) !== safeUrlOrigin(tab.url)) { + throw new Error("Artifact previews cannot navigate outside their capability origin."); + } tab.url = nextUrl; tab.title = defaultTitleForUrl(nextUrl); tab.lastCommittedUrl = null; @@ -884,7 +986,12 @@ export class DesktopBrowserManager { newTab(input: BrowserNewTabInput): ThreadBrowserState { const state = this.ensureWorkspace(input.threadId); - const tab = createBrowserTab(normalizeUrlInput(input.url)); + const tab = createBrowserTab( + normalizeUrlInput(input.url), + input.kind ?? "web", + input.displayUrl, + ); + this.configureTabSession(input.threadId, tab); state.tabs = [...state.tabs, tab]; if (input.activate !== false || !state.activeTabId) { state.activeTabId = tab.id; @@ -908,6 +1015,7 @@ export class DesktopBrowserManager { closeTab(input: BrowserTabInput): ThreadBrowserState { const state = this.ensureWorkspace(input.threadId); + const closedTab = state.tabs.find((tab) => tab.id === input.tabId); const nextTabs = state.tabs.filter((tab) => tab.id !== input.tabId); if (nextTabs.length === state.tabs.length) { return this.snapshotThreadState(input.threadId, state); @@ -916,6 +1024,9 @@ export class DesktopBrowserManager { this.closePopupWindowsForTab(input.threadId, input.tabId); this.destroyRuntime(input.threadId, input.tabId); state.tabs = nextTabs; + if (closedTab) { + this.clearArtifactSession(input.threadId, closedTab); + } if (nextTabs.length === 0) { // Closing the last tab keeps the browser open on a fresh blank tab (the same state @@ -1509,9 +1620,15 @@ export class DesktopBrowserManager { } private createLiveRuntime(threadId: ThreadId, tabId: string): LiveTabRuntime { + const state = this.ensureWorkspace(threadId); + const tab = this.resolveTab(state, tabId); + const partition = browserSessionPartition(tab.kind, threadId, tab.id); + if (tab.kind !== "web") { + this.configureTabSession(threadId, tab); + } const view = new WebContentsView({ webPreferences: { - partition: BROWSER_SESSION_PARTITION, + partition, contextIsolation: true, nodeIntegration: false, sandbox: true, @@ -1532,6 +1649,10 @@ export class DesktopBrowserManager { private configureRuntimeWebContents(runtime: LiveTabRuntime): void { const { threadId, tabId, webContents } = runtime; + const state = this.ensureWorkspace(threadId); + const sourceTab = this.getTab(state, tabId); + const tabKind = sourceTab?.kind ?? "web"; + const artifactOrigin = tabKind === "artifact" ? safeUrlOrigin(sourceTab?.url) : null; // Belt-and-suspenders alongside the session-level UA: also covers an adopted renderer // for any navigation after it attaches. @@ -1542,17 +1663,23 @@ export class DesktopBrowserManager { const isWebUrl = url.startsWith("http://") || url.startsWith("https://") || url === ABOUT_BLANK_URL; if (!isWebUrl) { - void shell.openExternal(url); + if (tabKind === "web") { + void shell.openExternal(url); + } return { action: "deny" }; } - const kind = classifyBrowserWindowOpen({ + if (tabKind === "artifact") { + return { action: "deny" }; + } + + const windowKind = classifyBrowserWindowOpen({ url, frameName: details.frameName, features: details.features, disposition: details.disposition, }); - if (kind === "popup") { + if (tabKind === "web" && windowKind === "popup") { // Allow (don't deny) so Electron creates a real child window that keeps // `window.opener`, which the OAuth callback needs to message the page back. return { @@ -1564,6 +1691,7 @@ export class DesktopBrowserManager { this.newTab({ threadId, url, + kind: tabKind, activate: true, }); const bounds = this.getVisibleBoundsForThread(threadId); @@ -1573,6 +1701,26 @@ export class DesktopBrowserManager { return { action: "deny" }; }); + if (artifactOrigin) { + const blockArtifactFrameNavigation = ( + event: Electron.Event, + ) => { + if ( + !artifactPreviewNavigationAllowed({ + url: event.url, + allowedOrigin: artifactOrigin, + isMainFrame: event.isMainFrame, + }) + ) { + event.preventDefault(); + } + }; + webContents.on("will-frame-navigate", blockArtifactFrameNavigation); + runtime.listenerDisposers.push(() => { + webContents.removeListener("will-frame-navigate", blockArtifactFrameNavigation); + }); + } + const didCreateWindow = (childWindow: BrowserWindow) => { this.registerOAuthPopupWindow(childWindow, { threadId, tabId }); }; @@ -1944,11 +2092,17 @@ export class DesktopBrowserManager { return BROWSER_INACTIVE_TAB_SUSPEND_DELAY_MS; } - private ensureWorkspace(threadId: ThreadId, initialUrl?: string): ThreadBrowserState { + private ensureWorkspace( + threadId: ThreadId, + initialUrl?: string, + kind: BrowserTabKind = "web", + displayUrl?: string, + ): ThreadBrowserState { this.ensureSessionConfigured(); const state = this.getOrCreateState(threadId); if (state.tabs.length === 0) { - const initialTab = createBrowserTab(normalizeUrlInput(initialUrl)); + const initialTab = createBrowserTab(normalizeUrlInput(initialUrl), kind, displayUrl); + this.configureTabSession(threadId, initialTab); state.tabs = [initialTab]; state.activeTabId = initialTab.id; } @@ -1995,6 +2149,9 @@ export class DesktopBrowserManager { ): string | null { const state = this.states.get(threadId); const tab = state ? this.getTab(state, tabId) : null; + if (tab?.kind === "artifact") { + return null; + } const liveUrl = runtime && !runtime.webContents.isDestroyed() ? runtime.webContents.getURL() : null; return resolveCopyableBrowserTabUrl(tab, liveUrl); @@ -2007,7 +2164,7 @@ export class DesktopBrowserManager { return; } clipboard.writeText(url); - const event: BrowserCopyLinkEvent = { threadId, url }; + const event: BrowserCopyLinkEvent = { threadId, tabId, url }; for (const listener of this.copyLinkListeners) { listener(event); } diff --git a/apps/desktop/src/browserUsePipeServer.test.ts b/apps/desktop/src/browserUsePipeServer.test.ts index 60fdb6009..fad78f268 100644 --- a/apps/desktop/src/browserUsePipeServer.test.ts +++ b/apps/desktop/src/browserUsePipeServer.test.ts @@ -107,7 +107,9 @@ function makeFakeBrowserManager() { tabs: [ { id: "tab-a", + kind: "web", url: "https://example.test/a", + displayUrl: null, title: "Tab A", status: "live", isLoading: false, @@ -119,7 +121,9 @@ function makeFakeBrowserManager() { }, { id: "tab-b", + kind: "web", url: "https://example.test/b", + displayUrl: null, title: "Tab B", status: "live", isLoading: false, diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index a78450a61..519aa5039 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -108,7 +108,10 @@ import { shouldBroadcastDownloadProgress, shouldCheckForUpdatesOnForeground, } from "./updateState"; -import { registerDesktopVoiceTranscriptionHandler } from "./voiceTranscription"; +import { + disposeDesktopVoiceTranscription, + registerDesktopVoiceTranscriptionHandler, +} from "./voiceTranscription"; import { resolveDesktopMenuAccelerator, resolveKeyboardShortcutsMenuAccelerator, @@ -2996,6 +2999,7 @@ async function shutdownDesktopRuntime(reason: string): Promise { appSnapManager?.dispose(); appSnapManager = null; await disposeBrowserUsePipeServerForShutdown(reason); + await disposeDesktopVoiceTranscription(); await stopBackendAndWaitForExit(reason); browserManager.dispose(); restoreStdIoCapture?.(); @@ -3350,7 +3354,10 @@ function registerIpcHandlers(): void { if (appSnapManager) { registerAppSnapIpcHandlers(ipcMain, appSnapManager); } - registerDesktopVoiceTranscriptionHandler(); + registerDesktopVoiceTranscriptionHandler({ + scientHome: BASE_DIR, + stateDirectory: Path.join(BASE_DIR, isDevelopment ? "dev" : "userdata"), + }); startBrowserPerformanceLogging(); void ensureBrowserUsePipeServer().catch((error) => { console.warn("[Scient browser] Failed to start browser-use native pipe", error); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index dab31a3c9..7a4980282 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -6,7 +6,7 @@ import { normalizeDesktopWsUrl, resolveDesktopWsUrlFromEnv, } from "./desktopWsBridge"; -import { SERVER_TRANSCRIBE_VOICE_CHANNEL } from "./voiceTranscription"; +import { DESKTOP_VOICE_IPC_CHANNELS } from "./voice/voiceIpcChannels"; import { STORAGE_MIGRATION_IPC_CHANNELS } from "./desktopStorageMigration"; import { APPSNAP_IPC_CHANNELS } from "./appSnapIpc"; import { DESKTOP_CONNECTION_WAKE_CHANNEL } from "./desktopConnectionWake"; @@ -182,7 +182,15 @@ contextBridge.exposeInMainWorld("desktopBridge", { acknowledgeSnapshot: () => ipcRenderer.invoke(STORAGE_MIGRATION_IPC_CHANNELS.acknowledge), }, server: { - transcribeVoice: (input) => ipcRenderer.invoke(SERVER_TRANSCRIBE_VOICE_CHANNEL, input), + transcribeVoice: (input) => ipcRenderer.invoke(DESKTOP_VOICE_IPC_CHANNELS.transcribe, input), + cancelVoiceTranscription: () => + ipcRenderer.invoke(DESKTOP_VOICE_IPC_CHANNELS.cancelTranscription), + }, + voice: { + getState: () => ipcRenderer.invoke(DESKTOP_VOICE_IPC_CHANNELS.getState), + downloadModel: () => ipcRenderer.invoke(DESKTOP_VOICE_IPC_CHANNELS.downloadModel), + removeModel: () => ipcRenderer.invoke(DESKTOP_VOICE_IPC_CHANNELS.removeModel), + repairModel: () => ipcRenderer.invoke(DESKTOP_VOICE_IPC_CHANNELS.repairModel), }, browser: { open: (input) => ipcRenderer.invoke(BROWSER_IPC_CHANNELS.open, input), diff --git a/apps/desktop/src/voice/chatGptVoiceTranscription.test.ts b/apps/desktop/src/voice/chatGptVoiceTranscription.test.ts new file mode 100644 index 000000000..3a60288ff --- /dev/null +++ b/apps/desktop/src/voice/chatGptVoiceTranscription.test.ts @@ -0,0 +1,180 @@ +import { Buffer } from "node:buffer"; +import { EventEmitter } from "node:events"; + +import type { NormalizedVoiceClip } from "@synara/shared/voiceTranscription"; +import { net } from "electron"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("electron", () => ({ + app: { getVersion: () => "test" }, + net: { request: vi.fn() }, +})); + +import { + createDesktopChatGptVoiceTranscriptionBackend, + requestDesktopVoiceTranscription, +} from "./chatGptVoiceTranscription"; + +function chatGptToken(accountId: string | null = "account-123"): string { + const encode = (value: unknown) => + Buffer.from(JSON.stringify(value), "utf8").toString("base64url"); + return `${encode({ alg: "none" })}.${encode( + accountId === null + ? { subject: "user-123" } + : { + "https://api.openai.com/auth": { chatgpt_account_id: accountId }, + }, + )}.`; +} + +const clip: NormalizedVoiceClip = { + audioBytes: Buffer.from("RIFF0000WAVE", "ascii"), + mimeType: "audio/wav", + sampleRateHz: 24_000, + durationMs: 1_000, + cwd: "/tmp/project", +}; + +describe("createDesktopChatGptVoiceTranscriptionBackend", () => { + it("passes token-bound account context to the desktop upload", async () => { + const upload = vi.fn(async () => ({ statusCode: 200, body: '{"text":" hello "}' })); + const backend = createDesktopChatGptVoiceTranscriptionBackend({ + cwd: clip.cwd, + resolveAuth: async () => ({ token: chatGptToken() }), + upload, + }); + + await expect( + backend.transcribe(clip, { signal: new AbortController().signal }), + ).resolves.toEqual({ text: "hello" }); + expect(upload).toHaveBeenCalledWith( + expect.objectContaining({ + token: chatGptToken(), + accountId: "account-123", + }), + ); + }); + + it("reuses the successful eligibility account context for the immediate transcription", async () => { + const resolveAuth = vi.fn(async () => ({ token: chatGptToken() })); + const upload = vi.fn(async () => ({ statusCode: 200, body: '{"text":"hello"}' })); + const backend = createDesktopChatGptVoiceTranscriptionBackend({ + cwd: clip.cwd, + resolveAuth, + upload, + }); + const signal = new AbortController().signal; + + await expect(backend.getAvailability({ signal })).resolves.toMatchObject({ state: "ready" }); + await expect(backend.transcribe(clip, { signal })).resolves.toEqual({ text: "hello" }); + + expect(resolveAuth).toHaveBeenCalledOnce(); + }); + + it("refreshes auth exactly once after a forbidden response", async () => { + const upload = vi + .fn() + .mockResolvedValueOnce({ statusCode: 403, body: "{}" }) + .mockResolvedValueOnce({ statusCode: 200, body: '{"transcript":"hello"}' }); + const resolveAuth = vi.fn(async (refreshToken: boolean) => ({ + token: chatGptToken(refreshToken ? "fresh-account" : "stale-account"), + })); + const backend = createDesktopChatGptVoiceTranscriptionBackend({ + cwd: clip.cwd, + resolveAuth, + upload, + }); + + await backend.transcribe(clip, { signal: new AbortController().signal }); + + expect(resolveAuth).toHaveBeenNthCalledWith(1, false, expect.any(AbortSignal)); + expect(resolveAuth).toHaveBeenNthCalledWith(2, true, expect.any(AbortSignal)); + expect(upload).toHaveBeenCalledTimes(2); + expect(upload.mock.calls[1]?.[0]).toMatchObject({ accountId: "fresh-account" }); + }); + + it("does not upload without a token-bound account context", async () => { + const upload = vi.fn(); + const backend = createDesktopChatGptVoiceTranscriptionBackend({ + cwd: clip.cwd, + resolveAuth: async () => ({ token: chatGptToken(null) }), + upload, + }); + + await expect( + backend.transcribe(clip, { signal: new AbortController().signal }), + ).rejects.toMatchObject({ kind: "authentication", fallbackAllowed: true }); + expect(upload).not.toHaveBeenCalled(); + }); + + it("marks persistent entitlement failures as safe to fall back", async () => { + const backend = createDesktopChatGptVoiceTranscriptionBackend({ + cwd: clip.cwd, + resolveAuth: async () => ({ token: chatGptToken() }), + upload: async () => ({ statusCode: 403, body: "{}" }), + }); + + await expect( + backend.transcribe(clip, { signal: new AbortController().signal }), + ).rejects.toMatchObject({ kind: "entitlement", fallbackAllowed: true }); + }); + + it("propagates cancellation while checking ChatGPT eligibility", async () => { + const controller = new AbortController(); + let observedSignal: AbortSignal | undefined; + const backend = createDesktopChatGptVoiceTranscriptionBackend({ + cwd: clip.cwd, + resolveAuth: (_refreshToken, signal) => + new Promise((_resolve, reject) => { + observedSignal = signal; + signal?.addEventListener("abort", () => reject(signal.reason), { once: true }); + }), + upload: vi.fn(), + }); + + const availability = backend.getAvailability({ signal: controller.signal }); + controller.abort(new Error("cancelled")); + + await expect(availability).resolves.toMatchObject({ state: "unavailable" }); + expect(observedSignal?.aborted).toBe(true); + }); +}); + +describe("requestDesktopVoiceTranscription", () => { + it("lets Electron frame the request body without setting restricted Content-Length", async () => { + const response = Object.assign(new EventEmitter(), { + statusCode: 200, + headers: {}, + }); + const setHeader = vi.fn((name: string) => { + if (name.toLowerCase() === "content-length") { + throw new Error("Electron forbids setting Content-Length"); + } + }); + const request = Object.assign(new EventEmitter(), { + abort: vi.fn(), + setHeader, + write: vi.fn(), + end: vi.fn(() => { + queueMicrotask(() => { + request.emit("response", response); + response.emit("data", Buffer.from('{"text":"hello"}', "utf8")); + response.emit("end"); + }); + }), + }); + vi.mocked(net.request).mockReturnValue(request as never); + + await expect( + requestDesktopVoiceTranscription({ + clip, + token: chatGptToken(), + accountId: "account-123", + signal: new AbortController().signal, + timeoutMs: 1_000, + }), + ).resolves.toMatchObject({ statusCode: 200, body: '{"text":"hello"}' }); + + expect(setHeader).not.toHaveBeenCalledWith("Content-Length", expect.any(String)); + }); +}); diff --git a/apps/desktop/src/voice/chatGptVoiceTranscription.ts b/apps/desktop/src/voice/chatGptVoiceTranscription.ts new file mode 100644 index 000000000..0011e3c83 --- /dev/null +++ b/apps/desktop/src/voice/chatGptVoiceTranscription.ts @@ -0,0 +1,616 @@ +// FILE: chatGptVoiceTranscription.ts +// Purpose: Implements ChatGPT-subscription transcription in the Electron main process. +// Layer: Desktop voice backend +// Depends on: Codex auth discovery, Electron net, and shared voice contracts. + +import * as ChildProcess from "node:child_process"; +import * as Crypto from "node:crypto"; + +import { app, net } from "electron"; +import type { + ServerVoiceTranscriptionInput, + ServerVoiceTranscriptionResult, +} from "@synara/contracts"; +import { extractChatGptAccountIdFromToken } from "@synara/shared/chatGptVoiceAuth"; +import { + type NormalizedVoiceClip, + type VoiceTranscript, + type VoiceTranscriptionBackend, + VoiceTranscriptionBackendError, + type VoiceTranscriptionErrorKind, +} from "@synara/shared/voiceTranscription"; +import { prepareWindowsSafeProcess } from "@synara/shared/windowsProcess"; + +export const CHATGPT_TRANSCRIPTIONS_URL = "https://chatgpt.com/backend-api/transcribe"; + +const MAX_AUDIO_BYTES = 10 * 1024 * 1024; +const MAX_DURATION_MS = 120_000; +const DEFAULT_REQUEST_TIMEOUT_MS = 30_000; +const AUTH_DISCOVERY_TIMEOUT_MS = 10_000; +const AUTH_CONTEXT_CACHE_MS = 15_000; + +interface ChatGptAccountContext { + readonly token: string; + readonly accountId: string; +} + +interface DesktopVoiceUploadResult { + readonly statusCode: number; + readonly body: string; + readonly retryAfter?: string; +} + +type ResolveDesktopVoiceAuth = ( + refreshToken: boolean, + signal?: AbortSignal, +) => Promise<{ readonly token: string }>; + +export interface DesktopVoiceProcessContext { + readonly binaryPath: string; + readonly env: NodeJS.ProcessEnv; +} + +type UploadDesktopVoice = (input: { + readonly clip: NormalizedVoiceClip; + readonly token: string; + readonly accountId: string; + readonly signal: AbortSignal; + readonly timeoutMs: number; +}) => Promise; + +export interface DesktopChatGptVoiceBackendOptions { + readonly cwd: string; + readonly resolveAuth?: ResolveDesktopVoiceAuth; + readonly resolveProcessContext?: () => Promise; + readonly upload?: UploadDesktopVoice; + readonly requestTimeoutMs?: number; +} + +function backendError(input: { + kind: VoiceTranscriptionErrorKind; + fallbackAllowed?: boolean; + safeMessage: string; + retryAfterMs?: number; + cause?: unknown; +}): VoiceTranscriptionBackendError { + return new VoiceTranscriptionBackendError({ + kind: input.kind, + fallbackAllowed: input.fallbackAllowed ?? true, + safeMessage: input.safeMessage, + ...(input.retryAfterMs !== undefined ? { retryAfterMs: input.retryAfterMs } : {}), + ...(input.cause !== undefined ? { cause: input.cause } : {}), + }); +} + +function readNonEmptyString(value: unknown): string | null { + const normalized = typeof value === "string" ? value.trim() : ""; + return normalized.length > 0 ? normalized : null; +} + +function validateClip(clip: NormalizedVoiceClip): void { + if (clip.mimeType !== "audio/wav") { + throw backendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage: "Only WAV audio is supported for voice transcription.", + }); + } + if (clip.sampleRateHz !== 24_000) { + throw backendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage: "Voice transcription requires 24 kHz mono WAV audio.", + }); + } + if (clip.durationMs <= 0 || clip.durationMs > MAX_DURATION_MS) { + throw backendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage: + clip.durationMs <= 0 + ? "Voice messages must include a positive duration." + : "Voice messages are limited to 120 seconds.", + }); + } + if (clip.audioBytes.byteLength === 0 || clip.audioBytes.byteLength > MAX_AUDIO_BYTES) { + throw backendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage: + clip.audioBytes.byteLength === 0 + ? "The recorded audio is empty." + : "Voice messages are limited to 10 MB.", + }); + } + + const header = Buffer.from( + clip.audioBytes.buffer, + clip.audioBytes.byteOffset, + clip.audioBytes.byteLength, + ); + if ( + header.byteLength < 12 || + header.toString("ascii", 0, 4) !== "RIFF" || + header.toString("ascii", 8, 12) !== "WAVE" + ) { + throw backendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage: "The recorded audio is not a valid WAV file.", + }); + } +} + +function requireAccountContext(auth: { readonly token: string }): ChatGptAccountContext { + const token = auth.token.trim(); + if (!token) { + throw backendError({ + kind: "authentication", + safeMessage: "No ChatGPT session is available. Sign in to ChatGPT in Codex.", + }); + } + + const accountId = extractChatGptAccountIdFromToken(token); + if (!accountId) { + throw backendError({ + kind: "authentication", + safeMessage: + "The ChatGPT session does not include an account context. Sign in to ChatGPT in Codex again.", + }); + } + return { token, accountId }; +} + +async function resolveAccountContext( + resolveAuth: ResolveDesktopVoiceAuth, + refreshToken: boolean, + signal?: AbortSignal, +): Promise { + try { + return requireAccountContext(await resolveAuth(refreshToken, signal)); + } catch (cause) { + if (cause instanceof VoiceTranscriptionBackendError) throw cause; + throw backendError({ + kind: "authentication", + safeMessage: "Scient could not read the ChatGPT session from Codex.", + cause, + }); + } +} + +export async function resolveDesktopVoiceAuth( + cwd: string, + refreshToken: boolean, + processContext: DesktopVoiceProcessContext = { binaryPath: "codex", env: process.env }, + signal?: AbortSignal, +): Promise<{ readonly token: string }> { + signal?.throwIfAborted(); + return new Promise((resolve, reject) => { + const prepared = prepareWindowsSafeProcess(processContext.binaryPath, ["app-server"], { + cwd, + env: processContext.env, + }); + const child = ChildProcess.spawn(prepared.command, prepared.args, { + cwd, + env: processContext.env, + stdio: ["pipe", "pipe", "pipe"], + shell: prepared.shell, + windowsHide: prepared.windowsHide, + windowsVerbatimArguments: prepared.windowsVerbatimArguments, + }); + + let settled = false; + let refreshAttempted = refreshToken; + let stdoutBuffer = ""; + const initializeTimer = setTimeout(() => { + send({ + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + clientInfo: { + name: "scient-desktop", + title: "Scient Desktop", + version: app.getVersion(), + }, + capabilities: { experimentalApi: true }, + }, + }); + }, 100); + const discoveryTimer = setTimeout(() => { + rejectOnce(new Error("Timed out while reading ChatGPT auth from Codex.")); + }, AUTH_DISCOVERY_TIMEOUT_MS); + discoveryTimer.unref(); + + function cleanup(): void { + clearTimeout(initializeTimer); + clearTimeout(discoveryTimer); + child.kill(); + signal?.removeEventListener("abort", abortAuthDiscovery); + } + function rejectOnce(error: Error): void { + if (settled) return; + settled = true; + cleanup(); + reject(error); + } + function resolveOnce(value: { readonly token: string }): void { + if (settled) return; + settled = true; + cleanup(); + resolve(value); + } + function send(payload: Record): void { + if (!settled && child.stdin.writable) { + child.stdin.write(`${JSON.stringify(payload)}\n`); + } + } + function abortAuthDiscovery(): void { + rejectOnce(new Error("ChatGPT auth discovery was cancelled.")); + } + + signal?.addEventListener("abort", abortAuthDiscovery, { once: true }); + + child.once("error", (error) => { + rejectOnce(new Error(`Could not start Codex auth discovery: ${error.message}`)); + }); + child.once("exit", (code, signal) => { + if (!settled) { + rejectOnce( + new Error(`Codex auth discovery exited before responding (${signal ?? String(code)}).`), + ); + } + }); + child.stderr.on("data", () => { + // Codex may log to stderr; the JSON-RPC response remains authoritative. + }); + child.stdout.on("data", (chunk) => { + stdoutBuffer += chunk.toString(); + const lines = stdoutBuffer.split(/\n/u); + stdoutBuffer = lines.pop() ?? ""; + + for (const line of lines) { + let message: Record; + try { + message = JSON.parse(line) as Record; + } catch { + continue; + } + + if (message.id === 1) { + send({ jsonrpc: "2.0", method: "initialized", params: {} }); + send({ + jsonrpc: "2.0", + id: 2, + method: "getAuthStatus", + params: { includeToken: true, refreshToken }, + }); + continue; + } + if (message.id !== 2 && message.id !== 3) continue; + + const result = + typeof message.result === "object" && message.result !== null + ? (message.result as Record) + : null; + const authMethod = readNonEmptyString(result?.authMethod); + const token = readNonEmptyString(result?.authToken); + if (!token) { + if (!refreshAttempted) { + refreshAttempted = true; + send({ + jsonrpc: "2.0", + id: 3, + method: "getAuthStatus", + params: { includeToken: true, refreshToken: true }, + }); + continue; + } + rejectOnce( + new Error("No ChatGPT session token is available. Sign in to ChatGPT in Codex."), + ); + return; + } + if (authMethod !== "chatgpt" && authMethod !== "chatgptAuthTokens") { + rejectOnce( + new Error("Voice transcription requires a ChatGPT-authenticated Codex session."), + ); + return; + } + resolveOnce({ token }); + } + }); + }); +} + +export async function requestDesktopVoiceTranscription(input: { + readonly clip: NormalizedVoiceClip; + readonly token: string; + readonly accountId: string; + readonly signal: AbortSignal; + readonly timeoutMs: number; +}): Promise { + if (input.signal.aborted) { + throw backendError({ + kind: "cancelled", + fallbackAllowed: false, + safeMessage: "Voice transcription was cancelled.", + cause: input.signal.reason, + }); + } + + const boundary = `ScientVoice-${Crypto.randomUUID()}`; + const audioBuffer = Buffer.from( + input.clip.audioBytes.buffer, + input.clip.audioBytes.byteOffset, + input.clip.audioBytes.byteLength, + ); + const preamble = Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="voice.wav"\r\nContent-Type: ${input.clip.mimeType}\r\n\r\n`, + "utf8", + ); + const closing = Buffer.from(`\r\n--${boundary}--\r\n`, "utf8"); + const body = Buffer.concat([preamble, audioBuffer, closing]); + + return new Promise((resolve, reject) => { + let settled = false; + let timedOut = false; + const request = net.request({ method: "POST", url: CHATGPT_TRANSCRIPTIONS_URL }); + const timeout = setTimeout(() => { + timedOut = true; + request.abort(); + rejectOnce( + backendError({ + kind: "timeout", + safeMessage: "ChatGPT voice transcription timed out.", + }), + ); + }, input.timeoutMs); + const abortFromCaller = () => { + request.abort(); + rejectOnce( + backendError({ + kind: "cancelled", + fallbackAllowed: false, + safeMessage: "Voice transcription was cancelled.", + cause: input.signal.reason, + }), + ); + }; + + function cleanup(): void { + clearTimeout(timeout); + input.signal.removeEventListener("abort", abortFromCaller); + } + function rejectOnce(error: VoiceTranscriptionBackendError): void { + if (settled) return; + settled = true; + cleanup(); + reject(error); + } + function resolveOnce(value: DesktopVoiceUploadResult): void { + if (settled) return; + settled = true; + cleanup(); + resolve(value); + } + + input.signal.addEventListener("abort", abortFromCaller, { once: true }); + request.setHeader("Accept", "application/json"); + request.setHeader("Authorization", `Bearer ${input.token}`); + request.setHeader("ChatGPT-Account-Id", input.accountId); + request.setHeader("Origin", "https://chatgpt.com"); + request.setHeader("User-Agent", "Scient Desktop"); + request.setHeader("originator", "scient-desktop"); + request.setHeader("Content-Type", `multipart/form-data; boundary=${boundary}`); + + request.once("error", (cause) => { + if (settled || timedOut) return; + rejectOnce( + backendError({ + kind: "network", + safeMessage: "Scient could not reach ChatGPT voice transcription.", + cause, + }), + ); + }); + request.on("response", (response) => { + let responseBody = ""; + response.on("data", (chunk) => { + responseBody += chunk.toString(); + }); + response.once("end", () => { + const retryAfterHeader = response.headers["retry-after"]; + const retryAfter = Array.isArray(retryAfterHeader) ? retryAfterHeader[0] : retryAfterHeader; + resolveOnce({ + statusCode: response.statusCode, + body: responseBody, + ...(retryAfter ? { retryAfter } : {}), + }); + }); + response.once("error", (cause) => { + rejectOnce( + backendError({ + kind: "network", + safeMessage: "ChatGPT voice transcription response was interrupted.", + cause, + }), + ); + }); + }); + + request.write(body); + request.end(); + }); +} + +function parseRetryAfterMs(value: string | undefined): number | undefined { + const normalized = value?.trim(); + if (!normalized) return undefined; + const seconds = Number(normalized); + if (Number.isFinite(seconds) && seconds >= 0) return Math.round(seconds * 1_000); + const retryAt = Date.parse(normalized); + return Number.isFinite(retryAt) ? Math.max(0, retryAt - Date.now()) : undefined; +} + +function statusFailure(response: DesktopVoiceUploadResult): VoiceTranscriptionBackendError { + if (response.statusCode === 401) { + return backendError({ + kind: "authentication", + safeMessage: "ChatGPT rejected the current login. Sign in to ChatGPT in Codex again.", + }); + } + if (response.statusCode === 403) { + return backendError({ + kind: "entitlement", + safeMessage: "ChatGPT voice transcription is not available for this session.", + }); + } + if (response.statusCode === 429) { + const retryAfterMs = parseRetryAfterMs(response.retryAfter); + return backendError({ + kind: "rate-limit", + safeMessage: "ChatGPT voice transcription is temporarily rate limited.", + ...(retryAfterMs !== undefined ? { retryAfterMs } : {}), + }); + } + return backendError({ + kind: "provider-error", + safeMessage: `ChatGPT voice transcription failed with status ${response.statusCode}.`, + }); +} + +function parseTranscript(response: DesktopVoiceUploadResult): VoiceTranscript { + let payload: unknown; + try { + payload = JSON.parse(response.body) as unknown; + } catch (cause) { + throw backendError({ + kind: "malformed-response", + safeMessage: "ChatGPT returned an invalid transcription response.", + cause, + }); + } + const record = + typeof payload === "object" && payload !== null ? (payload as Record) : null; + const value = + typeof record?.text === "string" + ? record.text + : typeof record?.transcript === "string" + ? record.transcript + : ""; + const text = value.trim(); + if (!text) { + throw backendError({ + kind: "malformed-response", + safeMessage: "ChatGPT returned an empty transcription response.", + }); + } + return { text }; +} + +export function createDesktopChatGptVoiceTranscriptionBackend( + options: DesktopChatGptVoiceBackendOptions, +): VoiceTranscriptionBackend { + const cwd = options.cwd.trim() || process.cwd(); + const resolveAuth = + options.resolveAuth ?? + (async (refreshToken: boolean, signal?: AbortSignal) => + resolveDesktopVoiceAuth( + cwd, + refreshToken, + options.resolveProcessContext + ? await options.resolveProcessContext() + : { binaryPath: "codex", env: process.env }, + signal, + )); + const upload = options.upload ?? requestDesktopVoiceTranscription; + const timeoutMs = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; + let cachedAccount: { + readonly account: ChatGptAccountContext; + readonly expiresAt: number; + } | null = null; + + const takeCachedAccount = (): ChatGptAccountContext | null => { + const cached = cachedAccount; + cachedAccount = null; + return cached && cached.expiresAt > Date.now() ? cached.account : null; + }; + + return { + id: "chatgpt", + async getAvailability({ signal }: { readonly signal?: AbortSignal } = {}) { + try { + const account = await resolveAccountContext(resolveAuth, false, signal); + signal?.throwIfAborted(); + cachedAccount = { account, expiresAt: Date.now() + AUTH_CONTEXT_CACHE_MS }; + return { state: "ready" }; + } catch (cause) { + cachedAccount = null; + return { + state: "unavailable", + reason: cause instanceof VoiceTranscriptionBackendError ? cause.kind : "authentication", + }; + } + }, + async transcribe(clip, { signal }) { + validateClip(clip); + let account = + takeCachedAccount() ?? (await resolveAccountContext(resolveAuth, false, signal)); + let response = await upload({ clip, ...account, signal, timeoutMs }); + + if (response.statusCode === 401 || response.statusCode === 403) { + cachedAccount = null; + account = await resolveAccountContext(resolveAuth, true, signal); + response = await upload({ clip, ...account, signal, timeoutMs }); + } + if (response.statusCode < 200 || response.statusCode >= 300) { + throw statusFailure(response); + } + return parseTranscript(response); + }, + }; +} + +function decodeDesktopRequest(request: ServerVoiceTranscriptionInput): NormalizedVoiceClip { + if (request.mimeType !== "audio/wav") { + throw backendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage: "Only WAV audio is supported for voice transcription.", + }); + } + const normalized = request.audioBase64.trim().replace(/\s+/gu, ""); + if (!normalized || !/^[A-Za-z0-9+/]+={0,2}$/u.test(normalized)) { + throw backendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage: "The recorded audio could not be decoded.", + }); + } + const audioBytes = Buffer.from(normalized, "base64"); + if (!audioBytes.byteLength || audioBytes.toString("base64") !== normalized) { + throw backendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage: "The recorded audio could not be decoded.", + }); + } + return { + audioBytes, + mimeType: request.mimeType, + sampleRateHz: request.sampleRateHz, + durationMs: request.durationMs, + cwd: request.cwd, + ...(request.threadId ? { threadId: request.threadId } : {}), + }; +} + +export async function transcribeVoiceViaDesktopBridge( + input: ServerVoiceTranscriptionInput, +): Promise { + const backend = createDesktopChatGptVoiceTranscriptionBackend({ cwd: input.cwd }); + return backend.transcribe(decodeDesktopRequest(input), { + signal: new AbortController().signal, + }); +} diff --git a/apps/desktop/src/voice/desktopCodexVoiceRuntime.test.ts b/apps/desktop/src/voice/desktopCodexVoiceRuntime.test.ts new file mode 100644 index 000000000..d5530421c --- /dev/null +++ b/apps/desktop/src/voice/desktopCodexVoiceRuntime.test.ts @@ -0,0 +1,126 @@ +import * as FS from "node:fs/promises"; +import * as OS from "node:os"; +import * as Path from "node:path"; +import { createHash } from "node:crypto"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { resolveDesktopCodexVoiceProcessContext } from "./desktopCodexVoiceRuntime"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => FS.rm(directory, { recursive: true, force: true })), + ); +}); + +async function makeRuntimeRoot(): Promise<{ root: string; stateDirectory: string }> { + const root = await FS.mkdtemp(Path.join(OS.tmpdir(), "scient-voice-runtime-")); + temporaryDirectories.push(root); + const stateDirectory = Path.join(root, "userdata"); + await FS.mkdir(stateDirectory, { recursive: true }); + return { root, stateDirectory }; +} + +describe("resolveDesktopCodexVoiceProcessContext", () => { + it("uses configured Codex binary and the prepared auth overlay", async () => { + const { root, stateDirectory } = await makeRuntimeRoot(); + const overlay = Path.join(root, "codex-home-overlay"); + await FS.mkdir(overlay, { recursive: true }); + await FS.writeFile(Path.join(overlay, "auth.json"), "{}", "utf8"); + await FS.writeFile( + Path.join(stateDirectory, "settings.json"), + JSON.stringify({ + providers: { codex: { binaryPath: "/opt/scient/codex", homePath: "/source/codex" } }, + }), + "utf8", + ); + + await expect( + resolveDesktopCodexVoiceProcessContext({ + stateDirectory, + scientHome: root, + env: { PATH: "" }, + }), + ).resolves.toMatchObject({ + binaryPath: "/opt/scient/codex", + env: { CODEX_HOME: overlay, SCIENT_HOME: root }, + }); + }); + + it("uses the active managed Codex runtime when no system binary is available", async () => { + const { root, stateDirectory } = await makeRuntimeRoot(); + const managedRoot = Path.join(stateDirectory, "provider-runtimes", "codex"); + const executablePath = Path.join(managedRoot, "releases", "v1", "bin", "codex"); + const executableContents = "runtime"; + await FS.mkdir(Path.dirname(executablePath), { recursive: true }); + await FS.writeFile(executablePath, executableContents, "utf8"); + await FS.writeFile( + Path.join(managedRoot, "current.json"), + JSON.stringify({ + version: 1, + provider: "codex", + releaseId: "v1", + executableRelativePath: Path.join("bin", "codex"), + executablePath, + executableDigest: createHash("sha256").update(executableContents).digest("hex"), + }), + "utf8", + ); + + await expect( + resolveDesktopCodexVoiceProcessContext({ + stateDirectory, + scientHome: root, + env: { PATH: "", CODEX_HOME: "/source/codex" }, + }), + ).resolves.toMatchObject({ + binaryPath: executablePath, + env: { CODEX_HOME: "/source/codex" }, + }); + }); + + it("fails closed when a managed Codex executable no longer matches its digest", async () => { + const { root, stateDirectory } = await makeRuntimeRoot(); + const managedRoot = Path.join(stateDirectory, "provider-runtimes", "codex"); + const executablePath = Path.join(managedRoot, "releases", "v1", "bin", "codex"); + await FS.mkdir(Path.dirname(executablePath), { recursive: true }); + await FS.writeFile(executablePath, "tampered", "utf8"); + await FS.writeFile( + Path.join(managedRoot, "current.json"), + JSON.stringify({ + version: 1, + provider: "codex", + releaseId: "v1", + executableRelativePath: Path.join("bin", "codex"), + executablePath, + executableDigest: createHash("sha256").update("original").digest("hex"), + }), + "utf8", + ); + + await expect( + resolveDesktopCodexVoiceProcessContext({ + stateDirectory, + scientHome: root, + env: { PATH: "" }, + }), + ).rejects.toThrow(/refused an unverified managed Codex runtime/i); + }); + + it("rejects remote voice when Codex is disabled", async () => { + const { root, stateDirectory } = await makeRuntimeRoot(); + await FS.writeFile( + Path.join(stateDirectory, "settings.json"), + JSON.stringify({ providers: { codex: { enabled: false } } }), + "utf8", + ); + + await expect( + resolveDesktopCodexVoiceProcessContext({ stateDirectory, scientHome: root, env: {} }), + ).rejects.toThrow(/disabled/i); + }); +}); diff --git a/apps/desktop/src/voice/desktopCodexVoiceRuntime.ts b/apps/desktop/src/voice/desktopCodexVoiceRuntime.ts new file mode 100644 index 000000000..28592c360 --- /dev/null +++ b/apps/desktop/src/voice/desktopCodexVoiceRuntime.ts @@ -0,0 +1,196 @@ +// FILE: desktopCodexVoiceRuntime.ts +// Purpose: Resolves the same configured or managed Codex runtime context used by Scient sessions. +// Layer: Desktop voice runtime configuration + +import * as FS from "node:fs/promises"; +import { constants as FS_CONSTANTS, createReadStream } from "node:fs"; +import { createHash } from "node:crypto"; +import * as OS from "node:os"; +import * as Path from "node:path"; + +import type { DesktopVoiceProcessContext } from "./chatGptVoiceTranscription"; + +interface CodexSettingsRecord { + readonly providers?: { + readonly codex?: { + readonly enabled?: unknown; + readonly binaryPath?: unknown; + readonly homePath?: unknown; + }; + }; +} + +export interface DesktopCodexVoiceRuntimeOptions { + readonly stateDirectory: string; + readonly scientHome: string; + readonly env?: NodeJS.ProcessEnv; +} + +export async function resolveDesktopCodexVoiceProcessContext( + options: DesktopCodexVoiceRuntimeOptions, +): Promise { + const env = { ...(options.env ?? process.env) }; + const settings = await readCodexSettings(Path.join(options.stateDirectory, "settings.json")); + if (settings.enabled === false) { + throw new Error("Codex is disabled in Scient settings."); + } + + const configuredBinary = settings.binaryPath || "codex"; + const binaryPath = await resolveCodexBinaryPath({ + configuredBinary, + stateDirectory: options.stateDirectory, + env, + }); + const sourceHome = + settings.homePath || env.CODEX_HOME?.trim() || Path.join(OS.homedir(), ".codex"); + const overlayHome = Path.join(options.scientHome, "codex-home-overlay"); + const effectiveHome = (await pathExists(Path.join(overlayHome, "auth.json"))) + ? overlayHome + : sourceHome; + + return { + binaryPath, + env: { + ...env, + SCIENT_HOME: options.scientHome, + CODEX_HOME: effectiveHome, + }, + }; +} + +async function readCodexSettings(settingsPath: string): Promise<{ + readonly enabled?: boolean; + readonly binaryPath: string; + readonly homePath: string; +}> { + try { + const parsed = JSON.parse(await FS.readFile(settingsPath, "utf8")) as CodexSettingsRecord; + const codex = parsed.providers?.codex; + return { + ...(typeof codex?.enabled === "boolean" ? { enabled: codex.enabled } : {}), + binaryPath: boundedString(codex?.binaryPath), + homePath: boundedString(codex?.homePath), + }; + } catch { + return { binaryPath: "", homePath: "" }; + } +} + +async function resolveCodexBinaryPath(input: { + readonly configuredBinary: string; + readonly stateDirectory: string; + readonly env: NodeJS.ProcessEnv; +}): Promise { + if (input.configuredBinary !== "codex") return input.configuredBinary; + if (await findExecutableOnPath("codex", input.env)) return "codex"; + + const managedRoot = Path.join(input.stateDirectory, "provider-runtimes", "codex"); + const currentRecordPath = Path.join(managedRoot, "current.json"); + let rawRecord: string; + try { + rawRecord = await FS.readFile(currentRecordPath, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return "codex"; + throw new Error("Scient could not verify the managed Codex runtime.", { cause: error }); + } + + try { + const record = JSON.parse(rawRecord) as Record; + const releaseId = boundedString(record.releaseId); + const executableRelativePath = boundedString(record.executableRelativePath); + const executablePath = boundedString(record.executablePath); + const executableDigest = boundedString(record.executableDigest); + if ( + record.version !== 1 || + record.provider !== "codex" || + !releaseId || + !executableRelativePath || + !executablePath || + !/^[a-f0-9]{64}$/u.test(executableDigest) + ) { + throw new Error("The managed Codex runtime record is invalid."); + } + const releaseRoot = Path.resolve(managedRoot, "releases", releaseId); + const expectedPath = Path.resolve(releaseRoot, executableRelativePath); + if (expectedPath !== Path.resolve(executablePath)) { + throw new Error("The managed Codex executable path is invalid."); + } + const relative = Path.relative(releaseRoot, expectedPath); + if ( + !relative || + relative === ".." || + relative.startsWith(`..${Path.sep}`) || + Path.isAbsolute(relative) + ) { + throw new Error("The managed Codex executable escapes its release directory."); + } + const stats = await FS.lstat(executablePath); + if (!stats.isFile() || stats.isSymbolicLink()) { + throw new Error("The managed Codex executable is not a regular file."); + } + const [realRoot, realExecutable] = await Promise.all([ + FS.realpath(releaseRoot), + FS.realpath(executablePath), + ]); + const realRelative = Path.relative(realRoot, realExecutable); + if ( + !realRelative || + realRelative === ".." || + realRelative.startsWith(`..${Path.sep}`) || + Path.isAbsolute(realRelative) + ) { + throw new Error("The managed Codex executable resolves outside its release directory."); + } + if ((await hashExecutable(executablePath)) !== executableDigest) { + throw new Error("The managed Codex executable checksum changed after installation."); + } + return executablePath; + } catch (error) { + throw new Error("Scient refused an unverified managed Codex runtime.", { cause: error }); + } +} + +async function hashExecutable(filePath: string): Promise { + const hash = createHash("sha256"); + await new Promise((resolve, reject) => { + const stream = createReadStream(filePath); + stream.on("data", (chunk) => hash.update(chunk)); + stream.on("error", reject); + stream.on("end", resolve); + }); + return hash.digest("hex"); +} + +async function findExecutableOnPath(command: string, env: NodeJS.ProcessEnv): Promise { + const directories = (env.PATH ?? "").split(Path.delimiter).filter(Boolean); + const extensions = + process.platform === "win32" + ? (env.PATHEXT ?? ".EXE;.CMD;.BAT;.COM").split(";").filter(Boolean) + : [""]; + for (const directory of directories) { + for (const extension of extensions) { + try { + await FS.access(Path.join(directory, `${command}${extension}`), FS_CONSTANTS.X_OK); + return true; + } catch { + // Continue through the bounded PATH candidates. + } + } + } + return false; +} + +async function pathExists(filePath: string): Promise { + try { + await FS.access(filePath); + return true; + } catch { + return false; + } +} + +function boundedString(value: unknown): string { + if (typeof value !== "string") return ""; + const trimmed = value.trim(); + return trimmed.length <= 4_096 ? trimmed : ""; +} diff --git a/apps/desktop/src/voice/desktopVoiceService.test.ts b/apps/desktop/src/voice/desktopVoiceService.test.ts new file mode 100644 index 000000000..95d8fcf65 --- /dev/null +++ b/apps/desktop/src/voice/desktopVoiceService.test.ts @@ -0,0 +1,257 @@ +// FILE: desktopVoiceService.test.ts +// Purpose: Verifies desktop composition, provider-neutral routing, model setup, and safe errors. +// Layer: Desktop voice tests + +import { describe, expect, it, vi } from "vitest"; +import type { VoiceTranscriptionBackend } from "@synara/shared/voiceTranscription"; +import { VoiceTranscriptionBackendError } from "@synara/shared/voiceTranscription"; +import { DesktopVoiceService } from "./desktopVoiceService"; + +function wavBase64(): string { + const bytes = Buffer.alloc(46); + bytes.write("RIFF", 0, "ascii"); + bytes.writeUInt32LE(38, 4); + bytes.write("WAVEfmt ", 8, "ascii"); + bytes.writeUInt32LE(16, 16); + bytes.writeUInt16LE(1, 20); + bytes.writeUInt16LE(1, 22); + bytes.writeUInt32LE(24_000, 24); + bytes.writeUInt32LE(48_000, 28); + bytes.writeUInt16LE(2, 32); + bytes.writeUInt16LE(16, 34); + bytes.write("data", 36, "ascii"); + bytes.writeUInt32LE(2, 40); + return bytes.toString("base64"); +} + +const request = { + provider: "grok", + cwd: "/workspace", + mimeType: "audio/wav", + sampleRateHz: 24_000, + durationMs: 1, + audioBase64: wavBase64(), +} as const; + +function manager(state: "missing" | "ready" = "ready") { + let current = state; + return { + getStatus: vi.fn(async () => + current === "ready" + ? ({ state: "ready", modelPath: "/model.bin", byteSize: 190 } as const) + : ({ state: "missing" } as const), + ), + ensureInstalled: vi.fn(async () => { + current = "ready"; + return "/model.bin"; + }), + isDownloading: vi.fn(() => false), + repair: vi.fn(async () => "/model.bin"), + remove: vi.fn(async () => { + current = "missing"; + }), + }; +} + +function runtime() { + return { + isInstalled: vi.fn(async () => true), + isBusy: vi.fn(() => false), + stopIdle: vi.fn(async () => undefined), + transcribe: vi.fn(async () => ({ text: "local text" })), + dispose: vi.fn(async () => undefined), + }; +} + +function remote( + transcribe: VoiceTranscriptionBackend["transcribe"] = vi.fn(async () => ({ text: "cloud text" })), +): VoiceTranscriptionBackend { + return { + id: "chatgpt", + getAvailability: vi.fn(async () => ({ state: "ready" as const })), + transcribe, + }; +} + +describe("DesktopVoiceService", () => { + it("routes independently of the active text provider and prefers ChatGPT", async () => { + const service = new DesktopVoiceService({ + modelManager: manager() as never, + runtime: runtime(), + createRemoteBackend: () => remote(), + }); + + await expect(service.transcribe(request)).resolves.toMatchObject({ + text: "cloud text", + engine: "chatgpt", + fallbackUsed: false, + }); + }); + + it("quietly falls back locally after a recoverable ChatGPT failure", async () => { + const service = new DesktopVoiceService({ + modelManager: manager() as never, + runtime: runtime(), + createRemoteBackend: () => + remote(async () => { + throw new VoiceTranscriptionBackendError({ + kind: "network", + fallbackAllowed: true, + safeMessage: "Network failed.", + }); + }), + }); + + await expect(service.transcribe(request)).resolves.toMatchObject({ + text: "local text", + engine: "local", + fallbackUsed: true, + fallbackReason: "network", + }); + }); + + it("keeps automatic ChatGPT voice available while the offline model downloads", async () => { + let finishDownload!: () => void; + const modelManager = manager("missing"); + modelManager.ensureInstalled.mockImplementationOnce( + () => + new Promise((resolve) => { + finishDownload = () => resolve("/model.bin"); + }), + ); + const service = new DesktopVoiceService({ + modelManager: modelManager as never, + runtime: runtime(), + createRemoteBackend: () => remote(), + }); + + const download = service.downloadModel(); + await vi.waitFor(() => expect(modelManager.ensureInstalled).toHaveBeenCalledOnce()); + await expect(service.transcribe({ ...request, mode: "automatic" })).resolves.toMatchObject({ + text: "cloud text", + engine: "chatgpt", + }); + await expect(service.transcribe({ ...request, mode: "offline-only" })).rejects.toThrow( + /maintenance/i, + ); + finishDownload(); + await expect(download).resolves.toMatchObject({ model: { state: "missing" } }); + }); + + it("propagates user cancellation to an active backend request", async () => { + let observedSignal: AbortSignal | null = null; + const service = new DesktopVoiceService({ + modelManager: manager() as never, + runtime: runtime(), + createRemoteBackend: () => + remote( + (_clip, { signal }) => + new Promise((_resolve, reject) => { + observedSignal = signal; + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + }), + ), + }); + + const transcription = service.transcribe(request); + await vi.waitFor(() => expect(observedSignal).not.toBeNull()); + service.cancelActiveTranscriptions(); + + await expect(transcription).rejects.toThrow(/cancelled/i); + expect((observedSignal as AbortSignal | null)?.aborted).toBe(true); + }); + + it("aborts and drains active transcription before disposal completes", async () => { + let backendFinished = false; + const whisperRuntime = runtime(); + const service = new DesktopVoiceService({ + modelManager: manager() as never, + runtime: whisperRuntime, + createRemoteBackend: () => + remote( + (_clip, { signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener( + "abort", + () => { + backendFinished = true; + reject(signal.reason); + }, + { once: true }, + ); + }), + ), + }); + + const transcription = service.transcribe(request); + await Promise.resolve(); + await service.dispose(); + + await expect(transcription).rejects.toThrow(/cancelled/i); + expect(backendFinished).toBe(true); + expect(whisperRuntime.dispose).toHaveBeenCalledOnce(); + }); + + it("exposes download/removal lifecycle and disposes the runtime", async () => { + const modelManager = manager("missing"); + const whisperRuntime = runtime(); + const service = new DesktopVoiceService({ + modelManager: modelManager as never, + runtime: whisperRuntime, + createRemoteBackend: () => remote(), + }); + + await expect(service.getState()).resolves.toMatchObject({ model: { state: "missing" } }); + await expect(service.downloadModel()).resolves.toMatchObject({ model: { state: "ready" } }); + await expect(service.removeModel()).resolves.toMatchObject({ model: { state: "missing" } }); + await service.dispose(); + expect(whisperRuntime.dispose).toHaveBeenCalledOnce(); + }); + + it("repairs without removing the installed model first", async () => { + const modelManager = manager(); + const whisperRuntime = runtime(); + const service = new DesktopVoiceService({ + modelManager: modelManager as never, + runtime: whisperRuntime, + createRemoteBackend: () => remote(), + }); + + await expect(service.repairModel()).resolves.toMatchObject({ model: { state: "ready" } }); + expect(whisperRuntime.stopIdle).toHaveBeenCalledOnce(); + expect(modelManager.repair).toHaveBeenCalledOnce(); + expect(whisperRuntime.stopIdle.mock.invocationCallOrder[0]).toBeLessThan( + modelManager.repair.mock.invocationCallOrder[0] ?? Number.MAX_SAFE_INTEGER, + ); + expect(modelManager.remove).not.toHaveBeenCalled(); + }); + + it("does not remove the model during active local inference", async () => { + const modelManager = manager(); + const whisperRuntime = runtime(); + whisperRuntime.isBusy.mockReturnValue(true); + const service = new DesktopVoiceService({ + modelManager: modelManager as never, + runtime: whisperRuntime, + createRemoteBackend: () => remote(), + }); + + await expect(service.removeModel()).rejects.toThrow(/current voice transcription/i); + expect(modelManager.remove).not.toHaveBeenCalled(); + }); + + it("does not remove or repair the model while a download is active", async () => { + const modelManager = manager(); + modelManager.isDownloading.mockReturnValue(true); + const service = new DesktopVoiceService({ + modelManager: modelManager as never, + runtime: runtime(), + createRemoteBackend: () => remote(), + }); + + await expect(service.removeModel()).rejects.toThrow(/wait for/i); + await expect(service.repairModel()).rejects.toThrow(/wait for/i); + expect(modelManager.remove).not.toHaveBeenCalled(); + expect(modelManager.repair).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/desktop/src/voice/desktopVoiceService.ts b/apps/desktop/src/voice/desktopVoiceService.ts new file mode 100644 index 000000000..105ce6ac3 --- /dev/null +++ b/apps/desktop/src/voice/desktopVoiceService.ts @@ -0,0 +1,217 @@ +// FILE: desktopVoiceService.ts +// Purpose: Composes ChatGPT-first routing, verified local fallback, and model lifecycle. +// Layer: Desktop voice application service + +import type { + DesktopVoiceState, + ServerVoiceTranscriptionInput, + ServerVoiceTranscriptionResult, +} from "@synara/contracts"; +import { + isVoiceTranscriptionBackendError, + type VoiceTranscriptionBackend, +} from "@synara/shared/voiceTranscription"; +import { + VoiceTranscriptionRouter, + VoiceTranscriptionRoutingError, +} from "@synara/shared/voiceTranscriptionRouter"; + +import { + createDesktopChatGptVoiceTranscriptionBackend, + type DesktopVoiceProcessContext, +} from "./chatGptVoiceTranscription"; +import { LocalVoiceModelManager } from "./localVoiceModelManager"; +import { LOCAL_VOICE_MODEL } from "./localVoiceModelManifest"; +import { LocalWhisperBackend, type LocalWhisperRuntimeLike } from "./localWhisperBackend"; +import { normalizeDesktopVoiceRequest } from "./voiceRequest"; + +export interface DesktopVoiceServiceOptions { + readonly modelManager: LocalVoiceModelManager; + readonly runtime: LocalWhisperRuntimeLike; + readonly createRemoteBackend?: (cwd: string) => VoiceTranscriptionBackend; + readonly resolveRemoteProcessContext?: () => Promise; +} + +export class DesktopVoiceService { + private readonly localBackend: LocalWhisperBackend; + private readonly createRemoteBackend: (cwd: string) => VoiceTranscriptionBackend; + private readonly routers = new Map(); + private readonly activeControllers = new Set(); + private readonly activeTranscriptions = new Set>(); + private disposed = false; + private modelMutationActive = false; + + constructor(private readonly options: DesktopVoiceServiceOptions) { + this.localBackend = new LocalWhisperBackend( + options.modelManager, + options.runtime, + () => this.modelMutationActive, + ); + this.createRemoteBackend = + options.createRemoteBackend ?? + ((cwd) => + createDesktopChatGptVoiceTranscriptionBackend({ + cwd, + ...(options.resolveRemoteProcessContext + ? { resolveProcessContext: options.resolveRemoteProcessContext } + : {}), + })); + } + + async getState(): Promise { + const [runtimeAvailable, status] = await Promise.all([ + this.options.runtime.isInstalled(), + this.options.modelManager.getStatus(), + ]); + return { + runtimeAvailable, + model: toDesktopModelStatus(status), + modelName: LOCAL_VOICE_MODEL.displayName, + modelByteSize: LOCAL_VOICE_MODEL.byteSize, + }; + } + + async downloadModel(): Promise { + return this.withModelMutation(async () => { + await this.options.modelManager.ensureInstalled(new AbortController().signal); + return this.getState(); + }); + } + + async removeModel(): Promise { + if (this.options.runtime.isBusy() || this.options.modelManager.isDownloading()) { + throw new Error( + "Wait for the current voice transcription to finish before removing the model.", + ); + } + return this.withModelMutation(async () => { + await this.options.runtime.stopIdle(); + await this.options.modelManager.remove(); + return this.getState(); + }); + } + + async repairModel(): Promise { + if (this.options.runtime.isBusy() || this.options.modelManager.isDownloading()) { + throw new Error("Wait for current voice activity to finish before repairing the model."); + } + return this.withModelMutation(async () => { + await this.options.runtime.stopIdle(); + await this.options.modelManager.repair(new AbortController().signal); + return this.getState(); + }); + } + + transcribe(input: unknown): Promise { + if (this.disposed) return Promise.reject(new Error("Voice transcription is shutting down.")); + const controller = new AbortController(); + this.activeControllers.add(controller); + const transcription = this.transcribeWithSignal(input, controller.signal).finally(() => { + this.activeControllers.delete(controller); + this.activeTranscriptions.delete(transcription); + }); + this.activeTranscriptions.add(transcription); + return transcription; + } + + cancelActiveTranscriptions(): void { + for (const controller of this.activeControllers) { + controller.abort(new Error("Voice transcription was cancelled.")); + } + } + + private async transcribeWithSignal( + input: unknown, + signal: AbortSignal, + ): Promise { + const request = normalizeDesktopVoiceRequest(input); + const router = this.getRouter(request.clip.cwd); + try { + const transcript = await router.transcribe(request.clip, { + signal, + mode: request.mode, + }); + return { + text: transcript.text, + engine: transcript.engine, + fallbackUsed: transcript.fallbackUsed, + ...(transcript.fallbackReason ? { fallbackReason: transcript.fallbackReason } : {}), + }; + } catch (error) { + if (signal.aborted) { + throw new Error("Voice transcription was cancelled.", { cause: error }); + } + throw new Error(safeVoiceErrorMessage(error), { cause: error }); + } + } + + async dispose(): Promise { + if (this.disposed) return; + this.disposed = true; + this.cancelActiveTranscriptions(); + this.routers.clear(); + await Promise.allSettled([...this.activeTranscriptions]); + await this.localBackend.dispose(); + } + + private async withModelMutation(operation: () => Promise): Promise { + if (this.disposed) throw new Error("Voice transcription is shutting down."); + if ( + this.modelMutationActive || + this.activeTranscriptions.size > 0 || + this.options.runtime.isBusy() || + this.options.modelManager.isDownloading() + ) { + throw new Error("Wait for current voice activity to finish before changing the model."); + } + this.modelMutationActive = true; + try { + return await operation(); + } finally { + this.modelMutationActive = false; + } + } + + private getRouter(cwd: string): VoiceTranscriptionRouter { + const existing = this.routers.get(cwd); + if (existing) return existing; + const router = new VoiceTranscriptionRouter({ + remote: this.createRemoteBackend(cwd), + local: this.localBackend, + }); + this.routers.set(cwd, router); + return router; + } +} + +function toDesktopModelStatus( + status: Awaited>, +): DesktopVoiceState["model"] { + switch (status.state) { + case "missing": + return { state: "missing" }; + case "downloading": + return { + state: "downloading", + downloadedBytes: status.downloadedBytes, + totalBytes: status.totalBytes, + }; + case "ready": + return { state: "ready", byteSize: status.byteSize }; + case "error": + return { state: "error", message: status.message }; + } +} + +function safeVoiceErrorMessage(error: unknown): string { + if (isVoiceTranscriptionBackendError(error)) return error.safeMessage; + if (error instanceof VoiceTranscriptionRoutingError) { + const localMessage = safeVoiceErrorMessage(error.localError); + if (localMessage !== "Voice transcription failed.") return localMessage; + const remoteMessage = safeVoiceErrorMessage(error.remoteError); + if (remoteMessage !== "Voice transcription failed.") return remoteMessage; + } + return "Voice transcription failed."; +} + +export type DesktopVoiceTranscriptionInput = ServerVoiceTranscriptionInput; diff --git a/apps/desktop/src/voice/localVoiceModelManager.test.ts b/apps/desktop/src/voice/localVoiceModelManager.test.ts new file mode 100644 index 000000000..1ac803af7 --- /dev/null +++ b/apps/desktop/src/voice/localVoiceModelManager.test.ts @@ -0,0 +1,175 @@ +// FILE: localVoiceModelManager.test.ts +// Purpose: Verifies resumable, checksum-pinned offline voice model lifecycle behavior. +// Layer: Desktop voice runtime tests + +import { createHash } from "node:crypto"; +import * as FS from "node:fs/promises"; +import * as OS from "node:os"; +import * as Path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { LocalVoiceModelManager } from "./localVoiceModelManager"; +import type { LocalVoiceModelManifest } from "./localVoiceModelManifest"; + +const temporaryDirectories: string[] = []; + +async function makeFixture( + bytes = new Uint8Array([0x6c, 0x6d, 0x67, 0x67, ...new TextEncoder().encode(" model")]), +) { + const directory = await FS.mkdtemp(Path.join(OS.tmpdir(), "scient-voice-model-test-")); + temporaryDirectories.push(directory); + const manifest: LocalVoiceModelManifest = { + id: "test-small-q5", + fileName: "model.bin", + displayName: "Test model", + byteSize: bytes.byteLength, + sha256: createHash("sha256").update(bytes).digest("hex"), + headerHex: "6c6d6767", + sourceRevision: "test-revision", + downloadUrl: "https://models.invalid/model.bin", + license: "MIT", + }; + return { bytes, directory, manifest }; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => FS.rm(directory, { recursive: true, force: true })), + ); +}); + +describe("LocalVoiceModelManager", () => { + it("downloads, verifies, receipts, and reuses a pinned model", async () => { + const fixture = await makeFixture(); + const fetchImpl = vi.fn(async () => new Response(fixture.bytes, { status: 200 })); + const manager = new LocalVoiceModelManager({ + modelsDirectory: fixture.directory, + manifest: fixture.manifest, + fetchImpl: fetchImpl as typeof fetch, + }); + const progress = vi.fn(); + + await expect(manager.getStatus()).resolves.toEqual({ state: "missing" }); + await expect(manager.ensureInstalled(new AbortController().signal, progress)).resolves.toBe( + manager.modelPath, + ); + await expect(manager.verifyInstalledModel()).resolves.toBe(true); + await expect(manager.getStatus()).resolves.toEqual({ + state: "ready", + modelPath: manager.modelPath, + byteSize: fixture.bytes.byteLength, + }); + await manager.ensureInstalled(new AbortController().signal); + + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(progress).toHaveBeenLastCalledWith({ + downloadedBytes: fixture.bytes.byteLength, + totalBytes: fixture.bytes.byteLength, + }); + }); + + it("resumes an interrupted partial download with an HTTP range", async () => { + const fixture = await makeFixture( + new Uint8Array([0x6c, 0x6d, 0x67, 0x67, ...new TextEncoder().encode("456789abcdef")]), + ); + const manager = new LocalVoiceModelManager({ + modelsDirectory: fixture.directory, + manifest: fixture.manifest, + fetchImpl: (async (_url, init) => { + expect(new Headers(init?.headers).get("Range")).toBe("bytes=5-"); + return new Response(fixture.bytes.slice(5), { status: 206 }); + }) as typeof fetch, + }); + await FS.writeFile(manager.partialPath, fixture.bytes.slice(0, 5)); + + await manager.ensureInstalled(new AbortController().signal); + + await expect(manager.verifyInstalledModel()).resolves.toBe(true); + }); + + it("rejects checksum-mismatched content without installing it", async () => { + const fixture = await makeFixture(); + const corrupted = new TextEncoder().encode("corrupted whisper bytes"); + const manager = new LocalVoiceModelManager({ + modelsDirectory: fixture.directory, + manifest: { ...fixture.manifest, byteSize: corrupted.byteLength }, + fetchImpl: (async () => new Response(corrupted, { status: 200 })) as typeof fetch, + }); + + await expect(manager.ensureInstalled(new AbortController().signal)).rejects.toThrow( + /checksum verification failed/i, + ); + await expect(FS.stat(manager.modelPath)).rejects.toMatchObject({ + code: "ENOENT", + }); + await expect(FS.stat(manager.partialPath)).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it("rejects a checksum-valid file with an invalid GGML header", async () => { + const invalidHeader = new TextEncoder().encode("notg-model"); + const fixture = await makeFixture(invalidHeader); + const manager = new LocalVoiceModelManager({ + modelsDirectory: fixture.directory, + manifest: fixture.manifest, + fetchImpl: (async () => new Response(invalidHeader, { status: 200 })) as typeof fetch, + }); + + await expect(manager.ensureInstalled(new AbortController().signal)).rejects.toThrow( + /header verification failed/i, + ); + await expect(FS.stat(manager.modelPath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("removes the installed model and receipt", async () => { + const fixture = await makeFixture(); + const manager = new LocalVoiceModelManager({ + modelsDirectory: fixture.directory, + manifest: fixture.manifest, + fetchImpl: (async () => new Response(fixture.bytes, { status: 200 })) as typeof fetch, + }); + await manager.ensureInstalled(new AbortController().signal); + + await manager.remove(); + + await expect(manager.getStatus()).resolves.toEqual({ state: "missing" }); + }); + + it("repairs through a separate verified file and preserves the installed model on failure", async () => { + const fixture = await makeFixture(); + const fetchImpl = vi + .fn() + .mockResolvedValueOnce(new Response(fixture.bytes, { status: 200 })) + .mockRejectedValueOnce(new Error("offline")); + const manager = new LocalVoiceModelManager({ + modelsDirectory: fixture.directory, + manifest: fixture.manifest, + fetchImpl: fetchImpl as typeof fetch, + }); + await manager.ensureInstalled(new AbortController().signal); + const before = await FS.readFile(manager.modelPath); + + await expect(manager.repair(new AbortController().signal)).rejects.toThrow("offline"); + await expect(FS.readFile(manager.modelPath)).resolves.toEqual(before); + await expect(manager.verifyInstalledModel()).resolves.toBe(true); + await expect(FS.stat(manager.repairPartialPath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("re-downloads and atomically replaces a valid installed model during repair", async () => { + const fixture = await makeFixture(); + const fetchImpl = vi.fn(async () => new Response(fixture.bytes, { status: 200 })); + const manager = new LocalVoiceModelManager({ + modelsDirectory: fixture.directory, + manifest: fixture.manifest, + fetchImpl: fetchImpl as typeof fetch, + }); + await manager.ensureInstalled(new AbortController().signal); + await manager.repair(new AbortController().signal); + + expect(fetchImpl).toHaveBeenCalledTimes(2); + await expect(manager.verifyInstalledModel()).resolves.toBe(true); + }); +}); diff --git a/apps/desktop/src/voice/localVoiceModelManager.ts b/apps/desktop/src/voice/localVoiceModelManager.ts new file mode 100644 index 000000000..afac2f422 --- /dev/null +++ b/apps/desktop/src/voice/localVoiceModelManager.ts @@ -0,0 +1,338 @@ +// FILE: localVoiceModelManager.ts +// Purpose: Downloads, verifies, resumes, repairs, and removes the offline Whisper model. +// Layer: Desktop voice runtime + +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import * as FS from "node:fs/promises"; +import * as Path from "node:path"; + +import type { LocalVoiceModelManifest } from "./localVoiceModelManifest"; + +export type LocalVoiceModelStatus = + | { readonly state: "missing" } + | { + readonly state: "downloading"; + readonly downloadedBytes: number; + readonly totalBytes: number; + readonly readyModelPath?: string; + } + | { + readonly state: "ready"; + readonly modelPath: string; + readonly byteSize: number; + } + | { readonly state: "error"; readonly message: string }; + +export interface LocalVoiceModelDownloadProgress { + readonly downloadedBytes: number; + readonly totalBytes: number; +} + +interface LocalVoiceModelReceipt { + readonly id: string; + readonly fileName: string; + readonly byteSize: number; + readonly sha256: string; + readonly sourceRevision: string; + readonly verifiedAt: string; +} + +export interface LocalVoiceModelManagerOptions { + readonly modelsDirectory: string; + readonly manifest: LocalVoiceModelManifest; + readonly fetchImpl?: typeof fetch; +} + +export class LocalVoiceModelManager { + readonly modelPath: string; + readonly partialPath: string; + readonly repairPartialPath: string; + readonly receiptPath: string; + + private readonly fetchImpl: typeof fetch; + private activeDownload: Promise | null = null; + private activeTransferPath: string | null = null; + private activeOperation: "install" | "repair" | null = null; + + constructor(private readonly options: LocalVoiceModelManagerOptions) { + this.fetchImpl = options.fetchImpl ?? globalThis.fetch; + this.modelPath = Path.join(options.modelsDirectory, options.manifest.fileName); + this.partialPath = `${this.modelPath}.partial`; + this.repairPartialPath = `${this.modelPath}.repair.partial`; + this.receiptPath = `${this.modelPath}.json`; + } + + async getStatus(): Promise { + if (this.activeDownload) { + const downloadedBytes = await fileSize(this.activeTransferPath ?? this.partialPath); + const readyModelPath = + this.activeOperation === "repair" && (await this.hasVerifiedReceipt()) + ? this.modelPath + : undefined; + return { + state: "downloading", + downloadedBytes, + totalBytes: this.options.manifest.byteSize, + ...(readyModelPath ? { readyModelPath } : {}), + }; + } + + const ready = await this.hasVerifiedReceipt(); + return ready + ? { + state: "ready", + modelPath: this.modelPath, + byteSize: this.options.manifest.byteSize, + } + : { state: "missing" }; + } + + isDownloading(): boolean { + return this.activeDownload !== null; + } + + async ensureInstalled( + signal: AbortSignal, + onProgress?: (progress: LocalVoiceModelDownloadProgress) => void, + ): Promise { + if (await this.hasVerifiedReceipt()) { + return this.modelPath; + } + if (this.activeDownload) { + return this.activeDownload; + } + + this.activeTransferPath = this.partialPath; + this.activeOperation = "install"; + const download = this.downloadAndVerify(signal, onProgress, this.partialPath).finally(() => { + if (this.activeDownload === download) { + this.activeDownload = null; + this.activeTransferPath = null; + this.activeOperation = null; + } + }); + this.activeDownload = download; + return download; + } + + async repair( + signal: AbortSignal, + onProgress?: (progress: LocalVoiceModelDownloadProgress) => void, + ): Promise { + if (this.activeDownload) { + throw new Error("The offline voice model is already downloading."); + } + await FS.rm(this.repairPartialPath, { force: true }); + this.activeTransferPath = this.repairPartialPath; + this.activeOperation = "repair"; + const repair = this.downloadAndVerify(signal, onProgress, this.repairPartialPath).finally( + async () => { + if (this.activeDownload === repair) { + this.activeDownload = null; + this.activeTransferPath = null; + this.activeOperation = null; + } + await FS.rm(this.repairPartialPath, { force: true }); + }, + ); + this.activeDownload = repair; + return repair; + } + + async verifyInstalledModel(): Promise { + const stats = await statOrNull(this.modelPath); + if (!stats?.isFile() || stats.size !== this.options.manifest.byteSize) { + return false; + } + return ( + (await sha256File(this.modelPath)) === this.options.manifest.sha256 && + (await fileStartsWithHex(this.modelPath, this.options.manifest.headerHex)) + ); + } + + async remove(): Promise { + if (this.activeDownload) { + throw new Error("The offline voice model cannot be removed while it is downloading."); + } + await Promise.all([ + FS.rm(this.modelPath, { force: true }), + FS.rm(this.partialPath, { force: true }), + FS.rm(this.repairPartialPath, { force: true }), + FS.rm(this.receiptPath, { force: true }), + ]); + } + + private async hasVerifiedReceipt(): Promise { + const [modelStats, receipt] = await Promise.all([ + statOrNull(this.modelPath), + readReceipt(this.receiptPath), + ]); + const manifest = this.options.manifest; + return Boolean( + modelStats?.isFile() && + modelStats.size === manifest.byteSize && + receipt?.id === manifest.id && + receipt.fileName === manifest.fileName && + receipt.byteSize === manifest.byteSize && + receipt.sha256 === manifest.sha256 && + receipt.sourceRevision === manifest.sourceRevision, + ); + } + + private async downloadAndVerify( + signal: AbortSignal, + onProgress?: (progress: LocalVoiceModelDownloadProgress) => void, + partialPath = this.partialPath, + ): Promise { + if (typeof this.fetchImpl !== "function") { + throw new Error("Offline voice model downloads are unavailable in this runtime."); + } + + await FS.mkdir(this.options.modelsDirectory, { + recursive: true, + mode: 0o700, + }); + const manifest = this.options.manifest; + let existingBytes = Math.min(await fileSize(partialPath), manifest.byteSize); + if (existingBytes === manifest.byteSize) { + const valid = (await sha256File(partialPath)) === manifest.sha256; + if (!valid) { + await FS.rm(partialPath, { force: true }); + existingBytes = 0; + } + } + + if (existingBytes < manifest.byteSize) { + const response = await this.fetchImpl(manifest.downloadUrl, { + signal, + ...(existingBytes > 0 ? { headers: { Range: `bytes=${existingBytes}-` } } : {}), + }); + if (!response.ok) { + throw new Error(`Offline voice model download failed with status ${response.status}.`); + } + if (!response.body) { + throw new Error("Offline voice model download returned no data."); + } + + const resumed = existingBytes > 0 && response.status === 206; + if (!resumed && existingBytes > 0) { + existingBytes = 0; + await FS.rm(partialPath, { force: true }); + } + + const handle = await FS.open(partialPath, existingBytes > 0 ? "a" : "w", 0o600); + let downloadedBytes = existingBytes; + try { + const reader = response.body.getReader(); + while (true) { + signal.throwIfAborted(); + const next = await reader.read(); + if (next.done) break; + if (downloadedBytes + next.value.byteLength > manifest.byteSize) { + throw new Error("Offline voice model download exceeded its expected size."); + } + await handle.write(next.value); + downloadedBytes += next.value.byteLength; + onProgress?.({ downloadedBytes, totalBytes: manifest.byteSize }); + } + } finally { + await handle.close(); + } + } + + const downloadedStats = await statOrNull(partialPath); + if (downloadedStats?.size !== manifest.byteSize) { + throw new Error( + `Offline voice model download is incomplete (${downloadedStats?.size ?? 0}/${manifest.byteSize} bytes).`, + ); + } + + const digest = await sha256File(partialPath); + if (digest !== manifest.sha256) { + await FS.rm(partialPath, { force: true }); + throw new Error("Offline voice model checksum verification failed."); + } + if (!(await fileStartsWithHex(partialPath, manifest.headerHex))) { + await FS.rm(partialPath, { force: true }); + throw new Error("Offline voice model header verification failed."); + } + + // Rename only after all verification passes. Node uses replacement rename + // semantics for files, so repair never exposes an unverified model path. + await FS.rename(partialPath, this.modelPath); + const receipt: LocalVoiceModelReceipt = { + id: manifest.id, + fileName: manifest.fileName, + byteSize: manifest.byteSize, + sha256: manifest.sha256, + sourceRevision: manifest.sourceRevision, + verifiedAt: new Date().toISOString(), + }; + const pendingReceiptPath = `${this.receiptPath}.tmp-${process.pid}`; + await FS.writeFile(pendingReceiptPath, `${JSON.stringify(receipt, null, 2)}\n`, { + mode: 0o600, + }); + await FS.rm(this.receiptPath, { force: true }); + await FS.rename(pendingReceiptPath, this.receiptPath); + return this.modelPath; + } +} + +async function statOrNull(path: string) { + try { + return await FS.stat(path); + } catch (error) { + if (isNodeError(error) && error.code === "ENOENT") return null; + throw error; + } +} + +async function fileSize(path: string): Promise { + return (await statOrNull(path))?.size ?? 0; +} + +async function readReceipt(path: string): Promise { + try { + const value = JSON.parse(await FS.readFile(path, "utf8")) as Partial; + if ( + typeof value.id !== "string" || + typeof value.fileName !== "string" || + typeof value.byteSize !== "number" || + typeof value.sha256 !== "string" || + typeof value.sourceRevision !== "string" || + typeof value.verifiedAt !== "string" + ) { + return null; + } + return value as LocalVoiceModelReceipt; + } catch (error) { + if (isNodeError(error) && error.code === "ENOENT") return null; + if (error instanceof SyntaxError) return null; + throw error; + } +} + +async function sha256File(path: string): Promise { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) { + hash.update(chunk as Buffer); + } + return hash.digest("hex"); +} + +async function fileStartsWithHex(path: string, expectedHex: string): Promise { + const expected = Buffer.from(expectedHex, "hex"); + const handle = await FS.open(path, "r"); + try { + const actual = Buffer.alloc(expected.byteLength); + const { bytesRead } = await handle.read(actual, 0, actual.byteLength, 0); + return bytesRead === expected.byteLength && actual.equals(expected); + } finally { + await handle.close(); + } +} + +function isNodeError(value: unknown): value is NodeJS.ErrnoException { + return value instanceof Error && "code" in value; +} diff --git a/apps/desktop/src/voice/localVoiceModelManifest.ts b/apps/desktop/src/voice/localVoiceModelManifest.ts new file mode 100644 index 000000000..24f481720 --- /dev/null +++ b/apps/desktop/src/voice/localVoiceModelManifest.ts @@ -0,0 +1,30 @@ +// FILE: localVoiceModelManifest.ts +// Purpose: Pins the single supported offline multilingual Whisper model and its provenance. +// Layer: Desktop voice runtime + +export interface LocalVoiceModelManifest { + readonly id: string; + readonly fileName: string; + readonly displayName: string; + readonly byteSize: number; + readonly sha256: string; + readonly headerHex: "6c6d6767"; + readonly sourceRevision: string; + readonly downloadUrl: string; + readonly license: "MIT"; +} + +const SOURCE_REVISION = "5359861c739e955e79d9a303bcbc70fb988958b1"; + +export const LOCAL_VOICE_MODEL: LocalVoiceModelManifest = Object.freeze({ + id: "whisper-small-multilingual-q5_1", + fileName: "ggml-small-q5_1-ae85e4a9.bin", + displayName: "Multilingual Small", + byteSize: 190_085_487, + sha256: "ae85e4a935d7a567bd102fe55afc16bb595bdb618e11b2fc7591bc08120411bb", + // whisper.cpp GGML model magic: bytes 6c 6d 67 67 ("lmgg"). + headerHex: "6c6d6767", + sourceRevision: SOURCE_REVISION, + downloadUrl: `https://huggingface.co/ggerganov/whisper.cpp/resolve/${SOURCE_REVISION}/ggml-small-q5_1.bin?download=true`, + license: "MIT", +}); diff --git a/apps/desktop/src/voice/localWhisperBackend.test.ts b/apps/desktop/src/voice/localWhisperBackend.test.ts new file mode 100644 index 000000000..767e614b0 --- /dev/null +++ b/apps/desktop/src/voice/localWhisperBackend.test.ts @@ -0,0 +1,98 @@ +// FILE: localWhisperBackend.test.ts +// Purpose: Verifies offline model/runtime availability and safe local failure normalization. +// Layer: Desktop voice runtime tests + +import type { NormalizedVoiceClip } from "@synara/shared/voiceTranscription"; +import { describe, expect, it, vi } from "vitest"; +import { LocalWhisperBackend, type LocalWhisperRuntimeLike } from "./localWhisperBackend"; +import { LocalWhisperRuntimeError } from "./localWhisperRuntime"; + +const clip: NormalizedVoiceClip = { + audioBytes: new Uint8Array([1]), + mimeType: "audio/wav", + sampleRateHz: 24_000, + durationMs: 100, + cwd: "/workspace", +}; + +function modelManager(state: "missing" | "ready" | "downloading") { + return { + getStatus: vi.fn(async () => + state === "ready" + ? ({ state: "ready", modelPath: "/models/small.bin", byteSize: 10 } as const) + : state === "downloading" + ? ({ state: "downloading", downloadedBytes: 1, totalBytes: 10 } as const) + : ({ state: "missing" } as const), + ), + }; +} + +function runtime(overrides: Partial = {}): LocalWhisperRuntimeLike { + return { + isInstalled: vi.fn(async () => true), + isBusy: vi.fn(() => false), + stopIdle: vi.fn(async () => undefined), + transcribe: vi.fn(async () => ({ text: "offline transcript" })), + dispose: vi.fn(async () => undefined), + ...overrides, + }; +} + +describe("LocalWhisperBackend", () => { + it("requires setup when the model has not been installed", async () => { + const backend = new LocalWhisperBackend(modelManager("missing") as never, runtime()); + await expect(backend.getAvailability()).resolves.toEqual({ + state: "requires-setup", + reason: "model-not-installed", + }); + }); + + it("reports ready and transcribes with the verified model path", async () => { + const whisperRuntime = runtime(); + const backend = new LocalWhisperBackend(modelManager("ready") as never, whisperRuntime); + const signal = new AbortController().signal; + + await expect(backend.getAvailability()).resolves.toEqual({ state: "ready" }); + await expect(backend.transcribe(clip, { signal })).resolves.toEqual({ + text: "offline transcript", + }); + expect(whisperRuntime.transcribe).toHaveBeenCalledWith("/models/small.bin", clip, signal); + }); + + it("normalizes runtime crashes as safe non-fallback local failures", async () => { + const backend = new LocalWhisperBackend( + modelManager("ready") as never, + runtime({ + transcribe: vi.fn(async () => { + throw new Error("native crash details"); + }), + }), + ); + + await expect( + backend.transcribe(clip, { signal: new AbortController().signal }), + ).rejects.toMatchObject({ + kind: "provider-error", + fallbackAllowed: false, + safeMessage: "Offline voice transcription failed.", + }); + }); + + it("preserves the typed inference timeout", async () => { + const backend = new LocalWhisperBackend( + modelManager("ready") as never, + runtime({ + transcribe: vi.fn(async () => { + throw new LocalWhisperRuntimeError("timeout", "timed out"); + }), + }), + ); + + await expect( + backend.transcribe(clip, { signal: new AbortController().signal }), + ).rejects.toMatchObject({ + kind: "timeout", + safeMessage: "Offline voice transcription timed out.", + }); + }); +}); diff --git a/apps/desktop/src/voice/localWhisperBackend.ts b/apps/desktop/src/voice/localWhisperBackend.ts new file mode 100644 index 000000000..5d4b4c952 --- /dev/null +++ b/apps/desktop/src/voice/localWhisperBackend.ts @@ -0,0 +1,128 @@ +// FILE: localWhisperBackend.ts +// Purpose: Implements the provider-neutral local backend with the verified Small Q5 model. +// Layer: Desktop voice runtime + +import { + type NormalizedVoiceClip, + type VoiceTranscriptionBackend, + type VoiceTranscriptionBackendAvailability, + VoiceTranscriptionBackendError, + type VoiceTranscript, +} from "@synara/shared/voiceTranscription"; +import type { LocalVoiceModelManager } from "./localVoiceModelManager"; +import { LocalWhisperRuntimeError } from "./localWhisperRuntime"; + +export interface LocalWhisperRuntimeLike { + isInstalled(): Promise; + isBusy(): boolean; + stopIdle(): Promise; + transcribe( + modelPath: string, + clip: NormalizedVoiceClip, + signal: AbortSignal, + ): Promise; + dispose(): Promise; +} + +export class LocalWhisperBackend implements VoiceTranscriptionBackend { + readonly id = "local" as const; + + constructor( + private readonly modelManager: LocalVoiceModelManager, + private readonly runtime: LocalWhisperRuntimeLike, + private readonly isMaintenanceActive: () => boolean = () => false, + ) {} + + async getAvailability(): Promise { + if (this.isMaintenanceActive()) { + return { + state: "temporarily-unavailable", + reason: "Offline voice model maintenance is in progress.", + }; + } + const runtimeInstalled = await this.runtime.isInstalled(); + if (!runtimeInstalled) { + return { state: "unavailable", reason: "The bundled offline voice runtime is missing." }; + } + const modelStatus = await this.modelManager.getStatus(); + if ( + modelStatus.state === "ready" || + (modelStatus.state === "downloading" && modelStatus.readyModelPath) + ) { + return { state: "ready" }; + } + if (modelStatus.state === "downloading") { + return { + state: "temporarily-unavailable", + reason: "The offline voice model is still downloading.", + }; + } + return { state: "requires-setup", reason: "model-not-installed" }; + } + + async transcribe( + clip: NormalizedVoiceClip, + options: { readonly signal: AbortSignal }, + ): Promise { + options.signal.throwIfAborted(); + if (this.isMaintenanceActive()) { + throw new VoiceTranscriptionBackendError({ + kind: "backend-unavailable", + fallbackAllowed: false, + safeMessage: "Wait for offline voice model maintenance to finish.", + }); + } + const status = await this.modelManager.getStatus(); + const modelPath = + status.state === "ready" + ? status.modelPath + : status.state === "downloading" + ? status.readyModelPath + : undefined; + if (!modelPath) { + throw new VoiceTranscriptionBackendError({ + kind: "backend-unavailable", + fallbackAllowed: false, + safeMessage: "Set up offline voice transcription before using the microphone.", + }); + } + try { + return await this.runtime.transcribe(modelPath, clip, options.signal); + } catch (error) { + if (options.signal.aborted) { + throw new VoiceTranscriptionBackendError({ + kind: "cancelled", + fallbackAllowed: false, + safeMessage: "Voice transcription was cancelled.", + cause: error, + }); + } + if (error instanceof LocalWhisperRuntimeError && error.kind === "timeout") { + throw new VoiceTranscriptionBackendError({ + kind: "timeout", + fallbackAllowed: false, + safeMessage: "Offline voice transcription timed out.", + cause: error, + }); + } + if (error instanceof LocalWhisperRuntimeError && error.kind === "disposed") { + throw new VoiceTranscriptionBackendError({ + kind: "backend-unavailable", + fallbackAllowed: false, + safeMessage: "Offline voice transcription is shutting down.", + cause: error, + }); + } + throw new VoiceTranscriptionBackendError({ + kind: "provider-error", + fallbackAllowed: false, + safeMessage: "Offline voice transcription failed.", + cause: error, + }); + } + } + + dispose(): Promise { + return this.runtime.dispose(); + } +} diff --git a/apps/desktop/src/voice/localWhisperRuntime.test.ts b/apps/desktop/src/voice/localWhisperRuntime.test.ts new file mode 100644 index 000000000..50e03ce54 --- /dev/null +++ b/apps/desktop/src/voice/localWhisperRuntime.test.ts @@ -0,0 +1,161 @@ +// FILE: localWhisperRuntime.test.ts +// Purpose: Verifies private loopback runtime paths and non-shell process arguments. +// Layer: Desktop voice runtime tests + +import { EventEmitter } from "node:events"; +import { describe, expect, it, vi } from "vitest"; +import type { NormalizedVoiceClip } from "@synara/shared/voiceTranscription"; +import { + buildWhisperServerArguments, + LocalWhisperRuntime, + lowerWhisperProcessPriority, + resolveWhisperRuntimePaths, + resolveWhisperInferenceTimeoutMs, +} from "./localWhisperRuntime"; + +describe("localWhisperRuntime", () => { + it("resolves packaged Windows and development runtime binaries", () => { + expect( + resolveWhisperRuntimePaths({ + isPackaged: true, + resourcesPath: "C:\\Scient\\resources", + desktopRuntimeDirectory: "C:\\repo\\.electron-runtime", + platform: "win32", + }), + ).toEqual({ + runtimeDirectory: "C:\\Scient\\resources\\whisper-runtime", + executablePath: "C:\\Scient\\resources\\whisper-runtime\\whisper-server.exe", + }); + expect( + resolveWhisperRuntimePaths({ + isPackaged: false, + resourcesPath: "/Applications/Scient.app/Contents/Resources", + desktopRuntimeDirectory: "/repo/.electron-runtime", + platform: "darwin", + }), + ).toEqual({ + runtimeDirectory: "/repo/.electron-runtime/whisper-runtime", + executablePath: "/repo/.electron-runtime/whisper-runtime/whisper-server", + }); + }); + + it("builds explicit loopback-only arguments with multilingual auto-detection", () => { + expect( + buildWhisperServerArguments({ + modelPath: "/models/small q5.bin", + port: 43_210, + requestPath: "/scient-secret", + threads: 4, + }), + ).toEqual([ + "--model", + "/models/small q5.bin", + "--host", + "127.0.0.1", + "--port", + "43210", + "--request-path", + "/scient-secret", + "--inference-path", + "/inference", + "--threads", + "4", + "--language", + "auto", + "--no-timestamps", + ]); + }); + + it("best-effort lowers helper priority without making startup depend on it", () => { + const calls: Array<[number, number]> = []; + expect(lowerWhisperProcessPriority(42, (pid, priority) => calls.push([pid, priority]))).toBe( + true, + ); + expect(calls).toEqual([[42, 10]]); + expect( + lowerWhisperProcessPriority(42, () => { + throw new Error("unsupported"); + }), + ).toBe(false); + }); + + it("scales the hard inference deadline for long recordings with a release-safe cap", () => { + expect(resolveWhisperInferenceTimeoutMs(5_000)).toBe(45_000); + expect(resolveWhisperInferenceTimeoutMs(120_000)).toBe(360_000); + expect(resolveWhisperInferenceTimeoutMs(999_000)).toBe(360_000); + }); + + it("makes disposal terminal so queued work cannot respawn the helper", async () => { + const runtime = new LocalWhisperRuntime({ runtimeDirectory: "/missing" }); + let releaseQueue!: () => void; + const queuedBehind = new Promise((resolve) => { + releaseQueue = resolve; + }); + (runtime as unknown as { queueTail: Promise }).queueTail = queuedBehind; + const clip: NormalizedVoiceClip = { + audioBytes: new Uint8Array([1]), + mimeType: "audio/wav", + sampleRateHz: 24_000, + durationMs: 1, + cwd: "/workspace", + }; + const queued = runtime.transcribe("/model.bin", clip, new AbortController().signal); + const disposing = runtime.dispose(); + releaseQueue(); + + await expect(queued).rejects.toMatchObject({ kind: "disposed" }); + await expect(disposing).resolves.toBeUndefined(); + await expect( + runtime.transcribe("/model.bin", clip, new AbortController().signal), + ).rejects.toMatchObject({ kind: "disposed" }); + }); + + it("cannot spawn a helper when disposal races in-flight startup", async () => { + const runtime = new LocalWhisperRuntime({ runtimeDirectory: "/runtime" }); + let releaseInstallationCheck!: () => void; + const installationCheck = new Promise((resolve) => { + releaseInstallationCheck = resolve; + }); + const installed = vi.spyOn(runtime, "isInstalled").mockImplementation(async () => { + await installationCheck; + return true; + }); + const clip: NormalizedVoiceClip = { + audioBytes: new Uint8Array([1]), + mimeType: "audio/wav", + sampleRateHz: 24_000, + durationMs: 1, + cwd: "/workspace", + }; + + const transcription = runtime.transcribe("/model.bin", clip, new AbortController().signal); + await vi.waitFor(() => expect(installed).toHaveBeenCalledOnce()); + const disposing = runtime.dispose(); + releaseInstallationCheck(); + + await expect(transcription).rejects.toMatchObject({ kind: "disposed" }); + await expect(disposing).resolves.toBeUndefined(); + expect((runtime as unknown as { child: unknown }).child).toBeNull(); + }); + + it("stops and waits for an idle helper before model mutation", async () => { + const runtime = new LocalWhisperRuntime({ runtimeDirectory: "/missing" }); + const child = new EventEmitter() as EventEmitter & { + exitCode: number | null; + killed: boolean; + kill: ReturnType; + }; + child.exitCode = null; + child.killed = false; + child.kill = vi.fn(() => { + child.killed = true; + child.exitCode = 0; + queueMicrotask(() => child.emit("exit", 0, null)); + return true; + }); + (runtime as unknown as { child: typeof child }).child = child; + + await expect(runtime.stopIdle()).resolves.toBeUndefined(); + expect(child.kill).toHaveBeenCalledOnce(); + }); +}); diff --git a/apps/desktop/src/voice/localWhisperRuntime.ts b/apps/desktop/src/voice/localWhisperRuntime.ts new file mode 100644 index 000000000..e79e1f0e5 --- /dev/null +++ b/apps/desktop/src/voice/localWhisperRuntime.ts @@ -0,0 +1,477 @@ +// FILE: localWhisperRuntime.ts +// Purpose: Owns the private loopback whisper.cpp server process and serialized inference requests. +// Layer: Desktop voice runtime + +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import * as FS from "node:fs/promises"; +import { createServer } from "node:net"; +import * as OS from "node:os"; +import * as Path from "node:path"; + +import type { NormalizedVoiceClip, VoiceTranscript } from "@synara/shared/voiceTranscription"; + +const DEFAULT_IDLE_TIMEOUT_MS = 5 * 60_000; +const STARTUP_TIMEOUT_MS = 30_000; +const DEFAULT_INFERENCE_TIMEOUT_MS = 45_000; +const MAX_INFERENCE_TIMEOUT_MS = 6 * 60_000; +const STOP_TIMEOUT_MS = 5_000; +const FORCE_STOP_TIMEOUT_MS = 2_000; + +export interface LocalWhisperRuntimeOptions { + readonly runtimeDirectory: string; + readonly fetchImpl?: typeof fetch; + readonly idleTimeoutMs?: number; + readonly threads?: number; + readonly inferenceTimeoutMs?: number; +} + +export interface WhisperRuntimePaths { + readonly runtimeDirectory: string; + readonly executablePath: string; +} + +export class LocalWhisperRuntimeError extends Error { + constructor( + readonly kind: "timeout" | "busy" | "disposed", + message: string, + ) { + super(message); + this.name = "LocalWhisperRuntimeError"; + } +} + +export function resolveWhisperInferenceTimeoutMs( + clipDurationMs: number, + configuredFloorMs = DEFAULT_INFERENCE_TIMEOUT_MS, +): number { + return Math.min( + MAX_INFERENCE_TIMEOUT_MS, + Math.max(configuredFloorMs, Math.max(1, clipDurationMs) * 3), + ); +} + +export function resolveWhisperRuntimePaths(input: { + readonly isPackaged: boolean; + readonly resourcesPath: string; + readonly desktopRuntimeDirectory: string; + readonly platform?: NodeJS.Platform; +}): WhisperRuntimePaths { + const platform = input.platform ?? process.platform; + const pathApi = platform === "win32" ? Path.win32 : Path.posix; + const runtimeDirectory = input.isPackaged + ? pathApi.join(input.resourcesPath, "whisper-runtime") + : pathApi.join(input.desktopRuntimeDirectory, "whisper-runtime"); + return { + runtimeDirectory, + executablePath: pathApi.join( + runtimeDirectory, + platform === "win32" ? "whisper-server.exe" : "whisper-server", + ), + }; +} + +export function buildWhisperServerArguments(input: { + readonly modelPath: string; + readonly port: number; + readonly requestPath: string; + readonly threads: number; +}): string[] { + return [ + "--model", + input.modelPath, + "--host", + "127.0.0.1", + "--port", + String(input.port), + "--request-path", + input.requestPath, + "--inference-path", + "/inference", + "--threads", + String(input.threads), + "--language", + "auto", + "--no-timestamps", + ]; +} + +export class LocalWhisperRuntime { + private readonly fetchImpl: typeof fetch; + private readonly idleTimeoutMs: number; + private readonly threads: number; + private readonly inferenceTimeoutMs: number; + private child: ChildProcessWithoutNullStreams | null = null; + private endpoint: string | null = null; + private activeModelPath: string | null = null; + private starting: Promise | null = null; + private idleTimer: NodeJS.Timeout | null = null; + private queueTail: Promise = Promise.resolve(); + private activeRequests = 0; + private disposed = false; + private readonly lifecycleController = new AbortController(); + + constructor(private readonly options: LocalWhisperRuntimeOptions) { + this.fetchImpl = options.fetchImpl ?? globalThis.fetch; + this.idleTimeoutMs = options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS; + this.threads = Math.max(1, Math.min(4, options.threads ?? 4)); + this.inferenceTimeoutMs = Math.max( + 1_000, + options.inferenceTimeoutMs ?? DEFAULT_INFERENCE_TIMEOUT_MS, + ); + } + + isBusy(): boolean { + return this.activeRequests > 0; + } + + async isInstalled(): Promise { + const { executablePath } = this.paths(); + try { + const stats = await FS.stat(executablePath); + return stats.isFile(); + } catch (error) { + if (isNodeError(error) && error.code === "ENOENT") return false; + throw error; + } + } + + async transcribe( + modelPath: string, + clip: NormalizedVoiceClip, + signal: AbortSignal, + ): Promise { + if (this.disposed) { + throw new LocalWhisperRuntimeError("disposed", "Offline voice runtime is shutting down."); + } + this.activeRequests += 1; + const previous = this.queueTail; + let releaseQueue!: () => void; + this.queueTail = new Promise((resolve) => { + releaseQueue = resolve; + }); + await previous; + try { + if (this.disposed) { + throw new LocalWhisperRuntimeError("disposed", "Offline voice runtime is shutting down."); + } + const runtimeSignal = AbortSignal.any([signal, this.lifecycleController.signal]); + runtimeSignal.throwIfAborted(); + return await this.transcribeNow(modelPath, clip, runtimeSignal); + } finally { + releaseQueue(); + this.activeRequests -= 1; + } + } + + async dispose(): Promise { + if (!this.disposed) { + this.disposed = true; + this.lifecycleController.abort( + new LocalWhisperRuntimeError("disposed", "Offline voice runtime is shutting down."), + ); + this.clearIdleTimer(); + this.stopProcess(); + } + await this.queueTail; + } + + async stopIdle(): Promise { + if (this.activeRequests > 0) { + throw new LocalWhisperRuntimeError("busy", "Offline voice transcription is active."); + } + const child = this.child; + this.stopProcess(); + if (!child || child.exitCode !== null) return; + + if (await waitForChildExit(child, STOP_TIMEOUT_MS)) return; + if (child.exitCode === null) child.kill("SIGKILL"); + if (!(await waitForChildExit(child, FORCE_STOP_TIMEOUT_MS))) { + throw new Error("Offline voice helper did not stop before model maintenance."); + } + } + + private async transcribeNow( + modelPath: string, + clip: NormalizedVoiceClip, + signal: AbortSignal, + ): Promise { + const endpoint = await this.ensureStarted(modelPath, signal); + signal.throwIfAborted(); + this.clearIdleTimer(); + + const abortRuntime = () => this.stopProcess(); + signal.addEventListener("abort", abortRuntime, { once: true }); + const inferenceController = new AbortController(); + const forwardAbort = () => inferenceController.abort(signal.reason); + signal.addEventListener("abort", forwardAbort, { once: true }); + const timeout = setTimeout( + () => { + inferenceController.abort( + new LocalWhisperRuntimeError("timeout", "Offline voice transcription timed out."), + ); + this.stopProcess(); + }, + resolveWhisperInferenceTimeoutMs(clip.durationMs, this.inferenceTimeoutMs), + ); + timeout.unref(); + try { + const form = new FormData(); + form.append( + "file", + new Blob([Buffer.from(clip.audioBytes)], { type: clip.mimeType }), + "voice.wav", + ); + form.append("response_format", "json"); + form.append("language", "auto"); + form.append("temperature", "0.0"); + form.append("temperature_inc", "0.2"); + const response = await this.fetchImpl(endpoint, { + method: "POST", + body: form, + signal: inferenceController.signal, + }); + if (!response.ok) { + throw new Error(`Offline transcription failed with status ${response.status}.`); + } + const payload = (await response.json().catch(() => null)) as { text?: unknown } | null; + const text = typeof payload?.text === "string" ? payload.text.trim() : ""; + if (!text) { + throw new Error("Offline transcription returned no text."); + } + return { text }; + } catch (error) { + if (inferenceController.signal.reason instanceof LocalWhisperRuntimeError) { + throw inferenceController.signal.reason; + } + throw error; + } finally { + clearTimeout(timeout); + signal.removeEventListener("abort", abortRuntime); + signal.removeEventListener("abort", forwardAbort); + if (!this.disposed) this.armIdleTimer(); + } + } + + private async ensureStarted(modelPath: string, signal: AbortSignal): Promise { + if (this.endpoint && this.child && this.activeModelPath === modelPath) { + return this.endpoint; + } + if (this.activeModelPath !== modelPath) { + this.stopProcess(); + } + if (this.starting) return this.starting; + + const starting = this.startProcess(modelPath, signal).finally(() => { + if (this.starting === starting) this.starting = null; + }); + this.starting = starting; + return starting; + } + + private async startProcess(modelPath: string, signal: AbortSignal): Promise { + if (typeof this.fetchImpl !== "function") { + throw new Error("Offline transcription is unavailable in this runtime."); + } + const paths = this.paths(); + if (!(await this.isInstalled())) { + throw new Error("The bundled offline transcription runtime is missing."); + } + signal.throwIfAborted(); + + const port = await reserveLoopbackPort(); + signal.throwIfAborted(); + const requestPath = `/scient-${randomBytes(24).toString("hex")}`; + const endpoint = `http://127.0.0.1:${port}${requestPath}/inference`; + const child = spawn( + paths.executablePath, + buildWhisperServerArguments({ modelPath, port, requestPath, threads: this.threads }), + { + cwd: paths.runtimeDirectory, + env: runtimeEnvironment(paths.runtimeDirectory), + shell: false, + windowsHide: true, + stdio: ["pipe", "pipe", "pipe"], + }, + ); + this.child = child; + this.activeModelPath = modelPath; + lowerWhisperProcessPriority(child.pid); + child.stdout.on("data", () => undefined); + child.stderr.on("data", () => undefined); + child.once("exit", () => { + if (this.child === child) { + this.child = null; + this.endpoint = null; + this.activeModelPath = null; + } + }); + let rejectSpawn: ((error: Error) => void) | null = null; + const spawnFailure = new Promise((_resolve, reject) => { + rejectSpawn = reject; + }); + const onSpawnError = (error: Error) => rejectSpawn?.(error); + child.once("error", onSpawnError); + child.on("error", () => { + if (this.child === child) this.stopProcess(); + }); + + try { + await Promise.race([ + waitForServer({ + child, + endpoint, + fetchImpl: this.fetchImpl, + signal, + timeoutMs: STARTUP_TIMEOUT_MS, + }), + spawnFailure, + ]); + this.endpoint = endpoint; + this.armIdleTimer(); + return endpoint; + } catch (error) { + this.stopProcess(); + throw error; + } finally { + child.removeListener("error", onSpawnError); + } + } + + private paths(): WhisperRuntimePaths { + const executableName = process.platform === "win32" ? "whisper-server.exe" : "whisper-server"; + return { + runtimeDirectory: this.options.runtimeDirectory, + executablePath: Path.join(this.options.runtimeDirectory, executableName), + }; + } + + private armIdleTimer(): void { + this.clearIdleTimer(); + this.idleTimer = setTimeout(() => this.stopProcess(), this.idleTimeoutMs); + this.idleTimer.unref(); + } + + private clearIdleTimer(): void { + if (this.idleTimer) clearTimeout(this.idleTimer); + this.idleTimer = null; + } + + private stopProcess(): void { + this.clearIdleTimer(); + const child = this.child; + this.child = null; + this.endpoint = null; + this.activeModelPath = null; + if (child && child.exitCode === null && !child.killed) child.kill(); + } +} + +async function waitForChildExit( + child: ChildProcessWithoutNullStreams, + timeoutMs: number, +): Promise { + if (child.exitCode !== null) return true; + return new Promise((resolve) => { + let settled = false; + const finish = (exited: boolean) => { + if (settled) return; + settled = true; + clearTimeout(timer); + child.removeListener("exit", onExit); + resolve(exited); + }; + const onExit = () => finish(true); + const timer = setTimeout(() => finish(child.exitCode !== null), timeoutMs); + timer.unref(); + child.once("exit", onExit); + }); +} + +export function lowerWhisperProcessPriority( + pid: number | undefined, + setPriority: (pid: number, priority: number) => void = OS.setPriority, +): boolean { + if (!pid) return false; + try { + setPriority(pid, 10); + return true; + } catch { + return false; + } +} + +function runtimeEnvironment(runtimeDirectory: string): NodeJS.ProcessEnv { + const environment = { ...process.env }; + if (process.platform === "linux") { + environment.LD_LIBRARY_PATH = [runtimeDirectory, process.env.LD_LIBRARY_PATH] + .filter(Boolean) + .join(":"); + } + return environment; +} + +async function reserveLoopbackPort(): Promise { + return new Promise((resolve, reject) => { + const server = createServer(); + server.unref(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + server.close((error) => { + if (error) reject(error); + else if (port <= 0) reject(new Error("Could not reserve an offline transcription port.")); + else resolve(port); + }); + }); + }); +} + +async function waitForServer(input: { + readonly child: ChildProcessWithoutNullStreams; + readonly endpoint: string; + readonly fetchImpl: typeof fetch; + readonly signal: AbortSignal; + readonly timeoutMs: number; +}): Promise { + const startedAt = Date.now(); + while (Date.now() - startedAt < input.timeoutMs) { + input.signal.throwIfAborted(); + if (input.child.exitCode !== null) { + throw new Error(`Offline transcription runtime exited with code ${input.child.exitCode}.`); + } + try { + const response = await input.fetchImpl(input.endpoint, { + method: "OPTIONS", + signal: input.signal, + }); + if (response.ok) return; + } catch (error) { + if (input.signal.aborted) throw error; + } + await delay(100, input.signal); + } + throw new Error("Timed out while starting offline transcription."); +} + +async function delay(milliseconds: number, signal: AbortSignal): Promise { + await new Promise((resolve, reject) => { + const finish = () => { + signal.removeEventListener("abort", onAbort); + resolve(); + }; + const timeout = setTimeout(finish, milliseconds); + const onAbort = () => { + clearTimeout(timeout); + signal.removeEventListener("abort", onAbort); + reject(signal.reason); + }; + signal.addEventListener("abort", onAbort, { once: true }); + timeout.unref(); + }); +} + +function isNodeError(value: unknown): value is NodeJS.ErrnoException { + return value instanceof Error && "code" in value; +} diff --git a/apps/desktop/src/voice/voiceIpcChannels.ts b/apps/desktop/src/voice/voiceIpcChannels.ts new file mode 100644 index 000000000..6c8275565 --- /dev/null +++ b/apps/desktop/src/voice/voiceIpcChannels.ts @@ -0,0 +1,12 @@ +// FILE: voiceIpcChannels.ts +// Purpose: Defines dependency-free IPC names shared by Electron main and preload. +// Layer: Desktop voice IPC contract + +export const DESKTOP_VOICE_IPC_CHANNELS = { + transcribe: "desktop:server-transcribe-voice", + cancelTranscription: "desktop:voice-cancel-transcription", + getState: "desktop:voice-get-state", + downloadModel: "desktop:voice-download-model", + removeModel: "desktop:voice-remove-model", + repairModel: "desktop:voice-repair-model", +} as const; diff --git a/apps/desktop/src/voice/voiceRequest.test.ts b/apps/desktop/src/voice/voiceRequest.test.ts new file mode 100644 index 000000000..a5e48b46c --- /dev/null +++ b/apps/desktop/src/voice/voiceRequest.test.ts @@ -0,0 +1,74 @@ +// FILE: voiceRequest.test.ts +// Purpose: Verifies the provider-neutral desktop IPC audio boundary. +// Layer: Desktop voice tests + +import { describe, expect, it } from "vitest"; +import { normalizeDesktopVoiceRequest } from "./voiceRequest"; + +function wavBase64(sampleRateHz = 24_000): string { + const bytes = Buffer.alloc(46); + bytes.write("RIFF", 0, "ascii"); + bytes.writeUInt32LE(38, 4); + bytes.write("WAVEfmt ", 8, "ascii"); + bytes.writeUInt32LE(16, 16); + bytes.writeUInt16LE(1, 20); + bytes.writeUInt16LE(1, 22); + bytes.writeUInt32LE(sampleRateHz, 24); + bytes.writeUInt32LE(sampleRateHz * 2, 28); + bytes.writeUInt16LE(2, 32); + bytes.writeUInt16LE(16, 34); + bytes.write("data", 36, "ascii"); + bytes.writeUInt32LE(2, 40); + return bytes.toString("base64"); +} + +describe("normalizeDesktopVoiceRequest", () => { + it("decodes a validated WAV once and defaults to automatic routing", () => { + const result = normalizeDesktopVoiceRequest({ + provider: "claudeAgent", + cwd: " /workspace ", + threadId: "thread-1", + mimeType: "audio/wav", + sampleRateHz: 24_000, + durationMs: 1, + audioBase64: wavBase64(), + }); + + expect(result.mode).toBe("automatic"); + expect(result.clip).toMatchObject({ + cwd: "/workspace", + threadId: "thread-1", + sampleRateHz: 24_000, + mimeType: "audio/wav", + }); + expect(result.clip.audioBytes.byteLength).toBe(46); + }); + + it("accepts the provider-neutral offline-only mode", () => { + expect( + normalizeDesktopVoiceRequest({ + cwd: "/workspace", + mode: "offline-only", + mimeType: "audio/wav", + sampleRateHz: 24_000, + durationMs: 1, + audioBase64: wavBase64(), + }).mode, + ).toBe("offline-only"); + }); + + it("rejects malformed and wrong-format WAV before either backend", () => { + const base = { + cwd: "/workspace", + mimeType: "audio/wav", + sampleRateHz: 24_000, + durationMs: 1, + }; + expect(() => normalizeDesktopVoiceRequest({ ...base, audioBase64: "not base64" })).toThrow( + "The recorded audio could not be decoded.", + ); + expect(() => normalizeDesktopVoiceRequest({ ...base, audioBase64: wavBase64(16_000) })).toThrow( + "The recorded audio is not a valid 24 kHz mono PCM WAV file.", + ); + }); +}); diff --git a/apps/desktop/src/voice/voiceRequest.ts b/apps/desktop/src/voice/voiceRequest.ts new file mode 100644 index 000000000..9be8173de --- /dev/null +++ b/apps/desktop/src/voice/voiceRequest.ts @@ -0,0 +1,113 @@ +// FILE: voiceRequest.ts +// Purpose: Validates untrusted renderer voice payloads once before provider routing. +// Layer: Desktop voice IPC boundary + +import { Buffer } from "node:buffer"; + +import { + type NormalizedVoiceClip, + VoiceTranscriptionBackendError, +} from "@synara/shared/voiceTranscription"; +import type { VoiceTranscriptionMode } from "@synara/shared/voiceTranscriptionRouter"; + +const MAX_AUDIO_BYTES = 10 * 1024 * 1024; +const MAX_AUDIO_BASE64_CHARS = Math.ceil((MAX_AUDIO_BYTES * 4) / 3) + 4; +const MAX_DURATION_MS = 120_000; +const TARGET_SAMPLE_RATE_HZ = 24_000; + +export interface NormalizedDesktopVoiceRequest { + readonly clip: NormalizedVoiceClip; + readonly mode: VoiceTranscriptionMode; +} + +export function normalizeDesktopVoiceRequest(input: unknown): NormalizedDesktopVoiceRequest { + if (!isRecord(input)) throw invalidAudio("The voice transcription request is invalid."); + const cwd = readRequiredString(input.cwd, "The voice transcription workspace is missing."); + const mimeType = readRequiredString(input.mimeType, "The recorded audio type is missing."); + if (mimeType !== "audio/wav") { + throw invalidAudio("Only WAV audio is supported for voice transcription."); + } + if (input.sampleRateHz !== TARGET_SAMPLE_RATE_HZ) { + throw invalidAudio("Voice transcription requires 24 kHz mono WAV audio."); + } + if ( + !Number.isInteger(input.durationMs) || + (input.durationMs as number) <= 0 || + (input.durationMs as number) > MAX_DURATION_MS + ) { + throw invalidAudio("Voice messages must be between 1 ms and 120 seconds."); + } + const encoded = readRequiredString( + input.audioBase64, + "The recorded audio could not be decoded.", + ).replace(/\s+/gu, ""); + if (encoded.length > MAX_AUDIO_BASE64_CHARS || !/^[A-Za-z0-9+/]+={0,2}$/u.test(encoded)) { + throw invalidAudio("The recorded audio could not be decoded."); + } + const bytes = Buffer.from(encoded, "base64"); + if ( + bytes.byteLength === 0 || + bytes.byteLength > MAX_AUDIO_BYTES || + bytes.toString("base64") !== encoded + ) { + throw invalidAudio( + bytes.byteLength > MAX_AUDIO_BYTES + ? "Voice messages are limited to 10 MB." + : "The recorded audio could not be decoded.", + ); + } + validateScientWav(bytes); + + const threadId = + typeof input.threadId === "string" && input.threadId.trim().length > 0 + ? input.threadId.trim() + : undefined; + const mode: VoiceTranscriptionMode = input.mode === "offline-only" ? "offline-only" : "automatic"; + return { + clip: { + audioBytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength), + mimeType: "audio/wav", + sampleRateHz: TARGET_SAMPLE_RATE_HZ, + durationMs: input.durationMs as number, + cwd, + ...(threadId !== undefined ? { threadId } : {}), + }, + mode, + }; +} + +function validateScientWav(bytes: Buffer): void { + if ( + bytes.byteLength < 44 || + bytes.toString("ascii", 0, 4) !== "RIFF" || + bytes.toString("ascii", 8, 12) !== "WAVE" || + bytes.toString("ascii", 12, 16) !== "fmt " || + bytes.readUInt32LE(16) !== 16 || + bytes.readUInt16LE(20) !== 1 || + bytes.readUInt16LE(22) !== 1 || + bytes.readUInt32LE(24) !== TARGET_SAMPLE_RATE_HZ || + bytes.readUInt16LE(34) !== 16 || + bytes.toString("ascii", 36, 40) !== "data" || + bytes.readUInt32LE(40) !== bytes.byteLength - 44 + ) { + throw invalidAudio("The recorded audio is not a valid 24 kHz mono PCM WAV file."); + } +} + +function readRequiredString(value: unknown, message: string): string { + const normalized = typeof value === "string" ? value.trim() : ""; + if (!normalized) throw invalidAudio(message); + return normalized; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function invalidAudio(safeMessage: string): VoiceTranscriptionBackendError { + return new VoiceTranscriptionBackendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage, + }); +} diff --git a/apps/desktop/src/voiceTranscription.ts b/apps/desktop/src/voiceTranscription.ts index 448f3a083..27ad070e6 100644 --- a/apps/desktop/src/voiceTranscription.ts +++ b/apps/desktop/src/voiceTranscription.ts @@ -1,305 +1,68 @@ // FILE: voiceTranscription.ts -// Purpose: Owns the desktop-specific voice transcription flow for Electron builds. -// Layer: Desktop IPC + ChatGPT upload bridge -// Depends on: Codex auth discovery, Electron net uploads, and the shared server voice contract. +// Purpose: Registers provider-neutral desktop voice and offline-model IPC entrypoints. +// Layer: Desktop IPC -import * as ChildProcess from "node:child_process"; -import * as Crypto from "node:crypto"; +import * as Path from "node:path"; -import { app, ipcMain, net } from "electron"; -import type { - ServerVoiceTranscriptionInput, - ServerVoiceTranscriptionResult, -} from "@synara/contracts"; -import { prepareWindowsSafeProcess } from "@synara/shared/windowsProcess"; +import { app, ipcMain } from "electron"; -export const SERVER_TRANSCRIBE_VOICE_CHANNEL = "desktop:server-transcribe-voice"; +import { DesktopVoiceService } from "./voice/desktopVoiceService"; +import { resolveDesktopCodexVoiceProcessContext } from "./voice/desktopCodexVoiceRuntime"; +import { LocalVoiceModelManager } from "./voice/localVoiceModelManager"; +import { LOCAL_VOICE_MODEL } from "./voice/localVoiceModelManifest"; +import { LocalWhisperRuntime, resolveWhisperRuntimePaths } from "./voice/localWhisperRuntime"; +import { DESKTOP_VOICE_IPC_CHANNELS } from "./voice/voiceIpcChannels"; -const CHATGPT_TRANSCRIPTIONS_URL = "https://chatgpt.com/backend-api/transcribe"; -const MAX_VOICE_AUDIO_BYTES = 10 * 1024 * 1024; -const MAX_VOICE_DURATION_MS = 120_000; +let service: DesktopVoiceService | null = null; +let voiceRuntimeOptions: DesktopVoiceRuntimeOptions | null = null; -// --- Input validation ------------------------------------------------------ - -function normalizeVoiceBase64(value: string): string | null { - const normalized = value.trim().replace(/\s+/g, ""); - return normalized.length > 0 ? normalized : null; -} - -function isLikelyVoiceBase64(value: string): boolean { - return /^[A-Za-z0-9+/]+={0,2}$/.test(value); +export interface DesktopVoiceRuntimeOptions { + readonly stateDirectory: string; + readonly scientHome: string; } -function isLikelyWavBuffer(buffer: Buffer): boolean { - return ( - buffer.length >= 12 && - buffer.toString("ascii", 0, 4) === "RIFF" && - buffer.toString("ascii", 8, 12) === "WAVE" +export function registerDesktopVoiceTranscriptionHandler( + options: DesktopVoiceRuntimeOptions, +): void { + voiceRuntimeOptions = options; + const voiceService = getDesktopVoiceService(); + for (const channel of Object.values(DESKTOP_VOICE_IPC_CHANNELS)) ipcMain.removeHandler(channel); + ipcMain.handle(DESKTOP_VOICE_IPC_CHANNELS.transcribe, async (_event, input: unknown) => + voiceService.transcribe(input), ); -} - -function decodeDesktopVoiceAudio(input: ServerVoiceTranscriptionInput): Buffer { - if (input.mimeType !== "audio/wav") { - throw new Error("Only WAV audio is supported for voice transcription."); - } - if (input.sampleRateHz !== 24_000) { - throw new Error("Voice transcription requires 24 kHz mono WAV audio."); - } - if (input.durationMs <= 0) { - throw new Error("Voice messages must include a positive duration."); - } - if (input.durationMs > MAX_VOICE_DURATION_MS) { - throw new Error("Voice messages are limited to 120 seconds."); - } - - const normalizedBase64 = normalizeVoiceBase64(input.audioBase64); - if (!normalizedBase64 || !isLikelyVoiceBase64(normalizedBase64)) { - throw new Error("The recorded audio could not be decoded."); - } - - const audioBuffer = Buffer.from(normalizedBase64, "base64"); - if (!audioBuffer.length || audioBuffer.toString("base64") !== normalizedBase64) { - throw new Error("The recorded audio could not be decoded."); - } - if (audioBuffer.length > MAX_VOICE_AUDIO_BYTES) { - throw new Error("Voice messages are limited to 10 MB."); - } - if (!isLikelyWavBuffer(audioBuffer)) { - throw new Error("The recorded audio is not a valid WAV file."); - } - - return audioBuffer; -} - -function readNonEmptyString(value: unknown): string | null { - const normalized = typeof value === "string" ? value.trim() : ""; - return normalized.length > 0 ? normalized : null; -} - -// --- Auth discovery -------------------------------------------------------- - -async function resolveDesktopVoiceAuth( - cwd: string, -): Promise<{ token: string; transcriptionUrl: string }> { - return new Promise((resolve, reject) => { - const prepared = prepareWindowsSafeProcess("codex", ["app-server"], { - cwd, - env: process.env, - }); - const child = ChildProcess.spawn(prepared.command, prepared.args, { - cwd, - env: process.env, - stdio: ["pipe", "pipe", "pipe"], - shell: prepared.shell, - windowsHide: prepared.windowsHide, - windowsVerbatimArguments: prepared.windowsVerbatimArguments, - }); - - let settled = false; - let stdoutBuffer = ""; - const rejectOnce = (error: Error) => { - if (settled) { - return; - } - settled = true; - child.kill(); - reject(error); - }; - const resolveOnce = (value: { token: string; transcriptionUrl: string }) => { - if (settled) { - return; - } - settled = true; - child.kill(); - resolve(value); - }; - const send = (payload: Record) => { - child.stdin.write(`${JSON.stringify(payload)}\n`); - }; - - child.once("error", (error) => { - rejectOnce(new Error(`Could not start Codex auth discovery: ${error.message}`)); - }); - child.stderr.on("data", () => { - // Ignore stderr noise from the discovery process; the JSON-RPC result is authoritative. - }); - child.stdout.on("data", (chunk) => { - stdoutBuffer += chunk.toString(); - const lines = stdoutBuffer.split(/\n/); - stdoutBuffer = lines.pop() ?? ""; - for (const line of lines) { - let message: Record; - try { - message = JSON.parse(line) as Record; - } catch { - continue; - } - - if (message.id === 1) { - send({ jsonrpc: "2.0", method: "initialized", params: {} }); - send({ - jsonrpc: "2.0", - id: 2, - method: "getAuthStatus", - params: { includeToken: true, refreshToken: true }, - }); - continue; - } - - if (message.id !== 2) { - continue; - } - - const result = - typeof message.result === "object" && message.result !== null - ? (message.result as Record) - : null; - const authMethod = readNonEmptyString(result?.authMethod); - const token = readNonEmptyString(result?.authToken); - if (!token) { - rejectOnce( - new Error("No ChatGPT session token is available. Sign in to ChatGPT in Codex."), - ); - return; - } - if (authMethod !== "chatgpt" && authMethod !== "chatgptAuthTokens") { - rejectOnce( - new Error("Voice transcription requires a ChatGPT-authenticated Codex session."), - ); - return; - } - - resolveOnce({ - token, - transcriptionUrl: - readNonEmptyString(result?.transcriptionUrl) ?? CHATGPT_TRANSCRIPTIONS_URL, - }); - } - }); - - setTimeout(() => { - send({ - jsonrpc: "2.0", - id: 1, - method: "initialize", - params: { - clientInfo: { - name: "scient-desktop", - title: "Scient Desktop", - version: app.getVersion(), - }, - capabilities: { experimentalApi: true }, - }, - }); - }, 100); - - setTimeout(() => { - rejectOnce(new Error("Timed out while reading ChatGPT auth from Codex.")); - }, 10_000).unref(); + ipcMain.handle(DESKTOP_VOICE_IPC_CHANNELS.cancelTranscription, async () => { + voiceService.cancelActiveTranscriptions(); }); -} - -// --- Network upload -------------------------------------------------------- - -async function requestDesktopVoiceTranscription(input: { - readonly audioBuffer: Buffer; - readonly mimeType: string; - readonly token: string; - readonly transcriptionUrl: string; -}): Promise<{ statusCode: number; body: string }> { - const boundary = `ScientVoice-${Crypto.randomUUID()}`; - const preamble = Buffer.from( - `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="voice.wav"\r\nContent-Type: ${input.mimeType}\r\n\r\n`, - "utf8", + ipcMain.handle(DESKTOP_VOICE_IPC_CHANNELS.getState, async () => voiceService.getState()); + ipcMain.handle(DESKTOP_VOICE_IPC_CHANNELS.downloadModel, async () => + voiceService.downloadModel(), ); - const closing = Buffer.from(`\r\n--${boundary}--\r\n`, "utf8"); - const body = Buffer.concat([preamble, input.audioBuffer, closing]); - - return new Promise((resolve, reject) => { - const requestUrl = readNonEmptyString(input.transcriptionUrl) ?? CHATGPT_TRANSCRIPTIONS_URL; - const request = net.request({ - method: "POST", - url: requestUrl, - }); - request.setHeader("Authorization", `Bearer ${input.token}`); - request.setHeader("Content-Type", `multipart/form-data; boundary=${boundary}`); - - request.once("error", (error) => { - reject(new Error(`Voice transcription request failed: ${error.message}`)); - }); - request.on("response", (response) => { - let responseBody = ""; - response.on("data", (chunk) => { - responseBody += chunk.toString(); - }); - response.once("end", () => { - resolve({ - statusCode: response.statusCode, - body: responseBody, - }); - }); - response.once("error", (error) => { - reject(new Error(`Voice transcription response failed: ${error.message}`)); - }); - }); - - request.write(body); - request.end(); - }); + ipcMain.handle(DESKTOP_VOICE_IPC_CHANNELS.removeModel, async () => voiceService.removeModel()); + ipcMain.handle(DESKTOP_VOICE_IPC_CHANNELS.repairModel, async () => voiceService.repairModel()); } -function readVoiceResponseErrorMessage(statusCode: number, body: string): string { - try { - const payload = JSON.parse(body) as { error?: { message?: unknown }; message?: unknown }; - const providerMessage = - readNonEmptyString(payload.error?.message) ?? readNonEmptyString(payload.message); - if (providerMessage) { - return providerMessage; - } - } catch { - // Fall back to a status-based message when the upstream body is not JSON. - } - - if (statusCode === 401) { - return "Your ChatGPT login has expired. Sign in again."; - } - if (statusCode === 403) { - return "ChatGPT rejected the transcription request. Your Codex login is present, but this desktop upload was forbidden."; - } - - return `Transcription failed with status ${statusCode}.`; +export async function disposeDesktopVoiceTranscription(): Promise { + const active = service; + service = null; + if (active) await active.dispose(); } -// --- IPC entrypoint -------------------------------------------------------- - -async function transcribeVoiceViaDesktopBridge( - input: ServerVoiceTranscriptionInput, -): Promise { - const audioBuffer = decodeDesktopVoiceAudio(input); - const auth = await resolveDesktopVoiceAuth(input.cwd?.trim() || process.cwd()); - const response = await requestDesktopVoiceTranscription({ - audioBuffer, - mimeType: input.mimeType, - token: auth.token, - transcriptionUrl: auth.transcriptionUrl, +function getDesktopVoiceService(): DesktopVoiceService { + if (service) return service; + if (!voiceRuntimeOptions) throw new Error("Desktop voice runtime was not configured."); + const runtimeOptions = voiceRuntimeOptions; + const runtimeDirectory = resolveWhisperRuntimePaths({ + isPackaged: app.isPackaged, + resourcesPath: process.resourcesPath, + desktopRuntimeDirectory: Path.resolve(__dirname, "..", ".electron-runtime"), + }).runtimeDirectory; + service = new DesktopVoiceService({ + modelManager: new LocalVoiceModelManager({ + modelsDirectory: Path.join(app.getPath("userData"), "voice", "models"), + manifest: LOCAL_VOICE_MODEL, + }), + runtime: new LocalWhisperRuntime({ runtimeDirectory }), + resolveRemoteProcessContext: () => resolveDesktopCodexVoiceProcessContext(runtimeOptions), }); - if (response.statusCode < 200 || response.statusCode >= 300) { - throw new Error(readVoiceResponseErrorMessage(response.statusCode, response.body)); - } - - const payload = JSON.parse(response.body) as { text?: unknown; transcript?: unknown }; - const text = readNonEmptyString(payload.text) ?? readNonEmptyString(payload.transcript); - if (!text) { - throw new Error("The transcription response did not include any text."); - } - - return { text }; -} - -export function registerDesktopVoiceTranscriptionHandler(): void { - ipcMain.removeHandler(SERVER_TRANSCRIBE_VOICE_CHANNEL); - ipcMain.handle( - SERVER_TRANSCRIBE_VOICE_CHANNEL, - async (_event, input: ServerVoiceTranscriptionInput) => transcribeVoiceViaDesktopBridge(input), - ); + return service; } diff --git a/apps/server/package.json b/apps/server/package.json index ed8fefe55..ca396136a 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -20,7 +20,8 @@ "start": "node dist/index.mjs", "prepare": "effect-language-service patch", "typecheck": "tsc --noEmit", - "test": "vitest run --maxWorkers=1 --no-file-parallelism" + "test": "vitest run --maxWorkers=1 --no-file-parallelism", + "test:html-preview-platform": "vitest run --maxWorkers=1 --no-file-parallelism src/htmlPreview/HtmlArtifactPreview.platform.test.ts src/localServerMonitor.platform.test.ts" }, "dependencies": { "@anthropic-ai/claude-agent-sdk": "0.3.207", @@ -34,8 +35,10 @@ "@pierre/diffs": "^1.2.12", "@xterm/headless": "^6.0.0", "effect": "catalog:", + "es-module-lexer": "^2.0.0", "node-pty": "^1.1.0", "open": "^10.1.0", + "parse5": "^7.3.0", "tar": "^7.5.20", "tree-kill": "^1.2.2", "unzipper": "^0.12.3", diff --git a/apps/server/src/devServerManager.test.ts b/apps/server/src/devServerManager.test.ts index 28416a23f..0da6b32a3 100644 --- a/apps/server/src/devServerManager.test.ts +++ b/apps/server/src/devServerManager.test.ts @@ -6,11 +6,16 @@ import { describe, expect, it } from "vitest"; import { ProjectId, type ProjectDevServer, type ServerLocalServerProcess } from "@synara/contracts"; -import { findProjectDevServerForLocalServer } from "./devServerManager"; +import { + failProjectDevServerGeneration, + findProjectDevServerForLocalServer, + waitForProjectDevServerReadiness, +} from "./devServerManager"; function makeDevServer(overrides: Partial = {}): ProjectDevServer { return { projectId: ProjectId.makeUnsafe("project-1"), + runId: "run-1", command: "pnpm run dev", cwd: "/repo/app", pid: 100, @@ -48,15 +53,15 @@ describe("findProjectDevServerForLocalServer", () => { ).toBe(devServer); }); - it("uses the shared local-server ownership rule for cwd matches", () => { - const devServer = makeDevServer({ cwd: "/repo/app", pid: null }); + it("does not treat a matching cwd as process ownership", () => { + const devServer = makeDevServer({ cwd: "/repo/app", pid: 100 }); expect( findProjectDevServerForLocalServer({ localServer: makeLocalServer({ cwd: "/repo/app/packages/web", pid: 200 }), devServers: [devServer], }), - ).toBe(devServer); + ).toBeNull(); }); it("does not match sibling folders with the same prefix", () => { @@ -68,3 +73,63 @@ describe("findProjectDevServerForLocalServer", () => { ).toBeNull(); }); }); + +describe("waitForProjectDevServerReadiness", () => { + it("returns only after the tracked run owns a reachable listener", async () => { + let calls = 0; + const result = await waitForProjectDevServerReadiness(makeDevServer({ status: "starting" }), { + timeoutMs: 100, + pollMs: 0, + discover: async () => { + calls += 1; + return calls === 1 ? [] : [makeLocalServer({ cwd: "/repo/app", ppid: 100 })]; + }, + probe: async () => true, + sleep: async () => undefined, + }); + + expect(calls).toBe(2); + expect(result).toEqual({ url: "http://127.0.0.1:5173", ports: [5173] }); + }); + + it("does not probe an unrelated reachable listener with the same cwd", async () => { + let probes = 0; + const result = await waitForProjectDevServerReadiness( + makeDevServer({ pid: 100, status: "starting" }), + { + timeoutMs: 0, + discover: async () => [makeLocalServer({ cwd: "/repo/app", pid: 200, ppid: 1 })], + probe: async () => { + probes += 1; + return true; + }, + sleep: async () => undefined, + }, + ); + + expect(result).toBeNull(); + expect(probes).toBe(0); + }); + + it("fails closed when no listener becomes ready", async () => { + expect( + await waitForProjectDevServerReadiness(makeDevServer({ status: "starting" }), { + timeoutMs: 0, + discover: async () => [], + sleep: async () => undefined, + }), + ).toBeNull(); + }); +}); + +describe("failProjectDevServerGeneration", () => { + it("ignores a delayed exit from a superseded run", () => { + const current = makeDevServer({ runId: "new-run", status: "starting" }); + expect(failProjectDevServerGeneration(current, "old-run", "late exit")).toBeNull(); + expect(failProjectDevServerGeneration(current, "new-run", "current exit")).toMatchObject({ + runId: "new-run", + status: "failed", + error: "current exit", + }); + }); +}); diff --git a/apps/server/src/devServerManager.ts b/apps/server/src/devServerManager.ts index 08b472507..78b1058b2 100644 --- a/apps/server/src/devServerManager.ts +++ b/apps/server/src/devServerManager.ts @@ -11,7 +11,6 @@ * @module DevServerManager */ import { - DEFAULT_TERMINAL_ID, ProjectId, type ProjectDevServer, type ProjectDevServerEvent, @@ -22,16 +21,21 @@ import { type ProjectStopDevServerResult, type ServerLocalServerProcess, } from "@synara/contracts"; -import { localServerMatchesRun } from "@synara/shared/localServers"; +import * as Crypto from "node:crypto"; +import { localServerProcessMatchesRun } from "@synara/shared/localServers"; import { Effect, Layer, PubSub, Ref, ServiceMap, Stream } from "effect"; import { TerminalManager, type TerminalError } from "./terminal/Services/Manager"; +import { listLocalServers } from "./localServerMonitor"; // Dev servers reuse the terminal infrastructure under a reserved synthetic // thread namespace so their PTYs never collide with real chat-thread terminals. const DEV_SERVER_THREAD_PREFIX = "dev-server:"; +const DEV_SERVER_TERMINAL_PREFIX = "run:"; const DEV_SERVER_TERMINAL_COLS = 120; const DEV_SERVER_TERMINAL_ROWS = 30; +const DEV_SERVER_READINESS_TIMEOUT_MS = 30_000; +const DEV_SERVER_READINESS_POLL_MS = 400; const devServerThreadId = (projectId: ProjectId): string => `${DEV_SERVER_THREAD_PREFIX}${projectId}`; @@ -44,18 +48,87 @@ const parseDevServerProjectId = (threadId: string): ProjectId | null => { return raw.length > 0 ? ProjectId.makeUnsafe(raw) : null; }; +const devServerTerminalId = (runId: string): string => `${DEV_SERVER_TERMINAL_PREFIX}${runId}`; + +const parseDevServerRunId = (terminalId: string): string | null => + terminalId.startsWith(DEV_SERVER_TERMINAL_PREFIX) + ? terminalId.slice(DEV_SERVER_TERMINAL_PREFIX.length) || null + : null; + export function findProjectDevServerForLocalServer(input: { localServer: ServerLocalServerProcess; devServers: readonly ProjectDevServer[]; }): ProjectDevServer | null { for (const devServer of input.devServers) { - if (localServerMatchesRun(input.localServer, devServer)) { + if (localServerProcessMatchesRun(input.localServer, devServer)) { return devServer; } } return null; } +export function failProjectDevServerGeneration( + current: ProjectDevServer | undefined, + runId: string, + error: string, +): ProjectDevServer | null { + return current?.runId === runId ? { ...current, status: "failed", error } : null; +} + +function preferredLocalServerUrl(server: ServerLocalServerProcess): string | null { + return ( + server.addresses.find((address) => address.host === "127.0.0.1")?.url ?? + server.addresses.find((address) => address.host === "localhost")?.url ?? + server.addresses[0]?.url ?? + null + ); +} + +export async function waitForProjectDevServerReadiness( + server: ProjectDevServer, + options: { + timeoutMs?: number; + pollMs?: number; + discover?: () => Promise; + probe?: (url: string) => Promise; + sleep?: (milliseconds: number) => Promise; + } = {}, +): Promise<{ url: string; ports: readonly number[] } | null> { + const timeoutMs = options.timeoutMs ?? DEV_SERVER_READINESS_TIMEOUT_MS; + const pollMs = options.pollMs ?? DEV_SERVER_READINESS_POLL_MS; + const discover = + options.discover ?? + (async () => (await listLocalServers({ includePageTitles: false })).servers); + const sleep = + options.sleep ?? + ((milliseconds: number) => new Promise((resolve) => setTimeout(resolve, milliseconds))); + const probe = + options.probe ?? + (async (url: string) => { + try { + await globalThis.fetch(url, { + redirect: "manual", + signal: AbortSignal.timeout(800), + }); + return true; + } catch { + return false; + } + }); + const deadline = Date.now() + timeoutMs; + + while (Date.now() <= deadline) { + const localServers = await discover().catch(() => []); + const match = localServers.find((candidate) => localServerProcessMatchesRun(candidate, server)); + const url = match ? preferredLocalServerUrl(match) : null; + if (match && url && (await probe(url))) { + return { url, ports: match.ports }; + } + await sleep(pollMs); + } + return null; +} + export interface DevServerManagerShape { /** Start (or restart) the dev server for a project and return its descriptor. */ readonly run: ( @@ -85,20 +158,19 @@ export const DevServerManagerLive = Layer.effect( const publish = (event: ProjectDevServerEvent) => PubSub.publish(pubsub, event); - // Reap a tracked dev server whose PTY exited or errored. Guarded so that a - // deliberate stop (which removes the entry first) cannot double-publish, and - // so a stale exit for an already-replaced project is ignored. - const reapExited = (projectId: ProjectId) => + // Preserve failures in the registry so the UI can explain why a launch did + // not become ready. Deliberate stops remove the entry before PTY teardown. + const markExited = (projectId: ProjectId, runId: string, error: string) => Ref.modify(registry, (current) => { - if (!current[projectId]) { - return [false, current] as const; + const existing = current[projectId]; + const failed = failProjectDevServerGeneration(existing, runId, error); + if (!failed) { + return [null, current] as const; } - const next = { ...current }; - delete next[projectId]; - return [true, next] as const; + return [failed, { ...current, [projectId]: failed }] as const; }).pipe( - Effect.flatMap((removed) => - removed ? publish({ type: "removed", projectId, reason: "exited" }) : Effect.void, + Effect.flatMap((failed) => + failed ? publish({ type: "upserted", server: failed }) : Effect.void, ), ); @@ -107,16 +179,27 @@ export const DevServerManagerLive = Layer.effect( return; } const projectId = parseDevServerProjectId(event.threadId); - if (!projectId) { + const runId = parseDevServerRunId(event.terminalId); + if (!projectId || !runId) { return; } - Effect.runFork(reapExited(projectId)); + Effect.runFork( + markExited( + projectId, + runId, + event.type === "error" + ? "The development server process reported an error." + : "The development server process exited before it was stopped.", + ), + ); }); yield* Effect.addFinalizer(() => Effect.sync(unsubscribe)); const run: DevServerManagerShape["run"] = (input) => Effect.gen(function* () { const threadId = devServerThreadId(input.projectId); + const runId = Crypto.randomUUID(); + const terminalId = devServerTerminalId(runId); // If a dev server is already tracked for this project, tear its PTY down // first so the command always lands in a fresh shell. A deliberate close @@ -124,13 +207,17 @@ export const DevServerManagerLive = Layer.effect( const existing = (yield* Ref.get(registry))[input.projectId]; if (existing) { yield* terminalManager - .close({ threadId, deleteHistory: true }) + .close({ + threadId, + terminalId: devServerTerminalId(existing.runId), + deleteHistory: true, + }) .pipe(Effect.catch(() => Effect.void)); } const snapshot = yield* terminalManager.open({ threadId, - terminalId: DEFAULT_TERMINAL_ID, + terminalId, cwd: input.cwd, cols: DEV_SERVER_TERMINAL_COLS, rows: DEV_SERVER_TERMINAL_ROWS, @@ -142,19 +229,54 @@ export const DevServerManagerLive = Layer.effect( yield* terminalManager.write({ threadId, - terminalId: DEFAULT_TERMINAL_ID, + terminalId, data: `${input.command}\r`, }); - const server: ProjectDevServer = { + const startingServer: ProjectDevServer = { projectId: input.projectId, + runId, command: input.command, cwd: input.cwd, pid: snapshot.pid, startedAt: new Date().toISOString(), - status: "running", + status: "starting", }; - yield* Ref.update(registry, (current) => ({ ...current, [input.projectId]: server })); + yield* Ref.update(registry, (current) => ({ + ...current, + [input.projectId]: startingServer, + })); + yield* publish({ type: "upserted", server: startingServer }); + + const readiness = yield* Effect.promise(() => + waitForProjectDevServerReadiness(startingServer), + ); + const current = (yield* Ref.get(registry))[input.projectId]; + if (current?.runId === runId && current.status === "failed") { + return { server: current }; + } + if (!current || current.runId !== runId) { + return { + server: { + ...startingServer, + status: "failed", + error: "This development-server launch was superseded by a newer run.", + }, + }; + } + const server: ProjectDevServer = readiness + ? { + ...startingServer, + status: "running", + url: readiness.url, + ports: readiness.ports, + } + : { + ...startingServer, + status: "failed", + error: "No local HTTP listener became ready within 30 seconds.", + }; + yield* Ref.update(registry, (entries) => ({ ...entries, [input.projectId]: server })); yield* publish({ type: "upserted", server }); return { server }; }); @@ -164,19 +286,24 @@ export const DevServerManagerLive = Layer.effect( // Remove from the registry *before* closing so the PTY teardown cannot be // mistaken for a crash by the reaper. const removed = yield* Ref.modify(registry, (current) => { - if (!current[input.projectId]) { - return [false, current] as const; + const existing = current[input.projectId]; + if (!existing) { + return [null, current] as const; } const next = { ...current }; delete next[input.projectId]; - return [true, next] as const; + return [existing, next] as const; }); if (!removed) { return { stopped: false }; } yield* publish({ type: "removed", projectId: input.projectId, reason: "stopped" }); yield* terminalManager - .close({ threadId: devServerThreadId(input.projectId), deleteHistory: true }) + .close({ + threadId: devServerThreadId(input.projectId), + terminalId: devServerTerminalId(removed.runId), + deleteHistory: true, + }) .pipe(Effect.catch(() => Effect.void)); return { stopped: true }; }); diff --git a/apps/server/src/git/Services/TextGeneration.ts b/apps/server/src/git/Services/TextGeneration.ts index 01b1960b2..92cea54e5 100644 --- a/apps/server/src/git/Services/TextGeneration.ts +++ b/apps/server/src/git/Services/TextGeneration.ts @@ -26,7 +26,7 @@ export interface CommitMessageGenerationInput { codexHomePath?: string; /** When true, the model also returns a semantic branch name for the change. */ includeBranch?: boolean; - /** Model to use for generation. Defaults to gpt-5.4-mini if not specified. */ + /** Model to use for generation. Defaults to gpt-5.6-luna if not specified. */ model?: string; /** Optional provider-aware selection for providers that need more than a raw model slug. */ modelSelection?: ModelSelection; @@ -49,7 +49,7 @@ export interface PrContentGenerationInput { diffSummary: string; diffPatch: string; codexHomePath?: string; - /** Model to use for generation. Defaults to gpt-5.4-mini if not specified. */ + /** Model to use for generation. Defaults to gpt-5.6-luna if not specified. */ model?: string; /** Optional provider-aware selection for providers that need more than a raw model slug. */ modelSelection?: ModelSelection; @@ -66,7 +66,7 @@ export interface DiffSummaryGenerationInput { cwd: string; patch: string; codexHomePath?: string; - /** Model to use for generation. Defaults to gpt-5.4-mini if not specified. */ + /** Model to use for generation. Defaults to gpt-5.6-luna if not specified. */ model?: string; /** Optional provider-aware selection for providers that need more than a raw model slug. */ modelSelection?: ModelSelection; @@ -82,7 +82,7 @@ export interface BranchNameGenerationInput { cwd: string; message: string; attachments?: ReadonlyArray | undefined; - /** Model to use for generation. Defaults to gpt-5.4-mini if not specified. */ + /** Model to use for generation. Defaults to gpt-5.6-luna if not specified. */ model?: string; /** Optional provider-aware selection for providers that need more than a raw model slug. */ modelSelection?: ModelSelection; @@ -98,7 +98,7 @@ export interface ThreadTitleGenerationInput { cwd: string; message: string; attachments?: ReadonlyArray | undefined; - /** Model to use for generation. Defaults to gpt-5.4-mini if not specified. */ + /** Model to use for generation. Defaults to gpt-5.6-luna if not specified. */ model?: string; /** Optional provider-aware selection for providers that need more than a raw model slug. */ modelSelection?: ModelSelection; @@ -116,7 +116,7 @@ export interface ThreadRecapGenerationInput { newMaterial: string; currentState?: string | undefined; codexHomePath?: string; - /** Model to use for generation. Defaults to gpt-5.4-mini if not specified. */ + /** Model to use for generation. Defaults to gpt-5.6-luna if not specified. */ model?: string; /** Optional provider-aware selection for providers that need more than a raw model slug. */ modelSelection?: ModelSelection; @@ -134,7 +134,7 @@ export interface AutomationIntentGenerationInput { defaultMode?: AutomationMode; nowIso: string; codexHomePath?: string; - /** Model to use for generation. Defaults to gpt-5.4-mini if not specified. */ + /** Model to use for generation. Defaults to gpt-5.6-luna if not specified. */ model?: string; /** Optional provider-aware selection for providers that need more than a raw model slug. */ modelSelection?: ModelSelection; @@ -153,7 +153,7 @@ export interface AutomationCompletionEvaluationInput { runAssistantText: string; threadContext?: string | undefined; codexHomePath?: string; - /** Model to use for generation. Defaults to gpt-5.4-mini if not specified. */ + /** Model to use for generation. Defaults to gpt-5.6-luna if not specified. */ model?: string; /** Optional provider-aware selection for providers that need more than a raw model slug. */ modelSelection?: ModelSelection; diff --git a/apps/server/src/htmlPreview/HtmlArtifactPreview.platform.test.ts b/apps/server/src/htmlPreview/HtmlArtifactPreview.platform.test.ts new file mode 100644 index 000000000..ed3c6ae08 --- /dev/null +++ b/apps/server/src/htmlPreview/HtmlArtifactPreview.platform.test.ts @@ -0,0 +1,50 @@ +// FILE: HtmlArtifactPreview.platform.test.ts +// Purpose: Verifies capability hostnames through the current OS localhost resolver. + +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { Effect } from "effect"; +import { expect, it } from "vitest"; + +import { HtmlArtifactPreview } from "./Services/HtmlArtifactPreview"; +import { HtmlArtifactPreviewLive } from "./Layers/HtmlArtifactPreview"; + +it( + "loads the generated capability hostname through the platform localhost resolver", + { timeout: 10_000 }, + async () => { + const workspace = await fs.mkdtemp(path.join(os.tmpdir(), "scient-html-preview-platform-")); + try { + await fs.writeFile(path.join(workspace, "index.html"), "

Resolver smoke

"); + await fs.writeFile(path.join(workspace, "second.html"), "

Second origin

"); + await Effect.runPromise( + Effect.scoped( + Effect.gen(function* () { + const service = yield* HtmlArtifactPreview; + const prepared = yield* service.prepare({ cwd: workspace, path: "index.html" }); + const second = yield* service.prepare({ cwd: workspace, path: "second.html" }); + const previewUrl = new URL(prepared.previewUrl!); + const secondUrl = new URL(second.previewUrl!); + expect(secondUrl.origin).not.toBe(previewUrl.origin); + expect( + process.platform === "win32" + ? previewUrl.hostname === "127.0.0.1" + : previewUrl.hostname.endsWith(".preview.localhost"), + ).toBe(true); + const response = yield* Effect.promise(() => + fetch(prepared.previewUrl!, { + signal: AbortSignal.timeout(5_000), + }), + ); + expect(response.status).toBe(200); + expect(yield* Effect.promise(() => response.text())).toContain("Resolver smoke"); + }).pipe(Effect.provide(HtmlArtifactPreviewLive)), + ), + ); + } finally { + await fs.rm(workspace, { recursive: true, force: true }); + } + }, +); diff --git a/apps/server/src/htmlPreview/Inspector.test.ts b/apps/server/src/htmlPreview/Inspector.test.ts new file mode 100644 index 000000000..ae8c1ac18 --- /dev/null +++ b/apps/server/src/htmlPreview/Inspector.test.ts @@ -0,0 +1,123 @@ +// FILE: Inspector.test.ts +// Purpose: Regression coverage for fail-safe HTML artifact classification. + +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { inspectHtmlArtifact } from "./Inspector"; + +const temporaryDirectories: string[] = []; + +async function makeWorkspace(): Promise { + const workspace = await fs.mkdtemp(path.join(os.tmpdir(), "scient-html-inspector-")); + temporaryDirectories.push(workspace); + return workspace; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => fs.rm(directory, { recursive: true, force: true })), + ); +}); + +describe("inspectHtmlArtifact", () => { + it("classifies a standalone document with local presentation assets", async () => { + const workspace = await makeWorkspace(); + await fs.writeFile(path.join(workspace, "lesson.css"), "body { color: green; }"); + await fs.writeFile( + path.join(workspace, "lesson.html"), + 'Lesson

Study

', + ); + + const inspected = await inspectHtmlArtifact({ cwd: workspace, path: "lesson.html" }); + + expect(inspected.result).toEqual({ + mode: "static-document", + title: "Lesson", + warnings: [], + }); + expect(inspected.absolutePath).toBe(await fs.realpath(path.join(workspace, "lesson.html"))); + expect(inspected.allowedResourcePaths).toEqual([ + await fs.realpath(path.join(workspace, "lesson.css")), + ]); + }); + + it("classifies inline and browser-ready JavaScript as an interactive bundle", async () => { + const workspace = await makeWorkspace(); + await fs.writeFile(path.join(workspace, "app.js"), "document.body.dataset.ready = 'yes';"); + await fs.writeFile( + path.join(workspace, "index.html"), + '', + ); + + const inspected = await inspectHtmlArtifact({ cwd: workspace, path: "index.html" }); + + expect(inspected.result.mode).toBe("interactive-bundle"); + expect(inspected.result.warnings).toEqual([]); + }); + + it("routes Vite TSX source entrypoints to the nearest package run target", async () => { + const workspace = await makeWorkspace(); + const appDirectory = path.join(workspace, "packages", "app"); + await fs.mkdir(path.join(appDirectory, "src"), { recursive: true }); + await fs.writeFile(path.join(appDirectory, "bun.lock"), ""); + await fs.writeFile( + path.join(appDirectory, "package.json"), + JSON.stringify({ scripts: { dev: "vite" } }), + ); + await fs.writeFile(path.join(appDirectory, "src", "main.tsx"), "export {};\n"); + await fs.writeFile( + path.join(appDirectory, "index.html"), + '
', + ); + + const inspected = await inspectHtmlArtifact({ + cwd: workspace, + path: "packages/app/index.html", + }); + + expect(inspected.result).toMatchObject({ + mode: "dev-server-entrypoint", + runTarget: { + cwd: await fs.realpath(appDirectory), + command: "bun run dev", + scriptName: "dev", + }, + }); + }); + + it("reports blocked external resources without granting them network access", async () => { + const workspace = await makeWorkspace(); + await fs.writeFile( + path.join(workspace, "index.html"), + '', + ); + + const inspected = await inspectHtmlArtifact({ cwd: workspace, path: "index.html" }); + + expect(inspected.result.mode).toBe("interactive-bundle"); + expect(inspected.result.warnings).toEqual([ + { + code: "external-resource-blocked", + message: "External script blocked in preview: https://cdn.example/app.js", + }, + ]); + }); + + it("fails closed for files outside the workspace", async () => { + const workspace = await makeWorkspace(); + const outside = await makeWorkspace(); + const outsideFile = path.join(outside, "outside.html"); + await fs.writeFile(outsideFile, "

private

"); + + const inspected = await inspectHtmlArtifact({ cwd: workspace, path: outsideFile }); + + expect(inspected.result.mode).toBe("unsupported"); + expect(inspected.absolutePath).toBeNull(); + }); +}); diff --git a/apps/server/src/htmlPreview/Inspector.ts b/apps/server/src/htmlPreview/Inspector.ts new file mode 100644 index 000000000..1993026e5 --- /dev/null +++ b/apps/server/src/htmlPreview/Inspector.ts @@ -0,0 +1,399 @@ +// FILE: Inspector.ts +// Purpose: Classify local HTML artifacts before any executable preview capability is issued. +// Layer: Server HTML-preview domain logic + +import fs from "node:fs/promises"; +import path from "node:path"; + +import type { + ProjectHtmlArtifactRunTarget, + ProjectHtmlArtifactWarning, + ProjectInspectHtmlArtifactInput, + ProjectInspectHtmlArtifactResult, +} from "@synara/contracts"; +import { init as initializeModuleLexer, parse as parseModuleImports } from "es-module-lexer"; +import { isSupportedLocalHtmlPath, lowerCaseExtensionOf } from "@synara/shared/localPreviewFiles"; +import { parse, type DefaultTreeAdapterMap } from "parse5"; + +import { commandForProjectPackageScript, detectProjectPackageManager } from "../workspaceEntries"; + +const HTML_INSPECTION_MAX_BYTES = 1_000_000; +const PACKAGE_JSON_MAX_BYTES = 1_000_000; +const MAX_WARNINGS = 20; +const RESOURCE_GRAPH_MAX_FILES = 250; +const RESOURCE_GRAPH_PARSE_MAX_BYTES = 1_000_000; +const DEV_SOURCE_EXTENSIONS = new Set([".ts", ".tsx", ".jsx"]); +const BROWSER_SCRIPT_EXTENSIONS = new Set([".js", ".mjs"]); +const ALLOWED_LOCAL_RESOURCE_EXTENSIONS = new Set([ + ".avif", + ".bmp", + ".css", + ".gif", + ".heic", + ".heif", + ".ico", + ".jpeg", + ".jpg", + ".js", + ".mjs", + ".otf", + ".png", + ".svg", + ".tiff", + ".ttf", + ".webp", + ".woff", + ".woff2", +]); + +type DocumentNode = DefaultTreeAdapterMap["document"]; +type Node = DefaultTreeAdapterMap["node"]; +type Element = DefaultTreeAdapterMap["element"]; + +export interface InspectedHtmlArtifact { + readonly result: ProjectInspectHtmlArtifactResult; + readonly absolutePath: string | null; + readonly baseDirectory: string | null; + readonly allowedResourcePaths: readonly string[]; +} + +function isPathInside(candidate: string, root: string): boolean { + const relative = path.relative(root, candidate); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + +function isElement(node: Node): node is Element { + return "tagName" in node && typeof node.tagName === "string"; +} + +function attributeOf(element: Element, name: string): string | null { + return element.attrs.find((attribute) => attribute.name.toLowerCase() === name)?.value ?? null; +} + +function textContentOf(node: Node): string { + if ("value" in node && typeof node.value === "string") { + return node.value; + } + return "childNodes" in node ? node.childNodes.map((child) => textContentOf(child)).join("") : ""; +} + +function visit(node: Node, visitor: (element: Element) => void): void { + if (isElement(node)) { + visitor(node); + } + if ("childNodes" in node) { + for (const child of node.childNodes) { + visit(child, visitor); + } + } +} + +function isExternalResource(value: string): boolean { + const trimmed = value.trim(); + return ( + trimmed.startsWith("//") || + (/^[a-z][a-z\d+.-]*:/i.test(trimmed) && + !trimmed.startsWith("data:") && + !trimmed.startsWith("blob:")) + ); +} + +function resolveLocalResourcePath(value: string, baseDirectory: string): string | null { + const withoutQuery = value.trim().split(/[?#]/, 1)[0] ?? ""; + if ( + withoutQuery.length === 0 || + withoutQuery.startsWith("#") || + withoutQuery.startsWith("data:") || + withoutQuery.startsWith("blob:") || + isExternalResource(withoutQuery) + ) { + return null; + } + let decoded: string; + try { + decoded = decodeURIComponent(withoutQuery); + } catch { + return null; + } + if (decoded.includes("\0") || decoded.includes("\\")) { + return null; + } + return path.resolve(baseDirectory, decoded.replace(/^\/+/, "")); +} + +async function resourceExists(value: string, baseDirectory: string): Promise { + const resolved = resolveLocalResourcePath(value, baseDirectory); + if (!resolved) return true; + const stat = await fs.stat(resolved).catch(() => null); + return Boolean(stat?.isFile()); +} + +async function nearestRunTarget( + entryPath: string, + workspaceRoot: string, +): Promise { + let directory = path.dirname(entryPath); + while (isPathInside(directory, workspaceRoot)) { + const packageJsonPath = path.join(directory, "package.json"); + const stat = await fs.stat(packageJsonPath).catch(() => null); + if (stat?.isFile() && stat.size <= PACKAGE_JSON_MAX_BYTES) { + const parsed = await fs + .readFile(packageJsonPath, "utf8") + .then((contents) => JSON.parse(contents) as unknown) + .catch(() => null); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + const scripts = (parsed as { scripts?: unknown }).scripts; + if (scripts && typeof scripts === "object" && !Array.isArray(scripts)) { + const scriptRecord = scripts as Record; + const scriptName = ["dev", "start"].find( + (name) => + typeof scriptRecord[name] === "string" && scriptRecord[name].trim().length > 0, + ); + if (scriptName) { + const manager = await detectProjectPackageManager(directory); + return { + cwd: directory, + command: commandForProjectPackageScript(manager, scriptName), + scriptName, + }; + } + } + } + } + const parent = path.dirname(directory); + if (parent === directory) break; + directory = parent; + } + return undefined; +} + +function unsupported(reason: string): InspectedHtmlArtifact { + return { + result: { mode: "unsupported", reason, warnings: [] }, + absolutePath: null, + baseDirectory: null, + allowedResourcePaths: [], + }; +} + +const CSS_RESOURCE_PATTERN = /(?:url\(\s*|@import\s+(?:url\(\s*)?)["']?([^"')\s]+)["']?\s*\)?/gi; + +async function collectAllowedResourcePaths( + resources: readonly string[], + baseDirectory: string, +): Promise { + const pending = resources + .map((resource) => resolveLocalResourcePath(resource, baseDirectory)) + .filter((resource): resource is string => resource !== null); + const allowed = new Set(); + + while (pending.length > 0 && allowed.size < RESOURCE_GRAPH_MAX_FILES) { + const candidate = pending.shift(); + if (!candidate) continue; + const canonical = await fs.realpath(candidate).catch(() => null); + if (!canonical || !isPathInside(canonical, baseDirectory) || allowed.has(canonical)) continue; + const stat = await fs.stat(canonical).catch(() => null); + if (!stat?.isFile()) continue; + allowed.add(canonical); + + const extension = path.extname(canonical).toLowerCase(); + if ( + stat.size > RESOURCE_GRAPH_PARSE_MAX_BYTES || + (extension !== ".css" && extension !== ".js" && extension !== ".mjs") + ) { + continue; + } + const contents = await fs.readFile(canonical, "utf8"); + const dependencyDirectory = path.dirname(canonical); + if (extension === ".css") { + for (const match of contents.matchAll(CSS_RESOURCE_PATTERN)) { + const dependency = match[1]; + const resolved = dependency + ? resolveLocalResourcePath( + dependency, + dependency.startsWith("/") ? baseDirectory : dependencyDirectory, + ) + : null; + if (resolved) pending.push(resolved); + } + continue; + } + + await initializeModuleLexer; + const [imports] = parseModuleImports(contents); + for (const moduleImport of imports) { + const dependency = moduleImport.n; + if (!dependency || (!dependency.startsWith(".") && !dependency.startsWith("/"))) { + continue; + } + const resolved = dependency + ? resolveLocalResourcePath( + dependency, + dependency.startsWith("/") ? baseDirectory : dependencyDirectory, + ) + : null; + if (resolved) pending.push(resolved); + } + } + + return [...allowed]; +} + +export async function inspectHtmlArtifact( + input: ProjectInspectHtmlArtifactInput, +): Promise { + const canonicalWorkspaceRoot = await fs.realpath(path.resolve(input.cwd)).catch(() => null); + if (!canonicalWorkspaceRoot) { + return unsupported("The workspace is not available."); + } + + const requestedPath = path.isAbsolute(input.path) + ? path.resolve(input.path) + : path.resolve(canonicalWorkspaceRoot, input.path); + const absolutePath = await fs.realpath(requestedPath).catch(() => null); + if (!absolutePath || !isPathInside(absolutePath, canonicalWorkspaceRoot)) { + return unsupported("The HTML file is outside the active workspace or no longer exists."); + } + if (!isSupportedLocalHtmlPath(absolutePath)) { + return unsupported("Only HTML files can be inspected for browser preview."); + } + + const stat = await fs.stat(absolutePath).catch(() => null); + if (!stat?.isFile()) { + return unsupported("The HTML artifact is not a file."); + } + if (stat.size > HTML_INSPECTION_MAX_BYTES) { + return unsupported("The HTML artifact is too large to inspect safely."); + } + + const source = await fs.readFile(absolutePath, "utf8"); + const document = parse(source) as DocumentNode; + const baseDirectory = path.dirname(absolutePath); + const warnings: ProjectHtmlArtifactWarning[] = []; + const localResources: Array<{ value: string; executable: boolean }> = []; + let title: string | undefined; + let hasInlineScript = false; + let hasBrowserScript = false; + let hasDevSource = /(?:\/@vite\/client|react-refresh|\.tsx?(?:[?"'])|\.jsx(?:[?"']))/i.test( + source, + ); + let hasUnsupportedExecutable = false; + + const addWarning = (warning: ProjectHtmlArtifactWarning) => { + if ( + warnings.length < MAX_WARNINGS && + !warnings.some((entry) => entry.message === warning.message) + ) { + warnings.push(warning); + } + }; + + visit(document, (element) => { + const tagName = element.tagName.toLowerCase(); + if (tagName === "title" && !title) { + const candidate = textContentOf(element).replace(/\s+/g, " ").trim(); + if (candidate) title = candidate.slice(0, 500); + return; + } + + if (tagName === "script") { + const sourcePath = attributeOf(element, "src"); + if (!sourcePath) { + hasInlineScript = textContentOf(element).trim().length > 0; + return; + } + if (isExternalResource(sourcePath)) { + hasBrowserScript = true; + addWarning({ + code: "external-resource-blocked", + message: `External script blocked in preview: ${sourcePath.slice(0, 300)}`, + }); + return; + } + const extension = lowerCaseExtensionOf(sourcePath.split(/[?#]/, 1)[0] ?? ""); + if (extension && DEV_SOURCE_EXTENSIONS.has(extension)) { + hasDevSource = true; + } else if (extension && BROWSER_SCRIPT_EXTENSIONS.has(extension)) { + hasBrowserScript = true; + } else { + hasUnsupportedExecutable = true; + addWarning({ + code: "unsupported-local-resource", + message: `Unsupported script type: ${sourcePath.slice(0, 300)}`, + }); + } + localResources.push({ value: sourcePath, executable: true }); + return; + } + + const resourceAttribute = + tagName === "link" ? attributeOf(element, "href") : attributeOf(element, "src"); + if (!resourceAttribute) return; + if (isExternalResource(resourceAttribute)) { + addWarning({ + code: "external-resource-blocked", + message: `External resource blocked in preview: ${resourceAttribute.slice(0, 300)}`, + }); + return; + } + localResources.push({ value: resourceAttribute, executable: false }); + }); + + for (const resource of localResources) { + const cleanValue = resource.value.split(/[?#]/, 1)[0] ?? ""; + const extension = lowerCaseExtensionOf(cleanValue); + if ( + extension && + !DEV_SOURCE_EXTENSIONS.has(extension) && + !ALLOWED_LOCAL_RESOURCE_EXTENSIONS.has(extension) + ) { + addWarning({ + code: "unsupported-local-resource", + message: `Unsupported local preview resource: ${resource.value.slice(0, 300)}`, + }); + if (resource.executable) hasUnsupportedExecutable = true; + continue; + } + if (!(await resourceExists(resource.value, baseDirectory))) { + addWarning({ + code: "missing-local-resource", + message: `Local preview resource was not found: ${resource.value.slice(0, 300)}`, + }); + } + } + + const runTarget = hasDevSource + ? await nearestRunTarget(absolutePath, canonicalWorkspaceRoot) + : undefined; + const mode = hasDevSource + ? "dev-server-entrypoint" + : hasUnsupportedExecutable + ? "unsupported" + : hasInlineScript || hasBrowserScript + ? "interactive-bundle" + : "static-document"; + const reason = + mode === "dev-server-entrypoint" + ? runTarget + ? "This HTML file references source modules and must run through its development server." + : "This HTML file references source modules, but no dev or start script was found." + : mode === "unsupported" + ? "The HTML file references an executable resource type that the safe preview cannot run." + : undefined; + + return { + result: { + mode, + ...(title ? { title } : {}), + ...(reason ? { reason } : {}), + warnings, + ...(runTarget ? { runTarget } : {}), + }, + absolutePath, + baseDirectory, + allowedResourcePaths: await collectAllowedResourcePaths( + localResources.map((resource) => resource.value), + baseDirectory, + ), + }; +} diff --git a/apps/server/src/htmlPreview/Layers/HtmlArtifactPreview.test.ts b/apps/server/src/htmlPreview/Layers/HtmlArtifactPreview.test.ts new file mode 100644 index 000000000..5a1d9d764 --- /dev/null +++ b/apps/server/src/htmlPreview/Layers/HtmlArtifactPreview.test.ts @@ -0,0 +1,286 @@ +// FILE: HtmlArtifactPreview.test.ts +// Purpose: Security and rendering coverage for the isolated HTML preview listener. + +import fs from "node:fs/promises"; +import http from "node:http"; +import os from "node:os"; +import path from "node:path"; + +import { Effect } from "effect"; +import { afterEach, describe, expect, it } from "vitest"; + +import { + HtmlArtifactPreview, + type HtmlArtifactPreviewShape, +} from "../Services/HtmlArtifactPreview"; +import { HtmlArtifactPreviewLive } from "./HtmlArtifactPreview"; + +const temporaryDirectories: string[] = []; + +async function makeWorkspace(): Promise { + const workspace = await fs.mkdtemp(path.join(os.tmpdir(), "scient-html-preview-")); + temporaryDirectories.push(workspace); + return workspace; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => fs.rm(directory, { recursive: true, force: true })), + ); +}); + +interface PreviewResponse { + readonly status: number; + readonly headers: http.IncomingHttpHeaders; + readonly body: string; +} + +async function requestPreview( + previewUrl: string, + pathname = "/", + input: { method?: string; host?: string } = {}, +): Promise { + const url = new URL(previewUrl); + return new Promise((resolve, reject) => { + const request = http.request( + { + host: "127.0.0.1", + port: Number(url.port), + path: pathname, + method: input.method ?? "GET", + headers: { Host: input.host ?? url.host }, + }, + (response) => { + let body = ""; + response.setEncoding("utf8"); + response.on("data", (chunk: string) => { + body += chunk; + }); + response.on("end", () => { + resolve({ status: response.statusCode ?? 0, headers: response.headers, body }); + }); + }, + ); + request.on("error", reject); + request.end(); + }); +} + +function withPreviewService(use: (service: HtmlArtifactPreviewShape) => Promise): Promise { + return Effect.runPromise( + Effect.scoped( + Effect.gen(function* () { + const service = yield* HtmlArtifactPreview; + return yield* Effect.promise(() => use(service)); + }).pipe(Effect.provide(HtmlArtifactPreviewLive)), + ), + ); +} + +describe("HtmlArtifactPreviewLive", () => { + it("serves static HTML and presentation assets while refusing JavaScript", async () => { + const workspace = await makeWorkspace(); + await fs.writeFile( + path.join(workspace, "report.css"), + "body { color: green; background-image: url('./paper.png'); }", + ); + await fs.writeFile(path.join(workspace, "paper.png"), "image-bytes"); + await fs.writeFile(path.join(workspace, "unreferenced.css"), "body { color: red; }"); + await fs.writeFile(path.join(workspace, "ignored.js"), "window.pwned = true;"); + await fs.writeFile( + path.join(workspace, "report.html"), + '

Report

', + ); + + await withPreviewService(async (service) => { + const prepared = await Effect.runPromise( + service.prepare({ cwd: workspace, path: "report.html" }), + ); + expect(prepared.mode).toBe("static-document"); + expect(prepared.previewUrl).toBeDefined(); + + const document = await requestPreview(prepared.previewUrl!); + expect(document.status).toBe(200); + expect(document.body).toContain("

Report

"); + expect(document.headers["content-security-policy"]).toContain("script-src 'none'"); + expect(document.headers["permissions-policy"]).toContain("camera=()"); + + const stylesheet = await requestPreview(prepared.previewUrl!, "/report.css"); + expect(stylesheet.status).toBe(200); + expect(stylesheet.body).toContain("color: green"); + await expect(requestPreview(prepared.previewUrl!, "/paper.png")).resolves.toMatchObject({ + status: 200, + }); + await expect( + requestPreview(prepared.previewUrl!, "/unreferenced.css"), + ).resolves.toMatchObject({ status: 404 }); + + await expect(requestPreview(prepared.previewUrl!, "/ignored.js")).resolves.toMatchObject({ + status: 404, + }); + }); + }); + + it("serves browser-ready JavaScript only under the interactive policy", async () => { + const workspace = await makeWorkspace(); + await fs.writeFile( + path.join(workspace, "app.js"), + "import './chunk.js'; document.body.dataset.ready = 'yes';", + ); + await fs.writeFile(path.join(workspace, "chunk.js"), "export const ready = true;"); + await fs.writeFile(path.join(workspace, "secret.js"), "export const secret = 'no';"); + await fs.writeFile( + path.join(workspace, "index.html"), + '', + ); + + const previous = process.env.SCIENT_EXECUTABLE_HTML_PREVIEW; + process.env.SCIENT_EXECUTABLE_HTML_PREVIEW = "1"; + try { + await withPreviewService(async (service) => { + const prepared = await Effect.runPromise( + service.prepare({ cwd: workspace, path: "index.html" }), + ); + const document = await requestPreview(prepared.previewUrl!); + expect(document.headers["content-security-policy"]).toContain( + "script-src 'self' 'unsafe-inline'", + ); + expect(document.headers["content-security-policy"]).not.toContain("unsafe-eval"); + await expect(requestPreview(prepared.previewUrl!, "/app.js")).resolves.toMatchObject({ + status: 200, + }); + await expect(requestPreview(prepared.previewUrl!, "/chunk.js")).resolves.toMatchObject({ + status: 200, + }); + await expect(requestPreview(prepared.previewUrl!, "/secret.js")).resolves.toMatchObject({ + status: 404, + }); + }); + } finally { + if (previous === undefined) delete process.env.SCIENT_EXECUTABLE_HTML_PREVIEW; + else process.env.SCIENT_EXECUTABLE_HTML_PREVIEW = previous; + } + }); + + it("exposes no application routes and rejects invalid hosts, methods, dotfiles, and traversal", async () => { + const workspace = await makeWorkspace(); + await fs.writeFile(path.join(workspace, ".env"), "SECRET=value"); + await fs.writeFile(path.join(workspace, "index.html"), "

Safe

"); + + await withPreviewService(async (service) => { + const prepared = await Effect.runPromise( + service.prepare({ cwd: workspace, path: "index.html" }), + ); + const url = prepared.previewUrl!; + + for (const pathname of ["/api/auth/session", "/ws", "/.env", "/../.env", "/%2e%2e/.env"]) { + await expect(requestPreview(url, pathname)).resolves.toMatchObject({ status: 404 }); + } + await expect(requestPreview(url, "/", { method: "POST" })).resolves.toMatchObject({ + status: 404, + }); + await expect( + requestPreview(url, "/", { host: `invalid.preview.localhost:${new URL(url).port}` }), + ).resolves.toMatchObject({ status: 404 }); + }); + }); + + it("rejects sibling symlinks that escape the granted directory", async () => { + const workspace = await makeWorkspace(); + const outside = await makeWorkspace(); + await fs.writeFile(path.join(outside, "secret.css"), "body::before { content: 'secret'; }"); + await fs.symlink(path.join(outside, "secret.css"), path.join(workspace, "secret.css")); + await fs.writeFile( + path.join(workspace, "index.html"), + '', + ); + + await withPreviewService(async (service) => { + const prepared = await Effect.runPromise( + service.prepare({ cwd: workspace, path: "index.html" }), + ); + await expect(requestPreview(prepared.previewUrl!, "/secret.css")).resolves.toMatchObject({ + status: 404, + }); + }); + }); + + it("revalidates the root file after it is replaced by an escaping symlink", async () => { + const workspace = await makeWorkspace(); + const outside = await makeWorkspace(); + const entryPath = path.join(workspace, "index.html"); + await fs.writeFile(entryPath, "

Original

"); + await fs.writeFile(path.join(outside, "secret.html"), "

Secret

"); + + await withPreviewService(async (service) => { + const prepared = await Effect.runPromise( + service.prepare({ cwd: workspace, path: "index.html" }), + ); + await fs.rm(entryPath); + await fs.symlink(path.join(outside, "secret.html"), entryPath); + + const response = await requestPreview(prepared.previewUrl!); + expect(response.status).toBe(404); + expect(response.body).not.toContain("Secret"); + }); + }); + + it("revokes a capability explicitly and refuses every later request", async () => { + const workspace = await makeWorkspace(); + await fs.writeFile(path.join(workspace, "index.html"), "

Short lived

"); + + await withPreviewService(async (service) => { + const prepared = await Effect.runPromise( + service.prepare({ cwd: workspace, path: "index.html" }), + ); + expect((await requestPreview(prepared.previewUrl!)).status).toBe(200); + await expect( + Effect.runPromise(service.revoke({ previewUrl: prepared.previewUrl! })), + ).resolves.toEqual({ revoked: true }); + await expect(requestPreview(prepared.previewUrl!)).resolves.toMatchObject({ status: 404 }); + }); + }); + + it("bounds abandoned grants and evicts the oldest capability", async () => { + const workspace = await makeWorkspace(); + await fs.writeFile(path.join(workspace, "index.html"), "

Bounded

"); + + await withPreviewService(async (service) => { + const urls: string[] = []; + for (let index = 0; index < 129; index += 1) { + const prepared = await Effect.runPromise( + service.prepare({ cwd: workspace, path: "index.html" }), + ); + urls.push(prepared.previewUrl!); + } + await expect(requestPreview(urls[0]!)).resolves.toMatchObject({ status: 404 }); + await expect(requestPreview(urls.at(-1)!)).resolves.toMatchObject({ status: 200 }); + }); + }); + + it("keeps executable previews fail-closed behind an operational rollout switch", async () => { + const workspace = await makeWorkspace(); + await fs.writeFile(path.join(workspace, "app.js"), "document.body.dataset.ready = 'yes';"); + await fs.writeFile( + path.join(workspace, "index.html"), + '', + ); + const previous = process.env.SCIENT_EXECUTABLE_HTML_PREVIEW; + delete process.env.SCIENT_EXECUTABLE_HTML_PREVIEW; + try { + await withPreviewService(async (service) => { + const prepared = await Effect.runPromise( + service.prepare({ cwd: workspace, path: "index.html" }), + ); + expect(prepared.mode).toBe("unsupported"); + expect(prepared.previewUrl).toBeUndefined(); + expect(prepared.reason).toContain("rollout switch"); + }); + } finally { + if (previous === undefined) delete process.env.SCIENT_EXECUTABLE_HTML_PREVIEW; + else process.env.SCIENT_EXECUTABLE_HTML_PREVIEW = previous; + } + }); +}); diff --git a/apps/server/src/htmlPreview/Layers/HtmlArtifactPreview.ts b/apps/server/src/htmlPreview/Layers/HtmlArtifactPreview.ts new file mode 100644 index 000000000..66d054930 --- /dev/null +++ b/apps/server/src/htmlPreview/Layers/HtmlArtifactPreview.ts @@ -0,0 +1,433 @@ +// FILE: HtmlArtifactPreview.ts +// Purpose: Runs the capability-scoped, loopback-only HTML artifact preview listener. +// Layer: Server HTML-preview live implementation + +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import http from "node:http"; +import path from "node:path"; + +import type { + ProjectHtmlArtifactMode, + ProjectInspectHtmlArtifactInput, + ProjectPrepareHtmlArtifactPreviewInput, + ProjectRevokeHtmlArtifactPreviewInput, +} from "@synara/contracts"; +import { SUPPORTED_LOCAL_IMAGE_EXTENSIONS } from "@synara/shared/localPreviewFiles"; +import { Effect, Layer } from "effect"; + +import { inspectHtmlArtifact } from "../Inspector"; +import { + HtmlArtifactPreview, + HtmlArtifactPreviewError, + type HtmlArtifactPreviewShape, +} from "../Services/HtmlArtifactPreview"; + +const PREVIEW_GRANT_TTL_MS = 15 * 60 * 1000; +const PREVIEW_MAX_ACTIVE_GRANTS = 128; +const PREVIEW_MAX_ASSET_BYTES = 25 * 1024 * 1024; +const PREVIEW_HOST_SUFFIX = ".preview.localhost"; +const PREVIEW_ASSET_EXTENSIONS: ReadonlySet = new Set([ + ...SUPPORTED_LOCAL_IMAGE_EXTENSIONS, + ".css", + ".js", + ".mjs", + ".otf", + ".ttf", + ".woff", + ".woff2", +]); +const STATIC_PREVIEW_ASSET_EXTENSIONS: ReadonlySet = new Set( + [...PREVIEW_ASSET_EXTENSIONS].filter((extension) => extension !== ".js" && extension !== ".mjs"), +); + +function executableHtmlPreviewEnabled(): boolean { + const value = process.env.SCIENT_EXECUTABLE_HTML_PREVIEW?.trim().toLowerCase(); + return value === "1" || value === "true"; +} + +interface PreviewGrant { + readonly id: string; + readonly entryPath: string; + readonly baseDirectory: string; + readonly mode: Extract; + readonly expiresAtMs: number; + readonly allowedFiles: ReadonlySet; + readonly listenerPort: number; + readonly dedicatedServer?: http.Server; +} + +function isPathInside(candidate: string, root: string): boolean { + const relative = path.relative(root, candidate); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + +function previewContentSecurityPolicy(mode: PreviewGrant["mode"]): string { + const executable = mode === "interactive-bundle"; + return [ + "default-src 'none'", + executable ? "script-src 'self' 'unsafe-inline'" : "script-src 'none'", + "style-src 'self' 'unsafe-inline'", + "img-src 'self' data: blob:", + "font-src 'self' data:", + "media-src 'none'", + executable ? "connect-src 'self'" : "connect-src 'none'", + "worker-src 'none'", + "manifest-src 'none'", + "object-src 'none'", + "frame-src 'none'", + "child-src 'none'", + "base-uri 'none'", + "form-action 'none'", + "frame-ancestors http://localhost:* http://127.0.0.1:* scient:", + executable ? "sandbox allow-scripts allow-same-origin" : "sandbox", + ].join("; "); +} + +function contentTypeFor(filePath: string): string { + switch (path.extname(filePath).toLowerCase()) { + case ".html": + case ".htm": + return "text/html; charset=utf-8"; + case ".css": + return "text/css; charset=utf-8"; + case ".js": + case ".mjs": + return "text/javascript; charset=utf-8"; + case ".svg": + return "image/svg+xml"; + case ".png": + return "image/png"; + case ".jpg": + case ".jpeg": + return "image/jpeg"; + case ".gif": + return "image/gif"; + case ".webp": + return "image/webp"; + case ".avif": + return "image/avif"; + case ".ico": + return "image/x-icon"; + case ".woff": + return "font/woff"; + case ".woff2": + return "font/woff2"; + case ".ttf": + return "font/ttf"; + case ".otf": + return "font/otf"; + default: + return "application/octet-stream"; + } +} + +function writeNotFound(response: http.ServerResponse): void { + response.writeHead(404, { + "Cache-Control": "no-store", + "Content-Type": "text/plain; charset=utf-8", + "X-Content-Type-Options": "nosniff", + }); + response.end("Not Found"); +} + +function normalizedHostName(hostHeader: string | undefined): string | null { + if (!hostHeader) return null; + const hostname = hostHeader.trim().toLowerCase().replace(/:\d+$/, ""); + return hostname.length > 0 ? hostname : null; +} + +function grantIdFromHost(hostHeader: string | undefined): string | null { + const hostname = normalizedHostName(hostHeader); + if (!hostname?.startsWith("g-") || !hostname.endsWith(PREVIEW_HOST_SUFFIX)) return null; + const grantId = hostname.slice(2, -PREVIEW_HOST_SUFFIX.length); + return /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(grantId) + ? grantId + : null; +} + +function decodeRequestedAssetPath(rawUrl: string | undefined): string | null { + const rawPathname = (rawUrl ?? "").split(/[?#]/, 1)[0] ?? ""; + let decoded: string; + try { + decoded = decodeURIComponent(rawPathname); + } catch { + return null; + } + if (decoded.includes("\0") || decoded.includes("\\") || decoded.includes("%")) return null; + const relativePath = decoded.replace(/^\/+/, ""); + if (relativePath.length > 2_048) return null; + const segments = relativePath.split("/"); + if (segments.some((segment) => segment === "." || segment === ".." || segment.startsWith("."))) { + return null; + } + return relativePath; +} + +async function resolveGrantedFile( + grant: PreviewGrant, + rawUrl: string | undefined, +): Promise { + const relativePath = decodeRequestedAssetPath(rawUrl); + if (relativePath === null) return null; + if (relativePath.length > 0) { + const extension = path.extname(relativePath).toLowerCase(); + const allowedExtensions = + grant.mode === "interactive-bundle" + ? PREVIEW_ASSET_EXTENSIONS + : STATIC_PREVIEW_ASSET_EXTENSIONS; + if (!allowedExtensions.has(extension)) return null; + } + + const candidate = + relativePath.length === 0 ? grant.entryPath : path.resolve(grant.baseDirectory, relativePath); + const canonicalFile = await fs.realpath(candidate).catch(() => null); + if (!canonicalFile || !isPathInside(canonicalFile, grant.baseDirectory)) return null; + if (!grant.allowedFiles.has(canonicalFile)) return null; + const stat = await fs.stat(canonicalFile).catch(() => null); + return stat?.isFile() && stat.size <= PREVIEW_MAX_ASSET_BYTES ? canonicalFile : null; +} + +function securityHeaders(grant: PreviewGrant): Record { + return { + "Cache-Control": "no-store", + "Content-Security-Policy": previewContentSecurityPolicy(grant.mode), + "Cross-Origin-Opener-Policy": "same-origin", + "Cross-Origin-Resource-Policy": "same-origin", + "Origin-Agent-Cluster": "?1", + "Permissions-Policy": + "camera=(), microphone=(), geolocation=(), display-capture=(), fullscreen=(), payment=(), usb=(), serial=(), hid=(), clipboard-read=(), clipboard-write=()", + "Referrer-Policy": "no-referrer", + "X-Content-Type-Options": "nosniff", + }; +} + +async function closeServer(server: http.Server): Promise { + if (!server.listening) return; + await new Promise((resolve) => server.close(() => resolve())); +} + +async function listenOnLoopback(server: http.Server): Promise { + return new Promise((resolve, reject) => { + const onError = (cause: Error) => { + server.off("listening", onListening); + reject(cause); + }; + const onListening = () => { + server.off("error", onError); + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + if (port <= 0) { + reject(new Error("The listener did not expose a usable port.")); + return; + } + resolve(port); + }; + server.once("error", onError); + server.once("listening", onListening); + server.listen({ host: "127.0.0.1", port: 0 }); + }); +} + +export const HtmlArtifactPreviewLive = Layer.effect( + HtmlArtifactPreview, + Effect.gen(function* () { + const grants = new Map(); + let listenerPort = 0; + + const removeGrant = (id: string): boolean => { + const grant = grants.get(id); + if (!grant) return false; + grants.delete(id); + if (grant.dedicatedServer) { + void closeServer(grant.dedicatedServer); + } + return true; + }; + + const pruneExpiredGrants = (nowMs: number): void => { + for (const [id, grant] of grants) { + if (grant.expiresAtMs <= nowMs) removeGrant(id); + } + }; + + const reserveGrantCapacity = (nowMs: number): void => { + pruneExpiredGrants(nowMs); + while (grants.size >= PREVIEW_MAX_ACTIVE_GRANTS) { + const oldestId = grants.keys().next().value as string | undefined; + if (!oldestId) break; + removeGrant(oldestId); + } + }; + + const createServer = (dedicatedGrantId?: string): http.Server => + http.createServer((request, response) => { + void (async () => { + if (request.method !== "GET" && request.method !== "HEAD") { + writeNotFound(response); + return; + } + const nowMs = Date.now(); + pruneExpiredGrants(nowMs); + const grantId = dedicatedGrantId + ? normalizedHostName(request.headers.host) === "127.0.0.1" + ? dedicatedGrantId + : null + : grantIdFromHost(request.headers.host); + const grant = grantId ? grants.get(grantId) : undefined; + if (!grant || grant.expiresAtMs <= nowMs) { + writeNotFound(response); + return; + } + const filePath = await resolveGrantedFile(grant, request.url); + if (!filePath) { + writeNotFound(response); + return; + } + const stat = await fs.stat(filePath).catch(() => null); + if (!stat?.isFile() || stat.size > PREVIEW_MAX_ASSET_BYTES) { + writeNotFound(response); + return; + } + response.writeHead(200, { + ...securityHeaders(grant), + "Content-Length": String(stat.size), + "Content-Type": contentTypeFor(filePath), + }); + if (request.method === "HEAD") { + response.end(); + return; + } + const file = await fs.open(filePath, "r"); + const stream = file.createReadStream(); + stream.on("error", () => response.destroy()); + stream.on("close", () => void file.close().catch(() => undefined)); + stream.pipe(response); + })().catch(() => { + if (!response.headersSent) writeNotFound(response); + else response.destroy(); + }); + }); + + const server = createServer(); + + yield* Effect.acquireRelease( + Effect.tryPromise({ + try: async () => { + listenerPort = await listenOnLoopback(server); + }, + catch: (cause) => + new HtmlArtifactPreviewError({ + message: "Failed to start the isolated HTML preview listener.", + cause, + }), + }), + () => + Effect.promise(async () => { + const dedicatedServers = [...grants.values()].flatMap((grant) => + grant.dedicatedServer ? [grant.dedicatedServer] : [], + ); + grants.clear(); + await Promise.all([closeServer(server), ...dedicatedServers.map(closeServer)]); + }), + ); + + const inspect: HtmlArtifactPreviewShape["inspect"] = (input: ProjectInspectHtmlArtifactInput) => + Effect.tryPromise({ + try: async () => (await inspectHtmlArtifact(input)).result, + catch: (cause) => + new HtmlArtifactPreviewError({ message: "Failed to inspect the HTML artifact.", cause }), + }); + + const prepare: HtmlArtifactPreviewShape["prepare"] = ( + input: ProjectPrepareHtmlArtifactPreviewInput, + ) => + Effect.tryPromise({ + try: async () => { + const inspected = await inspectHtmlArtifact(input); + if ( + !executableHtmlPreviewEnabled() && + (inspected.result.mode === "interactive-bundle" || + inspected.result.mode === "dev-server-entrypoint") + ) { + const { runTarget: _runTarget, ...inspectionWithoutRunTarget } = inspected.result; + return { + ...inspectionWithoutRunTarget, + mode: "unsupported" as const, + reason: + "Executable HTML previews are disabled by the SCIENT_EXECUTABLE_HTML_PREVIEW rollout switch.", + }; + } + if ( + !inspected.absolutePath || + !inspected.baseDirectory || + (inspected.result.mode !== "static-document" && + inspected.result.mode !== "interactive-bundle") + ) { + return inspected.result; + } + const canonicalBaseDirectory = await fs.realpath(inspected.baseDirectory); + const nowMs = Date.now(); + reserveGrantCapacity(nowMs); + const expiresAtMs = nowMs + PREVIEW_GRANT_TTL_MS; + const id = crypto.randomUUID(); + const dedicatedServer = process.platform === "win32" ? createServer(id) : undefined; + const grantListenerPort = dedicatedServer + ? await listenOnLoopback(dedicatedServer) + : listenerPort; + grants.set(id, { + id, + entryPath: inspected.absolutePath, + baseDirectory: canonicalBaseDirectory, + mode: inspected.result.mode, + expiresAtMs, + allowedFiles: new Set([inspected.absolutePath, ...inspected.allowedResourcePaths]), + listenerPort: grantListenerPort, + ...(dedicatedServer ? { dedicatedServer } : {}), + }); + return { + ...inspected.result, + previewUrl: dedicatedServer + ? `http://127.0.0.1:${grantListenerPort}/` + : `http://g-${id}${PREVIEW_HOST_SUFFIX}:${grantListenerPort}/`, + expiresAt: new Date(expiresAtMs).toISOString(), + }; + }, + catch: (cause) => + new HtmlArtifactPreviewError({ + message: "Failed to prepare the HTML artifact preview.", + cause, + }), + }); + + const revoke: HtmlArtifactPreviewShape["revoke"] = ( + input: ProjectRevokeHtmlArtifactPreviewInput, + ) => + Effect.promise(async () => { + try { + const previewUrl = new URL(input.previewUrl); + const grantId = + grantIdFromHost(previewUrl.host) ?? + (previewUrl.hostname === "127.0.0.1" + ? ([...grants.values()].find( + (grant) => + grant.dedicatedServer && String(grant.listenerPort) === previewUrl.port, + )?.id ?? null) + : null); + const grant = grantId ? grants.get(grantId) : undefined; + if (!grant || String(grant.listenerPort) !== previewUrl.port) { + return { revoked: false }; + } + grants.delete(grant.id); + if (grant.dedicatedServer) { + await closeServer(grant.dedicatedServer); + } + return { revoked: true }; + } catch { + return { revoked: false }; + } + }); + + return HtmlArtifactPreview.of({ inspect, prepare, revoke }); + }), +); diff --git a/apps/server/src/htmlPreview/Services/HtmlArtifactPreview.ts b/apps/server/src/htmlPreview/Services/HtmlArtifactPreview.ts new file mode 100644 index 000000000..59dd98e98 --- /dev/null +++ b/apps/server/src/htmlPreview/Services/HtmlArtifactPreview.ts @@ -0,0 +1,35 @@ +// FILE: HtmlArtifactPreview.ts +// Purpose: Service contract for inspecting and preparing isolated HTML artifact previews. +// Layer: Server HTML-preview service boundary + +import type { + ProjectInspectHtmlArtifactInput, + ProjectInspectHtmlArtifactResult, + ProjectPrepareHtmlArtifactPreviewInput, + ProjectPrepareHtmlArtifactPreviewResult, + ProjectRevokeHtmlArtifactPreviewInput, + ProjectRevokeHtmlArtifactPreviewResult, +} from "@synara/contracts"; +import { Data, Effect, ServiceMap } from "effect"; + +export class HtmlArtifactPreviewError extends Data.TaggedError("HtmlArtifactPreviewError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +export interface HtmlArtifactPreviewShape { + readonly inspect: ( + input: ProjectInspectHtmlArtifactInput, + ) => Effect.Effect; + readonly prepare: ( + input: ProjectPrepareHtmlArtifactPreviewInput, + ) => Effect.Effect; + readonly revoke: ( + input: ProjectRevokeHtmlArtifactPreviewInput, + ) => Effect.Effect; +} + +export class HtmlArtifactPreview extends ServiceMap.Service< + HtmlArtifactPreview, + HtmlArtifactPreviewShape +>()("synara/htmlPreview/Services/HtmlArtifactPreview") {} diff --git a/apps/server/src/localServerMonitor.platform.test.ts b/apps/server/src/localServerMonitor.platform.test.ts new file mode 100644 index 000000000..1e67d5193 --- /dev/null +++ b/apps/server/src/localServerMonitor.platform.test.ts @@ -0,0 +1,107 @@ +// FILE: localServerMonitor.platform.test.ts +// Purpose: Exercises native listener and process-lineage discovery against a real loopback server. + +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; + +import { expect, it } from "vitest"; + +import { listLocalServers } from "./localServerMonitor"; + +const SUPPORTED_PLATFORM = new Set(["darwin", "linux", "win32"]); +const DISCOVERY_ATTEMPTS = 5; +const DISCOVERY_RETRY_MS = 400; + +const delay = (milliseconds: number): Promise => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + +async function waitForPort(child: ChildProcessWithoutNullStreams): Promise { + return new Promise((resolve, reject) => { + let stdout = ""; + const timeout = setTimeout(() => { + reject(new Error(`Timed out waiting for smoke server port. stderr=${child.stderr.read()}`)); + }, 5_000); + const cleanup = () => { + clearTimeout(timeout); + child.stdout.removeListener("data", onData); + child.removeListener("exit", onExit); + }; + const onData = (chunk: Buffer) => { + stdout += chunk.toString("utf8"); + const line = stdout.split(/\r?\n/, 1)[0]?.trim() ?? ""; + const port = Number(line); + if (Number.isInteger(port) && port > 0) { + cleanup(); + resolve(port); + } + }; + const onExit = (code: number | null) => { + cleanup(); + reject(new Error(`Smoke server exited before listening (code ${String(code)}).`)); + }; + child.stdout.on("data", onData); + child.once("exit", onExit); + }); +} + +async function stopChild(child: ChildProcessWithoutNullStreams): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + const exited = new Promise((resolve) => child.once("exit", () => resolve())); + child.kill(); + await Promise.race([exited, delay(2_000)]); + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + } +} + +it.runIf(SUPPORTED_PLATFORM.has(process.platform))( + "discovers a real Vite-labelled loopback listener through the native OS path", + { timeout: 45_000 }, + async () => { + const child = spawn( + process.execPath, + [ + "-e", + [ + "const http = require('node:http');", + "const server = http.createServer((_request, response) => response.end('ok'));", + "server.listen(0, '127.0.0.1', () => console.log(server.address().port));", + "process.on('SIGTERM', () => server.close(() => process.exit(0)));", + ].join(" "), + "vite-localhost-smoke", + ], + { stdio: ["pipe", "pipe", "pipe"] }, + ); + + try { + const port = await waitForPort(child); + let discovered = null; + let lastSnapshot = await listLocalServers({ includePageTitles: false }); + for (let attempt = 0; attempt < DISCOVERY_ATTEMPTS; attempt += 1) { + discovered = lastSnapshot.servers.find( + (server) => server.pid === child.pid && server.ports.includes(port), + ); + if (discovered) break; + await delay(DISCOVERY_RETRY_MS); + lastSnapshot = await listLocalServers({ includePageTitles: false }); + } + + expect( + discovered, + `Expected pid=${String(child.pid)} port=${String(port)}; found ${JSON.stringify( + lastSnapshot.servers.map((server) => ({ + pid: server.pid, + ports: server.ports, + command: server.command, + args: server.args, + })), + )}`, + ).toMatchObject({ + pid: child.pid, + args: expect.stringContaining("vite-localhost-smoke"), + ports: expect.arrayContaining([port]), + }); + } finally { + await stopChild(child); + } + }, +); diff --git a/apps/server/src/localServerMonitor.test.ts b/apps/server/src/localServerMonitor.test.ts index 8d0ea8fef..ba1301217 100644 --- a/apps/server/src/localServerMonitor.test.ts +++ b/apps/server/src/localServerMonitor.test.ts @@ -13,6 +13,7 @@ import { isLikelyDevServerProcess, parseLsofCwdOutput, parseLsofTcpListenOutput, + parseWindowsTcpListenOutput, type LocalServerProcessInfo, } from "./localServerMonitor"; @@ -59,6 +60,41 @@ describe("localServerMonitor", () => { ]); }); + it("parses PowerShell listener rows with command lineage for Windows", () => { + const commandLine = Buffer.from( + "node.exe C:\\repo\\node_modules\\vite\\bin\\vite.js || bun.exe run dev", + "utf8", + ).toString("base64"); + const snapshot = parseWindowsTcpListenOutput( + `127.0.0.1\t5173\t4321\tnode.exe\t4000\t${commandLine}\t4000,3900`, + ); + + expect(snapshot.listeners).toEqual([ + { + pid: 4321, + command: "node.exe", + protocol: "tcp", + host: "127.0.0.1", + port: 5173, + family: "tcp4", + }, + ]); + expect(snapshot.processInfoByPid.get(4321)).toEqual({ + ppid: 4000, + commandLine: "node.exe C:\\repo\\node_modules\\vite\\bin\\vite.js || bun.exe run dev", + ancestorPids: [4000, 3900], + }); + expect( + buildLocalServerProcesses(snapshot.listeners, snapshot.processInfoByPid)[0], + ).toMatchObject({ + pid: 4321, + ppid: 4000, + ancestorPids: [4000, 3900], + displayName: "Vite", + ports: [5173], + }); + }); + it("extracts a readable title from local server HTML", () => { expect( extractLocalServerPageTitle(` diff --git a/apps/server/src/localServerMonitor.ts b/apps/server/src/localServerMonitor.ts index ae6910d0b..eba234a32 100644 --- a/apps/server/src/localServerMonitor.ts +++ b/apps/server/src/localServerMonitor.ts @@ -16,6 +16,7 @@ import type { } from "@synara/contracts"; const PROCESS_OUTPUT_MAX_BUFFER_BYTES = 2 * 1024 * 1024; +const PROCESS_DISCOVERY_TIMEOUT_MS = 5_000; const STOP_SIGNAL_SETTLE_MS = 450; const MAX_PROCESS_ARGS_CHARS = 1_000; const PROCESS_LINEAGE_MAX_DEPTH = 4; @@ -41,6 +42,12 @@ export interface ParsedLsofListener { export interface LocalServerProcessInfo { readonly ppid: number; readonly commandLine: string; + readonly ancestorPids?: readonly number[]; +} + +export interface ParsedWindowsListenerSnapshot { + readonly listeners: readonly ParsedLsofListener[]; + readonly processInfoByPid: ReadonlyMap; } interface DevServerCandidateInput { @@ -125,7 +132,12 @@ function execFileText(command: string, args: readonly string[]): Promise execFile( command, [...args], - { encoding: "utf8", maxBuffer: PROCESS_OUTPUT_MAX_BUFFER_BYTES }, + { + encoding: "utf8", + maxBuffer: PROCESS_OUTPUT_MAX_BUFFER_BYTES, + timeout: PROCESS_DISCOVERY_TIMEOUT_MS, + windowsHide: true, + }, (error, stdout) => { if (error) { reject(error); @@ -273,6 +285,61 @@ function parseProcessInfo(output: string): Map { return rows; } +export function parseWindowsTcpListenOutput(output: string): ParsedWindowsListenerSnapshot { + const listeners: ParsedLsofListener[] = []; + const processInfoByPid = new Map(); + for (const rawLine of output.split(/\r?\n/g)) { + const [ + hostValue, + portValue, + pidValue, + commandValue, + ppidValue, + encodedCommand, + ancestorsValue, + ] = rawLine.split("\t"); + const port = Number(portValue); + const pid = Number(pidValue); + const ppid = Number(ppidValue); + const host = hostValue?.trim() ?? ""; + if ( + host.length === 0 || + !Number.isInteger(port) || + port <= 0 || + port > 65_535 || + !Number.isInteger(pid) || + pid <= 0 + ) { + continue; + } + const commandLine = (() => { + try { + return redactProcessArgs(Buffer.from(encodedCommand ?? "", "base64").toString("utf8")); + } catch { + return commandValue?.trim() || "unknown"; + } + })(); + const ancestorPids = (ancestorsValue ?? "") + .split(",") + .map(Number) + .filter((ancestorPid) => Number.isInteger(ancestorPid) && ancestorPid > 1); + listeners.push({ + pid, + command: commandValue?.trim() || "unknown", + protocol: "tcp", + host, + port, + family: host.includes(":") ? "tcp6" : "tcp4", + }); + processInfoByPid.set(pid, { + ppid: Number.isInteger(ppid) && ppid > 0 ? ppid : 1, + commandLine: commandLine || commandValue?.trim() || "unknown", + ...(ancestorPids.length > 0 ? { ancestorPids } : {}), + }); + } + return { listeners, processInfoByPid }; +} + function tokenizeCommandLine(commandLine: string): string[] { return [...commandLine.matchAll(/"([^"]*)"|'([^']*)'|(\S+)/g)] .map((match) => match[1] ?? match[2] ?? match[3] ?? "") @@ -712,7 +779,7 @@ async function mapWithConcurrency( limit: number, mapper: (item: T) => Promise, ): Promise { - const results = new Array(items.length); + const results: R[] = []; let nextIndex = 0; async function worker(): Promise { while (nextIndex < items.length) { @@ -801,6 +868,9 @@ function toServerProcess( ...(typeof processInfo?.ppid === "number" && processInfo.ppid > 0 ? { ppid: processInfo.ppid } : {}), + ...(processInfo?.ancestorPids && processInfo.ancestorPids.length > 0 + ? { ancestorPids: [...processInfo.ancestorPids] } + : {}), command, displayName: formatDisplayName(command, detectionArgs), ...(cwd ? { cwd } : {}), @@ -853,15 +923,45 @@ function groupListenersByPid( } async function readLsofListeners(): Promise { - if (process.platform === "win32") { - return []; - } const output = await execFileText("lsof", ["-nP", "-iTCP", "-sTCP:LISTEN", "-F", "pcPn"]).catch( () => "", ); return parseLsofTcpListenOutput(output); } +async function readWindowsListenerSnapshot(): Promise { + const script = [ + "$ErrorActionPreference = 'SilentlyContinue'", + "$connections = Get-NetTCPConnection -State Listen", + "foreach ($connection in $connections) {", + " $process = Get-CimInstance Win32_Process -Filter ('ProcessId = ' + $connection.OwningProcess)", + " if ($null -eq $process) { continue }", + " $lineage = @()", + " $ancestorIds = @()", + " $current = $process", + " for ($depth = 0; $depth -lt 4 -and $null -ne $current; $depth++) {", + " if ($current.CommandLine) { $lineage += $current.CommandLine }", + " $parentId = [int]$current.ParentProcessId", + " if ($parentId -le 1) { break }", + " $ancestorIds += $parentId", + " $current = Get-CimInstance Win32_Process -Filter ('ProcessId = ' + $parentId)", + " }", + " $commandLine = [string]::Join(' || ', $lineage)", + " $encoded = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($commandLine))", + " $fields = @($connection.LocalAddress, $connection.LocalPort, $connection.OwningProcess, $process.Name, $process.ParentProcessId, $encoded, [string]::Join(',', $ancestorIds))", + " [Console]::WriteLine([string]::Join([char]9, $fields))", + "}", + ].join("\n"); + const output = await execFileText("powershell.exe", [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-Command", + script, + ]).catch(() => ""); + return parseWindowsTcpListenOutput(output); +} + async function readProcessInfoBatch( pids: readonly number[], ): Promise> { @@ -934,17 +1034,25 @@ export function buildLocalServerProcesses( ); } -export async function listLocalServers(): Promise { - const listeners = await readLsofListeners(); +export async function listLocalServers(options?: { + includePageTitles?: boolean; +}): Promise { + const windowsSnapshot = process.platform === "win32" ? await readWindowsListenerSnapshot() : null; + const listeners = windowsSnapshot?.listeners ?? (await readLsofListeners()); const pids = [...new Set(listeners.map((listener) => listener.pid))]; - const processInfoByPid = await readProcessInfoWithAncestors(pids); + const processInfoByPid = windowsSnapshot + ? new Map(windowsSnapshot.processInfoByPid) + : await readProcessInfoWithAncestors(pids); // Resolve cwd across the full lineage so a generic port-holding child can fall // back to its dev-tool parent's directory (cwd is inherited across fork/exec). const cwdByPid = await readProcessCwdBatch([...new Set([...pids, ...processInfoByPid.keys()])]); const servers = buildLocalServerProcesses(listeners, processInfoByPid, cwdByPid); return { generatedAt: new Date().toISOString(), - servers: await enrichLocalServerProcessesWithPageTitles(servers), + servers: + options?.includePageTitles === false + ? servers + : await enrichLocalServerProcessesWithPageTitles(servers), }; } diff --git a/apps/server/src/provider/Layers/DroidAdapter.ts b/apps/server/src/provider/Layers/DroidAdapter.ts index 04f8e7bab..ca3c4c888 100644 --- a/apps/server/src/provider/Layers/DroidAdapter.ts +++ b/apps/server/src/provider/Layers/DroidAdapter.ts @@ -2052,7 +2052,6 @@ export function makeDroidAdapter( cwd, clientName: "Scient Model Discovery", }); - yield* runtime.start(); const result = yield* discoverDroidAcpModels(runtime); const commands = yield* runtime.getAvailableCommands; setDroidDiscoveryCacheEntry(commandDiscoveryCache, cacheKey, { diff --git a/apps/server/src/provider/acp/DroidAcpSupport.test.ts b/apps/server/src/provider/acp/DroidAcpSupport.test.ts index 663610f55..3c3a66879 100644 --- a/apps/server/src/provider/acp/DroidAcpSupport.test.ts +++ b/apps/server/src/provider/acp/DroidAcpSupport.test.ts @@ -246,7 +246,9 @@ describe("applyDroidAcpInteractionMode", () => { }); describe("discoverDroidAcpModels", () => { - it("reads each model's reasoning choices from session config options", async () => { + it("starts the session before reading each model's reasoning choices", async () => { + let startCalls = 0; + let started = false; let currentModel = "model-a"; const configOptions = (): ReadonlyArray => [ { @@ -283,7 +285,22 @@ describe("discoverDroidAcpModels", () => { }, ]; const runtime = { - getConfigOptions: Effect.sync(configOptions), + start: () => + Effect.sync(() => { + startCalls += 1; + started = true; + return { + sessionId: "droid-model-discovery-test", + initializeResult: { protocolVersion: 1 }, + sessionSetupResult: { + sessionId: "droid-model-discovery-test", + configOptions: configOptions(), + }, + modelConfigId: "model", + sessionSetupMethod: "new" as const, + }; + }), + getConfigOptions: Effect.sync(() => (started ? configOptions() : [])), setConfigOption: (configId: string, value: string | boolean) => { if (configId === "model") { currentModel = String(value); @@ -293,6 +310,7 @@ describe("discoverDroidAcpModels", () => { }; const result = await Effect.runPromise(discoverDroidAcpModels(runtime)); + expect(startCalls).toBe(1); expect(result.models).toEqual([ expect.objectContaining({ slug: "model-a", diff --git a/apps/server/src/provider/acp/DroidAcpSupport.ts b/apps/server/src/provider/acp/DroidAcpSupport.ts index 592386e1e..e0bdb473f 100644 --- a/apps/server/src/provider/acp/DroidAcpSupport.ts +++ b/apps/server/src/provider/acp/DroidAcpSupport.ts @@ -400,13 +400,18 @@ function droidModelDescriptor( } /** - * Reads the model catalog from ACP and reselects each model so Droid returns that - * model's current reasoning choices. Discovery runs in a disposable session. + * Starts the disposable ACP session, reads its model catalog, and reselects each + * model so Droid returns that model's current reasoning choices. + * + * Owning startup here keeps every discovery caller on the same lifecycle path; + * reading configuration before `session/new` completes always produces an empty + * inventory. */ export function discoverDroidAcpModels( - runtime: Pick, + runtime: Pick, ): Effect.Effect { return Effect.gen(function* () { + yield* runtime.start(); const initialOptions = yield* runtime.getConfigOptions; const modelConfig = findDroidSelectConfig(initialOptions, { id: DROID_MODEL_CONFIG_ID, diff --git a/apps/server/src/serverLayers.ts b/apps/server/src/serverLayers.ts index 17e45cec1..4b47fed35 100644 --- a/apps/server/src/serverLayers.ts +++ b/apps/server/src/serverLayers.ts @@ -16,6 +16,7 @@ import { ThreadDeletionReactorLive } from "./orchestration/Layers/ThreadDeletion import { OrchestrationLayerLive } from "./orchestration/runtimeLayer"; import { DevServerManagerLive } from "./devServerManager"; +import { HtmlArtifactPreviewLive } from "./htmlPreview/Layers/HtmlArtifactPreview"; import { KeybindingsLive } from "./keybindings"; import { GitCoreLive } from "./git/Layers/GitCore"; import { GitLayerLive, TextGenerationLayerLive } from "./git/runtimeLayer"; @@ -139,6 +140,7 @@ export function makeServerRuntimeServicesLayer() { orchestrationReactorLayer, threadDeletionReactorLayer, devServerManagerLayer, + HtmlArtifactPreviewLive, GitLayerLive, TextGenerationLayerLive, TerminalLayerLive, diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 14f84c2e7..8370b0e4d 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1,6 +1,6 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import path from "node:path"; -import { DEFAULT_MODEL_BY_PROVIDER } from "@synara/contracts"; +import { DEFAULT_GIT_TEXT_GENERATION_MODEL, DEFAULT_MODEL_BY_PROVIDER } from "@synara/contracts"; import { Effect, FileSystem, Layer } from "effect"; import { describe, expect, it } from "vitest"; import { ServerConfig } from "./config"; @@ -31,6 +31,10 @@ describe("ServerSettingsService", () => { expect(settings.defaultThreadEnvMode).toBe("local"); expect(settings.enableProviderUpdateChecks).toBe(true); expect(settings.telemetryPrivacyLevel).toBe("essential"); + expect(settings.textGenerationModelSelection).toEqual({ + provider: "codex", + model: DEFAULT_GIT_TEXT_GENERATION_MODEL, + }); }); it("persists updates and reloads them", async () => { diff --git a/apps/server/src/voice/chatGptVoiceTranscription.ts b/apps/server/src/voice/chatGptVoiceTranscription.ts new file mode 100644 index 000000000..c93d05d08 --- /dev/null +++ b/apps/server/src/voice/chatGptVoiceTranscription.ts @@ -0,0 +1,404 @@ +// FILE: chatGptVoiceTranscription.ts +// Purpose: Implements the isolated ChatGPT-subscription voice backend. +// Layer: Server voice backend +// Depends on: Codex-provided ChatGPT OAuth tokens and the private transcription endpoint. + +import { Buffer } from "node:buffer"; + +import type { + ServerVoiceTranscriptionInput, + ServerVoiceTranscriptionResult, +} from "@synara/contracts"; +import { extractChatGptAccountIdFromToken } from "@synara/shared/chatGptVoiceAuth"; +import { + type NormalizedVoiceClip, + type VoiceTranscript, + type VoiceTranscriptionBackend, + VoiceTranscriptionBackendError, + type VoiceTranscriptionErrorKind, +} from "@synara/shared/voiceTranscription"; + +export const CHATGPT_TRANSCRIPTIONS_URL = "https://chatgpt.com/backend-api/transcribe"; + +const MAX_AUDIO_BYTES = 10 * 1024 * 1024; +const MAX_DURATION_MS = 120_000; +const DEFAULT_REQUEST_TIMEOUT_MS = 30_000; + +export interface ChatGptVoiceAuthContext { + readonly token: string; +} + +type ResolveChatGptVoiceAuth = ( + refreshToken: boolean, + signal?: AbortSignal, +) => Promise; + +interface ChatGptVoiceBackendOptions { + readonly resolveAuth: ResolveChatGptVoiceAuth; + readonly fetchImpl?: typeof fetch; + readonly requestTimeoutMs?: number; +} + +function backendError(input: { + kind: VoiceTranscriptionErrorKind; + fallbackAllowed?: boolean; + safeMessage: string; + retryAfterMs?: number; + cause?: unknown; +}): VoiceTranscriptionBackendError { + return new VoiceTranscriptionBackendError({ + kind: input.kind, + fallbackAllowed: input.fallbackAllowed ?? true, + safeMessage: input.safeMessage, + ...(input.retryAfterMs !== undefined ? { retryAfterMs: input.retryAfterMs } : {}), + ...(input.cause !== undefined ? { cause: input.cause } : {}), + }); +} + +function requireChatGptAccountContext(auth: ChatGptVoiceAuthContext): { + readonly token: string; + readonly accountId: string; +} { + const token = auth.token.trim(); + if (!token) { + throw backendError({ + kind: "authentication", + safeMessage: "No ChatGPT session is available. Sign in to ChatGPT in Codex.", + }); + } + + const accountId = extractChatGptAccountIdFromToken(token); + if (!accountId) { + throw backendError({ + kind: "authentication", + safeMessage: + "The ChatGPT session does not include an account context. Sign in to ChatGPT in Codex again.", + }); + } + + return { token, accountId }; +} + +async function resolveAccountContext( + resolveAuth: ResolveChatGptVoiceAuth, + refreshToken: boolean, + signal?: AbortSignal, +): Promise<{ readonly token: string; readonly accountId: string }> { + try { + return requireChatGptAccountContext(await resolveAuth(refreshToken, signal)); + } catch (cause) { + if (cause instanceof VoiceTranscriptionBackendError) throw cause; + throw backendError({ + kind: "authentication", + safeMessage: "Scient could not read the ChatGPT session from Codex.", + cause, + }); + } +} + +function validateClip(clip: NormalizedVoiceClip): void { + if (clip.mimeType !== "audio/wav") { + throw backendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage: "Only WAV audio is supported for voice transcription.", + }); + } + if (clip.sampleRateHz !== 24_000) { + throw backendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage: "Voice transcription requires 24 kHz mono WAV audio.", + }); + } + if (clip.durationMs <= 0 || clip.durationMs > MAX_DURATION_MS) { + throw backendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage: + clip.durationMs <= 0 + ? "Voice messages must include a positive duration." + : "Voice messages are limited to 120 seconds.", + }); + } + if (clip.audioBytes.byteLength === 0 || clip.audioBytes.byteLength > MAX_AUDIO_BYTES) { + throw backendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage: + clip.audioBytes.byteLength === 0 + ? "The recorded audio is empty." + : "Voice messages are limited to 10 MB.", + }); + } + const header = Buffer.from( + clip.audioBytes.buffer, + clip.audioBytes.byteOffset, + clip.audioBytes.byteLength, + ); + if ( + header.byteLength < 12 || + header.toString("ascii", 0, 4) !== "RIFF" || + header.toString("ascii", 8, 12) !== "WAVE" + ) { + throw backendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage: "The recorded audio is not a valid WAV file.", + }); + } +} + +function parseRetryAfterMs(response: Response): number | undefined { + const value = response.headers.get("retry-after")?.trim(); + if (!value) return undefined; + + const seconds = Number(value); + if (Number.isFinite(seconds) && seconds >= 0) return Math.round(seconds * 1_000); + + const retryAt = Date.parse(value); + return Number.isFinite(retryAt) ? Math.max(0, retryAt - Date.now()) : undefined; +} + +function statusFailure(response: Response): VoiceTranscriptionBackendError { + if (response.status === 401) { + return backendError({ + kind: "authentication", + safeMessage: "ChatGPT rejected the current login. Sign in to ChatGPT in Codex again.", + }); + } + if (response.status === 403) { + return backendError({ + kind: "entitlement", + safeMessage: "ChatGPT voice transcription is not available for this session.", + }); + } + if (response.status === 429) { + const retryAfterMs = parseRetryAfterMs(response); + return backendError({ + kind: "rate-limit", + safeMessage: "ChatGPT voice transcription is temporarily rate limited.", + ...(retryAfterMs !== undefined ? { retryAfterMs } : {}), + }); + } + return backendError({ + kind: "provider-error", + safeMessage: `ChatGPT voice transcription failed with status ${response.status}.`, + }); +} + +async function postTranscription(input: { + readonly fetchImpl: typeof fetch; + readonly clip: NormalizedVoiceClip; + readonly token: string; + readonly accountId: string; + readonly signal: AbortSignal; + readonly timeoutMs: number; +}): Promise { + if (input.signal.aborted) { + throw backendError({ + kind: "cancelled", + fallbackAllowed: false, + safeMessage: "Voice transcription was cancelled.", + cause: input.signal.reason, + }); + } + + const controller = new AbortController(); + let timedOut = false; + const abortFromCaller = () => controller.abort(input.signal.reason); + input.signal.addEventListener("abort", abortFromCaller, { once: true }); + const timeout = setTimeout(() => { + timedOut = true; + controller.abort(); + }, input.timeoutMs); + + try { + const bytes = Buffer.from( + input.clip.audioBytes.buffer, + input.clip.audioBytes.byteOffset, + input.clip.audioBytes.byteLength, + ); + const formData = new FormData(); + formData.append("file", new Blob([bytes], { type: input.clip.mimeType }), "voice.wav"); + return await input.fetchImpl(CHATGPT_TRANSCRIPTIONS_URL, { + method: "POST", + headers: { + Accept: "application/json", + Authorization: `Bearer ${input.token}`, + "ChatGPT-Account-Id": input.accountId, + Origin: "https://chatgpt.com", + "User-Agent": "Scient Desktop", + originator: "scient-desktop", + }, + body: formData, + signal: controller.signal, + }); + } catch (cause) { + if (input.signal.aborted) { + throw backendError({ + kind: "cancelled", + fallbackAllowed: false, + safeMessage: "Voice transcription was cancelled.", + cause, + }); + } + if (timedOut) { + throw backendError({ + kind: "timeout", + safeMessage: "ChatGPT voice transcription timed out.", + cause, + }); + } + throw backendError({ + kind: "network", + safeMessage: "Scient could not reach ChatGPT voice transcription.", + cause, + }); + } finally { + clearTimeout(timeout); + input.signal.removeEventListener("abort", abortFromCaller); + } +} + +async function parseTranscript(response: Response): Promise { + let payload: unknown; + try { + payload = JSON.parse(await response.text()) as unknown; + } catch (cause) { + throw backendError({ + kind: "malformed-response", + safeMessage: "ChatGPT returned an invalid transcription response.", + cause, + }); + } + + const record = + typeof payload === "object" && payload !== null ? (payload as Record) : null; + const textValue = + typeof record?.text === "string" + ? record.text + : typeof record?.transcript === "string" + ? record.transcript + : ""; + const text = textValue.trim(); + if (!text) { + throw backendError({ + kind: "malformed-response", + safeMessage: "ChatGPT returned an empty transcription response.", + }); + } + return { text }; +} + +export function createChatGptVoiceTranscriptionBackend( + options: ChatGptVoiceBackendOptions, +): VoiceTranscriptionBackend { + const fetchImpl = options.fetchImpl ?? globalThis.fetch; + const timeoutMs = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; + + return { + id: "chatgpt", + async getAvailability({ signal }: { readonly signal?: AbortSignal } = {}) { + if (typeof fetchImpl !== "function") { + return { state: "unavailable", reason: "fetch-unavailable" }; + } + try { + await resolveAccountContext(options.resolveAuth, false, signal); + return { state: "ready" }; + } catch (cause) { + return { + state: "unavailable", + reason: cause instanceof VoiceTranscriptionBackendError ? cause.kind : "authentication", + }; + } + }, + async transcribe(clip, { signal }) { + validateClip(clip); + if (typeof fetchImpl !== "function") { + throw backendError({ + kind: "backend-unavailable", + safeMessage: "ChatGPT voice transcription is unavailable in this runtime.", + }); + } + + let account = await resolveAccountContext(options.resolveAuth, false, signal); + let response = await postTranscription({ + fetchImpl, + clip, + ...account, + signal, + timeoutMs, + }); + + if (response.status === 401 || response.status === 403) { + account = await resolveAccountContext(options.resolveAuth, true, signal); + response = await postTranscription({ + fetchImpl, + clip, + ...account, + signal, + timeoutMs, + }); + } + + if (!response.ok) throw statusFailure(response); + return parseTranscript(response); + }, + }; +} + +function decodeServerRequest(request: ServerVoiceTranscriptionInput): NormalizedVoiceClip { + if (request.mimeType !== "audio/wav") { + throw backendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage: "Only WAV audio is supported for voice transcription.", + }); + } + + const normalized = request.audioBase64.trim().replace(/\s+/gu, ""); + if (!normalized || !/^[A-Za-z0-9+/]+={0,2}$/u.test(normalized)) { + throw backendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage: "The recorded audio could not be decoded.", + }); + } + + const audioBytes = Buffer.from(normalized, "base64"); + if (!audioBytes.byteLength || audioBytes.toString("base64") !== normalized) { + throw backendError({ + kind: "invalid-audio", + fallbackAllowed: false, + safeMessage: "The recorded audio could not be decoded.", + }); + } + + return { + audioBytes, + mimeType: request.mimeType, + sampleRateHz: request.sampleRateHz, + durationMs: request.durationMs, + cwd: request.cwd, + ...(request.threadId ? { threadId: request.threadId } : {}), + }; +} + +// Compatibility entrypoint used until the application-level router owns the request. +export async function transcribeVoiceWithChatGptSession(input: { + readonly request: ServerVoiceTranscriptionInput; + readonly resolveAuth: ResolveChatGptVoiceAuth; + readonly fetchImpl?: typeof fetch; + readonly signal?: AbortSignal; + readonly requestTimeoutMs?: number; +}): Promise { + const backend = createChatGptVoiceTranscriptionBackend({ + resolveAuth: input.resolveAuth, + ...(input.fetchImpl ? { fetchImpl: input.fetchImpl } : {}), + ...(input.requestTimeoutMs !== undefined ? { requestTimeoutMs: input.requestTimeoutMs } : {}), + }); + return backend.transcribe(decodeServerRequest(input.request), { + signal: input.signal ?? new AbortController().signal, + }); +} diff --git a/apps/server/src/voiceTranscription.test.ts b/apps/server/src/voiceTranscription.test.ts index 26a0a56a4..76d9e8033 100644 --- a/apps/server/src/voiceTranscription.test.ts +++ b/apps/server/src/voiceTranscription.test.ts @@ -5,12 +5,27 @@ // Depends on: voiceTranscription utility and mocked fetch responses. import type { ServerVoiceTranscriptionInput } from "@synara/contracts"; +import { VoiceTranscriptionBackendError } from "@synara/shared/voiceTranscription"; import { describe, expect, it, vi } from "vitest"; import { transcribeVoiceWithChatGptSession } from "./voiceTranscription"; const WAV_BASE64 = Buffer.from("RIFF0000WAVE", "ascii").toString("base64"); +function encodeJwtPart(value: unknown): string { + return Buffer.from(JSON.stringify(value), "utf8").toString("base64url"); +} + +function chatGptToken(accountId = "account-123"): string { + return `${encodeJwtPart({ alg: "none" })}.${encodeJwtPart({ + "https://api.openai.com/auth": { chatgpt_account_id: accountId }, + })}.`; +} + +function tokenWithoutAccount(): string { + return `${encodeJwtPart({ alg: "none" })}.${encodeJwtPart({ subject: "user-123" })}.`; +} + const baseRequest: ServerVoiceTranscriptionInput = { provider: "codex", cwd: "/tmp/project", @@ -28,13 +43,16 @@ describe("transcribeVoiceWithChatGptSession", () => { await transcribeVoiceWithChatGptSession({ request: baseRequest, - resolveAuth: async () => ({ token: "chatgpt-token" }), + resolveAuth: async () => ({ token: chatGptToken() }), fetchImpl, }); const [url, init] = vi.mocked(fetchImpl).mock.calls[0] ?? []; + expect(init).toBeDefined(); expect(url).toBe("https://chatgpt.com/backend-api/transcribe"); - expect((init?.body as FormData).get("model")).toBeNull(); + expect((init?.body as FormData | undefined)?.get("model")).toBeNull(); + expect(new Headers(init?.headers).get("chatgpt-account-id")).toBe("account-123"); + expect(new Headers(init?.headers).get("originator")).toBe("scient-desktop"); }); it("refreshes the ChatGPT session once when the upload is unauthorized", async () => { @@ -43,7 +61,7 @@ describe("transcribeVoiceWithChatGptSession", () => { .mockResolvedValueOnce(new Response("{}", { status: 401 })) .mockResolvedValueOnce(new Response(JSON.stringify({ text: "hello" }), { status: 200 })); const resolveAuth = vi.fn(async (refreshToken: boolean) => ({ - token: refreshToken ? "fresh-chatgpt-token" : "stale-chatgpt-token", + token: refreshToken ? chatGptToken("fresh-account") : chatGptToken("stale-account"), })); await transcribeVoiceWithChatGptSession({ @@ -52,8 +70,133 @@ describe("transcribeVoiceWithChatGptSession", () => { fetchImpl: fetchImpl as unknown as typeof fetch, }); - expect(resolveAuth).toHaveBeenNthCalledWith(1, false); - expect(resolveAuth).toHaveBeenNthCalledWith(2, true); + expect(resolveAuth).toHaveBeenNthCalledWith(1, false, expect.any(AbortSignal)); + expect(resolveAuth).toHaveBeenNthCalledWith(2, true, expect.any(AbortSignal)); expect(fetchImpl).toHaveBeenCalledTimes(2); + const secondRequest = vi.mocked(fetchImpl).mock.calls[1]?.[1]; + expect(new Headers(secondRequest?.headers).get("chatgpt-account-id")).toBe("fresh-account"); + }); + + it("refreshes once when the first request is forbidden", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValueOnce(new Response("{}", { status: 403 })) + .mockResolvedValueOnce(new Response(JSON.stringify({ text: "hello" }), { status: 200 })); + const resolveAuth = vi.fn(async () => ({ token: chatGptToken() })); + + await transcribeVoiceWithChatGptSession({ + request: baseRequest, + resolveAuth, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + expect(resolveAuth).toHaveBeenNthCalledWith(1, false, expect.any(AbortSignal)); + expect(resolveAuth).toHaveBeenNthCalledWith(2, true, expect.any(AbortSignal)); + }); + + it("fails before upload when the session has no account context", async () => { + const fetchImpl = vi.fn(); + + const promise = transcribeVoiceWithChatGptSession({ + request: baseRequest, + resolveAuth: async () => ({ token: tokenWithoutAccount() }), + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + await expect(promise).rejects.toMatchObject({ + kind: "authentication", + fallbackAllowed: true, + }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it("classifies a persistent forbidden response as an entitlement failure", async () => { + const fetchImpl = vi.fn(async () => new Response("{}", { status: 403 })); + + const promise = transcribeVoiceWithChatGptSession({ + request: baseRequest, + resolveAuth: async () => ({ token: chatGptToken() }), + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + await expect(promise).rejects.toMatchObject({ + kind: "entitlement", + fallbackAllowed: true, + }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + + it("preserves retry timing for rate limits", async () => { + const fetchImpl = vi.fn( + async () => new Response("{}", { status: 429, headers: { "Retry-After": "3" } }), + ); + + const promise = transcribeVoiceWithChatGptSession({ + request: baseRequest, + resolveAuth: async () => ({ token: chatGptToken() }), + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + await expect(promise).rejects.toMatchObject({ kind: "rate-limit", retryAfterMs: 3_000 }); + }); + + it("reports malformed successful responses", async () => { + const fetchImpl = vi.fn(async () => new Response("not-json", { status: 200 })); + + const promise = transcribeVoiceWithChatGptSession({ + request: baseRequest, + resolveAuth: async () => ({ token: chatGptToken() }), + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + await expect(promise).rejects.toMatchObject({ kind: "malformed-response" }); + }); + + it("preserves caller cancellation without falling back", async () => { + const controller = new AbortController(); + controller.abort(new Error("cancelled by caller")); + + const promise = transcribeVoiceWithChatGptSession({ + request: baseRequest, + resolveAuth: async () => ({ token: chatGptToken() }), + fetchImpl: vi.fn() as unknown as typeof fetch, + signal: controller.signal, + }); + + await expect(promise).rejects.toMatchObject({ + kind: "cancelled", + fallbackAllowed: false, + }); + }); + + it("classifies an expired request deadline as a fallback-safe timeout", async () => { + const fetchImpl = vi.fn((_url: unknown, init?: RequestInit) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => reject(init.signal?.reason), { + once: true, + }); + }); + }); + + const promise = transcribeVoiceWithChatGptSession({ + request: baseRequest, + resolveAuth: async () => ({ token: chatGptToken() }), + fetchImpl: fetchImpl as unknown as typeof fetch, + requestTimeoutMs: 1, + }); + + await expect(promise).rejects.toMatchObject({ kind: "timeout", fallbackAllowed: true }); + }); + + it("does not allow fallback for invalid audio", async () => { + const promise = transcribeVoiceWithChatGptSession({ + request: { ...baseRequest, mimeType: "audio/webm" }, + resolveAuth: async () => ({ token: chatGptToken() }), + fetchImpl: vi.fn() as unknown as typeof fetch, + }); + + const error = await promise.catch((cause: unknown) => cause); + expect(error).toBeInstanceOf(VoiceTranscriptionBackendError); + expect(error).toMatchObject({ kind: "invalid-audio", fallbackAllowed: false }); }); }); diff --git a/apps/server/src/voiceTranscription.ts b/apps/server/src/voiceTranscription.ts index 6c44f3088..31ed460c2 100644 --- a/apps/server/src/voiceTranscription.ts +++ b/apps/server/src/voiceTranscription.ts @@ -1,167 +1,7 @@ -// FILE: voiceTranscription.ts -// Purpose: Validates Remodex-style WAV payloads and proxies them to ChatGPT transcription. -// Layer: Server utility -// Exports: transcribeVoiceWithChatGptSession -// Depends on: ChatGPT session auth supplied by Codex app-server callers. - -import { Buffer } from "node:buffer"; - -import type { - ServerVoiceTranscriptionInput, - ServerVoiceTranscriptionResult, -} from "@synara/contracts"; - -const CHATGPT_TRANSCRIPTIONS_URL = "https://chatgpt.com/backend-api/transcribe"; -const MAX_AUDIO_BYTES = 10 * 1024 * 1024; -const MAX_DURATION_MS = 120_000; - -export interface ChatGptVoiceAuthContext { - readonly token: string; - readonly transcriptionUrl?: string; -} - -// Validate the captured WAV clip and retry once if the ChatGPT session needs a refresh. -export async function transcribeVoiceWithChatGptSession(input: { - readonly request: ServerVoiceTranscriptionInput; - readonly resolveAuth: (refreshToken: boolean) => Promise; - readonly fetchImpl?: typeof fetch; -}): Promise { - const fetchImpl = input.fetchImpl ?? globalThis.fetch; - if (typeof fetchImpl !== "function") { - throw new Error("Voice transcription is unavailable in this runtime."); - } - - const audioBuffer = decodeVoiceAudio(input.request); - let auth = await input.resolveAuth(false); - let response = await requestTranscription({ - fetchImpl, - audioBuffer, - mimeType: input.request.mimeType, - token: auth.token, - ...(auth.transcriptionUrl ? { transcriptionUrl: auth.transcriptionUrl } : {}), - }); - - if (response.status === 401 || response.status === 403) { - auth = await input.resolveAuth(true); - response = await requestTranscription({ - fetchImpl, - audioBuffer, - mimeType: input.request.mimeType, - token: auth.token, - ...(auth.transcriptionUrl ? { transcriptionUrl: auth.transcriptionUrl } : {}), - }); - } - - if (!response.ok) { - throw new Error(await readTranscriptionErrorMessage(response)); - } - - const payload = (await response.json().catch(() => null)) as { - text?: unknown; - transcript?: unknown; - } | null; - const text = readString(payload?.text) ?? readString(payload?.transcript); - if (!text) { - throw new Error("The transcription response did not include any text."); - } - - return { text }; -} - -// Keep the server-side contract strict so the private backend only sees normalized clips. -function decodeVoiceAudio(input: ServerVoiceTranscriptionInput): Buffer { - if (input.mimeType !== "audio/wav") { - throw new Error("Only WAV audio is supported for voice transcription."); - } - if (input.sampleRateHz !== 24_000) { - throw new Error("Voice transcription requires 24 kHz mono WAV audio."); - } - if (input.durationMs <= 0) { - throw new Error("Voice messages must include a positive duration."); - } - if (input.durationMs > MAX_DURATION_MS) { - throw new Error("Voice messages are limited to 120 seconds."); - } - - const normalizedBase64 = normalizeBase64(input.audioBase64); - if (!normalizedBase64 || !isLikelyBase64(normalizedBase64)) { - throw new Error("The recorded audio could not be decoded."); - } - - const audioBuffer = Buffer.from(normalizedBase64, "base64"); - if (!audioBuffer.length || audioBuffer.toString("base64") !== normalizedBase64) { - throw new Error("The recorded audio could not be decoded."); - } - if (audioBuffer.length > MAX_AUDIO_BYTES) { - throw new Error("Voice messages are limited to 10 MB."); - } - if (!isLikelyWavBuffer(audioBuffer)) { - throw new Error("The recorded audio is not a valid WAV file."); - } - - return audioBuffer; -} - -async function requestTranscription(input: { - readonly fetchImpl: typeof fetch; - readonly audioBuffer: Buffer; - readonly mimeType: string; - readonly token: string; - readonly transcriptionUrl?: string; -}): Promise { - const formData = new FormData(); - formData.append("file", new Blob([input.audioBuffer], { type: input.mimeType }), "voice.wav"); - - return input.fetchImpl(input.transcriptionUrl ?? CHATGPT_TRANSCRIPTIONS_URL, { - method: "POST", - headers: { - Authorization: `Bearer ${input.token}`, - }, - body: formData, - }); -} - -async function readTranscriptionErrorMessage(response: Response): Promise { - let errorMessage = `Transcription failed with status ${response.status}.`; - try { - const payload = (await response.json()) as { - error?: { message?: unknown }; - message?: unknown; - } | null; - const providerMessage = - readString(payload?.error?.message) ?? readString(payload?.message) ?? null; - if (providerMessage) { - errorMessage = providerMessage; - } - } catch { - // Keep the generic status-based message when the provider body is empty or invalid. - } - - if (response.status === 401 || response.status === 403) { - return "Your ChatGPT login has expired. Sign in again."; - } - - return errorMessage; -} - -function normalizeBase64(value: string): string | null { - const normalized = value.trim().replace(/\s+/g, ""); - return normalized.length > 0 ? normalized : null; -} - -function isLikelyBase64(value: string): boolean { - return /^[A-Za-z0-9+/]+={0,2}$/.test(value); -} - -function isLikelyWavBuffer(buffer: Buffer): boolean { - return ( - buffer.length >= 12 && - buffer.toString("ascii", 0, 4) === "RIFF" && - buffer.toString("ascii", 8, 12) === "WAVE" - ); -} - -function readString(value: unknown): string | null { - const normalized = typeof value === "string" ? value.trim() : ""; - return normalized.length > 0 ? normalized : null; -} +// Compatibility facade. Application-level routing lives outside the ChatGPT adapter. +export { + CHATGPT_TRANSCRIPTIONS_URL, + createChatGptVoiceTranscriptionBackend, + transcribeVoiceWithChatGptSession, + type ChatGptVoiceAuthContext, +} from "./voice/chatGptVoiceTranscription.ts"; diff --git a/apps/server/src/workspaceEntries.ts b/apps/server/src/workspaceEntries.ts index ab58a0661..f13c18eec 100644 --- a/apps/server/src/workspaceEntries.ts +++ b/apps/server/src/workspaceEntries.ts @@ -228,7 +228,7 @@ function isPathInIgnoredDirectory(relativePath: string): boolean { return IGNORED_DIRECTORY_NAMES.has(firstSegment); } -type ProjectPackageManager = "bun" | "pnpm" | "yarn" | "npm"; +export type ProjectPackageManager = "bun" | "pnpm" | "yarn" | "npm"; const PROJECT_PACKAGE_MANAGER_LOCKFILES: ReadonlyArray<{ readonly manager: ProjectPackageManager; @@ -254,7 +254,9 @@ async function pathExists(absolutePath: string): Promise { } } -async function detectPackageManager(packageDir: string): Promise { +export async function detectProjectPackageManager( + packageDir: string, +): Promise { for (const candidate of PROJECT_PACKAGE_MANAGER_LOCKFILES) { for (const filename of candidate.filenames) { if (await pathExists(path.join(packageDir, filename))) { @@ -265,7 +267,10 @@ async function detectPackageManager(packageDir: string): Promise typeof command === "string" && name.trim().length > 0 && command.trim().length > 0 ? [ { name: name.trim(), - command: commandForPackageScript(manager, name.trim()), + command: commandForProjectPackageScript(manager, name.trim()), }, ] : [], diff --git a/apps/server/src/wsRpc.ts b/apps/server/src/wsRpc.ts index 0b421699c..41be16c22 100644 --- a/apps/server/src/wsRpc.ts +++ b/apps/server/src/wsRpc.ts @@ -43,9 +43,10 @@ import { GitManager } from "./git/Services/GitManager"; import { GitHubCliError } from "./git/Errors"; import { GitStatusBroadcaster } from "./git/Services/GitStatusBroadcaster"; import { TextGeneration } from "./git/Services/TextGeneration"; +import { HtmlArtifactPreview } from "./htmlPreview/Services/HtmlArtifactPreview"; import { Keybindings } from "./keybindings"; import { createLocalPreviewGrant } from "./localImageFiles"; -import { listLocalServers, stopLocalServer } from "./localServerMonitor"; +import { listLocalServers } from "./localServerMonitor"; import { Open, resolveAvailableEditors } from "./open"; import { makeDispatchCommandNormalizer } from "./orchestration/dispatchCommandNormalization"; import { makeImportThreadHandler } from "./orchestration/importThreadRoute"; @@ -267,6 +268,7 @@ export const makeWsRpcLayer = () => const git = yield* GitCore; const gitManager = yield* GitManager; const gitStatusBroadcaster = yield* GitStatusBroadcaster; + const htmlArtifactPreview = yield* HtmlArtifactPreview; const keybindings = yield* Keybindings; const open = yield* Open; const orchestrationEngine = yield* OrchestrationEngineService; @@ -463,20 +465,52 @@ export const makeWsRpcLayer = () => localServerSnapshot.servers.find( (server) => server.pid === input.pid && server.ports.includes(input.port), ) ?? null; - const result = yield* Effect.promise(() => stopLocalServer(input, localServer)); - if (localServer?.isStoppable) { - const devServers = yield* devServerManager.list; - const trackedServer = findProjectDevServerForLocalServer({ - localServer, - devServers: devServers.servers, - }); - if (trackedServer) { - yield* devServerManager - .stop({ projectId: trackedServer.projectId }) - .pipe(Effect.catch(() => Effect.void)); - } + const devServers = yield* devServerManager.list; + const trackedServer = localServer + ? findProjectDevServerForLocalServer({ + localServer, + devServers: devServers.servers, + }) + : null; + if (!localServer || !trackedServer) { + return { + pid: input.pid, + stopped: false, + message: + "Scient only stops development servers that it launched and can identify safely.", + }; } - return result; + const stopped = yield* devServerManager.stop({ projectId: trackedServer.projectId }); + return { + pid: input.pid, + stopped: stopped.stopped, + message: stopped.stopped + ? "Development server stopped." + : "That development server is no longer running.", + }; + }); + + const listLocalServersWithStopOwnership = Effect.gen(function* () { + const snapshot = yield* Effect.promise(() => listLocalServers()); + const devServers = yield* devServerManager.list; + return { + ...snapshot, + servers: snapshot.servers.map((server) => { + const tracked = findProjectDevServerForLocalServer({ + localServer: server, + devServers: devServers.servers, + }); + if (tracked) { + const ownedServer = Object.assign({}, server, { isStoppable: true }); + Reflect.deleteProperty(ownedServer, "stopDisabledReason"); + return ownedServer; + } + return Object.assign({}, server, { + isStoppable: false, + stopDisabledReason: "Only Scient-managed servers can be stopped here.", + }); + }), + }; }); const loadServerConfig = Effect.gen(function* () { @@ -693,6 +727,12 @@ export const makeWsRpcLayer = () => Effect.promise(() => createLocalPreviewGrant({ requestedPath: input.path })), "Failed to create local file preview grant", ), + [WS_METHODS.projectsInspectHtmlArtifact]: (input) => + rpcEffect(htmlArtifactPreview.inspect(input), "Failed to inspect HTML artifact"), + [WS_METHODS.projectsPrepareHtmlArtifactPreview]: (input) => + rpcEffect(htmlArtifactPreview.prepare(input), "Failed to prepare HTML artifact preview"), + [WS_METHODS.projectsRevokeHtmlArtifactPreview]: (input) => + rpcEffect(htmlArtifactPreview.revoke(input), "Failed to revoke HTML artifact preview"), [WS_METHODS.projectsWriteFile]: (input) => rpcEffect(workspaceFileSystem.writeFile(input), "Failed to write workspace file"), [WS_METHODS.scientProjectInitializationPreview]: (input) => @@ -797,6 +837,13 @@ export const makeWsRpcLayer = () => ), [WS_METHODS.filesystemBrowse]: (input) => rpcEffect(workspaceEntries.browse(input), "Failed to browse filesystem"), + [WS_METHODS.filesystemCreateDirectory]: (input) => + rpcEffect( + canonicalizeProjectWorkspaceRoot(input.path, { createIfMissing: true }).pipe( + Effect.map((createdPath) => ({ path: createdPath })), + ), + "Failed to create directory", + ), [WS_METHODS.shellOpenInEditor]: (input) => rpcEffect(open.openInEditor(input), "Failed to open editor"), @@ -1125,10 +1172,7 @@ export const makeWsRpcLayer = () => ), [WS_METHODS.serverListWorktrees]: () => Effect.succeed({ worktrees: [] }), [WS_METHODS.serverListLocalServers]: () => - rpcEffect( - Effect.promise(() => listLocalServers()), - "Failed to list local servers", - ), + rpcEffect(listLocalServersWithStopOwnership, "Failed to list local servers"), [WS_METHODS.serverStopLocalServer]: (input) => rpcEffect(stopLocalServerAndTrackedProjectRun(input), "Failed to stop local server"), [WS_METHODS.statsGetProfileStats]: (input) => @@ -1178,19 +1222,23 @@ export const makeWsRpcLayer = () => ), [WS_METHODS.serverTranscribeVoice]: (input) => rpcEffect( - providerAdapterRegistry - .getByProvider(input.provider) - .pipe( - Effect.flatMap((adapter) => - adapter.transcribeVoice - ? adapter.transcribeVoice(input) - : Effect.fail( - new Error( - `Voice transcription is unavailable for provider '${input.provider}'.`, - ), - ), + input.mode === "offline-only" + ? Effect.fail( + new Error("Offline voice transcription is available in the Scient desktop app."), + ) + : providerAdapterRegistry.getByProvider("codex").pipe( + Effect.flatMap((adapter) => + adapter.transcribeVoice + ? adapter.transcribeVoice(input).pipe( + Effect.map((result) => ({ + ...result, + engine: "chatgpt" as const, + fallbackUsed: false, + })), + ) + : Effect.fail(new Error("Voice transcription is unavailable right now.")), + ), ), - ), "Voice transcription failed", ), [WS_METHODS.serverGenerateThreadRecap]: (input) => diff --git a/apps/web/src/appSettings.test.ts b/apps/web/src/appSettings.test.ts index f0c1145e8..93350e894 100644 --- a/apps/web/src/appSettings.test.ts +++ b/apps/web/src/appSettings.test.ts @@ -9,6 +9,7 @@ import { describe, expect, it } from "vitest"; import { AppSettingsSchema, CUSTOM_MODEL_EDITOR_PROVIDER_SETTINGS, + CURRENT_APP_SETTINGS_VERSION, DEFAULT_CHAT_FONT_SIZE_PX, DEFAULT_SIDEBAR_PROJECT_SORT_ORDER, DEFAULT_TERMINAL_FONT_SIZE_PX, @@ -308,6 +309,50 @@ describe("chat font size defaults", () => { }); }); +describe("app settings default migration", () => { + it("starts new profiles with Studio hidden and the 15px base font", () => { + expect(AppSettingsSchema.makeUnsafe({})).toMatchObject({ + appSettingsVersion: CURRENT_APP_SETTINGS_VERSION, + chatFontSizePx: 15, + showStudioSection: false, + showWorkspaceSection: false, + }); + }); + + it("overrides existing profile preferences once", () => { + const decode = Schema.decodeSync(Schema.fromJsonString(AppSettingsSchema)); + const existingSettings = decode( + JSON.stringify({ + chatFontSizePx: 18, + showStudioSection: true, + }), + ); + + expect(existingSettings.appSettingsVersion).toBe(0); + expect(normalizeStoredAppSettings(existingSettings)).toMatchObject({ + appSettingsVersion: CURRENT_APP_SETTINGS_VERSION, + chatFontSizePx: 15, + showStudioSection: false, + }); + }); + + it("preserves preferences changed after the one-time migration", () => { + const codec = Schema.fromJsonString(AppSettingsSchema); + const currentSettings = AppSettingsSchema.makeUnsafe({ + appSettingsVersion: CURRENT_APP_SETTINGS_VERSION, + chatFontSizePx: 13, + showStudioSection: true, + }); + const reloadedSettings = Schema.decodeSync(codec)(Schema.encodeSync(codec)(currentSettings)); + + expect(normalizeStoredAppSettings(reloadedSettings)).toMatchObject({ + appSettingsVersion: CURRENT_APP_SETTINGS_VERSION, + chatFontSizePx: 13, + showStudioSection: true, + }); + }); +}); + describe("terminal font size defaults", () => { it("defaults terminal font size to 12px", () => { expect(DEFAULT_TERMINAL_FONT_SIZE_PX).toBe(12); @@ -372,6 +417,7 @@ describe("normalizeStoredAppSettings", () => { it("preserves an explicitly stored updated_at project sort order", () => { const decodedSettings = Schema.decodeSync(Schema.fromJsonString(AppSettingsSchema))( JSON.stringify({ + appSettingsVersion: CURRENT_APP_SETTINGS_VERSION, sidebarProjectSortOrder: "updated_at", chatFontSizePx: 99, terminalFontSizePx: 3, @@ -804,6 +850,7 @@ describe("AppSettingsSchema", () => { ), ).toMatchObject({ claudeBinaryPath: "", + appSettingsVersion: 0, uiDensity: "comfortable", chatFontSizePx: DEFAULT_CHAT_FONT_SIZE_PX, codexBinaryPath: "/usr/local/bin/codex", @@ -815,9 +862,10 @@ describe("AppSettingsSchema", () => { enableAppSnap: false, appSnapPlaySound: true, enableAssistantStreaming: true, + voiceTranscriptionMode: "automatic", sidebarProjectSortOrder: DEFAULT_SIDEBAR_PROJECT_SORT_ORDER, sidebarThreadSortOrder: DEFAULT_SIDEBAR_THREAD_SORT_ORDER, - showStudioSection: true, + showStudioSection: false, timestampFormat: DEFAULT_TIMESTAMP_FORMAT, customCodexModels: [], customClaudeModels: [], @@ -830,6 +878,13 @@ describe("AppSettingsSchema", () => { }); }); + it("preserves an explicit offline-only voice preference", () => { + const decode = Schema.decodeSync(Schema.fromJsonString(AppSettingsSchema)); + expect(decode(JSON.stringify({ voiceTranscriptionMode: "offline-only" }))).toMatchObject({ + voiceTranscriptionMode: "offline-only", + }); + }); + it("migrates the former AppSnap feature flag", () => { const decode = Schema.decodeSync(Schema.fromJsonString(AppSettingsSchema)); diff --git a/apps/web/src/appSettings.ts b/apps/web/src/appSettings.ts index 84de9e4fd..01c9de954 100644 --- a/apps/web/src/appSettings.ts +++ b/apps/web/src/appSettings.ts @@ -45,6 +45,7 @@ const APP_SETTINGS_STORAGE_KEY = "scient:app-settings:v1"; const SERVER_SETTINGS_MIGRATION_STORAGE_KEY = "scient:server-settings-migrated:v1"; const MAX_CUSTOM_MODEL_COUNT = 32; export const MAX_CUSTOM_MODEL_LENGTH = 256; +export const CURRENT_APP_SETTINGS_VERSION = 1; export const MIN_CHAT_FONT_SIZE_PX = 11; export const MAX_CHAT_FONT_SIZE_PX = 18; export const DEFAULT_CHAT_FONT_SIZE_PX = 15; @@ -86,6 +87,9 @@ export const UiDensity = Schema.Literals(UI_DENSITY_MODES); export type UiDensity = typeof UiDensity.Type; export { DEFAULT_UI_DENSITY }; +export const VoiceTranscriptionMode = Schema.Literals(["automatic", "offline-only"]); +export type VoiceTranscriptionMode = typeof VoiceTranscriptionMode.Type; + export function getDefaultNativeFontSmoothing(platform = globalThis.navigator?.platform ?? "") { return /mac|iphone|ipad|ipod/i.test(platform); } @@ -157,6 +161,10 @@ const PersistedProviderKind = Schema.Literals([ ); export const AppSettingsSchema = Schema.Struct({ + appSettingsVersion: Schema.Number.pipe( + Schema.withConstructorDefault(() => Option.some(CURRENT_APP_SETTINGS_VERSION)), + Schema.withDecodingDefault(() => 0), + ), telemetryPrivacyLevel: TelemetryPrivacyLevel.pipe(withDefaults(() => "essential" as const)), claudeBinaryPath: Schema.String.check(Schema.isMaxLength(4096)).pipe(withDefaults(() => "")), uiDensity: UiDensity.pipe(withDefaults(() => DEFAULT_UI_DENSITY)), @@ -199,7 +207,7 @@ export const AppSettingsSchema = Schema.Struct({ // (rootless chats not tied to a project). `showStudioSection` and // `showWorkspaceSection` control optional tabs in the section switcher. showChatsSection: Schema.Boolean.pipe(withDefaults(() => true)), - showStudioSection: Schema.Boolean.pipe(withDefaults(() => true)), + showStudioSection: Schema.Boolean.pipe(withDefaults(() => false)), showWorkspaceSection: Schema.Boolean.pipe(withDefaults(() => false)), // Local-only UI preferences: which optional sections of the chat Environment panel are // shown. The git block (Changes/Worktree/branch/Commit and Push) is always visible; these @@ -221,6 +229,9 @@ export const AppSettingsSchema = Schema.Struct({ enableNativeFontSmoothing: Schema.Boolean.pipe(withDefaults(getDefaultNativeFontSmoothing)), enableTaskCompletionToasts: Schema.Boolean.pipe(withDefaults(() => true)), enableSystemTaskCompletionNotifications: Schema.Boolean.pipe(withDefaults(() => true)), + // Local UI preference. Automatic prefers an eligible ChatGPT subscription + // and always falls back to the verified on-device model. + voiceTranscriptionMode: VoiceTranscriptionMode.pipe(withDefaults(() => "automatic" as const)), // Local desktop preference. Native capability/permission state remains owned by Electron. // AppSnap is opt-in because enabling its Settings toggle requests macOS // Input Monitoring and Screen Recording permissions. @@ -785,7 +796,20 @@ function buildInitialServerSettingsMigrationPatch(settings: AppSettings): Server } export function normalizeStoredAppSettings(settings: AppSettings): AppSettings { - return normalizeAppSettings(settings); + const normalizedSettings = normalizeAppSettings(settings); + if (normalizedSettings.appSettingsVersion >= CURRENT_APP_SETTINGS_VERSION) { + return normalizedSettings; + } + + // v1 intentionally applies the new product defaults once to every existing local profile, + // including profiles with explicit values. Persisting the version alongside the settings + // ensures later user changes remain untouched. + return normalizeAppSettings({ + ...normalizedSettings, + appSettingsVersion: CURRENT_APP_SETTINGS_VERSION, + chatFontSizePx: DEFAULT_CHAT_FONT_SIZE_PX, + showStudioSection: false, + }); } export function getCustomModelsForProvider( @@ -1166,13 +1190,18 @@ export function useAppSettings() { [], ); + const migratedLocalSettings = useMemo( + () => normalizeStoredAppSettings(localSettings), + [localSettings], + ); + const settings = useMemo( () => normalizeAppSettings({ - ...localSettings, + ...migratedLocalSettings, ...(serverSettingsQuery.data ? serverSettingsToAppSettings(serverSettingsQuery.data) : {}), }), - [localSettings, serverSettingsQuery.data], + [migratedLocalSettings, serverSettingsQuery.data], ); useEffect(() => { @@ -1215,7 +1244,9 @@ export function useAppSettings() { const updateSettings = useCallback( (patch: Partial) => { - setSettings((prev) => normalizeAppSettings({ ...prev, ...patch })); + setSettings((previous) => + normalizeAppSettings({ ...normalizeStoredAppSettings(previous), ...patch }), + ); if (touchesProviderDiscoverySettings(patch)) { void queryClient.invalidateQueries({ queryKey: providerDiscoveryQueryKeys.all }); } diff --git a/apps/web/src/appSnap.logic.test.ts b/apps/web/src/appSnap.logic.test.ts index 75225fee0..84cbc4acf 100644 --- a/apps/web/src/appSnap.logic.test.ts +++ b/apps/web/src/appSnap.logic.test.ts @@ -1,3 +1,5 @@ +import { readFileSync } from "node:fs"; + import { ThreadId } from "@synara/contracts"; import { describe, expect, it } from "vitest"; @@ -24,6 +26,31 @@ describe("createLatestAppSnapRequestGuard", () => { expect(guard.isCurrent(enableRequest)).toBe(false); expect(guard.isCurrent(disableRequest)).toBe(true); }); + + it("keeps only the newest result current across repeated preference changes", () => { + const guard = createLatestAppSnapRequestGuard(); + const first = guard.begin(); + const second = guard.begin(); + const third = guard.begin(); + + expect(guard.isCurrent(first)).toBe(false); + expect(guard.isCurrent(second)).toBe(false); + expect(guard.isCurrent(third)).toBe(true); + }); + + it("guards coordinator preference results and dismisses recovery alerts when actions start", () => { + const source = readFileSync( + new URL("./components/AppSnapCoordinator.tsx", import.meta.url), + "utf8", + ); + + expect(source).toContain("appSnapPreferenceRequestGuardRef.current.begin()"); + expect( + source.match(/appSnapPreferenceRequestGuardRef\.current\.isCurrent\(requestId\)/g), + ).toHaveLength(2); + expect(source.match(/transientAlertManager\.close\(alertId\)/g)).toHaveLength(2); + expect(source).toContain('title: "Restarting AppSnap"'); + }); }); describe("didAppSnapHydrationInputsChange", () => { diff --git a/apps/web/src/browserStateStore.test.ts b/apps/web/src/browserStateStore.test.ts index cfb9689bb..e701de692 100644 --- a/apps/web/src/browserStateStore.test.ts +++ b/apps/web/src/browserStateStore.test.ts @@ -2,6 +2,7 @@ import { ThreadId } from "@synara/contracts"; import { describe, expect, it } from "vitest"; import { + browserHistoryAfterThreadState, createDedupedBrowserStateStorage, sanitizeRecentHistoryByThreadId, selectThreadBrowserHistory, @@ -25,6 +26,34 @@ describe("browserStateStore selectors", () => { expect(first).toBe(second); expect(first).toEqual([]); }); + + it("never persists artifact capability URLs into browser history", () => { + const history = browserHistoryAfterThreadState([], { + threadId: THREAD_ID, + version: 1, + open: true, + activeTabId: "artifact", + tabs: [ + { + id: "artifact", + kind: "artifact", + url: "http://g-secret.preview.localhost:5000/", + displayUrl: "/workspace/report.html", + title: "Report", + status: "live", + isLoading: false, + canGoBack: false, + canGoForward: false, + faviconUrl: null, + lastCommittedUrl: "http://g-secret.preview.localhost:5000/", + lastError: null, + }, + ], + lastError: null, + }); + + expect(history).toEqual([]); + }); }); describe("createDedupedBrowserStateStorage", () => { diff --git a/apps/web/src/browserStateStore.ts b/apps/web/src/browserStateStore.ts index 9f80dbd1c..1047ded35 100644 --- a/apps/web/src/browserStateStore.ts +++ b/apps/web/src/browserStateStore.ts @@ -82,6 +82,27 @@ function sameBrowserHistoryEntries( }); } +export function browserHistoryAfterThreadState( + previousHistory: BrowserHistoryEntry[], + state: ThreadBrowserState, +): BrowserHistoryEntry[] { + const activeTab = state.tabs.find((tab) => tab.id === state.activeTabId) ?? null; + const orderedTabs = activeTab + ? [activeTab, ...state.tabs.filter((tab) => tab.id !== activeTab.id)] + : state.tabs; + return orderedTabs.reduce( + (entries, tab) => + tab.kind === "artifact" + ? entries + : upsertRecentHistoryEntry(entries, { + url: tab.lastCommittedUrl ?? tab.url, + title: tab.title, + tabId: tab.id, + }), + previousHistory, + ); +} + function sanitizeBrowserHistoryEntry(rawEntry: unknown): BrowserHistoryEntry | null { if (!isPlainObject(rawEntry)) { return null; @@ -148,21 +169,9 @@ export const useBrowserStateStore = create()( if (previousState?.version === state.version) { return current; } - const activeTab = state.tabs.find((tab) => tab.id === state.activeTabId) ?? null; - const orderedTabs = activeTab - ? [activeTab, ...state.tabs.filter((tab) => tab.id !== activeTab.id)] - : state.tabs; const previousHistory = current.recentHistoryByThreadId[state.threadId] ?? EMPTY_BROWSER_HISTORY; - const nextHistory = orderedTabs.reduce( - (entries, tab) => - upsertRecentHistoryEntry(entries, { - url: tab.lastCommittedUrl ?? tab.url, - title: tab.title, - tabId: tab.id, - }), - previousHistory, - ); + const nextHistory = browserHistoryAfterThreadState(previousHistory, state); const historyChanged = !sameBrowserHistoryEntries(previousHistory, nextHistory); return { diff --git a/apps/web/src/components/AddProjectDialog.browser.tsx b/apps/web/src/components/AddProjectDialog.browser.tsx index b5bd450cd..387ba78c4 100644 --- a/apps/web/src/components/AddProjectDialog.browser.tsx +++ b/apps/web/src/components/AddProjectDialog.browser.tsx @@ -5,7 +5,7 @@ import "../index.css"; import type { NativeApi } from "@synara/contracts"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { page } from "vitest/browser"; +import { page, userEvent } from "vitest/browser"; import { afterEach, describe, expect, it, vi } from "vitest"; import { render } from "vitest-browser-react"; @@ -16,6 +16,7 @@ function installNativeApi(overrides: { statuses: NativeApi["projects"]["repositorySourceStatuses"]; cloneSource?: NativeApi["projects"]["cloneSource"]; browse?: NativeApi["filesystem"]["browse"]; + createDirectory?: NativeApi["filesystem"]["createDirectory"]; }) { const previousNativeApi = window.nativeApi; const baseApi = readNativeApi(); @@ -34,6 +35,8 @@ function installNativeApi(overrides: { browse: overrides.browse ?? vi.fn().mockResolvedValue({ parentPath: "/Users/tester", entries: [] }), + createDirectory: + overrides.createDirectory ?? vi.fn(async ({ path }: { path: string }) => ({ path })), }, dialogs: { ...baseApi.dialogs, @@ -49,7 +52,10 @@ function installNativeApi(overrides: { }; } -function renderDialog(onAddProjectPath = vi.fn().mockResolvedValue(true)) { +function renderDialog( + onAddProjectPath = vi.fn().mockResolvedValue(true), + options: { homeDir?: string; defaultCloneDirectory?: string } = {}, +) { const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); render( @@ -57,8 +63,8 @@ function renderDialog(onAddProjectPath = vi.fn().mockResolvedValue(true)) { open onOpenChange={() => undefined} onAddProjectPath={onAddProjectPath} - homeDir="/Users/tester" - defaultCloneDirectory="/Users/tester" + homeDir={options.homeDir ?? "/Users/tester"} + defaultCloneDirectory={options.defaultCloneDirectory ?? "/Users/tester"} /> , ); @@ -165,7 +171,7 @@ describe("AddProjectDialog", () => { await vi.waitFor(() => expect(browse).toHaveBeenCalledWith({ partialPath: "/Users/tester/Research /notes/" }), ); - await page.getByRole("button", { name: /Add/ }).click(); + await page.getByRole("button", { name: "Open Enter", exact: true }).click(); expect(onAddProjectPath).toHaveBeenCalledWith("/Users/tester/Research /notes", { createIfMissing: false, @@ -173,6 +179,216 @@ describe("AddProjectDialog", () => { restore(); }); + it.each([ + { + platform: "MacIntel", + homeDir: "/Users/tester", + rootQuery: "/Users/tester/", + parentPath: "/Users/tester/Parent", + parentQuery: "/Users/tester/Parent/", + childPath: "/Users/tester/Parent/Child", + }, + { + platform: "Linux x86_64", + homeDir: "/home/tester", + rootQuery: "/home/tester/", + parentPath: "/home/tester/Parent", + parentQuery: "/home/tester/Parent/", + childPath: "/home/tester/Parent/Child", + }, + { + platform: "Win32", + homeDir: "C:\\Users\\tester", + rootQuery: "C:\\Users\\tester\\", + parentPath: "C:\\Users\\tester\\Parent", + parentQuery: "C:\\Users\\tester\\Parent\\", + childPath: "C:\\Users\\tester\\Parent\\Child", + }, + ])( + "opens the current non-leaf folder with Enter on $platform", + async ({ platform, homeDir, rootQuery, parentPath, parentQuery, childPath }) => { + vi.spyOn(window.navigator, "platform", "get").mockReturnValue(platform); + const browse = vi.fn(async ({ partialPath }: { partialPath: string }) => { + if (partialPath === rootQuery) { + return { + parentPath: homeDir, + entries: [{ name: "Parent", fullPath: parentPath }], + }; + } + if (partialPath === parentQuery) { + return { + parentPath, + entries: [{ name: "Child", fullPath: childPath }], + }; + } + throw new Error(`Unexpected browse path: ${partialPath}`); + }); + const onAddProjectPath = vi.fn().mockResolvedValue(false); + const restore = installNativeApi({ + statuses: vi.fn().mockResolvedValue({ sources: [] }), + browse, + }); + renderDialog(onAddProjectPath, { homeDir, defaultCloneDirectory: homeDir }); + + await page.getByText("Local folder", { exact: true }).click(); + await page.getByText("Parent", { exact: true }).click(); + const pathInput = page.getByPlaceholder("Type or browse a folder path"); + await expect.element(pathInput).toHaveValue(parentQuery); + await expect.element(page.getByText("Child", { exact: true })).toBeVisible(); + await expect + .element(page.getByRole("button", { name: "Open Enter", exact: true })) + .toBeVisible(); + + await userEvent.keyboard("{Enter}"); + + await vi.waitFor(() => + expect(onAddProjectPath).toHaveBeenCalledWith(parentPath, { createIfMissing: false }), + ); + expect(onAddProjectPath).toHaveBeenCalledTimes(1); + await expect.element(pathInput).toHaveValue(parentQuery); + restore(); + }, + ); + + it("opens a displayed folder only once when Enter is pressed twice quickly", async () => { + let finishOpen!: (shouldClose: boolean) => void; + const onAddProjectPath = vi.fn( + () => + new Promise((resolve) => { + finishOpen = resolve; + }), + ); + const restore = installNativeApi({ + statuses: vi.fn().mockResolvedValue({ sources: [] }), + }); + renderDialog(onAddProjectPath); + + await page.getByText("Local folder", { exact: true }).click(); + await userEvent.keyboard("{Enter}{Enter}"); + + await vi.waitFor(() => expect(onAddProjectPath).toHaveBeenCalledTimes(1)); + finishOpen(false); + restore(); + }); + + it("uses ArrowRight to browse highlighted folders and parent entries", async () => { + const browse = vi.fn(async ({ partialPath }: { partialPath: string }) => { + if (partialPath === "/Users/tester/") { + return { + parentPath: "/Users/tester", + entries: [{ name: "Parent", fullPath: "/Users/tester/Parent" }], + }; + } + if (partialPath === "/Users/tester/Parent/") { + return { + parentPath: "/Users/tester/Parent", + entries: [{ name: "Child", fullPath: "/Users/tester/Parent/Child" }], + }; + } + throw new Error(`Unexpected browse path: ${partialPath}`); + }); + const onAddProjectPath = vi.fn().mockResolvedValue(false); + const restore = installNativeApi({ + statuses: vi.fn().mockResolvedValue({ sources: [] }), + browse, + }); + renderDialog(onAddProjectPath); + + await page.getByText("Local folder", { exact: true }).click(); + const pathInput = page.getByPlaceholder("Type or browse a folder path"); + await userEvent.keyboard("{ArrowDown}{ArrowDown}{ArrowRight}"); + await expect.element(pathInput).toHaveValue("/Users/tester/Parent/"); + + await userEvent.keyboard("{ArrowDown}{ArrowRight}"); + await expect.element(pathInput).toHaveValue("/Users/tester/"); + expect(onAddProjectPath).not.toHaveBeenCalled(); + restore(); + }); + + it("steps the back arrow through parent folders before returning to sources", async () => { + const browse = vi.fn(async ({ partialPath }: { partialPath: string }) => { + if (partialPath === "/Users/tester/") { + return { + parentPath: "/Users/tester", + entries: [{ name: "Parent", fullPath: "/Users/tester/Parent" }], + }; + } + if (partialPath === "/Users/tester/Parent/") { + return { + parentPath: "/Users/tester/Parent", + entries: [{ name: "Child", fullPath: "/Users/tester/Parent/Child" }], + }; + } + throw new Error(`Unexpected browse path: ${partialPath}`); + }); + const restore = installNativeApi({ + statuses: vi.fn().mockResolvedValue({ sources: [] }), + browse, + }); + renderDialog(); + + await page.getByText("Local folder", { exact: true }).click(); + await page.getByText("Parent", { exact: true }).click(); + const pathInput = page.getByPlaceholder("Type or browse a folder path"); + await expect.element(pathInput).toHaveValue("/Users/tester/Parent/"); + + await page.getByRole("button", { name: "Parent folder", exact: true }).click(); + await expect.element(pathInput).toHaveValue("/Users/tester/"); + await expect.element(page.getByText("Parent", { exact: true })).toBeVisible(); + + await page.getByRole("button", { name: "Back", exact: true }).click(); + await expect.element(page.getByText("Git URL", { exact: true })).toBeVisible(); + restore(); + }); + + it("offers a neutral new-folder action and creates a collision-safe child", async () => { + const browse = vi.fn().mockResolvedValue({ + parentPath: "/Users/tester", + entries: [ + { name: "New folder", fullPath: "/Users/tester/New folder" }, + { name: "New folder 2", fullPath: "/Users/tester/New folder 2" }, + ], + }); + const onAddProjectPath = vi.fn().mockResolvedValue(false); + const createDirectory = vi.fn(async ({ path }: { path: string }) => ({ path })); + const restore = installNativeApi({ + statuses: vi.fn().mockResolvedValue({ sources: [] }), + browse, + createDirectory, + }); + renderDialog(onAddProjectPath); + + await page.getByText("Local folder", { exact: true }).click(); + await page.getByRole("button", { name: "New folder", exact: true }).click(); + + const pathInput = page.getByPlaceholder("Type or browse a folder path"); + await expect.element(pathInput).toHaveValue("/Users/tester/New folder 3"); + await vi.waitFor(() => { + const input = document.activeElement as HTMLInputElement | null; + expect(input?.value.slice(input.selectionStart ?? 0, input.selectionEnd ?? 0)).toBe( + "New folder 3", + ); + }); + await expect + .element(page.getByRole("button", { name: "Create & Open Enter", exact: true })) + .toBeVisible(); + + await userEvent.keyboard("{Enter}"); + + await vi.waitFor(() => + expect(createDirectory).toHaveBeenCalledWith({ + path: "/Users/tester/New folder 3", + }), + ); + await vi.waitFor(() => + expect(onAddProjectPath).toHaveBeenCalledWith("/Users/tester/New folder 3", { + createIfMissing: true, + }), + ); + expect(onAddProjectPath).toHaveBeenCalledTimes(1); + restore(); + }); + it("clones an available GitHub repository and passes the result to Scient initialization", async () => { const cloneSource = vi.fn().mockResolvedValue({ path: "/Users/tester/scient" }); const onAddProjectPath = vi.fn().mockResolvedValue(true); diff --git a/apps/web/src/components/AddProjectDialog.logic.test.ts b/apps/web/src/components/AddProjectDialog.logic.test.ts index a3077d5dc..5f4ffef11 100644 --- a/apps/web/src/components/AddProjectDialog.logic.test.ts +++ b/apps/web/src/components/AddProjectDialog.logic.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { buildCloneProjectSourceInput, + getAvailableNewFolderName, inferCloneDirectoryName, joinProjectPath, } from "./AddProjectDialog.logic"; @@ -21,6 +22,12 @@ describe("AddProjectDialog logic", () => { ); }); + it("chooses a collision-safe new folder name across case-sensitive filesystems", () => { + expect(getAvailableNewFolderName([])).toBe("New folder"); + expect(getAvailableNewFolderName(["New folder", "New folder 2"])).toBe("New folder 3"); + expect(getAvailableNewFolderName(["NEW FOLDER"])).toBe("New folder 2"); + }); + it("builds the source-specific clone payload", () => { expect( buildCloneProjectSourceInput({ diff --git a/apps/web/src/components/AddProjectDialog.logic.ts b/apps/web/src/components/AddProjectDialog.logic.ts index 79e80c57b..7cda76bbc 100644 --- a/apps/web/src/components/AddProjectDialog.logic.ts +++ b/apps/web/src/components/AddProjectDialog.logic.ts @@ -29,6 +29,24 @@ export function joinProjectPath(basePath: string, childName: string): string { return `${basePath.replace(/[\\/]+$/, "")}${separator}${childName}`; } +export function getAvailableNewFolderName( + directoryNames: readonly string[], + baseName = "New folder", +): string { + // Avoid a name that collides on case-insensitive filesystems as well as on Linux. + const existingNames = new Set(directoryNames.map((name) => name.toLowerCase())); + if (!existingNames.has(baseName.toLowerCase())) { + return baseName; + } + + for (let suffix = 2; ; suffix += 1) { + const candidate = `${baseName} ${suffix}`; + if (!existingNames.has(candidate.toLowerCase())) { + return candidate; + } + } +} + export function buildCloneProjectSourceInput(input: { source: Exclude; repositoryInput: string; diff --git a/apps/web/src/components/AddProjectDialog.tsx b/apps/web/src/components/AddProjectDialog.tsx index b42c4b65a..2304b7f82 100644 --- a/apps/web/src/components/AddProjectDialog.tsx +++ b/apps/web/src/components/AddProjectDialog.tsx @@ -40,6 +40,7 @@ import { import { Kbd, KbdGroup } from "./ui/kbd"; import { buildCloneProjectSourceInput, + getAvailableNewFolderName, type AddProjectSource, inferCloneDirectoryName, joinProjectPath, @@ -93,26 +94,35 @@ function sourceStatus( return statuses?.find((status) => status.provider === provider) ?? null; } -function NavigationKeyHints(props: { enterLabel: string }) { +function NavigationKeyHints(props: { + enterLabel: string; + browseDirectories?: boolean; + compact?: boolean; +}) { return (
↑ ↓ + {props.browseDirectories ? → : null} Navigate Enter {props.enterLabel} - - Backspace - Back - - - Esc - Close - + {!props.compact ? ( + <> + + Backspace + Back + + + Esc + Close + + + ) : null}
); } @@ -134,6 +144,11 @@ function ProjectPathBrowser(props: PathBrowserProps & { homeDir: string | null } const [query, setQuery] = useState(props.initialQuery); const [highlightedValue, setHighlightedValue] = useState(null); const [error, setError] = useState(null); + const inputRef = useRef(null); + const submitInFlightRef = useRef(false); + const pendingNameSelectionRef = useRef<{ query: string; start: number; end: number } | null>( + null, + ); const platform = typeof navigator === "undefined" ? "" : navigator.platform; const trimmedQuery = query.trim(); const unsupportedWindowsPath = isUnsupportedWindowsProjectPath(trimmedQuery, platform); @@ -167,14 +182,11 @@ function ProjectPathBrowser(props: PathBrowserProps & { homeDir: string | null } leafSegment.length > 0 ? (filteredEntries.find((entry) => entry.name === leafSegment) ?? null) : null; - const highlightedPath = highlightedValue?.startsWith("folder:") - ? highlightedValue.slice("folder:".length) - : null; const browseParentPath = getBrowseParentPath(query); const canBrowseUp = canNavigateUp(query); + const canStepBackWithinBrowser = query !== props.initialQuery && browseParentPath !== null; const willCreatePath = !props.cloneDirectoryName && - !highlightedPath && trimmedQuery.length > 0 && !hasTrailingPathSeparator(query) && exactEntry === null && @@ -183,19 +195,21 @@ function ProjectPathBrowser(props: PathBrowserProps & { homeDir: string | null } useEffect(() => { setError(null); setHighlightedValue(null); + + const pendingSelection = pendingNameSelectionRef.current; + if (pendingSelection?.query === query) { + pendingNameSelectionRef.current = null; + inputRef.current?.focus(); + inputRef.current?.setSelectionRange(pendingSelection.start, pendingSelection.end); + } }, [query]); const resolvePath = (): { path: string; createIfMissing: boolean } => { - const selectedDirectory = highlightedPath - ? highlightedPath - : hasTrailingPathSeparator(query) - ? (browseResult?.parentPath ?? expandHome(trimmedQuery, props.homeDir)) - : (exactEntry?.fullPath ?? expandHome(trimmedQuery, props.homeDir)); + const selectedDirectory = hasTrailingPathSeparator(query) + ? (browseResult?.parentPath ?? expandHome(trimmedQuery, props.homeDir)) + : (exactEntry?.fullPath ?? expandHome(trimmedQuery, props.homeDir)); const normalized = normalizeProjectPathForDispatch(selectedDirectory); - if ( - props.cloneDirectoryName && - (highlightedPath || exactEntry || hasTrailingPathSeparator(query)) - ) { + if (props.cloneDirectoryName && (exactEntry || hasTrailingPathSeparator(query))) { return { path: joinProjectPath(normalized, props.cloneDirectoryName), createIfMissing: false, @@ -205,8 +219,8 @@ function ProjectPathBrowser(props: PathBrowserProps & { homeDir: string | null } }; const submit = async () => { - if (props.isBusy) return; - if (!trimmedQuery && !highlightedPath) { + if (props.isBusy || submitInFlightRef.current) return; + if (!trimmedQuery) { setError("Enter a folder path."); return; } @@ -214,33 +228,56 @@ function ProjectPathBrowser(props: PathBrowserProps & { homeDir: string | null } setError("Windows paths are not supported on this platform."); return; } - if (!highlightedPath && isExplicitRelativeProjectPath(trimmedQuery)) { + if (isExplicitRelativeProjectPath(trimmedQuery)) { setError("Use an absolute path or start with ~/."); return; } + submitInFlightRef.current = true; try { const resolved = resolvePath(); - await props.onSubmit(resolved.path, { createIfMissing: resolved.createIfMissing }); + if (!resolved.createIfMissing) { + await props.onSubmit(resolved.path, { createIfMissing: false }); + return; + } + + const api = readNativeApi(); + if (!api) throw new Error("The app server is unavailable."); + const created = await api.filesystem.createDirectory({ path: resolved.path }); + await props.onSubmit(created.path, { createIfMissing: true }); } catch (cause) { setError(cause instanceof Error ? cause.message : "Unable to add project."); + } finally { + submitInFlightRef.current = false; } }; - const isMac = platform.toLowerCase().includes("mac"); - const modifier = isMac ? "⌘" : "Ctrl"; - const hasHighlightedFolder = highlightedPath !== null; - const hasHighlightedBrowseItem = hasHighlightedFolder || highlightedValue === "__browse_up__"; + const browseHighlightedDirectory = (): boolean => { + if (highlightedValue === "__browse_up__") { + if (!browseParentPath) return false; + setQuery(browseParentPath); + return true; + } + if (!highlightedValue?.startsWith("folder:")) return false; - const handleInputKeyDown = (event: KeyboardEvent) => { - const modifierPressed = isMac ? event.metaKey : event.ctrlKey; - if ( - event.key === "Enter" && - (!hasHighlightedBrowseItem || (hasHighlightedFolder && modifierPressed)) - ) { + const highlightedPath = highlightedValue.slice("folder:".length); + const highlightedEntry = filteredEntries.find((entry) => entry.fullPath === highlightedPath); + if (!highlightedEntry) return false; + setQuery(appendBrowsePathSegment(query, highlightedEntry.name)); + return true; + }; + + const handleInputKeyDownCapture = (event: KeyboardEvent) => { + if (event.key === "Enter") { event.preventDefault(); + event.stopPropagation(); void submit(); return; } + if (event.key === "ArrowRight" && browseHighlightedDirectory()) { + event.preventDefault(); + event.stopPropagation(); + return; + } if (event.key === "Backspace" && query === "") { event.preventDefault(); props.onBack(); @@ -258,25 +295,50 @@ function ProjectPathBrowser(props: PathBrowserProps & { homeDir: string | null } } }; + const beginNewFolder = () => { + const basePath = getBrowseDirectoryPath(query); + const folderName = getAvailableNewFolderName(browseEntries.map((entry) => entry.name)); + const nextQuery = `${basePath}${folderName}`; + pendingNameSelectionRef.current = { + query: nextQuery, + start: basePath.length, + end: nextQuery.length, + }; + setQuery(nextQuery); + }; + + const goBack = () => { + if (canStepBackWithinBrowser) { + setQuery(browseParentPath); + return; + } + props.onBack(); + }; + return ( setHighlightedValue(typeof value === "string" ? value : null)} >
- + setQuery(event.currentTarget.value)} - onKeyDown={handleInputKeyDown} - className="pe-32 ps-8" + onKeyDownCapture={handleInputKeyDownCapture} + className={cn("ps-8", willCreatePath ? "pe-40" : "pe-32")} />
@@ -341,30 +403,59 @@ function ProjectPathBrowser(props: PathBrowserProps & { homeDir: string | null } - - + +
+ {!props.cloneDirectoryName ? ( + + ) : null} + +
); @@ -512,8 +603,8 @@ export function AddProjectDialog(props: AddProjectDialogProps) { { diff --git a/apps/web/src/components/AppSnapCoordinator.tsx b/apps/web/src/components/AppSnapCoordinator.tsx index 5f8701217..7d487a7a7 100644 --- a/apps/web/src/components/AppSnapCoordinator.tsx +++ b/apps/web/src/components/AppSnapCoordinator.tsx @@ -11,6 +11,7 @@ import { useAppSettings } from "../appSettings"; import { type AppSnapThreadTarget, type TimedAppSnapThreadTarget, + createLatestAppSnapRequestGuard, didAppSnapHydrationInputsChange, hasHydratedAppSnapCapture, hasPersistedAppSnapCapture, @@ -43,12 +44,23 @@ import { isComposerAppSnapCaptureSource, } from "../lib/composerImageSource"; import { resolveRecentThreadSplitActivation } from "../recentViewActivation.logic"; +import { activityManager } from "../notifications/activityStore"; +import { transientAlertManager } from "../notifications/transientAlert"; import { useSplitViewStore } from "../splitViewStore"; import { useStore } from "../store"; import { useTerminalStateStore } from "../terminalStateStore"; -import { toastManager } from "./ui/toast"; const MAX_REMEMBERED_CAPTURE_IDS = 100; +const APPSNAP_LISTENER_ACTIVITY_KEY = "appsnap:listener"; +const APPSNAP_PENDING_RESTORE_ACTIVITY_KEY = "appsnap:pending-restore"; + +function appSnapCaptureActivityKey(captureId: string): string { + return `appsnap:capture:${captureId}`; +} + +function errorDescription(error: unknown, fallback: string): string { + return error instanceof Error ? error.message : fallback; +} interface PersistedAppSnapHydrationTarget { attachments: ReadonlyArray; @@ -123,6 +135,7 @@ export function AppSnapCoordinator() { playCaptureSoundRef.current = settings.appSnapPlaySound; const enableAppSnapRef = useRef(settings.enableAppSnap); enableAppSnapRef.current = settings.enableAppSnap; + const appSnapPreferenceRequestGuardRef = useRef(createLatestAppSnapRequestGuard()); useEffect(() => { let disposed = false; @@ -283,11 +296,33 @@ export function AppSnapCoordinator() { useEffect(() => { const bridge = window.desktopBridge?.appSnap; if (!bridge) return; + const requestId = appSnapPreferenceRequestGuardRef.current.begin(); + let disposed = false; // The opt-in preference lives in the renderer settings store. This root // coordinator is mounted for the full UI lifetime and owns the native listener. - void bridge.setEnabled(settings.enableAppSnap).catch((error) => { - console.warn("[appsnap] Could not update native listener state", error); - }); + void bridge + .setEnabled(settings.enableAppSnap) + .then(() => { + if (disposed || !appSnapPreferenceRequestGuardRef.current.isCurrent(requestId)) return; + activityManager.remove(APPSNAP_LISTENER_ACTIVITY_KEY); + }) + .catch((error) => { + if (disposed || !appSnapPreferenceRequestGuardRef.current.isCurrent(requestId)) return; + const description = errorDescription(error, "Could not update AppSnap capture state."); + console.warn("[appsnap] Could not update native listener state", error); + activityManager.publish({ + dedupeKey: APPSNAP_LISTENER_ACTIVITY_KEY, + source: "system", + status: "needs_attention", + tone: "error", + title: "AppSnap needs attention", + description, + destination: { type: "settings", section: "appsnap" }, + }); + }); + return () => { + disposed = true; + }; }, [settings.enableAppSnap]); const activateExistingTarget = useCallback( @@ -432,18 +467,22 @@ export function AppSnapCoordinator() { } lastAppSnapRef.current = { ...target, atMs: captureAtMs }; requestComposerFocus(target.threadId); - toastManager.add({ - type: persistenceResult === "unverified" ? "warning" : "success", - title: - persistenceResult === "unverified" ? "AppSnap added with a warning" : "AppSnap added", - description: - persistenceResult === "unverified" - ? "The capture is attached, but Scient could not verify its draft metadata. If it is missing after a reload, Scient will attach it again." - : capture.sourceAppName - ? `Captured ${capture.sourceAppName} and added it to the composer.` - : "The frontmost window was added to the composer.", - data: { allowCrossThreadVisibility: true }, - }); + const captureActivityKey = appSnapCaptureActivityKey(capture.id); + if (persistenceResult === "unverified") { + activityManager.publish({ + dedupeKey: captureActivityKey, + source: "system", + status: "needs_attention", + tone: "warning", + title: "AppSnap attached, but not fully verified", + description: + "The capture is visible in the composer, but Scient could not verify its saved draft metadata. It will retry from the recovery copy after a reload.", + destination: { type: "thread", threadId: target.threadId }, + }); + } else { + // The attachment chip is the local success state; clear any prior failure for this capture. + activityManager.remove(captureActivityKey); + } return persistenceResult; }, [activateExistingTarget, handleNewChat, openChatThreadPage], @@ -496,13 +535,36 @@ export function AppSnapCoordinator() { if (!attach) throw new Error("The AppSnap composer is not ready yet."); persistence = await attach(capture); } catch (error) { - toastManager.add({ + const description = errorDescription(error, "AppSnap capture failed."); + const captureActivityKey = appSnapCaptureActivityKey(capture.id); + activityManager.publish({ + dedupeKey: captureActivityKey, + source: "system", + status: "needs_attention", + tone: "error", + title: "AppSnap could not be added", + description, + }); + // This coordinator has no owning visual surface. Keep the rare global error solely + // to expose the immediate Retry action; the durable record lives in Activity. + let alertId: ReturnType; + alertId = transientAlertManager.add({ type: "error", title: "AppSnap could not be added", - description: error instanceof Error ? error.message : "AppSnap capture failed.", + description, actionProps: { children: "Retry", onClick: () => { + transientAlertManager.close(alertId); + activityManager.publish({ + dedupeKey: captureActivityKey, + source: "system", + status: "in_progress", + tone: "info", + title: "Retrying AppSnap", + description: "Scient is trying to attach the recovered capture again.", + preserveRead: true, + }); captureIdsRef.current.delete(capture.id); enqueueCapture(capture); }, @@ -531,24 +593,59 @@ export function AppSnapCoordinator() { enqueueCapture(capture); }); const unsubscribeError = bridge.onError((error) => { - toastManager.add({ - type: "error", + activityManager.publish({ + dedupeKey: APPSNAP_LISTENER_ACTIVITY_KEY, + source: "system", + status: "needs_attention", + tone: "error", title: "AppSnap failed", description: error.message, - ...(error.code === "helper-stopped" - ? { - actionProps: { - children: "Restart", - onClick: () => { - void bridge - .setEnabled(enableAppSnapRef.current) - .catch((restartError) => - console.warn("[appsnap] Could not restart native listener", restartError), - ); - }, - }, - } - : {}), + destination: { type: "settings", section: "appsnap" }, + }); + if (error.code !== "helper-stopped") return; + + // The native helper stopped outside any visible owning surface. Retain one global error + // only because Activity does not yet host actions and immediate Restart must remain available. + let alertId: ReturnType; + alertId = transientAlertManager.add({ + type: "error", + title: "AppSnap stopped", + description: error.message, + actionProps: { + children: "Restart", + onClick: () => { + transientAlertManager.close(alertId); + activityManager.publish({ + dedupeKey: APPSNAP_LISTENER_ACTIVITY_KEY, + source: "system", + status: "in_progress", + tone: "info", + title: "Restarting AppSnap", + description: "Scient is restarting the AppSnap listener.", + preserveRead: true, + destination: { type: "settings", section: "appsnap" }, + }); + void bridge + .setEnabled(enableAppSnapRef.current) + .then(() => activityManager.remove(APPSNAP_LISTENER_ACTIVITY_KEY)) + .catch((restartError) => { + const description = errorDescription( + restartError, + "Scient could not restart AppSnap.", + ); + console.warn("[appsnap] Could not restart native listener", restartError); + activityManager.publish({ + dedupeKey: APPSNAP_LISTENER_ACTIVITY_KEY, + source: "system", + status: "needs_attention", + tone: "error", + title: "AppSnap could not restart", + description, + destination: { type: "settings", section: "appsnap" }, + }); + }); + }, + }, data: { allowCrossThreadVisibility: true, copyText: `${error.code}: ${error.message}`, @@ -557,8 +654,23 @@ export function AppSnapCoordinator() { }); void bridge .listPendingCaptures() - .then((captures) => captures.forEach(enqueueCapture)) - .catch((error) => console.warn("[appsnap] Could not restore pending captures", error)); + .then((captures) => { + activityManager.remove(APPSNAP_PENDING_RESTORE_ACTIVITY_KEY); + captures.forEach(enqueueCapture); + }) + .catch((error) => { + const description = errorDescription(error, "Could not restore pending AppSnaps."); + console.warn("[appsnap] Could not restore pending captures", error); + activityManager.publish({ + dedupeKey: APPSNAP_PENDING_RESTORE_ACTIVITY_KEY, + source: "system", + status: "needs_attention", + tone: "warning", + title: "Pending AppSnaps could not be restored", + description, + destination: { type: "settings", section: "appsnap" }, + }); + }); return () => { disposed = true; diff --git a/apps/web/src/components/BranchToolbarBranchSelector.tsx b/apps/web/src/components/BranchToolbarBranchSelector.tsx index b86f6e9e7..c218f9b48 100644 --- a/apps/web/src/components/BranchToolbarBranchSelector.tsx +++ b/apps/web/src/components/BranchToolbarBranchSelector.tsx @@ -57,7 +57,7 @@ import { ComboboxTrigger, } from "./ui/combobox"; import { Input } from "./ui/input"; -import { toastManager } from "./ui/toast"; +import { transientAlertManager } from "../notifications/transientAlert"; import { ENVIRONMENT_ROW_CLASS_NAME, ENVIRONMENT_ROW_ICON_CLASS_NAME, @@ -106,18 +106,18 @@ const UNRESOLVED_INDEX_PATTERN = /you need to resolve your current index/i; const GIT_INDEX_LOCK_PATTERN = /(?:Unable to create '([^']*\.git\/index\.lock)'|Another git process seems to be running|\.git\/index\.lock.*File exists)/i; const GIT_INDEX_WRITE_PATTERN = /could not write index/i; -let activeBranchRecoveryToastId: ReturnType | null = null; +let activeBranchRecoveryAlertId: ReturnType | null = null; -function closeActiveBranchRecoveryToast(): void { - if (!activeBranchRecoveryToastId) return; - toastManager.close(activeBranchRecoveryToastId); - activeBranchRecoveryToastId = null; +function closeActiveBranchRecoveryAlert(): void { + if (!activeBranchRecoveryAlertId) return; + transientAlertManager.close(activeBranchRecoveryAlertId); + activeBranchRecoveryAlertId = null; } -function addBranchRecoveryToast(input: Parameters[0]) { - closeActiveBranchRecoveryToast(); - activeBranchRecoveryToastId = toastManager.add(input); - return activeBranchRecoveryToastId; +function addBranchRecoveryAlert(input: Parameters[0]) { + closeActiveBranchRecoveryAlert(); + activeBranchRecoveryAlertId = transientAlertManager.add(input); + return activeBranchRecoveryAlertId; } function parseDirtyWorktreeError(error: unknown): { branch: string; files: string[] } | null { @@ -187,13 +187,13 @@ function handleCheckoutError( input.onSuccess(); }; - const addGitIndexLockToast = (error: unknown): void => { + const addGitIndexLockAlert = (error: unknown): void => { const lockError = parseGitIndexLockError(error); if (!lockError) return; const lockFileLabel = lockError.lockPath ? lockError.lockPath.split("/").slice(-2).join("/") : ".git/index.lock"; - addBranchRecoveryToast({ + addBranchRecoveryAlert({ type: "error", title: "Git index is locked.", description: `${lockFileLabel} already exists. Close any running Git operation, remove the stale lock file if none is running, then retry.`, @@ -214,8 +214,8 @@ function handleCheckoutError( }); }; - const addGitIndexWriteToast = (error: unknown): void => { - addBranchRecoveryToast({ + const addGitIndexWriteAlert = (error: unknown): void => { + addBranchRecoveryAlert({ type: "error", title: "Git index could not be written.", description: @@ -239,7 +239,7 @@ function handleCheckoutError( const dirtyWorktree = parseDirtyWorktreeError(error); if (dirtyWorktree) { const copyText = toBranchActionErrorMessage(error); - const dirtyToastId = addBranchRecoveryToast({ + addBranchRecoveryAlert({ type: "warning", title: "Uncommitted changes block checkout.", description: formatDirtyWorktreeDescription(dirtyWorktree.files), @@ -247,23 +247,23 @@ function handleCheckoutError( actionProps: { children: "Stash & Switch", onClick: () => { - closeActiveBranchRecoveryToast(); + closeActiveBranchRecoveryAlert(); input.runBranchAction(async () => { try { await retryStashAndCheckout(); } catch (stashError) { if (parseGitIndexLockError(stashError)) { - addGitIndexLockToast(stashError); + addGitIndexLockAlert(stashError); return; } if (isGitIndexWriteError(stashError)) { - addGitIndexWriteToast(stashError); + addGitIndexWriteAlert(stashError); return; } if (isStashConflictError(stashError)) { await invalidateGitQueries(input.queryClient); input.onSuccess(); - const stashConflictToastId = addBranchRecoveryToast({ + addBranchRecoveryAlert({ type: "warning", title: "Changes saved, but not reapplied.", description: @@ -274,7 +274,7 @@ function handleCheckoutError( className: "border-destructive bg-destructive text-white shadow-destructive/24 hover:bg-destructive/90", onClick: () => { - closeActiveBranchRecoveryToast(); + closeActiveBranchRecoveryAlert(); input.onRequestDiscardStash({ cwd: input.cwd }); }, }, @@ -282,7 +282,7 @@ function handleCheckoutError( return; } if (parseDirtyWorktreeError(stashError)) { - addBranchRecoveryToast({ + addBranchRecoveryAlert({ type: "error", title: "Cannot switch branches.", description: @@ -291,7 +291,7 @@ function handleCheckoutError( }); return; } - addBranchRecoveryToast({ + addBranchRecoveryAlert({ type: "error", title: "Failed to stash and switch.", description: toBranchActionErrorMessage(stashError), @@ -306,15 +306,15 @@ function handleCheckoutError( } if (parseGitIndexLockError(error)) { - addGitIndexLockToast(error); + addGitIndexLockAlert(error); return; } if (isGitIndexWriteError(error)) { - addGitIndexWriteToast(error); + addGitIndexWriteAlert(error); return; } - addBranchRecoveryToast({ + addBranchRecoveryAlert({ type: "error", title: isUnresolvedIndexError(error) ? "Unresolved conflicts in the repository." @@ -382,6 +382,7 @@ export function BranchToolbarBranchSelector({ const [isBranchMenuOpen, setIsBranchMenuOpen] = useState(false); const [isCreateBranchDialogOpen, setIsCreateBranchDialogOpen] = useState(false); const [createBranchName, setCreateBranchName] = useState(""); + const [createBranchError, setCreateBranchError] = useState(null); const [branchQuery, setBranchQuery] = useState(""); const deferredBranchQuery = useDeferredValue(branchQuery); @@ -476,6 +477,7 @@ export function BranchToolbarBranchSelector({ const openCreateBranchDialog = useCallback(() => { setCreateBranchName(canPrefillCreateBranch && !hasExactBranchMatch ? trimmedBranchQuery : ""); + setCreateBranchError(null); setIsBranchMenuOpen(false); setIsCreateBranchDialogOpen(true); }, [canPrefillCreateBranch, hasExactBranchMatch, trimmedBranchQuery]); @@ -519,6 +521,12 @@ export function BranchToolbarBranchSelector({ try { await api.git.stashDrop({ cwd: dialog.cwd }); setStashDiscardDialog(null); + } catch (error) { + setStashDiscardDialog((current) => + current?.cwd === dialog.cwd + ? { ...current, error: toBranchActionErrorMessage(error), loading: false } + : current, + ); } finally { setIsDroppingStash(false); } @@ -605,10 +613,14 @@ export function BranchToolbarBranchSelector({ const createBranch = (rawName: string) => { const name = rawName.trim(); const api = readNativeApi(); - if (!api || !branchCwd || !name || isBranchActionPending) return; + if (!name || isBranchActionPending) return; + if (!api || !branchCwd) { + setCreateBranchError("Branch creation is unavailable."); + return; + } + setCreateBranchError(null); setIsBranchMenuOpen(false); - onComposerFocusRequest?.(); runBranchAction(async () => { setOptimisticBranch(name); @@ -618,6 +630,12 @@ export function BranchToolbarBranchSelector({ try { await api.git.checkout({ cwd: branchCwd, branch: name }); } catch (error) { + // The branch now exists, so leave creation mode before offering + // checkout-specific recovery actions. Retrying creation would be wrong. + setIsCreateBranchDialogOpen(false); + setCreateBranchName(""); + setCreateBranchError(null); + onComposerFocusRequest?.(); handleCheckoutError(error, { api, branch: name, @@ -639,11 +657,7 @@ export function BranchToolbarBranchSelector({ return; } } catch (error) { - toastManager.add({ - type: "error", - title: "Failed to create branch.", - description: toBranchActionErrorMessage(error), - }); + setCreateBranchError(toBranchActionErrorMessage(error)); return; } @@ -654,6 +668,9 @@ export function BranchToolbarBranchSelector({ }); setBranchQuery(""); setCreateBranchName(""); + setCreateBranchError(null); + setIsCreateBranchDialogOpen(false); + onComposerFocusRequest?.(); }); }; @@ -918,6 +935,7 @@ export function BranchToolbarBranchSelector({ setIsCreateBranchDialogOpen(open); if (!open) { setCreateBranchName(""); + setCreateBranchError(null); } }} > @@ -937,7 +955,6 @@ export function BranchToolbarBranchSelector({ if (!nextName || branchByName.has(nextName)) { return; } - setIsCreateBranchDialogOpen(false); createBranch(nextName); }} > @@ -950,12 +967,20 @@ export function BranchToolbarBranchSelector({ id="branch-create-name" placeholder="feature/my-change" value={createBranchName} - onChange={(event) => setCreateBranchName(event.target.value)} + onChange={(event) => { + setCreateBranchName(event.target.value); + setCreateBranchError(null); + }} /> {branchByName.has(createBranchName.trim()) ? (

A branch with this name already exists.

) : null} + {createBranchError ? ( +

+ {createBranchError} +

+ ) : null} diff --git a/apps/web/src/components/BrowserPanel.browser.tsx b/apps/web/src/components/BrowserPanel.browser.tsx new file mode 100644 index 000000000..f5b5c4cae --- /dev/null +++ b/apps/web/src/components/BrowserPanel.browser.tsx @@ -0,0 +1,133 @@ +// FILE: BrowserPanel.browser.tsx +// Purpose: Browser-level coverage for tab-scoped, local copy feedback. + +import "../index.css"; + +import type { ThreadBrowserState, ThreadId } from "@synara/contracts"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { page } from "vitest/browser"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { render } from "vitest-browser-react"; + +vi.mock("~/lib/serverReactQuery", async (importOriginal) => ({ + ...(await importOriginal()), + serverLocalServersQueryOptions: () => ({ + queryKey: ["browser-panel-test", "local-servers"], + queryFn: async () => ({ servers: [] }), + staleTime: Number.POSITIVE_INFINITY, + }), +})); + +vi.mock("~/nativeApi", async (importOriginal) => ({ + ...(await importOriginal()), + readNativeApi: () => undefined, +})); + +import { useBrowserStateStore } from "../browserStateStore"; +import { BrowserPanel } from "./BrowserPanel"; + +const THREAD_ID = "thread-browser-copy" as ThreadId; + +function browserState(activeTabId: string): ThreadBrowserState { + return { + threadId: THREAD_ID, + version: activeTabId === "tab-1" ? 1 : 2, + open: true, + activeTabId, + lastError: null, + tabs: [ + { + id: "tab-1", + kind: "web", + url: "https://scientfactory.com/", + displayUrl: null, + title: "ScientFactory", + status: "suspended", + isLoading: false, + canGoBack: false, + canGoForward: false, + faviconUrl: null, + lastCommittedUrl: "https://scientfactory.com/", + lastError: null, + }, + { + id: "tab-2", + kind: "web", + url: "https://example.com/", + displayUrl: null, + title: "Example", + status: "suspended", + isLoading: false, + canGoBack: false, + canGoForward: false, + faviconUrl: null, + lastCommittedUrl: "https://example.com/", + lastError: null, + }, + ], + }; +} + +function renderPanel() { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return render( + + undefined} + /> + , + ); +} + +describe("BrowserPanel copy feedback", () => { + beforeEach(() => { + useBrowserStateStore.getState().upsertThreadState(browserState("tab-1")); + }); + + afterEach(() => { + useBrowserStateStore.getState().removeThreadState(THREAD_ID); + vi.restoreAllMocks(); + document.body.innerHTML = ""; + }); + + it("surfaces clipboard rejection locally", async () => { + const writeText = vi + .spyOn(navigator.clipboard, "writeText") + .mockRejectedValue(new Error("Clipboard denied")); + await renderPanel(); + + ( + (await page.getByRole("button", { name: "Copy link" }).element()) as HTMLButtonElement + ).click(); + + await vi.waitFor(() => expect(writeText).toHaveBeenCalledWith("https://scientfactory.com/")); + await vi.waitFor(() => { + const localStatus = page + .getByRole("status") + .elements() + .find((element) => element.tagName === "SPAN"); + expect(localStatus?.textContent).toBe("Couldn't complete that browser action."); + }); + }); + + it("does not show late copy success on a different active tab", async () => { + let resolveCopy: (() => void) | undefined; + vi.spyOn(navigator.clipboard, "writeText").mockImplementation( + () => new Promise((resolve) => (resolveCopy = resolve)), + ); + await renderPanel(); + ( + (await page.getByRole("button", { name: "Copy link" }).element()) as HTMLButtonElement + ).click(); + + useBrowserStateStore.getState().upsertThreadState(browserState("tab-2")); + resolveCopy?.(); + await new Promise((resolve) => window.setTimeout(resolve, 0)); + + await expect.element(page.getByRole("button", { name: "Copy link" })).toBeVisible(); + expect(page.getByText("Link copied").query()).toBeNull(); + }); +}); diff --git a/apps/web/src/components/BrowserPanel.logic.test.ts b/apps/web/src/components/BrowserPanel.logic.test.ts index fd6feb13a..dc6ee4285 100644 --- a/apps/web/src/components/BrowserPanel.logic.test.ts +++ b/apps/web/src/components/BrowserPanel.logic.test.ts @@ -2,12 +2,44 @@ import { describe, expect, it } from "vitest"; import { browserAddressDisplayValue, + browserCopyFeedbackMatches, buildBrowserAddressSuggestions, normalizeBrowserAddressInput, resolveBrowserChromeStatus, resolveBrowserAddressSync, } from "./BrowserPanel.logic"; +describe("browserCopyFeedbackMatches", () => { + const feedback = { + item: "link" as const, + tabId: "tab-1", + url: "https://scientfactory.com/", + tone: "success" as const, + message: "Link copied", + }; + + it("keeps feedback scoped to the exact tab and URL that was copied", () => { + expect( + browserCopyFeedbackMatches(feedback, { + tabId: "tab-1", + url: "https://scientfactory.com/", + }), + ).toBe(true); + expect( + browserCopyFeedbackMatches(feedback, { + tabId: "tab-2", + url: "https://scientfactory.com/", + }), + ).toBe(false); + expect( + browserCopyFeedbackMatches(feedback, { + tabId: "tab-1", + url: "https://scientfactory.com/docs", + }), + ).toBe(false); + }); +}); + describe("browserAddressDisplayValue", () => { it("hides about:blank for new tabs", () => { expect(browserAddressDisplayValue({ url: "about:blank" })).toBe(""); @@ -16,6 +48,15 @@ describe("browserAddressDisplayValue", () => { it("keeps real urls visible", () => { expect(browserAddressDisplayValue({ url: "https://x.com/" })).toBe("https://x.com/"); }); + + it("shows an artifact source path instead of its capability URL", () => { + expect( + browserAddressDisplayValue({ + url: "http://g-secret.preview.localhost:5000/", + displayUrl: "/workspace/report.html", + }), + ).toBe("/workspace/report.html"); + }); }); describe("resolveBrowserAddressSync", () => { diff --git a/apps/web/src/components/BrowserPanel.logic.ts b/apps/web/src/components/BrowserPanel.logic.ts index a9479f61c..673a07840 100644 --- a/apps/web/src/components/BrowserPanel.logic.ts +++ b/apps/web/src/components/BrowserPanel.logic.ts @@ -55,11 +55,26 @@ export interface BrowserChromeStatus { label: string; } +export interface BrowserCopyFeedback { + item: "link" | "screenshot"; + tabId: string; + url: string; + tone: "success" | "error"; + message: string; +} + +export function browserCopyFeedbackMatches( + feedback: BrowserCopyFeedback | null, + scope: { tabId: string; url: string } | null, +): feedback is BrowserCopyFeedback { + return Boolean(feedback && scope && feedback.tabId === scope.tabId && feedback.url === scope.url); +} + // Hides about:blank from the address bar so new tabs behave like real browsers. export function browserAddressDisplayValue( - tab: Pick | null | undefined, + tab: { url: string; displayUrl?: string | null } | null | undefined, ): string { - const nextUrl = tab?.url?.trim() ?? ""; + const nextUrl = tab?.displayUrl?.trim() || tab?.url?.trim() || ""; return nextUrl === BROWSER_BLANK_URL ? "" : nextUrl; } diff --git a/apps/web/src/components/BrowserPanel.tsx b/apps/web/src/components/BrowserPanel.tsx index 204243aa1..5c7eb285a 100644 --- a/apps/web/src/components/BrowserPanel.tsx +++ b/apps/web/src/components/BrowserPanel.tsx @@ -18,6 +18,7 @@ import { ArrowLeftIcon, ArrowRightIcon, CameraIcon, + CheckIcon, EllipsisIcon, ExternalLinkIcon, GlobeIcon, @@ -32,13 +33,11 @@ import { import { localServerPrimaryLabel } from "@synara/shared/localServers"; import { BROWSER_BLANK_URL, + browserSessionPartition, isBlankBrowserTabUrl, resolveCopyableBrowserTabUrl, } from "@synara/shared/browserSession"; -import { - BROWSER_COPY_LINK_TOAST_TITLE, - isBrowserCopyLinkChord, -} from "@synara/shared/browserShortcuts"; +import { isBrowserCopyLinkChord } from "@synara/shared/browserShortcuts"; import { isElectron } from "~/env"; import { readNativeApi } from "~/nativeApi"; @@ -54,18 +53,19 @@ import { selectThreadBrowserState, } from "../browserStateStore"; import { useComposerDraftStore } from "../composerDraftStore"; -import { anchoredToastManager } from "./ui/toast"; import { composerImageFromBrowserScreenshot, screenshotAttachmentName, } from "../lib/browserPromptContext"; import { browserAddressDisplayValue, + browserCopyFeedbackMatches, buildBrowserAddressSuggestions, normalizeBrowserAddressInput, resolveBrowserChromeStatus, resolveBrowserAddressSync, type BrowserAddressSuggestion, + type BrowserCopyFeedback, } from "./BrowserPanel.logic"; import { DiffPanelLoadingState, DiffPanelShell, type DiffPanelMode } from "./DiffPanelShell"; import { LocalServerIdentity } from "./LocalServerIdentity"; @@ -74,7 +74,6 @@ import { ComposerPickerMenuPopup } from "./chat/ComposerPickerMenuPopup"; import { Input } from "./ui/input"; import { Menu, MenuItem, MenuSeparator, MenuTrigger } from "./ui/menu"; import { Skeleton } from "./ui/skeleton"; -import { toastManager } from "./ui/toast"; interface BrowserPanelProps { mode: DiffPanelMode; @@ -86,7 +85,6 @@ interface BrowserPanelProps { const BROWSER_BOUNDS_SYNC_BURST_FRAMES = 30; const BROWSER_BOUNDS_SYNC_STABLE_FRAME_TARGET = 2; -const BROWSER_WEBVIEW_PARTITION = "persist:scient-browser"; const BROWSER_PERF_SAMPLE_INTERVAL_MS = 5_000; const SYNARA_BROWSER_LABEL = "Scient browser"; // The address field and tab pills share one chrome-control surface so the whole row reads @@ -526,9 +524,15 @@ export function BrowserPanel({ const browserWebviewTabIdRef = useRef(null); const browserWebviewAttachKeyRef = useRef(null); const copyScreenshotButtonRef = useRef(null); + const [copyFeedback, setCopyFeedback] = useState(null); const addressDraftsByTabIdRef = useRef(new Map()); const lastSyncedAddressByTabIdRef = useRef(new Map()); const previousActiveTabIdRef = useRef(null); + const artifactPreviewUrlsRef = useRef( + new Set( + threadBrowserState?.tabs.filter((tab) => tab.kind === "artifact").map((tab) => tab.url) ?? [], + ), + ); const lastSentBoundsRef = useRef(null); const lastMeasuredBoundsKeyRef = useRef(null); const lastOverlayObscuredRef = useRef(false); @@ -558,6 +562,17 @@ export function BrowserPanel({ threadBrowserState?.tabs.find((tab) => tab.id === threadBrowserState.activeTabId) ?? threadBrowserState?.tabs[0] ?? null; + const activeCopyScope = activeTab + ? { + tabId: activeTab.id, + url: resolveCopyableBrowserTabUrl(activeTab) ?? activeTab.url, + } + : null; + const visibleCopyFeedback = browserCopyFeedbackMatches(copyFeedback, activeCopyScope) + ? copyFeedback + : null; + const copiedBrowserItem = + visibleCopyFeedback?.tone === "success" ? visibleCopyFeedback.item : null; const loading = activeTab?.isLoading ?? false; const activeTabIsBlank = isBlankBrowserTabUrl(activeTab); const showLocalServersHome = isLiveRuntime && workspaceReady && (!activeTab || activeTabIsBlank); @@ -573,11 +588,15 @@ export function BrowserPanel({ const browserAddressSuggestions = buildBrowserAddressSuggestions({ query: addressValue, activeTabId: activeTab?.id ?? null, - tabs: threadBrowserState?.tabs ?? [], + tabs: threadBrowserState?.tabs.filter((tab) => tab.kind !== "artifact") ?? [], recentHistory, }); const showBrowserAddressSuggestions = - isLiveRuntime && isAddressFocused && browserAddressSuggestions.length > 0 && runtimeReady; + activeTab?.kind !== "artifact" && + isLiveRuntime && + isAddressFocused && + browserAddressSuggestions.length > 0 && + runtimeReady; const requestLiveRuntime = useCallback(() => { onRequestLive?.(); @@ -634,9 +653,22 @@ export function BrowserPanel({ } return api.browser.onState((state) => { + if (state.threadId === threadId) { + const nextArtifactUrls = new Set( + state.tabs.filter((tab) => tab.kind === "artifact").map((tab) => tab.url), + ); + for (const previewUrl of artifactPreviewUrlsRef.current) { + if (!nextArtifactUrls.has(previewUrl)) { + void api.projects + .revokeHtmlArtifactPreview({ previewUrl }) + .catch(() => ({ revoked: false })); + } + } + artifactPreviewUrlsRef.current = nextArtifactUrls; + } upsertThreadState(state); }); - }, [api, isLiveRuntime, upsertThreadState]); + }, [api, isLiveRuntime, threadId, upsertThreadState]); useEffect(() => { if (!api || !isLiveRuntime) { @@ -708,6 +740,11 @@ export function BrowserPanel({ } let webview = browserWebviewRef.current; + const expectedPartition = browserSessionPartition(activeTab.kind, threadId, activeTab.id); + if (webview?.getAttribute("partition") !== expectedPartition) { + detachRendererBrowserWebview(); + webview = null; + } if (!webview) { webview = document.createElement("webview") as BrowserWebviewElement; webview.className = "h-full w-full"; @@ -715,13 +752,15 @@ export function BrowserPanel({ webview.style.width = "100%"; webview.style.height = "100%"; webview.style.backgroundColor = "#0d0d0d"; - webview.setAttribute("partition", BROWSER_WEBVIEW_PARTITION); + webview.setAttribute("partition", expectedPartition); webview.setAttribute("webpreferences", "contextIsolation=yes,nodeIntegration=no,sandbox=yes"); // A blocks window.open() unless `allowpopups` is set. Without it, clicking // "Continue with Google" (and any OAuth/popup flow) is silently dropped before the main // process's window-open handler ever runs. Enabling it lets the popup classifier in // browserManager decide popup-vs-tab and keep the OAuth `window.opener` handshake alive. - webview.setAttribute("allowpopups", "true"); + if (activeTab.kind === "web") { + webview.setAttribute("allowpopups", "true"); + } // No `useragent` attribute on purpose: the desktop main process spoofs a desktop Chrome // UA on the shared persistent partition, so this webview (and OAuth popups) inherit the // same identity. This keeps in-app Google/OAuth sign-in working without duplicating the @@ -1154,33 +1193,29 @@ export function BrowserPanel({ return; } - void runBrowserAction(() => - api.browser.copyScreenshotToClipboard({ threadId, tabId: activeTab.id }), - ).then((result) => { - if (result === null) { - return; - } - const anchor = copyScreenshotButtonRef.current; - if (anchor) { - anchoredToastManager.add({ - data: { - tooltipStyle: true, - }, - positionerProps: { - anchor, - }, - timeout: 1_200, - title: "Browser screenshot copied", + const scope = { + tabId: activeTab.id, + url: resolveCopyableBrowserTabUrl(activeTab) ?? activeTab.url, + }; + void api.browser.copyScreenshotToClipboard({ threadId, tabId: activeTab.id }).then( + () => { + setCopyFeedback({ + ...scope, + item: "screenshot", + tone: "success", + message: "Screenshot copied", }); - return; - } - - toastManager.add({ - type: "success", - title: "Browser screenshot copied", - }); - }); - }, [activeTab, api, ensureLiveRuntime, runBrowserAction, threadId]); + }, + (error) => { + setCopyFeedback({ + ...scope, + item: "screenshot", + tone: "error", + message: formatBrowserActionError(error) ?? "Could not copy the screenshot.", + }); + }, + ); + }, [activeTab, api, ensureLiveRuntime, threadId]); const copyActiveTabLink = useCallback(() => { if (!activeTab) { @@ -1190,8 +1225,19 @@ export function BrowserPanel({ // with "Document is not focused" while the native browser view holds focus, so this // mirrors the keyboard chord — main writes the URL and emits onCopyLink, which surfaces // the toast in the listener below. + const scope = { + tabId: activeTab.id, + url: resolveCopyableBrowserTabUrl(activeTab) ?? activeTab.url, + }; if (isElectron && api) { - void runBrowserAction(() => api.browser.copyLink({ threadId, tabId: activeTab.id })); + void api.browser.copyLink({ threadId, tabId: activeTab.id }).catch((error) => { + setCopyFeedback({ + ...scope, + item: "link", + tone: "error", + message: formatBrowserActionError(error) ?? "Could not copy the link.", + }); + }); return; } const url = resolveCopyableBrowserTabUrl(activeTab); @@ -1200,17 +1246,28 @@ export function BrowserPanel({ } const clipboard = typeof navigator !== "undefined" ? navigator.clipboard : undefined; if (!clipboard) { + setCopyFeedback({ + ...scope, + item: "link", + tone: "error", + message: "Clipboard access is unavailable.", + }); return; } void clipboard.writeText(url).then( () => { - toastManager.add({ type: "success", title: BROWSER_COPY_LINK_TOAST_TITLE }); + setCopyFeedback({ ...scope, item: "link", tone: "success", message: "Link copied" }); }, - () => { - // Clipboard writes can reject without user gesture; nothing actionable to surface. + (error) => { + setCopyFeedback({ + ...scope, + item: "link", + tone: "error", + message: formatBrowserActionError(error) ?? "Could not copy the link.", + }); }, ); - }, [activeTab, api, runBrowserAction, threadId]); + }, [activeTab, api, threadId]); // React chrome focus path: the native page handles the chord through the desktop main // process, so this only fires when the address bar/tab strip (not the page) is focused. @@ -1244,7 +1301,7 @@ export function BrowserPanel({ }; }, [copyActiveTabLink, isLiveRuntime]); - // Native page focus path: main already wrote the URL to the clipboard, so just toast. + // Native page focus path: main already wrote the URL to the clipboard. useEffect(() => { if (!api || !isLiveRuntime) { return; @@ -1253,10 +1310,22 @@ export function BrowserPanel({ if (event.threadId !== threadId) { return; } - toastManager.add({ type: "success", title: BROWSER_COPY_LINK_TOAST_TITLE }); + setCopyFeedback({ + item: "link", + tabId: event.tabId, + url: event.url, + tone: "success", + message: "Link copied", + }); }); }, [api, isLiveRuntime, threadId]); + useEffect(() => { + if (!copyFeedback) return; + const timeoutId = window.setTimeout(() => setCopyFeedback(null), 2_500); + return () => window.clearTimeout(timeoutId); + }, [copyFeedback]); + const onCloseTab = useCallback( (tabId: string) => { if (!ensureLiveRuntime()) { @@ -1265,7 +1334,15 @@ export function BrowserPanel({ if (!api) { return; } - void runBrowserAction(() => api.browser.closeTab({ threadId, tabId })).then((state) => { + const closingTab = threadBrowserState?.tabs.find((tab) => tab.id === tabId); + void runBrowserAction(async () => { + if (closingTab?.kind === "artifact") { + await api.projects + .revokeHtmlArtifactPreview({ previewUrl: closingTab.url }) + .catch(() => ({ revoked: false })); + } + return api.browser.closeTab({ threadId, tabId }); + }).then((state) => { if (!state) { return; } @@ -1275,7 +1352,15 @@ export function BrowserPanel({ } }); }, - [api, ensureLiveRuntime, onClosePanel, runBrowserAction, threadId, upsertThreadState], + [ + api, + ensureLiveRuntime, + onClosePanel, + runBrowserAction, + threadBrowserState?.tabs, + threadId, + upsertThreadState, + ], ); const header = ( @@ -1361,6 +1446,12 @@ export function BrowserPanel({ { if (!isLiveRuntime) { requestLiveRuntime(); @@ -1426,6 +1517,16 @@ export function BrowserPanel({ ) : null}
+ + {visibleCopyFeedback?.message ?? ""} + void; + restore: () => void; +} { + const mediaDevicesDescriptor = Object.getOwnPropertyDescriptor(navigator, "mediaDevices"); + const audioContextDescriptor = Object.getOwnPropertyDescriptor(globalThis, "AudioContext"); + const trackStop = vi.fn(); + const processor = { + onaudioprocess: null as + | ((event: { + inputBuffer: { + numberOfChannels: number; + length: number; + getChannelData: (channel: number) => Float32Array; + }; + }) => void) + | null, + connect: vi.fn(), + disconnect: vi.fn(), + }; + const source = { connect: vi.fn(), disconnect: vi.fn() }; + const gain = { + gain: { value: 1 }, + connect: vi.fn(), + disconnect: vi.fn(), + }; + + class FakeAudioContext { + readonly sampleRate = 48_000; + readonly destination = {}; + + resume = vi.fn(async () => undefined); + close = vi.fn(async () => undefined); + createMediaStreamSource = vi.fn(() => source); + createScriptProcessor = vi.fn(() => processor); + createGain = vi.fn(() => gain); + } + + Object.defineProperty(navigator, "mediaDevices", { + configurable: true, + value: { + getUserMedia: vi.fn(async () => ({ + getTracks: () => [{ stop: trackStop }], + })), + }, + }); + Object.defineProperty(globalThis, "AudioContext", { + configurable: true, + value: FakeAudioContext, + }); + + return { + emitSamples: () => { + const samples = new Float32Array(4_096).fill(0.25); + processor.onaudioprocess?.({ + inputBuffer: { + numberOfChannels: 1, + length: samples.length, + getChannelData: () => samples, + }, + }); + }, + restore: () => { + if (mediaDevicesDescriptor) { + Object.defineProperty(navigator, "mediaDevices", mediaDevicesDescriptor); + } else { + Reflect.deleteProperty(navigator, "mediaDevices"); + } + if (audioContextDescriptor) { + Object.defineProperty(globalThis, "AudioContext", audioContextDescriptor); + } else { + Reflect.deleteProperty(globalThis, "AudioContext"); + } + }, + }; +} + +function appendActiveThreadActivity(activity: OrchestrationThreadActivity): void { + const snapshot: OrchestrationReadModel = { + ...fixture.snapshot, + snapshotSequence: fixture.snapshot.snapshotSequence + 1, + threads: fixture.snapshot.threads.map((thread) => + thread.id === THREAD_ID + ? { + ...thread, + activities: [...thread.activities, activity], + updatedAt: activity.createdAt, + } + : thread, + ), + updatedAt: activity.createdAt, + }; + fixture = { ...fixture, snapshot }; + useStore.getState().syncServerReadModel(snapshot); +} + +function configureSuccessfulVoiceTranscription(transcript: string): (api: NativeApi) => NativeApi { + return (api) => ({ + ...api, + server: { + ...api.server, + transcribeVoice: vi.fn(async () => ({ + text: transcript, + engine: "local" as const, + })), + cancelVoiceTranscription: vi.fn(async () => undefined), + }, + }); +} + function createSnapshotForTargetUser(options: { targetMessageId: MessageId; targetText: string; @@ -1242,89 +1356,91 @@ function resolveWsRpc(body: WsRequestEnvelope["body"]): unknown { return {}; } -function installDeterministicActionNativeApi(): () => void { +function installDeterministicActionNativeApi( + configure?: (api: NativeApi) => NativeApi, +): () => void { const previousNativeApi = window.nativeApi; const wsNativeApi = readNativeApi(); if (!wsNativeApi) { throw new Error("Expected browser native API fixture."); } - Object.defineProperty(window, "nativeApi", { - configurable: true, - value: { - ...wsNativeApi, - shell: { - ...wsNativeApi.shell, - openInEditor: async ( - cwd: Parameters[0], - editor: Parameters[1], - ) => { - wsRequests.push({ - _tag: WS_METHODS.shellOpenInEditor, - cwd, - editor, - }); - }, + const deterministicApi: NativeApi = { + ...wsNativeApi, + shell: { + ...wsNativeApi.shell, + openInEditor: async ( + cwd: Parameters[0], + editor: Parameters[1], + ) => { + wsRequests.push({ + _tag: WS_METHODS.shellOpenInEditor, + cwd, + editor, + }); }, - git: { - ...wsNativeApi.git, - createWorktree: async (input: Parameters[0]) => { - const request: WsRequestEnvelope["body"] = { - _tag: WS_METHODS.gitCreateWorktree, - ...input, - }; - wsRequests.push(request); - return resolveWsRpc(request) as Awaited< - ReturnType - >; - }, + }, + git: { + ...wsNativeApi.git, + createWorktree: async (input: Parameters[0]) => { + const request: WsRequestEnvelope["body"] = { + _tag: WS_METHODS.gitCreateWorktree, + ...input, + }; + wsRequests.push(request); + return resolveWsRpc(request) as Awaited>; }, - terminal: { - ...wsNativeApi.terminal, - open: async (input: Parameters[0]) => { - const request: WsRequestEnvelope["body"] = { - _tag: WS_METHODS.terminalOpen, - ...input, - }; - wsRequests.push(request); - return resolveWsRpc(request) as Awaited>; - }, - write: async (input: Parameters[0]) => { - wsRequests.push({ - _tag: WS_METHODS.terminalWrite, - ...input, - }); - }, + }, + terminal: { + ...wsNativeApi.terminal, + open: async (input: Parameters[0]) => { + const request: WsRequestEnvelope["body"] = { + _tag: WS_METHODS.terminalOpen, + ...input, + }; + wsRequests.push(request); + return resolveWsRpc(request) as Awaited>; }, - orchestration: { - ...wsNativeApi.orchestration, - dispatchCommand: async ( - command: Parameters[0], - ) => { - wsRequests.push({ - _tag: ORCHESTRATION_WS_METHODS.dispatchCommand, - command, - }); - const recordedFork = recordThreadForkCreateCommand(command); - return { - sequence: recordedFork - ? fixture.snapshot.snapshotSequence - : fixture.snapshot.snapshotSequence + 1, - }; - }, + write: async (input: Parameters[0]) => { + wsRequests.push({ + _tag: WS_METHODS.terminalWrite, + ...input, + }); }, - automation: { - ...wsNativeApi.automation, - create: async (input: Parameters[0]) => { - const request: WsRequestEnvelope["body"] = { - _tag: WS_METHODS.automationCreate, - ...input, - }; - wsRequests.push(request); - return resolveWsRpc(request) as Awaited>; - }, + }, + orchestration: { + ...wsNativeApi.orchestration, + dispatchCommand: async ( + command: Parameters[0], + ) => { + wsRequests.push({ + _tag: ORCHESTRATION_WS_METHODS.dispatchCommand, + command, + }); + const recordedFork = recordThreadForkCreateCommand(command); + return { + sequence: recordedFork + ? fixture.snapshot.snapshotSequence + : fixture.snapshot.snapshotSequence + 1, + }; }, }, + automation: { + ...wsNativeApi.automation, + create: async (input: Parameters[0]) => { + const request: WsRequestEnvelope["body"] = { + _tag: WS_METHODS.automationCreate, + ...input, + }; + wsRequests.push(request); + return resolveWsRpc(request) as Awaited>; + }, + }, + }; + + Object.defineProperty(window, "nativeApi", { + configurable: true, + value: configure ? configure(deterministicApi) : deterministicApi, }); return () => { @@ -1508,10 +1624,11 @@ async function waitForURL( return pathname; } -async function waitForComposerEditor(): Promise { +async function waitForComposerEditor(timeout = 8_000): Promise { return waitForElement( () => document.querySelector('[contenteditable="true"]'), "Unable to find composer editor.", + timeout, ); } @@ -1802,6 +1919,7 @@ async function mountChatView(options: { viewport: ViewportSpec; snapshot: OrchestrationReadModel; configureFixture?: (fixture: TestFixture) => void; + configureNativeApi?: (api: NativeApi) => NativeApi; initialEntry?: string; }): Promise { fixture = buildFixture(options.snapshot); @@ -1810,7 +1928,7 @@ async function mountChatView(options: { // transport-level contract. Record mutating native actions synchronously so // a slow Linux WebSocket round trip cannot outlive unmount and dispose the // next test's Effect runtime. - const restoreNativeApi = installDeterministicActionNativeApi(); + const restoreNativeApi = installDeterministicActionNativeApi(options.configureNativeApi); await setViewport(options.viewport); await waitForProductionStyles(); @@ -1938,6 +2056,102 @@ describe("ChatView timeline estimator parity (full app)", () => { document.body.innerHTML = ""; }); + it("keeps unavailable provider setup out of an empty chat until the user tries to send", async () => { + const snapshot = createSnapshotForTargetUser({ + targetMessageId: "msg-user-empty-provider-health" as MessageId, + targetText: "This message is removed to create an empty thread", + }); + const emptyThreadSnapshot: OrchestrationReadModel = { + ...snapshot, + threads: snapshot.threads.map((thread) => ({ + ...thread, + latestTurn: null, + messages: [], + })), + }; + + const mounted = await mountChatView({ + viewport: DEFAULT_VIEWPORT, + snapshot: emptyThreadSnapshot, + configureFixture: (nextFixture) => { + nextFixture.serverConfig = { + ...nextFixture.serverConfig, + providers: [ + { + provider: "codex", + status: "error", + available: false, + authStatus: "unauthenticated", + message: "Codex is not installed.", + checkedAt: NOW_ISO, + }, + ], + }; + }, + }); + + try { + await waitForComposerEditor(20_000); + expect(document.body.textContent).not.toContain("Codex provider status"); + expect(document.body.textContent).not.toContain("Codex is not installed."); + + useComposerDraftStore.getState().setPrompt(THREAD_ID, "Help me connect Codex"); + const sendButton = await waitForSendButton(); + await vi.waitFor(() => expect(sendButton.disabled).toBe(false)); + sendButton.click(); + + await vi.waitFor(() => { + expect(useProviderConnectionDialogStore.getState()).toMatchObject({ + isOpen: true, + provider: "codex", + source: "send", + }); + }); + } finally { + useProviderConnectionDialogStore.getState().setOpen(false); + await mounted.cleanup(); + } + }); + + it("still shows provider health when an existing conversation loses its provider", async () => { + const mounted = await mountChatView({ + viewport: DEFAULT_VIEWPORT, + snapshot: createSnapshotForTargetUser({ + targetMessageId: "msg-user-provider-health" as MessageId, + targetText: "Keep this existing conversation visible", + }), + configureFixture: (nextFixture) => { + nextFixture.serverConfig = { + ...nextFixture.serverConfig, + providers: [ + { + provider: "codex", + status: "error", + available: false, + authStatus: "unauthenticated", + message: "Codex is not installed.", + checkedAt: NOW_ISO, + }, + ], + }; + }, + }); + + try { + await waitForElement( + () => + Array.from(document.querySelectorAll("[data-slot='alert-title']")).find( + (element) => element.textContent === "Codex provider status", + ) ?? null, + "Unable to find provider health for the existing conversation.", + 20_000, + ); + expect(document.body.textContent).toContain("Codex is not installed."); + } finally { + await mounted.cleanup(); + } + }); + it("dispatches a bounded fork command from the message action and navigates to the new task", async () => { const sourceMessageId = MessageId.makeUnsafe("msg-user-message-fork-source"); const sourceSnapshot = createSnapshotForTargetUser({ @@ -3514,6 +3728,36 @@ describe("ChatView timeline estimator parity (full app)", () => { ); expect(getComputedStyle(stopButton).cursor).toBe("pointer"); + expect(document.querySelector('button[aria-label="Record voice note"]')).not.toBeNull(); + expect(document.querySelector('button[aria-label="Queue follow-up"]')).toBeNull(); + } finally { + await mounted.cleanup(); + } + }); + + it("keeps slash-command validation feedback inside the composer controls", async () => { + useComposerDraftStore.getState().setPrompt(THREAD_ID, "/review unsupported-target"); + const mounted = await mountChatView({ + viewport: DEFAULT_VIEWPORT, + snapshot: createSnapshotForTargetUser({ + targetMessageId: "msg-user-local-slash-feedback" as MessageId, + targetText: "local slash feedback target", + }), + }); + + try { + const composerForm = await waitForElement( + () => document.querySelector('form[data-chat-composer-form="true"]'), + "Unable to find composer form.", + ); + composerForm.requestSubmit(); + + const feedback = await waitForElement( + () => document.querySelector('[data-composer-local-feedback="true"]'), + "Unable to find composer-local slash feedback.", + ); + expect(feedback.textContent).toContain("Invalid /review command"); + expect(feedback.closest('[data-chat-composer-footer="true"]')).not.toBeNull(); } finally { await mounted.cleanup(); } @@ -3532,11 +3776,18 @@ describe("ChatView timeline estimator parity (full app)", () => { }); try { - const composerForm = await waitForElement( - () => document.querySelector('form[data-chat-composer-form="true"]'), - "Unable to find composer form.", - ); - composerForm.requestSubmit(); + const queueButton = await waitForElement( + () => document.querySelector('button[aria-label="Queue follow-up"]'), + "Unable to find the running-turn queue button.", + ); + expect(queueButton.title).toContain("Enter"); + expect(queueButton.title).toContain("Cmd/Ctrl+Enter"); + expect(document.querySelector('button[aria-label="Stop generation"]')).toBeNull(); + expect(document.querySelector('button[aria-label="Record voice note"]')).not.toBeNull(); + await mounted.setViewport(TEXT_VIEWPORT_MATRIX[3]); + expect(document.querySelector('button[aria-label="Queue follow-up"]')).not.toBeNull(); + expect(document.querySelector('button[aria-label="Record voice note"]')).not.toBeNull(); + document.querySelector('button[aria-label="Queue follow-up"]')?.click(); await vi.waitFor( () => { @@ -3557,6 +3808,219 @@ describe("ChatView timeline estimator parity (full app)", () => { "Unable to find stop generation button.", ); expect(stopButton).not.toBeNull(); + expect(document.querySelector('button[aria-label="Record voice note"]')).not.toBeNull(); + expect(hasDispatchedCommandType("thread.turn.interrupt")).toBe(false); + } finally { + await mounted.cleanup(); + } + }); + + it("keeps active recorder controls usable when an approval arrives", async () => { + const microphone = installFakeMicrophoneCapture(); + const mounted = await mountChatView({ + viewport: DEFAULT_VIEWPORT, + snapshot: createSnapshotForTargetUser({ + targetMessageId: "msg-user-voice-approval-transition" as MessageId, + targetText: "voice approval transition target", + sessionStatus: "running", + }), + configureNativeApi: configureSuccessfulVoiceTranscription("approval-safe transcript"), + }); + + try { + const recordButton = await waitForElement( + () => document.querySelector('button[aria-label="Record voice note"]'), + "Unable to find voice recording button.", + ); + recordButton.click(); + await waitForElement( + () => + document.querySelector('button[aria-label="Cancel voice recording"]'), + "Voice recorder did not start.", + ); + microphone.emitSamples(); + + appendActiveThreadActivity({ + id: EventId.makeUnsafe("approval-during-voice"), + createdAt: isoAt(1_200), + kind: "approval.requested", + summary: "Command approval requested", + tone: "approval", + turnId: null, + payload: { + requestId: "approval-during-voice", + requestKind: "command", + detail: "bun run test", + }, + }); + + await vi.waitFor( + () => { + expect(document.body.textContent).toContain("bun run test"); + expect( + document.querySelector( + 'button[aria-label="Cancel voice recording"]', + ), + ).not.toBeNull(); + }, + { timeout: 8_000, interval: 16 }, + ); + + const insertButton = document.querySelector( + 'button[aria-label="Stop and insert voice note"]', + ); + const sendButton = document.querySelector( + 'button[aria-label="Send voice note"]', + ); + expect(insertButton?.disabled).toBe(false); + expect(sendButton?.disabled).toBe(false); + + document + .querySelector('button[aria-label="Cancel voice recording"]') + ?.click(); + await vi.waitFor(() => { + expect(document.querySelector('[data-chat-composer-footer="true"]')).toBeNull(); + }); + } finally { + await mounted.cleanup(); + microphone.restore(); + } + }); + + it("queues voice Send instead of answering a question that arrives mid-recording", async () => { + const microphone = installFakeMicrophoneCapture(); + useComposerDraftStore.getState().setPrompt(THREAD_ID, "existing draft"); + const mounted = await mountChatView({ + viewport: DEFAULT_VIEWPORT, + snapshot: createSnapshotForTargetUser({ + targetMessageId: "msg-user-voice-question-transition" as MessageId, + targetText: "voice question transition target", + sessionStatus: "running", + }), + configureNativeApi: configureSuccessfulVoiceTranscription("spoken follow-up"), + }); + + try { + const recordButton = await waitForElement( + () => document.querySelector('button[aria-label="Record voice note"]'), + "Unable to find voice recording button.", + ); + recordButton.click(); + await waitForElement( + () => + document.querySelector('button[aria-label="Cancel voice recording"]'), + "Voice recorder did not start.", + ); + microphone.emitSamples(); + await new Promise((resolve) => window.setTimeout(resolve, 300)); + + appendActiveThreadActivity({ + id: EventId.makeUnsafe("question-during-voice"), + createdAt: isoAt(1_210), + kind: "user-input.requested", + summary: "User input requested", + tone: "info", + turnId: null, + payload: { + requestId: "question-during-voice", + questions: [ + { + id: "release_choice", + header: "Release", + question: "Which release path should be used?", + options: [ + { + label: "safe", + description: "Use the safe release path", + }, + ], + }, + ], + }, + }); + + await vi.waitFor( + () => { + expect(document.body.textContent).toContain("Which release path should be used?"); + expect( + document.querySelector('button[aria-label="Send voice note"]'), + ).not.toBeNull(); + }, + { timeout: 8_000, interval: 16 }, + ); + + document.querySelector('button[aria-label="Send voice note"]')?.click(); + + await vi.waitFor( + () => { + const queuedRow = document.querySelector( + '[data-testid="queued-follow-up-row"]', + ); + const queuedTurn = + useComposerDraftStore.getState().draftsByThreadId[THREAD_ID]?.queuedTurns[0]; + expect(queuedRow?.textContent).toContain("existing draft"); + expect(queuedTurn?.kind).toBe("chat"); + expect(queuedTurn?.kind === "chat" ? queuedTurn.prompt : null).toBe( + "existing draft\nspoken follow-up", + ); + }, + { timeout: 8_000, interval: 16 }, + ); + expect( + wsRequests.some( + (request) => + request._tag === ORCHESTRATION_WS_METHODS.dispatchCommand && + typeof request.command === "object" && + request.command !== null && + "type" in request.command && + request.command.type === "thread.user-input.respond", + ), + ).toBe(false); + expect(document.body.textContent).toContain("Which release path should be used?"); + } finally { + await mounted.cleanup(); + microphone.restore(); + } + }); + + it("keeps Cmd/Ctrl+Enter as the immediate steering shortcut while a turn is running", async () => { + useComposerDraftStore.getState().setPrompt(THREAD_ID, "steer this follow-up now"); + + const mounted = await mountChatView({ + viewport: DEFAULT_VIEWPORT, + snapshot: createSnapshotForTargetUser({ + targetMessageId: "msg-user-running-steer-shortcut" as MessageId, + targetText: "running steer shortcut target", + sessionStatus: "running", + }), + }); + + try { + const composerEditor = await waitForComposerEditor(); + const useMetaForMod = isMacPlatform(navigator.platform); + composerEditor.dispatchEvent( + new KeyboardEvent("keydown", { + key: "Enter", + metaKey: useMetaForMod, + ctrlKey: !useMetaForMod, + bubbles: true, + cancelable: true, + }), + ); + + await vi.waitFor( + () => { + const steeredTurn = wsRequests + .map((request) => readDispatchedCommand(request)) + .find( + (command) => + command?.type === "thread.turn.start" && command.dispatchMode === "steer", + ); + expect(steeredTurn).toBeTruthy(); + }, + { timeout: 8_000, interval: 16 }, + ); + expect(document.querySelector('[data-testid="queued-follow-up-row"]')).toBeNull(); } finally { await mounted.cleanup(); } @@ -4042,6 +4506,15 @@ describe("ChatView timeline estimator parity (full app)", () => { }); it("coalesces repeated Studio new-chat clicks and stays in Studio after navigation settles", async () => { + // Studio is hidden by default; this Studio-specific regression test opts in explicitly. + localStorage.setItem( + "scient:app-settings:v1", + JSON.stringify({ + appSettingsVersion: CURRENT_APP_SETTINGS_VERSION, + showStudioSection: true, + }), + ); + useComposerDraftStore.setState({ draftThreadsByThreadId: { [STUDIO_DRAFT_THREAD_ID]: { diff --git a/apps/web/src/components/ChatView.logic.test.ts b/apps/web/src/components/ChatView.logic.test.ts index f76a960ae..0ef5c78a0 100644 --- a/apps/web/src/components/ChatView.logic.test.ts +++ b/apps/web/src/components/ChatView.logic.test.ts @@ -3,9 +3,11 @@ import { describe, expect, it } from "vitest"; import { appendVoiceTranscriptToPrompt, + completeComposerVoiceTranscript, buildComposerMenuSelectionKey, createLocalDispatchSnapshot, createWorktreeSetupSnapshot, + deriveComposerFooterActionPlan, derivePromptHistoryFromMessages, failWorktreeSetupSnapshot, filterSidechatTranscriptMessages, @@ -42,6 +44,8 @@ import { shouldEnableComposerPastedTextCollapse, shouldHandlePromptHistoryNavigationKey, shouldRenderProviderHealthBanner, + shouldRenderComposerFooter, + shouldRouteComposerSendToPendingInput, shouldShowComposerModelBootstrapSkeleton, shouldStartActiveTurnLayoutGrace, shouldRenderTerminalWorkspace, @@ -631,6 +635,94 @@ describe("voice helpers", () => { expect(appendVoiceTranscriptToPrompt("Hello", " ")).toBeNull(); }); + it("inserts a completed voice transcript without sending", async () => { + const inserted: string[] = []; + const sent: string[] = []; + + await expect( + completeComposerVoiceTranscript({ + intent: "insert", + currentPrompt: "Existing draft", + transcript: "Voice text", + insertTranscript: (transcript) => { + inserted.push(transcript); + return true; + }, + sendPrompt: async (prompt) => { + sent.push(prompt); + return true; + }, + }), + ).resolves.toBe("inserted"); + expect(inserted).toEqual(["Voice text"]); + expect(sent).toEqual([]); + }); + + it("sends the existing draft and transcript as one completed voice message", async () => { + const inserted: string[] = []; + const sent: string[] = []; + + await expect( + completeComposerVoiceTranscript({ + intent: "send", + currentPrompt: "Existing draft ", + transcript: " Voice text ", + insertTranscript: (transcript) => { + inserted.push(transcript); + return true; + }, + sendPrompt: async (prompt) => { + sent.push(prompt); + return true; + }, + }), + ).resolves.toBe("sent"); + expect(sent).toEqual(["Existing draft\nVoice text"]); + expect(inserted).toEqual([]); + }); + + it("keeps the transcript in the composer when direct send cannot start", async () => { + const inserted: string[] = []; + + await expect( + completeComposerVoiceTranscript({ + intent: "send", + currentPrompt: "Existing draft", + transcript: "Voice text", + insertTranscript: (transcript) => { + inserted.push(transcript); + return true; + }, + sendPrompt: async () => false, + }), + ).resolves.toBe("inserted"); + expect(inserted).toEqual(["Voice text"]); + }); + + it("does not duplicate a transcript already restored by the send pipeline", async () => { + let composerPrompt = "Existing draft"; + + await expect( + completeComposerVoiceTranscript({ + intent: "send", + currentPrompt: composerPrompt, + transcript: "Voice text", + insertTranscript: (_transcript, completedPrompt) => { + if (composerPrompt === completedPrompt) { + return false; + } + composerPrompt = completedPrompt; + return true; + }, + sendPrompt: async (completedPrompt) => { + composerPrompt = completedPrompt; + return false; + }, + }), + ).resolves.toBe("preserved"); + expect(composerPrompt).toBe("Existing draft\nVoice text"); + }); + it("sanitizes inline stack traces from voice errors", () => { expect( sanitizeVoiceErrorMessage( @@ -685,6 +777,20 @@ describe("voice helpers", () => { canStartVoiceNotes: false, showVoiceNotesControl: true, }); + + expect( + deriveComposerVoiceState({ + authStatus: "unauthenticated", + voiceTranscriptionAvailable: false, + desktopVoiceAvailable: true, + isRecording: false, + isTranscribing: false, + }), + ).toEqual({ + canRenderVoiceNotes: true, + canStartVoiceNotes: true, + showVoiceNotesControl: true, + }); }); }); @@ -1316,6 +1422,149 @@ describe("deriveComposerSendState", () => { }); }); +describe("deriveComposerFooterActionPlan", () => { + const baseOptions = { + hasLiveTurn: false, + hasSendableContent: false, + hasActivePendingProgress: false, + hasPendingApproval: false, + hasPendingUserInput: false, + isVoiceActive: false, + showPlanFollowUpPrompt: false, + canShowVoiceNotes: true, + }; + + it("keeps Stop and the microphone available for an empty active-turn composer", () => { + expect( + deriveComposerFooterActionPlan({ + ...baseOptions, + hasLiveTurn: true, + }), + ).toEqual({ primaryAction: "stop-generation", showVoiceButton: true }); + }); + + it("morphs Stop into Queue when active-turn content becomes sendable", () => { + expect( + deriveComposerFooterActionPlan({ + ...baseOptions, + hasLiveTurn: true, + hasSendableContent: true, + }), + ).toEqual({ primaryAction: "queue-message", showVoiceButton: true }); + }); + + it("morphs to Queue after a finalized voice transcript is inserted", () => { + const prompt = appendVoiceTranscriptToPrompt("", "voice follow-up") ?? ""; + const sendState = deriveComposerSendState({ + prompt, + imageCount: 0, + fileCount: 0, + assistantSelectionCount: 0, + fileCommentCount: 0, + terminalContexts: [], + pastedTexts: [], + }); + + expect( + deriveComposerFooterActionPlan({ + ...baseOptions, + hasLiveTurn: true, + hasSendableContent: sendState.hasSendableContent, + }), + ).toEqual({ primaryAction: "queue-message", showVoiceButton: true }); + }); + + it("leaves the dedicated recorder controls exclusive during active-turn voice capture", () => { + expect( + deriveComposerFooterActionPlan({ + ...baseOptions, + hasLiveTurn: true, + hasSendableContent: true, + isVoiceActive: true, + }), + ).toEqual({ primaryAction: "none", showVoiceButton: false }); + }); + + it("keeps an active recorder exclusive when a question or approval arrives", () => { + expect( + deriveComposerFooterActionPlan({ + ...baseOptions, + hasLiveTurn: true, + hasActivePendingProgress: true, + isVoiceActive: true, + }), + ).toEqual({ primaryAction: "none", showVoiceButton: false }); + expect( + deriveComposerFooterActionPlan({ + ...baseOptions, + hasLiveTurn: true, + hasPendingApproval: true, + isVoiceActive: true, + }), + ).toEqual({ primaryAction: "none", showVoiceButton: false }); + }); + + it("keeps approval, pending-input, and plan-follow-up controls exclusive", () => { + expect( + deriveComposerFooterActionPlan({ + ...baseOptions, + hasLiveTurn: true, + hasSendableContent: true, + hasActivePendingProgress: true, + }), + ).toEqual({ primaryAction: "pending-input", showVoiceButton: false }); + expect( + deriveComposerFooterActionPlan({ + ...baseOptions, + hasLiveTurn: true, + hasSendableContent: true, + hasPendingApproval: true, + }), + ).toEqual({ primaryAction: "none", showVoiceButton: false }); + expect( + deriveComposerFooterActionPlan({ + ...baseOptions, + showPlanFollowUpPrompt: true, + }), + ).toEqual({ primaryAction: "plan-follow-up", showVoiceButton: false }); + }); +}); + +describe("composer voice transition invariants", () => { + it("routes ordinary pending-question submits to the question flow", () => { + expect( + shouldRouteComposerSendToPendingInput({ + hasActivePendingProgress: true, + hasVoicePromptOverride: false, + }), + ).toBe(true); + }); + + it("keeps a completed voice prompt on the message path when a question arrives", () => { + expect( + shouldRouteComposerSendToPendingInput({ + hasActivePendingProgress: true, + hasVoicePromptOverride: true, + }), + ).toBe(false); + }); + + it("keeps the footer mounted for active voice and hides it after voice settles", () => { + expect( + shouldRenderComposerFooter({ + hasPendingApproval: true, + isVoiceActive: true, + }), + ).toBe(true); + expect( + shouldRenderComposerFooter({ + hasPendingApproval: true, + isVoiceActive: false, + }), + ).toBe(false); + }); +}); + describe("buildExpiredTerminalContextToastCopy", () => { it("formats clear empty-state guidance", () => { expect(buildExpiredTerminalContextToastCopy(1, "empty")).toEqual({ @@ -1416,11 +1665,22 @@ describe("resolveProjectScriptTerminalTarget", () => { }); describe("shouldRenderProviderHealthBanner", () => { + it("does not show provider setup health on a new empty chat", () => { + expect( + shouldRenderProviderHealthBanner({ + threadEntryPoint: "chat", + terminalWorkspaceTerminalTabActive: false, + hasConversationActivity: false, + }), + ).toBe(false); + }); + it("does not show chat provider health while a terminal thread is active", () => { expect( shouldRenderProviderHealthBanner({ threadEntryPoint: "terminal", terminalWorkspaceTerminalTabActive: false, + hasConversationActivity: true, }), ).toBe(false); }); @@ -1430,15 +1690,17 @@ describe("shouldRenderProviderHealthBanner", () => { shouldRenderProviderHealthBanner({ threadEntryPoint: "chat", terminalWorkspaceTerminalTabActive: true, + hasConversationActivity: true, }), ).toBe(false); }); - it("shows chat provider health only on the chat surface", () => { + it("shows provider health for a started conversation on the chat surface", () => { expect( shouldRenderProviderHealthBanner({ threadEntryPoint: "chat", terminalWorkspaceTerminalTabActive: false, + hasConversationActivity: true, }), ).toBe(true); }); diff --git a/apps/web/src/components/ChatView.logic.ts b/apps/web/src/components/ChatView.logic.ts index a30e99483..997d10514 100644 --- a/apps/web/src/components/ChatView.logic.ts +++ b/apps/web/src/components/ChatView.logic.ts @@ -43,6 +43,7 @@ import { } from "../session-logic"; import { localSubagentThreadId } from "./ChatView.selectors"; import type { ProviderModelOption } from "../providerModelOptions"; +import type { ComposerVoiceCompletionIntent } from "./chat/composerVoiceState"; export const LAST_INVOKED_SCRIPT_BY_PROJECT_KEY = "scient:last-invoked-script-by-project"; export const DISMISSED_PROVIDER_HEALTH_BANNERS_KEY = "scient:dismissed-provider-health-banners"; @@ -121,8 +122,13 @@ export function resolveRuntimeModeAfterApprovalDecision( export function shouldRenderProviderHealthBanner(input: { threadEntryPoint: ThreadPrimarySurface; terminalWorkspaceTerminalTabActive: boolean; + hasConversationActivity: boolean; }): boolean { - return input.threadEntryPoint === "chat" && !input.terminalWorkspaceTerminalTabActive; + return ( + input.hasConversationActivity && + input.threadEntryPoint === "chat" && + !input.terminalWorkspaceTerminalTabActive + ); } // Big-paste cards are sent only by the normal chat path; non-chat composer flows @@ -624,6 +630,23 @@ export function appendVoiceTranscriptToPrompt( : `${currentPrompt.replace(/\s+$/, "")}\n${trimmedTranscript}`; } +export async function completeComposerVoiceTranscript(input: { + intent: ComposerVoiceCompletionIntent; + currentPrompt: string; + transcript: string; + insertTranscript: (transcript: string, completedPrompt: string) => boolean; + sendPrompt: (prompt: string) => Promise; +}): Promise<"empty" | "inserted" | "preserved" | "sent"> { + const nextPrompt = appendVoiceTranscriptToPrompt(input.currentPrompt, input.transcript); + if (!nextPrompt) { + return "empty"; + } + if (input.intent === "send" && (await input.sendPrompt(nextPrompt))) { + return "sent"; + } + return input.insertTranscript(input.transcript, nextPrompt) ? "inserted" : "preserved"; +} + export function sanitizeVoiceErrorMessage(message: string): string { const normalized = message.trim(); if (normalized.length === 0) { @@ -678,6 +701,7 @@ export function describeVoiceRecordingStartError(error: unknown): string { export function deriveComposerVoiceState(input: { authStatus: ServerProviderAuthStatus | null | undefined; voiceTranscriptionAvailable: boolean | undefined; + desktopVoiceAvailable?: boolean; isRecording: boolean; isTranscribing: boolean; }): { @@ -685,8 +709,11 @@ export function deriveComposerVoiceState(input: { canStartVoiceNotes: boolean; showVoiceNotesControl: boolean; } { - const canRenderVoiceNotes = input.authStatus !== "unauthenticated"; - const canStartVoiceNotes = canRenderVoiceNotes && input.voiceTranscriptionAvailable !== false; + const canRenderVoiceNotes = + input.desktopVoiceAvailable === true || input.authStatus !== "unauthenticated"; + const canStartVoiceNotes = + input.desktopVoiceAvailable === true || + (canRenderVoiceNotes && input.voiceTranscriptionAvailable !== false); return { canRenderVoiceNotes, @@ -1057,6 +1084,78 @@ export function deriveComposerSendState(options: { }; } +export type ComposerFooterPrimaryAction = + | "none" + | "pending-input" + | "plan-follow-up" + | "stop-generation" + | "queue-message" + | "send-message"; + +export function deriveComposerFooterActionPlan(options: { + hasLiveTurn: boolean; + hasSendableContent: boolean; + hasActivePendingProgress: boolean; + hasPendingApproval: boolean; + hasPendingUserInput: boolean; + isVoiceActive: boolean; + showPlanFollowUpPrompt: boolean; + canShowVoiceNotes: boolean; +}): { + primaryAction: ComposerFooterPrimaryAction; + showVoiceButton: boolean; +} { + if (options.isVoiceActive) { + return { primaryAction: "none", showVoiceButton: false }; + } + + if (options.hasActivePendingProgress) { + return { primaryAction: "pending-input", showVoiceButton: false }; + } + + if (options.hasPendingApproval) { + return { primaryAction: "none", showVoiceButton: false }; + } + + if (options.hasLiveTurn) { + const composerCanAcceptActions = !options.hasPendingUserInput; + return { + primaryAction: + composerCanAcceptActions && options.hasSendableContent + ? "queue-message" + : "stop-generation", + showVoiceButton: composerCanAcceptActions && options.canShowVoiceNotes, + }; + } + + if (options.hasPendingUserInput) { + return { primaryAction: "none", showVoiceButton: false }; + } + + if (options.showPlanFollowUpPrompt) { + return { primaryAction: "plan-follow-up", showVoiceButton: false }; + } + + return { + primaryAction: "send-message", + showVoiceButton: options.canShowVoiceNotes, + }; +} + +export function shouldRouteComposerSendToPendingInput(options: { + hasActivePendingProgress: boolean; + hasVoicePromptOverride: boolean; +}): boolean { + return options.hasActivePendingProgress && !options.hasVoicePromptOverride; +} + +export function shouldRenderComposerFooter(options: { + hasPendingApproval: boolean; + isVoiceActive: boolean; +}): boolean { + return !options.hasPendingApproval || options.isVoiceActive; +} + export function collectUserMessageAssistantSelections( message: ChatMessage, ): ChatAssistantSelectionAttachment[] { diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 7686f000f..c76845b03 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -123,6 +123,7 @@ import { saveConfirmedCustomBinaryPaths, } from "../confirmedCustomBinaryPathStore"; import { isElectron } from "../env"; +import { ensureDesktopVoiceReady, hasDesktopVoiceRuntime } from "../lib/desktopVoiceSetup"; import { stripDiffSearchParams } from "../diffRouteSearch"; import { resolveSubagentPresentationForThread } from "../lib/subagentPresentation"; import { ensureHomeChatProject, isHomeChatContainerProject } from "../lib/chatProjects"; @@ -299,7 +300,8 @@ import { Skeleton } from "./ui/skeleton"; import { Menu, MenuItem, MenuTrigger } from "./ui/menu"; import { disposeAndCloseTerminalSession, randomTerminalId } from "./terminal/terminalSession"; import { cn, isMacPlatform, randomUUID } from "~/lib/utils"; -import { toastManager } from "./ui/toast"; +import { activityManager } from "../notifications/activityStore"; +import { transientAlertManager } from "../notifications/transientAlert"; import { decodeProjectScriptKeybindingRule } from "~/lib/projectScriptKeybindings"; import { type NewProjectScriptInput } from "./ProjectScriptsControl"; import { @@ -384,6 +386,7 @@ import { deriveLatestContextWindowSnapshot, deriveSelectedContextWindowSnapshot, } from "../lib/contextWindow"; +import { LiveVoicePreviewSession } from "../lib/liveVoicePreview"; import { formatVoiceRecordingDuration, useVoiceRecorder } from "../lib/voiceRecorder"; import { composerFooterPlanForTier, @@ -469,11 +472,16 @@ import { ComposerInputBanners } from "./chat/ComposerInputBanners"; import { ComposerPendingUserInputPanel } from "./chat/ComposerPendingUserInputPanel"; import { ComposerVoiceButton } from "./chat/ComposerVoiceButton"; import { ComposerVoiceRecorderBar } from "./chat/ComposerVoiceRecorderBar"; +import type { ComposerVoiceCompletionIntent } from "./chat/composerVoiceState"; import { ComposerReferenceAttachments } from "./chat/ComposerReferenceAttachments"; import { TranscriptSelectionActionLayer } from "./chat/TranscriptSelectionActionLayer"; import { ComposerActiveTaskListCard } from "./chat/ComposerActiveTaskListCard"; import { ComposerColumnFrame } from "./chat/ComposerColumnFrame"; import { useTranscriptAssistantSelectionAction } from "./chat/useTranscriptAssistantSelectionAction"; +import type { + ComposerLocalFeedback, + ReportComposerLocalFeedback, +} from "./chat/useComposerVoiceController"; import { resolveTranscriptMarkerRange } from "./chat/chatSelectionActions"; import { dispatchThreadMarkerAdd, @@ -510,6 +518,7 @@ import { import { ACTIVE_TURN_LAYOUT_SETTLE_DELAY_MS, appendVoiceTranscriptToPrompt, + completeComposerVoiceTranscript, describeVoiceRecordingStartError, isVoiceAuthExpiredMessage, sanitizeVoiceErrorMessage, @@ -521,9 +530,12 @@ import { DismissedProviderHealthBannersSchema, shouldRenderTerminalWorkspace, collectUserMessageBlobPreviewUrls, + deriveComposerFooterActionPlan, deriveComposerSendState, failWorktreeSetupSnapshot, filterSidechatTranscriptMessages, + shouldRenderComposerFooter, + shouldRouteComposerSendToPendingInput, hasServerAcknowledgedLocalDispatch, resolveNextLocalDispatchSnapshot, WORKTREE_SETUP_ERROR_HOLD_MS, @@ -1155,10 +1167,47 @@ export default function ChatView({ durationMs: voiceRecordingDurationMs, waveformLevels: voiceWaveformLevels, startRecording: startVoiceRecording, + snapshotRecording: snapshotVoiceRecording, stopRecording: stopVoiceRecording, cancelRecording: cancelVoiceRecording, } = useVoiceRecorder(); - const [isVoiceTranscribing, setIsVoiceTranscribing] = useState(false); + const [voiceCompletionIntent, setVoiceCompletionIntent] = + useState(null); + const [liveVoicePreview, setLiveVoicePreview] = useState(null); + const liveVoicePreviewSessionRef = useRef(null); + if (liveVoicePreviewSessionRef.current === null) { + liveVoicePreviewSessionRef.current = new LiveVoicePreviewSession(); + } + const isVoiceTranscribing = voiceCompletionIntent !== null; + const isVoiceActive = isVoiceRecording || isVoiceTranscribing; + const sendVoiceTranscriptRef = useRef<(prompt: string) => Promise>(async () => false); + const composerFeedbackIdRef = useRef(0); + const [composerLocalFeedback, setComposerLocalFeedback] = useState< + (ComposerLocalFeedback & { id: number }) | null + >(null); + const reportComposerFeedback = useCallback((feedback) => { + composerFeedbackIdRef.current += 1; + setComposerLocalFeedback({ ...feedback, id: composerFeedbackIdRef.current }); + }, []); + useEffect(() => { + if (!composerLocalFeedback) { + return; + } + const timeout = window.setTimeout( + () => setComposerLocalFeedback(null), + composerLocalFeedback.type === "success" ? 4_000 : 8_000, + ); + return () => window.clearTimeout(timeout); + }, [composerLocalFeedback]); + useEffect(() => { + setComposerLocalFeedback(null); + }, [threadId]); + useEffect( + () => () => { + void liveVoicePreviewSessionRef.current?.stop(); + }, + [], + ); const composerSendState = useMemo( () => deriveComposerSendState({ @@ -2049,6 +2098,9 @@ export default function ChatView({ activeThread.messages.length > 0 || activeThread.session !== null), ); + const hasConversationActivity = Boolean( + activeThread && (activeThread.latestTurn !== null || activeThread.messages.length > 0), + ); const lockedProvider: ProviderKind | null = hasThreadStarted ? (sessionProvider ?? threadProvider ?? selectedProviderByThreadId ?? null) : null; @@ -2061,6 +2113,7 @@ export default function ChatView({ provider: ProviderKind; } | null>(null); const voiceTranscriptionRequestIdRef = useRef(0); + const voiceCompletionInFlightRef = useRef(false); const voiceThreadIdRef = useRef(threadId); const voiceProviderRef = useRef(selectedProvider); const voiceRecordingStartedAtRef = useRef(null); @@ -2859,7 +2912,11 @@ export default function ChatView({ activeThreadId === null ? null : `${activeThreadId}:${activeLatestTurn?.turnId ?? "idle"}`; const activeTurnInProgress = activeTurnLayoutLive || keepSettledActiveTurnLayout; const isComposerApprovalState = activePendingApproval !== null; - const isComposerEditorDisabled = isConnecting || isComposerApprovalState; + const liveVoicePreviewPrompt = liveVoicePreview + ? (appendVoiceTranscriptToPrompt(prompt, liveVoicePreview) ?? prompt) + : null; + const isComposerEditorDisabled = + isConnecting || isComposerApprovalState || isVoiceActive || liveVoicePreview !== null; const canCollapsePastedTextToDraft = shouldEnableComposerPastedTextCollapse({ isComposerApprovalState, hasPendingUserInput: pendingUserInputs.length > 0, @@ -2915,8 +2972,26 @@ export default function ChatView({ }, [activeLatestTurn?.startedAt, activeTurnLayoutKey, activeTurnLayoutLive]); useEffect(() => { + // A provider question can arrive while microphone capture is already active. + // Keep promptRef anchored to the untouched chat draft until voice settles; + // otherwise the question's empty custom answer drops that draft from Send. + if (isVoiceActive) { + return; + } const nextCustomAnswer = activePendingProgress?.customAnswer; if (typeof nextCustomAnswer !== "string") { + if (lastSyncedPendingInputRef.current !== null) { + // The question temporarily borrowed promptRef without replacing the + // persisted chat draft. Restore that draft when the question settles + // so the next voice note or keyboard send cannot reuse a stale answer. + promptRef.current = prompt; + const nextCursor = collapseExpandedComposerCursor(prompt, prompt.length); + setComposerCursor(nextCursor); + setComposerTrigger( + detectComposerTrigger(prompt, expandCollapsedComposerCursor(prompt, nextCursor)), + ); + setComposerHighlightedItemId(null); + } lastSyncedPendingInputRef.current = null; return; } @@ -2950,6 +3025,8 @@ export default function ChatView({ activePendingProgress?.customAnswer, activePendingUserInput?.requestId, activePendingProgress?.activeQuestion?.id, + isVoiceActive, + prompt, ]); useEffect(() => { attachmentPreviewHandoffByMessageIdRef.current = attachmentPreviewHandoffByMessageId; @@ -3156,6 +3233,7 @@ export default function ChatView({ }; }, [markerMessageIds, pinnedMessageIds, timelineMessages]); const { + pinLimitMessageId, handleTogglePinMessage, handleTogglePinnedMessageDone, handleUnpinMessage, @@ -3183,16 +3261,9 @@ export default function ChatView({ if (nextNotes === threadNotes) { return; } - void handleNotesChange(activeThreadId, nextNotes) - .then(() => { - toastManager.add({ - type: "success", - title: "Project instructions added to notepad.", - }); - }) - .catch(() => { - // `handleNotesChange` already surfaces the save failure through the shared notes toast. - }); + void handleNotesChange(activeThreadId, nextNotes).catch(() => { + // `handleNotesChange` already records the save failure in Activity. + }); }, [activeThreadId, handleNotesChange, projectInstructions, threadNotes]); const handleJumpToPinnedMessage = useCallback((messageId: MessageId) => { timelineControllerRef.current?.scrollToMessage(messageId); @@ -3207,7 +3278,7 @@ export default function ChatView({ } void dispatchThreadMarkerRemove(activeThreadId, markerId).catch((error) => { console.error("Failed to remove thread marker", error); - toastManager.add({ + transientAlertManager.add({ type: "error", title: "Could not remove marker.", }); @@ -3226,7 +3297,7 @@ export default function ChatView({ } void dispatchThreadMarkerDoneSet(activeThreadId, markerId, !marker.done).catch((error) => { console.error("Failed to update thread marker", error); - toastManager.add({ + transientAlertManager.add({ type: "error", title: "Could not update marker.", }); @@ -3241,7 +3312,7 @@ export default function ChatView({ } void dispatchThreadMarkerLabelSet(activeThreadId, markerId, label).catch((error) => { console.error("Failed to rename thread marker", error); - toastManager.add({ + transientAlertManager.add({ type: "error", title: "Could not rename marker.", }); @@ -3743,16 +3814,34 @@ export default function ChatView({ () => findProviderStatus(providerStatuses, "codex"), [providerStatuses], ); + const desktopVoiceAvailable = hasDesktopVoiceRuntime(); const refreshProviderStatuses = useRefreshProviderStatusesNow(); const voiceRecordingDurationLabel = useMemo( () => formatVoiceRecordingDuration(voiceRecordingDurationMs), [voiceRecordingDurationMs], ); - const canRenderVoiceNotes = voiceProviderStatus?.authStatus !== "unauthenticated"; + const canRenderVoiceNotes = + desktopVoiceAvailable || voiceProviderStatus?.authStatus !== "unauthenticated"; const canStartVoiceNotes = - voiceProviderStatus?.authStatus !== "unauthenticated" && - voiceProviderStatus?.voiceTranscriptionAvailable !== false; + desktopVoiceAvailable || + (voiceProviderStatus?.authStatus !== "unauthenticated" && + voiceProviderStatus?.voiceTranscriptionAvailable !== false); const showVoiceNotesControl = canRenderVoiceNotes || isVoiceRecording || isVoiceTranscribing; + const composerFooterActionPlan = deriveComposerFooterActionPlan({ + hasLiveTurn, + hasSendableContent: composerSendState.hasSendableContent, + hasActivePendingProgress: activePendingProgress !== null, + hasPendingApproval: isComposerApprovalState, + hasPendingUserInput: pendingUserInputs.length > 0, + isVoiceActive, + showPlanFollowUpPrompt, + canShowVoiceNotes: showVoiceNotesControl, + }); + const composerSubmitIsQueue = composerFooterActionPlan.primaryAction === "queue-message"; + const composerSubmitLabel = composerSubmitIsQueue ? "Queue follow-up" : "Send message"; + const composerSubmitTitle = composerSubmitIsQueue + ? "Queue follow-up (Enter). Press Cmd/Ctrl+Enter to steer the current response instead." + : undefined; const activeProjectCwd = activeProject?.cwd ?? null; const activeThreadWorktreePath = activeThread?.worktreePath ?? null; const hasNativeUserMessages = useMemo( @@ -3871,7 +3960,7 @@ export default function ChatView({ (url: string) => { const api = readNativeApi(); void api?.browser.open({ threadId, initialUrl: url }).catch((error) => { - toastManager.add({ + transientAlertManager.add({ type: "error", title: "Could not open repository", description: @@ -3925,6 +4014,7 @@ export default function ChatView({ const shouldShowProviderHealthBanner = shouldRenderProviderHealthBanner({ threadEntryPoint: terminalState.entryPoint, terminalWorkspaceTerminalTabActive, + hasConversationActivity, }); // Terminal-only threads should not pay to mount the hidden chat/composer pane. const shouldRenderChatPaneContent = !( @@ -4800,8 +4890,6 @@ export default function ChatView({ if (!activeProject) return; const nextScripts = activeProject.scripts.filter((script) => script.id !== scriptId); - const deletedName = activeProject.scripts.find((s) => s.id === scriptId)?.name; - try { await persistProjectScripts({ projectId: activeProject.id, @@ -4811,12 +4899,8 @@ export default function ChatView({ keybinding: null, keybindingCommand: commandForProjectScript(scriptId), }); - toastManager.add({ - type: "success", - title: `Deleted action "${deletedName ?? "Unknown"}"`, - }); } catch (error) { - toastManager.add({ + transientAlertManager.add({ type: "error", title: "Could not delete action", description: error instanceof Error ? error.message : "An unexpected error occurred.", @@ -4849,7 +4933,7 @@ export default function ChatView({ setPendingServerRuntimeMode((pending) => pending?.threadId === threadId && pending.mode === mode ? null : pending, ); - toastManager.add({ + reportComposerFeedback({ type: "error", title: "Could not update access mode", description: @@ -4862,6 +4946,7 @@ export default function ChatView({ }, [ isLocalDraftThread, + reportComposerFeedback, runtimeMode, scheduleComposerFocus, serverThread, @@ -4890,7 +4975,7 @@ export default function ChatView({ createdAt: new Date().toISOString(), }) .catch((error) => { - toastManager.add({ + reportComposerFeedback({ type: "error", title: "Could not update plan mode", description: @@ -4904,6 +4989,7 @@ export default function ChatView({ [ interactionMode, isLocalDraftThread, + reportComposerFeedback, scheduleComposerFocus, serverThread, setComposerDraftInteractionMode, @@ -5172,6 +5258,7 @@ export default function ChatView({ addComposerAssistantSelectionToDraft, canReferenceAssistantSelection: (selection) => !isPendingSetupBubbleId(MessageId.makeUnsafe(selection.assistantMessageId)), + reportComposerFeedback, scheduleComposerFocus, onMessagesClickCaptureBase, onMessagesPointerCancelBase, @@ -5198,7 +5285,7 @@ export default function ChatView({ } const message = timelineMessages.find((candidate) => candidate.id === messageId); if (!message) { - toastManager.add({ + reportComposerFeedback({ type: "warning", title: "Could not find the selected message.", }); @@ -5209,7 +5296,7 @@ export default function ChatView({ selectedText: pendingSelection.selection.text, }); if (!range) { - toastManager.add({ + reportComposerFeedback({ type: "warning", title: "Select a unique phrase to mark it.", description: "Try including a few more words so Scient can find the exact place.", @@ -5229,7 +5316,7 @@ export default function ChatView({ for (const marker of sameStyleOverlappingMarkers) { void dispatchThreadMarkerRemove(activeThreadId, marker.id).catch((error) => { console.error("Failed to remove thread marker", error); - toastManager.add({ + reportComposerFeedback({ type: "error", title: "Could not remove marker.", }); @@ -5248,7 +5335,7 @@ export default function ChatView({ color, }).catch((error) => { console.error("Failed to create thread marker", error); - toastManager.add({ + reportComposerFeedback({ type: "error", title: "Could not create marker.", }); @@ -5259,6 +5346,7 @@ export default function ChatView({ dismissTranscriptSelectionAction, isPendingSetupBubbleId, pendingTranscriptSelectionAction, + reportComposerFeedback, threadMarkers, timelineMessages, ], @@ -5529,9 +5617,13 @@ export default function ChatView({ useEffect(() => { voiceTranscriptionRequestIdRef.current += 1; + voiceCompletionInFlightRef.current = false; voiceRecordingStartedAtRef.current = null; + setLiveVoicePreview(null); + void liveVoicePreviewSessionRef.current?.stop(); + void readNativeApi()?.server.cancelVoiceTranscription?.(); void cancelVoiceRecording(); - setIsVoiceTranscribing(false); + setVoiceCompletionIntent(null); setOptimisticUserMessages((existing) => { if (existing.length === 0) return existing; for (const message of existing) { @@ -5558,9 +5650,12 @@ export default function ChatView({ isVoiceRecording, }); voiceTranscriptionRequestIdRef.current += 1; + voiceCompletionInFlightRef.current = false; voiceRecordingStartedAtRef.current = null; + setLiveVoicePreview(null); + void liveVoicePreviewSessionRef.current?.stop(); void cancelVoiceRecording(); - setIsVoiceTranscribing(false); + setVoiceCompletionIntent(null); }, [ canStartVoiceNotes, cancelVoiceRecording, @@ -6184,30 +6279,66 @@ export default function ChatView({ if (!activeProject) { return; } - if (voiceProviderStatus?.authStatus === "unauthenticated") { + if (!desktopVoiceAvailable && voiceProviderStatus?.authStatus === "unauthenticated") { useProviderConnectionDialogStore.getState().openDialog("codex", "runtime_error"); return; } if (!canStartVoiceNotes) { - toastManager.add({ + reportComposerFeedback({ type: "error", - title: "Voice notes require a ChatGPT-authenticated Codex session.", + title: "Voice transcription is unavailable in this browser session.", }); return; } if (pendingUserInputs.length > 0) { - toastManager.add({ + reportComposerFeedback({ type: "error", title: "Answer plan questions before recording a voice note.", }); return; } + if (!(await ensureDesktopVoiceReady(settings.voiceTranscriptionMode, reportComposerFeedback))) { + return; + } + try { + await liveVoicePreviewSessionRef.current?.stop(); await startVoiceRecording(); voiceRecordingStartedAtRef.current = performance.now(); + setLiveVoicePreview(null); + if (desktopVoiceAvailable) { + const api = readNativeApi(); + if (api) { + try { + liveVoicePreviewSessionRef.current?.start({ + getRecordingDurationMs: () => { + const startedAt = voiceRecordingStartedAtRef.current; + return startedAt === null + ? Number.POSITIVE_INFINITY + : Math.max(0, performance.now() - startedAt); + }, + captureSnapshot: snapshotVoiceRecording, + transcribeSnapshot: async (payload) => { + const result = await api.server.transcribeVoice({ + mode: "offline-only", + cwd: activeProject.cwd, + ...(activeThread ? { threadId: activeThread.id } : {}), + ...payload, + }); + return result.text; + }, + cancelActiveTranscription: () => + api.server.cancelVoiceTranscription?.() ?? Promise.resolve(), + onPreview: setLiveVoicePreview, + }); + } catch { + // Preview is opportunistic; the authoritative Stop/Send pass still works. + } + } + } } catch (error) { - toastManager.add({ + reportComposerFeedback({ type: "error", title: "Could not start recording", description: describeVoiceRecordingStartError(error), @@ -6215,138 +6346,166 @@ export default function ChatView({ } }, [ activeProject, + activeThread, canStartVoiceNotes, + desktopVoiceAvailable, pendingUserInputs.length, + reportComposerFeedback, + settings.voiceTranscriptionMode, + snapshotVoiceRecording, startVoiceRecording, voiceProviderStatus?.authStatus, ]); - const submitComposerVoiceRecording = useCallback(async () => { - if (!activeProject || !isVoiceRecording) { - return; - } - const recordedForMs = - voiceRecordingStartedAtRef.current === null - ? null - : Math.round(performance.now() - voiceRecordingStartedAtRef.current); - if ( - recordedForMs !== null && - recordedForMs >= 0 && - recordedForMs < VOICE_RECORDER_ACTION_ARM_DELAY_MS - ) { - warnVoiceGuard("ignored recorder action immediately after start", { - recordedForMs, - }); - return; - } - - const api = readNativeApi(); - if (!api) { - toastManager.add({ - type: "error", - title: "Voice transcription is unavailable right now.", - }); - void cancelVoiceRecording(); - return; - } - - setIsVoiceTranscribing(true); - const requestId = voiceTranscriptionRequestIdRef.current + 1; - voiceTranscriptionRequestIdRef.current = requestId; - const requestThreadId = threadId; - const requestProvider = selectedProvider; - const isCurrentVoiceRequest = () => - voiceTranscriptionRequestIdRef.current === requestId && - voiceThreadIdRef.current === requestThreadId && - voiceProviderRef.current === requestProvider; - - try { - const payload = await stopVoiceRecording(); - if (!isCurrentVoiceRequest()) { + const finishComposerVoiceRecording = useCallback( + async (intent: ComposerVoiceCompletionIntent) => { + if (!activeProject || !isVoiceRecording || voiceCompletionInFlightRef.current) { return; } - if (!payload) { - toastManager.add({ - type: "warning", - title: "No audio was captured.", + const recordedForMs = + voiceRecordingStartedAtRef.current === null + ? null + : Math.round(performance.now() - voiceRecordingStartedAtRef.current); + if ( + recordedForMs !== null && + recordedForMs >= 0 && + recordedForMs < VOICE_RECORDER_ACTION_ARM_DELAY_MS + ) { + warnVoiceGuard("ignored recorder action immediately after start", { + recordedForMs, }); return; } - const result = await api.server.transcribeVoice({ - provider: "codex", - cwd: activeProject.cwd, - ...(activeThread ? { threadId: activeThread.id } : {}), - ...payload, - }); - if (!isCurrentVoiceRequest()) { - return; - } - appendVoiceTranscriptToComposer(result.text); - } catch (error) { - if (!isCurrentVoiceRequest()) { + + const api = readNativeApi(); + if (!api) { + reportComposerFeedback({ + type: "error", + title: "Voice transcription is unavailable right now.", + }); + void cancelVoiceRecording(); return; } - const description = - error instanceof Error - ? sanitizeVoiceErrorMessage(error.message) - : "The voice note could not be transcribed."; - const authExpired = isVoiceAuthExpiredMessage(description); - if (authExpired) { - void refreshProviderStatuses(); - } - toastManager.add({ - type: "error", - title: authExpired ? "Sign in to ChatGPT again" : "Couldn't transcribe voice note", - description: authExpired - ? "Voice transcription uses your ChatGPT session in Codex. That session was rejected, so sign in again there and retry." - : description, - ...(authExpired - ? { - actionProps: { - children: "Refresh status", - onClick: () => { - void refreshProviderStatuses(); - }, - }, + + setVoiceCompletionIntent(intent); + voiceCompletionInFlightRef.current = true; + const requestId = voiceTranscriptionRequestIdRef.current + 1; + voiceTranscriptionRequestIdRef.current = requestId; + const requestThreadId = threadId; + const requestProvider = selectedProvider; + const isCurrentVoiceRequest = () => + voiceTranscriptionRequestIdRef.current === requestId && + voiceThreadIdRef.current === requestThreadId && + voiceProviderRef.current === requestProvider; + + try { + // A partial uses the same serialized local helper as the final pass. + // Cancel and drain it first so Stop/Send never waits behind stale work. + await liveVoicePreviewSessionRef.current?.stop(); + if (!isCurrentVoiceRequest()) { + return; + } + const payload = await stopVoiceRecording(); + if (!isCurrentVoiceRequest()) { + return; + } + if (!payload) { + reportComposerFeedback({ + type: "warning", + title: "No audio was captured.", + }); + return; + } + const result = await api.server.transcribeVoice({ + mode: settings.voiceTranscriptionMode, + cwd: activeProject.cwd, + ...(activeThread ? { threadId: activeThread.id } : {}), + ...payload, + }); + if (!isCurrentVoiceRequest()) { + return; + } + const completion = await completeComposerVoiceTranscript({ + intent, + currentPrompt: promptRef.current, + transcript: result.text, + insertTranscript: (transcript, completedPrompt) => { + if (promptRef.current === completedPrompt) { + return false; } - : {}), - }); - } finally { - if (isCurrentVoiceRequest()) { - voiceRecordingStartedAtRef.current = null; - setIsVoiceTranscribing(false); + appendVoiceTranscriptToComposer(transcript); + return true; + }, + sendPrompt: (nextPrompt) => sendVoiceTranscriptRef.current(nextPrompt), + }); + if (completion === "empty") { + reportComposerFeedback({ + type: "warning", + title: "No speech was detected.", + }); + } + } catch (error) { + if (!isCurrentVoiceRequest()) { + return; + } + const description = + error instanceof Error + ? sanitizeVoiceErrorMessage(error.message) + : "The voice note could not be transcribed."; + const authExpired = !desktopVoiceAvailable && isVoiceAuthExpiredMessage(description); + if (authExpired) { + void refreshProviderStatuses(); + } + reportComposerFeedback({ + type: "error", + title: authExpired ? "Sign in to ChatGPT again" : "Couldn't transcribe voice note", + description: authExpired + ? "Your ChatGPT session was rejected. Sign in again and retry." + : description, + ...(authExpired + ? { + actionProps: { + children: "Refresh status", + onClick: () => { + void refreshProviderStatuses(); + }, + }, + } + : {}), + }); + } finally { + if (isCurrentVoiceRequest()) { + voiceCompletionInFlightRef.current = false; + voiceRecordingStartedAtRef.current = null; + setLiveVoicePreview(null); + setVoiceCompletionIntent(null); + } } - } - }, [ - activeProject, - activeThread, - appendVoiceTranscriptToComposer, - cancelVoiceRecording, - isVoiceRecording, - refreshProviderStatuses, - selectedProvider, - stopVoiceRecording, - threadId, - ]); + }, + [ + activeProject, + activeThread, + appendVoiceTranscriptToComposer, + cancelVoiceRecording, + desktopVoiceAvailable, + isVoiceRecording, + reportComposerFeedback, + refreshProviderStatuses, + selectedProvider, + settings.voiceTranscriptionMode, + stopVoiceRecording, + threadId, + ], + ); const cancelComposerVoiceRecording = useCallback(() => { - const recordedForMs = - voiceRecordingStartedAtRef.current === null - ? null - : Math.round(performance.now() - voiceRecordingStartedAtRef.current); - if ( - recordedForMs !== null && - recordedForMs >= 0 && - recordedForMs < VOICE_RECORDER_ACTION_ARM_DELAY_MS - ) { - warnVoiceGuard("ignored recorder action immediately after start", { - recordedForMs, - }); - return; - } voiceTranscriptionRequestIdRef.current += 1; + voiceCompletionInFlightRef.current = false; voiceRecordingStartedAtRef.current = null; - setIsVoiceTranscribing(false); + setLiveVoicePreview(null); + setVoiceCompletionIntent(null); + void liveVoicePreviewSessionRef.current?.stop(); + void readNativeApi()?.server.cancelVoiceTranscription?.(); void cancelVoiceRecording(); }, [cancelVoiceRecording]); @@ -6357,7 +6516,7 @@ export default function ChatView({ return; } if (isVoiceRecording) { - void submitComposerVoiceRecording(); + void finishComposerVoiceRecording("insert"); return; } void startComposerVoiceRecording(); @@ -6365,7 +6524,7 @@ export default function ChatView({ isVoiceRecording, isVoiceTranscribing, startComposerVoiceRecording, - submitComposerVoiceRecording, + finishComposerVoiceRecording, ]); // --- Composer attachment entry points ------------------------------------- @@ -6374,7 +6533,7 @@ export default function ChatView({ if (!activeThreadId || files.length === 0) return; if (pendingUserInputs.length > 0) { - toastManager.add({ + reportComposerFeedback({ type: "error", title: "Attach images after answering plan questions.", }); @@ -6393,14 +6552,16 @@ export default function ChatView({ } else if (nextImages.length > 1) { addComposerImagesToDraft(nextImages); } - setThreadError(activeThreadId, error); + if (error) { + reportComposerFeedback({ type: "error", title: error }); + } }, [ activeThreadId, addComposerImage, addComposerImagesToDraft, pendingUserInputs.length, - setThreadError, + reportComposerFeedback, ], ); @@ -6413,7 +6574,7 @@ export default function ChatView({ if (!activeThreadId || files.length === 0) return; if (pendingUserInputs.length > 0) { - toastManager.add({ + reportComposerFeedback({ type: "error", title: "Attach files after answering plan questions.", }); @@ -6430,9 +6591,11 @@ export default function ChatView({ if (nextFiles.length > 0) { addComposerFilesToDraft(nextFiles); } - setThreadError(activeThreadId, error); + if (error) { + reportComposerFeedback({ type: "error", title: error }); + } }, - [activeThreadId, addComposerFilesToDraft, pendingUserInputs.length, setThreadError], + [activeThreadId, addComposerFilesToDraft, pendingUserInputs.length, reportComposerFeedback], ); const removeComposerFile = (fileId: string) => { @@ -6565,7 +6728,7 @@ export default function ChatView({ try { await createThreadHandoff(activeThread, targetProvider); } catch (error) { - toastManager.add({ + reportComposerFeedback({ type: "error", title: "Could not create handoff thread", description: @@ -6575,7 +6738,7 @@ export default function ChatView({ }); } }, - [activeThread, createThreadHandoff, handoffDisabled], + [activeThread, createThreadHandoff, handoffDisabled, reportComposerFeedback], ); const clearComposerInput = useCallback( @@ -6730,7 +6893,7 @@ export default function ChatView({ createdAt, }); } catch { - toastManager.add({ + reportComposerFeedback({ type: "warning", title: "Thread note not added", description: @@ -6742,14 +6905,14 @@ export default function ChatView({ void queryClient.invalidateQueries({ queryKey: automationQueryKey }); clearComposerInput(activeThread?.id ?? threadId); resetAutomationDraftState(); - toastManager.add({ + reportComposerFeedback({ type: "success", title: "Automation created", description: `${definition.name} - ${formatCadence(definition.schedule)}`, }); return true; } catch (error) { - toastManager.add({ + reportComposerFeedback({ type: "error", title: "Could not create automation", description: @@ -6768,6 +6931,7 @@ export default function ChatView({ isServerThread, providerOptionsForDispatch, queryClient, + reportComposerFeedback, resetAutomationDraftState, threadId, ], @@ -6782,7 +6946,7 @@ export default function ChatView({ }): Promise => { const api = readNativeApi(); if (!api || !activeProject || !activeThread) { - toastManager.add({ + reportComposerFeedback({ type: "warning", title: "Chat required", description: "Open a chat before creating a chat-bound automation.", @@ -6818,7 +6982,7 @@ export default function ChatView({ { force: true }, ); if (result === "unavailable") { - toastManager.add({ + reportComposerFeedback({ type: "error", title: "Could not create chat", description: "Scient could not promote this draft before saving the automation.", @@ -6838,7 +7002,7 @@ export default function ChatView({ return activeThread.id; } catch (error) { - toastManager.add({ + reportComposerFeedback({ type: "error", title: "Could not create chat", description: @@ -6849,7 +7013,14 @@ export default function ChatView({ return null; } }, - [activeProject, activeThread, activeThreadAssociatedWorktree, isServerThread, threadNotes], + [ + activeProject, + activeThread, + activeThreadAssociatedWorktree, + isServerThread, + reportComposerFeedback, + threadNotes, + ], ); const prepareAutomationFormForCreate = useCallback( @@ -6950,7 +7121,7 @@ export default function ChatView({ updateInputFromForm(input.definition, input.form, providerOptions, acknowledgedRisks), ); resetAutomationDraftState(); - toastManager.add({ + reportComposerFeedback({ type: "success", title: "Automation updated", description: `${updated.name} - ${formatCadence(updated.schedule)}`, @@ -6963,7 +7134,12 @@ export default function ChatView({ setIsAutomationDraftSubmitting(false); } }, - [automationUpdateMutation, providerOptionsForDispatch, resetAutomationDraftState], + [ + automationUpdateMutation, + providerOptionsForDispatch, + reportComposerFeedback, + resetAutomationDraftState, + ], ); const submitAutomationDraft = useCallback(async () => { @@ -7100,6 +7276,7 @@ export default function ChatView({ e?: { preventDefault: () => void }, dispatchMode: "queue" | "steer" = "queue", queuedTurn?: QueuedComposerChatTurn, + voicePromptOverride?: string, ): Promise => { e?.preventDefault(); const api = readNativeApi(); @@ -7108,13 +7285,19 @@ export default function ChatView({ !activeThread || isSendBusy || isConnecting || - isVoiceTranscribing || + (isVoiceTranscribing && voicePromptOverride === undefined) || sendPreflightInFlightRef.current || sendInFlightRef.current ) { return false; } - if (activePendingProgress) { + if ( + shouldRouteComposerSendToPendingInput({ + hasActivePendingProgress: activePendingProgress !== null, + hasVoicePromptOverride: voicePromptOverride !== undefined, + }) && + activePendingProgress + ) { const activeQuestion = activePendingProgress.activeQuestion; const liveComposerSnapshot = composerEditorRef.current?.readSnapshot() ?? null; const livePendingAnswerText = liveComposerSnapshot?.value ?? promptRef.current; @@ -7152,7 +7335,11 @@ export default function ChatView({ const queuedChatTurn = queuedTurn ?? null; const liveComposerSnapshot = queuedChatTurn === null ? (composerEditorRef.current?.readSnapshot() ?? null) : null; - let promptForSend = queuedChatTurn?.prompt ?? liveComposerSnapshot?.value ?? promptRef.current; + let promptForSend = + queuedChatTurn?.prompt ?? + voicePromptOverride ?? + liveComposerSnapshot?.value ?? + promptRef.current; let composerImagesForSend = queuedChatTurn?.images ?? composerImages; // AppSnap captures persist as IndexedDB blobs and hydrate into `images` // asynchronously (see AppSnapCoordinator). Right after a reload the user can @@ -7308,7 +7495,7 @@ export default function ChatView({ expiredTerminalContextCount, "empty", ); - toastManager.add({ + reportComposerFeedback({ type: "warning", title: toastCopy.title, description: toastCopy.description, @@ -7345,7 +7532,7 @@ export default function ChatView({ // clearing the composer would drop attachments/mentions Cancel can't restore, // and ephemeral setup bubbles must not anchor a running turn's work rows. if (!hasPromptOnlySendableContent || hasLiveTurn) { - toastManager.add({ + reportComposerFeedback({ type: "warning", title: "Automation needs a bit more detail", description: @@ -7486,7 +7673,7 @@ export default function ChatView({ if (nextAttachmentCount <= PROVIDER_SEND_TURN_MAX_ATTACHMENTS) { composerImagesForSend = [...composerImagesForSend, browserPromptAttachment.image]; } else { - toastManager.add({ + reportComposerFeedback({ type: "warning", title: `You can attach up to ${PROVIDER_SEND_TURN_MAX_ATTACHMENTS} references per message.`, description: @@ -7502,7 +7689,7 @@ export default function ChatView({ : browserPromptAttachment.reason === "attachment-too-large" ? `The browser screenshot exceeded the ${IMAGE_SIZE_LIMIT_LABEL} attachment limit.` : "The current browser context could not be attached."; - toastManager.add({ + reportComposerFeedback({ type: "warning", title: "Couldn’t attach the in-app browser context", description, @@ -7830,7 +8017,7 @@ export default function ChatView({ expiredTerminalContextCount, "omitted", ); - toastManager.add({ + reportComposerFeedback({ type: "warning", title: toastCopy.title, description: toastCopy.description, @@ -8118,6 +8305,8 @@ export default function ChatView({ } return turnStartSucceeded; }; + sendVoiceTranscriptRef.current = (voicePrompt) => + onSend(undefined, "queue", undefined, voicePrompt); const onRespondToApproval = useCallback( async (requestId: ApprovalRequestId, decision: ProviderApprovalDecision) => { @@ -8149,6 +8338,7 @@ export default function ChatView({ return; } if (durableRuntimeMode) { + const runtimeModeActivityKey = `thread:${activeThreadId}:runtime-mode-persistence`; setPendingServerRuntimeMode({ threadId: activeThreadId, mode: durableRuntimeMode }); try { await api.orchestration.dispatchCommand({ @@ -8158,19 +8348,24 @@ export default function ChatView({ runtimeMode: durableRuntimeMode, createdAt: new Date().toISOString(), }); + activityManager.remove(runtimeModeActivityKey); } catch (err: unknown) { setPendingServerRuntimeMode((pending) => pending?.threadId === activeThreadId && pending.mode === durableRuntimeMode ? null : pending, ); - toastManager.add({ - type: "warning", + activityManager.publish({ + dedupeKey: runtimeModeActivityKey, + source: "thread", + status: "needs_attention", + tone: "warning", title: "Approval sent, but agent access was not saved", description: err instanceof Error ? err.message : "Choose Unrestricted again to keep it for future turns.", + destination: { type: "thread", threadId: activeThreadId }, }); } } @@ -8840,7 +9035,7 @@ export default function ChatView({ useStore.getState().removeDeletedThreadFromClientState, }); } - toastManager.add({ + reportComposerFeedback({ type: "error", title: "Could not start implementation thread", description: @@ -8863,6 +9058,7 @@ export default function ChatView({ selectedPromptEffort, selectedModelSelection, providerOptionsForDispatch, + reportComposerFeedback, rememberCustomBinaryPathForDispatch, selectedProvider, assistantDeliveryMode, @@ -9579,7 +9775,7 @@ export default function ChatView({ }), handleClearConversation: async () => { if (!activeProject) { - toastManager.add({ + reportComposerFeedback({ type: "warning", title: "Clear is unavailable", description: "Open a project before starting a fresh thread.", @@ -9597,6 +9793,7 @@ export default function ChatView({ setComposerCommandPicker("review-target"); setComposerHighlightedItemId("review-target:changes"); }, + reportComposerFeedback, setComposerDraftProviderModelOptions, editorActions: slashEditorActions, }); @@ -10212,7 +10409,7 @@ export default function ChatView({ } : undefined, }).catch((error) => { - toastManager.add({ + transientAlertManager.add({ type: "error", title: "Failed to rename thread", description: error instanceof Error ? error.message : "An error occurred.", @@ -10221,7 +10418,7 @@ export default function ChatView({ }); if (outcome === "empty") { - toastManager.add({ + transientAlertManager.add({ type: "warning", title: "Thread title cannot be empty", }); @@ -10634,13 +10831,17 @@ export default function ChatView({
- {/* Bottom toolbar — hidden while an approval takes over the composer, - since the approve/decline actions live in the detached approval card - floating above (see ComposerPendingApprovalPanel). */} - {activePendingApproval ? null : ( + {/* An idle approval owns the composer footer. If it arrives during active + voice capture, keep recorder controls mounted until voice settles. */} + {shouldRenderComposerFooter({ + hasPendingApproval: activePendingApproval !== null, + isVoiceActive, + }) ? (
+ + {composerLocalFeedback.title} + {composerLocalFeedback.description ? ( + + {" — "} + {composerLocalFeedback.description} + + ) : null} + + {composerLocalFeedback.actionProps ? ( + + ) : null} + + ) : null} + {!isVoiceRecording && !isVoiceTranscribing ? ( <> {interactionMode === "plan" ? ( @@ -10769,24 +11021,26 @@ export default function ChatView({ {!isVoiceRecording && !isVoiceTranscribing ? composerPickerControls : null} {showVoiceNotesControl && (isVoiceRecording || isVoiceTranscribing) ? ( void finishComposerVoiceRecording("insert")} + onSend={() => void finishComposerVoiceRecording("send")} + /> + ) : null} + {composerFooterActionPlan.showVoiceButton ? ( + { - if (isVoiceRecording) { - void submitComposerVoiceRecording(); - return; - } - cancelComposerVoiceRecording(); - }} - onSubmit={() => { - void submitComposerVoiceRecording(); - }} + onClick={toggleComposerVoiceRecording} /> ) : null} - {activePendingProgress ? ( + {composerFooterActionPlan.primaryAction === "pending-input" && + activePendingProgress ? ( - ) : pendingUserInputs.length === 0 && - !isVoiceRecording && - !isVoiceTranscribing ? ( + ) : composerFooterActionPlan.primaryAction === "plan-follow-up" ? ( showPlanFollowUpPrompt ? ( prompt.trim().length > 0 ? (
) - ) : ( - <> - {showVoiceNotesControl ? ( - - ) : null} - - - ) + + + ) : ( +