diff --git a/.changeset/stable-composer-controls.md b/.changeset/stable-composer-controls.md index d1df963bd70..b599dc558e4 100644 --- a/.changeset/stable-composer-controls.md +++ b/.changeset/stable-composer-controls.md @@ -2,6 +2,8 @@ "@hashintel/petrinaut": patch --- -Add a generic host-rendered AI composer control with stable finalized-text submission, -conversation identity, stop handling, schema-validated interactive-tool text mapping, and an -explicit separate-message target for corrections. +Add generic host-rendered AI composer controls and a persistent interview stage with docked and +detached placements, protected active conversations, keyboard fallback, and one-answer buffering +while the normal chat stream settles. Add the Chat / Interview mode switch and export +`PetrinautAiInteractionMode`, with the selected interaction mode and mode-change callback available +to host-rendered interview stages. diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts index e9b0915734f..3df1bb427a3 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts @@ -51,7 +51,7 @@ const captureSchema = z.object({ supersedes: z.string().optional(), }); -const sweepOutputSchema = z.discriminatedUnion("status", [ +export const sweepOutputSchema = z.discriminatedUnion("status", [ z.object({ status: z.literal("no-settled-range") }), z.object({ status: z.literal("refused"), diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx index 441be1df455..4893ff001a7 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx @@ -5,7 +5,7 @@ import { isValidElement, type ReactNode } from "react"; import { describe, expect, test, vi } from "vitest"; import { VoiceInterviewControl } from "../voice-interview/voice-interview-control"; -import { getBrunchVoiceComposerControl } from "./local-storage-demo-app"; +import { getBrunchVoiceInterviewStage } from "./local-storage-demo-app"; const defaultTransportOptions = vi.hoisted(() => ({ current: null as unknown, @@ -28,16 +28,28 @@ vi.mock("@hashintel/petrinaut/ui", () => ({ describe("local storage demo Brunch voice integration", () => { test("does not install voice on the generic local chat fallback", () => { - expect(getBrunchVoiceComposerControl(false)).toBeUndefined(); + expect(getBrunchVoiceInterviewStage(null)).toBeUndefined(); }); test("installs the app-owned voice control for a configured Brunch transport", () => { - const renderControl = getBrunchVoiceComposerControl(true); - const control = renderControl?.({ + const config = { available: true as const, connectionTimeoutMs: 15_000 }; + const stage = getBrunchVoiceInterviewStage(config); + const control = stage?.({ + canAcceptInterviewAnswer: true, conversationId: "petrinaut-preview:net-1", + focusComposer: vi.fn(), + interactionMode: "chat", messages: [], + openSidebar: vi.fn(), + placement: "sidebar", + setActive: vi.fn(), + setInteractionMode: vi.fn(), status: "ready", stop: vi.fn(async () => undefined), + submitInterviewAnswer: vi.fn(async () => ({ + kind: "message" as const, + messageId: "message-1", + })), submitText: vi.fn(async () => ({ kind: "message" as const, messageId: "message-1", @@ -48,7 +60,10 @@ describe("local storage demo Brunch voice integration", () => { if (!isValidElement(control)) { throw new Error("Expected the configured composer control to render."); } - expect(control.type).toBe(VoiceInterviewControl); + expect(control).toMatchObject({ + props: { config }, + type: VoiceInterviewControl, + }); }); test("correlates the existing Brunch transport request", () => { diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index aeeaddd27bf..5eabfe8ba34 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -13,15 +13,19 @@ import { DefaultChatTransport, Petrinaut, type PetrinautAiChatTransport, - type PetrinautAiComposerControl, - type PetrinautAiComposerControlContext, + type PetrinautAiInterviewStage, + type PetrinautAiInterviewStageContext, type PetrinautAiMessage, WalkthroughProvider, } from "@hashintel/petrinaut/ui"; import { VOICE_REQUEST_ID_HEADER } from "../../../voice-diagnostics"; import { useSentryFeedbackAction } from "../sentry-feedback-button"; -import { VoiceInterviewControl } from "../voice-interview/voice-interview-control"; +import { + loadOpenAIVoiceConfig, + type OpenAIVoiceConfig, + VoiceInterviewControl, +} from "../voice-interview/voice-interview-control"; import { brunchAskInteractiveTool } from "./brunch-ask-interactive-tool"; import { createBrunchPanelTransport } from "./brunch-panel-transport"; import { @@ -88,18 +92,14 @@ const brunchPreviewConfig = resolveBrunchPreviewConfig( import.meta.env.VITE_BRUNCH_CHAT_ENDPOINT, ); -const renderBrunchVoiceComposerControl = ( - context: PetrinautAiComposerControlContext, -) => ; - -export const getBrunchVoiceComposerControl = ( - isBrunchConfigured: boolean, -): PetrinautAiComposerControl | undefined => - isBrunchConfigured ? renderBrunchVoiceComposerControl : undefined; - -const brunchVoiceComposerControl = getBrunchVoiceComposerControl( - brunchPreviewConfig.isBrunchConfigured, -); +export const getBrunchVoiceInterviewStage = ( + config: OpenAIVoiceConfig | null | undefined, +): PetrinautAiInterviewStage | undefined => + config + ? (context: PetrinautAiInterviewStageContext) => ( + + ) + : undefined; const createHandle = (net: SDCPNInLocalStorage): PetrinautDocHandle => createJsonDocHandle({ @@ -152,11 +152,38 @@ const createActiveHandle = (net: SDCPNInLocalStorage): ActiveHandle => ({ */ export const LocalStorageDemoApp = () => { const sentryFeedbackAction = useSentryFeedbackAction(); + const [openAIVoiceConfig, setOpenAIVoiceConfig] = + useState(); const { aiMessagesByNetId, setAiMessagesByNetId } = useLocalStorageAiMessages(); const { storedSDCPNs, setStoredSDCPNs } = useLocalStorageSDCPNs(); const storedSDCPNsForDisplay = getStoredSDCPNsForDisplay(storedSDCPNs); + useEffect(() => { + if (!brunchPreviewConfig.isBrunchConfigured) { + // eslint-disable-next-line react-hooks-js/set-state-in-effect -- Resolve the loading sentinel when voice is not configured. + setOpenAIVoiceConfig(null); + return; + } + + const abortController = new AbortController(); + void loadOpenAIVoiceConfig( + globalThis.fetch.bind(globalThis), + abortController.signal, + ).then((config) => { + if (!abortController.signal.aborted) { + setOpenAIVoiceConfig(config); + } + }); + + return () => abortController.abort(); + }, []); + + const brunchVoiceInterviewStage = useMemo( + () => getBrunchVoiceInterviewStage(openAIVoiceConfig), + [openAIVoiceConfig], + ); + // Pick the most recently modified net const mostRecentlyModifiedNet = Object.values(storedSDCPNsForDisplay).sort( @@ -320,13 +347,18 @@ export const LocalStorageDemoApp = () => { return next; }); }, - ...(brunchVoiceComposerControl + ...(brunchVoiceInterviewStage ? { - renderComposerControl: brunchVoiceComposerControl, + renderInterviewStage: brunchVoiceInterviewStage, } : {}), }), - [aiMessagesByNetId, currentNetId, setAiMessagesByNetId], + [ + aiMessagesByNetId, + brunchVoiceInterviewStage, + currentNetId, + setAiMessagesByNetId, + ], ); if (!currentNet) { diff --git a/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.test.ts new file mode 100644 index 00000000000..6f1ba59c8c6 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, test } from "vitest"; + +import { SWEEP_TOOL_NAME } from "@hashintel/brunch-agent-transport-aisdk/client-tools"; + +import { selectInterviewCoverage } from "./interview-coverage"; + +import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui"; + +describe("interview coverage", () => { + test("uses only authoritative completion results for covered and open topics", () => { + const messages = [ + { + id: "assistant-sweep", + role: "assistant", + parts: [ + { + type: "dynamic-tool", + toolCallId: "sweep-1", + toolName: SWEEP_TOOL_NAME, + state: "output-available", + input: {}, + output: { + status: "applied", + appliedCaptureIds: ["capture-owner"], + captures: [ + { + id: "capture-owner", + status: "active", + epistemicStatus: "explicit", + confidence: "high", + content: { + value: { + type: "slot-asserted", + kind: "activity", + node: "approval", + slot: "who performs it", + precision: "named", + assertion: { value: "shift lead" }, + }, + }, + }, + { + id: "capture-old", + status: "superseded", + epistemicStatus: "explicit", + confidence: "high", + content: { + value: { + type: "slot-asserted", + kind: "activity", + node: "approval", + slot: "how long it takes", + assertion: { value: "one hour" }, + }, + }, + }, + ], + completion: { + complete: false, + pluginVersion: "sdcpn/1", + revision: "revision-1", + failures: [ + { + diagnostic: "unaddressed", + nodeId: "activity:approval", + kind: "activity", + slot: "how long it takes", + requirement: "spread", + actual: "not mentioned", + message: "Duration is still unknown.", + captureIds: [], + }, + ], + sliceNodeIds: ["activity:approval", "activity:dispatch"], + outsideSlice: [], + }, + }, + }, + ], + }, + ] as unknown as PetrinautAiMessage[]; + + expect(selectInterviewCoverage(messages)).toEqual({ + complete: false, + covered: ["dispatch"], + stillExploring: ["approval — how long it takes"], + }); + }); + + test("omits coverage when no validated completion report exists", () => { + expect(selectInterviewCoverage([])).toBeNull(); + }); +}); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.ts b/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.ts new file mode 100644 index 00000000000..ecbbf0ed7ca --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.ts @@ -0,0 +1,66 @@ +import { SWEEP_TOOL_NAME } from "@hashintel/brunch-agent-transport-aisdk/client-tools"; + +import { sweepOutputSchema } from "../local-storage-demo/brunch-panel-transport"; + +import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui"; + +export interface InterviewCoverage { + readonly complete: boolean; + readonly covered: readonly string[]; + readonly stillExploring: readonly string[]; +} + +const unique = (items: string[]): string[] => [...new Set(items)]; + +const nodeLabel = (nodeId: string): string => { + const separator = nodeId.indexOf(":"); + return separator === -1 ? nodeId : nodeId.slice(separator + 1); +}; + +export const selectInterviewCoverage = ( + messages: PetrinautAiMessage[], +): InterviewCoverage | null => { + for (const message of messages.toReversed()) { + for (const part of message.parts.toReversed()) { + if ( + part.type !== "dynamic-tool" || + part.toolName !== SWEEP_TOOL_NAME || + part.state !== "output-available" + ) { + continue; + } + const parsed = sweepOutputSchema.safeParse(part.output); + if ( + !parsed.success || + parsed.data.status !== "applied" || + parsed.data.completion === undefined + ) { + continue; + } + + const { completion } = parsed.data; + const nodesWithFailures = new Set( + completion.failures.flatMap((failure) => + failure.nodeId === undefined ? [] : [failure.nodeId], + ), + ); + const covered = completion.sliceNodeIds + .filter((nodeId) => !nodesWithFailures.has(nodeId)) + .map(nodeLabel); + const stillExploring = unique( + completion.failures.map((failure) => + failure.nodeId === undefined + ? failure.message + : `${nodeLabel(failure.nodeId)} — ${failure.slot ?? failure.message}`, + ), + ); + + return { + complete: completion.complete, + covered, + stillExploring, + }; + } + } + return null; +}; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts index 095375e8ab5..b98da14b733 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts @@ -11,6 +11,7 @@ class FakeDataChannel extends EventTarget { public readonly close = vi.fn(() => { this.readyState = "closed"; }); + public readonly send = vi.fn(); public open() { this.readyState = "open"; @@ -26,8 +27,34 @@ class FakeDataChannel extends EventTarget { } } -const createHarness = () => { +const createHarness = ({ + createAudioContext, +}: { + readonly createAudioContext?: () => AudioContext; +} = {}) => { let requestNumber = 0; + const animationFrames: FrameRequestCallback[] = []; + const analyser = { + fftSize: 0, + getByteTimeDomainData: vi.fn((data: Uint8Array) => { + data.fill(160); + }), + }; + const mediaSource = { connect: vi.fn() }; + const tracks: Array<{ enabled: boolean; stop: ReturnType }> = + []; + const trackEnabledWhenMeterConnected: boolean[] = []; + const audioContext = { + close: vi.fn(async () => undefined), + createAnalyser: vi.fn(() => analyser), + createMediaStreamSource: vi.fn(() => { + trackEnabledWhenMeterConnected.push(tracks.at(-1)?.enabled ?? true); + return mediaSource; + }), + resume: vi.fn(async () => undefined), + state: "suspended" as AudioContextState, + }; + const cancelAnimationFrame = vi.fn(); const channels: FakeDataChannel[] = []; const peers: Array<{ addTrack: ReturnType; @@ -39,8 +66,6 @@ const createHarness = () => { setLocalDescription: ReturnType; setRemoteDescription: ReturnType; }> = []; - const tracks: Array<{ enabled: boolean; stop: ReturnType }> = - []; const fetch = vi.fn( async () => new Response("v=0\r\no=OpenAI answer", { @@ -78,18 +103,29 @@ const createHarness = () => { return peer as unknown as RTCPeerConnection; }; const session = new OpenAIRealtimeSession({ + cancelAnimationFrame, connectionTimeoutMs: 15_000, + createAudioContext: + createAudioContext ?? (() => audioContext as unknown as AudioContext), createRequestId: () => `voice-request-${++requestNumber}`, createPeerConnection, fetch, getUserMedia, now: () => 100, reportDiagnostic, + requestAnimationFrame: (callback) => { + animationFrames.push(callback); + return animationFrames.length; + }, }); const events: OpenAIRealtimeSessionEvent[] = []; session.subscribe((event) => events.push(event)); return { + analyser, + animationFrames, + audioContext, + cancelAnimationFrame, channels, events, fetch, @@ -97,6 +133,7 @@ const createHarness = () => { peers, reportDiagnostic, session, + trackEnabledWhenMeterConnected, tracks, }; }; @@ -119,6 +156,7 @@ describe("OpenAIRealtimeSession", () => { }, }); expect(harness.tracks[0]!.enabled).toBe(false); + expect(harness.trackEnabledWhenMeterConnected).toEqual([false]); expect(harness.fetch).toHaveBeenCalledWith( "/api/voice/realtime-call", expect.objectContaining({ @@ -149,6 +187,95 @@ describe("OpenAIRealtimeSession", () => { }); }); + test("reports real input level only while the microphone track is enabled", async () => { + const harness = createHarness(); + await harness.session.connect(); + + expect(harness.animationFrames).toHaveLength(0); + harness.session.setMicrophoneEnabled(true); + expect(harness.animationFrames).toHaveLength(1); + harness.animationFrames.shift()?.(0); + + expect(harness.events.at(-1)).toMatchObject({ + type: "microphone-level", + }); + expect((harness.events.at(-1) as { level: number }).level).toBeGreaterThan( + 0, + ); + + harness.session.setMicrophoneEnabled(false); + expect(harness.tracks[0]!.enabled).toBe(false); + expect(harness.cancelAnimationFrame).toHaveBeenCalled(); + expect(harness.events.at(-1)).toEqual({ + level: 0, + type: "microphone-level", + }); + }); + + test("resumes a suspended input meter before waiting for microphone access", async () => { + const harness = createHarness(); + const track = { enabled: true, stop: vi.fn() }; + const stream = { + getAudioTracks: () => [track], + getTracks: () => [track], + } as unknown as MediaStream; + let resolveMedia: ((mediaStream: MediaStream) => void) | undefined; + harness.getUserMedia.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveMedia = resolve; + }), + ); + + const connection = harness.session.connect(); + const resumeCallsBeforeMedia = + harness.audioContext.resume.mock.calls.length; + const sourceCallsBeforeMedia = + harness.audioContext.createMediaStreamSource.mock.calls.length; + resolveMedia?.(stream); + await connection; + + expect(resumeCallsBeforeMedia).toBe(1); + expect(sourceCallsBeforeMedia).toBe(0); + expect(harness.audioContext.createMediaStreamSource).toHaveBeenCalledWith( + stream, + ); + }); + + test("connects without metering when audio context construction throws", async () => { + const harness = createHarness({ + createAudioContext: () => { + throw new Error("AudioContext unavailable"); + }, + }); + + await expect(harness.session.connect()).resolves.toBe(1); + harness.session.setMicrophoneEnabled(true); + + expect(harness.fetch).toHaveBeenCalledOnce(); + expect(harness.peers[0]!.addTrack).toHaveBeenCalledOnce(); + expect(harness.tracks[0]!.enabled).toBe(true); + expect(harness.animationFrames).toHaveLength(0); + expect(harness.events).toEqual([]); + }); + + test("connects without metering when meter initialization throws", async () => { + const harness = createHarness(); + harness.audioContext.createMediaStreamSource.mockImplementationOnce(() => { + throw new Error("Media stream source unavailable"); + }); + + await expect(harness.session.connect()).resolves.toBe(1); + harness.session.setMicrophoneEnabled(true); + + expect(harness.fetch).toHaveBeenCalledOnce(); + expect(harness.peers[0]!.addTrack).toHaveBeenCalledOnce(); + expect(harness.tracks[0]!.enabled).toBe(true); + expect(harness.audioContext.close).toHaveBeenCalledOnce(); + expect(harness.animationFrames).toHaveLength(0); + expect(harness.events).toEqual([]); + }); + test("emits only strict input transcription events with stable source identity", async () => { const harness = createHarness(); await harness.session.connect(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts index 6cb0c5a0298..1ce16c37ad8 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts @@ -28,6 +28,7 @@ export type OpenAIRealtimeSessionEvent = readonly text: string; readonly type: "partial" | "completed"; } + | { readonly level: number; readonly type: "microphone-level" } | { readonly code: VoiceErrorCode; readonly message: string; @@ -36,7 +37,9 @@ export type OpenAIRealtimeSessionEvent = }; interface OpenAIRealtimeSessionDependencies { + readonly cancelAnimationFrame: (handle: number) => void; readonly connectionTimeoutMs: number; + readonly createAudioContext: () => AudioContext; readonly createRequestId?: () => string; readonly createPeerConnection: () => RTCPeerConnection; readonly fetch: typeof globalThis.fetch; @@ -45,6 +48,7 @@ interface OpenAIRealtimeSessionDependencies { ) => Promise; readonly now?: () => number; readonly reportDiagnostic?: VoiceDiagnosticReporter; + readonly requestAnimationFrame: (callback: FrameRequestCallback) => number; } type SessionListener = (event: OpenAIRealtimeSessionEvent) => void; @@ -104,6 +108,8 @@ const waitForAbort = ( export class OpenAIRealtimeSession { readonly #dependencies: OpenAIRealtimeSessionDependencies; readonly #listeners = new Set(); + #analyser: AnalyserNode | null = null; + #audioContext: AudioContext | null = null; #abortController: AbortController | null = null; #activeEpoch: number | null = null; #connected = false; @@ -112,6 +118,9 @@ export class OpenAIRealtimeSession { #dataChannel: RTCDataChannel | null = null; #epoch = 0; #mediaStream: MediaStream | null = null; + #meterFrame: number | null = null; + #meterHasSample = false; + #meterSamples: Uint8Array | null = null; #messageListener: ((event: MessageEvent) => void) | null = null; #microphoneTrack: MediaStreamTrack | null = null; #peerConnection: RTCPeerConnection | null = null; @@ -147,6 +156,21 @@ export class OpenAIRealtimeSession { }, this.#dependencies.connectionTimeoutMs); try { + let audioContext: AudioContext | null = null; + try { + audioContext = this.#dependencies.createAudioContext(); + this.#audioContext = audioContext; + if (audioContext.state === "suspended") { + try { + void audioContext.resume().catch(() => undefined); + } catch { + // Input metering is optional and must not block voice connection. + } + } + } catch { + // Input metering is optional and must not block voice connection. + } + let mediaStream: MediaStream; try { const mediaStreamPromise = this.#dependencies.getUserMedia({ @@ -209,6 +233,13 @@ export class OpenAIRealtimeSession { } microphoneTrack.enabled = false; this.#microphoneTrack = microphoneTrack; + if (audioContext) { + try { + this.#initializeMeter(audioContext, mediaStream); + } catch { + this.#releaseMeterResources(); + } + } const peerConnection = this.#dependencies.createPeerConnection(); this.#peerConnection = peerConnection; @@ -344,7 +375,13 @@ export class OpenAIRealtimeSession { public setMicrophoneEnabled(enabled: boolean): void { if (this.#microphoneTrack) { - this.#microphoneTrack.enabled = enabled && this.#connected; + const isEnabled = enabled && this.#connected; + this.#microphoneTrack.enabled = isEnabled; + if (isEnabled) { + this.#startMeter(); + } else { + this.#stopMeter(); + } } } @@ -410,6 +447,72 @@ export class OpenAIRealtimeSession { }); } + #initializeMeter(audioContext: AudioContext, mediaStream: MediaStream): void { + const analyser = audioContext.createAnalyser(); + analyser.fftSize = 256; + audioContext.createMediaStreamSource(mediaStream).connect(analyser); + this.#analyser = analyser; + this.#meterSamples = new Uint8Array(analyser.fftSize); + } + + #startMeter(): void { + if (this.#meterFrame !== null || !this.#analyser || !this.#meterSamples) { + return; + } + + const sample = () => { + if ( + !this.#microphoneTrack?.enabled || + !this.#analyser || + !this.#meterSamples + ) { + this.#stopMeter(); + return; + } + this.#analyser.getByteTimeDomainData(this.#meterSamples); + let squaredTotal = 0; + for (const value of this.#meterSamples) { + const normalized = (value - 128) / 128; + squaredTotal += normalized * normalized; + } + this.#emit({ + level: Math.min(1, Math.sqrt(squaredTotal / this.#meterSamples.length)), + type: "microphone-level", + }); + this.#meterHasSample = true; + this.#meterFrame = this.#dependencies.requestAnimationFrame(sample); + }; + + this.#meterFrame = this.#dependencies.requestAnimationFrame(sample); + } + + #stopMeter(): void { + if (this.#meterFrame === null) { + return; + } + this.#dependencies.cancelAnimationFrame(this.#meterFrame); + this.#meterFrame = null; + if (this.#meterHasSample) { + this.#emit({ level: 0, type: "microphone-level" }); + this.#meterHasSample = false; + } + } + + #releaseMeterResources(): void { + this.#stopMeter(); + this.#analyser = null; + this.#meterSamples = null; + const audioContext = this.#audioContext; + this.#audioContext = null; + if (audioContext) { + try { + void audioContext.close().catch(() => undefined); + } catch { + // Input metering cleanup is best-effort. + } + } + } + #handleMessage(event: MessageEvent, connectionEpoch: number): void { const parsed = parseRealtimeEvent(event.data); if (!parsed || typeof parsed.type !== "string") { @@ -539,6 +642,7 @@ export class OpenAIRealtimeSession { this.#connectionRequestId = null; this.#abortController?.abort(); this.#abortController = null; + this.#releaseMeterResources(); if (this.#dataChannel && this.#messageListener) { this.#dataChannel.removeEventListener("message", this.#messageListener); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx index 248bdafa010..cfb1d86a555 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx @@ -1,37 +1,205 @@ /** * @vitest-environment jsdom */ -import { act, StrictMode } from "react"; -import { createRoot } from "react-dom/client"; +import { + cleanup, + fireEvent, + render, + screen, + waitFor, +} from "@testing-library/react"; +import { StrictMode, useState } from "react"; import { renderToStaticMarkup } from "react-dom/server"; import { afterEach, describe, expect, test, vi } from "vitest"; import { + acknowledgeVoiceInterviewDisclosure, + isVoiceInterviewDisclosureAcknowledged, loadOpenAIVoiceConfig, + VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, VoiceInterviewControl, VoiceInterviewControlView, + type VoiceInterviewControlViewProps, } from "./voice-interview-control"; -describe("voice interview control", () => { - afterEach(() => { - vi.unstubAllGlobals(); +import type { PetrinautAiInterviewStageContext } from "@hashintel/petrinaut/ui"; + +const snapshot = { + canReviseLastAnswer: false, + currentQuestion: "What happens after approval?", + errorCode: null, + errorMessage: "", + errorRequestId: "", + lastCommittedText: "", + microphoneEnabled: true, + microphoneLevel: 0.24, + partialText: "The request goes to", + phase: "listening" as const, +}; + +const config = { available: true as const, connectionTimeoutMs: 15_000 }; + +const viewProps = ( + overrides: Partial = {}, +): VoiceInterviewControlViewProps => ({ + consented: true, + correction: "", + coverage: null, + editing: false, + microphoneCheck: "", + onCheckMicrophone: vi.fn(), + onConsentChange: vi.fn(), + onCorrectionChange: vi.fn(), + onDoneSpeaking: vi.fn(), + onEdit: vi.fn(), + onEnd: vi.fn(), + onExpand: vi.fn(), + onInterrupt: vi.fn(), + onMinimize: vi.fn(), + onPause: vi.fn(), + onReconnect: vi.fn(), + onRedo: vi.fn(), + onResume: vi.fn(), + onStart: vi.fn(), + onSubmitCorrection: vi.fn(), + onTypeInstead: vi.fn(), + placement: "sidebar", + presentation: "full", + snapshot, + ...overrides, +}); + +const StatefulVoiceInterviewHarness = ({ + onFocusComposer = vi.fn(), + onOpenSidebar, +}: { + onFocusComposer?: () => void; + onOpenSidebar: () => void; +}) => { + "use no memo"; + + const [active, setActive] = useState(false); + const [interactionMode, setInteractionMode] = + useState("chat"); + const [sidebarOpenRequests, setSidebarOpenRequests] = useState(0); + const context: PetrinautAiInterviewStageContext = { + canAcceptInterviewAnswer: true, + conversationId: "interview-test", + focusComposer: onFocusComposer, + interactionMode, + messages: [], + openSidebar: () => { + onOpenSidebar(); + setSidebarOpenRequests((requests) => requests + 1); + }, + placement: "sidebar", + setActive, + setInteractionMode, + status: "ready", + stop: vi.fn(async () => undefined), + submitInterviewAnswer: vi.fn(async () => ({ + kind: "message" as const, + messageId: "voice-answer", + })), + submitText: vi.fn(async () => ({ + kind: "message" as const, + messageId: "typed-answer", + })), + }; + + return ( + <> + + + {active ? "Interview active" : "Interview inactive"} + + {interactionMode === "chat" ? "Chat mode" : "Interview mode"} + + {sidebarOpenRequests} sidebar open requests + + + ); +}; + +const stubUnavailableMicrophone = () => { + const getUserMedia = vi.fn(async () => { + throw new DOMException("Permission denied", "NotAllowedError"); + }); + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ available: true, connectionTimeoutMs: 15_000 }), + ), + ); + vi.stubGlobal( + "AudioContext", + class { + public readonly state = "suspended"; + public readonly close = vi.fn(async () => undefined); + public readonly resume = vi.fn(async () => undefined); + }, + ); + vi.stubGlobal("navigator", { + mediaDevices: { getUserMedia }, + }); + return getUserMedia; +}; + +afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + window.localStorage.clear(); +}); + +describe("voice interview stage", () => { + test("stores and reads the current disclosure acknowledgement", () => { + const values = new Map(); + const storage = { + getItem: (key: string) => values.get(key) ?? null, + setItem: (key: string, value: string) => values.set(key, value), + }; + + expect(isVoiceInterviewDisclosureAcknowledged(storage)).toBe(false); + acknowledgeVoiceInterviewDisclosure(storage); + expect(values.get(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY)).toBe( + "acknowledged", + ); + expect(isVoiceInterviewDisclosureAcknowledged(storage)).toBe(true); + }); + + test("fails safe when disclosure storage is unavailable", () => { + const unavailableStorage = { + getItem: () => { + throw new DOMException("Blocked", "SecurityError"); + }, + setItem: () => { + throw new DOMException("Blocked", "SecurityError"); + }, + }; + + expect(isVoiceInterviewDisclosureAcknowledged(unavailableStorage)).toBe( + false, + ); + expect(() => + acknowledgeVoiceInterviewDisclosure(unavailableStorage), + ).not.toThrow(); }); - test("loads only a schema-valid, available server configuration", async () => { + test("loads only a schema-valid available server configuration", async () => { const fetch = vi.fn(async () => Response.json({ available: true, connectionTimeoutMs: 15_000 }), ); - await expect(loadOpenAIVoiceConfig(fetch)).resolves.toEqual({ available: true, connectionTimeoutMs: 15_000, }); const [url, request] = fetch.mock.calls[0]!; expect(url).toBe("/api/voice/config"); - expect(request).toMatchObject({ - cache: "no-store", - method: "GET", - }); + expect(request).toMatchObject({ cache: "no-store", method: "GET" }); expect(request?.signal).toBeInstanceOf(AbortSignal); fetch.mockResolvedValueOnce( @@ -44,179 +212,458 @@ describe("voice interview control", () => { await expect(loadOpenAIVoiceConfig(fetch)).resolves.toBeNull(); }); - test("renders an accessible idle voice action and live status", () => { + test("shows disclosure and requires consent before starting", () => { const html = renderToStaticMarkup( , ); - expect(html).toContain("Start voice input"); - expect(html).toContain('aria-live="polite"'); - expect(html).toContain("Voice input is off."); + expect(html).toContain("Voice interview"); + expect(html).toContain("Talk through your process with AI"); + expect(html).toContain("transcribed by OpenAI"); + expect(html).toContain("keeps finalized answers"); + expect(html).toContain("not the audio"); + expect(html.indexOf("Start interview")).toBeLessThan( + html.indexOf("Check microphone"), + ); + expect(html).toMatch(/]*disabled[^>]*>Start interview/u); + expect(html).toContain(">Use text instead<"); + expect(html).toContain("Check microphone"); + expect(html).toContain("pos_absolute"); + expect(html).not.toContain("pos_fixed"); }); - test("labels the half-duplex listening state and keeps partial text visibly provisional", () => { + test("keeps diagnostic recovery details visible without reopening the microphone", () => { const html = renderToStaticMarkup( , ); - expect(html).toContain("Microphone on. Listening."); - expect(html).toContain("Live transcript (not sent)"); - expect(html).toContain("The next activity"); + expect(html).toContain("We couldn’t connect"); expect(html).toContain( - "Microphone on. Listening. Live transcript (not sent): The next activity", + "Allow microphone access in your browser settings, then reconnect voice input.", ); - expect(html).toContain("End voice input"); - expect(html).toContain("Correct last voice answer"); - expect(html).toContain("Send correction"); + expect(html).toContain("Technical details"); + expect(html).toContain("microphone-permission"); + expect(html).toContain("voice-request-permission"); + expect(html).toContain(">Reconnect<"); + expect(html).toContain(">Use text instead<"); + expect(html).not.toContain(">Type instead<"); }); - test("offers reconnection without reopening the microphone after failure", () => { - const html = renderToStaticMarkup( - , + test("starts in the full stage and keeps recovery visible under Strict Mode", async () => { + const getUserMedia = stubUnavailableMicrophone(); + const openSidebar = vi.fn(); + + render( + + + , ); - expect(html).toContain( - "Microphone off. Allow microphone access in your browser settings, then reconnect voice input.", + fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + fireEvent.click(screen.getByRole("checkbox")); + fireEvent.click(screen.getByRole("button", { name: "Start interview" })); + + expect( + await screen.findByRole("region", { name: "Voice interview stage" }), + ).not.toBeNull(); + expect( + await screen.findAllByText( + /Microphone off · Allow microphone access in your browser settings, then reconnect voice input\./u, + ), + ).toHaveLength(2); + expect(screen.getByRole("button", { name: "Reconnect" })).not.toBeNull(); + expect(screen.getByText("Interview active")).not.toBeNull(); + expect(screen.getByText("Interview mode")).not.toBeNull(); + expect(screen.getByText("1 sidebar open requests")).not.toBeNull(); + expect(openSidebar).toHaveBeenCalledOnce(); + expect(getUserMedia).toHaveBeenCalledOnce(); + + fireEvent.click(screen.getByRole("button", { name: "Select Chat" })); + expect( + screen.getByRole("region", { name: "Voice interview mini bar" }), + ).not.toBeNull(); + expect(screen.getByText("Chat mode")).not.toBeNull(); + expect(openSidebar).toHaveBeenCalledOnce(); + }); + + test("uses full Interview and compact Chat presentations without ending", async () => { + window.localStorage.setItem( + VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, + "acknowledged", ); - expect(html).toContain("Error code: microphone-permission."); - expect(html).toContain("Diagnostic reference: voice-request-permission."); - expect(html).toContain("Reconnect voice input"); + const getUserMedia = stubUnavailableMicrophone(); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + expect( + await screen.findByRole("region", { name: "Voice interview stage" }), + ).not.toBeNull(); + + fireEvent.click( + screen.getByRole("button", { name: "Minimize voice interview" }), + ); + expect( + screen.getByRole("region", { name: "Voice interview mini bar" }), + ).not.toBeNull(); + expect(screen.getByText("Interview active")).not.toBeNull(); + + fireEvent.click( + screen.getByRole("button", { name: /Expand voice interview/u }), + ); + expect( + screen.getByRole("region", { name: "Voice interview stage" }), + ).not.toBeNull(); + expect(screen.getByText("Interview mode")).not.toBeNull(); + expect(getUserMedia).toHaveBeenCalledOnce(); }); - test("remains interactive after Strict Mode replays its effects", async () => { - const fetch = vi.fn(async () => - Response.json({ available: true, connectionTimeoutMs: 15_000 }), + test("ends the interview and returns to Chat", async () => { + window.localStorage.setItem( + VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, + "acknowledged", ); - const getUserMedia = vi.fn(async () => { - throw new DOMException("Permission denied", "NotAllowedError"); + stubUnavailableMicrophone(); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + expect( + await screen.findByRole("region", { name: "Voice interview stage" }), + ).not.toBeNull(); + + fireEvent.click(screen.getByRole("button", { name: "End interview" })); + + expect(screen.getByText("Chat mode")).not.toBeNull(); + expect(screen.getByText("Interview inactive")).not.toBeNull(); + await waitFor(() => { + expect( + screen.queryByRole("region", { name: "Voice interview stage" }), + ).toBeNull(); + expect( + screen.queryByRole("region", { name: "Voice interview mini bar" }), + ).toBeNull(); }); - vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); - vi.stubGlobal("fetch", fetch); - vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); - - const container = document.createElement("div"); - document.body.append(container); - const root = createRoot(container); - - try { - await act(async () => { - root.render( - - undefined)} - submitText={vi.fn(async () => ({ - kind: "message" as const, - messageId: "message-1", - }))} - /> - , - ); - }); - - const startButton = container.querySelector( - 'button[aria-label="Start voice input"]', - ); - expect(startButton).not.toBeNull(); - - await act(async () => { - startButton!.click(); - }); - - expect(getUserMedia).toHaveBeenCalledOnce(); - expect(container.textContent).toContain( - "Allow microphone access in your browser settings, then reconnect voice input.", - ); + }); + + test("records acknowledgement only when the interview starts", async () => { + window.localStorage.clear(); + stubUnavailableMicrophone(); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + fireEvent.click(screen.getByRole("button", { name: "Check microphone" })); + expect( + window.localStorage.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY), + ).toBeNull(); + + fireEvent.click(screen.getByRole("checkbox")); + fireEvent.click(screen.getByRole("button", { name: "Start interview" })); + expect( + window.localStorage.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY), + ).toBe("acknowledged"); + }); + + test("does not record acknowledgement when choosing text instead", async () => { + window.localStorage.clear(); + const focusComposer = vi.fn(); + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ available: true, connectionTimeoutMs: 15_000 }), + ), + ); + render( + , + ); + + fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + fireEvent.click(screen.getByRole("button", { name: "Use text instead" })); + expect( + window.localStorage.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY), + ).toBeNull(); + expect(screen.getByText("Chat mode")).not.toBeNull(); + expect(focusComposer).toHaveBeenCalledOnce(); + }); + + test("uses text from an active interview without ending the session", async () => { + window.localStorage.setItem( + VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, + "acknowledged", + ); + const focusComposer = vi.fn(); + stubUnavailableMicrophone(); + render( + , + ); + + fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + expect( + await screen.findByRole("region", { name: "Voice interview stage" }), + ).not.toBeNull(); + + fireEvent.click(screen.getByRole("button", { name: "Use text instead" })); + + expect(screen.getByText("Chat mode")).not.toBeNull(); + expect(screen.getByText("Interview active")).not.toBeNull(); + expect( + screen.getByRole("region", { name: "Voice interview mini bar" }), + ).not.toBeNull(); + expect(focusComposer).toHaveBeenCalledOnce(); + }); + + test("skips the disclosure after it has been acknowledged", async () => { + stubUnavailableMicrophone(); + window.localStorage.setItem( + VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, + "acknowledged", + ); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + + expect( + screen.queryByRole("region", { name: "Start voice interview" }), + ).toBeNull(); + expect( + await screen.findByRole("region", { name: "Voice interview stage" }), + ).not.toBeNull(); + }); + + test("keeps the question visible, distinguishes provisional text, and names microphone level", () => { + const html = renderToStaticMarkup( + , + ); + + expect(html).toContain("What happens after approval?"); + expect(html).toContain("What we’re hearing · Not sent yet"); + expect(html).toContain("Microphone on · Listening"); + expect(html).toContain("Microphone input level: Medium"); + expect(html).toContain("Done speaking"); + expect(html).toContain("motionReduce:vis_hidden"); + expect(html).toContain("pos_relative"); + expect(html).not.toContain("pos_fixed"); + expect(html).toContain('aria-live="polite"'); + }); + + test("renders icons for the listening controls", () => { + render(); + + for (const name of [ + "Minimize voice interview", + "End interview", + "Done speaking", + "Pause", + ]) { expect( - container.querySelector('button[aria-label="Reconnect voice input"]'), + screen.getByRole("button", { name }).querySelector("svg"), ).not.toBeNull(); - } finally { - await act(async () => root.unmount()); - container.remove(); } }); - test("announces synthesis, playback, and the AI-generated voice disclosure", () => { - const renderPhase = (phase: "synthesizing" | "playing") => - renderToStaticMarkup( - { + const waitingHtml = renderToStaticMarkup( + , + ); + + expect(waitingHtml).not.toContain( + "Microphone input level unavailable while microphone is off", + ); + }); + + test("renders committed repair actions separately from pause, minimize, and end", () => { + const html = renderToStaticMarkup( + , + ); + + for (const name of [ + "Minimize voice interview", + "End interview", + "Redo answer", + "Edit text", + "Use text instead", + ]) { + expect(html).toContain(name); + } + }); + + test("enables repair actions only while the last answer can be revised", () => { + const rendered = render( + , + ); + + expect( + screen + .getByRole("button", { name: "Redo answer" }) + .hasAttribute("disabled"), + ).toBe(true); + expect( + screen + .getByRole("button", { name: "Edit text" }) + .hasAttribute("disabled"), + ).toBe(true); + + rendered.rerender( + , + ); + + expect( + screen + .getByRole("button", { name: "Redo answer" }) + .hasAttribute("disabled"), + ).toBe(false); + expect( + screen + .getByRole("button", { name: "Edit text" }) + .hasAttribute("disabled"), + ).toBe(false); + }); + + test("offers deterministic interrupt instead of listening during playback", () => { + const html = renderToStaticMarkup( + , - ); + phase: "playing", + }, + })} + />, + ); + + expect(html).toContain("Microphone off · Interviewer speaking"); + expect(html).toContain("Interrupt and speak"); + expect(html).not.toContain(">Pause<"); + }); + + test("uses a detached bottom mini bar with independent expand, type, pause, and end controls", () => { + const html = renderToStaticMarkup( + , + ); - const synthesizing = renderPhase("synthesizing"); - expect(synthesizing).toContain( - "Microphone off. Creating AI-generated speech.", + expect(html).toContain('aria-label="Voice interview mini bar"'); + expect(html).toContain( + 'aria-label="Expand voice interview. Microphone on · Listening. Question: What happens after approval?"', ); - expect(synthesizing).toContain( - "Spoken responses use an AI-generated OpenAI voice.", + expect(html).toContain("Microphone on · Listening"); + expect(html).toContain("What happens after approval?"); + expect(html).toContain("--voice-interview-right"); + expect(html).toContain("[@media_(min-width:_768px)]"); + expect(html).not.toContain("md:right_4"); + expect(html).toContain('aria-label="Use text instead"'); + expect(html).toContain(">Pause<"); + expect(html).toContain('aria-label="End interview"'); + + render( + , ); + const endButton = screen.getByRole("button", { name: "End interview" }); + expect(endButton.querySelector("svg")).not.toBeNull(); + expect(endButton.parentElement?.getAttribute("data-part")).toBe("trigger"); + expect(endButton.parentElement?.getAttribute("data-scope")).toBe("tooltip"); + }); - const playing = renderPhase("playing"); - expect(playing).toContain("Microphone off. Playing AI-generated speech."); - expect(playing).toContain( - "Spoken responses use an AI-generated OpenAI voice.", + test("announces compact question and provisional transcript context", () => { + render( + , ); + + expect( + screen.getByRole("button", { + name: "Expand voice interview. Microphone on · Listening. Question: What happens after approval?", + }), + ).not.toBeNull(); + expect(screen.getByRole("status").textContent).toBe( + "Microphone on · Listening. Question: What happens after approval? Not sent yet: The request goes to", + ); + }); + + test("shows authoritative covered and still-exploring facts without a question count", () => { + const html = renderToStaticMarkup( + , + ); + + expect(html).toContain("Covered"); + expect(html).toContain("Still exploring"); + expect(html).not.toMatch(/\d+ of \d+/u); }); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index a435e2b2e1c..8f34aebbd07 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -1,30 +1,93 @@ import { type FormEvent, useEffect, + useRef, useState, useSyncExternalStore, } from "react"; -import { FaMicrophone, FaMicrophoneSlash } from "react-icons/fa6"; +import { + FaCheck, + FaKeyboard, + FaMicrophone, + FaMicrophoneSlash, + FaMinus, + FaPause, + FaXmark, +} from "react-icons/fa6"; import { Button } from "@hashintel/ds-components"; -import { css } from "@hashintel/ds-helpers/css"; +import { css, cva } from "@hashintel/ds-helpers/css"; import { reportVoiceDiagnostic } from "../../../voice-diagnostics"; import { selectCanonicalSpeechSegments } from "./canonical-speech"; +import { + type InterviewCoverage, + selectInterviewCoverage, +} from "./interview-coverage"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { SpeechPlaybackController } from "./speech-playback-controller"; import { VoiceTurnController, + type VoiceLatencyEvent, type VoiceTurnSnapshot, } from "./voice-turn-controller"; -import type { PetrinautAiComposerControlContext } from "@hashintel/petrinaut/ui"; +import type { PetrinautAiInterviewStageContext } from "@hashintel/petrinaut/ui"; export interface OpenAIVoiceConfig { readonly available: true; readonly connectionTimeoutMs: number; } +export const VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY = + "petrinaut:voice-interview-disclosure:v1"; +const VOICE_INTERVIEW_DISCLOSURE_ACKNOWLEDGED = "acknowledged"; + +const getVoiceInterviewDisclosureStorage = (): Storage | null => { + if (typeof window === "undefined") { + return null; + } + try { + return window.localStorage; + } catch { + return null; + } +}; + +export const isVoiceInterviewDisclosureAcknowledged = ( + storage: Pick< + Storage, + "getItem" + > | null = getVoiceInterviewDisclosureStorage(), +): boolean => { + try { + return ( + storage?.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY) === + VOICE_INTERVIEW_DISCLOSURE_ACKNOWLEDGED + ); + } catch { + return false; + } +}; + +export const acknowledgeVoiceInterviewDisclosure = ( + storage: Pick< + Storage, + "setItem" + > | null = getVoiceInterviewDisclosureStorage(), +): void => { + try { + storage?.setItem( + VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, + VOICE_INTERVIEW_DISCLOSURE_ACKNOWLEDGED, + ); + } catch { + // Storage is optional; the disclosure will appear again next time. + } +}; + +type Presentation = "start" | "full" | "mini"; + const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null; @@ -38,10 +101,7 @@ export const loadOpenAIVoiceConfig = async ( method: "GET", signal, }); - if (!response.ok) { - return null; - } - + if (!response.ok) return null; const body: unknown = await response.json(); if ( !isRecord(body) || @@ -61,77 +121,198 @@ export const loadOpenAIVoiceConfig = async ( } }; -const controlStyle = css({ - position: "relative", - flexShrink: "0", +const rootStyle = cva({ + base: { + zIndex: "overlay", + pointerEvents: "auto", + }, + variants: { + presentation: { + start: { + position: "absolute", + right: "0", + bottom: "[-2px]", + width: "full", + }, + full: { + position: "relative", + width: "full", + }, + mini: { + position: "relative", + width: "full", + }, + detached: { + position: "fixed", + "--voice-interview-right": "0px", + "--voice-interview-bottom": "0px", + "--voice-interview-left": "0px", + "--voice-interview-width": "100%", + right: "[var(--voice-interview-right)]", + bottom: "[var(--voice-interview-bottom)]", + left: "[var(--voice-interview-left)]", + width: "[var(--voice-interview-width)]", + "@media (min-width: 768px)": { + "--voice-interview-right": "var(--spacing-4)", + "--voice-interview-bottom": "var(--spacing-4)", + "--voice-interview-left": "auto", + "--voice-interview-width": "440px", + }, + }, + }, + }, }); -const panelStyle = css({ - position: "absolute", - right: "0", - bottom: "[calc(100% + 8px)]", - zIndex: "overlay", +const cardStyle = css({ display: "flex", - width: "[280px]", flexDirection: "column", - gap: "2", + gap: "3", + padding: "4", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + borderTopLeftRadius: "xl", + borderTopRightRadius: "xl", + borderBottomRightRadius: "xl", + borderBottomLeftRadius: "xl", + backgroundColor: "neutral.s00", + boxShadow: "xl", +}); + +const stageStyle = css({ + display: "flex", + maxHeight: "[72vh]", + flexDirection: "column", + gap: "3", padding: "3", + overflowY: "auto", borderWidth: "thin", borderStyle: "solid", borderColor: "neutral.a20", - borderRadius: "lg", backgroundColor: "neutral.s00", - boxShadow: "lg", + boxShadow: "[0 -8px 24px rgba(0,0,0,0.06)]", + borderRadius: "lg", }); -const statusStyle = css({ - color: "neutral.s90", - fontSize: "xs", - fontWeight: "medium", - lineHeight: "relaxed", +const headerStyle = css({ + display: "flex", + alignItems: "center", + gap: "2", +}); + +const startHeaderStyle = css({ + display: "flex", + alignItems: "center", + gap: "2", }); -const disclosureStyle = css({ - color: "neutral.s70", +const titleStyle = css({ + flex: "1", + color: "neutral.s100", + fontSize: "sm", + fontWeight: "semibold", +}); + +const subtitleStyle = css({ + color: "neutral.s80", fontSize: "xs", - lineHeight: "relaxed", + lineHeight: "snug", }); -const liveRegionStyle = css({ - position: "absolute", - width: "[1px]", - height: "[1px]", - padding: "0", - margin: "[-1px]", +const questionStyle = css({ + color: "neutral.s110", + fontSize: "lg", + fontWeight: "semibold", + lineHeight: "snug", +}); + +const contextStyle = css({ + display: "block", overflow: "hidden", - clip: "[rect(0, 0, 0, 0)]", + color: "neutral.s80", + fontSize: "xs", + lineHeight: "snug", + textOverflow: "ellipsis", whiteSpace: "nowrap", - borderWidth: "0", }); -const partialStyle = css({ +const miniTextStyle = css({ display: "flex", + minWidth: "0", flexDirection: "column", +}); + +const listeningStyle = css({ + display: "flex", + minHeight: "[84px]", + flexDirection: "column", + alignItems: "center", + justifyContent: "center", + gap: "2", + borderRadius: "lg", + backgroundColor: "blue.a10", +}); + +const meterStyle = css({ + display: "flex", + height: "[34px]", + alignItems: "center", gap: "1", - padding: "2", - borderRadius: "md", + _motionReduce: { visibility: "hidden" }, +}); + +const meterBarStyle = css({ + width: "[5px]", + minHeight: "[4px]", + borderRadius: "full", + backgroundColor: "blue.s70", + transition: "[height 80ms linear]", + _motionReduce: { transition: "[none]" }, +}); + +const statusStyle = css({ + color: "neutral.s90", + fontSize: "sm", + fontWeight: "medium", +}); + +const transcriptStyle = css({ + display: "flex", + flexDirection: "column", + gap: "1", + padding: "2.5", + borderRadius: "lg", backgroundColor: "neutral.s10", color: "neutral.s100", fontSize: "sm", }); -const partialLabelStyle = css({ +const labelStyle = css({ color: "neutral.s80", fontSize: "xs", + fontWeight: "semibold", +}); + +const phaseStyle = css({ + color: "blue.s80", + fontSize: "xs", + fontWeight: "semibold", }); -const correctionFormStyle = css({ +const technicalDetailsStyle = css({ + color: "neutral.s80", + fontSize: "xs", + _open: { color: "neutral.s90" }, +}); + +const actionsStyle = css({ display: "flex", - flexDirection: "column", + flexWrap: "wrap", + alignItems: "center", gap: "2", }); -const correctionInputStyle = css({ +const inputStyle = css({ width: "full", paddingX: "2", paddingY: "1.5", @@ -143,75 +324,337 @@ const correctionInputStyle = css({ color: "neutral.s100", fontSize: "sm", _focusVisible: { - borderColor: "blue.a70", outline: "2px solid", - outlineColor: "blue.a30", + outlineColor: "blue.a40", outlineOffset: "[1px]", }, }); -const panelActionsStyle = css({ +const miniStyle = css({ display: "flex", - justifyContent: "flex-end", + minHeight: "[60px]", + alignItems: "center", gap: "2", + padding: "2", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + borderTopLeftRadius: "lg", + borderTopRightRadius: "lg", + borderBottomRightRadius: "lg", + borderBottomLeftRadius: "lg", + backgroundColor: "neutral.s00", + boxShadow: "lg", +}); + +const miniExpandStyle = css({ + display: "flex", + minWidth: "0", + flex: "1", + alignItems: "center", + gap: "2", + padding: "2", + color: "neutral.s100", + textAlign: "left", + background: "[transparent]", + border: "none", + cursor: "pointer", + _focusVisible: { outline: "2px solid", outlineColor: "blue.a50" }, +}); + +const liveRegionStyle = css({ + position: "absolute", + width: "[1px]", + height: "[1px]", + padding: "0", + margin: "[-1px]", + overflow: "hidden", + clip: "[rect(0,0,0,0)]", + whiteSpace: "nowrap", + borderWidth: "0", }); const statusText = (snapshot: VoiceTurnSnapshot): string => { switch (snapshot.phase) { case "idle": - return "Voice input is off."; + return "Microphone off · Interview not started"; case "connecting": - return "Microphone off. Connecting voice input."; + return "Microphone off · Joining the interview"; case "listening": - return "Microphone on. Listening."; + return "Microphone on · Listening"; + case "paused": + return "Microphone off · Paused"; case "transcribing": - return "Microphone off. Finalizing the transcript."; + return "Microphone off · Finishing your answer"; case "delivering": - return "Microphone off. Sending the finalized transcript to Brunch."; + return "Microphone off · Answer recorded"; case "waiting": - return "Microphone off. Waiting for Brunch."; + return "Microphone off · Writing that down"; case "synthesizing": - return "Microphone off. Creating AI-generated speech."; + return "Microphone off · Preparing the next question"; case "playing": - return "Microphone off. Playing AI-generated speech."; + return "Microphone off · Interviewer speaking"; case "recoverable-error": { - const diagnostic = - snapshot.errorCode === null - ? "" - : ` Error code: ${snapshot.errorCode}.${ - snapshot.errorRequestId - ? ` Diagnostic reference: ${snapshot.errorRequestId}.` - : "" - }`; - return `Microphone off. ${snapshot.errorMessage}${diagnostic}`; + return `Microphone off · ${snapshot.errorMessage}`; } } }; -interface VoiceInterviewControlViewProps { +const inputLevelText = (level: number): string => + level >= 0.35 + ? "High" + : level >= 0.12 + ? "Medium" + : level > 0 + ? "Low" + : "Quiet"; + +const Meter = ({ snapshot }: { snapshot: VoiceTurnSnapshot }) => { + const level = snapshot.microphoneLevel; + return ( + <> + + + {`Microphone input level: ${inputLevelText(level)}`} + + + ); +}; + +const Coverage = ({ coverage }: { coverage: InterviewCoverage | null }) => { + if (!coverage) return null; + return ( +
+ + {coverage.complete ? "Coverage complete" : "Interview coverage"} + + {coverage.covered.length > 0 && ( +
+ Covered +
    + {coverage.covered.map((item) => ( +
  • {item}
    • + ))} +
+
+ )} + {coverage.stillExploring.length > 0 && ( +
+ Still exploring +
    + {coverage.stillExploring.map((item) => ( +
  • {item}
    • + ))} +
+
+ )} +
+ ); +}; + +export interface VoiceInterviewControlViewProps { + readonly consented: boolean; readonly correction: string; + readonly coverage: InterviewCoverage | null; + readonly editing: boolean; + readonly microphoneCheck: string; + readonly onCheckMicrophone: () => void; + readonly onConsentChange: (consented: boolean) => void; readonly onCorrectionChange: (value: string) => void; + readonly onDoneSpeaking: () => void; + readonly onEdit: () => void; readonly onEnd: () => void; + readonly onExpand: () => void; + readonly onInterrupt: () => void; + readonly onMinimize: () => void; + readonly onPause: () => void; readonly onReconnect: () => void; + readonly onRedo: () => void; + readonly onResume: () => void; readonly onStart: () => void; readonly onSubmitCorrection: () => void; + readonly onTypeInstead: () => void; + readonly placement: "sidebar" | "detached"; + readonly presentation: Presentation; readonly snapshot: VoiceTurnSnapshot; } export const VoiceInterviewControlView = ({ + consented, correction, + coverage, + editing, + microphoneCheck, + onCheckMicrophone, + onConsentChange, onCorrectionChange, + onDoneSpeaking, + onEdit, onEnd, + onExpand, + onInterrupt, + onMinimize, + onPause, onReconnect, + onRedo, + onResume, onStart, onSubmitCorrection, + onTypeInstead, + placement, + presentation, snapshot, }: VoiceInterviewControlViewProps) => { - const isIdle = snapshot.phase === "idle"; - const isConnecting = snapshot.phase === "connecting"; - const hasError = snapshot.phase === "recoverable-error"; - const canCorrect = - snapshot.phase === "listening" && snapshot.lastCommittedText.length > 0; + if (presentation === "start") { + if (placement === "detached") return null; + return ( +
+
+
+
+ Voice interview + + Talk through your process with AI + +
+ +
+

+ Your speech is transcribed by OpenAI. Petrinaut keeps finalized + answers in this conversation, not the audio. +

+ + {microphoneCheck &&

{microphoneCheck}

} +
+ + +
+
+
+ ); + } + + const effectivePresentation = + placement === "detached" ? "detached" : presentation; + const status = statusText(snapshot); + const isSpeaking = + snapshot.phase === "playing" || snapshot.phase === "synthesizing"; + const compactQuestionContext = snapshot.currentQuestion + ? ` Question: ${snapshot.currentQuestion}` + : ""; + const compactLiveOutput = `${status}.${compactQuestionContext}${ + snapshot.partialText ? ` Not sent yet: ${snapshot.partialText}` : "" + }`; + + if ( + effectivePresentation === "mini" || + effectivePresentation === "detached" + ) { + return ( +
+
+ + {isSpeaking ? ( + + ) : snapshot.phase === "paused" ? ( + + ) : ( + + )} +
+ + {compactLiveOutput} + +
+ ); + } const submitCorrection = (event: FormEvent) => { event.preventDefault(); @@ -219,122 +662,214 @@ export const VoiceInterviewControlView = ({ }; return ( -
- {isIdle ? ( - + + )} + +
+ {isSpeaking && ( + + )} + {snapshot.phase === "listening" && ( + <> + - + )} -
- {hasError ? ( + {snapshot.phase === "paused" && ( + + )} + {snapshot.phase === "recoverable-error" && ( + + )} + {snapshot.lastCommittedText && !snapshot.partialText && ( + <> - ) : ( - )} -
- - )} -
+ + )} + +
+ + + + + {status} + {snapshot.partialText && ` Not sent yet: ${snapshot.partialText}`} + + ); }; +const recordLatency = (event: VoiceLatencyEvent): void => { + try { + performance.measure(`voice-interview:${event.name}`, { + detail: { questionId: event.questionId }, + duration: event.elapsedMs, + start: 0, + }); + } catch { + // Performance measurement is optional and must not interrupt the interview. + } +}; + const AvailableVoiceInterviewControl = ({ config, context, }: { config: OpenAIVoiceConfig; - context: PetrinautAiComposerControlContext & { conversationId: string }; + context: PetrinautAiInterviewStageContext & { conversationId: string }; }) => { const [store] = useState(() => { const session = new OpenAIRealtimeSession({ + cancelAnimationFrame: (handle) => globalThis.cancelAnimationFrame(handle), connectionTimeoutMs: config.connectionTimeoutMs, + createAudioContext: () => new AudioContext(), createPeerConnection: () => new RTCPeerConnection(), fetch: globalThis.fetch.bind(globalThis), getUserMedia: (constraints) => navigator.mediaDevices.getUserMedia(constraints), reportDiagnostic: reportVoiceDiagnostic, + requestAnimationFrame: (callback) => + globalThis.requestAnimationFrame(callback), }); const playback = new SpeechPlaybackController({ createAudio: (source) => new Audio(source), @@ -345,9 +880,10 @@ const AvailableVoiceInterviewControl = ({ }); const controller = new VoiceTurnController({ conversationId: context.conversationId, + onLatencyEvent: recordLatency, playback, session, - submitText: context.submitText, + submitText: context.submitInterviewAnswer, }); return { controller, @@ -361,14 +897,67 @@ const AvailableVoiceInterviewControl = ({ store.getSnapshot, store.getSnapshot, ); + const [presentation, setPresentation] = useState("start"); + const [consented, setConsented] = useState(false); + const [microphoneCheck, setMicrophoneCheck] = useState(""); const [correction, setCorrection] = useState(""); + const [editing, setEditing] = useState(false); + const { interactionMode, openSidebar, setActive, setInteractionMode } = + context; + const openSidebarRef = useRef(openSidebar); + const handledInterviewSelectionRef = useRef(false); + const coverage = selectInterviewCoverage(context.messages); useEffect(() => { store.controller.updateChat({ + canAcceptInterviewAnswer: context.canAcceptInterviewAnswer, canonicalSegments: selectCanonicalSpeechSegments(context.messages), status: context.status, }); - }, [context.messages, context.status, store]); + }, [ + context.canAcceptInterviewAnswer, + context.messages, + context.status, + store, + ]); + + const active = snapshot.phase !== "idle"; + + useEffect(() => { + setActive(active); + }, [active, setActive]); + + useEffect(() => { + openSidebarRef.current = openSidebar; + }, [openSidebar]); + + useEffect(() => { + if (interactionMode === "chat") { + handledInterviewSelectionRef.current = false; + return; + } + if (handledInterviewSelectionRef.current || active) { + handledInterviewSelectionRef.current = true; + return; + } + + handledInterviewSelectionRef.current = true; + if (isVoiceInterviewDisclosureAcknowledged()) { + // eslint-disable-next-line react-hooks-js/set-state-in-effect -- Mode selection intentionally drives the host presentation. + setPresentation("full"); + setActive(true); + void store.controller.start(); + } else { + setPresentation("start"); + } + }, [active, interactionMode, setActive, store]); + + useEffect(() => { + if (snapshot.phase === "recoverable-error") { + setInteractionMode("interview"); + openSidebarRef.current(); + } + }, [setInteractionMode, snapshot.phase]); useEffect( () => () => { @@ -377,50 +966,114 @@ const AvailableVoiceInterviewControl = ({ [store], ); + const end = () => { + setEditing(false); + context.setActive(false); + context.setInteractionMode("chat"); + void store.controller.end(); + }; + + const minimize = () => context.setInteractionMode("chat"); + + const expand = () => { + context.openSidebar(); + context.setInteractionMode("interview"); + }; + + const useTextInstead = () => { + context.setInteractionMode("chat"); + context.focusComposer(); + }; + + const startInterview = () => { + setPresentation("full"); + context.setActive(true); + void store.controller.start(); + }; + + const visiblePresentation = + snapshot.phase === "recoverable-error" + ? context.interactionMode === "chat" + ? "mini" + : "full" + : context.placement === "detached" + ? active + ? "mini" + : null + : context.interactionMode === "chat" + ? active + ? "mini" + : null + : presentation; + + if (visiblePresentation === null) { + return null; + } + return ( { + setMicrophoneCheck("Checking microphone…"); + void navigator.mediaDevices.getUserMedia({ audio: true }).then( + (stream) => { + for (const track of stream.getTracks()) track.stop(); + setMicrophoneCheck("Microphone ready."); + }, + () => setMicrophoneCheck("Microphone access was not available."), + ); + }} + onConsentChange={setConsented} onCorrectionChange={setCorrection} - onEnd={() => void store.controller.end()} - onReconnect={() => void store.controller.reconnect()} - onStart={() => void store.controller.start()} + onDoneSpeaking={() => store.controller.doneSpeaking()} + onEdit={() => setEditing(true)} + onEnd={end} + onExpand={expand} + onInterrupt={() => store.controller.interruptAndSpeak()} + onMinimize={minimize} + onPause={() => store.controller.pause()} + onReconnect={() => { + setPresentation("full"); + void store.controller.reconnect(); + }} + onRedo={() => store.controller.redoAnswer()} + onResume={() => store.controller.resume()} + onStart={() => { + acknowledgeVoiceInterviewDisclosure(); + startInterview(); + }} onSubmitCorrection={() => { const value = correction; - setCorrection(""); - void store.controller.submitCorrection(value); + void store.controller.submitCorrection(value).then((accepted) => { + if (accepted) { + setCorrection(""); + setEditing(false); + } + }); }} + onTypeInstead={useTextInstead} + placement={context.placement} + presentation={visiblePresentation} snapshot={snapshot} /> ); }; -export const VoiceInterviewControl = ( - context: PetrinautAiComposerControlContext, -) => { - const [config, setConfig] = useState(); - - useEffect(() => { - const abortController = new AbortController(); - void loadOpenAIVoiceConfig( - globalThis.fetch.bind(globalThis), - abortController.signal, - ).then((loadedConfig) => { - if (!abortController.signal.aborted) { - setConfig(loadedConfig); - } - }); - return () => abortController.abort(); - }, []); - - if (!config || !context.conversationId) { - return null; - } - +export const VoiceInterviewControl = ({ + config, + ...context +}: PetrinautAiInterviewStageContext & { + readonly config: OpenAIVoiceConfig; +}) => { return ( ); }; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts index eddbfd80a52..60423bb73b5 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts @@ -161,14 +161,25 @@ describe("controlled voice preview", () => { }; let requestNumber = 0; const createRequestId = () => requestIds[requestNumber++]!; + const audioContext = { + close: vi.fn(async () => undefined), + createAnalyser: vi.fn(() => ({ + fftSize: 0, + getByteTimeDomainData: vi.fn(), + })), + createMediaStreamSource: vi.fn(() => ({ connect: vi.fn() })), + }; const session = new OpenAIRealtimeSession({ + cancelAnimationFrame: vi.fn(), connectionTimeoutMs: 15_000, + createAudioContext: () => audioContext as unknown as AudioContext, createPeerConnection: () => peer as unknown as RTCPeerConnection, createRequestId, fetch: browserFetch, getUserMedia: async () => mediaStream, now: clock, reportDiagnostic, + requestAnimationFrame: vi.fn(() => 1), }); const audio = createAudioHarness(); const revokeObjectURL = vi.fn(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index 6f35e329567..d927ec77eb6 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -24,6 +24,11 @@ const createHarness = () => { }), }; const submitText = vi.fn(async () => ({ kind: "message" as const })); + const latencyEvents: Array<{ + elapsedMs: number; + name: string; + questionId: string; + }> = []; const playback = { cancel: vi.fn(), play: vi.fn( @@ -37,6 +42,14 @@ const createHarness = () => { }; const controller = new VoiceTurnController({ conversationId: "preview/net 1", + now: (() => { + let now = 1_000; + return () => { + now += 100; + return now; + }; + })(), + onLatencyEvent: (event) => latencyEvents.push(event), playback, session, submitText, @@ -45,6 +58,7 @@ const createHarness = () => { return { controller, emit: (event: OpenAIRealtimeSessionEvent) => listener?.(event), + latencyEvents, playback, session, submitText, @@ -75,6 +89,312 @@ const updateChatStatus = ( ) => controller.updateChat({ canonicalSegments: [], status }); describe("VoiceTurnController", () => { + test("speaks a pending interview question and opens capture before generic chat settlement", async () => { + const harness = createHarness(); + let finishPlayback: (() => void) | undefined; + harness.playback.play.mockImplementationOnce( + async (_segment, events = {}) => { + events.onPlaying?.(); + await new Promise((resolve) => { + finishPlayback = resolve; + }); + }, + ); + const question = { + ...canonicalSegment("ask-1", "What happens after approval?"), + source: "brunch-ask" as const, + }; + + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question], + status: "streaming", + }); + await harness.controller.start(); + + expect(harness.playback.play).toHaveBeenCalledWith( + question, + expect.any(Object), + ); + expect(harness.controller.getSnapshot()).toMatchObject({ + currentQuestion: "What happens after approval?", + microphoneEnabled: false, + phase: "playing", + }); + + finishPlayback?.(); + await vi.waitFor(() => + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: true, + phase: "listening", + }), + ); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); + }); + + test("cancels playback completely before enabling the microphone to interrupt", async () => { + const order: string[] = []; + const harness = createHarness(); + harness.playback.cancel.mockImplementation(() => order.push("cancel")); + harness.session.setMicrophoneEnabled.mockImplementation((enabled) => { + if (enabled) order.push("microphone-on"); + }); + harness.playback.play.mockImplementationOnce( + async (_segment, events = {}) => { + events.onPlaying?.(); + await new Promise(() => undefined); + }, + ); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [ + { + ...canonicalSegment("ask-1"), + source: "brunch-ask" as const, + }, + ], + status: "streaming", + }); + await harness.controller.start(); + + harness.controller.interruptAndSpeak(); + + expect(order.slice(-2)).toEqual(["cancel", "microphone-on"]); + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: true, + phase: "listening", + }); + }); + + test("relies on server VAD after the expert is done speaking", async () => { + const order: string[] = []; + const harness = createHarness(); + harness.session.setMicrophoneEnabled.mockImplementation((enabled) => { + if (!enabled) order.push("microphone-off"); + }); + await harness.controller.start(); + order.length = 0; + + harness.controller.doneSpeaking(); + + expect(order).toEqual(["microphone-off"]); + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: false, + phase: "transcribing", + }); + }); + + test("records answer-to-question visibility and speech latency", async () => { + const harness = createHarness(); + await harness.controller.start(); + harness.emit({ + key: key(1, "answer-1"), + text: "Approval sends it to dispatch.", + type: "completed", + }); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [ + { + ...canonicalSegment("ask-next"), + source: "brunch-ask" as const, + }, + ], + status: "streaming", + }); + + await vi.waitFor(() => + expect(harness.latencyEvents.map(({ name }) => name)).toEqual([ + "question-visible", + "question-spoken-started", + "question-spoken", + "answer-ready", + ]), + ); + expect(harness.latencyEvents.every(({ elapsedMs }) => elapsedMs >= 0)).toBe( + true, + ); + }); + + test("pauses without ending the connection and redoes an answer as an explicit correction", async () => { + const harness = createHarness(); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [ + { + ...canonicalSegment("ask-1"), + source: "brunch-ask" as const, + }, + ], + status: "ready", + }); + await harness.controller.start(); + + harness.controller.pause(); + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: false, + phase: "paused", + }); + expect(harness.session.disconnect).not.toHaveBeenCalled(); + + harness.controller.resume(); + harness.emit({ + key: key(1, "answer-1"), + text: "The operator approves it.", + type: "completed", + }); + await vi.waitFor(() => expect(harness.submitText).toHaveBeenCalledOnce()); + updateChatStatus(harness.controller, "streaming"); + updateChatStatus(harness.controller, "ready"); + harness.controller.redoAnswer(); + expect(harness.controller.getSnapshot().phase).toBe("listening"); + harness.emit({ + key: key(1, "answer-2"), + text: "The shift lead approves it.", + type: "completed", + }); + await vi.waitFor(() => expect(harness.submitText).toHaveBeenCalledTimes(2)); + expect(harness.submitText).toHaveBeenLastCalledWith({ + target: "message", + text: 'Correction to my previous voice answer "The operator approves it.": The shift lead approves it.', + }); + }); + + test("keeps redo disabled while the previous answer is pending", async () => { + const harness = createHarness(); + let finishDelivery: (() => void) | undefined; + harness.submitText.mockImplementationOnce( + () => + new Promise((resolve) => { + finishDelivery = () => resolve({ kind: "message" as const }); + }), + ); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [ + { + ...canonicalSegment("ask-pending-redo"), + source: "brunch-ask" as const, + }, + ], + status: "ready", + }); + await harness.controller.start(); + harness.emit({ + key: key(1, "answer-pending-redo"), + text: "The operator approves it.", + type: "completed", + }); + await vi.waitFor(() => expect(harness.submitText).toHaveBeenCalledOnce()); + + harness.controller.redoAnswer(); + + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: false, + phase: "delivering", + }); + finishDelivery?.(); + }); + + test("holds queued question speech while paused and starts it on resume", async () => { + const harness = createHarness(); + let finishPlayback: (() => void) | undefined; + harness.playback.play.mockImplementationOnce( + async (_segment, events = {}) => { + events.onPlaying?.(); + await new Promise((resolve) => { + finishPlayback = resolve; + }); + }, + ); + await harness.controller.start(); + harness.controller.pause(); + const question = { + ...canonicalSegment("ask-paused"), + source: "brunch-ask" as const, + }; + + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question], + status: "streaming", + }); + + expect(harness.playback.play).not.toHaveBeenCalled(); + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: false, + phase: "paused", + }); + + harness.controller.resume(); + await vi.waitFor(() => + expect(harness.controller.getSnapshot().phase).toBe("playing"), + ); + expect(harness.playback.play).toHaveBeenCalledWith( + question, + expect.any(Object), + ); + + finishPlayback?.(); + await vi.waitFor(() => + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: true, + phase: "listening", + }), + ); + }); + + test("answers a new question normally after redo was armed", async () => { + const harness = createHarness(); + const firstQuestion = { + ...canonicalSegment("ask-first"), + source: "brunch-ask" as const, + }; + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [firstQuestion], + status: "ready", + }); + await harness.controller.start(); + harness.emit({ + key: key(1, "answer-first"), + text: "The operator approves it.", + type: "completed", + }); + await vi.waitFor(() => expect(harness.submitText).toHaveBeenCalledOnce()); + updateChatStatus(harness.controller, "streaming"); + updateChatStatus(harness.controller, "ready"); + harness.controller.redoAnswer(); + + const nextQuestion = { + ...canonicalSegment("ask-next"), + source: "brunch-ask" as const, + }; + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [firstQuestion, nextQuestion], + status: "ready", + }); + await vi.waitFor(() => + expect(harness.controller.getSnapshot()).toMatchObject({ + currentQuestion: nextQuestion.text, + phase: "listening", + }), + ); + const nextAnswerKey = key(1, "answer-next"); + harness.emit({ + key: nextAnswerKey, + text: "The supervisor dispatches it.", + type: "completed", + }); + + await vi.waitFor(() => expect(harness.submitText).toHaveBeenCalledTimes(2)); + expect(harness.submitText).toHaveBeenLastCalledWith({ + id: createVoiceMessageId("preview/net 1", nextAnswerKey), + text: "The supervisor dispatches it.", + }); + }); + test("does not surface an unrelated Brunch error before voice starts", async () => { const harness = createHarness(); @@ -88,7 +408,7 @@ describe("VoiceTurnController", () => { await harness.controller.start(); expect(harness.controller.getSnapshot()).toMatchObject({ - errorMessage: "Wait for Brunch to finish before starting voice input.", + errorMessage: "Wait for the current response to finish before starting.", phase: "recoverable-error", }); }); @@ -104,7 +424,7 @@ describe("VoiceTurnController", () => { false, ); expect(harness.controller.getSnapshot()).toMatchObject({ - errorMessage: "Wait for Brunch to finish before starting voice input.", + errorMessage: "Wait for the current response to finish before starting.", phase: "recoverable-error", }); }); @@ -128,6 +448,38 @@ describe("VoiceTurnController", () => { expect(harness.controller.getSnapshot().phase).toBe("listening"); }); + test("waits for answer capacity before opening the microphone after connection", async () => { + const harness = createHarness(); + harness.controller.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "ready", + }); + + await harness.controller.start(); + + expect(harness.session.connect).toHaveBeenCalledOnce(); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( + false, + ); + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: false, + phase: "waiting", + }); + + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [], + status: "ready", + }); + + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: true, + phase: "listening", + }); + }); + test("preserves a Brunch error that occurs while voice is connecting", async () => { const harness = createHarness(); let finishConnection: (() => void) | undefined; @@ -146,7 +498,7 @@ describe("VoiceTurnController", () => { expect(harness.session.disconnect).toHaveBeenCalledOnce(); expect(harness.controller.getSnapshot()).toMatchObject({ errorMessage: - "Brunch could not complete the current turn. Use the composer to retry.", + "The interview could not complete that turn. Use the composer to retry.", phase: "recoverable-error", }); }); @@ -193,9 +545,11 @@ describe("VoiceTurnController", () => { type: "partial", }); expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: true, partialText: "The support", - phase: "transcribing", + phase: "listening", }); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); expect(harness.submitText).not.toHaveBeenCalled(); harness.emit({ @@ -226,22 +580,6 @@ describe("VoiceTurnController", () => { expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); }); - test("closes the microphone when a partial transcript arrives", async () => { - const harness = createHarness(); - await harness.controller.start(); - - harness.emit({ - key: key(1, "item-a"), - text: "The support", - type: "partial", - }); - - expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( - false, - ); - expect(harness.controller.getSnapshot().phase).toBe("transcribing"); - }); - test("ignores duplicate, stale, and out-of-order completed items", async () => { const harness = createHarness(); await harness.controller.start(); @@ -468,7 +806,8 @@ describe("VoiceTurnController", () => { expect.objectContaining({ text: "Accepted final" }), ); expect(harness.controller.getSnapshot()).toMatchObject({ - lastCommittedText: "Accepted final", + canReviseLastAnswer: false, + lastCommittedText: "", phase: "idle", }); }); @@ -497,6 +836,70 @@ describe("VoiceTurnController", () => { ); }); + test("reconnects to a pending question without waiting for generic chat settlement", async () => { + const harness = createHarness(); + const question = { + ...canonicalSegment("ask-reconnect", "What happens after approval?"), + source: "brunch-ask" as const, + }; + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question], + status: "streaming", + }); + await harness.controller.start(); + await vi.waitFor(() => + expect(harness.controller.getSnapshot().phase).toBe("listening"), + ); + harness.emit({ + code: "network", + message: "The voice connection failed. Try reconnecting.", + requestId: "voice-request-reconnect", + type: "error", + }); + + await harness.controller.reconnect(); + + await vi.waitFor(() => + expect(harness.controller.getSnapshot()).toMatchObject({ + currentQuestion: "What happens after approval?", + microphoneEnabled: true, + phase: "listening", + }), + ); + expect(harness.playback.play).toHaveBeenCalledTimes(2); + expect(harness.session.connect).toHaveBeenCalledTimes(2); + }); + + test("replays a pending question after ending and restarting", async () => { + const harness = createHarness(); + const question = { + ...canonicalSegment("ask-restart", "What happens after approval?"), + source: "brunch-ask" as const, + }; + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question], + status: "ready", + }); + await harness.controller.start(); + await vi.waitFor(() => + expect(harness.controller.getSnapshot().phase).toBe("listening"), + ); + + await harness.controller.end(); + await harness.controller.start(); + + await vi.waitFor(() => + expect(harness.controller.getSnapshot()).toMatchObject({ + currentQuestion: "What happens after approval?", + microphoneEnabled: true, + phase: "listening", + }), + ); + expect(harness.playback.play).toHaveBeenCalledTimes(2); + }); + test("does not let a late delivery overwrite a connection failure", async () => { const harness = createHarness(); let finishDelivery: (() => void) | undefined; @@ -555,6 +958,108 @@ describe("VoiceTurnController", () => { ); }); + test("reports a rejected correction so the caller can preserve its draft", async () => { + const harness = createHarness(); + await harness.controller.start(); + harness.emit({ + key: key(1, "item-a"), + text: "The support lead closes it.", + type: "completed", + }); + await vi.waitFor(() => expect(harness.submitText).toHaveBeenCalledOnce()); + updateChatStatus(harness.controller, "streaming"); + updateChatStatus(harness.controller, "ready"); + harness.submitText.mockRejectedValueOnce( + new Error("A queued answer already exists"), + ); + + const accepted = await harness.controller.submitCorrection( + "The incident manager closes it.", + ); + + expect(accepted).toBe(false); + expect(harness.controller.getSnapshot().phase).toBe("recoverable-error"); + }); + + test("ignores a typed correction while the previous answer is still pending", async () => { + const harness = createHarness(); + let finishDelivery: (() => void) | undefined; + harness.submitText + .mockImplementationOnce( + () => + new Promise((resolve) => { + finishDelivery = () => resolve({ kind: "message" as const }); + }), + ) + .mockRejectedValueOnce(new Error("A queued answer already exists")); + await harness.controller.start(); + harness.emit({ + key: key(1, "answer-pending"), + text: "The support lead closes it.", + type: "completed", + }); + await vi.waitFor(() => expect(harness.submitText).toHaveBeenCalledOnce()); + expect(harness.controller.getSnapshot()).toMatchObject({ + canReviseLastAnswer: false, + phase: "delivering", + }); + + const accepted = await harness.controller.submitCorrection( + "The incident manager closes it.", + ); + + expect(accepted).toBe(false); + expect(harness.submitText).toHaveBeenCalledOnce(); + expect(harness.controller.getSnapshot().phase).toBe("delivering"); + finishDelivery?.(); + }); + + test("closes capture and ignores correction while answer capacity is unavailable", async () => { + const harness = createHarness(); + await harness.controller.start(); + harness.emit({ + key: key(1, "answer-before-capacity-closes"), + text: "The support lead closes it.", + type: "completed", + }); + await vi.waitFor(() => expect(harness.submitText).toHaveBeenCalledOnce()); + updateChatStatus(harness.controller, "streaming"); + updateChatStatus(harness.controller, "ready"); + harness.controller.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "ready", + }); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( + false, + ); + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: false, + phase: "waiting", + }); + + await harness.controller.submitCorrection( + "The incident manager closes it.", + ); + + expect(harness.submitText).toHaveBeenCalledOnce(); + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: false, + phase: "waiting", + }); + + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [], + status: "ready", + }); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: true, + phase: "listening", + }); + }); + test("seeds finalized history without replaying it when voice starts or reconnects", async () => { const harness = createHarness(); const history = canonicalSegment( diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index a76dbf928dd..d3eebb4b7c9 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -10,6 +10,7 @@ export type VoiceTurnPhase = | "idle" | "connecting" | "listening" + | "paused" | "transcribing" | "delivering" | "waiting" @@ -18,14 +19,28 @@ export type VoiceTurnPhase = | "recoverable-error"; export interface VoiceTurnSnapshot { + readonly canReviseLastAnswer: boolean; + readonly currentQuestion: string; readonly errorCode: VoiceErrorCode | null; readonly errorMessage: string; readonly errorRequestId: string; readonly lastCommittedText: string; + readonly microphoneEnabled: boolean; + readonly microphoneLevel: number; readonly partialText: string; readonly phase: VoiceTurnPhase; } +export interface VoiceLatencyEvent { + readonly elapsedMs: number; + readonly name: + | "question-visible" + | "question-spoken-started" + | "question-spoken" + | "answer-ready"; + readonly questionId: string; +} + type ChatStatus = "ready" | "submitted" | "streaming" | "error"; interface RealtimeSession { @@ -51,12 +66,15 @@ interface SpeechPlayback { interface VoiceTurnControllerDependencies { readonly conversationId: string; + readonly now?: () => number; + readonly onLatencyEvent?: (event: VoiceLatencyEvent) => void; readonly playback: SpeechPlayback; readonly session: RealtimeSession; readonly submitText: (input: SubmitTextInput) => Promise; } interface ChatUpdate { + readonly canAcceptInterviewAnswer?: boolean; readonly canonicalSegments: CanonicalSpeechSegment[]; readonly status: ChatStatus; } @@ -79,10 +97,14 @@ export const createVoiceMessageId = ( ].join(":"); const initialSnapshot: VoiceTurnSnapshot = { + canReviseLastAnswer: false, + currentQuestion: "", errorCode: null, errorMessage: "", errorRequestId: "", lastCommittedText: "", + microphoneEnabled: false, + microphoneLevel: 0, partialText: "", phase: "idle", }; @@ -90,30 +112,45 @@ const initialSnapshot: VoiceTurnSnapshot = { export class VoiceTurnController { readonly #conversationId: string; readonly #listeners = new Set(); + readonly #now: () => number; + readonly #onLatencyEvent: ((event: VoiceLatencyEvent) => void) | undefined; readonly #playback: SpeechPlayback; readonly #session: RealtimeSession; readonly #submitText: (input: SubmitTextInput) => Promise; readonly #completedKeys = new Set(); readonly #seenSpeechSegmentIds = new Set(); readonly #speechQueue: CanonicalSpeechSegment[] = []; + #answerFinalizedAt: number | null = null; + #answerReadyQuestionId: string | null = null; #activeEpoch: number | null = null; #activeItemId: string | null = null; #activeKey: string | null = null; #activeSpeechSegmentId: string | null = null; #awaitingChatCycle = false; + #availableSegments: CanonicalSpeechSegment[] = []; + #canAcceptInterviewAnswer = true; #chatStatus: ChatStatus = "ready"; + #currentQuestionId: string | null = null; #generation = 0; + #paused = false; + #questionAnswered = false; + #questionPlaybackComplete = false; + #redoing = false; #sawBusyChatStatus = false; #snapshot = initialSnapshot; #speechLoopGeneration: number | null = null; public constructor({ conversationId, + now = () => performance.now(), + onLatencyEvent, playback, session, submitText, }: VoiceTurnControllerDependencies) { this.#conversationId = conversationId; + this.#now = now; + this.#onLatencyEvent = onLatencyEvent; this.#playback = playback; this.#session = session; this.#submitText = submitText; @@ -133,10 +170,12 @@ export class VoiceTurnController { if (this.#snapshot.phase !== "idle") { return; } - if (!this.#isChatReady()) { - this.#session.setMicrophoneEnabled(false); + this.#queueAvailableSegments(); + if (!this.#isChatReady() && !this.#hasPendingQuestion()) { + this.#setMicrophoneEnabled(false); this.#update({ - errorMessage: "Wait for Brunch to finish before starting voice input.", + errorMessage: + "Wait for the current response to finish before starting.", phase: "recoverable-error", }); return; @@ -153,8 +192,13 @@ export class VoiceTurnController { this.#activeItemId = null; this.#activeKey = null; this.#completedKeys.clear(); - const canListen = this.#isChatReady() && this.#speechQueue.length === 0; - this.#session.setMicrophoneEnabled(canListen); + this.#paused = false; + const canListen = + this.#canAcceptInterviewAnswer && + this.#isChatReady() && + this.#speechQueue.length === 0 && + !this.#hasAnswerableQuestion(); + this.#setMicrophoneEnabled(canListen); this.#update({ partialText: "", phase: canListen ? "listening" : "waiting", @@ -168,7 +212,7 @@ export class VoiceTurnController { error instanceof VoiceError ? error : new VoiceError("connection", "invalid-response", ""); - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); this.#update({ errorCode: voiceError.code, errorMessage: voiceError.message, @@ -179,50 +223,131 @@ export class VoiceTurnController { } public async end(): Promise { + if (!this.#questionAnswered && this.#currentQuestionId !== null) { + this.#seenSpeechSegmentIds.delete(this.#currentQuestionId); + } ++this.#generation; this.#activeEpoch = null; this.#activeItemId = null; this.#activeKey = null; this.#activeSpeechSegmentId = null; this.#awaitingChatCycle = false; + this.#answerFinalizedAt = null; + this.#answerReadyQuestionId = null; + this.#currentQuestionId = null; + this.#paused = false; + this.#questionAnswered = false; + this.#questionPlaybackComplete = false; + this.#redoing = false; this.#sawBusyChatStatus = false; this.#speechLoopGeneration = null; this.#speechQueue.length = 0; this.#playback.cancel(); - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); await this.#session.disconnect(); this.#update({ errorMessage: "", + currentQuestion: "", + lastCommittedText: "", + microphoneLevel: 0, partialText: "", phase: "idle", }); } public async reconnect(): Promise { + const pendingQuestionId = this.#currentQuestionId; await this.end(); + if ( + pendingQuestionId !== null && + this.#availableSegments.some( + (segment) => + segment.id === pendingQuestionId && segment.source === "brunch-ask", + ) + ) { + this.#seenSpeechSegmentIds.delete(pendingQuestionId); + } await this.start(); } - public async submitCorrection(correction: string): Promise { + public async submitCorrection(correction: string): Promise { const correctedText = correction.trim(); const previousText = this.#snapshot.lastCommittedText; - if ( - !correctedText || - !previousText || - this.#snapshot.phase !== "listening" - ) { - return; + if (!correctedText || !this.#canReviseLastAnswer()) { + return false; } - this.#session.setMicrophoneEnabled(false); + this.#questionAnswered = true; + this.#setMicrophoneEnabled(false); this.#update({ errorMessage: "", phase: "delivering" }); - await this.#deliver({ + return this.#deliver({ target: "message", text: `Correction to my previous voice answer "${previousText}": ${correctedText}`, }); } - public updateChat({ canonicalSegments, status }: ChatUpdate): void { + public interruptAndSpeak(): void { + if ( + this.#activeEpoch === null || + (this.#snapshot.phase !== "playing" && + this.#snapshot.phase !== "synthesizing") + ) { + return; + } + + ++this.#generation; + this.#speechLoopGeneration = null; + this.#activeSpeechSegmentId = null; + this.#speechQueue.length = 0; + this.#playback.cancel(); + this.#questionPlaybackComplete = Boolean(this.#snapshot.currentQuestion); + this.#paused = false; + this.#settleListeningIfReady(); + } + + public doneSpeaking(): void { + if (this.#activeEpoch === null || this.#snapshot.phase !== "listening") { + return; + } + this.#setMicrophoneEnabled(false); + this.#update({ phase: "transcribing" }); + } + + public pause(): void { + if (this.#activeEpoch === null || this.#snapshot.phase !== "listening") { + return; + } + this.#paused = true; + this.#setMicrophoneEnabled(false); + this.#update({ phase: "paused" }); + } + + public resume(): void { + if (this.#snapshot.phase !== "paused") { + return; + } + this.#paused = false; + this.#startSpeechQueueIfNeeded(); + this.#settleListeningIfReady(); + } + + public redoAnswer(): void { + if (!this.#canReviseLastAnswer()) { + return; + } + this.#redoing = true; + this.#questionAnswered = false; + this.#paused = false; + this.#settleListeningIfReady(); + } + + public updateChat({ + canAcceptInterviewAnswer = true, + canonicalSegments, + status, + }: ChatUpdate): void { + this.#availableSegments = canonicalSegments; + this.#canAcceptInterviewAnswer = canAcceptInterviewAnswer; this.#chatStatus = status; const canQueueSpeech = status !== "error" && @@ -232,9 +357,10 @@ export class VoiceTurnController { if (this.#seenSpeechSegmentIds.has(segment.id)) { continue; } - this.#seenSpeechSegmentIds.add(segment.id); if (canQueueSpeech) { - this.#speechQueue.push(segment); + this.#queueSpeechSegment(segment); + } else if (segment.source === "assistant-text") { + this.#seenSpeechSegmentIds.add(segment.id); } } @@ -243,8 +369,8 @@ export class VoiceTurnController { return; } const errorMessage = this.#awaitingChatCycle - ? "Brunch could not accept the voice turn. Use the composer to retry." - : "Brunch could not complete the current turn. Use the composer to retry."; + ? "The interview could not accept that answer. Use the composer to retry." + : "The interview could not complete that turn. Use the composer to retry."; ++this.#generation; this.#activeEpoch = null; this.#activeItemId = null; @@ -255,7 +381,7 @@ export class VoiceTurnController { this.#activeSpeechSegmentId = null; this.#speechLoopGeneration = null; this.#playback.cancel(); - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); void this.#session.disconnect(); this.#update({ errorMessage, phase: "recoverable-error" }); return; @@ -266,7 +392,11 @@ export class VoiceTurnController { if (this.#awaitingChatCycle) { this.#sawBusyChatStatus = true; } - this.#session.setMicrophoneEnabled(false); + if (this.#hasAnswerableQuestion() && !this.#paused) { + this.#settleListeningIfReady(); + return; + } + this.#setMicrophoneEnabled(false); if ( this.#speechLoopGeneration === null && (this.#snapshot.phase === "listening" || @@ -280,7 +410,7 @@ export class VoiceTurnController { this.#settleListeningIfReady(); } - async #deliver(input: SubmitTextInput): Promise { + async #deliver(input: SubmitTextInput): Promise { const generation = this.#generation; this.#awaitingChatCycle = true; this.#sawBusyChatStatus = false; @@ -290,23 +420,25 @@ export class VoiceTurnController { generation !== this.#generation || !this.#isAwaitingCurrentChatCycle() ) { - return; + return false; } if (this.#snapshot.phase === "delivering") { this.#update({ phase: "waiting" }); } this.#settleListeningIfReady(); + return true; } catch { if (generation !== this.#generation) { - return; + return false; } this.#awaitingChatCycle = false; - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); this.#update({ errorMessage: - "Brunch could not accept the voice turn. Use the composer to retry.", + "The interview could not accept that answer. Use the composer to retry.", phase: "recoverable-error", }); + return false; } } @@ -319,6 +451,12 @@ export class VoiceTurnController { } #handleSessionEvent(event: OpenAIRealtimeSessionEvent): void { + if (event.type === "microphone-level") { + if (this.#snapshot.microphoneEnabled) { + this.#update({ microphoneLevel: event.level }); + } + return; + } if (event.type === "error") { ++this.#generation; this.#activeEpoch = null; @@ -330,7 +468,7 @@ export class VoiceTurnController { this.#speechLoopGeneration = null; this.#speechQueue.length = 0; this.#playback.cancel(); - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); this.#update({ errorCode: event.code, errorMessage: event.message, @@ -349,7 +487,7 @@ export class VoiceTurnController { return; } this.#activeItemId = event.itemId; - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); this.#update({ phase: "transcribing" }); return; } @@ -376,10 +514,8 @@ export class VoiceTurnController { this.#activeKey = key; if (event.type === "partial") { - this.#session.setMicrophoneEnabled(false); this.#update({ partialText: `${this.#snapshot.partialText}${event.text}`, - phase: "transcribing", }); return; } @@ -397,31 +533,44 @@ export class VoiceTurnController { return; } - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); + const previousText = this.#snapshot.lastCommittedText; + const redoing = this.#redoing; + this.#redoing = false; + this.#questionAnswered = true; + this.#answerFinalizedAt = this.#now(); this.#update({ errorMessage: "", lastCommittedText: finalText, partialText: "", phase: "delivering", }); - void this.#deliver({ - id: createVoiceMessageId(this.#conversationId, event.key), - text: finalText, - }); + void this.#deliver( + redoing + ? { + target: "message", + text: `Correction to my previous voice answer "${previousText}": ${finalText}`, + } + : { + id: createVoiceMessageId(this.#conversationId, event.key), + text: finalText, + }, + ); } #startSpeechQueueIfNeeded(): void { if ( this.#speechLoopGeneration !== null || this.#speechQueue.length === 0 || - this.#activeEpoch === null + this.#activeEpoch === null || + this.#paused ) { return; } const generation = this.#generation; this.#speechLoopGeneration = generation; - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); this.#update({ errorMessage: "", phase: "synthesizing" }); void this.#drainSpeechQueue(generation); } @@ -438,7 +587,7 @@ export class VoiceTurnController { } this.#activeSpeechSegmentId = segment.id; - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); this.#update({ errorMessage: "", phase: "synthesizing" }); try { await this.#playback.play(segment, { @@ -449,6 +598,9 @@ export class VoiceTurnController { this.#activeSpeechSegmentId === segment.id ) { this.#update({ phase: "playing" }); + if (segment.source === "brunch-ask") { + this.#recordLatency("question-spoken-started", segment.id); + } } }, }); @@ -466,7 +618,7 @@ export class VoiceTurnController { this.#activeSpeechSegmentId = null; this.#speechLoopGeneration = null; this.#speechQueue.length = 0; - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); this.#update({ errorCode: voiceError.code, errorMessage: voiceError.message, @@ -483,6 +635,10 @@ export class VoiceTurnController { return; } this.#activeSpeechSegmentId = null; + if (segment.source === "brunch-ask") { + this.#questionPlaybackComplete = true; + this.#recordLatency("question-spoken", segment.id); + } } if ( @@ -500,15 +656,42 @@ export class VoiceTurnController { if ( this.#activeEpoch === null || this.#snapshot.phase === "recoverable-error" || + this.#paused || this.#speechLoopGeneration !== null || this.#speechQueue.length > 0 ) { + if (this.#paused) { + this.#setMicrophoneEnabled(false); + } + return; + } + + if (!this.#canAcceptInterviewAnswer) { + this.#setMicrophoneEnabled(false); + if ( + this.#snapshot.phase !== "transcribing" && + this.#snapshot.phase !== "delivering" + ) { + this.#update({ phase: "waiting" }); + } + return; + } + + if (this.#hasAnswerableQuestion()) { + this.#awaitingChatCycle = false; + this.#sawBusyChatStatus = false; + this.#setMicrophoneEnabled(true); + this.#update({ errorMessage: "", phase: "listening" }); + if (this.#answerReadyQuestionId !== this.#activeQuestionId()) { + this.#answerReadyQuestionId = this.#activeQuestionId(); + this.#recordLatency("answer-ready", this.#activeQuestionId()); + } return; } if (this.#awaitingChatCycle) { if (!this.#sawBusyChatStatus || this.#chatStatus !== "ready") { - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); if ( this.#snapshot.phase === "synthesizing" || this.#snapshot.phase === "playing" @@ -522,7 +705,7 @@ export class VoiceTurnController { } if (!this.#isChatReady()) { - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); if ( this.#snapshot.phase !== "transcribing" && this.#snapshot.phase !== "delivering" @@ -532,16 +715,102 @@ export class VoiceTurnController { return; } - this.#session.setMicrophoneEnabled(true); + this.#setMicrophoneEnabled(true); this.#update({ errorMessage: "", phase: "listening" }); } + #activeQuestionId(): string { + return this.#currentQuestionId ?? this.#snapshot.currentQuestion; + } + + #hasPendingQuestion(): boolean { + return ( + Boolean(this.#snapshot.currentQuestion) && this.#canAcceptInterviewAnswer + ); + } + + #hasAnswerableQuestion(): boolean { + return ( + Boolean(this.#snapshot.currentQuestion) && + this.#questionPlaybackComplete && + !this.#questionAnswered && + this.#canAcceptInterviewAnswer + ); + } + + #canReviseLastAnswer(snapshot = this.#snapshot): boolean { + return ( + Boolean(snapshot.lastCommittedText) && + this.#activeEpoch !== null && + snapshot.phase === "listening" && + !this.#awaitingChatCycle && + this.#speechLoopGeneration === null && + this.#canAcceptInterviewAnswer + ); + } + + #queueAvailableSegments(): void { + for (const segment of this.#availableSegments) { + if (!this.#seenSpeechSegmentIds.has(segment.id)) { + this.#queueSpeechSegment(segment); + } + } + } + + #queueSpeechSegment(segment: CanonicalSpeechSegment): void { + this.#seenSpeechSegmentIds.add(segment.id); + this.#speechQueue.push(segment); + if (segment.source === "brunch-ask") { + this.#currentQuestionId = segment.id; + this.#questionPlaybackComplete = false; + this.#questionAnswered = false; + this.#redoing = false; + this.#answerReadyQuestionId = null; + this.#awaitingChatCycle = false; + this.#sawBusyChatStatus = false; + this.#update({ + currentQuestion: segment.text, + lastCommittedText: "", + }); + this.#recordLatency("question-visible", segment.id); + } + } + + #recordLatency(name: VoiceLatencyEvent["name"], questionId: string): void { + if (this.#answerFinalizedAt === null) { + return; + } + this.#onLatencyEvent?.({ + elapsedMs: Math.max(0, this.#now() - this.#answerFinalizedAt), + name, + questionId, + }); + } + + #setMicrophoneEnabled(enabled: boolean): void { + const microphoneEnabled = enabled && this.#canAcceptInterviewAnswer; + this.#session.setMicrophoneEnabled(microphoneEnabled); + if ( + this.#snapshot.microphoneEnabled !== microphoneEnabled || + (!microphoneEnabled && this.#snapshot.microphoneLevel !== 0) + ) { + this.#update({ + microphoneEnabled, + ...(microphoneEnabled ? {} : { microphoneLevel: 0 }), + }); + } + } + #update(update: Partial): void { const clearedError = update.errorMessage !== undefined && !("errorCode" in update) ? { errorCode: null, errorRequestId: "" } : {}; - this.#snapshot = { ...this.#snapshot, ...clearedError, ...update }; + const snapshot = { ...this.#snapshot, ...clearedError, ...update }; + this.#snapshot = { + ...snapshot, + canReviseLastAnswer: this.#canReviseLastAnswer(snapshot), + }; for (const listener of this.#listeners) { listener(this.#snapshot); } diff --git a/docs/superpowers/specs/2026-08-27-chat-interview-mode-switch-design.md b/docs/superpowers/specs/2026-08-27-chat-interview-mode-switch-design.md new file mode 100644 index 00000000000..20f670a0984 --- /dev/null +++ b/docs/superpowers/specs/2026-08-27-chat-interview-mode-switch-design.md @@ -0,0 +1,105 @@ +# Chat and Interview Mode Switch + +## Goal + +Make text chat and voice interview feel like two uniform ways to work with the same Petrinaut AI assistant. Move voice discovery out of the composer, expose it beside Chat at the top of the relevant surfaces, and preserve clear microphone/session awareness when switching modes. + +## Scope + +- Replace the composer microphone trigger with a labeled `Chat` / `Interview` mode switch. +- Show the same switch in: + - the empty-canvas “Describe the process you want to create” card; and + - the AI assistant panel header. +- Change the empty-conversation composer placeholder from “Get creating...” to “Describe the process you want to create”. +- Reuse the existing disclosure, voice session, compact bar, recovery, and local-storage behavior. +- Keep Chat and Interview in one conversation. This is an input-mode change, not a separate assistant or thread. + +## Interaction Design + +### Shared mode tabs + +A reusable Petrinaut component renders two labeled tabs: + +- `Chat`, with the existing AI assistant icon. +- `Interview`, with a microphone icon. + +The tabs use Petrinaut’s current neutral/blue styling, visible selected state, keyboard-accessible buttons, and accessible selected-state semantics. They remain labeled rather than becoming icon-only so their meaning does not depend on tooltips. + +When no host interview stage is configured, Petrinaut renders the current chat-only UI and no mode switch. + +### Empty canvas + +Chat is selected by default and retains the current title, example input, and submit action. + +Selecting Interview changes the card body to a microphone-led introduction: + +- “Talk through your process with AI” +- a short explanation that guided questions will help create the model +- `Start interview` + +Selecting `Start interview` opens the AI panel in Interview mode. On first use, the existing disclosure is shown there; after acknowledgement, the existing behavior starts the session directly. The main card does not duplicate consent or own a voice connection. + +### AI assistant panel + +The panel header replaces the static `AI` label with the same `Chat` / `Interview` tabs. Clear and close controls remain on the right. + +Chat mode shows the existing conversation, prompt chips, and composer. For an empty conversation, the composer placeholder is “Describe the process you want to create”. + +Interview mode shows the same conversation plus the full existing interview stage, and hides the generic composer and prompt chips to keep one primary input surface. Existing interview actions such as `Use text instead` switch to Chat and focus the composer. + +### Active-session switching + +Switching from Interview to Chat does not pause or end an active session. The full stage becomes the existing compact status bar above the composer, preserving visible microphone/session state and a route back to Interview. + +The following actions also align with the mode switch: + +- Expand/open interview: select Interview. +- Minimize or `Use text instead`: select Chat. +- End interview: end the voice session and return to Chat. +- Close the AI panel: retain the existing detached compact bar behavior. + +## Architecture + +Petrinaut owns the provider-neutral mode selection because both entry points and the surrounding layout belong to Petrinaut. The host-owned voice implementation continues to own disclosure, microphone, realtime connection, transcripts, and interview lifecycle. + +Add a provider-neutral interaction mode type (`"chat" | "interview"`) and expose the selected mode plus a mode-change callback in `PetrinautAiInterviewStageContext`. + +`EditorView` carries the empty-card mode request into `AiAssistantPanel` when opening it. `AiAssistantPanel` owns the live mode for that mounted conversation and passes it to both the panel contents and the host interview stage. The stage remains mounted while modes change so an active voice session is not accidentally destroyed. + +`VoiceInterviewControl` derives its visible presentation from both its existing lifecycle and Petrinaut’s selected mode: + +- inactive + Chat: no voice trigger; +- inactive + Interview: existing start/disclosure presentation; +- active + Interview: full stage; +- active + Chat: compact bar; +- closed sidebar + active session: detached compact bar. + +## Error Handling + +Existing microphone, connection, and service recovery states remain in the interview stage. Switching to Chat during an error keeps any active session represented by the compact bar. `Use text instead` remains the reliable escape path and selects Chat. + +If interview configuration is unavailable, the mode tabs are omitted rather than exposing an Interview mode that cannot start. + +## Accessibility + +- The shared switch exposes a tablist with two named tabs and the selected tab. +- Both tabs have visible text and decorative icons hidden from assistive technology. +- Selecting Chat focuses the composer when initiated from an interview fallback action. +- An active interview remains visible and announced in compact form after switching to Chat. +- Existing live transcript announcements and icon-button labels remain unchanged. + +## Testing + +Add or update tests for: + +- shared tabs, labels, selected state, and disabled-feature fallback; +- empty-card Chat and Interview content and actions; +- carrying an Interview selection from the empty card into the panel; +- the new empty-conversation composer placeholder; +- hiding the generic composer in Interview mode; +- preserving the mounted interview stage when modes change; +- full-to-compact and compact-to-full transitions; +- `Use text instead`, minimize, expand, end, and panel-close mode behavior; +- existing disclosure persistence and voice-session tests. + +Update the Petrinaut AI assistant user guide to describe Chat / Interview mode selection and the compact active-session behavior. diff --git a/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md b/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md index 2d346d43a38..b4f80c71504 100644 --- a/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md +++ b/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md @@ -36,9 +36,9 @@ recovery or public availability. transcription policy and calls OpenAI's unified WebRTC initialization endpoint. The fixed model is `gpt-live-transcribe`. Turn detection uses that model's default server VAD because the unified endpoint currently times out when either VAD mode is configured during initialization; - semantic VAD remains a tunable evaluation setting once initialization supports it. Provider - keys, prompts, vocabulary, language policy, and model selection remain server-side. Realtime - never generates assistant responses. + semantic VAD remains a tunable evaluation setting once initialization supports it. Partial + transcription events do not disable capture. Provider keys, prompts, vocabulary, language + policy, and model selection remain server-side. Realtime never generates assistant responses. 4. **Completed provider items are the only admitted audio input.** Partials are display-only. Completed items are keyed by connection epoch, provider item ID, and content index, then enter the existing Petrinaut composer and AI SDK transport once. A pending `brunch_ask` uses its diff --git a/libs/@hashintel/petrinaut/CHANGELOG.md b/libs/@hashintel/petrinaut/CHANGELOG.md index 34856e63881..bc789007a1b 100644 --- a/libs/@hashintel/petrinaut/CHANGELOG.md +++ b/libs/@hashintel/petrinaut/CHANGELOG.md @@ -4,9 +4,9 @@ ### Patch Changes -- Add a generic host-rendered AI composer control with stable finalized-text submission, - conversation identity, stop handling, schema-validated interactive-tool text mapping, and an - explicit separate-message target for corrections. +- Add generic host-rendered AI composer controls and a persistent interview stage with docked and + detached placements, protected active conversations, keyboard fallback, and one-answer buffering + while the normal chat stream settles. ## 0.0.19 diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 94864c55d03..89a5550a08e 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -26,21 +26,59 @@ question is waiting for an answer, completes that question rather than starting message. A host can explicitly submit a separate message instead when the text is a correction or other follow-up that must not answer the pending question. -When the Brunch voice preview is enabled by the host, the additional control can accept finalized -microphone transcripts and speak finalized assistant responses. Live transcript fragments are -labelled **not sent** and do not enter the conversation. The microphone is off while Brunch is -working or a response is playing. Spoken responses use an AI-generated OpenAI voice, as disclosed -in the voice status panel. If speech fails, the response remains visible to read and the voice -control offers recovery instead of changing or regenerating the text. +When the Brunch voice preview is enabled and available, **Chat** and **Interview** tabs appear on +the first-run card and in the assistant panel header. If voice is unavailable, **Interview** is not +shown. On the first-run card, select **Interview** and then **Start interview** to open the panel at +the existing one-time disclosure. Review that OpenAI transcribes your speech and Petrinaut keeps +finalized answers in the conversation rather than the audio, optionally check your microphone, +confirm that you understand it, then select **Start interview** in the disclosure. Petrinaut +remembers that acknowledgement in this browser for the current disclosure version, so later +selections of **Interview** start directly. If browser storage is unavailable or the disclosure +changes, Petrinaut asks again. + +While **Interview** is selected, the full interview stage replaces the generic message composer. It +keeps the current question visible, labels live words **Not sent yet**, and shows **Answer +recorded** only after finalization. The words shown as the question are also the exact words spoken +by the AI-generated OpenAI voice. Choose **Use text instead** to select **Chat** and return focus to +the composer. + +The stage names the microphone state and shows input activity only while the microphone is really +on. During interviewer playback, **Interrupt and speak** first stops playback and then opens the +microphone; it does not listen and play at the same time. While listening, a natural pause lets +voice detection finish the answer automatically. Choose **Done speaking** to close capture so it +can finish sooner, or **Pause** to turn the microphone off temporarily without submitting. +Provisional transcript updates do not turn off the microphone. **Pause**, +**Minimize**, **End interview**, and **Interrupt and speak** are separate actions. The header uses +icon-only controls with tooltips **Minimize** and **End interview**. After an answer is recorded +and the interviewer is ready, use **Redo answer** to say an explicit correction or **Edit text** to +type one. These correction controls remain unavailable while the previous answer is still being +written down. Choose **Use text instead** to switch to the generic message composer as a keyboard +fallback. + +Selecting **Chat** or **Minimize** hides the full stage. If the interview is active, the same session +continues in a compact bar above the composer; expanding the bar reopens the sidebar and selects +**Interview**. Closing the AI sidebar during an active interview moves the compact bar over the +canvas; reopening the sidebar docks it again. These presentation changes do not reconnect or end +the voice session. On narrow screens the compact presentation remains a bottom bar. If the session +fails, the sidebar reopens to the full recovery view once; selecting **Chat** after that keeps the +recovery controls compact. **Clear AI chat** is unavailable while an interview is active. + +When Brunch has produced authoritative completion information, the stage can expand **Covered** and +**Still exploring** topics. It does not show a question count or a speculative model preview; the +Petrinaut canvas remains the model surface. If voice cannot continue, the status panel identifies the kind of problem. For microphone permission or device errors, allow access or connect/select a microphone before reconnecting. For -an interrupted request, network error, or timeout, check the connection and choose **Reconnect -voice input**. If the preview is unavailable, continue with the text composer. An invalid service -response includes a diagnostic reference you can give to an operator; that reference and its -diagnostic record do not contain your transcript or the response being spoken. +an interrupted request, network error, or timeout, check the connection and choose **Reconnect**. +If the preview is unavailable, continue with the text composer. An invalid service response +includes a diagnostic reference you can give to an operator; error codes and that reference sit +under collapsed **Technical details** so you can share them without them dominating the interview. +That reference and its diagnostic record do not contain your transcript or the response being +spoken. -**Clear AI chat** via the delete button in the top right of the panel: wipes the conversation, stops any in-flight stream, and tells the host app to forget the messages (if the host persists them). +When no interview is active, **Clear AI chat** via the delete button in the top right of the panel +wipes the conversation, stops any in-flight stream, and tells the host app to forget the messages +(if the host persists them). ## What the assistant can do @@ -77,7 +115,7 @@ When the assistant edits code surfaces (lambdas, kernels, dynamics, visualizers, ## Host configuration -Whether the assistant is available, which additional composer controls appear, where the +Whether the assistant is available, which additional composer controls or interview stages appear, where the conversation is stored (in-memory, in your host app's database, or anywhere else), and the model behind it are all controlled by the host application that embeds Petrinaut. Read-only documents and the simulate-mode restrictions described above always apply when applicable. diff --git a/libs/@hashintel/petrinaut/src/ui/index.ts b/libs/@hashintel/petrinaut/src/ui/index.ts index 2a515699524..31f869c6efb 100644 --- a/libs/@hashintel/petrinaut/src/ui/index.ts +++ b/libs/@hashintel/petrinaut/src/ui/index.ts @@ -21,6 +21,10 @@ export type { PetrinautAiComposerControlContext, PetrinautAiComposerStatus, PetrinautAiComposerSubmitTextResult, + PetrinautAiInteractionMode, + PetrinautAiInterviewStage, + PetrinautAiInterviewStageContext, + PetrinautAiInterviewStagePlacement, } from "./types/ai-assistant-composer-control"; export { definePetrinautAiInteractiveTool } from "./types/ai-interactive-tool"; export type { diff --git a/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx b/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx index 06f75043915..48ddadbb252 100644 --- a/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx +++ b/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx @@ -29,7 +29,10 @@ const editorRootStyle = css({ backgroundColor: "neutral.s25", }); -import type { PetrinautAiComposerControl } from "./types/ai-assistant-composer-control"; +import type { + PetrinautAiComposerControl, + PetrinautAiInterviewStage, +} from "./types/ai-assistant-composer-control"; import type { PetrinautAiInteractiveTool } from "./types/ai-interactive-tool"; import type { PetrinautAiMessage, @@ -48,6 +51,8 @@ export type PetrinautAiAssistant = { onMessages?: (messages: PetrinautAiMessage[]) => void; /** Render a host-owned control inside the assistant composer. */ renderComposerControl?: PetrinautAiComposerControl; + /** Render one persistent, provider-neutral interview stage. */ + renderInterviewStage?: PetrinautAiInterviewStage; transport: PetrinautAiTransport; }; diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts index d1fb912ae5d..b1405fe93b2 100644 --- a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts @@ -1,6 +1,9 @@ import type { PetrinautAiMessage } from "../views/Editor/panels/ai-assistant-panel/types"; import type { ReactNode } from "react"; +/** The active way a user is providing input to the AI assistant. */ +export type PetrinautAiInteractionMode = "chat" | "interview"; + /** Current lifecycle state of Petrinaut's AI SDK conversation. */ export type PetrinautAiComposerStatus = | "submitted" @@ -34,3 +37,33 @@ export type PetrinautAiComposerControlContext = { export type PetrinautAiComposerControl = ( context: PetrinautAiComposerControlContext, ) => ReactNode; + +/** Provider-neutral placement for one persistent host-owned interview stage. */ +export type PetrinautAiInterviewStagePlacement = "sidebar" | "detached"; + +/** Stable controls and placement supplied to a host-owned interview stage. */ +export type PetrinautAiInterviewStageContext = + PetrinautAiComposerControlContext & { + /** True when Petrinaut can retain one next answer, even while chat settles. */ + canAcceptInterviewAnswer: boolean; + /** Bring back the sidebar and place keyboard focus in its composer. */ + focusComposer: () => void; + interactionMode: PetrinautAiInteractionMode; + /** Open the AI sidebar without changing the interview session. */ + openSidebar: () => void; + placement: PetrinautAiInterviewStagePlacement; + /** Tell Petrinaut to protect the active conversation from accidental clearing. */ + setActive: (active: boolean) => void; + setInteractionMode: (mode: PetrinautAiInteractionMode) => void; + /** + * Accept one finalized answer immediately. If generic chat is still busy, + * Petrinaut retains it and submits it through the canonical composer path + * as soon as that path is ready. + */ + submitInterviewAnswer: PetrinautAiComposerControlContext["submitText"]; + }; + +/** Render callback for a persistent host-owned stage above the composer. */ +export type PetrinautAiInterviewStage = ( + context: PetrinautAiInterviewStageContext, +) => ReactNode; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-cta-modal.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-cta-modal.test.tsx new file mode 100644 index 00000000000..f3011f96e1d --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-cta-modal.test.tsx @@ -0,0 +1,96 @@ +/** + * @vitest-environment jsdom + */ +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, test, vi } from "vitest"; + +import { AiCtaModal } from "./ai-cta-modal"; + +afterEach(cleanup); + +describe("AiCtaModal", () => { + test("switches from Chat creation to the Interview entry point", () => { + const onStartInterview = vi.fn(); + render( + , + ); + + expect( + screen.getByRole("heading", { + name: "Describe the process you want to create", + }), + ).not.toBeNull(); + fireEvent.click(screen.getByRole("button", { name: "Interview" })); + expect( + screen.getByRole("heading", { + name: "Talk through your process with AI", + }), + ).not.toBeNull(); + + fireEvent.click(screen.getByRole("button", { name: "Start interview" })); + expect(onStartInterview).toHaveBeenCalledOnce(); + }); + + test("keeps the current chat-only card when interview is unavailable", () => { + render( + , + ); + + expect( + screen.queryByRole("group", { name: "AI interaction mode" }), + ).toBeNull(); + expect( + screen.getByLabelText("Describe the process you want to create"), + ).not.toBeNull(); + }); + + test("returns to Chat when Interview becomes unavailable", () => { + const { rerender } = render( + , + ); + + fireEvent.click(screen.getByRole("button", { name: "Interview" })); + expect( + screen.getByRole("heading", { + name: "Talk through your process with AI", + }), + ).not.toBeNull(); + + rerender( + , + ); + + expect( + screen.queryByRole("group", { name: "AI interaction mode" }), + ).toBeNull(); + expect( + screen.getByRole("heading", { + name: "Describe the process you want to create", + }), + ).not.toBeNull(); + }); +}); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-cta-modal.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-cta-modal.tsx index bb677eae955..60b3eafaa89 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-cta-modal.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-cta-modal.tsx @@ -4,6 +4,12 @@ import { Button, TextInput } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; import { AiAssistantIcon } from "../../../components/ai-assistant-icon"; +import { + AiInteractionModeTabs, + AiMicrophoneIcon, +} from "./ai-interaction-mode-tabs"; + +import type { PetrinautAiInteractionMode } from "../../../types/ai-assistant-composer-control"; const aiCtaModalLayerStyle = css({ position: "absolute", @@ -62,6 +68,12 @@ const aiCtaModalCopyStyle = css({ maxWidth: "[420px]", }); +const aiCtaModalDescriptionStyle = css({ + margin: "0", + color: "neutral.s80", + textStyle: "sm", +}); + const aiCtaModalTitleStyle = css({ margin: "0", color: "neutral.s110", @@ -73,14 +85,20 @@ const aiCtaModalTitleStyle = css({ export const AiCtaModal = ({ bottomClearance, + interviewAvailable, onDismiss, + onStartInterview, onSubmit, }: { bottomClearance: number; + interviewAvailable: boolean; onDismiss: () => void; + onStartInterview: () => void; onSubmit: (message: string) => void; }) => { const [promptInput, setPromptInput] = useState(""); + const [interactionMode, setInteractionMode] = + useState("chat"); const canSubmit = promptInput.trim().length > 0; const inputRef = useRef(null); @@ -117,6 +135,10 @@ export const AiCtaModal = ({ }; }, [onDismiss]); + const effectiveInteractionMode = interviewAvailable + ? interactionMode + : "chat"; + return (
-
- -
-
-

- Describe the process you want to create -

-
- - ), - }} - /> + {interviewAvailable && ( + + )} + {effectiveInteractionMode === "chat" ? ( + <> +
+ +
+
+

+ Describe the process you want to create +

+
+ + ), + }} + /> + + ) : ( + <> + +
+

+ Talk through your process with AI +

+

+ Answer a few guided questions and Petrinaut will create the + model. +

+
+ + + )}
); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-interaction-mode-tabs.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-interaction-mode-tabs.test.tsx new file mode 100644 index 00000000000..8d6797f8308 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-interaction-mode-tabs.test.tsx @@ -0,0 +1,50 @@ +/** + * @vitest-environment jsdom + */ +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, test, vi } from "vitest"; + +import { AiInteractionModeTabs } from "./ai-interaction-mode-tabs"; + +afterEach(cleanup); + +describe("AiInteractionModeTabs", () => { + test("shows Chat selected in a labeled button group", () => { + const onModeChange = vi.fn(); + render(); + + expect( + screen.getByRole("group", { name: "AI interaction mode" }), + ).not.toBeNull(); + expect( + screen.getByRole("button", { name: "Chat" }).getAttribute("aria-pressed"), + ).toBe("true"); + expect( + screen + .getByRole("button", { name: "Interview" }) + .getAttribute("aria-pressed"), + ).toBe("false"); + + fireEvent.click(screen.getByRole("button", { name: "Interview" })); + expect(onModeChange).toHaveBeenCalledWith("interview"); + }); + + test("shows Interview selected and returns to Chat", () => { + const onModeChange = vi.fn(); + render( + , + ); + + expect( + screen.getByRole("button", { name: "Chat" }).getAttribute("aria-pressed"), + ).toBe("false"); + expect( + screen + .getByRole("button", { name: "Interview" }) + .getAttribute("aria-pressed"), + ).toBe("true"); + + fireEvent.click(screen.getByRole("button", { name: "Chat" })); + expect(onModeChange).toHaveBeenCalledWith("chat"); + }); +}); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-interaction-mode-tabs.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-interaction-mode-tabs.tsx new file mode 100644 index 00000000000..89ff841be16 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-interaction-mode-tabs.tsx @@ -0,0 +1,91 @@ +import { css, cva } from "@hashintel/ds-helpers/css"; + +import { AiAssistantIcon } from "../../../components/ai-assistant-icon"; + +import type { PetrinautAiInteractionMode } from "../../../types/ai-assistant-composer-control"; + +const modeGroupStyle = css({ + display: "flex", + alignItems: "center", + gap: "1", + padding: "1", + borderRadius: "lg", + backgroundColor: "neutral.s20", +}); + +const tabStyle = cva({ + base: { + display: "flex", + alignItems: "center", + gap: "1", + height: "[28px]", + paddingX: "2", + border: "none", + borderRadius: "md", + backgroundColor: "[transparent]", + color: "neutral.s90", + cursor: "pointer", + fontSize: "xs", + fontWeight: "medium", + _focusVisible: { + outline: "[2px solid token(colors.blue.s70)]", + outlineOffset: "[2px]", + }, + }, + variants: { + selected: { + true: { + backgroundColor: "neutral.s00", + boxShadow: "xs", + color: "blue.s90", + }, + }, + }, +}); + +export const AiMicrophoneIcon = () => ( + +); + +export const AiInteractionModeTabs = ({ + mode, + onModeChange, +}: { + mode: PetrinautAiInteractionMode; + onModeChange: (mode: PetrinautAiInteractionMode) => void; +}) => ( +
+ + +
+); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/editor-view.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/editor-view.tsx index 1414f4da523..52d8899bb12 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/editor-view.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/editor-view.tsx @@ -50,6 +50,7 @@ import { PropertiesPanel } from "./panels/PropertiesPanel/panel"; import { SimulateView } from "./panels/SimulateView/simulate-view"; import type { PetrinautAiAssistant } from "../../petrinaut"; +import type { PetrinautAiInteractionMode } from "../../types/ai-assistant-composer-control"; import type { PetrinautSlots } from "../../types/petrinaut-slots"; import type { ViewportAction } from "../../types/viewport-action"; @@ -149,6 +150,8 @@ export const EditorView = ({ const [pendingAiAssistantMessage, setPendingAiAssistantMessage] = useState< string | null >(null); + const [pendingAiInteractionMode, setPendingAiInteractionMode] = + useState(null); const [isAiCtaDismissed, setIsAiCtaDismissed] = useState(false); const { compactNodes, showWalkthroughOnInit, setShowWalkthroughOnInit } = @@ -449,9 +452,17 @@ export const EditorView = ({ {showEmptyAiHero && ( setIsAiCtaDismissed(true)} + onStartInterview={() => { + setPendingAiInteractionMode("interview"); + setAiAssistantOpen(true); + }} onSubmit={(message) => { setPendingAiAssistantMessage(message); + setPendingAiInteractionMode("chat"); setAiAssistantOpen(true); }} /> @@ -475,9 +486,13 @@ export const EditorView = ({ key={petriNetId ?? "no-net"} aiAssistant={aiAssistant} initialMessage={pendingAiAssistantMessage} + initialInteractionMode={pendingAiInteractionMode} onInitialMessageConsumed={() => setPendingAiAssistantMessage(null) } + onInitialInteractionModeConsumed={() => + setPendingAiInteractionMode(null) + } /> )} diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index db1a81dad95..e72e40cf0c0 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -2,6 +2,7 @@ * @vitest-environment jsdom */ import { + act, cleanup, fireEvent, render, @@ -32,13 +33,20 @@ import { definePetrinautAiInteractiveTool } from "../../../types/ai-interactive- import { AiAssistantPanel } from "./ai-assistant-panel"; import type { PetrinautAiAssistant } from "../../../petrinaut"; -import type { PetrinautAiComposerControlContext } from "../../../types/ai-assistant-composer-control"; +import type { + PetrinautAiComposerControlContext, + PetrinautAiInteractionMode, + PetrinautAiInterviewStageContext, +} from "../../../types/ai-assistant-composer-control"; import type { PetrinautAiMessage, PetrinautAiTransport, } from "./ai-assistant-panel/types"; import type { UIMessageChunk } from "ai"; +let interviewStageMounts = 0; +let interviewStageUnmounts = 0; + const emptySDCPN: SDCPN = { places: [], transitions: [], @@ -132,10 +140,16 @@ const testInstances: ReturnType[] = []; const renderTestPanel = ({ aiAssistant, + editorContext = editorContextValue, + initialInteractionMode, initialMessage, + onInitialInteractionModeConsumed, }: { aiAssistant: PetrinautAiAssistant; + editorContext?: EditorContextValue; + initialInteractionMode?: PetrinautAiInteractionMode; initialMessage?: string; + onInitialInteractionModeConsumed?: () => void; }) => { const handle = createJsonDocHandle({ id: "ai-assistant-panel-test", @@ -156,24 +170,31 @@ const renderTestPanel = ({ getItemType: () => null, }; - const renderPanel = (nextAiAssistant: PetrinautAiAssistant) => ( + const renderPanel = ( + nextAiAssistant: PetrinautAiAssistant, + nextEditorContext: EditorContextValue, + ) => ( - + ); - const rendered = render(renderPanel(aiAssistant)); + const rendered = render(renderPanel(aiAssistant, editorContext)); return { ...rendered, - rerenderPanel: (nextAiAssistant: PetrinautAiAssistant) => - rendered.rerender(renderPanel(nextAiAssistant)), + rerenderPanel: ( + nextAiAssistant: PetrinautAiAssistant, + nextEditorContext = editorContext, + ) => rendered.rerender(renderPanel(nextAiAssistant, nextEditorContext)), }; }; @@ -185,6 +206,290 @@ afterEach(() => { }); describe("AiAssistantPanel composer submissions", () => { + test("redocks one mounted interview stage when the sidebar closes and reopens", () => { + interviewStageMounts = 0; + interviewStageUnmounts = 0; + const Stage = ({ placement }: { placement: string }) => { + useEffect(() => { + interviewStageMounts += 1; + return () => { + interviewStageUnmounts += 1; + }; + }, []); + return
{`Stage ${placement}`}
; + }; + const aiAssistant: PetrinautAiAssistant = { + renderInterviewStage: ({ placement }) => , + transport: { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(), + }, + }; + const rendered = renderTestPanel({ aiAssistant }); + + expect(screen.getByText("Stage sidebar")).not.toBeNull(); + rendered.rerenderPanel(aiAssistant, { + ...editorContextValue, + isAiAssistantOpen: false, + }); + expect(screen.getByText("Stage detached")).not.toBeNull(); + rendered.rerenderPanel(aiAssistant, editorContextValue); + + expect(screen.getByText("Stage sidebar")).not.toBeNull(); + expect(interviewStageMounts).toBe(1); + expect(interviewStageUnmounts).toBe(0); + }); + + test("switches modes without unmounting the interview stage", () => { + interviewStageMounts = 0; + interviewStageUnmounts = 0; + const Stage = (context: PetrinautAiInterviewStageContext) => { + useEffect(() => { + interviewStageMounts += 1; + return () => { + interviewStageUnmounts += 1; + }; + }, []); + return ( + + ); + }; + const aiAssistant: PetrinautAiAssistant = { + renderInterviewStage: (context) => , + transport: { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(), + }, + }; + + renderTestPanel({ aiAssistant }); + + fireEvent.click(screen.getByRole("button", { name: "Interview" })); + expect(screen.getByText("Stage interview")).not.toBeNull(); + fireEvent.click(screen.getByRole("button", { name: "Stage interview" })); + + expect( + screen.getByPlaceholderText("Describe the process you want to create"), + ).not.toBeNull(); + expect(interviewStageMounts).toBe(1); + expect(interviewStageUnmounts).toBe(0); + }); + + test("defers and consumes an initial Interview mode once, then falls back to Chat", () => { + let latestInteractionMode = "chat"; + const onInitialInteractionModeConsumed = vi.fn(); + const aiAssistant: PetrinautAiAssistant = { + renderInterviewStage: (context) => { + latestInteractionMode = context.interactionMode; + return
Interview stage
; + }, + transport: { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(), + }, + }; + const closedEditorContext = { + ...editorContextValue, + isAiAssistantOpen: false, + }; + const rendered = renderTestPanel({ + aiAssistant, + editorContext: closedEditorContext, + initialInteractionMode: "interview", + onInitialInteractionModeConsumed, + }); + + expect(latestInteractionMode).toBe("chat"); + expect(onInitialInteractionModeConsumed).not.toHaveBeenCalled(); + + rendered.rerenderPanel(aiAssistant, editorContextValue); + + expect(latestInteractionMode).toBe("interview"); + expect(onInitialInteractionModeConsumed).toHaveBeenCalledOnce(); + + const unavailableAssistant: PetrinautAiAssistant = { + transport: aiAssistant.transport, + }; + rendered.rerenderPanel(unavailableAssistant, editorContextValue); + + expect(screen.queryByText("Interview stage")).toBeNull(); + expect( + screen.getByPlaceholderText("Describe the process you want to create"), + ).not.toBeNull(); + expect(onInitialInteractionModeConsumed).toHaveBeenCalledOnce(); + }); + + test("accepts one interview answer while generic chat is streaming and submits it after settlement", async () => { + let firstStreamController: + | ReadableStreamDefaultController + | undefined; + let secondStreamController: + | ReadableStreamDefaultController + | undefined; + const requests: PetrinautAiMessage[][] = []; + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(({ messages }) => { + requests.push(structuredClone(messages)); + if (requests.length === 1) { + return Promise.resolve( + new ReadableStream({ + start(controller) { + firstStreamController = controller; + for (const chunk of textChunks("question", "Question ready")) { + controller.enqueue(chunk); + } + }, + }), + ); + } + if (requests.length === 2) { + return Promise.resolve( + new ReadableStream({ + start(controller) { + secondStreamController = controller; + for (const chunk of textChunks( + "acknowledgement", + "Answer accepted", + )) { + controller.enqueue(chunk); + } + }, + }), + ); + } + return Promise.resolve( + streamChunks(textChunks("next-answer", "Next answer accepted")), + ); + }), + }; + let latestStageContext: PetrinautAiInterviewStageContext | undefined; + + renderTestPanel({ + aiAssistant: { + renderInterviewStage: (context) => { + latestStageContext = context; + return ( + + ); + }, + transport, + }, + }); + + fireEvent.click(screen.getByRole("button", { name: "Begin" })); + await screen.findByRole("button", { name: "Answer now" }); + expect(latestStageContext?.canAcceptInterviewAnswer).toBe(true); + fireEvent.click(screen.getByRole("button", { name: "Answer now" })); + await waitFor(() => + expect(latestStageContext?.canAcceptInterviewAnswer).toBe(false), + ); + expect(requests).toHaveLength(1); + + await act(async () => firstStreamController?.close()); + await screen.findByText("Answer accepted"); + + expect(requests).toHaveLength(2); + expect(latestStageContext?.status).toBe("streaming"); + expect(latestStageContext?.canAcceptInterviewAnswer).toBe(true); + expect(requests[1]?.at(-1)).toMatchObject({ + role: "user", + parts: [{ type: "text", text: "Queued interview answer" }], + }); + + void latestStageContext?.submitInterviewAnswer({ + target: "message", + text: "Next interview answer", + }); + await waitFor(() => + expect(latestStageContext?.canAcceptInterviewAnswer).toBe(false), + ); + expect(requests).toHaveLength(2); + + await act(async () => secondStreamController?.close()); + await screen.findByText("Next answer accepted"); + expect(requests).toHaveLength(3); + }); + + test("reopens the interview answer buffer when the conversation changes", async () => { + let streamController: + | ReadableStreamDefaultController + | undefined; + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(() => + Promise.resolve( + new ReadableStream({ + start(controller) { + streamController = controller; + for (const chunk of textChunks("question", "Question ready")) { + controller.enqueue(chunk); + } + }, + }), + ), + ), + }; + let latestStageContext: PetrinautAiInterviewStageContext | undefined; + const createAiAssistant = ( + conversationId: string, + ): PetrinautAiAssistant => ({ + conversationId, + renderInterviewStage: (context) => { + latestStageContext = context; + return null; + }, + transport, + }); + const rendered = renderTestPanel({ + aiAssistant: createAiAssistant("conversation-1"), + }); + let initialSubmission: Promise | undefined; + act(() => { + initialSubmission = latestStageContext?.submitText({ text: "Begin" }); + }); + void initialSubmission?.catch(() => undefined); + await waitFor(() => expect(latestStageContext?.status).toBe("streaming")); + let queuedAnswer: Promise | undefined; + act(() => { + queuedAnswer = latestStageContext?.submitInterviewAnswer({ + target: "message", + text: "Queued interview answer", + }); + }); + const queuedAnswerRejection = expect(queuedAnswer).rejects.toThrow( + "The interview conversation changed.", + ); + await waitFor(() => + expect(latestStageContext?.canAcceptInterviewAnswer).toBe(false), + ); + + rendered.rerenderPanel(createAiAssistant("conversation-2")); + + await queuedAnswerRejection; + await waitFor(() => + expect(latestStageContext?.canAcceptInterviewAnswer).toBe(true), + ); + await act(async () => streamController?.close()); + }); + test("exposes the generated useChat conversation identity to host controls", async () => { const chatIds: string[] = []; const observedConversationIds = new Set(); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index 4ba9dd4242f..aa860fee552 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -58,6 +58,7 @@ import type { PetrinautAiAssistant } from "../../../petrinaut"; import type { PetrinautAiComposerControlContext, PetrinautAiComposerSubmitTextResult, + PetrinautAiInteractionMode, } from "../../../types/ai-assistant-composer-control"; import type { PetrinautAiMessage } from "./ai-assistant-panel/types"; @@ -91,6 +92,14 @@ const selectTarget = ( ); }; +type QueuedInterviewAnswer = { + readonly input: Parameters< + PetrinautAiComposerControlContext["submitText"] + >[0]; + readonly reject: (reason?: unknown) => void; + readonly resolve: (result: PetrinautAiComposerSubmitTextResult) => void; +}; + const isPetrinautAiMutationToolName = ( toolName: string, ): toolName is PetrinautAiMutationToolName => @@ -203,11 +212,15 @@ const applyPetrinautAiCommand = async ({ export const AiAssistantPanel = ({ aiAssistant, + initialInteractionMode, initialMessage, + onInitialInteractionModeConsumed, onInitialMessageConsumed, }: { aiAssistant: PetrinautAiAssistant; + initialInteractionMode?: PetrinautAiInteractionMode | null; initialMessage?: string | null; + onInitialInteractionModeConsumed?: () => void; onInitialMessageConsumed?: () => void; }) => { // The wrapped AI transport closes over several refs (diagnostics version, @@ -240,6 +253,23 @@ export const AiAssistantPanel = ({ const { petriNetDefinition, setTitle, title } = use(SDCPNContext); const [input, setInput] = useState(""); + const [composerFocusRequest, setComposerFocusRequest] = useState(0); + const [interviewActive, setInterviewActive] = useState(false); + const [interviewAnswerQueued, setInterviewAnswerQueued] = useState(false); + const [interactionMode, setInteractionMode] = + useState("chat"); + const selectInteractionMode = useCallback( + (nextMode: PetrinautAiInteractionMode) => { + setInteractionMode(nextMode); + if (nextMode === "chat") { + setComposerFocusRequest((request) => request + 1); + } + }, + [], + ); + const queuedInterviewAnswerRef = useRef(null); + const consumedInitialInteractionModeRef = + useRef(null); const submittedInitialMessageRef = useRef(null); const titleRef = useRef(title); @@ -739,12 +769,122 @@ export const AiAssistantPanel = ({ const stopStateRef = useLatest(stop); + const submitInterviewAnswer = useCallback< + PetrinautAiComposerControlContext["submitText"] + >( + (answer) => { + if (queuedInterviewAnswerRef.current) { + return Promise.reject( + new Error("The previous interview answer is still being submitted."), + ); + } + const currentStatus = composerSubmissionStateRef.current.status; + if (currentStatus === "error") { + return Promise.reject( + new Error("The interview is not ready to accept an answer."), + ); + } + if (currentStatus === "ready") { + return submitText(answer); + } + + setInterviewAnswerQueued(true); + return new Promise((resolve, reject) => { + queuedInterviewAnswerRef.current = { + input: answer, + reject, + resolve, + }; + }); + }, + [composerSubmissionStateRef, submitText], + ); + + useEffect(() => { + const queued = queuedInterviewAnswerRef.current; + if (!queued) { + return; + } + if (status === "error") { + queuedInterviewAnswerRef.current = null; + setInterviewAnswerQueued(false); + queued.reject(new Error("The interview could not accept that answer.")); + return; + } + if (status !== "ready") { + return; + } + + queuedInterviewAnswerRef.current = null; + setInterviewAnswerQueued(false); + void submitText(queued.input).then( + (result) => queued.resolve(result), + (caught: unknown) => queued.reject(caught), + ); + }, [status, submitText]); + + useEffect( + () => () => { + queuedInterviewAnswerRef.current?.reject( + new Error("The interview conversation changed."), + ); + queuedInterviewAnswerRef.current = null; + setInterviewAnswerQueued(false); + }, + [conversationId], + ); + // Like submitText, stop is exposed to host controls and must stay stable. const stopComposer = useCallback(async () => { stopRequestedRef.current = true; await stopStateRef.current(); }, [stopStateRef]); + useEffect(() => { + if ( + initialInteractionMode === undefined || + initialInteractionMode === null + ) { + consumedInitialInteractionModeRef.current = null; + return; + } + + if ( + !isAiAssistantOpen || + consumedInitialInteractionModeRef.current === initialInteractionMode + ) { + return; + } + + selectInteractionMode( + initialInteractionMode === "interview" && + aiAssistant.renderInterviewStage === undefined + ? "chat" + : initialInteractionMode, + ); + consumedInitialInteractionModeRef.current = initialInteractionMode; + onInitialInteractionModeConsumed?.(); + }, [ + aiAssistant.renderInterviewStage, + initialInteractionMode, + isAiAssistantOpen, + onInitialInteractionModeConsumed, + selectInteractionMode, + ]); + + useEffect(() => { + if ( + interactionMode === "interview" && + aiAssistant.renderInterviewStage === undefined + ) { + selectInteractionMode("chat"); + } + }, [ + aiAssistant.renderInterviewStage, + interactionMode, + selectInteractionMode, + ]); + useEffect(() => { const trimmedInitialMessage = initialMessage?.trim(); if (!trimmedInitialMessage) { @@ -776,7 +916,7 @@ export const AiAssistantPanel = ({ sendMessage, ]); - if (!isAiAssistantOpen || !instance) { + if (!instance) { return null; } @@ -807,14 +947,34 @@ export const AiAssistantPanel = ({ const composerControl = aiAssistant.renderComposerControl?.( composerControlContext, ); + const interviewStage = aiAssistant.renderInterviewStage?.({ + ...composerControlContext, + canAcceptInterviewAnswer: !interviewAnswerQueued, + focusComposer: () => { + selectInteractionMode("chat"); + setAiAssistantOpen(true); + }, + interactionMode, + openSidebar: () => setAiAssistantOpen(true), + placement: isAiAssistantOpen ? "sidebar" : "detached", + setActive: setInterviewActive, + setInteractionMode: selectInteractionMode, + submitInterviewAnswer, + }); /* eslint-enable react-hooks-js/refs */ return ( { // Clearing aborts any in-flight response too, which fires `onFinish` @@ -834,6 +994,7 @@ export const AiAssistantPanel = ({ }} onClose={() => setAiAssistantOpen(false)} onInputChange={setInput} + onInteractionModeChange={selectInteractionMode} onInteractiveToolSubmit={({ toolCallId, toolName, output }) => { if (!isPetrinautAiCommandToolName(toolName)) { if ( diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index 80d7ea3a7f5..f08d773d5ed 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -9,7 +9,7 @@ import { screen, waitFor, } from "@testing-library/react"; -import { createElement } from "react"; +import { createElement, useEffect } from "react"; import { afterEach, describe, expect, test, vi } from "vitest"; import { DEFAULT_PETRINAUT_EXTENSIONS } from "@hashintel/petrinaut-core"; @@ -20,6 +20,8 @@ import { AiAssistantContents } from "./ai-assistant-contents"; import type { PetrinautAiMessage } from "./types"; const renderMarkdown = vi.hoisted(() => vi.fn()); +let interviewStageMounts = 0; +let interviewStageUnmounts = 0; vi.mock("react-markdown", async (importOriginal) => { const actual = await importOriginal(); @@ -42,6 +44,141 @@ afterEach(() => { }); describe("AiAssistantContents", () => { + test("keeps one provider-neutral interview stage mounted while it moves between docked and detached presentation", () => { + interviewStageMounts = 0; + interviewStageUnmounts = 0; + const Stage = () => { + useEffect(() => { + interviewStageMounts += 1; + return () => { + interviewStageUnmounts += 1; + }; + }, []); + return
Interview stage
; + }; + const props = { + input: "", + interviewStage: , + messages: [] as PetrinautAiMessage[], + onClose: noop, + onInputChange: noop, + onStop: noop, + onSubmit: noop, + status: "ready" as const, + }; + const { rerender } = render( + , + ); + + expect(screen.getByText("Interview stage")).not.toBeNull(); + expect(screen.getByTestId("ai-interview-stage").dataset.placement).toBe( + "sidebar", + ); + rerender(); + + expect(screen.getByTestId("ai-interview-stage").dataset.placement).toBe( + "detached", + ); + expect( + screen + .getByRole("complementary", { name: "AI assistant" }) + .getAttribute("aria-hidden"), + ).toBeNull(); + expect(interviewStageMounts).toBe(1); + expect(interviewStageUnmounts).toBe(0); + }); + + test("hides a closed chat-only panel from the accessibility tree", () => { + const { container } = render( + , + ); + + expect( + container + .querySelector('aside[aria-label="AI assistant"]') + ?.getAttribute("aria-hidden"), + ).toBe("true"); + }); + + test("keeps keyboard drafting available and protects clear-chat during an active interview", () => { + render( + Active interview} + isOpen={true} + messages={[ + { + id: "assistant-1", + role: "assistant", + parts: [{ type: "text", text: "Question" }], + }, + ]} + onClearMessages={vi.fn()} + onClose={noop} + onInputChange={noop} + onStop={noop} + onSubmit={noop} + status="streaming" + />, + ); + + expect( + screen.getByRole("textbox", { + name: "Message AI assistant", + }).disabled, + ).toBe(false); + expect( + screen.getByRole("complementary", { name: "AI assistant" }).className, + ).toContain("z_[calc(var(--z-index-sticky)_+_2)]"); + expect( + screen.getByRole("button", { + name: "Clear AI chat", + }).disabled, + ).toBe(true); + }); + + test("switches the panel between Chat composer and Interview stage", () => { + const onInteractionModeChange = vi.fn(); + const props = { + input: "", + interactionMode: "chat" as const, + interviewAvailable: true, + interviewStage:
Interview stage
, + messages: [] as PetrinautAiMessage[], + onClose: noop, + onInputChange: noop, + onInteractionModeChange, + onStop: noop, + onSubmit: noop, + status: "ready" as const, + }; + const rendered = render(); + + expect( + screen.getByPlaceholderText("Describe the process you want to create"), + ).not.toBeNull(); + fireEvent.click(screen.getByRole("button", { name: "Interview" })); + expect(onInteractionModeChange).toHaveBeenCalledWith("interview"); + + rendered.rerender( + , + ); + expect( + screen.queryByRole("textbox", { name: "Message AI assistant" }), + ).toBeNull(); + expect(screen.getByText("Interview stage")).not.toBeNull(); + }); + test("keeps completed messages memoized when interactive tools are omitted", () => { const messages: PetrinautAiMessage[] = [ { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx index 1ddbf917d40..1002b69425a 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx @@ -13,6 +13,7 @@ import { Button } from "@hashintel/ds-components"; import { css, cva } from "@hashintel/ds-helpers/css"; import { AiAssistantIcon } from "../../../../components/ai-assistant-icon"; +import { AiInteractionModeTabs } from "../../components/ai-interaction-mode-tabs"; import { getMessageRenderItems } from "./ai-assistant-contents/get-message-render-items"; import { PromptChips, @@ -25,6 +26,7 @@ import { type OnInteractiveToolSubmit, } from "./ai-assistant-contents/tool-list"; +import type { PetrinautAiInteractionMode } from "../../../../types/ai-assistant-composer-control"; import type { PetrinautAiInteractiveTool } from "../../../../types/ai-interactive-tool"; import type { AiToolTarget } from "./tool-summaries"; import type { PetrinautAiMessage } from "./types"; @@ -34,14 +36,21 @@ type AiAssistantStatus = "submitted" | "streaming" | "ready" | "error"; const EMPTY_INTERACTIVE_TOOLS: readonly PetrinautAiInteractiveTool[] = []; export type AiAssistantContentsProps = { + clearMessagesDisabled?: boolean; composerControl?: ReactNode; + composerFocusRequest?: number; error?: Error; input: string; + interactionMode?: PetrinautAiInteractionMode; + interviewAvailable?: boolean; + interviewStage?: ReactNode; interactiveTools?: readonly PetrinautAiInteractiveTool[]; + isOpen?: boolean; messages: PetrinautAiMessage[]; onClearMessages?: () => void; onClose: () => void; onInputChange: (value: string) => void; + onInteractionModeChange?: (mode: PetrinautAiInteractionMode) => void; onInteractiveToolSubmit?: OnInteractiveToolSubmit; onSelectToolTarget?: (target: AiToolTarget) => void; onSendPrompt?: (prompt: string) => void; @@ -55,26 +64,41 @@ export type AiAssistantContentsProps = { const defaultAssistantWidth = 500; -const shellStyle = css({ - position: "absolute", - top: "0", - right: "0", - bottom: "0", - width: `[${defaultAssistantWidth}px]`, - maxWidth: "[calc(100vw - 32px)]", - zIndex: "sticky", - padding: "2", - pointerEvents: "auto", - transition: "[right 150ms ease-in-out]", - _before: { - content: '""', +const shellStyle = cva({ + base: { position: "absolute", - inset: "2", - borderRadius: "[14px]", - background: - "[radial-gradient(circle at 78% 28%, rgba(52,160,250,0.22), rgba(190,230,255,0.04) 54%, transparent 80%)]", - filter: "[blur(4px)]", - pointerEvents: "none", + right: "0", + zIndex: "[calc(var(--z-index-sticky) + 2)]", + pointerEvents: "auto", + transition: "[right 150ms ease-in-out]", + }, + variants: { + open: { + true: { + top: "0", + bottom: "0", + width: `[${defaultAssistantWidth}px]`, + maxWidth: "[calc(100vw - 32px)]", + padding: "2", + _before: { + content: '""', + position: "absolute", + inset: "2", + borderRadius: "[14px]", + background: + "[radial-gradient(circle at 78% 28%, rgba(52,160,250,0.22), rgba(190,230,255,0.04) 54%, transparent 80%)]", + filter: "[blur(4px)]", + pointerEvents: "none", + }, + }, + false: { + bottom: "0", + width: "[0px]", + height: "[0px]", + overflow: "visible", + pointerEvents: "none", + }, + }, }, }); @@ -105,16 +129,46 @@ const resizeHandleStyle = css({ }, }); -const cardStyle = css({ +const cardStyle = cva({ + base: { + position: "relative", + display: "flex", + flexDirection: "column", + }, + variants: { + open: { + true: { + height: "full", + overflow: "hidden", + backgroundColor: "neutral.s10", + borderRadius: "[12px]", + boxShadow: + "[0px 0px 0px 1px rgba(0,0,0,0.06), 0px 1px 1px -0.5px rgba(0,0,0,0.04), 0px 12px 12px -6px rgba(0,0,0,0.02), 0px 4px 4px -12px rgba(0,0,0,0.02)]", + }, + false: { + width: "[0px]", + height: "[0px]", + overflow: "visible", + pointerEvents: "none", + }, + }, + }, +}); + +const panelContentStyle = cva({ + variants: { + visible: { + false: { display: "none" }, + }, + }, +}); + +const interviewStageStyle = css({ position: "relative", - display: "flex", - flexDirection: "column", - height: "full", - overflow: "hidden", - backgroundColor: "neutral.s10", - borderRadius: "[12px]", - boxShadow: - "[0px 0px 0px 1px rgba(0,0,0,0.06), 0px 1px 1px -0.5px rgba(0,0,0,0.04), 0px 12px 12px -6px rgba(0,0,0,0.02), 0px 4px 4px -12px rgba(0,0,0,0.02)]", + zIndex: "[2]", + flexShrink: "0", + overflow: "visible", + pointerEvents: "auto", }); const headerStyle = css({ @@ -406,14 +460,21 @@ const AiAssistantMessage = memo( AiAssistantMessage.displayName = "AiAssistantMessage"; export const AiAssistantContents = ({ + clearMessagesDisabled = false, composerControl, + composerFocusRequest = 0, error, input, + interactionMode = "chat", + interviewAvailable = false, + interviewStage, interactiveTools = EMPTY_INTERACTIVE_TOOLS, + isOpen = true, messages, onClearMessages, onClose, onInputChange, + onInteractionModeChange, onInteractiveToolSubmit, onSelectToolTarget, onSendPrompt, @@ -480,8 +541,10 @@ export const AiAssistantContents = ({ }; useEffect(() => { - inputRef.current?.focus(); - }, []); + if (isOpen) { + inputRef.current?.focus(); + } + }, [composerFocusRequest, isOpen]); // Auto-grow the composer to fit its content (up to `composerMaxHeight`, // after which it scrolls internally). Resetting to `auto` before measuring @@ -519,19 +582,32 @@ export const AiAssistantContents = ({ return (