From da3878d4296eb7cd9626ec7011ca00240e6778e7 Mon Sep 17 00:00:00 2001 From: Amp Date: Thu, 27 Aug 2026 14:20:23 +0000 Subject: [PATCH 01/20] H-6763: Implement voice interview stage Co-authored-by: Kostandin Angjellari Amp-Thread-ID: https://ampcode.com/threads/T-01a0431a-f233-7239-971b-ed234c72dc6c --- .changeset/stable-composer-controls.md | 6 +- .../brunch-panel-transport.ts | 2 +- .../local-storage-demo-app.test.tsx | 15 +- .../local-storage-demo-app.tsx | 20 +- .../interview-coverage.test.ts | 93 ++ .../app/voice-interview/interview-coverage.ts | 66 ++ .../openai-realtime-session.test.ts | 69 ++ .../openai-realtime-session.ts | 97 ++- .../voice-interview-control.test.tsx | 405 +++++---- .../voice-interview-control.tsx | 809 ++++++++++++++---- .../voice-preview.integration.test.ts | 11 + .../voice-turn-controller.test.ts | 226 ++++- .../voice-interview/voice-turn-controller.ts | 308 ++++++- libs/@hashintel/petrinaut/CHANGELOG.md | 6 +- .../@hashintel/petrinaut/docs/ai-assistant.md | 45 +- libs/@hashintel/petrinaut/src/ui/index.ts | 3 + .../@hashintel/petrinaut/src/ui/petrinaut.tsx | 7 +- .../ui/types/ai-assistant-composer-control.ts | 28 + .../Editor/panels/ai-assistant-panel.test.tsx | 164 +++- .../Editor/panels/ai-assistant-panel.tsx | 94 +- .../ai-assistant-contents.test.tsx | 81 +- .../ai-assistant-contents.tsx | 159 +++- 22 files changed, 2268 insertions(+), 446 deletions(-) create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.test.ts create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.ts diff --git a/.changeset/stable-composer-controls.md b/.changeset/stable-composer-controls.md index d1df963bd70..1d4e3ac0cef 100644 --- a/.changeset/stable-composer-controls.md +++ b/.changeset/stable-composer-controls.md @@ -2,6 +2,6 @@ "@hashintel/petrinaut": patch --- -Add a generic host-rendered AI composer control with stable finalized-text submission, -conversation identity, stop handling, schema-validated interactive-tool text mapping, and an -explicit separate-message target for corrections. +Add generic host-rendered AI composer controls and a persistent interview stage with docked and +detached placements, protected active conversations, keyboard fallback, and one-answer buffering +while the normal chat stream settles. diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts index e9b0915734f..3df1bb427a3 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts @@ -51,7 +51,7 @@ const captureSchema = z.object({ supersedes: z.string().optional(), }); -const sweepOutputSchema = z.discriminatedUnion("status", [ +export const sweepOutputSchema = z.discriminatedUnion("status", [ z.object({ status: z.literal("no-settled-range") }), z.object({ status: z.literal("refused"), diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx index 441be1df455..1d04c601178 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx @@ -5,7 +5,7 @@ import { isValidElement, type ReactNode } from "react"; import { describe, expect, test, vi } from "vitest"; import { VoiceInterviewControl } from "../voice-interview/voice-interview-control"; -import { getBrunchVoiceComposerControl } from "./local-storage-demo-app"; +import { getBrunchVoiceInterviewStage } from "./local-storage-demo-app"; const defaultTransportOptions = vi.hoisted(() => ({ current: null as unknown, @@ -28,16 +28,25 @@ vi.mock("@hashintel/petrinaut/ui", () => ({ describe("local storage demo Brunch voice integration", () => { test("does not install voice on the generic local chat fallback", () => { - expect(getBrunchVoiceComposerControl(false)).toBeUndefined(); + expect(getBrunchVoiceInterviewStage(false)).toBeUndefined(); }); test("installs the app-owned voice control for a configured Brunch transport", () => { - const renderControl = getBrunchVoiceComposerControl(true); + const renderControl = getBrunchVoiceInterviewStage(true); const control = renderControl?.({ + canAcceptInterviewAnswer: true, conversationId: "petrinaut-preview:net-1", + focusComposer: vi.fn(), messages: [], + openSidebar: vi.fn(), + placement: "sidebar", + setActive: vi.fn(), status: "ready", stop: vi.fn(async () => undefined), + submitInterviewAnswer: vi.fn(async () => ({ + kind: "message" as const, + messageId: "message-1", + })), submitText: vi.fn(async () => ({ kind: "message" as const, messageId: "message-1", diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index aeeaddd27bf..6daeea98dd5 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -13,8 +13,8 @@ import { DefaultChatTransport, Petrinaut, type PetrinautAiChatTransport, - type PetrinautAiComposerControl, - type PetrinautAiComposerControlContext, + type PetrinautAiInterviewStage, + type PetrinautAiInterviewStageContext, type PetrinautAiMessage, WalkthroughProvider, } from "@hashintel/petrinaut/ui"; @@ -88,16 +88,16 @@ const brunchPreviewConfig = resolveBrunchPreviewConfig( import.meta.env.VITE_BRUNCH_CHAT_ENDPOINT, ); -const renderBrunchVoiceComposerControl = ( - context: PetrinautAiComposerControlContext, +const renderBrunchVoiceInterviewStage = ( + context: PetrinautAiInterviewStageContext, ) => ; -export const getBrunchVoiceComposerControl = ( +export const getBrunchVoiceInterviewStage = ( isBrunchConfigured: boolean, -): PetrinautAiComposerControl | undefined => - isBrunchConfigured ? renderBrunchVoiceComposerControl : undefined; +): PetrinautAiInterviewStage | undefined => + isBrunchConfigured ? renderBrunchVoiceInterviewStage : undefined; -const brunchVoiceComposerControl = getBrunchVoiceComposerControl( +const brunchVoiceInterviewStage = getBrunchVoiceInterviewStage( brunchPreviewConfig.isBrunchConfigured, ); @@ -320,9 +320,9 @@ export const LocalStorageDemoApp = () => { return next; }); }, - ...(brunchVoiceComposerControl + ...(brunchVoiceInterviewStage ? { - renderComposerControl: brunchVoiceComposerControl, + renderInterviewStage: brunchVoiceInterviewStage, } : {}), }), diff --git a/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.test.ts new file mode 100644 index 00000000000..6f1ba59c8c6 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, test } from "vitest"; + +import { SWEEP_TOOL_NAME } from "@hashintel/brunch-agent-transport-aisdk/client-tools"; + +import { selectInterviewCoverage } from "./interview-coverage"; + +import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui"; + +describe("interview coverage", () => { + test("uses only authoritative completion results for covered and open topics", () => { + const messages = [ + { + id: "assistant-sweep", + role: "assistant", + parts: [ + { + type: "dynamic-tool", + toolCallId: "sweep-1", + toolName: SWEEP_TOOL_NAME, + state: "output-available", + input: {}, + output: { + status: "applied", + appliedCaptureIds: ["capture-owner"], + captures: [ + { + id: "capture-owner", + status: "active", + epistemicStatus: "explicit", + confidence: "high", + content: { + value: { + type: "slot-asserted", + kind: "activity", + node: "approval", + slot: "who performs it", + precision: "named", + assertion: { value: "shift lead" }, + }, + }, + }, + { + id: "capture-old", + status: "superseded", + epistemicStatus: "explicit", + confidence: "high", + content: { + value: { + type: "slot-asserted", + kind: "activity", + node: "approval", + slot: "how long it takes", + assertion: { value: "one hour" }, + }, + }, + }, + ], + completion: { + complete: false, + pluginVersion: "sdcpn/1", + revision: "revision-1", + failures: [ + { + diagnostic: "unaddressed", + nodeId: "activity:approval", + kind: "activity", + slot: "how long it takes", + requirement: "spread", + actual: "not mentioned", + message: "Duration is still unknown.", + captureIds: [], + }, + ], + sliceNodeIds: ["activity:approval", "activity:dispatch"], + outsideSlice: [], + }, + }, + }, + ], + }, + ] as unknown as PetrinautAiMessage[]; + + expect(selectInterviewCoverage(messages)).toEqual({ + complete: false, + covered: ["dispatch"], + stillExploring: ["approval — how long it takes"], + }); + }); + + test("omits coverage when no validated completion report exists", () => { + expect(selectInterviewCoverage([])).toBeNull(); + }); +}); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.ts b/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.ts new file mode 100644 index 00000000000..ecbbf0ed7ca --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.ts @@ -0,0 +1,66 @@ +import { SWEEP_TOOL_NAME } from "@hashintel/brunch-agent-transport-aisdk/client-tools"; + +import { sweepOutputSchema } from "../local-storage-demo/brunch-panel-transport"; + +import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui"; + +export interface InterviewCoverage { + readonly complete: boolean; + readonly covered: readonly string[]; + readonly stillExploring: readonly string[]; +} + +const unique = (items: string[]): string[] => [...new Set(items)]; + +const nodeLabel = (nodeId: string): string => { + const separator = nodeId.indexOf(":"); + return separator === -1 ? nodeId : nodeId.slice(separator + 1); +}; + +export const selectInterviewCoverage = ( + messages: PetrinautAiMessage[], +): InterviewCoverage | null => { + for (const message of messages.toReversed()) { + for (const part of message.parts.toReversed()) { + if ( + part.type !== "dynamic-tool" || + part.toolName !== SWEEP_TOOL_NAME || + part.state !== "output-available" + ) { + continue; + } + const parsed = sweepOutputSchema.safeParse(part.output); + if ( + !parsed.success || + parsed.data.status !== "applied" || + parsed.data.completion === undefined + ) { + continue; + } + + const { completion } = parsed.data; + const nodesWithFailures = new Set( + completion.failures.flatMap((failure) => + failure.nodeId === undefined ? [] : [failure.nodeId], + ), + ); + const covered = completion.sliceNodeIds + .filter((nodeId) => !nodesWithFailures.has(nodeId)) + .map(nodeLabel); + const stillExploring = unique( + completion.failures.map((failure) => + failure.nodeId === undefined + ? failure.message + : `${nodeLabel(failure.nodeId)} — ${failure.slot ?? failure.message}`, + ), + ); + + return { + complete: completion.complete, + covered, + stillExploring, + }; + } + } + return null; +}; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts index 095375e8ab5..82df34ae6dc 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts @@ -11,6 +11,7 @@ class FakeDataChannel extends EventTarget { public readonly close = vi.fn(() => { this.readyState = "closed"; }); + public readonly send = vi.fn(); public open() { this.readyState = "open"; @@ -28,6 +29,20 @@ class FakeDataChannel extends EventTarget { const createHarness = () => { let requestNumber = 0; + const animationFrames: FrameRequestCallback[] = []; + const analyser = { + fftSize: 0, + getByteTimeDomainData: vi.fn((data: Uint8Array) => { + data.fill(160); + }), + }; + const mediaSource = { connect: vi.fn() }; + const audioContext = { + close: vi.fn(async () => undefined), + createAnalyser: vi.fn(() => analyser), + createMediaStreamSource: vi.fn(() => mediaSource), + }; + const cancelAnimationFrame = vi.fn(); const channels: FakeDataChannel[] = []; const peers: Array<{ addTrack: ReturnType; @@ -41,6 +56,7 @@ const createHarness = () => { }> = []; const tracks: Array<{ enabled: boolean; stop: ReturnType }> = []; + const trackEnabledWhenMeterCreated: boolean[] = []; const fetch = vi.fn( async () => new Response("v=0\r\no=OpenAI answer", { @@ -78,18 +94,31 @@ const createHarness = () => { return peer as unknown as RTCPeerConnection; }; const session = new OpenAIRealtimeSession({ + cancelAnimationFrame, connectionTimeoutMs: 15_000, + createAudioContext: () => { + trackEnabledWhenMeterCreated.push(tracks.at(-1)?.enabled ?? true); + return audioContext as unknown as AudioContext; + }, createRequestId: () => `voice-request-${++requestNumber}`, createPeerConnection, fetch, getUserMedia, now: () => 100, reportDiagnostic, + requestAnimationFrame: (callback) => { + animationFrames.push(callback); + return animationFrames.length; + }, }); const events: OpenAIRealtimeSessionEvent[] = []; session.subscribe((event) => events.push(event)); return { + analyser, + animationFrames, + audioContext, + cancelAnimationFrame, channels, events, fetch, @@ -97,6 +126,7 @@ const createHarness = () => { peers, reportDiagnostic, session, + trackEnabledWhenMeterCreated, tracks, }; }; @@ -119,6 +149,7 @@ describe("OpenAIRealtimeSession", () => { }, }); expect(harness.tracks[0]!.enabled).toBe(false); + expect(harness.trackEnabledWhenMeterCreated).toEqual([false]); expect(harness.fetch).toHaveBeenCalledWith( "/api/voice/realtime-call", expect.objectContaining({ @@ -149,6 +180,31 @@ describe("OpenAIRealtimeSession", () => { }); }); + test("reports real input level only while the microphone track is enabled", async () => { + const harness = createHarness(); + await harness.session.connect(); + + expect(harness.animationFrames).toHaveLength(0); + harness.session.setMicrophoneEnabled(true); + expect(harness.animationFrames).toHaveLength(1); + harness.animationFrames.shift()?.(0); + + expect(harness.events.at(-1)).toMatchObject({ + type: "microphone-level", + }); + expect((harness.events.at(-1) as { level: number }).level).toBeGreaterThan( + 0, + ); + + harness.session.setMicrophoneEnabled(false); + expect(harness.tracks[0]!.enabled).toBe(false); + expect(harness.cancelAnimationFrame).toHaveBeenCalled(); + expect(harness.events.at(-1)).toEqual({ + level: 0, + type: "microphone-level", + }); + }); + test("emits only strict input transcription events with stable source identity", async () => { const harness = createHarness(); await harness.session.connect(); @@ -275,6 +331,19 @@ describe("OpenAIRealtimeSession", () => { ]); }); + test("closes capture before explicitly committing buffered input", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + + harness.session.commitInput(); + + expect(harness.tracks[0]!.enabled).toBe(false); + expect(harness.channels[0]!.send).toHaveBeenCalledWith( + JSON.stringify({ type: "input_audio_buffer.commit" }), + ); + }); + test("disposes all WebRTC resources and rejects stale events after reconnect", async () => { const harness = createHarness(); await harness.session.connect(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts index 6cb0c5a0298..afafdb23332 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts @@ -28,6 +28,7 @@ export type OpenAIRealtimeSessionEvent = readonly text: string; readonly type: "partial" | "completed"; } + | { readonly level: number; readonly type: "microphone-level" } | { readonly code: VoiceErrorCode; readonly message: string; @@ -36,7 +37,9 @@ export type OpenAIRealtimeSessionEvent = }; interface OpenAIRealtimeSessionDependencies { + readonly cancelAnimationFrame: (handle: number) => void; readonly connectionTimeoutMs: number; + readonly createAudioContext: () => AudioContext; readonly createRequestId?: () => string; readonly createPeerConnection: () => RTCPeerConnection; readonly fetch: typeof globalThis.fetch; @@ -45,6 +48,7 @@ interface OpenAIRealtimeSessionDependencies { ) => Promise; readonly now?: () => number; readonly reportDiagnostic?: VoiceDiagnosticReporter; + readonly requestAnimationFrame: (callback: FrameRequestCallback) => number; } type SessionListener = (event: OpenAIRealtimeSessionEvent) => void; @@ -104,6 +108,8 @@ const waitForAbort = ( export class OpenAIRealtimeSession { readonly #dependencies: OpenAIRealtimeSessionDependencies; readonly #listeners = new Set(); + #analyser: AnalyserNode | null = null; + #audioContext: AudioContext | null = null; #abortController: AbortController | null = null; #activeEpoch: number | null = null; #connected = false; @@ -112,6 +118,9 @@ export class OpenAIRealtimeSession { #dataChannel: RTCDataChannel | null = null; #epoch = 0; #mediaStream: MediaStream | null = null; + #meterFrame: number | null = null; + #meterHasSample = false; + #meterSamples: Uint8Array | null = null; #messageListener: ((event: MessageEvent) => void) | null = null; #microphoneTrack: MediaStreamTrack | null = null; #peerConnection: RTCPeerConnection | null = null; @@ -209,6 +218,7 @@ export class OpenAIRealtimeSession { } microphoneTrack.enabled = false; this.#microphoneTrack = microphoneTrack; + this.#initializeMeter(mediaStream); const peerConnection = this.#dependencies.createPeerConnection(); this.#peerConnection = peerConnection; @@ -344,7 +354,32 @@ export class OpenAIRealtimeSession { public setMicrophoneEnabled(enabled: boolean): void { if (this.#microphoneTrack) { - this.#microphoneTrack.enabled = enabled && this.#connected; + const isEnabled = enabled && this.#connected; + this.#microphoneTrack.enabled = isEnabled; + if (isEnabled) { + this.#startMeter(); + } else { + this.#stopMeter(); + } + } + } + + public commitInput(): void { + this.setMicrophoneEnabled(false); + if ( + !this.#connected || + !this.#dataChannel || + this.#dataChannel.readyState !== "open" + ) { + this.#handleConnectionFailure("network", "transcription"); + return; + } + try { + this.#dataChannel.send( + JSON.stringify({ type: "input_audio_buffer.commit" }), + ); + } catch { + this.#handleConnectionFailure("network", "transcription"); } } @@ -410,6 +445,59 @@ export class OpenAIRealtimeSession { }); } + #initializeMeter(mediaStream: MediaStream): void { + const audioContext = this.#dependencies.createAudioContext(); + const analyser = audioContext.createAnalyser(); + analyser.fftSize = 256; + audioContext.createMediaStreamSource(mediaStream).connect(analyser); + this.#audioContext = audioContext; + this.#analyser = analyser; + this.#meterSamples = new Uint8Array(analyser.fftSize); + } + + #startMeter(): void { + if (this.#meterFrame !== null || !this.#analyser || !this.#meterSamples) { + return; + } + + const sample = () => { + if ( + !this.#microphoneTrack?.enabled || + !this.#analyser || + !this.#meterSamples + ) { + this.#stopMeter(); + return; + } + this.#analyser.getByteTimeDomainData(this.#meterSamples); + let squaredTotal = 0; + for (const value of this.#meterSamples) { + const normalized = (value - 128) / 128; + squaredTotal += normalized * normalized; + } + this.#emit({ + level: Math.min(1, Math.sqrt(squaredTotal / this.#meterSamples.length)), + type: "microphone-level", + }); + this.#meterHasSample = true; + this.#meterFrame = this.#dependencies.requestAnimationFrame(sample); + }; + + this.#meterFrame = this.#dependencies.requestAnimationFrame(sample); + } + + #stopMeter(): void { + if (this.#meterFrame === null) { + return; + } + this.#dependencies.cancelAnimationFrame(this.#meterFrame); + this.#meterFrame = null; + if (this.#meterHasSample) { + this.#emit({ level: 0, type: "microphone-level" }); + this.#meterHasSample = false; + } + } + #handleMessage(event: MessageEvent, connectionEpoch: number): void { const parsed = parseRealtimeEvent(event.data); if (!parsed || typeof parsed.type !== "string") { @@ -539,6 +627,7 @@ export class OpenAIRealtimeSession { this.#connectionRequestId = null; this.#abortController?.abort(); this.#abortController = null; + this.#stopMeter(); if (this.#dataChannel && this.#messageListener) { this.#dataChannel.removeEventListener("message", this.#messageListener); @@ -571,6 +660,12 @@ export class OpenAIRealtimeSession { this.#mediaStream = null; } this.#microphoneTrack = null; + this.#analyser = null; + this.#meterSamples = null; + if (this.#audioContext) { + void this.#audioContext.close(); + this.#audioContext = null; + } } #waitForDataChannelOpen( diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx index 248bdafa010..162e08810ce 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx @@ -1,8 +1,8 @@ /** * @vitest-environment jsdom */ -import { act, StrictMode } from "react"; -import { createRoot } from "react-dom/client"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { StrictMode, useState } from "react"; import { renderToStaticMarkup } from "react-dom/server"; import { afterEach, describe, expect, test, vi } from "vitest"; @@ -10,28 +10,112 @@ import { loadOpenAIVoiceConfig, VoiceInterviewControl, VoiceInterviewControlView, + type VoiceInterviewControlViewProps, } from "./voice-interview-control"; -describe("voice interview control", () => { - afterEach(() => { - vi.unstubAllGlobals(); - }); +import type { PetrinautAiInterviewStageContext } from "@hashintel/petrinaut/ui"; + +const snapshot = { + currentQuestion: "What happens after approval?", + errorCode: null, + errorMessage: "", + errorRequestId: "", + lastCommittedText: "", + microphoneEnabled: true, + microphoneLevel: 0.24, + partialText: "The request goes to", + phase: "listening" as const, +}; + +const viewProps = ( + overrides: Partial = {}, +): VoiceInterviewControlViewProps => ({ + consented: true, + correction: "", + coverage: null, + editing: false, + microphoneCheck: "", + onCheckMicrophone: vi.fn(), + onConsentChange: vi.fn(), + onCorrectionChange: vi.fn(), + onDoneSpeaking: vi.fn(), + onEdit: vi.fn(), + onEnd: vi.fn(), + onExpand: vi.fn(), + onInterrupt: vi.fn(), + onMinimize: vi.fn(), + onPause: vi.fn(), + onReconnect: vi.fn(), + onRedo: vi.fn(), + onResume: vi.fn(), + onShowStart: vi.fn(), + onStart: vi.fn(), + onSubmitCorrection: vi.fn(), + onTypeInstead: vi.fn(), + placement: "sidebar", + presentation: "full", + snapshot, + ...overrides, +}); + +const StatefulVoiceInterviewHarness = ({ + onOpenSidebar, +}: { + onOpenSidebar: () => void; +}) => { + "use no memo"; + + const [active, setActive] = useState(false); + const [sidebarOpenRequests, setSidebarOpenRequests] = useState(0); + const context: PetrinautAiInterviewStageContext = { + canAcceptInterviewAnswer: true, + conversationId: "interview-test", + focusComposer: vi.fn(), + messages: [], + openSidebar: () => { + onOpenSidebar(); + setSidebarOpenRequests((requests) => requests + 1); + }, + placement: "sidebar", + setActive, + status: "ready", + stop: vi.fn(async () => undefined), + submitInterviewAnswer: vi.fn(async () => ({ + kind: "message" as const, + messageId: "voice-answer", + })), + submitText: vi.fn(async () => ({ + kind: "message" as const, + messageId: "typed-answer", + })), + }; + + return ( + <> + {active ? "Interview active" : "Interview inactive"} + {sidebarOpenRequests} sidebar open requests + + + ); +}; + +afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); +}); - test("loads only a schema-valid, available server configuration", async () => { +describe("voice interview stage", () => { + test("loads only a schema-valid available server configuration", async () => { const fetch = vi.fn(async () => Response.json({ available: true, connectionTimeoutMs: 15_000 }), ); - await expect(loadOpenAIVoiceConfig(fetch)).resolves.toEqual({ available: true, connectionTimeoutMs: 15_000, }); const [url, request] = fetch.mock.calls[0]!; expect(url).toBe("/api/voice/config"); - expect(request).toMatchObject({ - cache: "no-store", - method: "GET", - }); + expect(request).toMatchObject({ cache: "no-store", method: "GET" }); expect(request?.signal).toBeInstanceOf(AbortSignal); fetch.mockResolvedValueOnce( @@ -44,179 +128,188 @@ describe("voice interview control", () => { await expect(loadOpenAIVoiceConfig(fetch)).resolves.toBeNull(); }); - test("renders an accessible idle voice action and live status", () => { + test("shows disclosure and requires consent before starting", () => { const html = renderToStaticMarkup( , ); - expect(html).toContain("Start voice input"); - expect(html).toContain('aria-live="polite"'); - expect(html).toContain("Voice input is off."); + expect(html).toContain("sent to OpenAI for transcription"); + expect(html).toContain("does not retain the audio"); + expect(html).toMatch(/]*disabled[^>]*>Start interview/u); + expect(html).toContain("Use text instead"); + expect(html).toContain("Check microphone"); + expect(html).toContain("pos_absolute"); + expect(html).not.toContain("pos_fixed"); }); - test("labels the half-duplex listening state and keeps partial text visibly provisional", () => { + test("keeps diagnostic recovery details visible without reopening the microphone", () => { const html = renderToStaticMarkup( , ); - expect(html).toContain("Microphone on. Listening."); - expect(html).toContain("Live transcript (not sent)"); - expect(html).toContain("The next activity"); expect(html).toContain( - "Microphone on. Listening. Live transcript (not sent): The next activity", + "Microphone off · Allow microphone access in your browser settings, then reconnect voice input.", ); - expect(html).toContain("End voice input"); - expect(html).toContain("Correct last voice answer"); - expect(html).toContain("Send correction"); + expect(html).toContain("Error code: microphone-permission."); + expect(html).toContain("Diagnostic reference: voice-request-permission."); + expect(html).toContain(">Reconnect<"); }); - test("offers reconnection without reopening the microphone after failure", () => { - const html = renderToStaticMarkup( - , + test("starts in the full stage and keeps recovery visible under Strict Mode", async () => { + const getUserMedia = vi.fn(async () => { + throw new DOMException("Permission denied", "NotAllowedError"); + }); + const openSidebar = vi.fn(); + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ available: true, connectionTimeoutMs: 15_000 }), + ), ); + vi.stubGlobal("navigator", { + mediaDevices: { getUserMedia }, + }); - expect(html).toContain( - "Microphone off. Allow microphone access in your browser settings, then reconnect voice input.", + render( + + + , ); - expect(html).toContain("Error code: microphone-permission."); - expect(html).toContain("Diagnostic reference: voice-request-permission."); - expect(html).toContain("Reconnect voice input"); + + fireEvent.click( + await screen.findByRole("button", { name: "Start voice interview" }), + ); + fireEvent.click(screen.getByRole("checkbox")); + fireEvent.click(screen.getByRole("button", { name: "Start interview" })); + + expect( + await screen.findByRole("region", { name: "Voice interview stage" }), + ).not.toBeNull(); + expect( + await screen.findAllByText( + /Microphone off · Allow microphone access in your browser settings, then reconnect voice input\. Error code: microphone-permission\./u, + ), + ).toHaveLength(2); + expect(screen.getByRole("button", { name: "Reconnect" })).not.toBeNull(); + expect(screen.getByText("Interview active")).not.toBeNull(); + expect(screen.getByText("1 sidebar open requests")).not.toBeNull(); + expect(openSidebar).toHaveBeenCalledOnce(); }); - test("remains interactive after Strict Mode replays its effects", async () => { - const fetch = vi.fn(async () => - Response.json({ available: true, connectionTimeoutMs: 15_000 }), + test("keeps the question visible, distinguishes provisional text, and names microphone level", () => { + const html = renderToStaticMarkup( + , ); - const getUserMedia = vi.fn(async () => { - throw new DOMException("Permission denied", "NotAllowedError"); - }); - vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); - vi.stubGlobal("fetch", fetch); - vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); - - const container = document.createElement("div"); - document.body.append(container); - const root = createRoot(container); - - try { - await act(async () => { - root.render( - - undefined)} - submitText={vi.fn(async () => ({ - kind: "message" as const, - messageId: "message-1", - }))} - /> - , - ); - }); - - const startButton = container.querySelector( - 'button[aria-label="Start voice input"]', - ); - expect(startButton).not.toBeNull(); - - await act(async () => { - startButton!.click(); - }); - - expect(getUserMedia).toHaveBeenCalledOnce(); - expect(container.textContent).toContain( - "Allow microphone access in your browser settings, then reconnect voice input.", - ); - expect( - container.querySelector('button[aria-label="Reconnect voice input"]'), - ).not.toBeNull(); - } finally { - await act(async () => root.unmount()); - container.remove(); + + expect(html).toContain("What happens after approval?"); + expect(html).toContain("What we’re hearing · Not sent yet"); + expect(html).toContain("Microphone on · Listening"); + expect(html).toContain("Microphone input level: Medium"); + expect(html).toContain("Done speaking"); + expect(html).toContain("motionReduce:vis_hidden"); + expect(html).toContain("pos_relative"); + expect(html).not.toContain("pos_fixed"); + expect(html).toContain('aria-live="polite"'); + }); + + test("renders committed repair actions separately from pause, minimize, and end", () => { + const html = renderToStaticMarkup( + , + ); + + for (const name of [ + "Minimize", + "End interview", + "Redo answer", + "Edit text", + "Type instead", + ]) { + expect(html).toContain(name); } }); - test("announces synthesis, playback, and the AI-generated voice disclosure", () => { - const renderPhase = (phase: "synthesizing" | "playing") => - renderToStaticMarkup( - { + const html = renderToStaticMarkup( + , - ); - - const synthesizing = renderPhase("synthesizing"); - expect(synthesizing).toContain( - "Microphone off. Creating AI-generated speech.", + phase: "playing", + }, + })} + />, ); - expect(synthesizing).toContain( - "Spoken responses use an AI-generated OpenAI voice.", + + expect(html).toContain("Microphone off · Interviewer speaking"); + expect(html).toContain("Interrupt and speak"); + expect(html).not.toContain(">Pause<"); + }); + + test("uses a detached bottom mini bar with independent expand, type, pause, and end controls", () => { + const html = renderToStaticMarkup( + , ); - const playing = renderPhase("playing"); - expect(playing).toContain("Microphone off. Playing AI-generated speech."); - expect(playing).toContain( - "Spoken responses use an AI-generated OpenAI voice.", + expect(html).toContain('aria-label="Voice interview mini bar"'); + expect(html).toContain( + 'aria-label="Expand voice interview. Microphone on · Listening"', ); + expect(html).toContain("Microphone on · Listening"); + expect(html).toContain("--voice-interview-right"); + expect(html).toContain("[@media_(min-width:_768px)]"); + expect(html).not.toContain("md:right_4"); + expect(html).toContain('aria-label="Type an interview answer"'); + expect(html).toContain(">Pause<"); + expect(html).toContain('aria-label="End interview"'); + }); + + test("shows authoritative covered and still-exploring facts without a question count", () => { + const html = renderToStaticMarkup( + , + ); + + expect(html).toContain("Covered"); + expect(html).toContain("Still exploring"); + expect(html).not.toMatch(/\d+ of \d+/u); }); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index a435e2b2e1c..04d648b810d 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -1,30 +1,38 @@ import { type FormEvent, useEffect, + useRef, useState, useSyncExternalStore, } from "react"; -import { FaMicrophone, FaMicrophoneSlash } from "react-icons/fa6"; +import { FaKeyboard, FaMicrophone, FaMicrophoneSlash } from "react-icons/fa6"; import { Button } from "@hashintel/ds-components"; -import { css } from "@hashintel/ds-helpers/css"; +import { css, cva } from "@hashintel/ds-helpers/css"; import { reportVoiceDiagnostic } from "../../../voice-diagnostics"; import { selectCanonicalSpeechSegments } from "./canonical-speech"; +import { + type InterviewCoverage, + selectInterviewCoverage, +} from "./interview-coverage"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { SpeechPlaybackController } from "./speech-playback-controller"; import { VoiceTurnController, + type VoiceLatencyEvent, type VoiceTurnSnapshot, } from "./voice-turn-controller"; -import type { PetrinautAiComposerControlContext } from "@hashintel/petrinaut/ui"; +import type { PetrinautAiInterviewStageContext } from "@hashintel/petrinaut/ui"; export interface OpenAIVoiceConfig { readonly available: true; readonly connectionTimeoutMs: number; } +type Presentation = "trigger" | "start" | "full" | "mini"; + const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null; @@ -38,10 +46,7 @@ export const loadOpenAIVoiceConfig = async ( method: "GET", signal, }); - if (!response.ok) { - return null; - } - + if (!response.ok) return null; const body: unknown = await response.json(); if ( !isRecord(body) || @@ -61,77 +66,164 @@ export const loadOpenAIVoiceConfig = async ( } }; -const controlStyle = css({ - position: "relative", - flexShrink: "0", +const rootStyle = cva({ + base: { + zIndex: "overlay", + pointerEvents: "auto", + }, + variants: { + presentation: { + trigger: { + position: "absolute", + right: "[44px]", + bottom: "[-44px]", + }, + start: { + position: "absolute", + right: "0", + bottom: "[-2px]", + width: "full", + }, + full: { + position: "relative", + width: "full", + }, + mini: { + position: "relative", + width: "full", + }, + detached: { + position: "fixed", + "--voice-interview-right": "0px", + "--voice-interview-bottom": "0px", + "--voice-interview-left": "0px", + "--voice-interview-width": "100%", + right: "[var(--voice-interview-right)]", + bottom: "[var(--voice-interview-bottom)]", + left: "[var(--voice-interview-left)]", + width: "[var(--voice-interview-width)]", + "@media (min-width: 768px)": { + "--voice-interview-right": "var(--spacing-4)", + "--voice-interview-bottom": "var(--spacing-4)", + "--voice-interview-left": "auto", + "--voice-interview-width": "440px", + }, + }, + }, + }, }); -const panelStyle = css({ - position: "absolute", - right: "0", - bottom: "[calc(100% + 8px)]", - zIndex: "overlay", +const cardStyle = css({ display: "flex", - width: "[280px]", flexDirection: "column", - gap: "2", - padding: "3", + gap: "3", + padding: "4", borderWidth: "thin", borderStyle: "solid", borderColor: "neutral.a20", - borderRadius: "lg", + borderTopLeftRadius: "xl", + borderTopRightRadius: "xl", + borderBottomRightRadius: "xl", + borderBottomLeftRadius: "xl", backgroundColor: "neutral.s00", - boxShadow: "lg", + boxShadow: "xl", }); -const statusStyle = css({ - color: "neutral.s90", - fontSize: "xs", - fontWeight: "medium", - lineHeight: "relaxed", +const stageStyle = css({ + display: "flex", + maxHeight: "[72vh]", + flexDirection: "column", + gap: "3", + padding: "3", + overflowY: "auto", + borderTopWidth: "thin", + borderBottomWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + backgroundColor: "neutral.s00", + boxShadow: "[0 -8px 24px rgba(0,0,0,0.06)]", + borderRadius: "lg", }); -const disclosureStyle = css({ - color: "neutral.s70", - fontSize: "xs", - lineHeight: "relaxed", +const headerStyle = css({ + display: "flex", + alignItems: "center", + gap: "2", }); -const liveRegionStyle = css({ - position: "absolute", - width: "[1px]", - height: "[1px]", - padding: "0", - margin: "[-1px]", - overflow: "hidden", - clip: "[rect(0, 0, 0, 0)]", - whiteSpace: "nowrap", - borderWidth: "0", +const titleStyle = css({ + flex: "1", + color: "neutral.s100", + fontSize: "sm", + fontWeight: "semibold", +}); + +const questionStyle = css({ + color: "neutral.s110", + fontSize: "lg", + fontWeight: "semibold", + lineHeight: "snug", }); -const partialStyle = css({ +const listeningStyle = css({ display: "flex", + minHeight: "[84px]", flexDirection: "column", + alignItems: "center", + justifyContent: "center", + gap: "2", + borderRadius: "lg", + backgroundColor: "blue.a10", +}); + +const meterStyle = css({ + display: "flex", + height: "[34px]", + alignItems: "center", gap: "1", - padding: "2", - borderRadius: "md", + _motionReduce: { visibility: "hidden" }, +}); + +const meterBarStyle = css({ + width: "[5px]", + minHeight: "[4px]", + borderRadius: "full", + backgroundColor: "blue.s70", + transition: "[height 80ms linear]", + _motionReduce: { transition: "[none]" }, +}); + +const statusStyle = css({ + color: "neutral.s90", + fontSize: "sm", + fontWeight: "medium", +}); + +const transcriptStyle = css({ + display: "flex", + flexDirection: "column", + gap: "1", + padding: "2.5", + borderRadius: "lg", backgroundColor: "neutral.s10", color: "neutral.s100", fontSize: "sm", }); -const partialLabelStyle = css({ +const labelStyle = css({ color: "neutral.s80", fontSize: "xs", + fontWeight: "semibold", }); -const correctionFormStyle = css({ +const actionsStyle = css({ display: "flex", - flexDirection: "column", + flexWrap: "wrap", + alignItems: "center", gap: "2", }); -const correctionInputStyle = css({ +const inputStyle = css({ width: "full", paddingX: "2", paddingY: "1.5", @@ -143,37 +235,76 @@ const correctionInputStyle = css({ color: "neutral.s100", fontSize: "sm", _focusVisible: { - borderColor: "blue.a70", outline: "2px solid", - outlineColor: "blue.a30", + outlineColor: "blue.a40", outlineOffset: "[1px]", }, }); -const panelActionsStyle = css({ +const miniStyle = css({ + display: "flex", + minHeight: "[60px]", + alignItems: "center", + gap: "2", + padding: "2", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + borderTopLeftRadius: "lg", + borderTopRightRadius: "lg", + borderBottomRightRadius: "lg", + borderBottomLeftRadius: "lg", + backgroundColor: "neutral.s00", + boxShadow: "lg", +}); + +const miniExpandStyle = css({ display: "flex", - justifyContent: "flex-end", + minWidth: "0", + flex: "1", + alignItems: "center", gap: "2", + padding: "2", + color: "neutral.s100", + textAlign: "left", + background: "[transparent]", + border: "none", + cursor: "pointer", + _focusVisible: { outline: "2px solid", outlineColor: "blue.a50" }, +}); + +const liveRegionStyle = css({ + position: "absolute", + width: "[1px]", + height: "[1px]", + padding: "0", + margin: "[-1px]", + overflow: "hidden", + clip: "[rect(0,0,0,0)]", + whiteSpace: "nowrap", + borderWidth: "0", }); const statusText = (snapshot: VoiceTurnSnapshot): string => { switch (snapshot.phase) { case "idle": - return "Voice input is off."; + return "Microphone off · Interview not started"; case "connecting": - return "Microphone off. Connecting voice input."; + return "Microphone off · Joining the interview"; case "listening": - return "Microphone on. Listening."; + return "Microphone on · Listening"; + case "paused": + return "Microphone off · Paused"; case "transcribing": - return "Microphone off. Finalizing the transcript."; + return "Microphone off · Finishing your answer"; case "delivering": - return "Microphone off. Sending the finalized transcript to Brunch."; + return "Microphone off · Answer recorded"; case "waiting": - return "Microphone off. Waiting for Brunch."; + return "Microphone off · Writing that down"; case "synthesizing": - return "Microphone off. Creating AI-generated speech."; + return "Microphone off · Preparing the next question"; case "playing": - return "Microphone off. Playing AI-generated speech."; + return "Microphone off · Interviewer speaking"; case "recoverable-error": { const diagnostic = snapshot.errorCode === null @@ -183,158 +314,415 @@ const statusText = (snapshot: VoiceTurnSnapshot): string => { ? ` Diagnostic reference: ${snapshot.errorRequestId}.` : "" }`; - return `Microphone off. ${snapshot.errorMessage}${diagnostic}`; + return `Microphone off · ${snapshot.errorMessage}${diagnostic}`; } } }; -interface VoiceInterviewControlViewProps { +const inputLevelText = (level: number): string => + level >= 0.35 + ? "High" + : level >= 0.12 + ? "Medium" + : level > 0 + ? "Low" + : "Quiet"; + +const Meter = ({ snapshot }: { snapshot: VoiceTurnSnapshot }) => { + const level = snapshot.microphoneEnabled ? snapshot.microphoneLevel : 0; + return ( + <> + + + {snapshot.microphoneEnabled + ? `Microphone input level: ${inputLevelText(level)}` + : "Microphone input level unavailable while microphone is off"} + + + ); +}; + +const Coverage = ({ coverage }: { coverage: InterviewCoverage | null }) => { + if (!coverage) return null; + return ( +
+ + {coverage.complete ? "Coverage complete" : "Interview coverage"} + + {coverage.covered.length > 0 && ( +
+ Covered +
    + {coverage.covered.map((item) => ( +
  • {item}
    • + ))} +
+
+ )} + {coverage.stillExploring.length > 0 && ( +
+ Still exploring +
    + {coverage.stillExploring.map((item) => ( +
  • {item}
    • + ))} +
+
+ )} +
+ ); +}; + +export interface VoiceInterviewControlViewProps { + readonly consented: boolean; readonly correction: string; + readonly coverage: InterviewCoverage | null; + readonly editing: boolean; + readonly microphoneCheck: string; + readonly onCheckMicrophone: () => void; + readonly onConsentChange: (consented: boolean) => void; readonly onCorrectionChange: (value: string) => void; + readonly onDoneSpeaking: () => void; + readonly onEdit: () => void; readonly onEnd: () => void; + readonly onExpand: () => void; + readonly onInterrupt: () => void; + readonly onMinimize: () => void; + readonly onPause: () => void; readonly onReconnect: () => void; + readonly onRedo: () => void; + readonly onResume: () => void; + readonly onShowStart: () => void; readonly onStart: () => void; readonly onSubmitCorrection: () => void; + readonly onTypeInstead: () => void; + readonly placement: "sidebar" | "detached"; + readonly presentation: Presentation; readonly snapshot: VoiceTurnSnapshot; } export const VoiceInterviewControlView = ({ + consented, correction, + coverage, + editing, + microphoneCheck, + onCheckMicrophone, + onConsentChange, onCorrectionChange, + onDoneSpeaking, + onEdit, onEnd, + onExpand, + onInterrupt, + onMinimize, + onPause, onReconnect, + onRedo, + onResume, + onShowStart, onStart, onSubmitCorrection, + onTypeInstead, + placement, + presentation, snapshot, }: VoiceInterviewControlViewProps) => { - const isIdle = snapshot.phase === "idle"; - const isConnecting = snapshot.phase === "connecting"; - const hasError = snapshot.phase === "recoverable-error"; - const canCorrect = - snapshot.phase === "listening" && snapshot.lastCommittedText.length > 0; - - const submitCorrection = (event: FormEvent) => { - event.preventDefault(); - onSubmitCorrection(); - }; - - return ( -
- {isIdle ? ( + if (presentation === "trigger") { + return placement === "sidebar" ? ( +
+ ) : null; + } - - {statusText(snapshot)} - {snapshot.partialText && - ` Live transcript (not sent): ${snapshot.partialText}`} - - - {!isIdle && ( -
-

{statusText(snapshot)}

-

- Spoken responses use an AI-generated OpenAI voice. +

+

Talk with the AI interviewer

+

+ Your speech is sent to OpenAI for transcription. Petrinaut keeps + finalized answers in this conversation; this app does not retain the + audio.

- {snapshot.partialText && ( -

- - Live transcript (not sent) - - {snapshot.partialText} -

+

+ Questions are spoken by an AI-generated OpenAI voice. You can pause, + type, correct an answer, or end the interview at any time. +

+ + {microphoneCheck &&

{microphoneCheck}

} +
+ + + +
+
+
+ ); + } + + const effectivePresentation = + placement === "detached" ? "detached" : presentation; + const status = statusText(snapshot); + const isSpeaking = + snapshot.phase === "playing" || snapshot.phase === "synthesizing"; + + if ( + effectivePresentation === "mini" || + effectivePresentation === "detached" + ) { + return ( +
+
+ + {isSpeaking ? ( + + ) : snapshot.phase === "paused" ? ( + + ) : ( + + )} + +
+ + {status} + +
+ ); + } + + const submitCorrection = (event: FormEvent) => { + event.preventDefault(); + onSubmitCorrection(); + }; + + return ( +
+
+
+ AI · Voice interview + + +
+ +

+ {snapshot.currentQuestion || "The next question will appear here."} +

+ +
+ {snapshot.microphoneEnabled ? ( +
+ + {snapshot.partialText && ( +
+ + What we’re hearing · Not sent yet + + {snapshot.partialText} +
+ )} + {!snapshot.partialText && snapshot.lastCommittedText && ( +
+ Answer recorded + {snapshot.lastCommittedText} +
+ )} + + {editing && ( + + + + onCorrectionChange(event.currentTarget.value) + } + /> + + + )} + +
+ {isSpeaking && ( + + )} + {snapshot.phase === "listening" && ( + <> + - + + + )} + {snapshot.phase === "paused" && ( + + )} + {snapshot.phase === "recoverable-error" && ( + )} -
- {hasError ? ( - - ) : ( - - )} -
-
- )} -
+ + )} + + + + + + + {status} + {snapshot.partialText && ` Not sent yet: ${snapshot.partialText}`} + + ); }; +const recordLatency = (event: VoiceLatencyEvent): void => { + performance.measure(`voice-interview:${event.name}`, { + detail: { questionId: event.questionId }, + duration: event.elapsedMs, + start: 0, + }); +}; + const AvailableVoiceInterviewControl = ({ config, context, }: { config: OpenAIVoiceConfig; - context: PetrinautAiComposerControlContext & { conversationId: string }; + context: PetrinautAiInterviewStageContext & { conversationId: string }; }) => { const [store] = useState(() => { const session = new OpenAIRealtimeSession({ + cancelAnimationFrame: (handle) => globalThis.cancelAnimationFrame(handle), connectionTimeoutMs: config.connectionTimeoutMs, + createAudioContext: () => new AudioContext(), createPeerConnection: () => new RTCPeerConnection(), fetch: globalThis.fetch.bind(globalThis), getUserMedia: (constraints) => navigator.mediaDevices.getUserMedia(constraints), reportDiagnostic: reportVoiceDiagnostic, + requestAnimationFrame: (callback) => + globalThis.requestAnimationFrame(callback), }); const playback = new SpeechPlaybackController({ createAudio: (source) => new Audio(source), @@ -345,9 +733,10 @@ const AvailableVoiceInterviewControl = ({ }); const controller = new VoiceTurnController({ conversationId: context.conversationId, + onLatencyEvent: recordLatency, playback, session, - submitText: context.submitText, + submitText: context.submitInterviewAnswer, }); return { controller, @@ -361,14 +750,51 @@ const AvailableVoiceInterviewControl = ({ store.getSnapshot, store.getSnapshot, ); + const [presentation, setPresentation] = useState("trigger"); + const [previousPlacement, setPreviousPlacement] = useState(context.placement); + const [consented, setConsented] = useState(false); + const [microphoneCheck, setMicrophoneCheck] = useState(""); const [correction, setCorrection] = useState(""); + const [editing, setEditing] = useState(false); + const { openSidebar, setActive } = context; + const openSidebarRef = useRef(openSidebar); + const coverage = selectInterviewCoverage(context.messages); + + if (previousPlacement !== context.placement) { + setPreviousPlacement(context.placement); + if (context.placement === "detached" && snapshot.phase !== "idle") { + setPresentation("mini"); + } + } useEffect(() => { store.controller.updateChat({ + canAcceptInterviewAnswer: context.canAcceptInterviewAnswer, canonicalSegments: selectCanonicalSpeechSegments(context.messages), status: context.status, }); - }, [context.messages, context.status, store]); + }, [ + context.canAcceptInterviewAnswer, + context.messages, + context.status, + store, + ]); + + const active = snapshot.phase !== "idle"; + + useEffect(() => { + setActive(active); + }, [active, setActive]); + + useEffect(() => { + openSidebarRef.current = openSidebar; + }, [openSidebar]); + + useEffect(() => { + if (snapshot.phase === "recoverable-error") { + openSidebarRef.current(); + } + }, [snapshot.phase]); useEffect( () => () => { @@ -377,25 +803,75 @@ const AvailableVoiceInterviewControl = ({ [store], ); + const end = () => { + setPresentation("trigger"); + setEditing(false); + context.setActive(false); + void store.controller.end(); + }; + return ( { + setMicrophoneCheck("Checking microphone…"); + void navigator.mediaDevices.getUserMedia({ audio: true }).then( + (stream) => { + for (const track of stream.getTracks()) track.stop(); + setMicrophoneCheck("Microphone ready."); + }, + () => setMicrophoneCheck("Microphone access was not available."), + ); + }} + onConsentChange={setConsented} onCorrectionChange={setCorrection} - onEnd={() => void store.controller.end()} - onReconnect={() => void store.controller.reconnect()} - onStart={() => void store.controller.start()} + onDoneSpeaking={() => store.controller.doneSpeaking()} + onEdit={() => setEditing(true)} + onEnd={end} + onExpand={() => { + context.openSidebar(); + setPresentation("full"); + }} + onInterrupt={() => store.controller.interruptAndSpeak()} + onMinimize={() => setPresentation("mini")} + onPause={() => store.controller.pause()} + onReconnect={() => { + setPresentation("full"); + void store.controller.reconnect(); + }} + onRedo={() => store.controller.redoAnswer()} + onResume={() => store.controller.resume()} + onShowStart={() => setPresentation("start")} + onStart={() => { + setPresentation("full"); + context.setActive(true); + void store.controller.start(); + }} onSubmitCorrection={() => { const value = correction; setCorrection(""); + setEditing(false); void store.controller.submitCorrection(value); }} + onTypeInstead={() => { + if (snapshot.phase === "idle") setPresentation("trigger"); + context.focusComposer(); + }} + placement={context.placement} + presentation={ + snapshot.phase === "recoverable-error" ? "full" : presentation + } snapshot={snapshot} /> ); }; export const VoiceInterviewControl = ( - context: PetrinautAiComposerControlContext, + context: PetrinautAiInterviewStageContext, ) => { const [config, setConfig] = useState(); @@ -405,17 +881,12 @@ export const VoiceInterviewControl = ( globalThis.fetch.bind(globalThis), abortController.signal, ).then((loadedConfig) => { - if (!abortController.signal.aborted) { - setConfig(loadedConfig); - } + if (!abortController.signal.aborted) setConfig(loadedConfig); }); return () => abortController.abort(); }, []); - if (!config || !context.conversationId) { - return null; - } - + if (!config || !context.conversationId) return null; return ( { }; let requestNumber = 0; const createRequestId = () => requestIds[requestNumber++]!; + const audioContext = { + close: vi.fn(async () => undefined), + createAnalyser: vi.fn(() => ({ + fftSize: 0, + getByteTimeDomainData: vi.fn(), + })), + createMediaStreamSource: vi.fn(() => ({ connect: vi.fn() })), + }; const session = new OpenAIRealtimeSession({ + cancelAnimationFrame: vi.fn(), connectionTimeoutMs: 15_000, + createAudioContext: () => audioContext as unknown as AudioContext, createPeerConnection: () => peer as unknown as RTCPeerConnection, createRequestId, fetch: browserFetch, getUserMedia: async () => mediaStream, now: clock, reportDiagnostic, + requestAnimationFrame: vi.fn(() => 1), }); const audio = createAudioHarness(); const revokeObjectURL = vi.fn(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index 6f35e329567..776b592878a 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -13,6 +13,7 @@ const createHarness = () => { let epoch = 0; let listener: ((event: OpenAIRealtimeSessionEvent) => void) | undefined; const session = { + commitInput: vi.fn(), connect: vi.fn(async () => ++epoch), disconnect: vi.fn(async () => undefined), setMicrophoneEnabled: vi.fn(), @@ -24,6 +25,11 @@ const createHarness = () => { }), }; const submitText = vi.fn(async () => ({ kind: "message" as const })); + const latencyEvents: Array<{ + elapsedMs: number; + name: string; + questionId: string; + }> = []; const playback = { cancel: vi.fn(), play: vi.fn( @@ -37,6 +43,14 @@ const createHarness = () => { }; const controller = new VoiceTurnController({ conversationId: "preview/net 1", + now: (() => { + let now = 1_000; + return () => { + now += 100; + return now; + }; + })(), + onLatencyEvent: (event) => latencyEvents.push(event), playback, session, submitText, @@ -45,6 +59,7 @@ const createHarness = () => { return { controller, emit: (event: OpenAIRealtimeSessionEvent) => listener?.(event), + latencyEvents, playback, session, submitText, @@ -75,6 +90,176 @@ const updateChatStatus = ( ) => controller.updateChat({ canonicalSegments: [], status }); describe("VoiceTurnController", () => { + test("speaks a pending interview question and opens capture before generic chat settlement", async () => { + const harness = createHarness(); + let finishPlayback: (() => void) | undefined; + harness.playback.play.mockImplementationOnce( + async (_segment, events = {}) => { + events.onPlaying?.(); + await new Promise((resolve) => { + finishPlayback = resolve; + }); + }, + ); + const question = { + ...canonicalSegment("ask-1", "What happens after approval?"), + source: "brunch-ask" as const, + }; + + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question], + status: "streaming", + }); + await harness.controller.start(); + + expect(harness.playback.play).toHaveBeenCalledWith( + question, + expect.any(Object), + ); + expect(harness.controller.getSnapshot()).toMatchObject({ + currentQuestion: "What happens after approval?", + microphoneEnabled: false, + phase: "playing", + }); + + finishPlayback?.(); + await vi.waitFor(() => + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: true, + phase: "listening", + }), + ); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); + }); + + test("cancels playback completely before enabling the microphone to interrupt", async () => { + const order: string[] = []; + const harness = createHarness(); + harness.playback.cancel.mockImplementation(() => order.push("cancel")); + harness.session.setMicrophoneEnabled.mockImplementation((enabled) => { + if (enabled) order.push("microphone-on"); + }); + harness.playback.play.mockImplementationOnce( + async (_segment, events = {}) => { + events.onPlaying?.(); + await new Promise(() => undefined); + }, + ); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [ + { + ...canonicalSegment("ask-1"), + source: "brunch-ask" as const, + }, + ], + status: "streaming", + }); + await harness.controller.start(); + + harness.controller.interruptAndSpeak(); + + expect(order.slice(-2)).toEqual(["cancel", "microphone-on"]); + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: true, + phase: "listening", + }); + }); + + test("closes the microphone before committing when the expert is done speaking", async () => { + const order: string[] = []; + const harness = createHarness(); + harness.session.setMicrophoneEnabled.mockImplementation((enabled) => { + if (!enabled) order.push("microphone-off"); + }); + harness.session.commitInput.mockImplementation(() => order.push("commit")); + await harness.controller.start(); + order.length = 0; + + harness.controller.doneSpeaking(); + + expect(order).toEqual(["microphone-off", "commit"]); + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: false, + phase: "transcribing", + }); + }); + + test("records answer-to-question visibility and speech latency", async () => { + const harness = createHarness(); + await harness.controller.start(); + harness.emit({ + key: key(1, "answer-1"), + text: "Approval sends it to dispatch.", + type: "completed", + }); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [ + { + ...canonicalSegment("ask-next"), + source: "brunch-ask" as const, + }, + ], + status: "streaming", + }); + + await vi.waitFor(() => + expect(harness.latencyEvents.map(({ name }) => name)).toEqual([ + "question-visible", + "question-spoken-started", + "question-spoken", + "answer-ready", + ]), + ); + expect(harness.latencyEvents.every(({ elapsedMs }) => elapsedMs >= 0)).toBe( + true, + ); + }); + + test("pauses without ending the connection and redoes an answer as an explicit correction", async () => { + const harness = createHarness(); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [ + { + ...canonicalSegment("ask-1"), + source: "brunch-ask" as const, + }, + ], + status: "ready", + }); + await harness.controller.start(); + + harness.controller.pause(); + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: false, + phase: "paused", + }); + expect(harness.session.disconnect).not.toHaveBeenCalled(); + + harness.controller.resume(); + harness.emit({ + key: key(1, "answer-1"), + text: "The operator approves it.", + type: "completed", + }); + await vi.waitFor(() => expect(harness.submitText).toHaveBeenCalledOnce()); + harness.controller.redoAnswer(); + expect(harness.controller.getSnapshot().phase).toBe("listening"); + harness.emit({ + key: key(1, "answer-2"), + text: "The shift lead approves it.", + type: "completed", + }); + await vi.waitFor(() => expect(harness.submitText).toHaveBeenCalledTimes(2)); + expect(harness.submitText).toHaveBeenLastCalledWith({ + target: "message", + text: 'Correction to my previous voice answer "The operator approves it.": The shift lead approves it.', + }); + }); + test("does not surface an unrelated Brunch error before voice starts", async () => { const harness = createHarness(); @@ -88,7 +273,7 @@ describe("VoiceTurnController", () => { await harness.controller.start(); expect(harness.controller.getSnapshot()).toMatchObject({ - errorMessage: "Wait for Brunch to finish before starting voice input.", + errorMessage: "Wait for the current response to finish before starting.", phase: "recoverable-error", }); }); @@ -104,7 +289,7 @@ describe("VoiceTurnController", () => { false, ); expect(harness.controller.getSnapshot()).toMatchObject({ - errorMessage: "Wait for Brunch to finish before starting voice input.", + errorMessage: "Wait for the current response to finish before starting.", phase: "recoverable-error", }); }); @@ -146,7 +331,7 @@ describe("VoiceTurnController", () => { expect(harness.session.disconnect).toHaveBeenCalledOnce(); expect(harness.controller.getSnapshot()).toMatchObject({ errorMessage: - "Brunch could not complete the current turn. Use the composer to retry.", + "The interview could not complete that turn. Use the composer to retry.", phase: "recoverable-error", }); }); @@ -497,6 +682,41 @@ describe("VoiceTurnController", () => { ); }); + test("reconnects to a pending question without waiting for generic chat settlement", async () => { + const harness = createHarness(); + const question = { + ...canonicalSegment("ask-reconnect", "What happens after approval?"), + source: "brunch-ask" as const, + }; + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question], + status: "streaming", + }); + await harness.controller.start(); + await vi.waitFor(() => + expect(harness.controller.getSnapshot().phase).toBe("listening"), + ); + harness.emit({ + code: "network", + message: "The voice connection failed. Try reconnecting.", + requestId: "voice-request-reconnect", + type: "error", + }); + + await harness.controller.reconnect(); + + await vi.waitFor(() => + expect(harness.controller.getSnapshot()).toMatchObject({ + currentQuestion: "What happens after approval?", + microphoneEnabled: true, + phase: "listening", + }), + ); + expect(harness.playback.play).toHaveBeenCalledTimes(2); + expect(harness.session.connect).toHaveBeenCalledTimes(2); + }); + test("does not let a late delivery overwrite a connection failure", async () => { const harness = createHarness(); let finishDelivery: (() => void) | undefined; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index a76dbf928dd..73a8149bcbf 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -10,6 +10,7 @@ export type VoiceTurnPhase = | "idle" | "connecting" | "listening" + | "paused" | "transcribing" | "delivering" | "waiting" @@ -18,17 +19,31 @@ export type VoiceTurnPhase = | "recoverable-error"; export interface VoiceTurnSnapshot { + readonly currentQuestion: string; readonly errorCode: VoiceErrorCode | null; readonly errorMessage: string; readonly errorRequestId: string; readonly lastCommittedText: string; + readonly microphoneEnabled: boolean; + readonly microphoneLevel: number; readonly partialText: string; readonly phase: VoiceTurnPhase; } +export interface VoiceLatencyEvent { + readonly elapsedMs: number; + readonly name: + | "question-visible" + | "question-spoken-started" + | "question-spoken" + | "answer-ready"; + readonly questionId: string; +} + type ChatStatus = "ready" | "submitted" | "streaming" | "error"; interface RealtimeSession { + commitInput(): void; connect(): Promise; disconnect(): Promise; setMicrophoneEnabled(enabled: boolean): void; @@ -51,12 +66,15 @@ interface SpeechPlayback { interface VoiceTurnControllerDependencies { readonly conversationId: string; + readonly now?: () => number; + readonly onLatencyEvent?: (event: VoiceLatencyEvent) => void; readonly playback: SpeechPlayback; readonly session: RealtimeSession; readonly submitText: (input: SubmitTextInput) => Promise; } interface ChatUpdate { + readonly canAcceptInterviewAnswer?: boolean; readonly canonicalSegments: CanonicalSpeechSegment[]; readonly status: ChatStatus; } @@ -79,10 +97,13 @@ export const createVoiceMessageId = ( ].join(":"); const initialSnapshot: VoiceTurnSnapshot = { + currentQuestion: "", errorCode: null, errorMessage: "", errorRequestId: "", lastCommittedText: "", + microphoneEnabled: false, + microphoneLevel: 0, partialText: "", phase: "idle", }; @@ -90,30 +111,45 @@ const initialSnapshot: VoiceTurnSnapshot = { export class VoiceTurnController { readonly #conversationId: string; readonly #listeners = new Set(); + readonly #now: () => number; + readonly #onLatencyEvent: ((event: VoiceLatencyEvent) => void) | undefined; readonly #playback: SpeechPlayback; readonly #session: RealtimeSession; readonly #submitText: (input: SubmitTextInput) => Promise; readonly #completedKeys = new Set(); readonly #seenSpeechSegmentIds = new Set(); readonly #speechQueue: CanonicalSpeechSegment[] = []; + #answerFinalizedAt: number | null = null; + #answerReadyQuestionId: string | null = null; #activeEpoch: number | null = null; #activeItemId: string | null = null; #activeKey: string | null = null; #activeSpeechSegmentId: string | null = null; #awaitingChatCycle = false; + #availableSegments: CanonicalSpeechSegment[] = []; + #canAcceptInterviewAnswer = true; #chatStatus: ChatStatus = "ready"; + #currentQuestionId: string | null = null; #generation = 0; + #paused = false; + #questionAnswered = false; + #questionPlaybackComplete = false; + #redoing = false; #sawBusyChatStatus = false; #snapshot = initialSnapshot; #speechLoopGeneration: number | null = null; public constructor({ conversationId, + now = () => performance.now(), + onLatencyEvent, playback, session, submitText, }: VoiceTurnControllerDependencies) { this.#conversationId = conversationId; + this.#now = now; + this.#onLatencyEvent = onLatencyEvent; this.#playback = playback; this.#session = session; this.#submitText = submitText; @@ -133,10 +169,12 @@ export class VoiceTurnController { if (this.#snapshot.phase !== "idle") { return; } - if (!this.#isChatReady()) { - this.#session.setMicrophoneEnabled(false); + this.#queueAvailableSegments(); + if (!this.#isChatReady() && !this.#hasPendingQuestion()) { + this.#setMicrophoneEnabled(false); this.#update({ - errorMessage: "Wait for Brunch to finish before starting voice input.", + errorMessage: + "Wait for the current response to finish before starting.", phase: "recoverable-error", }); return; @@ -153,8 +191,12 @@ export class VoiceTurnController { this.#activeItemId = null; this.#activeKey = null; this.#completedKeys.clear(); - const canListen = this.#isChatReady() && this.#speechQueue.length === 0; - this.#session.setMicrophoneEnabled(canListen); + this.#paused = false; + const canListen = + this.#isChatReady() && + this.#speechQueue.length === 0 && + !this.#hasAnswerableQuestion(); + this.#setMicrophoneEnabled(canListen); this.#update({ partialText: "", phase: canListen ? "listening" : "waiting", @@ -168,7 +210,7 @@ export class VoiceTurnController { error instanceof VoiceError ? error : new VoiceError("connection", "invalid-response", ""); - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); this.#update({ errorCode: voiceError.code, errorMessage: voiceError.message, @@ -185,21 +227,40 @@ export class VoiceTurnController { this.#activeKey = null; this.#activeSpeechSegmentId = null; this.#awaitingChatCycle = false; + this.#answerFinalizedAt = null; + this.#answerReadyQuestionId = null; + this.#currentQuestionId = null; + this.#paused = false; + this.#questionAnswered = false; + this.#questionPlaybackComplete = false; + this.#redoing = false; this.#sawBusyChatStatus = false; this.#speechLoopGeneration = null; this.#speechQueue.length = 0; this.#playback.cancel(); - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); await this.#session.disconnect(); this.#update({ errorMessage: "", + currentQuestion: "", + microphoneLevel: 0, partialText: "", phase: "idle", }); } public async reconnect(): Promise { + const pendingQuestionId = this.#currentQuestionId; await this.end(); + if ( + pendingQuestionId !== null && + this.#availableSegments.some( + (segment) => + segment.id === pendingQuestionId && segment.source === "brunch-ask", + ) + ) { + this.#seenSpeechSegmentIds.delete(pendingQuestionId); + } await this.start(); } @@ -209,12 +270,14 @@ export class VoiceTurnController { if ( !correctedText || !previousText || - this.#snapshot.phase !== "listening" + this.#activeEpoch === null || + this.#speechLoopGeneration !== null ) { return; } - this.#session.setMicrophoneEnabled(false); + this.#questionAnswered = true; + this.#setMicrophoneEnabled(false); this.#update({ errorMessage: "", phase: "delivering" }); await this.#deliver({ target: "message", @@ -222,7 +285,73 @@ export class VoiceTurnController { }); } - public updateChat({ canonicalSegments, status }: ChatUpdate): void { + public interruptAndSpeak(): void { + if ( + this.#activeEpoch === null || + (this.#snapshot.phase !== "playing" && + this.#snapshot.phase !== "synthesizing") + ) { + return; + } + + ++this.#generation; + this.#speechLoopGeneration = null; + this.#activeSpeechSegmentId = null; + this.#speechQueue.length = 0; + this.#playback.cancel(); + this.#questionPlaybackComplete = Boolean(this.#snapshot.currentQuestion); + this.#paused = false; + this.#settleListeningIfReady(); + } + + public doneSpeaking(): void { + if (this.#activeEpoch === null || this.#snapshot.phase !== "listening") { + return; + } + this.#setMicrophoneEnabled(false); + this.#update({ phase: "transcribing" }); + this.#session.commitInput(); + } + + public pause(): void { + if (this.#activeEpoch === null || this.#snapshot.phase !== "listening") { + return; + } + this.#paused = true; + this.#setMicrophoneEnabled(false); + this.#update({ phase: "paused" }); + } + + public resume(): void { + if (this.#snapshot.phase !== "paused") { + return; + } + this.#paused = false; + this.#settleListeningIfReady(); + } + + public redoAnswer(): void { + if ( + this.#activeEpoch === null || + !this.#snapshot.lastCommittedText || + this.#speechLoopGeneration !== null || + !this.#canAcceptInterviewAnswer + ) { + return; + } + this.#redoing = true; + this.#questionAnswered = false; + this.#paused = false; + this.#settleListeningIfReady(); + } + + public updateChat({ + canAcceptInterviewAnswer = true, + canonicalSegments, + status, + }: ChatUpdate): void { + this.#availableSegments = canonicalSegments; + this.#canAcceptInterviewAnswer = canAcceptInterviewAnswer; this.#chatStatus = status; const canQueueSpeech = status !== "error" && @@ -232,9 +361,10 @@ export class VoiceTurnController { if (this.#seenSpeechSegmentIds.has(segment.id)) { continue; } - this.#seenSpeechSegmentIds.add(segment.id); if (canQueueSpeech) { - this.#speechQueue.push(segment); + this.#queueSpeechSegment(segment); + } else if (segment.source === "assistant-text") { + this.#seenSpeechSegmentIds.add(segment.id); } } @@ -243,8 +373,8 @@ export class VoiceTurnController { return; } const errorMessage = this.#awaitingChatCycle - ? "Brunch could not accept the voice turn. Use the composer to retry." - : "Brunch could not complete the current turn. Use the composer to retry."; + ? "The interview could not accept that answer. Use the composer to retry." + : "The interview could not complete that turn. Use the composer to retry."; ++this.#generation; this.#activeEpoch = null; this.#activeItemId = null; @@ -255,7 +385,7 @@ export class VoiceTurnController { this.#activeSpeechSegmentId = null; this.#speechLoopGeneration = null; this.#playback.cancel(); - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); void this.#session.disconnect(); this.#update({ errorMessage, phase: "recoverable-error" }); return; @@ -266,7 +396,11 @@ export class VoiceTurnController { if (this.#awaitingChatCycle) { this.#sawBusyChatStatus = true; } - this.#session.setMicrophoneEnabled(false); + if (this.#hasAnswerableQuestion() && !this.#paused) { + this.#settleListeningIfReady(); + return; + } + this.#setMicrophoneEnabled(false); if ( this.#speechLoopGeneration === null && (this.#snapshot.phase === "listening" || @@ -301,10 +435,10 @@ export class VoiceTurnController { return; } this.#awaitingChatCycle = false; - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); this.#update({ errorMessage: - "Brunch could not accept the voice turn. Use the composer to retry.", + "The interview could not accept that answer. Use the composer to retry.", phase: "recoverable-error", }); } @@ -319,6 +453,12 @@ export class VoiceTurnController { } #handleSessionEvent(event: OpenAIRealtimeSessionEvent): void { + if (event.type === "microphone-level") { + if (this.#snapshot.microphoneEnabled) { + this.#update({ microphoneLevel: event.level }); + } + return; + } if (event.type === "error") { ++this.#generation; this.#activeEpoch = null; @@ -330,7 +470,7 @@ export class VoiceTurnController { this.#speechLoopGeneration = null; this.#speechQueue.length = 0; this.#playback.cancel(); - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); this.#update({ errorCode: event.code, errorMessage: event.message, @@ -349,7 +489,7 @@ export class VoiceTurnController { return; } this.#activeItemId = event.itemId; - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); this.#update({ phase: "transcribing" }); return; } @@ -376,7 +516,7 @@ export class VoiceTurnController { this.#activeKey = key; if (event.type === "partial") { - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); this.#update({ partialText: `${this.#snapshot.partialText}${event.text}`, phase: "transcribing", @@ -397,17 +537,29 @@ export class VoiceTurnController { return; } - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); + const previousText = this.#snapshot.lastCommittedText; + const redoing = this.#redoing; + this.#redoing = false; + this.#questionAnswered = true; + this.#answerFinalizedAt = this.#now(); this.#update({ errorMessage: "", lastCommittedText: finalText, partialText: "", phase: "delivering", }); - void this.#deliver({ - id: createVoiceMessageId(this.#conversationId, event.key), - text: finalText, - }); + void this.#deliver( + redoing + ? { + target: "message", + text: `Correction to my previous voice answer "${previousText}": ${finalText}`, + } + : { + id: createVoiceMessageId(this.#conversationId, event.key), + text: finalText, + }, + ); } #startSpeechQueueIfNeeded(): void { @@ -421,7 +573,7 @@ export class VoiceTurnController { const generation = this.#generation; this.#speechLoopGeneration = generation; - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); this.#update({ errorMessage: "", phase: "synthesizing" }); void this.#drainSpeechQueue(generation); } @@ -438,7 +590,7 @@ export class VoiceTurnController { } this.#activeSpeechSegmentId = segment.id; - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); this.#update({ errorMessage: "", phase: "synthesizing" }); try { await this.#playback.play(segment, { @@ -449,6 +601,9 @@ export class VoiceTurnController { this.#activeSpeechSegmentId === segment.id ) { this.#update({ phase: "playing" }); + if (segment.source === "brunch-ask") { + this.#recordLatency("question-spoken-started", segment.id); + } } }, }); @@ -466,7 +621,7 @@ export class VoiceTurnController { this.#activeSpeechSegmentId = null; this.#speechLoopGeneration = null; this.#speechQueue.length = 0; - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); this.#update({ errorCode: voiceError.code, errorMessage: voiceError.message, @@ -483,6 +638,10 @@ export class VoiceTurnController { return; } this.#activeSpeechSegmentId = null; + if (segment.source === "brunch-ask") { + this.#questionPlaybackComplete = true; + this.#recordLatency("question-spoken", segment.id); + } } if ( @@ -500,15 +659,31 @@ export class VoiceTurnController { if ( this.#activeEpoch === null || this.#snapshot.phase === "recoverable-error" || + this.#paused || this.#speechLoopGeneration !== null || this.#speechQueue.length > 0 ) { + if (this.#paused) { + this.#setMicrophoneEnabled(false); + } + return; + } + + if (this.#hasAnswerableQuestion()) { + this.#awaitingChatCycle = false; + this.#sawBusyChatStatus = false; + this.#setMicrophoneEnabled(true); + this.#update({ errorMessage: "", phase: "listening" }); + if (this.#answerReadyQuestionId !== this.#activeQuestionId()) { + this.#answerReadyQuestionId = this.#activeQuestionId(); + this.#recordLatency("answer-ready", this.#activeQuestionId()); + } return; } if (this.#awaitingChatCycle) { if (!this.#sawBusyChatStatus || this.#chatStatus !== "ready") { - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); if ( this.#snapshot.phase === "synthesizing" || this.#snapshot.phase === "playing" @@ -522,7 +697,7 @@ export class VoiceTurnController { } if (!this.#isChatReady()) { - this.#session.setMicrophoneEnabled(false); + this.#setMicrophoneEnabled(false); if ( this.#snapshot.phase !== "transcribing" && this.#snapshot.phase !== "delivering" @@ -532,10 +707,79 @@ export class VoiceTurnController { return; } - this.#session.setMicrophoneEnabled(true); + this.#setMicrophoneEnabled(true); this.#update({ errorMessage: "", phase: "listening" }); } + #activeQuestionId(): string { + return this.#currentQuestionId ?? this.#snapshot.currentQuestion; + } + + #hasPendingQuestion(): boolean { + return ( + Boolean(this.#snapshot.currentQuestion) && this.#canAcceptInterviewAnswer + ); + } + + #hasAnswerableQuestion(): boolean { + return ( + Boolean(this.#snapshot.currentQuestion) && + this.#questionPlaybackComplete && + !this.#questionAnswered && + this.#canAcceptInterviewAnswer + ); + } + + #queueAvailableSegments(): void { + for (const segment of this.#availableSegments) { + if (!this.#seenSpeechSegmentIds.has(segment.id)) { + this.#queueSpeechSegment(segment); + } + } + } + + #queueSpeechSegment(segment: CanonicalSpeechSegment): void { + this.#seenSpeechSegmentIds.add(segment.id); + this.#speechQueue.push(segment); + if (segment.source === "brunch-ask") { + this.#currentQuestionId = segment.id; + this.#questionPlaybackComplete = false; + this.#questionAnswered = false; + this.#answerReadyQuestionId = null; + this.#awaitingChatCycle = false; + this.#sawBusyChatStatus = false; + this.#update({ + currentQuestion: segment.text, + lastCommittedText: "", + }); + this.#recordLatency("question-visible", segment.id); + } + } + + #recordLatency(name: VoiceLatencyEvent["name"], questionId: string): void { + if (this.#answerFinalizedAt === null) { + return; + } + this.#onLatencyEvent?.({ + elapsedMs: Math.max(0, this.#now() - this.#answerFinalizedAt), + name, + questionId, + }); + } + + #setMicrophoneEnabled(enabled: boolean): void { + this.#session.setMicrophoneEnabled(enabled); + if ( + this.#snapshot.microphoneEnabled !== enabled || + (!enabled && this.#snapshot.microphoneLevel !== 0) + ) { + this.#update({ + microphoneEnabled: enabled, + ...(enabled ? {} : { microphoneLevel: 0 }), + }); + } + } + #update(update: Partial): void { const clearedError = update.errorMessage !== undefined && !("errorCode" in update) diff --git a/libs/@hashintel/petrinaut/CHANGELOG.md b/libs/@hashintel/petrinaut/CHANGELOG.md index 34856e63881..bc789007a1b 100644 --- a/libs/@hashintel/petrinaut/CHANGELOG.md +++ b/libs/@hashintel/petrinaut/CHANGELOG.md @@ -4,9 +4,9 @@ ### Patch Changes -- Add a generic host-rendered AI composer control with stable finalized-text submission, - conversation identity, stop handling, schema-validated interactive-tool text mapping, and an - explicit separate-message target for corrections. +- Add generic host-rendered AI composer controls and a persistent interview stage with docked and + detached placements, protected active conversations, keyboard fallback, and one-answer buffering + while the normal chat stream settles. ## 0.0.19 diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 94864c55d03..0201c96e549 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -26,21 +26,42 @@ question is waiting for an answer, completes that question rather than starting message. A host can explicitly submit a separate message instead when the text is a correction or other follow-up that must not answer the pending question. -When the Brunch voice preview is enabled by the host, the additional control can accept finalized -microphone transcripts and speak finalized assistant responses. Live transcript fragments are -labelled **not sent** and do not enter the conversation. The microphone is off while Brunch is -working or a response is playing. Spoken responses use an AI-generated OpenAI voice, as disclosed -in the voice status panel. If speech fails, the response remains visible to read and the voice -control offers recovery instead of changing or regenerating the text. +When the Brunch voice preview is enabled by the host, select **Start voice interview** beside the +composer. Review the transcription, retention, and AI-voice disclosure, optionally check your +microphone, confirm that you understand it, then select **Start interview**. The full interview +stage opens above the composer. It keeps the current question visible, labels live words **Not sent +yet**, and shows **Answer recorded** only after finalization. The words shown as the question are +also the exact words spoken by the AI-generated OpenAI voice. + +The stage names the microphone state and shows input activity only while the microphone is really +on. During interviewer playback, **Interrupt and speak** first stops playback and then opens the +microphone; it does not listen and play at the same time. While listening, choose **Done speaking** +to finalize immediately or **Pause** to turn the microphone off temporarily. **Pause**, +**Minimize**, **End interview**, and **Interrupt and speak** are separate actions. After an answer +is recorded, use **Redo answer** to say an explicit correction or **Edit text** to type one. The +normal message composer remains available as a keyboard fallback. + +Minimizing produces a compact bar above the composer. Closing the AI sidebar during an active +interview moves that same session to a bottom bar over the canvas; reopening the sidebar docks it +again. These presentation changes do not reconnect or end the voice session. On narrow screens the +full stage is a bottom sheet and the compact presentation is a bottom bar. If the session fails, +the sidebar reopens to the full recovery view. **Clear AI chat** is unavailable while an interview +is active. + +When Brunch has produced authoritative completion information, the stage can expand **Covered** and +**Still exploring** topics. It does not show a question count or a speculative model preview; the +Petrinaut canvas remains the model surface. If voice cannot continue, the status panel identifies the kind of problem. For microphone permission or device errors, allow access or connect/select a microphone before reconnecting. For -an interrupted request, network error, or timeout, check the connection and choose **Reconnect -voice input**. If the preview is unavailable, continue with the text composer. An invalid service -response includes a diagnostic reference you can give to an operator; that reference and its -diagnostic record do not contain your transcript or the response being spoken. +an interrupted request, network error, or timeout, check the connection and choose **Reconnect**. +If the preview is unavailable, continue with the text composer. An invalid service response +includes a diagnostic reference you can give to an operator; that reference and its diagnostic +record do not contain your transcript or the response being spoken. -**Clear AI chat** via the delete button in the top right of the panel: wipes the conversation, stops any in-flight stream, and tells the host app to forget the messages (if the host persists them). +When no interview is active, **Clear AI chat** via the delete button in the top right of the panel +wipes the conversation, stops any in-flight stream, and tells the host app to forget the messages +(if the host persists them). ## What the assistant can do @@ -77,7 +98,7 @@ When the assistant edits code surfaces (lambdas, kernels, dynamics, visualizers, ## Host configuration -Whether the assistant is available, which additional composer controls appear, where the +Whether the assistant is available, which additional composer controls or interview stages appear, where the conversation is stored (in-memory, in your host app's database, or anywhere else), and the model behind it are all controlled by the host application that embeds Petrinaut. Read-only documents and the simulate-mode restrictions described above always apply when applicable. diff --git a/libs/@hashintel/petrinaut/src/ui/index.ts b/libs/@hashintel/petrinaut/src/ui/index.ts index 2a515699524..68522324354 100644 --- a/libs/@hashintel/petrinaut/src/ui/index.ts +++ b/libs/@hashintel/petrinaut/src/ui/index.ts @@ -21,6 +21,9 @@ export type { PetrinautAiComposerControlContext, PetrinautAiComposerStatus, PetrinautAiComposerSubmitTextResult, + PetrinautAiInterviewStage, + PetrinautAiInterviewStageContext, + PetrinautAiInterviewStagePlacement, } from "./types/ai-assistant-composer-control"; export { definePetrinautAiInteractiveTool } from "./types/ai-interactive-tool"; export type { diff --git a/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx b/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx index 06f75043915..48ddadbb252 100644 --- a/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx +++ b/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx @@ -29,7 +29,10 @@ const editorRootStyle = css({ backgroundColor: "neutral.s25", }); -import type { PetrinautAiComposerControl } from "./types/ai-assistant-composer-control"; +import type { + PetrinautAiComposerControl, + PetrinautAiInterviewStage, +} from "./types/ai-assistant-composer-control"; import type { PetrinautAiInteractiveTool } from "./types/ai-interactive-tool"; import type { PetrinautAiMessage, @@ -48,6 +51,8 @@ export type PetrinautAiAssistant = { onMessages?: (messages: PetrinautAiMessage[]) => void; /** Render a host-owned control inside the assistant composer. */ renderComposerControl?: PetrinautAiComposerControl; + /** Render one persistent, provider-neutral interview stage. */ + renderInterviewStage?: PetrinautAiInterviewStage; transport: PetrinautAiTransport; }; diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts index d1fb912ae5d..83751176e39 100644 --- a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts @@ -34,3 +34,31 @@ export type PetrinautAiComposerControlContext = { export type PetrinautAiComposerControl = ( context: PetrinautAiComposerControlContext, ) => ReactNode; + +/** Provider-neutral placement for one persistent host-owned interview stage. */ +export type PetrinautAiInterviewStagePlacement = "sidebar" | "detached"; + +/** Stable controls and placement supplied to a host-owned interview stage. */ +export type PetrinautAiInterviewStageContext = + PetrinautAiComposerControlContext & { + /** True when Petrinaut can retain one next answer, even while chat settles. */ + canAcceptInterviewAnswer: boolean; + /** Bring back the sidebar and place keyboard focus in its composer. */ + focusComposer: () => void; + /** Open the AI sidebar without changing the interview session. */ + openSidebar: () => void; + placement: PetrinautAiInterviewStagePlacement; + /** Tell Petrinaut to protect the active conversation from accidental clearing. */ + setActive: (active: boolean) => void; + /** + * Accept one finalized answer immediately. If generic chat is still busy, + * Petrinaut retains it and submits it through the canonical composer path + * as soon as that path is ready. + */ + submitInterviewAnswer: PetrinautAiComposerControlContext["submitText"]; + }; + +/** Render callback for a persistent host-owned stage above the composer. */ +export type PetrinautAiInterviewStage = ( + context: PetrinautAiInterviewStageContext, +) => ReactNode; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index db1a81dad95..a97de9a45cb 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -2,6 +2,7 @@ * @vitest-environment jsdom */ import { + act, cleanup, fireEvent, render, @@ -32,13 +33,19 @@ import { definePetrinautAiInteractiveTool } from "../../../types/ai-interactive- import { AiAssistantPanel } from "./ai-assistant-panel"; import type { PetrinautAiAssistant } from "../../../petrinaut"; -import type { PetrinautAiComposerControlContext } from "../../../types/ai-assistant-composer-control"; +import type { + PetrinautAiComposerControlContext, + PetrinautAiInterviewStageContext, +} from "../../../types/ai-assistant-composer-control"; import type { PetrinautAiMessage, PetrinautAiTransport, } from "./ai-assistant-panel/types"; import type { UIMessageChunk } from "ai"; +let interviewStageMounts = 0; +let interviewStageUnmounts = 0; + const emptySDCPN: SDCPN = { places: [], transitions: [], @@ -132,9 +139,11 @@ const testInstances: ReturnType[] = []; const renderTestPanel = ({ aiAssistant, + editorContext = editorContextValue, initialMessage, }: { aiAssistant: PetrinautAiAssistant; + editorContext?: EditorContextValue; initialMessage?: string; }) => { const handle = createJsonDocHandle({ @@ -156,9 +165,12 @@ const renderTestPanel = ({ getItemType: () => null, }; - const renderPanel = (nextAiAssistant: PetrinautAiAssistant) => ( + const renderPanel = ( + nextAiAssistant: PetrinautAiAssistant, + nextEditorContext: EditorContextValue, + ) => ( - + ); - const rendered = render(renderPanel(aiAssistant)); + const rendered = render(renderPanel(aiAssistant, editorContext)); return { ...rendered, - rerenderPanel: (nextAiAssistant: PetrinautAiAssistant) => - rendered.rerender(renderPanel(nextAiAssistant)), + rerenderPanel: ( + nextAiAssistant: PetrinautAiAssistant, + nextEditorContext = editorContext, + ) => rendered.rerender(renderPanel(nextAiAssistant, nextEditorContext)), }; }; @@ -185,6 +199,144 @@ afterEach(() => { }); describe("AiAssistantPanel composer submissions", () => { + test("redocks one mounted interview stage when the sidebar closes and reopens", () => { + interviewStageMounts = 0; + interviewStageUnmounts = 0; + const Stage = ({ placement }: { placement: string }) => { + useEffect(() => { + interviewStageMounts += 1; + return () => { + interviewStageUnmounts += 1; + }; + }, []); + return
{`Stage ${placement}`}
; + }; + const aiAssistant: PetrinautAiAssistant = { + renderInterviewStage: ({ placement }) => , + transport: { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(), + }, + }; + const rendered = renderTestPanel({ aiAssistant }); + + expect(screen.getByText("Stage sidebar")).not.toBeNull(); + rendered.rerenderPanel(aiAssistant, { + ...editorContextValue, + isAiAssistantOpen: false, + }); + expect(screen.getByText("Stage detached")).not.toBeNull(); + rendered.rerenderPanel(aiAssistant, editorContextValue); + + expect(screen.getByText("Stage sidebar")).not.toBeNull(); + expect(interviewStageMounts).toBe(1); + expect(interviewStageUnmounts).toBe(0); + }); + + test("accepts one interview answer while generic chat is streaming and submits it after settlement", async () => { + let firstStreamController: + | ReadableStreamDefaultController + | undefined; + let secondStreamController: + | ReadableStreamDefaultController + | undefined; + const requests: PetrinautAiMessage[][] = []; + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(({ messages }) => { + requests.push(structuredClone(messages)); + if (requests.length === 1) { + return Promise.resolve( + new ReadableStream({ + start(controller) { + firstStreamController = controller; + for (const chunk of textChunks("question", "Question ready")) { + controller.enqueue(chunk); + } + }, + }), + ); + } + if (requests.length === 2) { + return Promise.resolve( + new ReadableStream({ + start(controller) { + secondStreamController = controller; + for (const chunk of textChunks( + "acknowledgement", + "Answer accepted", + )) { + controller.enqueue(chunk); + } + }, + }), + ); + } + return Promise.resolve( + streamChunks(textChunks("next-answer", "Next answer accepted")), + ); + }), + }; + let latestStageContext: PetrinautAiInterviewStageContext | undefined; + + renderTestPanel({ + aiAssistant: { + renderInterviewStage: (context) => { + latestStageContext = context; + return ( + + ); + }, + transport, + }, + }); + + fireEvent.click(screen.getByRole("button", { name: "Begin" })); + await screen.findByRole("button", { name: "Answer now" }); + expect(latestStageContext?.canAcceptInterviewAnswer).toBe(true); + fireEvent.click(screen.getByRole("button", { name: "Answer now" })); + await waitFor(() => + expect(latestStageContext?.canAcceptInterviewAnswer).toBe(false), + ); + expect(requests).toHaveLength(1); + + await act(async () => firstStreamController?.close()); + await screen.findByText("Answer accepted"); + + expect(requests).toHaveLength(2); + expect(latestStageContext?.status).toBe("streaming"); + expect(latestStageContext?.canAcceptInterviewAnswer).toBe(true); + expect(requests[1]?.at(-1)).toMatchObject({ + role: "user", + parts: [{ type: "text", text: "Queued interview answer" }], + }); + + void latestStageContext?.submitInterviewAnswer({ + target: "message", + text: "Next interview answer", + }); + await waitFor(() => + expect(latestStageContext?.canAcceptInterviewAnswer).toBe(false), + ); + expect(requests).toHaveLength(2); + + await act(async () => secondStreamController?.close()); + await screen.findByText("Next answer accepted"); + expect(requests).toHaveLength(3); + }); + test("exposes the generated useChat conversation identity to host controls", async () => { const chatIds: string[] = []; const observedConversationIds = new Set(); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index 4ba9dd4242f..64587f1f0c2 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -91,6 +91,14 @@ const selectTarget = ( ); }; +type QueuedInterviewAnswer = { + readonly input: Parameters< + PetrinautAiComposerControlContext["submitText"] + >[0]; + readonly reject: (reason?: unknown) => void; + readonly resolve: (result: PetrinautAiComposerSubmitTextResult) => void; +}; + const isPetrinautAiMutationToolName = ( toolName: string, ): toolName is PetrinautAiMutationToolName => @@ -240,6 +248,10 @@ export const AiAssistantPanel = ({ const { petriNetDefinition, setTitle, title } = use(SDCPNContext); const [input, setInput] = useState(""); + const [composerFocusRequest, setComposerFocusRequest] = useState(0); + const [interviewActive, setInterviewActive] = useState(false); + const [interviewAnswerQueued, setInterviewAnswerQueued] = useState(false); + const queuedInterviewAnswerRef = useRef(null); const submittedInitialMessageRef = useRef(null); const titleRef = useRef(title); @@ -739,6 +751,70 @@ export const AiAssistantPanel = ({ const stopStateRef = useLatest(stop); + const submitInterviewAnswer = useCallback< + PetrinautAiComposerControlContext["submitText"] + >( + (answer) => { + if (queuedInterviewAnswerRef.current) { + return Promise.reject( + new Error("The previous interview answer is still being submitted."), + ); + } + const currentStatus = composerSubmissionStateRef.current.status; + if (currentStatus === "error") { + return Promise.reject( + new Error("The interview is not ready to accept an answer."), + ); + } + if (currentStatus === "ready") { + return submitText(answer); + } + + setInterviewAnswerQueued(true); + return new Promise((resolve, reject) => { + queuedInterviewAnswerRef.current = { + input: answer, + reject, + resolve, + }; + }); + }, + [composerSubmissionStateRef, submitText], + ); + + useEffect(() => { + const queued = queuedInterviewAnswerRef.current; + if (!queued) { + return; + } + if (status === "error") { + queuedInterviewAnswerRef.current = null; + setInterviewAnswerQueued(false); + queued.reject(new Error("The interview could not accept that answer.")); + return; + } + if (status !== "ready") { + return; + } + + queuedInterviewAnswerRef.current = null; + setInterviewAnswerQueued(false); + void submitText(queued.input).then( + (result) => queued.resolve(result), + (caught: unknown) => queued.reject(caught), + ); + }, [status, submitText]); + + useEffect( + () => () => { + queuedInterviewAnswerRef.current?.reject( + new Error("The interview conversation changed."), + ); + queuedInterviewAnswerRef.current = null; + }, + [conversationId], + ); + // Like submitText, stop is exposed to host controls and must stay stable. const stopComposer = useCallback(async () => { stopRequestedRef.current = true; @@ -776,7 +852,7 @@ export const AiAssistantPanel = ({ sendMessage, ]); - if (!isAiAssistantOpen || !instance) { + if (!instance) { return null; } @@ -807,14 +883,30 @@ export const AiAssistantPanel = ({ const composerControl = aiAssistant.renderComposerControl?.( composerControlContext, ); + const interviewStage = aiAssistant.renderInterviewStage?.({ + ...composerControlContext, + canAcceptInterviewAnswer: !interviewAnswerQueued, + focusComposer: () => { + setAiAssistantOpen(true); + setComposerFocusRequest((request) => request + 1); + }, + openSidebar: () => setAiAssistantOpen(true), + placement: isAiAssistantOpen ? "sidebar" : "detached", + setActive: setInterviewActive, + submitInterviewAnswer, + }); /* eslint-enable react-hooks-js/refs */ return ( { // Clearing aborts any in-flight response too, which fires `onFinish` diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index 80d7ea3a7f5..a4ed952cb1a 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -9,7 +9,7 @@ import { screen, waitFor, } from "@testing-library/react"; -import { createElement } from "react"; +import { createElement, useEffect } from "react"; import { afterEach, describe, expect, test, vi } from "vitest"; import { DEFAULT_PETRINAUT_EXTENSIONS } from "@hashintel/petrinaut-core"; @@ -20,6 +20,8 @@ import { AiAssistantContents } from "./ai-assistant-contents"; import type { PetrinautAiMessage } from "./types"; const renderMarkdown = vi.hoisted(() => vi.fn()); +let interviewStageMounts = 0; +let interviewStageUnmounts = 0; vi.mock("react-markdown", async (importOriginal) => { const actual = await importOriginal(); @@ -42,6 +44,83 @@ afterEach(() => { }); describe("AiAssistantContents", () => { + test("keeps one provider-neutral interview stage mounted while it moves between docked and detached presentation", () => { + interviewStageMounts = 0; + interviewStageUnmounts = 0; + const Stage = () => { + useEffect(() => { + interviewStageMounts += 1; + return () => { + interviewStageUnmounts += 1; + }; + }, []); + return
Interview stage
; + }; + const props = { + input: "", + interviewStage: , + messages: [] as PetrinautAiMessage[], + onClose: noop, + onInputChange: noop, + onStop: noop, + onSubmit: noop, + status: "ready" as const, + }; + const { rerender } = render( + , + ); + + expect(screen.getByText("Interview stage")).not.toBeNull(); + expect(screen.getByTestId("ai-interview-stage").dataset.placement).toBe( + "sidebar", + ); + rerender(); + + expect(screen.getByTestId("ai-interview-stage").dataset.placement).toBe( + "detached", + ); + expect(interviewStageMounts).toBe(1); + expect(interviewStageUnmounts).toBe(0); + }); + + test("keeps keyboard drafting available and protects clear-chat during an active interview", () => { + render( + Active interview} + isOpen={true} + messages={[ + { + id: "assistant-1", + role: "assistant", + parts: [{ type: "text", text: "Question" }], + }, + ]} + onClearMessages={vi.fn()} + onClose={noop} + onInputChange={noop} + onStop={noop} + onSubmit={noop} + status="streaming" + />, + ); + + expect( + screen.getByRole("textbox", { + name: "Message AI assistant", + }).disabled, + ).toBe(false); + expect( + screen.getByRole("complementary", { name: "AI assistant" }).className, + ).toContain("z_[calc(var(--z-index-sticky)_+_2)]"); + expect( + screen.getByRole("button", { + name: "Clear AI chat", + }).disabled, + ).toBe(true); + }); + test("keeps completed messages memoized when interactive tools are omitted", () => { const messages: PetrinautAiMessage[] = [ { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx index 1ddbf917d40..2263f471a6e 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx @@ -34,10 +34,14 @@ type AiAssistantStatus = "submitted" | "streaming" | "ready" | "error"; const EMPTY_INTERACTIVE_TOOLS: readonly PetrinautAiInteractiveTool[] = []; export type AiAssistantContentsProps = { + clearMessagesDisabled?: boolean; composerControl?: ReactNode; + composerFocusRequest?: number; error?: Error; input: string; + interviewStage?: ReactNode; interactiveTools?: readonly PetrinautAiInteractiveTool[]; + isOpen?: boolean; messages: PetrinautAiMessage[]; onClearMessages?: () => void; onClose: () => void; @@ -55,26 +59,41 @@ export type AiAssistantContentsProps = { const defaultAssistantWidth = 500; -const shellStyle = css({ - position: "absolute", - top: "0", - right: "0", - bottom: "0", - width: `[${defaultAssistantWidth}px]`, - maxWidth: "[calc(100vw - 32px)]", - zIndex: "sticky", - padding: "2", - pointerEvents: "auto", - transition: "[right 150ms ease-in-out]", - _before: { - content: '""', +const shellStyle = cva({ + base: { position: "absolute", - inset: "2", - borderRadius: "[14px]", - background: - "[radial-gradient(circle at 78% 28%, rgba(52,160,250,0.22), rgba(190,230,255,0.04) 54%, transparent 80%)]", - filter: "[blur(4px)]", - pointerEvents: "none", + right: "0", + zIndex: "[calc(var(--z-index-sticky) + 2)]", + pointerEvents: "auto", + transition: "[right 150ms ease-in-out]", + }, + variants: { + open: { + true: { + top: "0", + bottom: "0", + width: `[${defaultAssistantWidth}px]`, + maxWidth: "[calc(100vw - 32px)]", + padding: "2", + _before: { + content: '""', + position: "absolute", + inset: "2", + borderRadius: "[14px]", + background: + "[radial-gradient(circle at 78% 28%, rgba(52,160,250,0.22), rgba(190,230,255,0.04) 54%, transparent 80%)]", + filter: "[blur(4px)]", + pointerEvents: "none", + }, + }, + false: { + bottom: "0", + width: "[0px]", + height: "[0px]", + overflow: "visible", + pointerEvents: "none", + }, + }, }, }); @@ -105,16 +124,46 @@ const resizeHandleStyle = css({ }, }); -const cardStyle = css({ +const cardStyle = cva({ + base: { + position: "relative", + display: "flex", + flexDirection: "column", + }, + variants: { + open: { + true: { + height: "full", + overflow: "hidden", + backgroundColor: "neutral.s10", + borderRadius: "[12px]", + boxShadow: + "[0px 0px 0px 1px rgba(0,0,0,0.06), 0px 1px 1px -0.5px rgba(0,0,0,0.04), 0px 12px 12px -6px rgba(0,0,0,0.02), 0px 4px 4px -12px rgba(0,0,0,0.02)]", + }, + false: { + width: "[0px]", + height: "[0px]", + overflow: "visible", + pointerEvents: "none", + }, + }, + }, +}); + +const panelContentStyle = cva({ + variants: { + visible: { + false: { display: "none" }, + }, + }, +}); + +const interviewStageStyle = css({ position: "relative", - display: "flex", - flexDirection: "column", - height: "full", - overflow: "hidden", - backgroundColor: "neutral.s10", - borderRadius: "[12px]", - boxShadow: - "[0px 0px 0px 1px rgba(0,0,0,0.06), 0px 1px 1px -0.5px rgba(0,0,0,0.04), 0px 12px 12px -6px rgba(0,0,0,0.02), 0px 4px 4px -12px rgba(0,0,0,0.02)]", + zIndex: "[2]", + flexShrink: "0", + overflow: "visible", + pointerEvents: "auto", }); const headerStyle = css({ @@ -406,10 +455,14 @@ const AiAssistantMessage = memo( AiAssistantMessage.displayName = "AiAssistantMessage"; export const AiAssistantContents = ({ + clearMessagesDisabled = false, composerControl, + composerFocusRequest = 0, error, input, + interviewStage, interactiveTools = EMPTY_INTERACTIVE_TOOLS, + isOpen = true, messages, onClearMessages, onClose, @@ -480,8 +533,10 @@ export const AiAssistantContents = ({ }; useEffect(() => { - inputRef.current?.focus(); - }, []); + if (isOpen) { + inputRef.current?.focus(); + } + }, [composerFocusRequest, isOpen]); // Auto-grow the composer to fit its content (up to `composerMaxHeight`, // after which it scrolls internally). Resetting to `auto` before measuring @@ -519,18 +574,23 @@ export const AiAssistantContents = ({ return ( ); From fa85c0a8b5ed84c5ff37bf67d9bba026929d24f4 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Thu, 27 Aug 2026 20:53:06 +0200 Subject: [PATCH 16/20] Harden Chat and Interview mode switching Co-authored-by: Cursor --- .../Editor/components/ai-cta-modal.test.tsx | 36 +++++ .../views/Editor/components/ai-cta-modal.tsx | 12 +- .../Editor/panels/ai-assistant-panel.test.tsx | 51 +++++++ .../Editor/panels/ai-assistant-panel.tsx | 12 +- .../ai-assistant-contents.tsx | 138 +++++++++--------- 5 files changed, 177 insertions(+), 72 deletions(-) diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-cta-modal.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-cta-modal.test.tsx index c5ac2355447..9d4332e15b8 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-cta-modal.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-cta-modal.test.tsx @@ -53,4 +53,40 @@ describe("AiCtaModal", () => { screen.getByLabelText("Describe the process you want to create"), ).not.toBeNull(); }); + + test("returns to Chat when Interview becomes unavailable", () => { + const { rerender } = render( + , + ); + + fireEvent.click(screen.getByRole("tab", { name: "Interview" })); + expect( + screen.getByRole("heading", { + name: "Talk through your process with AI", + }), + ).not.toBeNull(); + + rerender( + , + ); + + expect(screen.queryByRole("tablist")).toBeNull(); + expect( + screen.getByRole("heading", { + name: "Describe the process you want to create", + }), + ).not.toBeNull(); + }); }); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-cta-modal.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-cta-modal.tsx index e8ceb062f6e..2da4cefe08a 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-cta-modal.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-cta-modal.tsx @@ -68,6 +68,12 @@ const aiCtaModalCopyStyle = css({ maxWidth: "[420px]", }); +const aiCtaModalDescriptionStyle = css({ + margin: "0", + color: "neutral.s80", + textStyle: "sm", +}); + const aiCtaModalTitleStyle = css({ margin: "0", color: "neutral.s110", @@ -129,6 +135,8 @@ export const AiCtaModal = ({ }; }, [onDismiss]); + const effectiveInteractionMode = interviewAvailable ? interactionMode : "chat"; + return (
)} - {interactionMode === "chat" ? ( + {effectiveInteractionMode === "chat" ? ( <>
@@ -201,7 +209,7 @@ export const AiCtaModal = ({

Talk through your process with AI

-

+

Answer a few guided questions and Petrinaut will create the model.

diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index 446f54e7024..016254fc395 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -35,6 +35,7 @@ import { AiAssistantPanel } from "./ai-assistant-panel"; import type { PetrinautAiAssistant } from "../../../petrinaut"; import type { PetrinautAiComposerControlContext, + PetrinautAiInteractionMode, PetrinautAiInterviewStageContext, } from "../../../types/ai-assistant-composer-control"; import type { @@ -140,11 +141,15 @@ const testInstances: ReturnType[] = []; const renderTestPanel = ({ aiAssistant, editorContext = editorContextValue, + initialInteractionMode, initialMessage, + onInitialInteractionModeConsumed, }: { aiAssistant: PetrinautAiAssistant; editorContext?: EditorContextValue; + initialInteractionMode?: PetrinautAiInteractionMode; initialMessage?: string; + onInitialInteractionModeConsumed?: () => void; }) => { const handle = createJsonDocHandle({ id: "ai-assistant-panel-test", @@ -174,7 +179,9 @@ const renderTestPanel = ({ @@ -271,6 +278,50 @@ describe("AiAssistantPanel composer submissions", () => { expect(interviewStageUnmounts).toBe(0); }); + test("defers and consumes an initial Interview mode once, then falls back to Chat", () => { + let latestInteractionMode = "chat"; + const onInitialInteractionModeConsumed = vi.fn(); + const aiAssistant: PetrinautAiAssistant = { + renderInterviewStage: (context) => { + latestInteractionMode = context.interactionMode; + return
Interview stage
; + }, + transport: { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(), + }, + }; + const closedEditorContext = { + ...editorContextValue, + isAiAssistantOpen: false, + }; + const rendered = renderTestPanel({ + aiAssistant, + editorContext: closedEditorContext, + initialInteractionMode: "interview", + onInitialInteractionModeConsumed, + }); + + expect(latestInteractionMode).toBe("chat"); + expect(onInitialInteractionModeConsumed).not.toHaveBeenCalled(); + + rendered.rerenderPanel(aiAssistant, editorContextValue); + + expect(latestInteractionMode).toBe("interview"); + expect(onInitialInteractionModeConsumed).toHaveBeenCalledOnce(); + + const unavailableAssistant: PetrinautAiAssistant = { + transport: aiAssistant.transport, + }; + rendered.rerenderPanel(unavailableAssistant, editorContextValue); + + expect(screen.queryByText("Interview stage")).toBeNull(); + expect( + screen.getByPlaceholderText("Describe the process you want to create"), + ).not.toBeNull(); + expect(onInitialInteractionModeConsumed).toHaveBeenCalledOnce(); + }); + test("accepts one interview answer while generic chat is streaming and submits it after settlement", async () => { let firstStreamController: | ReadableStreamDefaultController diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index c430768163d..67b3db61301 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -268,6 +268,8 @@ export const AiAssistantPanel = ({ [], ); const queuedInterviewAnswerRef = useRef(null); + const consumedInitialInteractionModeRef = + useRef(null); const submittedInitialMessageRef = useRef(null); const titleRef = useRef(title); @@ -840,9 +842,16 @@ export const AiAssistantPanel = ({ useEffect(() => { if ( - !isAiAssistantOpen || initialInteractionMode === undefined || initialInteractionMode === null + ) { + consumedInitialInteractionModeRef.current = null; + return; + } + + if ( + !isAiAssistantOpen || + consumedInitialInteractionModeRef.current === initialInteractionMode ) { return; } @@ -853,6 +862,7 @@ export const AiAssistantPanel = ({ ? "chat" : initialInteractionMode, ); + consumedInitialInteractionModeRef.current = initialInteractionMode; onInitialInteractionModeConsumed?.(); }, [ aiAssistant.renderInterviewStage, diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx index 0dd3f7d12a5..c00b0d1ef61 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx @@ -599,10 +599,10 @@ export const AiAssistantContents = ({
- {interviewAvailable ? ( + {interviewAvailable && onInteractionModeChange ? ( {})} + onModeChange={onInteractionModeChange} /> ) : (
AI
@@ -671,75 +671,75 @@ export const AiAssistantContents = ({
- {showChips && ( - setChipsDismissed(true)} - onSelect={(prompt) => onSendPrompt(prompt)} - /> - )} - { - event.preventDefault(); - const submitter = (event.nativeEvent as SubmitEvent).submitter; - if ( - canSubmit && - submitter?.hasAttribute("data-ai-assistant-submit") - ) { - onSubmit(); - } - }} - > -
-