From f3f05bada904c8369855d576eb8dca90a3f1a08d Mon Sep 17 00:00:00 2001 From: maciej-konczal <142623144+maciej-konczal@users.noreply.github.com> Date: Thu, 28 May 2026 15:35:04 +0200 Subject: [PATCH 1/2] fix(onboarding): notify user when voice intro falls back due to missing Gemini key --- .env.example | 8 +++++- packages/gateway/src/onboarding/types.ts | 5 ++++ packages/gateway/src/onboarding/ws-handler.ts | 7 +++++ shell/src/components/OnboardingScreen.tsx | 7 +++++ shell/src/hooks/useOnboarding.ts | 6 +++++ tests/gateway/onboarding/ws-handler.test.ts | 26 +++++++++++++++++++ 6 files changed, 58 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 104e829ec..0b6e66bda 100644 --- a/.env.example +++ b/.env.example @@ -4,7 +4,13 @@ # Required: Anthropic API key for kernel AI features ANTHROPIC_API_KEY=sk-ant-... -# Optional: Gemini API key for image generation (Nano Banana) +# Optional: Gemini API key. Powers two features: +# 1. The voice-first first-boot onboarding (Gemini Live, "Aoede" voice). +# Without this key the intro silently falls back to a text-only mode +# and shows a notice in the shell. +# 2. Image generation (Nano Banana). The IPC tool returns a "not configured" +# message if invoked without a key. +# Get one at https://aistudio.google.com/apikey GEMINI_API_KEY= # Gateway diff --git a/packages/gateway/src/onboarding/types.ts b/packages/gateway/src/onboarding/types.ts index 65b118d41..3ec28d400 100644 --- a/packages/gateway/src/onboarding/types.ts +++ b/packages/gateway/src/onboarding/types.ts @@ -81,6 +81,11 @@ export const GatewayToShellSchema = z.discriminatedUnion("type", [ z.object({ type: z.literal("contextual_content"), content: ContextualContentSchema }), z.object({ type: z.literal("api_key_result"), valid: z.boolean(), error: z.string().optional() }), z.object({ type: z.literal("onboarding_already_complete") }), + z.object({ + type: z.literal("notice"), + code: z.enum(["gemini_unavailable"]), + message: z.string(), + }), z.object({ type: z.literal("error"), code: z.string(), diff --git a/packages/gateway/src/onboarding/ws-handler.ts b/packages/gateway/src/onboarding/ws-handler.ts index 821b04c9c..70aec0ee5 100644 --- a/packages/gateway/src/onboarding/ws-handler.ts +++ b/packages/gateway/src/onboarding/ws-handler.ts @@ -283,6 +283,13 @@ export function createOnboardingHandler(deps: OnboardingDeps) { } else { console.log("[onboarding] No voice mode — Gemini Live connection:", hasVoiceConnection ? "set" : "MISSING"); audioMode = false; + if (audioFormat === "pcm16" && !hasVoiceConnection) { + send({ + type: "notice", + code: "gemini_unavailable", + message: "Voice intro is unavailable — set GEMINI_API_KEY for the spoken onboarding, or continue in text below.", + }); + } send({ type: "mode_change", mode: "text" }); } } diff --git a/shell/src/components/OnboardingScreen.tsx b/shell/src/components/OnboardingScreen.tsx index 5aa87cb2d..e4dd7941f 100644 --- a/shell/src/components/OnboardingScreen.tsx +++ b/shell/src/components/OnboardingScreen.tsx @@ -295,6 +295,13 @@ export function OnboardingScreen({ onComplete, onOpenTerminal }: OnboardingScree )} + {/* Notice (informational, e.g. voice unavailable) */} + {ob.notice && !ob.error && ( +
+ {ob.notice} +
+ )} + {/* Error */} {ob.error && (
diff --git a/shell/src/hooks/useOnboarding.ts b/shell/src/hooks/useOnboarding.ts index 98195d680..afe336a5e 100644 --- a/shell/src/hooks/useOnboarding.ts +++ b/shell/src/hooks/useOnboarding.ts @@ -36,6 +36,7 @@ export interface OnboardingHook { transcripts: Transcript[]; suggestedApps: SuggestedApp[]; error: string | null; + notice: string | null; isVoiceMode: boolean; alreadyComplete: boolean; apiKeyResult: { valid: boolean; error?: string } | null; @@ -55,6 +56,7 @@ export function useOnboarding(): OnboardingHook { const [transcripts, setTranscripts] = useState([]); const [suggestedApps, setSuggestedApps] = useState([]); const [error, setError] = useState(null); + const [notice, setNotice] = useState(null); const [isVoiceMode, setIsVoiceMode] = useState(false); const [alreadyComplete, setAlreadyComplete] = useState(false); const [apiKeyResult, setApiKeyResult] = useState<{ valid: boolean; error?: string } | null>(null); @@ -331,6 +333,9 @@ export function useOnboarding(): OnboardingHook { case "onboarding_already_complete": setAlreadyComplete(true); break; + case "notice": + setNotice(msg.message as string); + break; case "error": setError(msg.message as string); break; @@ -433,6 +438,7 @@ export function useOnboarding(): OnboardingHook { transcripts, suggestedApps, error, + notice, isVoiceMode, alreadyComplete, apiKeyResult, diff --git a/tests/gateway/onboarding/ws-handler.test.ts b/tests/gateway/onboarding/ws-handler.test.ts index 3c9589356..21f07f74e 100644 --- a/tests/gateway/onboarding/ws-handler.test.ts +++ b/tests/gateway/onboarding/ws-handler.test.ts @@ -94,6 +94,32 @@ describe("onboarding websocket handler", () => { expect(existsSync(join(homePath, "system/onboarding-complete.json"))).toBe(true); }); + it("sends a gemini_unavailable notice when voice is requested but no Gemini key is configured", async () => { + const h = handler(); + await h.onOpen((msg) => sent.push(msg)); + + await h.onMessage(JSON.stringify({ type: "start", audioFormat: "pcm16" })); + + expect(geminiMock.clients).toHaveLength(0); + expect(sent).toContainEqual({ type: "mode_change", mode: "text" }); + const notice = sent.find((m) => m.type === "notice"); + expect(notice).toBeDefined(); + expect(notice).toMatchObject({ + type: "notice", + code: "gemini_unavailable", + }); + expect((notice as { message: string }).message).toMatch(/GEMINI_API_KEY/); + }); + + it("does not send a gemini_unavailable notice when the user explicitly chose text mode", async () => { + const h = handler(); + await h.onOpen((msg) => sent.push(msg)); + + await h.onMessage(JSON.stringify({ type: "start", audioFormat: "text" })); + + expect(sent.find((m) => m.type === "notice")).toBeUndefined(); + }); + it("closes an existing Gemini client before handling a duplicate start", async () => { const h = handler("test-gemini-key"); await h.onOpen((msg) => sent.push(msg)); From 90e5415e931b029992b00a8f5495846f7a4d3711 Mon Sep 17 00:00:00 2001 From: maciej-konczal <142623144+maciej-konczal@users.noreply.github.com> Date: Thu, 28 May 2026 17:02:33 +0200 Subject: [PATCH 2/2] fix(onboarding): gemini setup updates --- packages/gateway/src/onboarding/ws-handler.ts | 19 +- shell/src/hooks/useOnboarding.ts | 2 + tests/gateway/onboarding/ws-handler.test.ts | 179 +++++++++++++----- 3 files changed, 152 insertions(+), 48 deletions(-) diff --git a/packages/gateway/src/onboarding/ws-handler.ts b/packages/gateway/src/onboarding/ws-handler.ts index 07c1b05e3..2062de2bb 100644 --- a/packages/gateway/src/onboarding/ws-handler.ts +++ b/packages/gateway/src/onboarding/ws-handler.ts @@ -37,6 +37,15 @@ type SendFn = (msg: GatewayToShell) => void; const ONBOARDING_COMPLETE_FILE = "system/onboarding-complete.json"; const ONBOARDING_STATE_FILE = "system/onboarding-state.json"; +const GEMINI_VOICE_UNAVAILABLE_NOTICE = + "Voice intro is unavailable — set GEMINI_API_KEY for the spoken onboarding, or continue in text below."; +const GEMINI_VOICE_CONNECT_FAILED_NOTICE = + "Voice intro could not connect — continue in text below, or try again in a moment."; + +function sendGeminiVoiceNotice(send: SendFn, message: string): void { + send({ type: "notice", code: "gemini_unavailable", message }); +} + const TAIL_TO_DONE: Record = { greeting: ["interview", "extract_profile", "suggest_apps", "done"], interview: ["extract_profile", "suggest_apps", "done"], @@ -226,6 +235,9 @@ export function createOnboardingHandler(deps: OnboardingDeps) { client.on("disconnected", () => { if (gemini !== client) return; if (sm.current !== "done") { + if (audioMode) { + sendGeminiVoiceNotice(send, GEMINI_VOICE_CONNECT_FAILED_NOTICE); + } send({ type: "mode_change", mode: "text" }); audioMode = false; } @@ -282,6 +294,7 @@ export function createOnboardingHandler(deps: OnboardingDeps) { console.error("[onboarding] Gemini Live connection FAILED:", err instanceof Error ? err.message : String(err)); if (gemini !== client) return; gemini = null; + sendGeminiVoiceNotice(send, GEMINI_VOICE_CONNECT_FAILED_NOTICE); send({ type: "mode_change", mode: "text" }); audioMode = false; } @@ -289,11 +302,7 @@ export function createOnboardingHandler(deps: OnboardingDeps) { console.log("[onboarding] No voice mode — Gemini Live connection:", hasVoiceConnection ? "set" : "MISSING"); audioMode = false; if (audioFormat === "pcm16" && !hasVoiceConnection) { - send({ - type: "notice", - code: "gemini_unavailable", - message: "Voice intro is unavailable — set GEMINI_API_KEY for the spoken onboarding, or continue in text below.", - }); + sendGeminiVoiceNotice(send, GEMINI_VOICE_UNAVAILABLE_NOTICE); } send({ type: "mode_change", mode: "text" }); } diff --git a/shell/src/hooks/useOnboarding.ts b/shell/src/hooks/useOnboarding.ts index a34e4c824..5a5d7ae84 100644 --- a/shell/src/hooks/useOnboarding.ts +++ b/shell/src/hooks/useOnboarding.ts @@ -578,6 +578,8 @@ export function useOnboarding(): OnboardingHook { // Public API const start = useCallback((useVoice: boolean) => { + setNotice(null); + setError(null); setIsVoiceMode(useVoice); void connect() diff --git a/tests/gateway/onboarding/ws-handler.test.ts b/tests/gateway/onboarding/ws-handler.test.ts index ae767acc8..f8a1de9a6 100644 --- a/tests/gateway/onboarding/ws-handler.test.ts +++ b/tests/gateway/onboarding/ws-handler.test.ts @@ -8,9 +8,69 @@ import { import { join, resolve } from "node:path"; import { tmpdir } from "node:os"; import { createOnboardingHandler } from "../../../packages/gateway/src/onboarding/ws-handler.js"; +import type { + OnboardingGoalId, + OnboardingGoalSummary, + ReadinessResponse, + SelectGoalsResponse, +} from "../../../packages/gateway/src/onboarding/activation-contracts.js"; +import type { ReadinessService } from "../../../packages/gateway/src/onboarding/readiness-service.js"; import type { GatewayToShell } from "../../../packages/gateway/src/onboarding/types.js"; +type GoalSeed = Pick & + Partial>; + +function mockReadinessResponse(goals: GoalSeed[]): ReadinessResponse { + return { + overallStatus: "checking", + goals: goals.map((goal) => ({ + id: goal.id, + selected: goal.selected, + label: goal.label ?? goal.id, + description: goal.description ?? `${goal.id} goal`, + })), + gates: [], + systemAgent: "hermes", + activeAgents: ["hermes"], + agents: [], + codingHandoffStatus: null, + }; +} + +function mockReadinessService(overrides: { + getReadiness?: GoalSeed[] | (() => Promise); + selectGoals?: ( + ownerId: string, + goalIds: OnboardingGoalId[], + ) => Promise; +} = {}): Pick { + const getReadinessImpl = async (): Promise => { + const seed = overrides.getReadiness; + if (Array.isArray(seed)) { + return mockReadinessResponse(seed); + } + if (seed) { + return seed(); + } + return mockReadinessResponse([]); + }; + + const defaultSelectGoals = async ( + _ownerId: string, + _goalIds: OnboardingGoalId[], + ): Promise => ({ + goalIds: ["coding"], + steps: [], + }); + + return { + getReadiness: vi.fn(getReadinessImpl), + selectGoals: vi.fn(overrides.selectGoals ?? defaultSelectGoals), + }; +} + const geminiMock = vi.hoisted(() => ({ + connectBehavior: "resolve" as "resolve" | "reject", clients: [] as Array<{ on: ReturnType; connect: ReturnType; @@ -34,7 +94,11 @@ vi.mock("../../../packages/gateway/src/onboarding/gemini-live.js", () => ({ createGeminiLiveClient: vi.fn(() => { const client = { on: vi.fn(), - connect: vi.fn().mockResolvedValue(undefined), + connect: vi.fn().mockImplementation(() => ( + geminiMock.connectBehavior === "reject" + ? Promise.reject(new Error("connection failed")) + : Promise.resolve(undefined) + )), close: vi.fn(), sendText: vi.fn(), sendAudio: vi.fn(), @@ -53,6 +117,7 @@ describe("onboarding websocket handler", () => { homePath = resolve(mkdtempSync(join(tmpdir(), "onboarding-ws-"))); mkdirSync(join(homePath, "system"), { recursive: true }); sent = []; + geminiMock.connectBehavior = "resolve"; geminiMock.clients.length = 0; vi.stubEnv("ANTHROPIC_API_KEY", ""); vi.stubEnv("CLAUDE_CODE_AUTH", ""); @@ -128,18 +193,15 @@ describe("onboarding websocket handler", () => { }); it("returns goal steps for websocket goal selection", async () => { - const readinessService = { - getReadiness: vi.fn(async () => ({ - goals: [], - })), - selectGoals: vi.fn(async () => ({ - goalIds: ["coding" as const], + const readinessService = mockReadinessService({ + selectGoals: async () => ({ + goalIds: ["coding"], steps: [ - { id: "github.connected", required: true, title: "Connect GitHub", unlocks: ["coding" as const] }, - { id: "project.selected", required: true, title: "Choose a project", unlocks: ["coding" as const] }, + { id: "github.connected", required: true, title: "Connect GitHub", unlocks: ["coding"] }, + { id: "project.selected", required: true, title: "Choose a project", unlocks: ["coding"] }, ], - })), - }; + }), + }); const h = handler("", readinessService); await h.onOpen((msg) => sent.push(msg)); @@ -158,15 +220,12 @@ describe("onboarding websocket handler", () => { }); it("does not persist websocket goal selection without a resolved owner", async () => { - const readinessService = { - getReadiness: vi.fn(async () => ({ - goals: [], - })), - selectGoals: vi.fn(async () => ({ - goalIds: ["coding" as const], + const readinessService = mockReadinessService({ + selectGoals: async () => ({ + goalIds: ["coding"], steps: [], - })), - }; + }), + }); const h = handler("", readinessService, undefined); await h.onOpen((msg) => sent.push(msg)); @@ -184,21 +243,19 @@ describe("onboarding websocket handler", () => { }); it("preserves previously selected goals when websocket goal selection persists", async () => { - const readinessService = { - getReadiness: vi.fn(async () => ({ - goals: [ - { id: "assistant" as const, selected: true }, - { id: "coding" as const, selected: false }, - ], - })), - selectGoals: vi.fn(async () => ({ - goalIds: ["assistant" as const, "coding" as const], + const readinessService = mockReadinessService({ + getReadiness: [ + { id: "assistant", selected: true }, + { id: "coding", selected: false }, + ], + selectGoals: async () => ({ + goalIds: ["assistant", "coding"], steps: [ - { id: "integrations.capabilities", required: true, title: "Approve assistant capabilities", unlocks: ["assistant" as const] }, - { id: "github.connected", required: true, title: "Connect GitHub", unlocks: ["coding" as const] }, + { id: "integrations.capabilities", required: true, title: "Approve assistant capabilities", unlocks: ["assistant"] }, + { id: "github.connected", required: true, title: "Connect GitHub", unlocks: ["coding"] }, ], - })), - }; + }), + }); const h = handler("", readinessService); await h.onOpen((msg) => sent.push(msg)); @@ -216,15 +273,13 @@ describe("onboarding websocket handler", () => { }); it("serializes rapid websocket goal selections so later writes include earlier goals", async () => { - let selected: Array<"assistant" | "coding"> = []; - const readinessService = { - getReadiness: vi.fn(async () => ({ - goals: [ - { id: "assistant" as const, selected: selected.includes("assistant") }, - { id: "coding" as const, selected: selected.includes("coding") }, - ], - })), - selectGoals: vi.fn(async (_ownerId: string, goalIds: Array<"assistant" | "coding">) => { + let selected: OnboardingGoalId[] = []; + const readinessService = mockReadinessService({ + getReadiness: async () => mockReadinessResponse([ + { id: "assistant", selected: selected.includes("assistant") }, + { id: "coding", selected: selected.includes("coding") }, + ]), + selectGoals: async (_ownerId, goalIds) => { selected = goalIds; return { goalIds, @@ -235,8 +290,8 @@ describe("onboarding websocket handler", () => { unlocks: [id], })), }; - }), - }; + }, + }); const h = handler("", readinessService); await h.onOpen((msg) => sent.push(msg)); @@ -250,6 +305,44 @@ describe("onboarding websocket handler", () => { expect(sent.filter((msg) => msg.type === "goal_selected")).toHaveLength(2); }); + it("sends a gemini_unavailable notice when Gemini Live connect fails", async () => { + geminiMock.connectBehavior = "reject"; + const h = handler("test-gemini-key"); + await h.onOpen((msg) => sent.push(msg)); + + await h.onMessage(JSON.stringify({ type: "start", audioFormat: "pcm16" })); + + expect(sent).toContainEqual({ type: "mode_change", mode: "text" }); + expect(sent).toContainEqual({ + type: "notice", + code: "gemini_unavailable", + message: expect.stringMatching(/could not connect/i), + }); + }); + + it("sends a gemini_unavailable notice when Gemini Live disconnects during voice onboarding", async () => { + const h = handler("test-gemini-key"); + await h.onOpen((msg) => sent.push(msg)); + + await h.onMessage(JSON.stringify({ type: "start", audioFormat: "pcm16" })); + expect(geminiMock.clients).toHaveLength(1); + + const disconnectedHandler = geminiMock.clients[0].on.mock.calls.find( + (call) => call[0] === "disconnected", + )?.[1] as (() => void) | undefined; + expect(disconnectedHandler).toBeDefined(); + + sent.length = 0; + disconnectedHandler!(); + + expect(sent).toContainEqual({ type: "mode_change", mode: "text" }); + expect(sent).toContainEqual({ + type: "notice", + code: "gemini_unavailable", + message: expect.stringMatching(/could not connect/i), + }); + }); + it("closes an existing Gemini client before handling a duplicate start", async () => { const h = handler("test-gemini-key"); await h.onOpen((msg) => sent.push(msg));