From f3f05bada904c8369855d576eb8dca90a3f1a08d Mon Sep 17 00:00:00 2001
From: maciej-konczal <142623144+maciej-konczal@users.noreply.github.com>
Date: Thu, 28 May 2026 15:35:04 +0200
Subject: [PATCH 1/2] fix(onboarding): notify user when voice intro falls back
due to missing Gemini key
---
.env.example | 8 +++++-
packages/gateway/src/onboarding/types.ts | 5 ++++
packages/gateway/src/onboarding/ws-handler.ts | 7 +++++
shell/src/components/OnboardingScreen.tsx | 7 +++++
shell/src/hooks/useOnboarding.ts | 6 +++++
tests/gateway/onboarding/ws-handler.test.ts | 26 +++++++++++++++++++
6 files changed, 58 insertions(+), 1 deletion(-)
diff --git a/.env.example b/.env.example
index 104e829ec..0b6e66bda 100644
--- a/.env.example
+++ b/.env.example
@@ -4,7 +4,13 @@
# Required: Anthropic API key for kernel AI features
ANTHROPIC_API_KEY=sk-ant-...
-# Optional: Gemini API key for image generation (Nano Banana)
+# Optional: Gemini API key. Powers two features:
+# 1. The voice-first first-boot onboarding (Gemini Live, "Aoede" voice).
+# Without this key the intro silently falls back to a text-only mode
+# and shows a notice in the shell.
+# 2. Image generation (Nano Banana). The IPC tool returns a "not configured"
+# message if invoked without a key.
+# Get one at https://aistudio.google.com/apikey
GEMINI_API_KEY=
# Gateway
diff --git a/packages/gateway/src/onboarding/types.ts b/packages/gateway/src/onboarding/types.ts
index 65b118d41..3ec28d400 100644
--- a/packages/gateway/src/onboarding/types.ts
+++ b/packages/gateway/src/onboarding/types.ts
@@ -81,6 +81,11 @@ export const GatewayToShellSchema = z.discriminatedUnion("type", [
z.object({ type: z.literal("contextual_content"), content: ContextualContentSchema }),
z.object({ type: z.literal("api_key_result"), valid: z.boolean(), error: z.string().optional() }),
z.object({ type: z.literal("onboarding_already_complete") }),
+ z.object({
+ type: z.literal("notice"),
+ code: z.enum(["gemini_unavailable"]),
+ message: z.string(),
+ }),
z.object({
type: z.literal("error"),
code: z.string(),
diff --git a/packages/gateway/src/onboarding/ws-handler.ts b/packages/gateway/src/onboarding/ws-handler.ts
index 821b04c9c..70aec0ee5 100644
--- a/packages/gateway/src/onboarding/ws-handler.ts
+++ b/packages/gateway/src/onboarding/ws-handler.ts
@@ -283,6 +283,13 @@ export function createOnboardingHandler(deps: OnboardingDeps) {
} else {
console.log("[onboarding] No voice mode — Gemini Live connection:", hasVoiceConnection ? "set" : "MISSING");
audioMode = false;
+ if (audioFormat === "pcm16" && !hasVoiceConnection) {
+ send({
+ type: "notice",
+ code: "gemini_unavailable",
+ message: "Voice intro is unavailable — set GEMINI_API_KEY for the spoken onboarding, or continue in text below.",
+ });
+ }
send({ type: "mode_change", mode: "text" });
}
}
diff --git a/shell/src/components/OnboardingScreen.tsx b/shell/src/components/OnboardingScreen.tsx
index 5aa87cb2d..e4dd7941f 100644
--- a/shell/src/components/OnboardingScreen.tsx
+++ b/shell/src/components/OnboardingScreen.tsx
@@ -295,6 +295,13 @@ export function OnboardingScreen({ onComplete, onOpenTerminal }: OnboardingScree
)}
+ {/* Notice (informational, e.g. voice unavailable) */}
+ {ob.notice && !ob.error && (
+
+ {ob.notice}
+
+ )}
+
{/* Error */}
{ob.error && (
diff --git a/shell/src/hooks/useOnboarding.ts b/shell/src/hooks/useOnboarding.ts
index 98195d680..afe336a5e 100644
--- a/shell/src/hooks/useOnboarding.ts
+++ b/shell/src/hooks/useOnboarding.ts
@@ -36,6 +36,7 @@ export interface OnboardingHook {
transcripts: Transcript[];
suggestedApps: SuggestedApp[];
error: string | null;
+ notice: string | null;
isVoiceMode: boolean;
alreadyComplete: boolean;
apiKeyResult: { valid: boolean; error?: string } | null;
@@ -55,6 +56,7 @@ export function useOnboarding(): OnboardingHook {
const [transcripts, setTranscripts] = useState
([]);
const [suggestedApps, setSuggestedApps] = useState([]);
const [error, setError] = useState(null);
+ const [notice, setNotice] = useState(null);
const [isVoiceMode, setIsVoiceMode] = useState(false);
const [alreadyComplete, setAlreadyComplete] = useState(false);
const [apiKeyResult, setApiKeyResult] = useState<{ valid: boolean; error?: string } | null>(null);
@@ -331,6 +333,9 @@ export function useOnboarding(): OnboardingHook {
case "onboarding_already_complete":
setAlreadyComplete(true);
break;
+ case "notice":
+ setNotice(msg.message as string);
+ break;
case "error":
setError(msg.message as string);
break;
@@ -433,6 +438,7 @@ export function useOnboarding(): OnboardingHook {
transcripts,
suggestedApps,
error,
+ notice,
isVoiceMode,
alreadyComplete,
apiKeyResult,
diff --git a/tests/gateway/onboarding/ws-handler.test.ts b/tests/gateway/onboarding/ws-handler.test.ts
index 3c9589356..21f07f74e 100644
--- a/tests/gateway/onboarding/ws-handler.test.ts
+++ b/tests/gateway/onboarding/ws-handler.test.ts
@@ -94,6 +94,32 @@ describe("onboarding websocket handler", () => {
expect(existsSync(join(homePath, "system/onboarding-complete.json"))).toBe(true);
});
+ it("sends a gemini_unavailable notice when voice is requested but no Gemini key is configured", async () => {
+ const h = handler();
+ await h.onOpen((msg) => sent.push(msg));
+
+ await h.onMessage(JSON.stringify({ type: "start", audioFormat: "pcm16" }));
+
+ expect(geminiMock.clients).toHaveLength(0);
+ expect(sent).toContainEqual({ type: "mode_change", mode: "text" });
+ const notice = sent.find((m) => m.type === "notice");
+ expect(notice).toBeDefined();
+ expect(notice).toMatchObject({
+ type: "notice",
+ code: "gemini_unavailable",
+ });
+ expect((notice as { message: string }).message).toMatch(/GEMINI_API_KEY/);
+ });
+
+ it("does not send a gemini_unavailable notice when the user explicitly chose text mode", async () => {
+ const h = handler();
+ await h.onOpen((msg) => sent.push(msg));
+
+ await h.onMessage(JSON.stringify({ type: "start", audioFormat: "text" }));
+
+ expect(sent.find((m) => m.type === "notice")).toBeUndefined();
+ });
+
it("closes an existing Gemini client before handling a duplicate start", async () => {
const h = handler("test-gemini-key");
await h.onOpen((msg) => sent.push(msg));
From 90e5415e931b029992b00a8f5495846f7a4d3711 Mon Sep 17 00:00:00 2001
From: maciej-konczal <142623144+maciej-konczal@users.noreply.github.com>
Date: Thu, 28 May 2026 17:02:33 +0200
Subject: [PATCH 2/2] fix(onboarding): gemini setup updates
---
packages/gateway/src/onboarding/ws-handler.ts | 19 +-
shell/src/hooks/useOnboarding.ts | 2 +
tests/gateway/onboarding/ws-handler.test.ts | 179 +++++++++++++-----
3 files changed, 152 insertions(+), 48 deletions(-)
diff --git a/packages/gateway/src/onboarding/ws-handler.ts b/packages/gateway/src/onboarding/ws-handler.ts
index 07c1b05e3..2062de2bb 100644
--- a/packages/gateway/src/onboarding/ws-handler.ts
+++ b/packages/gateway/src/onboarding/ws-handler.ts
@@ -37,6 +37,15 @@ type SendFn = (msg: GatewayToShell) => void;
const ONBOARDING_COMPLETE_FILE = "system/onboarding-complete.json";
const ONBOARDING_STATE_FILE = "system/onboarding-state.json";
+const GEMINI_VOICE_UNAVAILABLE_NOTICE =
+ "Voice intro is unavailable — set GEMINI_API_KEY for the spoken onboarding, or continue in text below.";
+const GEMINI_VOICE_CONNECT_FAILED_NOTICE =
+ "Voice intro could not connect — continue in text below, or try again in a moment.";
+
+function sendGeminiVoiceNotice(send: SendFn, message: string): void {
+ send({ type: "notice", code: "gemini_unavailable", message });
+}
+
const TAIL_TO_DONE: Record = {
greeting: ["interview", "extract_profile", "suggest_apps", "done"],
interview: ["extract_profile", "suggest_apps", "done"],
@@ -226,6 +235,9 @@ export function createOnboardingHandler(deps: OnboardingDeps) {
client.on("disconnected", () => {
if (gemini !== client) return;
if (sm.current !== "done") {
+ if (audioMode) {
+ sendGeminiVoiceNotice(send, GEMINI_VOICE_CONNECT_FAILED_NOTICE);
+ }
send({ type: "mode_change", mode: "text" });
audioMode = false;
}
@@ -282,6 +294,7 @@ export function createOnboardingHandler(deps: OnboardingDeps) {
console.error("[onboarding] Gemini Live connection FAILED:", err instanceof Error ? err.message : String(err));
if (gemini !== client) return;
gemini = null;
+ sendGeminiVoiceNotice(send, GEMINI_VOICE_CONNECT_FAILED_NOTICE);
send({ type: "mode_change", mode: "text" });
audioMode = false;
}
@@ -289,11 +302,7 @@ export function createOnboardingHandler(deps: OnboardingDeps) {
console.log("[onboarding] No voice mode — Gemini Live connection:", hasVoiceConnection ? "set" : "MISSING");
audioMode = false;
if (audioFormat === "pcm16" && !hasVoiceConnection) {
- send({
- type: "notice",
- code: "gemini_unavailable",
- message: "Voice intro is unavailable — set GEMINI_API_KEY for the spoken onboarding, or continue in text below.",
- });
+ sendGeminiVoiceNotice(send, GEMINI_VOICE_UNAVAILABLE_NOTICE);
}
send({ type: "mode_change", mode: "text" });
}
diff --git a/shell/src/hooks/useOnboarding.ts b/shell/src/hooks/useOnboarding.ts
index a34e4c824..5a5d7ae84 100644
--- a/shell/src/hooks/useOnboarding.ts
+++ b/shell/src/hooks/useOnboarding.ts
@@ -578,6 +578,8 @@ export function useOnboarding(): OnboardingHook {
// Public API
const start = useCallback((useVoice: boolean) => {
+ setNotice(null);
+ setError(null);
setIsVoiceMode(useVoice);
void connect()
diff --git a/tests/gateway/onboarding/ws-handler.test.ts b/tests/gateway/onboarding/ws-handler.test.ts
index ae767acc8..f8a1de9a6 100644
--- a/tests/gateway/onboarding/ws-handler.test.ts
+++ b/tests/gateway/onboarding/ws-handler.test.ts
@@ -8,9 +8,69 @@ import {
import { join, resolve } from "node:path";
import { tmpdir } from "node:os";
import { createOnboardingHandler } from "../../../packages/gateway/src/onboarding/ws-handler.js";
+import type {
+ OnboardingGoalId,
+ OnboardingGoalSummary,
+ ReadinessResponse,
+ SelectGoalsResponse,
+} from "../../../packages/gateway/src/onboarding/activation-contracts.js";
+import type { ReadinessService } from "../../../packages/gateway/src/onboarding/readiness-service.js";
import type { GatewayToShell } from "../../../packages/gateway/src/onboarding/types.js";
+type GoalSeed = Pick &
+ Partial>;
+
+function mockReadinessResponse(goals: GoalSeed[]): ReadinessResponse {
+ return {
+ overallStatus: "checking",
+ goals: goals.map((goal) => ({
+ id: goal.id,
+ selected: goal.selected,
+ label: goal.label ?? goal.id,
+ description: goal.description ?? `${goal.id} goal`,
+ })),
+ gates: [],
+ systemAgent: "hermes",
+ activeAgents: ["hermes"],
+ agents: [],
+ codingHandoffStatus: null,
+ };
+}
+
+function mockReadinessService(overrides: {
+ getReadiness?: GoalSeed[] | (() => Promise);
+ selectGoals?: (
+ ownerId: string,
+ goalIds: OnboardingGoalId[],
+ ) => Promise;
+} = {}): Pick {
+ const getReadinessImpl = async (): Promise => {
+ const seed = overrides.getReadiness;
+ if (Array.isArray(seed)) {
+ return mockReadinessResponse(seed);
+ }
+ if (seed) {
+ return seed();
+ }
+ return mockReadinessResponse([]);
+ };
+
+ const defaultSelectGoals = async (
+ _ownerId: string,
+ _goalIds: OnboardingGoalId[],
+ ): Promise => ({
+ goalIds: ["coding"],
+ steps: [],
+ });
+
+ return {
+ getReadiness: vi.fn(getReadinessImpl),
+ selectGoals: vi.fn(overrides.selectGoals ?? defaultSelectGoals),
+ };
+}
+
const geminiMock = vi.hoisted(() => ({
+ connectBehavior: "resolve" as "resolve" | "reject",
clients: [] as Array<{
on: ReturnType;
connect: ReturnType;
@@ -34,7 +94,11 @@ vi.mock("../../../packages/gateway/src/onboarding/gemini-live.js", () => ({
createGeminiLiveClient: vi.fn(() => {
const client = {
on: vi.fn(),
- connect: vi.fn().mockResolvedValue(undefined),
+ connect: vi.fn().mockImplementation(() => (
+ geminiMock.connectBehavior === "reject"
+ ? Promise.reject(new Error("connection failed"))
+ : Promise.resolve(undefined)
+ )),
close: vi.fn(),
sendText: vi.fn(),
sendAudio: vi.fn(),
@@ -53,6 +117,7 @@ describe("onboarding websocket handler", () => {
homePath = resolve(mkdtempSync(join(tmpdir(), "onboarding-ws-")));
mkdirSync(join(homePath, "system"), { recursive: true });
sent = [];
+ geminiMock.connectBehavior = "resolve";
geminiMock.clients.length = 0;
vi.stubEnv("ANTHROPIC_API_KEY", "");
vi.stubEnv("CLAUDE_CODE_AUTH", "");
@@ -128,18 +193,15 @@ describe("onboarding websocket handler", () => {
});
it("returns goal steps for websocket goal selection", async () => {
- const readinessService = {
- getReadiness: vi.fn(async () => ({
- goals: [],
- })),
- selectGoals: vi.fn(async () => ({
- goalIds: ["coding" as const],
+ const readinessService = mockReadinessService({
+ selectGoals: async () => ({
+ goalIds: ["coding"],
steps: [
- { id: "github.connected", required: true, title: "Connect GitHub", unlocks: ["coding" as const] },
- { id: "project.selected", required: true, title: "Choose a project", unlocks: ["coding" as const] },
+ { id: "github.connected", required: true, title: "Connect GitHub", unlocks: ["coding"] },
+ { id: "project.selected", required: true, title: "Choose a project", unlocks: ["coding"] },
],
- })),
- };
+ }),
+ });
const h = handler("", readinessService);
await h.onOpen((msg) => sent.push(msg));
@@ -158,15 +220,12 @@ describe("onboarding websocket handler", () => {
});
it("does not persist websocket goal selection without a resolved owner", async () => {
- const readinessService = {
- getReadiness: vi.fn(async () => ({
- goals: [],
- })),
- selectGoals: vi.fn(async () => ({
- goalIds: ["coding" as const],
+ const readinessService = mockReadinessService({
+ selectGoals: async () => ({
+ goalIds: ["coding"],
steps: [],
- })),
- };
+ }),
+ });
const h = handler("", readinessService, undefined);
await h.onOpen((msg) => sent.push(msg));
@@ -184,21 +243,19 @@ describe("onboarding websocket handler", () => {
});
it("preserves previously selected goals when websocket goal selection persists", async () => {
- const readinessService = {
- getReadiness: vi.fn(async () => ({
- goals: [
- { id: "assistant" as const, selected: true },
- { id: "coding" as const, selected: false },
- ],
- })),
- selectGoals: vi.fn(async () => ({
- goalIds: ["assistant" as const, "coding" as const],
+ const readinessService = mockReadinessService({
+ getReadiness: [
+ { id: "assistant", selected: true },
+ { id: "coding", selected: false },
+ ],
+ selectGoals: async () => ({
+ goalIds: ["assistant", "coding"],
steps: [
- { id: "integrations.capabilities", required: true, title: "Approve assistant capabilities", unlocks: ["assistant" as const] },
- { id: "github.connected", required: true, title: "Connect GitHub", unlocks: ["coding" as const] },
+ { id: "integrations.capabilities", required: true, title: "Approve assistant capabilities", unlocks: ["assistant"] },
+ { id: "github.connected", required: true, title: "Connect GitHub", unlocks: ["coding"] },
],
- })),
- };
+ }),
+ });
const h = handler("", readinessService);
await h.onOpen((msg) => sent.push(msg));
@@ -216,15 +273,13 @@ describe("onboarding websocket handler", () => {
});
it("serializes rapid websocket goal selections so later writes include earlier goals", async () => {
- let selected: Array<"assistant" | "coding"> = [];
- const readinessService = {
- getReadiness: vi.fn(async () => ({
- goals: [
- { id: "assistant" as const, selected: selected.includes("assistant") },
- { id: "coding" as const, selected: selected.includes("coding") },
- ],
- })),
- selectGoals: vi.fn(async (_ownerId: string, goalIds: Array<"assistant" | "coding">) => {
+ let selected: OnboardingGoalId[] = [];
+ const readinessService = mockReadinessService({
+ getReadiness: async () => mockReadinessResponse([
+ { id: "assistant", selected: selected.includes("assistant") },
+ { id: "coding", selected: selected.includes("coding") },
+ ]),
+ selectGoals: async (_ownerId, goalIds) => {
selected = goalIds;
return {
goalIds,
@@ -235,8 +290,8 @@ describe("onboarding websocket handler", () => {
unlocks: [id],
})),
};
- }),
- };
+ },
+ });
const h = handler("", readinessService);
await h.onOpen((msg) => sent.push(msg));
@@ -250,6 +305,44 @@ describe("onboarding websocket handler", () => {
expect(sent.filter((msg) => msg.type === "goal_selected")).toHaveLength(2);
});
+ it("sends a gemini_unavailable notice when Gemini Live connect fails", async () => {
+ geminiMock.connectBehavior = "reject";
+ const h = handler("test-gemini-key");
+ await h.onOpen((msg) => sent.push(msg));
+
+ await h.onMessage(JSON.stringify({ type: "start", audioFormat: "pcm16" }));
+
+ expect(sent).toContainEqual({ type: "mode_change", mode: "text" });
+ expect(sent).toContainEqual({
+ type: "notice",
+ code: "gemini_unavailable",
+ message: expect.stringMatching(/could not connect/i),
+ });
+ });
+
+ it("sends a gemini_unavailable notice when Gemini Live disconnects during voice onboarding", async () => {
+ const h = handler("test-gemini-key");
+ await h.onOpen((msg) => sent.push(msg));
+
+ await h.onMessage(JSON.stringify({ type: "start", audioFormat: "pcm16" }));
+ expect(geminiMock.clients).toHaveLength(1);
+
+ const disconnectedHandler = geminiMock.clients[0].on.mock.calls.find(
+ (call) => call[0] === "disconnected",
+ )?.[1] as (() => void) | undefined;
+ expect(disconnectedHandler).toBeDefined();
+
+ sent.length = 0;
+ disconnectedHandler!();
+
+ expect(sent).toContainEqual({ type: "mode_change", mode: "text" });
+ expect(sent).toContainEqual({
+ type: "notice",
+ code: "gemini_unavailable",
+ message: expect.stringMatching(/could not connect/i),
+ });
+ });
+
it("closes an existing Gemini client before handling a duplicate start", async () => {
const h = handler("test-gemini-key");
await h.onOpen((msg) => sent.push(msg));