fix(ai): fall back to default gateway when swarm Bifrost is unreachable

src/lib/ai/provider.ts

@@ -112,6 +112,44 @@ export function getModel(
   });
 }
 
+/**
+ * Bifrost gateway reachability probe.
+ *
+ * Bifrost-routed LLM calls go through ONE gateway — the primary
+ * workspace's swarm proxy (`baseUrl`, e.g.
+ * `https://swarm38.sphinx.chat:8181`). When that swarm is unreachable
+ * (expired/self-signed TLS cert, connection refused, DNS, timeout) the
+ * call dies even when the default gateway is healthy.
+ *
+ * So before committing to the swarm route, callers pre-flight it: any
+ * resolved HTTP response (even a 404/401) means the TLS handshake + TCP
+ * connect succeeded → the gateway is reachable, keep the Bifrost route.
+ * A *rejection* (CERT_HAS_EXPIRED, ECONNREFUSED, timeout, fetch failed)
+ * means we can't talk to it → drop the entire Bifrost bundle (baseUrl +
+ * VK + macaroon) and fall back to the plain default gateway.
+ *
+ * We probe rather than catch a mid-stream `streamText` error because by
+ * the time that surfaces, the HTTP response is already returned to the
+ * client and the stream can't be restarted.
+ */
+export const GATEWAY_PROBE_TIMEOUT_MS = 3000;
+export async function isGatewayReachable(
+  baseUrl: string,
+  timeoutMs: number = GATEWAY_PROBE_TIMEOUT_MS,
+): Promise<boolean> {
+  try {
+    await fetch(baseUrl, {
+      method: "GET",
+      signal: AbortSignal.timeout(timeoutMs),
+    });
+    // Any resolved response (any status) means the connection — and
+    // therefore the certificate — is fine. We don't care about the body.
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 /**
  * Get provider tool with mock support
  */

src/lib/ai/runCanvasAgent.ts

@@ -67,7 +67,12 @@ import {
 } from "@/lib/ai/capabilities";
 import { getLinkedWorkspacesForInitiative } from "@/lib/canvas/linkedWorkspaces";
 import { sanitizeAndCompleteToolCalls } from "@/lib/ai/message-sanitizer";
-import { getModel, getApiKeyForProvider, type Provider } from "@/lib/ai/provider";
+import {
+  getModel,
+  getApiKeyForProvider,
+  isGatewayReachable,
+  type Provider,
+} from "@/lib/ai/provider";
 import { getProviderOptions } from "aieo";
 // Deep import — see comment in services/task-workflow.ts.
 import { getBifrostForLLM } from "@/services/bifrost/orchestrator";
@@ -851,13 +856,33 @@ export async function runCanvasAgent(
   const modelOverride =
     modelName && modelName.startsWith("anthropic/") ? modelName : undefined;
 
+  // Pre-flight the swarm Bifrost gateway. If we can't connect to it
+  // (expired cert / connection refused / timeout), discard the whole
+  // Bifrost bundle and fall back to the default gateway path — the same
+  // route a non-Bifrost workspace takes (default key, no baseUrl, no
+  // custom headers). This rescues the turn when a single swarm is down
+  // instead of failing the whole request. See `isGatewayReachable`.
+  let activeBifrost = bifrost;
+  if (bifrost?.baseUrl && !(await isGatewayReachable(bifrost.baseUrl))) {
+    console.warn(
+      "[runCanvasAgent] Bifrost gateway unreachable; falling back to default gateway",
+      {
+        workspaces: workspaceSlugs,
+        orgId: orgId ?? null,
+        baseUrl: bifrost.baseUrl,
+        agentName: bifrost.agentName,
+      },
+    );
+    activeBifrost = undefined;
+  }
+
   const model = getModel(
     provider,
-    bifrost?.apiKey ?? apiKey,
+    activeBifrost?.apiKey ?? apiKey,
     primarySlug,
     modelOverride,
-    bifrost
-      ? { baseUrl: bifrost.baseUrl, headers: bifrost.headers }
+    activeBifrost
+      ? { baseUrl: activeBifrost.baseUrl, headers: activeBifrost.headers }
       : undefined,
   );
 
@@ -886,9 +911,10 @@ export async function runCanvasAgent(
     orgId: orgId ?? null,
     readonly,
     silentPusher,
-    bifrost: bifrost
-      ? { runId: bifrost.runId, agentName: bifrost.agentName }
+    bifrost: activeBifrost
+      ? { runId: activeBifrost.runId, agentName: activeBifrost.agentName }
       : null,
+    bifrostFellBack: !!bifrost && !activeBifrost,
     cacheControl: (providerOptions as { anthropic?: { cacheControl?: unknown } })
       ?.anthropic?.cacheControl ?? null,
   });

src/services/canvas-turn-enrichments.ts

@@ -17,7 +17,11 @@
 
 import { ModelMessage, generateObject } from "ai";
 import { z } from "zod";
-import { getModel, getApiKeyForProvider } from "@/lib/ai/provider";
+import {
+  getModel,
+  getApiKeyForProvider,
+  isGatewayReachable,
+} from "@/lib/ai/provider";
 import { getBifrostForLLM } from "@/services/bifrost/orchestrator";
 import { getWorkspaceChannelName, PUSHER_EVENTS, pusherServer } from "@/lib/pusher";
 import { swarmFetch } from "@/lib/ai/concepts";
@@ -127,15 +131,30 @@ export async function emitFollowUpQuestions(args: {
       },
       { agentName },
     );
+    // Pre-flight the swarm Bifrost gateway; if it's unreachable (expired
+    // cert / connection refused / timeout), drop the whole Bifrost bundle
+    // and fall back to the default gateway — same resilience as the main
+    // stream in `runCanvasAgent`. See `isGatewayReachable`.
+    let activeFollowUpBifrost = followUpBifrost;
+    if (
+      followUpBifrost?.baseUrl &&
+      !(await isGatewayReachable(followUpBifrost.baseUrl))
+    ) {
+      console.warn(
+        "[emitFollowUpQuestions] Bifrost gateway unreachable; falling back to default gateway",
+        { primarySlug, baseUrl: followUpBifrost.baseUrl, agentName },
+      );
+      activeFollowUpBifrost = undefined;
+    }
     const followUpModel = getModel(
       "anthropic",
-      followUpBifrost?.apiKey ?? followUpApiKey,
+      activeFollowUpBifrost?.apiKey ?? followUpApiKey,
       primarySlug,
       undefined,
-      followUpBifrost
+      activeFollowUpBifrost
         ? {
-            baseUrl: followUpBifrost.baseUrl,
-            headers: followUpBifrost.headers,
+            baseUrl: activeFollowUpBifrost.baseUrl,
+            headers: activeFollowUpBifrost.headers,
           }
         : undefined,
     );