Surface transient memU backend outages instead of silently returning no memories

nerve/agent/tools/handlers/memory.py

@@ -26,6 +26,7 @@
     MEMORY_UPDATE_SCHEMA,
     SESSION_CONTEXT_SCHEMA,
 )
+from nerve.memory.memu_bridge import MemoryBackendUnavailable
 
 logger = logging.getLogger(__name__)
 
@@ -102,6 +103,13 @@ async def memory_recall_handler(ctx: ToolContext, args: dict) -> ToolResult:
             if cats:
                 header += f" + {len(cats)} related topics"
             memu_block = f"{header}:\n\n{body}"
+    except MemoryBackendUnavailable as e:
+        logger.warning("Memory recall: backend unavailable: %s", e)
+        memu_block = (
+            "⚠️ MEMORY BACKEND DOWN (transient proxy/auth error) — recall is UNAVAILABLE right now, "
+            "NOT empty. Do NOT conclude 'no relevant memories'. Reconstruct context from other "
+            "sources (git history, your own notes) and alert the operator that memory is down."
+        )
     except Exception as e:
         logger.error("Memory recall failed: %s", e)
         memu_block = f"Memory recall error: {e}"
@@ -394,6 +402,7 @@ async def memorize_handler(ctx: ToolContext, args: dict) -> ToolResult:
     # Primary write: memU (file + extraction pipeline).
     memu_ok = False
     memu_err: str | None = None
+    memu_down = False  # transient backend outage (distinct from a genuine failure)
     try:
         mem_dir = Path("~/.nerve/memu-manual").expanduser()
         mem_path = mem_dir / f"memorize-{int(time.time())}.txt"
@@ -405,6 +414,10 @@ def _write_memorize_file() -> None:
         await asyncio.to_thread(_write_memorize_file)
 
         memu_ok = await ctx.memory_bridge.memorize_file(str(mem_path), modality="document")
+    except MemoryBackendUnavailable as e:
+        logger.warning("Memorize (memU): backend unavailable: %s", e)
+        memu_err = str(e)
+        memu_down = True
     except Exception as e:
         logger.error("Memorize (memU) failed: %s", e)
         memu_err = str(e)
@@ -420,6 +433,12 @@ def _write_memorize_file() -> None:
     if memu_ok:
         suffix = " (+ xmemory)" if xmem_written else ""
         return ToolResult.text(f"Memorized: {content}{suffix}")
+    if memu_down:
+        return ToolResult.text(
+            "⚠️ NOT saved — MEMORY BACKEND DOWN (transient proxy/auth error). "
+            "The fact was NOT stored; retry shortly. If it persists, `nerve restart` "
+            f"clears a stuck proxy cooldown. ({memu_err})"
+        )
     if memu_err is not None:
         return ToolResult.text(f"Error: {memu_err}")
     return ToolResult.text("Failed to memorize.")

nerve/memory/memu_bridge.py

@@ -33,6 +33,43 @@
 
 logger = logging.getLogger(__name__)
 
+
+class MemoryBackendUnavailable(RuntimeError):
+    """Raised when a memU operation fails because the LLM backend (the shared
+    proxy route) is transiently unavailable — HTTP 429/5xx or an auth blip
+    (``auth_unavailable`` / revoked OAuth) — rather than because the query
+    genuinely had no results. The tool layer surfaces this LOUDLY so a session
+    never mistakes "memory is DOWN" for "no relevant memories"."""
+
+
+def _is_transient_llm_error(exc: BaseException) -> bool:
+    """True if ``exc`` looks like a transient proxy/auth failure — worth
+    retrying, and worth surfacing as backend-down — vs a permanent/logic error.
+
+    Detects the OpenAI-SDK ``APIStatusError`` family by HTTP status (429 / 5xx)
+    plus the proxy's ``auth_unavailable`` / revoked-OAuth signatures on 401/403.
+    Status is read from ``status_code`` (or ``response.status_code``), falling
+    back to parsing ``"Error code: NNN"`` from the message so this stays
+    decoupled from the ``openai`` import."""
+    msg = str(exc)
+    status = getattr(exc, "status_code", None)
+    if status is None:
+        status = getattr(getattr(exc, "response", None), "status_code", None)
+    if status is None and "Error code: " in msg:
+        try:
+            status = int(msg.split("Error code: ", 1)[1].strip()[:3])
+        except (ValueError, IndexError):
+            status = None
+    if status in (429, 500, 502, 503, 504):
+        return True
+    low = msg.lower()
+    if status in (401, 403) and ("auth_unavailable" in low or "revoked" in low):
+        return True
+    # Belt-and-suspenders: the proxy's stuck-cooldown signature regardless of
+    # how the status surfaced.
+    return "auth_unavailable" in low
+
+
 # Semantic dedup threshold — set from config at init time.
 _SEMANTIC_DEDUP_THRESHOLD = 0.85
 
@@ -1916,38 +1953,60 @@ async def _timeout_chat(
                     # omits it.  Default to 4096 to prevent 400 errors.
                     if max_tokens is None:
                         max_tokens = 4096
-                    t0 = time.monotonic()
-                    try:
-                        return await asyncio.wait_for(
-                            _orig(
-                                prompt,
-                                max_tokens=max_tokens,
-                                system_prompt=system_prompt,
-                                temperature=temperature,
-                            ),
-                            timeout=self._LLM_CALL_TIMEOUT,
-                        )
-                    except asyncio.TimeoutError:
-                        elapsed = time.monotonic() - t0
-                        in_flight = len(self._metrics.in_flight)
-                        # Dump httpx connection pool state for diagnosis
-                        pool_info = "unknown"
+                    # memU otherwise fails fast with SDK retries disabled
+                    # (see max_retries=0 above), so a transient proxy/auth blip
+                    # (503/429/auth_unavailable) that the agent SDK rides out via
+                    # its own retries takes memory down instead. Retry a few
+                    # times with short backoff on transient errors only; keep the
+                    # per-attempt wait_for so genuine hangs still fail fast.
+                    _RETRY_BACKOFFS = (0.5, 1.5, 3.0)
+                    for _attempt in range(len(_RETRY_BACKOFFS) + 1):
+                        t0 = time.monotonic()
                         try:
-                            base = self._service._llm_clients.get(_prof)
-                            sdk = getattr(base, "client", None)
-                            transport = getattr(sdk, "_client", None)
-                            pool = getattr(transport, "_pool", None) or getattr(transport, "_transport", None)
-                            if pool:
-                                pool_info = repr(pool)
-                        except Exception:
-                            pass
-                        logger.error(
-                            "memU LLM HUNG [%s]: no response after %.0fs "
-                            "(prompt=%d chars, in_flight=%d, pool=%s)",
-                            _prof, elapsed, len(prompt),
-                            in_flight, pool_info,
-                        )
-                        raise
+                            return await asyncio.wait_for(
+                                _orig(
+                                    prompt,
+                                    max_tokens=max_tokens,
+                                    system_prompt=system_prompt,
+                                    temperature=temperature,
+                                ),
+                                timeout=self._LLM_CALL_TIMEOUT,
+                            )
+                        except asyncio.TimeoutError:
+                            elapsed = time.monotonic() - t0
+                            in_flight = len(self._metrics.in_flight)
+                            # Dump httpx connection pool state for diagnosis
+                            pool_info = "unknown"
+                            try:
+                                base = self._service._llm_clients.get(_prof)
+                                sdk = getattr(base, "client", None)
+                                transport = getattr(sdk, "_client", None)
+                                pool = getattr(transport, "_pool", None) or getattr(transport, "_transport", None)
+                                if pool:
+                                    pool_info = repr(pool)
+                            except Exception:
+                                pass
+                            logger.error(
+                                "memU LLM HUNG [%s]: no response after %.0fs "
+                                "(prompt=%d chars, in_flight=%d, pool=%s)",
+                                _prof, elapsed, len(prompt),
+                                in_flight, pool_info,
+                            )
+                            raise
+                        except Exception as _e:
+                            # Transient proxy/auth blip → retry with backoff.
+                            # Anything else (or the final transient failure) is
+                            # re-raised so the recall/memorize layer can surface
+                            # it loudly as MemoryBackendUnavailable.
+                            if _is_transient_llm_error(_e) and _attempt < len(_RETRY_BACKOFFS):
+                                _delay = _RETRY_BACKOFFS[_attempt]
+                                logger.warning(
+                                    "memU LLM transient error [%s] attempt %d/%d: %s — retrying in %.1fs",
+                                    _prof, _attempt + 1, len(_RETRY_BACKOFFS) + 1, _e, _delay,
+                                )
+                                await asyncio.sleep(_delay)
+                                continue
+                            raise
 
                 _timeout_chat._nerve_timeout_wrapped = True  # type: ignore[attr-defined]
                 client.chat = _timeout_chat  # type: ignore[method-assign]
@@ -2212,6 +2271,11 @@ async def memorize_file(self, file_path: str, modality: str = "document", source
                     last_error = str(e)
                     logger.error("memU memorize_file failed %sfor %s: %s", label + " " if label else "", file_path, e)
                     self._metrics.end_op(op_id, success=False, error=last_error)
+                    # Distinguish a transient backend outage from a genuine
+                    # failure so the tool layer can tell the agent memory is
+                    # DOWN (not that the write silently "didn't happen").
+                    if _is_transient_llm_error(e):
+                        raise MemoryBackendUnavailable(f"memory backend unavailable: {e}") from e
                     return False
 
             attempt += 1
@@ -2651,6 +2715,11 @@ async def recall(
         except Exception as e:
             logger.error("memU recall failed: %s", e)
             self._metrics.end_op(op_id, success=False, error=str(e))
+            # A transient backend outage must not masquerade as "no results" —
+            # surface it so the caller can flag amnesia instead of trusting the
+            # empty list.
+            if _is_transient_llm_error(e):
+                raise MemoryBackendUnavailable(f"memory backend unavailable: {e}") from e
             return []
 
     async def expand_category(