diff --git a/nerve/agent/tools/handlers/memory.py b/nerve/agent/tools/handlers/memory.py index d212cd5..ee5478f 100644 --- a/nerve/agent/tools/handlers/memory.py +++ b/nerve/agent/tools/handlers/memory.py @@ -26,6 +26,7 @@ MEMORY_UPDATE_SCHEMA, SESSION_CONTEXT_SCHEMA, ) +from nerve.memory.memu_bridge import MemoryBackendUnavailable logger = logging.getLogger(__name__) @@ -102,6 +103,13 @@ async def memory_recall_handler(ctx: ToolContext, args: dict) -> ToolResult: if cats: header += f" + {len(cats)} related topics" memu_block = f"{header}:\n\n{body}" + except MemoryBackendUnavailable as e: + logger.warning("Memory recall: backend unavailable: %s", e) + memu_block = ( + "⚠️ MEMORY BACKEND DOWN (transient proxy/auth error) — recall is UNAVAILABLE right now, " + "NOT empty. Do NOT conclude 'no relevant memories'. Reconstruct context from other " + "sources (git history, your own notes) and alert the operator that memory is down." + ) except Exception as e: logger.error("Memory recall failed: %s", e) memu_block = f"Memory recall error: {e}" @@ -394,6 +402,7 @@ async def memorize_handler(ctx: ToolContext, args: dict) -> ToolResult: # Primary write: memU (file + extraction pipeline). memu_ok = False memu_err: str | None = None + memu_down = False # transient backend outage (distinct from a genuine failure) try: mem_dir = Path("~/.nerve/memu-manual").expanduser() mem_path = mem_dir / f"memorize-{int(time.time())}.txt" @@ -405,6 +414,10 @@ def _write_memorize_file() -> None: await asyncio.to_thread(_write_memorize_file) memu_ok = await ctx.memory_bridge.memorize_file(str(mem_path), modality="document") + except MemoryBackendUnavailable as e: + logger.warning("Memorize (memU): backend unavailable: %s", e) + memu_err = str(e) + memu_down = True except Exception as e: logger.error("Memorize (memU) failed: %s", e) memu_err = str(e) @@ -420,6 +433,12 @@ def _write_memorize_file() -> None: if memu_ok: suffix = " (+ xmemory)" if xmem_written else "" return ToolResult.text(f"Memorized: {content}{suffix}") + if memu_down: + return ToolResult.text( + "⚠️ NOT saved — MEMORY BACKEND DOWN (transient proxy/auth error). " + "The fact was NOT stored; retry shortly. If it persists, `nerve restart` " + f"clears a stuck proxy cooldown. ({memu_err})" + ) if memu_err is not None: return ToolResult.text(f"Error: {memu_err}") return ToolResult.text("Failed to memorize.") diff --git a/nerve/memory/memu_bridge.py b/nerve/memory/memu_bridge.py index 7a3fd78..9754b65 100644 --- a/nerve/memory/memu_bridge.py +++ b/nerve/memory/memu_bridge.py @@ -33,6 +33,43 @@ logger = logging.getLogger(__name__) + +class MemoryBackendUnavailable(RuntimeError): + """Raised when a memU operation fails because the LLM backend (the shared + proxy route) is transiently unavailable — HTTP 429/5xx or an auth blip + (``auth_unavailable`` / revoked OAuth) — rather than because the query + genuinely had no results. The tool layer surfaces this LOUDLY so a session + never mistakes "memory is DOWN" for "no relevant memories".""" + + +def _is_transient_llm_error(exc: BaseException) -> bool: + """True if ``exc`` looks like a transient proxy/auth failure — worth + retrying, and worth surfacing as backend-down — vs a permanent/logic error. + + Detects the OpenAI-SDK ``APIStatusError`` family by HTTP status (429 / 5xx) + plus the proxy's ``auth_unavailable`` / revoked-OAuth signatures on 401/403. + Status is read from ``status_code`` (or ``response.status_code``), falling + back to parsing ``"Error code: NNN"`` from the message so this stays + decoupled from the ``openai`` import.""" + msg = str(exc) + status = getattr(exc, "status_code", None) + if status is None: + status = getattr(getattr(exc, "response", None), "status_code", None) + if status is None and "Error code: " in msg: + try: + status = int(msg.split("Error code: ", 1)[1].strip()[:3]) + except (ValueError, IndexError): + status = None + if status in (429, 500, 502, 503, 504): + return True + low = msg.lower() + if status in (401, 403) and ("auth_unavailable" in low or "revoked" in low): + return True + # Belt-and-suspenders: the proxy's stuck-cooldown signature regardless of + # how the status surfaced. + return "auth_unavailable" in low + + # Semantic dedup threshold — set from config at init time. _SEMANTIC_DEDUP_THRESHOLD = 0.85 @@ -1916,38 +1953,60 @@ async def _timeout_chat( # omits it. Default to 4096 to prevent 400 errors. if max_tokens is None: max_tokens = 4096 - t0 = time.monotonic() - try: - return await asyncio.wait_for( - _orig( - prompt, - max_tokens=max_tokens, - system_prompt=system_prompt, - temperature=temperature, - ), - timeout=self._LLM_CALL_TIMEOUT, - ) - except asyncio.TimeoutError: - elapsed = time.monotonic() - t0 - in_flight = len(self._metrics.in_flight) - # Dump httpx connection pool state for diagnosis - pool_info = "unknown" + # memU otherwise fails fast with SDK retries disabled + # (see max_retries=0 above), so a transient proxy/auth blip + # (503/429/auth_unavailable) that the agent SDK rides out via + # its own retries takes memory down instead. Retry a few + # times with short backoff on transient errors only; keep the + # per-attempt wait_for so genuine hangs still fail fast. + _RETRY_BACKOFFS = (0.5, 1.5, 3.0) + for _attempt in range(len(_RETRY_BACKOFFS) + 1): + t0 = time.monotonic() try: - base = self._service._llm_clients.get(_prof) - sdk = getattr(base, "client", None) - transport = getattr(sdk, "_client", None) - pool = getattr(transport, "_pool", None) or getattr(transport, "_transport", None) - if pool: - pool_info = repr(pool) - except Exception: - pass - logger.error( - "memU LLM HUNG [%s]: no response after %.0fs " - "(prompt=%d chars, in_flight=%d, pool=%s)", - _prof, elapsed, len(prompt), - in_flight, pool_info, - ) - raise + return await asyncio.wait_for( + _orig( + prompt, + max_tokens=max_tokens, + system_prompt=system_prompt, + temperature=temperature, + ), + timeout=self._LLM_CALL_TIMEOUT, + ) + except asyncio.TimeoutError: + elapsed = time.monotonic() - t0 + in_flight = len(self._metrics.in_flight) + # Dump httpx connection pool state for diagnosis + pool_info = "unknown" + try: + base = self._service._llm_clients.get(_prof) + sdk = getattr(base, "client", None) + transport = getattr(sdk, "_client", None) + pool = getattr(transport, "_pool", None) or getattr(transport, "_transport", None) + if pool: + pool_info = repr(pool) + except Exception: + pass + logger.error( + "memU LLM HUNG [%s]: no response after %.0fs " + "(prompt=%d chars, in_flight=%d, pool=%s)", + _prof, elapsed, len(prompt), + in_flight, pool_info, + ) + raise + except Exception as _e: + # Transient proxy/auth blip → retry with backoff. + # Anything else (or the final transient failure) is + # re-raised so the recall/memorize layer can surface + # it loudly as MemoryBackendUnavailable. + if _is_transient_llm_error(_e) and _attempt < len(_RETRY_BACKOFFS): + _delay = _RETRY_BACKOFFS[_attempt] + logger.warning( + "memU LLM transient error [%s] attempt %d/%d: %s — retrying in %.1fs", + _prof, _attempt + 1, len(_RETRY_BACKOFFS) + 1, _e, _delay, + ) + await asyncio.sleep(_delay) + continue + raise _timeout_chat._nerve_timeout_wrapped = True # type: ignore[attr-defined] client.chat = _timeout_chat # type: ignore[method-assign] @@ -2212,6 +2271,11 @@ async def memorize_file(self, file_path: str, modality: str = "document", source last_error = str(e) logger.error("memU memorize_file failed %sfor %s: %s", label + " " if label else "", file_path, e) self._metrics.end_op(op_id, success=False, error=last_error) + # Distinguish a transient backend outage from a genuine + # failure so the tool layer can tell the agent memory is + # DOWN (not that the write silently "didn't happen"). + if _is_transient_llm_error(e): + raise MemoryBackendUnavailable(f"memory backend unavailable: {e}") from e return False attempt += 1 @@ -2651,6 +2715,11 @@ async def recall( except Exception as e: logger.error("memU recall failed: %s", e) self._metrics.end_op(op_id, success=False, error=str(e)) + # A transient backend outage must not masquerade as "no results" — + # surface it so the caller can flag amnesia instead of trusting the + # empty list. + if _is_transient_llm_error(e): + raise MemoryBackendUnavailable(f"memory backend unavailable: {e}") from e return [] async def expand_category(