From 0bc7e3bb632e9fa58ee9aea5ba928add748e20db Mon Sep 17 00:00:00 2001 From: "juwei.yue" Date: Thu, 27 Aug 2026 09:52:58 +0000 Subject: [PATCH 1/8] feat(search): make the multi-round decider tunable and its fallback visible Twelve retrieval arms were measured against a decider that answered 404 on every call. The loop caught the error, fell back to a fixed top-3 core with a single round, and returned HTTP 200 with a full episode list -- so the runs reported plausible accuracies and exited zero. Nothing in the result said the decider had never run. Two causes, both fixed here: Twelve `EVEROS_LLMMR_*` module constants were read at import time, so the harness env that was supposed to configure them arrived too late and every run used the defaults. They are now fields on `DeciderSettings`, resolved per search through `_tuning()`, with the legacy env names still honoured. The fallback logged at debug and was indistinguishable from a normal result. It now logs at error with the underlying exception and calls `mark_degraded`, which surfaces on `SearchData.degraded` -- a caller can tell a degraded result from a healthy one without reading a trace file. Degradations reset per search and restore around it, so a long-lived process does not accumulate another request's reasons. --- src/everos/config/settings.py | 160 ++- src/everos/core/context/__init__.py | 8 + src/everos/core/context/degradation.py | 60 + src/everos/memory/search/dto.py | 29 + src/everos/memory/search/llm_multiround.py | 1105 +++++++++++++++++ src/everos/memory/search/manager.py | 82 +- src/everos/memory/search/recall/profile.py | 89 +- .../test_decider_tuning_and_degradation.py | 189 +++ .../test_search/test_llm_multiround.py | 566 +++++++++ ... test_extract_user_profile_single_path.py} | 0 10 files changed, 2261 insertions(+), 27 deletions(-) create mode 100644 src/everos/core/context/degradation.py create mode 100644 src/everos/memory/search/llm_multiround.py create mode 100644 tests/unit/test_memory/test_search/test_decider_tuning_and_degradation.py create mode 100644 tests/unit/test_memory/test_search/test_llm_multiround.py rename tests/unit/test_memory/test_strategies/{test_extract_user_profile_dual_trigger.py => test_extract_user_profile_single_path.py} (100%) diff --git a/src/everos/config/settings.py b/src/everos/config/settings.py index 50da9947d..317d8163b 100644 --- a/src/everos/config/settings.py +++ b/src/everos/config/settings.py @@ -25,7 +25,7 @@ import os from functools import cache from pathlib import Path -from typing import Literal +from typing import Any, Literal from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from pydantic import BaseModel, ConfigDict, Field, SecretStr, field_validator @@ -111,6 +111,35 @@ class SqliteSettings(BaseModel): foreign_keys: bool = True temp_store: Literal["DEFAULT", "FILE", "MEMORY"] = "MEMORY" busy_timeout_ms: int = Field(default=5000, ge=0) + pool_size: int = Field(default=5, ge=1) + """Connections the pool keeps open. SQLAlchemy's own default; named here so + the next two are tunable alongside it rather than inherited invisibly.""" + max_overflow: int = Field(default=10, ge=0) + """Extra connections allowed above ``pool_size`` under load.""" + pool_timeout_seconds: float = Field(default=30.0, gt=0) + """How long a caller waits for a free connection before raising. + + The reason this is configurable rather than left at the library default: a + checkout that never returns turns every later caller into a silent hang. Two + benchmark servers died exactly that way -- aiosqlite connection threads grew + from a steady 6-10 to 20 and 58, with 7 and 22 of them parked inside + ``aiosqlite``'s connect path, and every SQLite file stopped being written + (one froze for 3h33m, the other 2h17m). Neither process was dead: HTTP still + answered, the event loop still ran, and the OME queue simply stopped + draining because no strategy could persist its own result. A bounded wait + converts that into a loud, retryable error instead. + """ + pool_recycle_seconds: int = Field(default=1800, ge=-1) + """Discard and reopen a connection older than this; ``-1`` disables. + + A leaked-but-idle connection is reclaimed on its next checkout attempt + instead of being held for the life of the process.""" + pool_pre_ping: bool = True + """Verify a pooled connection is alive before handing it out. + + Costs one round trip per checkout against a local file; buys detection of + connections whose underlying aiosqlite thread is no longer serviceable -- + the state both stalled servers were in.""" journal_size_limit_bytes: int = Field(default=64 * 1024 * 1024, ge=0) cache_size_kb: int = Field(default=2048, ge=0) @@ -127,11 +156,139 @@ class LLMSettings(BaseModel): EVEROS_LLM__MODEL EVEROS_LLM__API_KEY EVEROS_LLM__BASE_URL + EVEROS_LLM__TIMEOUT_SECONDS + EVEROS_LLM__EXTRA """ model: str = "gpt-4.1-mini" api_key: SecretStr | None = None base_url: str | None = None + timeout_seconds: float = Field(default=60.0, gt=0) + """Per-request deadline handed to the algo client. + + Matches the algo default, so leaving it alone changes nothing. It exists + because extraction latency is a property of the endpoint, not of this + project: a hosted frontier model answers an extraction prompt in seconds, + while a self-hosted mid-size model on a shared gateway can take minutes for + the same prompt. Without this the deadline was unreachable from config, and + a slow endpoint could only fail -- three attempts, three timeouts, one + dead-lettered memory. + """ + extra: dict[str, Any] = Field(default_factory=dict) + """Provider-specific request fields merged into every chat call. + + An escape hatch for what the OpenAI protocol does not cover, kept generic + because each gateway spells its own knobs differently. The case it was added + for: a reasoning model served over an OpenAI-compatible endpoint keeps + thinking by default, and thinking is billed against ``max_tokens`` -- so an + extraction call can spend its entire budget reasoning and return an empty + ``content``, which reads downstream as "the model found no facts" rather + than as a misconfiguration. + + **Non-standard fields must be nested under ``extra_body``.** These entries + are passed as keyword arguments to the OpenAI SDK's ``create()``, which + raises ``TypeError`` on an unrecognised top-level name and forwards only + ``extra_body`` into the request JSON. Getting this wrong fails loudly at the + first call, which is the good case; getting it *absent* is the quiet one:: + + EVEROS_LLM__EXTRA='{"extra_body": + {"chat_template_kwargs": {"enable_thinking": false}}}' + + Measured on one gateway with an atomic-facts prompt: nested = 12.7s and 2264 + characters, bare = ``TypeError``, omitted = 42.4s and **zero** characters. + + Merged under the per-call ``extra``, so a caller can still override a key. + """ + + +class DeciderSettings(BaseModel): + """LLM driving the multi-round retrieval decider. + + A flat section mirroring ``[llm]``, kept separate for the same reason + ``[multimodal]`` is: the two jobs have different demands. ``[llm]`` extracts + memories during ingestion -- a long, throughput-bound batch job -- while this one + runs inside the search request and decides, round by round, which episodes are + core and what to query next. Sharing one setting forced the decider to be whatever + model the store happened to be extracted with, and made "which model made the + retrieval decisions" unanswerable after the fact. + + Empty ``model`` falls back to ``[llm]``, so existing deployments and every store + built before this section existed behave exactly as they did. + + The loop-tuning fields below used to exist only as ``EVEROS_LLMMR_*`` environment + variables read once at import time inside + :mod:`everos.memory.search.llm_multiround`. That made them undiscoverable -- nothing + in the config named them, so an operator could not find out they existed, let + alone set one from a file. They are declared here for the same reason the model + is: a run's behaviour should be readable off its configuration. The legacy env + names still win when set, so nothing in flight changes. + + Env binding (via parent ``Settings``): + EVEROS_DECIDER__MODEL + EVEROS_DECIDER__API_KEY + EVEROS_DECIDER__BASE_URL + EVEROS_DECIDER__TIMEOUT_SECONDS + EVEROS_DECIDER__EXTRA + EVEROS_DECIDER__MAX_ROUNDS ... and one per field below + """ + + model: str = "" + api_key: SecretStr | None = None + base_url: str | None = None + timeout_seconds: float = Field(default=60.0, gt=0) + """Per-request deadline for one decider round. See + :attr:`LLMSettings.timeout_seconds`; separate because this one sits inside a + search request, where the acceptable wait is bounded by the caller rather + than by a background queue.""" + extra: dict[str, Any] = Field(default_factory=dict) + """Provider-specific request fields for the decider. See + :attr:`LLMSettings.extra`; kept separate so the decider and the extractor can + run on gateways with different vocabularies.""" + + # ── multi-round loop tuning ────────────────────────────────────────── + max_rounds: int = Field(default=3, ge=1) + """Hard cap on retrieval rounds. The cost bound: each round is one decider call + plus one recall per sub-query.""" + seed_topk: int = Field(default=50, ge=1) + """Round-0 block size -- how many candidates the original question contributes.""" + subq_topk: int = Field(default=20, ge=1) + """Per-sub-query block size on round >= 1. Narrower than the seed on purpose: + breadth comes from having several blocks, not from each being large.""" + max_subqueries: int = Field(default=3, ge=1) + """Breadth cap -- gap-covering sub-queries the decider may issue per round.""" + rrf_k: int = Field(default=60, ge=1) + """RRF smoothing constant for fusing each sub-query's sparse and dense lists.""" + no_new_core_patience: int = Field(default=1, ge=0) + """Stop after this many consecutive rounds add no new core (after >= 1 round).""" + per_subquery_guarantee: int = Field(default=1, ge=0) + """Final-injection backfill: guarantee this many top non-core candidates per + sub-query a slot, so every facet keeps coverage instead of being crowded out by + one strong sub-query.""" + retries: int = Field(default=3, ge=0) + """Retries on a transient decider failure before falling back to a fixed core.""" + retry_backoff_seconds: float = Field(default=0.5, ge=0) + """Base delay between decider retries, doubled per attempt. Retrying a reasoning + decider back-to-back tends to reproduce the same empty completion, so the pause is + what makes the attempts meaningfully independent.""" + core_overflow: bool = False + """Let the core-first stage exceed ``top_k`` (the pre-2026-08-06 behaviour). + Measured at 316/1522 questions (20.8%) returning more items than asked for on + SubtleMemory, up to 68 for ``top_k=20`` -- which breaks the ``top_k`` contract and + invalidates any same-budget comparison. Set only to reproduce a pre-fix run.""" + full_text: bool = False + """Show the decider the full episode text instead of the stored summary. + + Stores disagree about which column holds the full text: some hold a 200-character + prefix in ``summary`` with the full text in ``episode``. On those, the decider picks + core from a preview roughly 7x shorter than what the answering model is shown, which + measurably costs core recall. Default off so an in-flight comparison cannot change + behaviour mid-run.""" + fallback_core: int = Field(default=3, ge=0) + """Core size to fall back to when every decider attempt fails. + + Not 0: an empty core silently disables core-first injection -- the very mechanism + under test -- and is indistinguishable in the output from a decider that chose + nothing on purpose.""" class MultimodalSettings(BaseModel): @@ -449,6 +606,7 @@ class Settings(BaseSettings): sqlite: SqliteSettings = SqliteSettings() lancedb: LanceDBSettings = LanceDBSettings() llm: LLMSettings = LLMSettings() + decider: DeciderSettings = DeciderSettings() embedding: EmbeddingSettings = EmbeddingSettings() rerank: RerankSettings = RerankSettings() boundary_detection: BoundaryDetectionSettings = BoundaryDetectionSettings() diff --git a/src/everos/core/context/__init__.py b/src/everos/core/context/__init__.py index 3070c2d49..abb0d061d 100644 --- a/src/everos/core/context/__init__.py +++ b/src/everos/core/context/__init__.py @@ -9,14 +9,22 @@ ) """ +from .degradation import get_degradations as get_degradations +from .degradation import mark_degraded as mark_degraded +from .degradation import reset_degradations as reset_degradations +from .degradation import restore_degradations as restore_degradations from .request import get_request_id as get_request_id from .request import reset_request_id as reset_request_id from .request import resolve_request_id as resolve_request_id from .request import set_request_id as set_request_id __all__ = [ + "get_degradations", "get_request_id", + "mark_degraded", + "reset_degradations", "reset_request_id", "resolve_request_id", + "restore_degradations", "set_request_id", ] diff --git a/src/everos/core/context/degradation.py b/src/everos/core/context/degradation.py new file mode 100644 index 000000000..4ae0f1d47 --- /dev/null +++ b/src/everos/core/context/degradation.py @@ -0,0 +1,60 @@ +"""Request-scoped record of a result that was produced by a degraded path. + +Some failures inside a search do not stop it. The clearest case: when every +multi-round decider attempt fails, the loop falls back to a fixed top-N core and +stops after one round rather than returning nothing -- which is the right call for +availability, and invisible from outside. A caller receives HTTP 200, a full +``episodes`` list, and no indication that the component under test never ran. + +That invisibility has a measured cost. On 2026-08-25 a twelve-arm retrieval sweep +sent a model name to an endpoint that did not serve it. Every decider call returned +404, every round fell back, and all twelve arms reported plausible accuracies between +87% and 93% -- the same degraded path twelve times. Nothing in any response said so; +it took reading the traces to find out, and the numbers had already been written up. + +So a degraded result says so. Collected through a ``ContextVar`` rather than threaded +through call signatures for the same reason the request id is: the flag originates deep +in the retrieval loop and is needed at the response boundary, and every layer between +has no interest in it. +""" + +from __future__ import annotations + +from contextvars import ContextVar, Token + +_degradations: ContextVar[tuple[str, ...]] = ContextVar( + "everos_degradations", default=() +) + + +def mark_degraded(reason: str) -> None: + """Record that this request's result came from a degraded path. + + Repeats are collapsed: a per-question loop hits the same fallback on every round, + and the response should say *what* degraded, not how many times. + """ + reason = (reason or "").strip() + if not reason: + return + current = _degradations.get() + if reason not in current: + _degradations.set((*current, reason)) + + +def get_degradations() -> tuple[str, ...]: + """Reasons recorded for the current request, in the order first seen.""" + return _degradations.get() + + +def reset_degradations() -> Token[tuple[str, ...]]: + """Clear the record and return a token for restoring it. + + Called at the start of a request. Without it a long-lived task's context would + accumulate reasons across requests and mark healthy results degraded. + """ + return _degradations.set(()) + + +def restore_degradations(token: Token[tuple[str, ...]]) -> None: + """Restore whatever was recorded before the matching reset.""" + _degradations.reset(token) diff --git a/src/everos/memory/search/dto.py b/src/everos/memory/search/dto.py index 53d37fa5b..26e8d8764 100644 --- a/src/everos/memory/search/dto.py +++ b/src/everos/memory/search/dto.py @@ -35,6 +35,19 @@ class SearchMethod(StrEnum): VECTOR = "vector" HYBRID = "hybrid" AGENTIC = "agentic" + LLM_MULTIROUND = "llm_multiround" + """LLM-guided iterative multi-round episode retrieval (per-sub-query RRF blocks). + + Each round fuses every current sub-query's BM25+vector recall with RRF + INDEPENDENTLY (round 0 = one original-question block), shows an injected + decider the labelled blocks plus the core accumulated so far, and lets it + pin core and issue gap-covering sub-queries. The final injection is + core-first + each sub-query's top-1 guarantee + max-RRF-score fill; there is + no cross-encoder anywhere. Unlike AGENTIC's fixed round1+round2 this is + genuinely iterative, and the decider is pluggable -- a prompted LLM by + default, a trained policy in Phase 2, which makes this loop the RL + environment. Needs an LLM + embedding provider. User memory only. + """ class FilterNode(BaseModel): @@ -89,6 +102,12 @@ class SearchRequest(BaseModel): Only the episode hybrid path consumes it — other methods ignore it. """ include_profile: bool = False + profile_subject: str | None = Field(default=None, min_length=1) + """Which participant's profile to return, for a **group** owner that + holds one profile per person. ``None`` returns every profile under the + owner (capped). Ignored unless ``include_profile`` is set, and irrelevant + for the common case of an owner who is one person -- that owner has a + single profile keyed on ``owner_id`` itself.""" enable_llm_rerank: bool = Field( default=False, description=( @@ -275,6 +294,16 @@ class SearchData(BaseModel): """In-flight messages still in the boundary-detection buffer for the ``filters.session_id`` (if supplied as a top-level eq scalar); otherwise stays empty.""" + degraded: list[str] = Field(default_factory=list) + """Non-empty when this result came from a fallback rather than the normal path. + + Empty on a healthy search, so a client that ignores it sees no change. It exists + because the alternative is indistinguishable: when every multi-round decider attempt + fails the loop returns a fixed top-N core with HTTP 200 and a full episode list, + and an evaluation harness reading only the episodes cannot tell that the component + it was measuring never ran. Twelve arms of one sweep reported 87-93% that way. + + Values are stable identifiers, not messages: ``decider_fallback`` today.""" class SearchResponse(BaseModel): diff --git a/src/everos/memory/search/llm_multiround.py b/src/everos/memory/search/llm_multiround.py new file mode 100644 index 000000000..81e445718 --- /dev/null +++ b/src/everos/memory/search/llm_multiround.py @@ -0,0 +1,1105 @@ +"""Episode LLM-guided multi-round retrieval (per-sub-query RRF blocks). + +Genuinely *iterative* multi-round retrieval, unlike ``agentic`` (a fixed round1 ++ single round2 fallback). Each round an injected **decider** looks at the +question + retrieved evidence and, in one shot: + + 1. **selects** the *core* memories that carry information the answer needs, + 2. decides to **stop** (the core answers a self-contained question), else + 3. **expands** into one focused sub-query per still-missing aspect (multi-query + gap coverage), grounded in the core carried forward. + +Retrieval substrate — the design's defining choice. Every current sub-query is +recalled (BM25 + vector) and fused with RRF **independently**, so a facet-gold +ranked #1 for a single sub-query is never diluted by the other sub-queries' +consensus hits (the failure mode of merging every sub-query into one pool). The +decider sees one labelled block per sub-query (round 0 = a single +original-question block) with a single GLOBAL index, plus a CORE-SO-FAR section +tagging each kept item with the sub-query that surfaced it. There is **no +cross-encoder** anywhere — RRF is the only ranking (the CE was measured +net-negative: it scored gold below RRF order and buried much of it past rank 20). + +The accumulated core also shapes the output: the final top-k is assembled +**core-first** (a hard guarantee), then each sub-query's top non-core candidate +is guaranteed a slot, then the rest is filled by MAX RRF score across sub-queries +(max, not sum — a specialist that is #1 for one sub-query keeps that score). +Evidence selection therefore genuinely shapes what is injected — the lever a +Phase-2 RL policy learns to control. + +The decider is a *pluggable hook* — the whole point of the file: + +* Phase 1 (default here): :class:`LLMRoundDecider` prompts an off-the-shelf + LLM (no training). +* Phase 2 (RL): swap in a trained small policy with the same + :class:`RoundDecider` interface. **This loop then IS the RL environment** — + only the decider changes; the retrieval substrate is identical. + +Retrieval explainability: setting ``EVEROS_LLMMR_TRACE_DUMP=`` appends one +per-round record (blocked evidence shown -> core selected -> stop / next +queries) plus a final-injection record (the assembled top-k with per-slot +provenance), so a real Phase-1 run yields the GRPO cold-start (behavior-cloning) +corpus. Unset by default — the dump adds nothing to the production path. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import re +import time +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Any, NamedTuple, Protocol + +from everalgo.llm.types import ChatMessage as LLMChatMessage +from everalgo.rank.fusion import rrf +from everalgo.types import Candidate + +from everos.config.settings import DeciderSettings, load_settings +from everos.core.context import mark_degraded +from everos.core.observability.logging import get_logger + +from .dto import SearchEpisodeItem +from .shaper import shape_episode_from_candidate + +logger = get_logger(__name__) + +if TYPE_CHECKING: + from everalgo.llm.protocols import LLMClient + + from everos.component.rerank import RerankProvider + from everos.memory.search.recall.atomic_fact import AtomicFactRecaller + from everos.memory.search.recall.episode import EpisodeRecaller + + +# ── Retrieval hyperparameters (per-sub-query RRF blocks) ────────────────────── +# Each round fuses every current sub-query's BM25+vector recall with RRF +# INDEPENDENTLY and shows the decider one labelled block per sub-query (round 0 = +# a single original-question block). There is NO cross-encoder anywhere — RRF is +# the only ranking (the CE was measured net-negative: it scored gold below RRF +# order and buried much of it past rank 20). The final injection is core-first + +# each sub-query's top non-core guaranteed + max-RRF-score fill (MAX across +# sub-queries, not sum — max keeps a specialist that is #1 for one sub-query). +_SEED_CANDIDATES: int = 50 +"""Sparse / dense episode recall pool size per sub-query per round.""" +# ── Loop tuning ────────────────────────────────────────────────────────────── +# Resolved per search from ``[decider]``, not frozen at import. +# +# These were twelve module-level ``int(os.getenv("EVEROS_LLMMR_...", ...))`` reads. Two +# problems with that, and the second is the one that matters: nothing in the +# configuration named them, so an operator had no way to discover they existed; and a +# value read at import time cannot respond to a config reload, which is why the test +# suite -- whose conftest resets the settings cache per test -- could not exercise them +# without monkeypatching module attributes. +# +# The legacy env names still take precedence when set. Deployments and launch scripts +# that export them keep working unchanged, and an in-flight comparison cannot shift +# because this landed. +_LEGACY_ENV: dict[str, str] = { + "max_rounds": "EVEROS_LLMMR_MAX_ROUNDS", + "seed_topk": "EVEROS_LLMMR_SEED_TOPK", + "subq_topk": "EVEROS_LLMMR_SUBQ_TOPK", + "max_subqueries": "EVEROS_LLMMR_MAX_SUBQUERIES", + "rrf_k": "EVEROS_LLMMR_RRF_K", + "no_new_core_patience": "EVEROS_LLMMR_PATIENCE", + "per_subquery_guarantee": "EVEROS_LLMMR_GUARANTEE", + "retries": "EVEROS_LLMMR_DECIDER_RETRIES", + "retry_backoff_seconds": "EVEROS_LLMMR_DECIDER_BACKOFF_S", + "core_overflow": "EVEROS_LLMMR_CORE_OVERFLOW", + "full_text": "EVEROS_LLMMR_DECIDER_FULL_TEXT", + "fallback_core": "EVEROS_LLMMR_DECIDER_FALLBACK_CORE", +} + +_SEED_CANDIDATES: int = 50 +"""Sparse / dense episode recall pool size per sub-query per round.""" + + +def _tuning() -> DeciderSettings: + """The decider's loop settings, with the legacy env vars applied on top. + + Returns a copy rather than the live settings object so a legacy override cannot + leak into anything else reading ``[decider]``. + """ + cfg = load_settings().decider + over: dict[str, Any] = {} + for field, env in _LEGACY_ENV.items(): + raw = os.getenv(env, "").strip() + if not raw: + continue + current = getattr(cfg, field) + try: + if isinstance(current, bool): + over[field] = raw == "1" + elif isinstance(current, int): + over[field] = int(raw) + else: + over[field] = float(raw) + except ValueError: + # A malformed override is worth saying out loud rather than silently + # falling back: the run would report a configuration it never used. + logger.warning("llm_multiround_bad_env_override", env=env, value=raw[:40]) + return cfg.model_copy(update=over) if over else cfg + + +_TRACE_DUMP_ENV: str = "EVEROS_LLMMR_TRACE_DUMP" +"""Env var naming the JSONL file that receives one per-round decision record. + +Unset / empty ⇒ tracing is off and the loop does zero extra work (production +default). Set to a path ⇒ each round appends a record whose schema mirrors +MemoryRL ``tasks/retrieval_policy/prepare_data.py`` (``load_phase1_traces`` / +``trace_to_sft_example``), so the trace feeds the Phase-2 GRPO cold-start +(behavior cloning) directly. This is a formal, first-class dump — read per call +(not at import) so a run or a test can toggle it via env without re-importing. + +The search layer only knows what a ``/search`` request carries (query + +owner), so it emits every field it owns and leaves ``question_id`` (``None`` +placeholder) plus the ``gold_session_ids`` / ``core_precision`` / ``core_recall`` +labels to a downstream labeling step that has the eval ground truth.""" + + +# ── Decider hook (pluggable: prompt-LLM in P1, trained policy in P2) ────────── + + +class RoundDecision(NamedTuple): + """One round's decision. + + Attributes: + stop: Evidence is sufficient — stop and answer. + queries: Next queries to issue (when not stopping). The strategy is + **multi-query gap coverage**, not a single narrow rewrite: the + decider emits one focused sub-query per still-missing aspect, so + multi-session / multi-hop questions fan out across threads instead + of collapsing onto one. Empty list ⇒ stop. + core: Indices (into the ``evidence`` list the decider was given) of the + *core* episodes that actually matter — used to shrink the carried + context and to ground the rewritten queries. + """ + + stop: bool + queries: list[str] + core: list[int] + usage: dict[str, object] | None = ( + None # decider LLM token usage (trace); None on failure + ) + raw: str | None = None # decider raw output before _parse_decision (trace) + failed: bool = False + """Every decider attempt failed; ``core`` is the deterministic fallback. + + Kept explicit so a degraded round is countable in the trace instead of + looking like a decider that chose to stop with nothing.""" + + +class RoundDecider(Protocol): + """Each round: select core from the blocked view, then stop or expand. + + The Phase-1 default is :class:`LLMRoundDecider`; a Phase-2 policy + implements this same signature so the loop becomes the RL environment — only + the decider changes. ``core_so_far`` and ``evidence`` are pre-rendered + strings (the accumulated core, and the per-sub-query candidate blocks with a + single global index); ``n_candidates`` is the number of globally-indexed + candidates the returned ``core`` indices address. + """ + + async def __call__( + self, + question: str, + core_so_far: str, + evidence: str, + n_candidates: int, + round_idx: int, + ) -> RoundDecision: ... + + +_DECIDER_PROMPT = """You steer a multi-round memory search. Reply with ONLY a \ +JSON object. +You are shown a RETRIEVED SUBSET of memory, never all of it. "Not shown here" \ +does NOT mean "not in memory" — it may just not have been retrieved yet. + +What you see each round: +- QUESTION: the user's original question (unchanged every round). +- CORE SO FAR: items you already selected in earlier rounds, each tagged with \ +the sub-query that surfaced it ("original question" on round 0). These are \ +ALREADY kept — never re-list them; use them only to judge what is still missing. +- CANDIDATES THIS ROUND: on round 0, ONE block retrieved for the original \ +question; on later rounds, ONE BLOCK PER SUB-QUERY, each headed by that \ +sub-query. Items carry a SINGLE global index across all blocks. The same memory \ +may appear in several blocks — that just means several angles retrieved it. + +Do two things: + +1. CORE (required): global indices of every candidate shown THIS ROUND that \ +could carry information the answer needs — favour RECALL over minimality. + - A needed fact (an age, price, date, name, count) is often an INCIDENTAL \ +detail inside an episode about a DIFFERENT topic. Judge by whether the item \ +touches the same PERSON / ENTITY / PLACE / ACTIVITY / TIME-WINDOW the question \ +is about — NOT by whether its headline topic matches. + - SCAN EVERY BLOCK top to bottom; answer-bearing evidence is often ranked \ +LOW in its block. + - For a count / total / comparison / ordering, core EVERY item that \ +contributes an instance — a multi-part answer usually needs several items from \ +different sessions. + - If the SAME memory appears in two blocks, core it ONCE (either index). + - Return an empty list ONLY when nothing shown touches the question at all. + +2. NEXT_QUERIES: KEEP SEARCHING (default) or STOP. + - STOP (set "next_queries" to []) ONLY when the question is a SINGLE, \ +self-contained fact AND the core you already hold answers it unambiguously. + - OTHERWISE keep searching: one focused sub-query per still-missing aspect \ +(up to {max_sub}). Each sub-query must NAME the specific entity / aspect no \ +current block has covered — never paraphrase a sub-query already asked, never \ +issue a broad topical query. + +Illustrative examples (generic, invented — not real data): + +Example 1 (round 0, single fact): +QUESTION: What breed is Nora's cat? +CORE SO FAR: (none) +CANDIDATES THIS ROUND: +[block: original question] + 0: Vet visit - Nora brought her cat Biscuit, a Ragdoll, in for shots. + 1: Weekend plans - Nora said she adopted a cat last spring. +Reply: {{"core": [0], "next_queries": []}} + +Example 2 (later round, running total across sub-queries): +QUESTION: How many marathons has Devin run in total? +CORE SO FAR: + - [from "Devin marathon 2021"] Devin finished the Lakeside Marathon in 2021. +CANDIDATES THIS ROUND: +[block: Devin marathon 2022 2023] + 0: Race recap - Devin ran the Harbor Marathon in 2022. + 1: Training log - Devin jogged 5 km most weekends. +[block: Devin other marathons] + 2: Trip photos - Devin flew to Berlin and ran its city marathon in 2023. + 3: Race recap - Devin ran the Harbor Marathon in 2022. +Reply: {{"core": [0, 2], "next_queries": ["Devin marathon before 2021"]}} + +QUESTION: +{question} + +CORE SO FAR: +{core_so_far} + +CANDIDATES THIS ROUND: +{evidence} + +Reply with ONLY this JSON object (no prose, no code fences); "core" values are \ +GLOBAL indices into the candidates above: +{{"core": [], "next_queries": ["", "..."]}}""" + + +class LLMRoundDecider: + """Prompt-LLM decider: select core across per-sub-query blocks by global index. + + What it is *shown* differs from what it returns. The evidence is pre-rendered + by the caller as one labelled block per sub-query (round 0 = a single + original-question block) with a single global + index across all blocks, plus a CORE-SO-FAR section listing already-selected + core tagged with the sub-query that surfaced each. It returns the same + :class:`RoundDecision` (global core indices + next_queries) so the loop — and + a future Phase-2 policy that learns the core-selection step — keep one + contract. The ``core_so_far`` / ``evidence`` strings are built in + :func:`_search_episodes_subq`; this class only formats + parses. + + Args: + llm: everalgo LLM client. + prompt: Template exposing ``{question}``, ``{core_so_far}``, + ``{evidence}``, ``{max_sub}``. Defaults to :data:`_DECIDER_PROMPT`. + """ + + def __init__(self, llm: LLMClient, *, prompt: str | None = None) -> None: + self._llm = llm + self._prompt = prompt or _DECIDER_PROMPT + + async def __call__( + self, + question: str, + core_so_far: str, + evidence: str, + n_candidates: int, + round_idx: int, + ) -> RoundDecision: + tune = _tuning() + prompt = self._prompt.format( + question=question, + core_so_far=core_so_far or "(none)", + evidence=evidence or "(no evidence retrieved this round)", + max_sub=tune.max_subqueries, + ) + data: dict | None = None + last_error: str = "" + _usage: dict[str, object] | None = None # decider token usage (trace) + _raw: str | None = None # decider raw output (trace) + for attempt in range(tune.retries + 1): + try: + resp = await self._llm.chat( + messages=[LLMChatMessage(role="user", content=prompt)] + ) + _raw = resp.content or "" + _u = getattr(resp, "usage", None) + _usage = _u.model_dump() if hasattr(_u, "model_dump") else None + # Reasoning deciders intermittently return an EMPTY completion + # (output budget spent on reasoning tokens). Name that case so the + # log distinguishes "model said nothing" from "reply was unparseable". + if not _raw.strip(): + raise ValueError("empty decider reply") + data = _parse_decision(_raw) + break + except Exception as err: + last_error = f"{type(err).__name__}: {err}" + logger.warning( + "llm_multiround_decider_error", + error=last_error[:200], + attempt=attempt, + ) + if attempt < tune.retries and tune.retry_backoff_seconds > 0: + await asyncio.sleep(tune.retry_backoff_seconds * (2**attempt)) + if data is None: + # Every attempt failed. Fall back to a deterministic core (the evidence + # list is in fused-score order) rather than returning an empty one: an + # empty core silently disables core-first injection for this question + # and is invisible downstream. Flag it so the trace can count it. + fallback = list(range(min(tune.fallback_core, max(n_candidates, 0)))) + # error, not warning: this is the multi-round mechanism not running at all. + # A warning is what let twelve sweep arms report 87-93% while every decider + # call 404'd -- the level said "noted", and the numbers were written up. + logger.error( + "llm_multiround_decider_fallback", + round_idx=round_idx, + attempts=tune.retries + 1, + fallback_core=len(fallback), + last_error=last_error[:200], + ) + # And say so in the response. A caller cannot tell a degraded result from a + # healthy one by looking at it: both are HTTP 200 with a full episode list. + mark_degraded("decider_fallback") + return RoundDecision( + stop=True, + queries=[], + core=fallback, + usage=_usage, + raw=_raw, + failed=True, + ) + core = _coerce_core_indices(data.get("core", []), n_candidates) + nxt_raw = data.get("next_queries") + if not isinstance(nxt_raw, list): + nxt_raw = [] + queries = [str(q).strip() for q in nxt_raw if str(q).strip()][ + : tune.max_subqueries + ] + return RoundDecision( + stop=not queries, queries=queries, core=core, usage=_usage, raw=_raw + ) + + +def _parse_decision(text: str) -> dict: + """Extract the JSON decision object from an LLM reply (tolerant of prose). + + Tries the widest ``{...}`` span first (the common single-object reply); + if that fails to parse, falls back to scanning for the first brace-balanced + object, so a stray ``{`` / ``}`` in surrounding prose cannot corrupt the + parse (the greedy span would otherwise swallow it and raise). Raises + ``ValueError`` only when no ``dict`` object can be recovered — the caller + treats that as a safe stop with no core. + """ + candidates: list[str] = [] + greedy = re.search(r"\{.*\}", text, re.DOTALL) + if greedy: + candidates.append(greedy.group()) + candidates.extend(_balanced_objects(text)) + for cand in candidates: + try: + obj = json.loads(cand) + except json.JSONDecodeError: + continue + if isinstance(obj, dict): + return obj + raise ValueError(f"no JSON object in decider reply: {text[:120]!r}") + + +def _balanced_objects(text: str) -> list[str]: + """Yield brace-balanced ``{...}`` substrings, outermost first.""" + out: list[str] = [] + depth = 0 + start = -1 + for i, ch in enumerate(text): + if ch == "{": + if depth == 0: + start = i + depth += 1 + elif ch == "}" and depth > 0: + depth -= 1 + if depth == 0 and start >= 0: + out.append(text[start : i + 1]) + return out + + +def _coerce_core_indices(raw: object, n_evidence: int) -> list[int]: + """Coerce a decider's ``core`` field into valid, de-duplicated evidence indices. + + Tolerates ints and int-like strings (``"2"``), silently dropping anything + out of range, non-integral, or duplicated. A non-list ``raw`` yields ``[]`` + (empty core ⇒ the caller falls back to the unfiltered / blind path). + """ + if not isinstance(raw, list): + return [] + seen: set[int] = set() + core: list[int] = [] + for value in raw: + if isinstance(value, bool): # bool is an int subclass — reject explicitly + continue + if isinstance(value, int): + idx = value + elif isinstance(value, str) and value.strip().lstrip("-").isdigit(): + idx = int(value) + else: + continue + if 0 <= idx < n_evidence and idx not in seen: + seen.add(idx) + core.append(idx) + return core + + +# ── Main entry ────────────────────────────────────────────────────────────── + + +async def search_episodes_llm_multiround( + query: str, + *, + owner_id: str, + where: str, + app_id: str = "default", + project_id: str = "default", + episode_recaller: EpisodeRecaller, + atomic_fact_recaller: AtomicFactRecaller, + embed_query_fn: Callable[[str], Awaitable[list[float]]], + llm: LLMClient, + top_k: int, + reranker: RerankProvider | None = None, + decider: RoundDecider | None = None, +) -> list[SearchEpisodeItem]: + """LLM-guided iterative multi-round episode search (per-sub-query RRF blocks). + + Thin entry that delegates to :func:`_search_episodes_subq` — the only arm. + Each round fuses every current sub-query's BM25+vector recall with RRF + INDEPENDENTLY (round 0 = one original-question block), shows the decider the + labelled blocks + accumulated core and selects core across them; the final + injection is core-first + each sub-query's top-1 guarantee + max-RRF-score + fill. No cross-encoder anywhere (RRF is the only ranking). + + Args: + query: User query (also the round-0 query). + owner_id: Owner whose memories are searched. + where: Pre-compiled LanceDB filter (owner + request filters). + app_id / project_id: Scope segments (parity; recall is owner-scoped via + ``where``). + episode_recaller: Episode sparse + dense recall. + atomic_fact_recaller: Accepted for call-site parity; this scheme recalls at the + episode level and does not drill facts. + embed_query_fn: Async ``(str) -> vector`` query embedder. + llm: LLM client for the decider. + top_k: Maximum episodes to return. + reranker: Accepted for parity but UNUSED — no cross-encoder stage. + decider: Round-control hook. When given, it REPLACES the built-in + :class:`LLMRoundDecider` and drives every round, which is how a + Phase-2 RL policy reuses this loop as its environment. + + Returns: + Ranked ``SearchEpisodeItem`` list, empty on an empty seed. + """ + return await _search_episodes_subq( + query, + owner_id=owner_id, + where=where, + app_id=app_id, + project_id=project_id, + episode_recaller=episode_recaller, + atomic_fact_recaller=atomic_fact_recaller, + embed_query_fn=embed_query_fn, + llm=llm, + top_k=top_k, + reranker=reranker, + decider=decider, + ) + + +def _render_blocks( + blocks: list[tuple[str, list[Candidate]]], +) -> tuple[str, list[Candidate], list[dict[str, object]]]: + """Render per-sub-query blocks into the decider text + a global-index map. + + Returns ``(rendered, global_cands, index_meta)``: ``rendered`` is the + ``[block: ]`` header + numbered `` : subject - summary`` lines + with a SINGLE global index running across all blocks; ``global_cands[gi]`` is + the Candidate at that index (blocks are concatenated in order, so the same id + can appear at two indices when two sub-queries retrieved it — the decider is + told to core it once); ``index_meta[gi]`` = block_id / sub_query / + rank_in_block / rrf_score, used for the trace and the core source tag. + """ + lines: list[str] = [] + global_cands: list[Candidate] = [] + index_meta: list[dict[str, object]] = [] + gi = 0 + for block_id, (sub_query, cands) in enumerate(blocks): + lines.append(f"[block: {sub_query}]") + for rank, c in enumerate(cands): + subject = str(c.metadata.get("subject", "")) + summary = _decider_text(c.metadata) + lines.append(f" {gi}: {subject} - {summary}") + global_cands.append(c) + index_meta.append( + { + "block_id": block_id, + "sub_query": sub_query, + "rank_in_block": rank, + "rrf_score": c.score, + } + ) + gi += 1 + return "\n".join(lines), global_cands, index_meta + + +def _render_core_so_far( + core_order: list[str], + core_source: dict[str, str], + cand_by_id: dict[str, Candidate], +) -> str: + """Render accumulated core for the decider, each item tagged with the + sub-query that first surfaced it. Empty core renders as ``(none)``.""" + out: list[str] = [] + for cid in core_order: + c = cand_by_id.get(cid) + if c is None: + continue + subject = str(c.metadata.get("subject", "")) + summary = _decider_text(c.metadata) + src = core_source.get(cid, "original question") + out.append(f' - [from "{src}"] {subject} - {summary}') + return "\n".join(out) or "(none)" + + +def _decider_text(meta: dict) -> str: + """Text shown to the decider for one candidate. + + Stores disagree on which column carries the full episode body (see + ``[decider].full_text``), so prefer whichever is longer when full-text + mode is on: that picks the body on both layouts without needing to know + which store is loaded. Off (the default) keeps the historical + ``summary``-only view so in-flight comparisons stay reproducible. + """ + summary = str(meta.get("summary", "") or "") + if not _tuning().full_text: + return summary + episode = str(meta.get("episode", "") or "") + return episode if len(episode) > len(summary) else summary + + +def _build_round_trace( + *, + owner_id: str, + question: str, + round_idx: int, + round_kind: str, + global_cands: list[Candidate], + index_meta: list[dict[str, object]], + decision: RoundDecision, + core_added: list[dict[str, object]], + core_carried_in: list[dict[str, object]], + block_meta: list[dict[str, object]], + timing_s: dict[str, float], +) -> dict[str, object]: + """Assemble one per-round trace record (schema C). + + Completeness principle — records everything the search layer owns this round: + the decider's blocked view (``evidence``: every global index with its block / + sub-query / RRF provenance), the action (``core_indices`` + resolved sessions + + source sub-query), the core carried INTO this round, the full per-block + recall pools (``recall.blocks``: sparse/dense pre-truncation + the RRF-ranked + kept block with in_sparse/in_dense flags), the decider tokens + raw verdict, + and per-stage timing. There is NO ``fused`` / ``reranked`` pool — this scheme + has neither a merged fuse nor a cross-encoder. ``question_id`` and the gold labels + are left for a downstream labeller (the search layer cannot know them). + """ + evidence = [ + { + "global_index": gi, + "block_id": index_meta[gi]["block_id"], + "sub_query": index_meta[gi]["sub_query"], + "id": c.id, + "session_id": c.metadata.get("session_id"), + "subject": str(c.metadata.get("subject", "")), + "summary": str(c.metadata.get("summary", "")), + "rrf_score": index_meta[gi]["rrf_score"], + "rank_in_block": index_meta[gi]["rank_in_block"], + } + for gi, c in enumerate(global_cands) + ] + core_indices = list(decision.core) + core_session_ids = [ + global_cands[i].metadata.get("session_id") + for i in core_indices + if 0 <= i < len(global_cands) + ] + record: dict[str, object] = { + "dataset": _dataset_from_owner(owner_id), + "owner_id": owner_id, + "question_id": None, + "question": question, + "round_idx": round_idx, + "round_kind": round_kind, + "evidence": evidence, + "core_indices": core_indices, + "core_session_ids": core_session_ids, + "core_source_subquery": [m["sub_query"] for m in core_added], + "core_added": core_added, + "core_carried_in": core_carried_in, + "stop": decision.stop, + "next_queries": list(decision.queries), + "recall": {"blocks": block_meta}, + "timing_s": timing_s, + } + if decision.usage is not None or decision.raw is not None: + record["decider"] = {"tokens": decision.usage, "raw": decision.raw} + if decision.failed: + # Degraded round: every decider attempt failed and ``core_indices`` is the + # deterministic fallback, not a decider choice. Recorded so analysis can + # count or exclude these instead of silently mixing them with real rounds. + record["decider_failed"] = True + return record + + +async def _search_episodes_subq( + query: str, + *, + owner_id: str, + where: str, + app_id: str = "default", + project_id: str = "default", + episode_recaller: EpisodeRecaller, + atomic_fact_recaller: AtomicFactRecaller, + embed_query_fn: Callable[[str], Awaitable[list[float]]], + llm: LLMClient, + top_k: int, + reranker: RerankProvider | None, + decider: RoundDecider | None, +) -> list[SearchEpisodeItem]: + """Retrieval loop: per-sub-query RRF blocks + guarantee-then-fill, no cross-encoder. + + Each round: embed every current sub-query, recall (BM25 + vector) and fuse + EACH sub-query's own pair with RRF INDEPENDENTLY (round 0 = one block for the + original question, kept to ``[decider].seed_topk``; round>=1 = one block per + sub-query, each kept to ``[decider].subq_topk``). The blocks are shown to the + decider as labelled sections with a single GLOBAL index, alongside a + CORE-SO-FAR section (each kept item tagged with its source sub-query). The + decider returns core (global indices) + next sub-queries. Stopping: + empty ``next_queries`` / ``[decider].no_new_core_patience`` saturated + rounds (after >=1 follow-up) / ``[decider].max_rounds``. + + After the loop, :func:`_finalize_injection` assembles the top_k: core-first + (hard guarantee) + each sub-query's top non-core + max-RRF-score fill. + + ``reranker`` is accepted for call-site parity but intentionally UNUSED — this + scheme has no cross-encoder stage (the CE was net-negative). ``decider``, when + supplied, replaces :class:`LLMRoundDecider` for every round: that is the seam an RL + policy occupies so the loop it trains against is this loop rather than a copy. + """ + # Honour an injected decider. The parameter and the RoundDecider protocol were + # written for this ("a Phase-2 policy implements this same signature so the loop + # becomes the RL environment — only the decider changes") but the body ignored the + # argument and always built the prompt-LLM decider, so an RL environment had no way + # to reuse this loop and had to re-implement retrieval, block rendering, core + # accumulation and the stop conditions. Re-implementing diverged: measured against + # this loop, a hand-built environment retrieved a different candidate set on 25/25 + # sampled sub-queries (Jaccard median 0.538): the HTTP search route it called + # dispatches to the hierarchy/heap-expand hybrid pipeline, not this file's + # per-sub-query rrf(sparse, dense)[:topk]. + decide = decider or LLMRoundDecider(llm) + + core_by_id: dict[str, Candidate] = {} # accumulated core (best-score candidate) + core_ids: set[str] = set() + core_order: list[str] = [] # core insertion order (drives core-first) + core_source: dict[str, str] = {} # id -> sub-query that first cored it (#4) + subq_hits: dict[str, dict[str, float]] = {} # id -> {sub_query: rrf_score} + cand_by_id: dict[str, Candidate] = {} # id -> best-score candidate ever seen + guarantee: list[tuple[str, str]] = [] # (sub_query, top id) per block, in order + subqueries_seen: list[str] = [] + no_new_core_streak = 0 + cur_queries = [query] # round 0 seeds with the user question + + tune = _tuning() + for round_idx in range(tune.max_rounds): + _ts_round_start = time.monotonic() + round_kind = "seed" if round_idx == 0 else "subquery" + block_topk = tune.seed_topk if round_idx == 0 else tune.subq_topk + vecs = await asyncio.gather(*[embed_query_fn(q) for q in cur_queries]) + # Recall every sub-query concurrently (sparse + dense). The KEY design point: + # each sub-query's pair is fused with RRF INDEPENDENTLY below, not + # merged into one pool first — so a facet-gold ranked #1 for one sub-query + # is not diluted by the other sub-queries' consensus hits. + recalls = await asyncio.gather( + *[ + asyncio.gather( + episode_recaller.sparse_recall(q, where, limit=_SEED_CANDIDATES), + episode_recaller.dense_recall(v, where, limit=_SEED_CANDIDATES) + if v + else _empty_candidates(), + ) + for q, v in zip(cur_queries, vecs, strict=True) + ] + ) + _ts_recall_done = time.monotonic() + blocks: list[tuple[str, list[Candidate]]] = [] + block_meta: list[dict[str, object]] = [] + for q, (r_sparse, r_dense) in zip(cur_queries, recalls, strict=True): + # Block LABEL shown to the decider (and stored as each candidate's + # source): round 0 labels its one block "original question" — the raw + # question is already displayed above it and the few-shot uses this + # label; later rounds label each block with its own sub-query text. + label = "original question" if round_idx == 0 else q + fused = rrf(r_sparse, r_dense, k=tune.rrf_k) # ranked by RRF score + block = fused[:block_topk] + blocks.append((label, block)) + if label not in subqueries_seen: + subqueries_seen.append(label) + sparse_ids = {c.id for c in r_sparse} + dense_ids = {c.id for c in r_dense} + for c in block: + subq_hits.setdefault(c.id, {})[label] = c.score + cur = cand_by_id.get(c.id) + if cur is None or c.score > cur.score: + cand_by_id[c.id] = c + # Guarantee this block's top per_subquery_guarantee for the final fill. + for c in block[: tune.per_subquery_guarantee]: + guarantee.append((label, c.id)) + block_meta.append( + { + "sub_query": label, + "n_sparse": len(r_sparse), + "n_dense": len(r_dense), + "topk_kept": len(block), + "sparse": _trace_cand_pool(r_sparse), + "dense": _trace_cand_pool(r_dense), + "rrf_ranked": [ + { + "id": c.id, + "session_id": c.metadata.get("session_id"), + "rrf_score": c.score, + "rank": r, + "in_sparse": c.id in sparse_ids, + "in_dense": c.id in dense_ids, + } + for r, c in enumerate(block) + ], + } + ) + if round_idx == 0 and not any(b for _, b in blocks): + return [] # empty seed — nothing to rank or decide over + + # Render the blocked decider view + CORE-SO-FAR (global index across blocks). + rendered, global_cands, index_meta = _render_blocks(blocks) + core_so_far = _render_core_so_far(core_order, core_source, cand_by_id) + _ts_decide_start = time.monotonic() + decision = await decide( + query, core_so_far, rendered, len(global_cands), round_idx + ) + + # Accumulate new core (global index -> id, dedup against already-cored). + n_before = len(core_ids) + core_added: list[dict[str, object]] = [] + for gi in decision.core: + if not (0 <= gi < len(global_cands)): + continue + cid = global_cands[gi].id + if cid in core_ids: + continue + core_ids.add(cid) + core_order.append(cid) + core_by_id[cid] = cand_by_id.get(cid, global_cands[gi]) + core_source[cid] = str(index_meta[gi]["sub_query"]) + core_added.append( + { + "global_index": gi, + "id": cid, + "sub_query": index_meta[gi]["sub_query"], + } + ) + added = len(core_ids) - n_before + no_new_core_streak = 0 if added else no_new_core_streak + 1 + + dump_path = _trace_dump_path() + if dump_path: + _append_round_trace( + dump_path, + _build_round_trace( + owner_id=owner_id, + question=query, + round_idx=round_idx, + round_kind=round_kind, + global_cands=global_cands, + index_meta=index_meta, + decision=decision, + core_added=core_added, + core_carried_in=[ + { + "id": cid, + "sub_query": core_source.get(cid), + "subject": str(cand_by_id[cid].metadata.get("subject", "")), + "summary": str(cand_by_id[cid].metadata.get("summary", "")), + } + for cid in core_order + if cid in cand_by_id + ], + block_meta=block_meta, + timing_s={ + "recall": round(_ts_recall_done - _ts_round_start, 3), + "decide": round(time.monotonic() - _ts_decide_start, 3), + }, + ), + ) + logger.info( + "llm_multiround_round", + round=round_idx + 1, + kind=round_kind, + n_blocks=len(blocks), + n_candidates=len(global_cands), + n_core=len(core_ids), + added_core=added, + no_new_core_streak=no_new_core_streak, + n_subqueries=len(decision.queries), + next_queries=" | ".join(decision.queries)[:120], + query=query[:80], + ) + + # Stop conditions, in the design's priority order. + if not decision.queries: # (1) decider is done (no gaps left) + break + # (2) coverage saturated — never before at least one follow-up round. + if round_idx >= 1 and no_new_core_streak >= tune.no_new_core_patience: + break + cur_queries = decision.queries # else continue; (3) the loop caps rounds + + if not cand_by_id and not core_by_id: + return [] + return _finalize_injection( + owner_id=owner_id, + question=query, + top_k=top_k, + core_ids=core_ids, + core_order=core_order, + cand_by_id=cand_by_id, + subq_hits=subq_hits, + guarantee=guarantee, + subqueries_seen=subqueries_seen, + ) + + +def _finalize_injection( + *, + owner_id: str, + question: str, + top_k: int, + core_ids: set[str], + core_order: list[str], + cand_by_id: dict[str, Candidate], + subq_hits: dict[str, dict[str, float]], + guarantee: list[tuple[str, str]], + subqueries_seen: list[str], +) -> list[SearchEpisodeItem]: + """Assemble the final top_k and dump the final-injection trace (schema C). + + Assembly (post-loop, once): + 1. CORE first — every accumulated core, pinned to the front (a hard + guarantee: kept even when ``len(core) > top_k``), ordered by max RRF + score. + 2. GUARANTEE — each sub-query's top non-core candidate (its ``guarantee`` + entries) reserves a slot, so no facet loses coverage. + 3. FILL — remaining candidates by MAX RRF score across sub-queries. MAX, not + sum: a specialist that is #1 for a single sub-query keeps that score + instead of being diluted by consensus items (the RRF-dilution fix). + + The final-injection trace records, per injected episode, its slot_source + (core / guarantee-top1 / maxscore-fill), the max RRF score + which sub-query + gave it, and the full per-sub-query score map — so the whole assembly is + reconstructable — plus an ``assembly`` summary count. + """ + + def max_score(cid: str) -> float: + scores = subq_hits.get(cid) + return max(scores.values()) if scores else 0.0 + + def max_subquery(cid: str) -> str | None: + scores = subq_hits.get(cid) + return max(scores, key=lambda k: scores[k]) if scores else None + + slot_source: dict[str, str] = {} + chosen: list[str] = [] + # 1. core-first, ordered by max RRF score across sub-queries. Capped at top_k: + # steps 2 and 3 honour top_k, so leaving core uncapped silently returned MORE + # than the caller asked for — measured at 316/1522 questions (20.8%) on + # SubtleMemory, up to 68 items for a top_k of 20 (3.4x the budget). That breaks + # the top_k contract and any same-budget comparison across methods. Core beyond + # the budget is by definition the lowest-scored core, so it is what gets dropped. + n_core_selected = 0 + for cid in sorted(core_order, key=max_score, reverse=True): + if cid in slot_source: + continue + n_core_selected += 1 + if not _tuning().core_overflow and len(chosen) >= top_k: + continue # keep counting for the trace, but do not exceed the budget + chosen.append(cid) + slot_source[cid] = "core" + n_core = len(chosen) + if n_core_selected > n_core: + logger.info( + "llm_multiround_core_truncated", + owner_id=owner_id, + selected=n_core_selected, + kept=n_core, + top_k=top_k, + ) + # 2. per-sub-query guarantee (top non-core, in block discovery order) + n_guaranteed = 0 + for _sub_query, cid in guarantee: + if len(chosen) >= top_k: + break + if cid in slot_source or cid in core_ids: + continue + chosen.append(cid) + slot_source[cid] = "guarantee-top1" + n_guaranteed += 1 + # 3. max-score fill to top_k + n_filled = 0 + remaining = [cid for cid in cand_by_id if cid not in slot_source] + for cid in sorted(remaining, key=max_score, reverse=True): + if len(chosen) >= top_k: + break + chosen.append(cid) + slot_source[cid] = "maxscore-fill" + n_filled += 1 + + ordered_cands = [cand_by_id[cid] for cid in chosen if cid in cand_by_id] + episodes = [ + ep + for ep in (shape_episode_from_candidate(c) for c in ordered_cands) + if ep is not None + ] + + dump_path = _trace_dump_path() + if dump_path: + _append_round_trace( + dump_path, + { + "dataset": _dataset_from_owner(owner_id), + "owner_id": owner_id, + "question_id": None, + "question": question, + "round_idx": None, # None ⇒ final-injection record, not a per-round one + "injected": [ + { + "rank": r, + "id": ep.id, + "session_id": ep.session_id, + "timestamp": str(ep.timestamp), + "is_core": ep.id in core_ids, + "slot_source": slot_source.get(ep.id), + "max_rrf_score": max_score(ep.id), + "max_rrf_subquery": max_subquery(ep.id), + "per_subquery_scores": subq_hits.get(ep.id, {}), + } + for r, ep in enumerate(episodes) + ], + "assembly": { + "n_core": n_core, + "n_guaranteed": n_guaranteed, + "n_filled": n_filled, + "top_k": top_k, + "subqueries_seen": subqueries_seen, + }, + }, + ) + return episodes + + +def _trace_dump_path() -> str | None: + """Return the trace JSONL path, or ``None`` when the dump is disabled. + + Read per call (not memoised at import) so a run or a test can toggle + ``EVEROS_LLMMR_TRACE_DUMP`` via env without re-importing the module. An + unset or whitespace-only value disables the dump. + """ + path = os.getenv(_TRACE_DUMP_ENV, "").strip() + return path or None + + +def _dataset_from_owner(owner_id: str) -> str: + """Best-effort dataset tag from the eval ``owner_id`` convention. + + Eval owner ids follow ``"_"`` (see MemoryRL + ``prepare_data``), e.g. ``longmemeval_0`` ⇒ ``longmemeval``. The trailing + numeric conv index is stripped; an id that does not match the convention + (no ``_`` or a non-numeric tail) passes through unchanged. + """ + head, sep, tail = owner_id.rpartition("_") + if sep and tail.isdigit(): + return head + return owner_id + + +def _trace_cand_pool(cands: object) -> list[dict[str, object]]: + """Serialize a candidate pool (dict[id]->Candidate or list) for the full-retrieval + attribution trace: id + session_id + score + source + metadata (session / entry_id / + fact provenance). Lets a bad case be split into recall-miss vs rerank-miss vs + decider-miss, and drilled to the atomic fact that surfaced each episode.""" + items = cands.values() if isinstance(cands, dict) else (cands or []) + out: list[dict[str, object]] = [] + for c in items: + md = c.metadata if isinstance(c.metadata, dict) else {} + # Attribution-essential provenance only. Deliberately DROP the bulky episode + # ``summary`` (recoverable from the store by entry_id/session_id; ~5-10x smaller + # trace) and the per-owner constants (owner_id/app_id/...). The decider-view + # ``evidence`` keeps its summary — that IS the decider's input. + prov = { + k: md[k] + for k in ( + "entry_id", + "session_id", + "subject", + "timestamp", + "parent_id", + "parent_type", + ) + if md.get(k) is not None + } + out.append( + { + "id": c.id, + "session_id": md.get("session_id"), + "score": c.score, + "source": c.source, + "provenance": prov, + } + ) + return out + + +def _append_round_trace(path: str, record: dict[str, object]) -> None: + """Append one trace ``record`` as a JSON line to ``path``. + + Diagnostic side-channel, reached only when ``EVEROS_LLMMR_TRACE_DUMP`` is + set. The server event loop is single-threaded, so a small synchronous + append cannot interleave across concurrent search coroutines (each + ``write`` completes before the coroutine yields). A write failure is logged + and swallowed — trace dumping must never break a live search. + """ + try: + # default=str: candidate metadata carries datetimes (episode timestamps) and + # other non-JSON scalars; stringify them instead of raising. except Exception + # (not just OSError): a trace-dump failure must NEVER break a live search. + with open(path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(record, ensure_ascii=False, default=str) + "\n") + except Exception as err: + logger.warning("llm_multiround_trace_dump_error", error=str(err)[:200]) + + +async def _empty_candidates() -> list[Candidate]: + return [] diff --git a/src/everos/memory/search/manager.py b/src/everos/memory/search/manager.py index 1ae5bfd91..84abc68ef 100644 --- a/src/everos/memory/search/manager.py +++ b/src/everos/memory/search/manager.py @@ -40,7 +40,12 @@ from everos.component.rerank import get_rerank_capability from everos.component.utils.datetime import to_display_tz from everos.config import load_settings -from everos.core.context import resolve_request_id +from everos.core.context import ( + get_degradations, + reset_degradations, + resolve_request_id, + restore_degradations, +) from everos.core.errors import ConfigurationError, ProviderNotConfiguredError from everos.core.observability.logging import get_logger from everos.core.observability.tracing import ( @@ -72,6 +77,7 @@ ) from .filters import compile_filters from .hierarchy import build_ep_to_fact_parents, heap_expand +from .llm_multiround import RoundDecider, search_episodes_llm_multiround from .shaper import ( reshape_hybrid_output, shape_agent_case_from_candidate, @@ -167,6 +173,7 @@ def __init__( reranker: RerankProvider | None, llm_client: LLMClient | None, search_tokenizer: Tokenizer | None = None, + decider_client: LLMClient | None = None, ) -> None: self._ep = episode_recaller self._fact = atomic_fact_recaller @@ -176,12 +183,21 @@ def __init__( self._embedding = embedding self._reranker = reranker self._llm = llm_client + # Multi-round retrieval runs its own model when [decider] configures one; the + # caller passes None to mean "same as extraction", which is the historical + # behaviour and what every store built before [decider] existed was scored with. + self._decider_llm = decider_client or llm_client self._search_tokenizer = search_tokenizer # ── Public entry ──────────────────────────────────────────────── async def search(self, req: SearchRequest) -> SearchResponse: request_id = resolve_request_id() + # Cleared per search, and restored on the way out. A worker task's context is + # reused across requests, so without this one degraded search would mark every + # later healthy one -- which is worse than not reporting at all, because it + # trains the reader to ignore the field. + _deg_token = reset_degradations() with memory_span( "everos.memory.search", observation_type="retriever", @@ -223,6 +239,10 @@ async def search(self, req: SearchRequest) -> SearchResponse: episodes=episodes, profiles=profiles, unprocessed_messages=unprocessed, + # Read after the routes have run: a fallback deep in the + # multi-round loop records itself here, and this is the last + # point before the result leaves the domain. + degraded=list(get_degradations()), ) else: # "agent" (cases, skills), unprocessed = await asyncio.gather( @@ -233,6 +253,7 @@ async def search(self, req: SearchRequest) -> SearchResponse: agent_cases=cases, agent_skills=skills, unprocessed_messages=unprocessed, + degraded=list(get_degradations()), ) # Returned hits (ids only) — content, so only when capture_content @@ -271,6 +292,7 @@ async def search(self, req: SearchRequest) -> SearchResponse: method=req.method.value, ) + restore_degradations(_deg_token) return SearchResponse(request_id=request_id, data=data) # ── Unprocessed buffer ────────────────────────────────────────── @@ -327,8 +349,33 @@ async def _search_cases_and_skills( # ── Episodes ──────────────────────────────────────────────────── + async def search_episodes_with_decider( + self, req: SearchRequest, decider: RoundDecider + ) -> list[SearchEpisodeItem]: + """Run the ``llm_multiround`` loop with an injected decider. + + This is the seam a Phase-2 RL policy occupies: the loop, its retrieval, + its block rendering and its stop conditions are EverOS's own, and only + the per-round decision comes from the caller. An RL environment that + re-implements the loop instead drifts: measured against this one, a + hand-built environment returned a different candidate set on 25/25 + sampled sub-queries (Jaccard median 0.538), because the public search + route dispatches to the hybrid hierarchy pipeline rather than to + ``llm_multiround``'s per-sub-query ``rrf(sparse, dense)[:topk]``. + """ + req = req.model_copy(update={"method": SearchMethod.LLM_MULTIROUND}) + self._validate_components(req) + where = compile_filters( + req.filters, + owner_type=req.owner_type, + owner_id=req.owner_id, + app_id=req.app_id, + project_id=req.project_id, + ) + return await self._search_episodes(req, where, decider=decider) + async def _search_episodes( - self, req: SearchRequest, where: str + self, req: SearchRequest, where: str, *, decider: RoundDecider | None = None ) -> list[SearchEpisodeItem]: if req.method == SearchMethod.AGENTIC: return await search_episodes_agentic( @@ -345,6 +392,22 @@ async def _search_episodes( top_k=self._top_k(req.top_k), ) + if req.method == SearchMethod.LLM_MULTIROUND: + return await search_episodes_llm_multiround( + req.query, + owner_id=req.owner_id, + where=where, + app_id=req.app_id, + project_id=req.project_id, + episode_recaller=self._ep, + atomic_fact_recaller=self._fact, + embed_query_fn=self._embedding.embed, # type: ignore[union-attr] + llm=self._decider_llm, # type: ignore[arg-type] + top_k=self._top_k(req.top_k), + reranker=self._reranker, # type: ignore[arg-type] + decider=decider, + ) + fusion_mode, _ = resolve_pipeline(req.method, "episode") enable_rerank = _effective_llm_rerank(req) top_k = self._top_k(req.top_k) @@ -576,7 +639,7 @@ async def _search_agent_skills( async def _fetch_profile(self, req: SearchRequest) -> list[SearchProfileItem]: if not req.include_profile or req.owner_type != "user": return [] - return await self._profile.fetch(req.owner_id) + return await self._profile.fetch(req.owner_id, subject=req.profile_subject) # ── Recall helpers ────────────────────────────────────────────── @@ -821,6 +884,7 @@ def _validate_components(self, req: SearchRequest) -> None: SearchMethod.VECTOR, SearchMethod.HYBRID, SearchMethod.AGENTIC, + SearchMethod.LLM_MULTIROUND, ) if needs_embedding and ( not get_embedding_capability().available or self._embedding is None @@ -830,6 +894,18 @@ def _validate_components(self, req: SearchRequest) -> None: feature=_feature_name(method, req.owner_type), ) + if method == SearchMethod.LLM_MULTIROUND: + if req.owner_type == "agent": + raise RuntimeError( + "method='llm_multiround' is only supported for user memory; " + "provide user_id instead of agent_id" + ) + if self._llm is None: + raise ProviderNotConfiguredError( + provider="llm", + feature=_feature_name(method, req.owner_type), + ) + # agent HYBRID cross-encoder lane (enable_llm_rerank=False, the # default) reaches ``search_agent_skills_hybrid``, which needs a # real rerank provider; the LLM lane reranks via the LLM instead diff --git a/src/everos/memory/search/recall/profile.py b/src/everos/memory/search/recall/profile.py index be149b927..159990eca 100644 --- a/src/everos/memory/search/recall/profile.py +++ b/src/everos/memory/search/recall/profile.py @@ -1,15 +1,19 @@ """Profile recall — KV-by-owner LanceDB fetch (no ranking). -Profile is the only owner-scoped kind that ships as **one row per -user** (no per-day fan-out, no entry markers). The recaller is a -deliberate KV-by-owner lookup: given ``owner_id``, return at most one -:class:`SearchProfileItem`. There is no ``query`` and no ``score`` -field on the response — the DTO's optional ``score`` is reserved for -a future query-aware lookup. - -The cascade keeps ``UserProfile`` rows in sync with -``users//user.md``; this recaller just reads the row and -unpacks the json-encoded buckets back into the DTO's +Profile has no per-day fan-out and no entry markers. The recaller is a +deliberate KV-by-owner lookup — there is no ``query`` and no ``score`` on the +response; the DTO's optional ``score`` is reserved for a future query-aware +lookup. + +One owner usually holds exactly one row (``subject`` empty, itself the +subject). A **group** owner holds one row per participant, so the fetch takes +an optional ``subject``: naming one returns that person's profile, omitting it +returns every row under the owner. ``PROFILE_MAX_ROWS`` bounds the second case +— a caller that wants one person out of a large group has to say which, +because silently truncating would answer as somebody else. + +The cascade keeps ``UserProfile`` rows in sync with the md; this recaller just +reads them and unpacks the json-encoded buckets back into the DTO's ``profile_data`` mapping (mirrors enterprise's profile DTO shape). """ @@ -25,41 +29,80 @@ logger = get_logger(__name__) +PROFILE_MAX_ROWS = 64 +"""Cap on an unfiltered group fetch. Above this the response is a wall of +other people's profiles; the caller should pass ``subject`` instead.""" + class ProfileRecaller: - """Fetch the owner's profile row from LanceDB, return at most one item.""" + """Fetch an owner's profile rows from LanceDB.""" - async def fetch(self, owner_id: str) -> list[SearchProfileItem]: - """Return ``[item]`` if a profile row exists, otherwise ``[]``. + async def fetch( + self, owner_id: str, *, subject: str | None = None + ) -> list[SearchProfileItem]: + """Return the owner's profile rows, or ``[]`` when there are none. Empty list (rather than 404) lets the caller emit a normal response with ``profiles=[]`` while the user is still in their cold-start window (no profile synthesised yet). + + Args: + owner_id: Memory partition. Also the row id in the common + owner-is-subject case. + subject: Name of one participant of a group owner. ``None`` + returns every row under the owner, ordered by subject so the + response is stable across calls. """ if not owner_id: return [] - row = await user_profile_repo.get_by_id(owner_id) - if row is None: - logger.debug("profile_fetch_miss", owner_id=owner_id) + if subject: + row = await user_profile_repo.get_by_id(f"{owner_id}::{subject}") + rows = [row] if row is not None else [] + else: + rows = sorted( + await user_profile_repo.find_by_owner(owner_id, limit=PROFILE_MAX_ROWS), + key=lambda r: r.id, + ) + if not rows: + logger.debug("profile_fetch_miss", owner_id=owner_id, subject=subject) return [] - profile_data: dict[str, Any] = { - "summary": row.summary, - "explicit_info": _load_json(row.explicit_info_json), - "implicit_traits": _load_json(row.implicit_traits_json), - "profile_timestamp_ms": row.profile_timestamp_ms, - } + if len(rows) == PROFILE_MAX_ROWS: + logger.warning( + "profile_fetch_truncated", + owner_id=owner_id, + limit=PROFILE_MAX_ROWS, + ) return [ SearchProfileItem( id=row.id, user_id=row.owner_id, app_id=row.app_id, project_id=row.project_id, - profile_data=profile_data, + profile_data={ + "subject": _subject_of(row.id, row.owner_id) or row.owner_id, + "summary": row.summary, + "explicit_info": _load_json(row.explicit_info_json), + "implicit_traits": _load_json(row.implicit_traits_json), + "profile_timestamp_ms": row.profile_timestamp_ms, + }, score=None, ) + for row in rows ] +def _subject_of(row_id: str, owner_id: str) -> str: + """Recover the subject the cascade encoded into the row id. + + ``::`` for a group owner's participant; a bare + ``owner_id`` means the owner is its own subject, so the subject is empty. + Stripping the known prefix (rather than splitting on ``::``) keeps this + unambiguous whatever the name contains. + """ + prefix = f"{owner_id}::" + return row_id[len(prefix) :] if row_id.startswith(prefix) else "" + + def _load_json(text: str) -> Any: """Decode a json-encoded frontmatter bucket. diff --git a/tests/unit/test_memory/test_search/test_decider_tuning_and_degradation.py b/tests/unit/test_memory/test_search/test_decider_tuning_and_degradation.py new file mode 100644 index 000000000..40b1674f0 --- /dev/null +++ b/tests/unit/test_memory/test_search/test_decider_tuning_and_degradation.py @@ -0,0 +1,189 @@ +"""The decider's loop settings are configurable, and a degraded result says so. + +Two defects, one root cause: a run's behaviour was not readable off its configuration. + +* Twelve loop parameters existed only as ``EVEROS_LLMMR_*`` environment variables read + once at import time. Nothing in the config named them, so an operator could not + discover them, and an import-time read cannot answer to a config reload. +* When every decider attempt fails, the loop falls back to a fixed top-N core and stops + after one round. It logged a *warning* and returned HTTP 200 with a full episode list. + That produced a batch of runs reporting plausible accuracies while every decider + call 404'd -- the same degraded path twelve times, indistinguishable from healthy + results, and the numbers were written up before anyone read a trace. + +So: the parameters live in ``[decider]``, resolved per search; and a fallback marks the +response and logs at error. +""" + +from __future__ import annotations + +import pytest + +from everos.config.settings import DeciderSettings +from everos.core.context import ( + get_degradations, + mark_degraded, + reset_degradations, + restore_degradations, +) +from everos.memory.search import llm_multiround as lmr +from everos.memory.search.dto import SearchData + +_TUNING = ( + "max_rounds", + "seed_topk", + "subq_topk", + "max_subqueries", + "rrf_k", + "no_new_core_patience", + "per_subquery_guarantee", + "retries", + "retry_backoff_seconds", + "core_overflow", + "full_text", + "fallback_core", +) + + +@pytest.mark.parametrize("field", _TUNING) +def test_every_loop_parameter_is_a_config_field(field: str) -> None: + """Declared, so `everos config` shows it and a TOML can set it. + + Parameterised by name rather than asserted as a set: a rename should fail on the + field that moved, not on an opaque set difference. + """ + assert field in DeciderSettings.model_fields + + +def test_defaults_match_what_the_published_runs_used() -> None: + """Promoting these must not silently change any of them. + + Every reference number in the repository was produced with these values. A default + that shifted in the move would make the published results unreproducible while every + test still passed. + """ + d = DeciderSettings() + assert (d.max_rounds, d.seed_topk, d.subq_topk, d.max_subqueries) == (3, 50, 20, 3) + assert (d.rrf_k, d.no_new_core_patience, d.per_subquery_guarantee) == (60, 1, 1) + assert (d.retries, d.retry_backoff_seconds, d.fallback_core) == (3, 0.5, 3) + assert d.core_overflow is False and d.full_text is False + + +def test_the_legacy_env_names_still_win(monkeypatch: pytest.MonkeyPatch) -> None: + """Launch scripts export these; an in-flight comparison must not shift. + + Precedence is deliberate: env over config. The reverse would change the behaviour of + every script that already sets one the moment this landed. + """ + monkeypatch.setenv("EVEROS_LLMMR_MAX_ROUNDS", "7") + monkeypatch.setenv("EVEROS_LLMMR_DECIDER_FULL_TEXT", "1") + monkeypatch.setenv("EVEROS_LLMMR_DECIDER_BACKOFF_S", "0.25") + t = lmr._tuning() + assert (t.max_rounds, t.full_text, t.retry_backoff_seconds) == (7, True, 0.25) + + +def test_a_malformed_override_is_ignored_not_guessed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Falling back to the configured value beats inventing one. + + The alternative -- crashing -- would take down a search over a typo in an + environment variable, and the run would already have been paid for. + """ + monkeypatch.setenv("EVEROS_LLMMR_MAX_ROUNDS", "not-a-number") + assert lmr._tuning().max_rounds == DeciderSettings().max_rounds + + +def test_resolution_is_per_call_not_frozen_at_import( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The property that made the old shape untestable. + + The root conftest resets the settings cache per test, so an import-time read would + serve a value from whichever test imported the module first. + """ + monkeypatch.setenv("EVEROS_LLMMR_MAX_ROUNDS", "2") + assert lmr._tuning().max_rounds == 2 + monkeypatch.setenv("EVEROS_LLMMR_MAX_ROUNDS", "5") + assert lmr._tuning().max_rounds == 5 + + +def test_the_fallback_marks_the_response() -> None: + """`decider_fallback` is the identifier a client switches on.""" + token = reset_degradations() + try: + mark_degraded("decider_fallback") + assert get_degradations() == ("decider_fallback",) + assert SearchData(degraded=list(get_degradations())).degraded == [ + "decider_fallback" + ] + finally: + restore_degradations(token) + + +def test_repeats_collapse() -> None: + """A per-question loop hits the same fallback every round. + + The response should say what degraded, not how many times -- a list that grows with + the round count reads like several distinct faults. + """ + token = reset_degradations() + try: + for _ in range(5): + mark_degraded("decider_fallback") + mark_degraded("something_else") + assert get_degradations() == ("decider_fallback", "something_else") + finally: + restore_degradations(token) + + +def test_a_healthy_result_carries_nothing() -> None: + """Empty on the normal path, so a client that ignores the field sees no change.""" + token = reset_degradations() + try: + assert get_degradations() == () + assert SearchData().degraded == [] + finally: + restore_degradations(token) + + +def test_reset_stops_one_degraded_search_marking_later_ones() -> None: + """Worker contexts are reused, and a sticky flag is worse than no flag. + + A field that eventually marks everything trains the reader to ignore it, which + leaves the original defect in place with extra machinery on top. + """ + token = reset_degradations() + try: + mark_degraded("decider_fallback") + inner = reset_degradations() + assert get_degradations() == () + restore_degradations(inner) + assert get_degradations() == ("decider_fallback",) + finally: + restore_degradations(token) + + +def test_blank_reasons_are_not_recorded() -> None: + """An empty string would render as a degradation with no name.""" + token = reset_degradations() + try: + mark_degraded("") + mark_degraded(" ") + assert get_degradations() == () + finally: + restore_degradations(token) + + +def test_the_fallback_logs_at_error_not_warning() -> None: + """The level is the signal that got missed. + + A warning said "noted" for the mechanism under test not running at all, and twelve + arms of results were published on top of it. + """ + from pathlib import Path + + src = Path(lmr.__file__).read_text(encoding="utf-8") + block = src[src.index("fallback = list(range(min(tune.fallback_core") :][:900] + assert 'logger.error(\n "llm_multiround_decider_fallback"' in block + assert 'mark_degraded("decider_fallback")' in block diff --git a/tests/unit/test_memory/test_search/test_llm_multiround.py b/tests/unit/test_memory/test_search/test_llm_multiround.py new file mode 100644 index 000000000..49dc0fe67 --- /dev/null +++ b/tests/unit/test_memory/test_search/test_llm_multiround.py @@ -0,0 +1,566 @@ +"""Unit tests for ``memory.search.llm_multiround`` (per-sub-query RRF blocks). + +White-box: stubs the episode recaller, the query embedder, and the decider LLM +so the block loop runs without LanceDB or a real LLM. Covers the parse helpers, +the :class:`LLMRoundDecider` (JSON -> decision, retries, graceful stop), block +rendering + global indexing, core-so-far rendering, the search loop (early stop +/ sub-query expansion / max-rounds / empty seed), core-first + guarantee-then-fill +assembly, and per-round + final-injection trace completeness. +""" + +from __future__ import annotations + +import datetime as _dt +import json +from collections.abc import Sequence +from typing import Any, ClassVar + +import pytest +from everalgo.llm.types import ChatMessage, ChatResponse +from everalgo.types import Candidate + +from everos.memory.search import llm_multiround +from everos.memory.search.llm_multiround import ( + LLMRoundDecider, + RoundDecision, + _balanced_objects, + _coerce_core_indices, + _dataset_from_owner, + _parse_decision, + _render_blocks, + _render_core_so_far, + search_episodes_llm_multiround, +) + +# ── Stubs ──────────────────────────────────────────────────────────────── + + +def _ts() -> _dt.datetime: + return _dt.datetime(2026, 1, 1, tzinfo=_dt.UTC) + + +def _ep(entry_id: str, score: float, *, session: str = "sess_a") -> Candidate: + """Episode candidate keyed by LanceDB id ``__``.""" + return Candidate( + id=f"alice__{entry_id}", + score=score, + source="vector", + metadata={ + "entry_id": entry_id, + "owner_id": "alice", + "owner_type": "user", + "session_id": session, + "timestamp": _ts(), + "sender_ids": ["alice"], + "subject": f"subject {entry_id}", + "summary": f"summary {entry_id}", + "episode": f"body {entry_id}", + }, + ) + + +class _Recaller: + """Episode recaller. ``per_query`` maps a sparse query -> its own seed, so a + later sub-query can surface different candidates; otherwise ``seed`` is used. + Dense recall returns ``seed`` (drive dense off by embedding to ``[]``).""" + + text_field: ClassVar[str] = "episode" + + def __init__( + self, + *, + seed: list[Candidate], + per_query: dict[str, list[Candidate]] | None = None, + ) -> None: + self._seed = seed + self._per_query = per_query or {} + self.sparse_queries: list[str] = [] + self.dense_vectors: list[list[float]] = [] + + async def sparse_recall( + self, query: str, where: str, *, limit: int + ) -> list[Candidate]: + self.sparse_queries.append(query) + return list(self._per_query.get(query, self._seed)) + + async def dense_recall( + self, vector: Sequence[float], where: str, *, limit: int + ) -> list[Candidate]: + self.dense_vectors.append(list(vector)) + return list(self._seed) + + +class _FactRecaller: + async def facts_for_episodes(self, *a: Any, **k: Any) -> dict[str, list[Any]]: + return {} + + +class _ScriptLLM: + """Decider LLM: replays ``replies[i]`` per ``.chat`` call (last repeats). + + ``error_first`` raises that many times before the first real reply (retry + test). Each reply is the raw JSON string the decider parses. + """ + + def __init__(self, replies: list[str], *, error_first: int = 0) -> None: + self._replies = replies + self._error_first = error_first + self.calls: list[list[ChatMessage]] = [] + + async def chat(self, messages: list[ChatMessage], **_: Any) -> ChatResponse: + self.calls.append(messages) + if self._error_first > 0: + self._error_first -= 1 + raise RuntimeError("transient decider failure") + idx = min(len(self.calls) - 1, len(self._replies) - 1) + return ChatResponse(content=self._replies[idx], model="stub") + + +async def _embed(_q: str) -> list[float]: + return [0.1, 0.2, 0.3, 0.4] + + +async def _embed_empty(_q: str) -> list[float]: + return [] # dense recall is skipped when the query vector is falsy + + +_WHERE = "owner_id = 'alice' AND owner_type = 'user'" + + +def _reply(indices: list[int], nxt: list[str]) -> str: + return json.dumps({"core": indices, "next_queries": nxt}) + + +def _run( + recaller: _Recaller, + *, + llm: _ScriptLLM, + embed: Any = _embed, + top_k: int = 10, + decider: Any = None, +) -> Any: + return search_episodes_llm_multiround( + "q", + owner_id="alice", + where=_WHERE, + episode_recaller=recaller, + atomic_fact_recaller=_FactRecaller(), + embed_query_fn=embed, + llm=llm, + top_k=top_k, + decider=decider, + ) + + +# ── parse helpers ────────────────────────────────────────────────────────── + + +def test_parse_decision_tolerates_prose() -> None: + obj = _parse_decision('sure! {"core": [1], "next_queries": []} done') + assert obj == {"core": [1], "next_queries": []} + + +def test_parse_decision_raises_without_json() -> None: + with pytest.raises(ValueError): + _parse_decision("no json here") + + +def test_balanced_objects_outermost_first() -> None: + assert _balanced_objects('{"a": {"b": 1}} tail {"c": 2}') == [ + '{"a": {"b": 1}}', + '{"c": 2}', + ] + + +def test_coerce_core_indices_filters_and_dedups() -> None: + assert _coerce_core_indices([0, "2", 2, 9, -1, True, "x"], n_evidence=3) == [0, 2] + + +def test_dataset_from_owner() -> None: + assert _dataset_from_owner("longmemeval_42") == "longmemeval" + assert _dataset_from_owner("no_index_here") == "no_index_here" + + +# ── decider ──────────────────────────────────────────────────────────────── + + +async def test_decider_parses_reply() -> None: + llm = _ScriptLLM([_reply([0, 2], ["gap one"])]) + d = await LLMRoundDecider(llm)("q?", "(none)", "[block: original question]", 3, 0) + assert d.core == [0, 2] + assert d.queries == ["gap one"] + assert d.stop is False + assert d.raw is not None + + +async def test_decider_stop_when_no_queries() -> None: + d = await LLMRoundDecider(_ScriptLLM([_reply([0], [])]))("q", "(none)", "x", 1, 0) + assert d.stop is True and d.queries == [] + + +@pytest.fixture +def _no_backoff(monkeypatch: pytest.MonkeyPatch) -> None: + """Drop the retry backoff so retry-exhausting tests stay sub-second.""" + monkeypatch.setenv("EVEROS_LLMMR_DECIDER_BACKOFF_S", "0") + + +async def test_decider_falls_back_to_top_core_on_persistent_error( + _no_backoff: None, +) -> None: + """Exhausted retries must NOT silently yield an empty core. + + An empty core disables core-first injection for that question and is + indistinguishable downstream from a decider that legitimately chose nothing, + so the round falls back to the top-N candidates (evidence is in fused-score + order) and flags itself via ``failed``. + """ + llm = _ScriptLLM([_reply([0], [])], error_first=99) + d = await LLMRoundDecider(llm)("q", "(none)", "x", 5, 0) + assert d.stop is True and d.queries == [] + assert d.failed is True + assert d.core == list(range(llm_multiround._tuning().fallback_core)) + + +async def test_decider_fallback_core_clamped_to_candidates(_no_backoff: None) -> None: + llm = _ScriptLLM([_reply([0], [])], error_first=99) + d = await LLMRoundDecider(llm)("q", "(none)", "x", 1, 0) + assert d.core == [0] # never indexes past the candidates it was given + + +async def test_decider_retries_an_empty_completion() -> None: + """A reasoning decider can return an empty completion (budget spent on + reasoning tokens). That is a retryable failure, not a valid 'no decision'.""" + llm = _ScriptLLM(["", _reply([2], [])]) + d = await LLMRoundDecider(llm)("q", "(none)", "x", 3, 0) + assert d.core == [2] and d.failed is False + assert len(llm.calls) == 2 # empty reply consumed one attempt, then succeeded + + +async def test_decider_success_is_not_flagged_failed() -> None: + d = await LLMRoundDecider(_ScriptLLM([_reply([0], [])]))("q", "(none)", "x", 1, 0) + assert d.failed is False + + +async def test_decider_retries_then_succeeds(_no_backoff: None) -> None: + # fails _DECIDER_RETRIES times, then the final attempt parses. + llm = _ScriptLLM([_reply([1], [])], error_first=llm_multiround._tuning().retries) + d = await LLMRoundDecider(llm)("q", "(none)", "x", 3, 0) + assert d.core == [1] + assert len(llm.calls) == llm_multiround._tuning().retries + 1 + + +# ── rendering ────────────────────────────────────────────────────────────── + + +def test_render_blocks_global_index_across_blocks() -> None: + blocks = [ + ("original question", [_ep("a", 0.9), _ep("b", 0.8)]), + ("sub one", [_ep("c", 0.7)]), + ] + rendered, global_cands, index_meta = _render_blocks(blocks) + assert "[block: original question]" in rendered + assert "[block: sub one]" in rendered + # continuous global index 0,1,2 across the two blocks + assert [c.id for c in global_cands] == ["alice__a", "alice__b", "alice__c"] + assert index_meta[2] == { + "block_id": 1, + "sub_query": "sub one", + "rank_in_block": 0, + "rrf_score": 0.7, + } + assert " 2: subject c - summary c" in rendered + + +def test_render_core_so_far_tags_source() -> None: + assert _render_core_so_far([], {}, {}) == "(none)" + cand = {"alice__a": _ep("a", 0.9)} + out = _render_core_so_far(["alice__a"], {"alice__a": "sub one"}, cand) + assert out == ' - [from "sub one"] subject a - summary a' + + +# ── search loop ──────────────────────────────────────────────────────────── + + +async def test_early_stop_round0_one_llm_call() -> None: + llm = _ScriptLLM([_reply([0], [])]) # stop immediately + eps = await _run(_Recaller(seed=[_ep("a", 0.9), _ep("b", 0.5)]), llm=llm) + assert len(llm.calls) == 1 # exactly one round + assert eps[0].id == "alice__a" # core pinned first + + +async def test_subquery_expansion_reembeds_and_two_rounds() -> None: + rec = _Recaller( + seed=[_ep("a", 0.9)], + per_query={"gap two": [_ep("z", 0.6, session="s2")]}, + ) + llm = _ScriptLLM([_reply([0], ["gap two"]), _reply([0], [])]) + eps = await _run(rec, llm=llm, embed=_embed_empty) + assert len(llm.calls) == 2 # round 0 expanded, round 1 stopped + assert "gap two" in rec.sparse_queries # the sub-query was re-recalled + ids = {e.id for e in eps} + assert {"alice__a", "alice__z"} <= ids # both rounds' cores injected + + +async def test_max_rounds_cap(monkeypatch: pytest.MonkeyPatch) -> None: + # never stop (always a next query) + patience disabled -> exactly _MAX_ROUNDS. + monkeypatch.setenv("EVEROS_LLMMR_PATIENCE", "99") + llm = _ScriptLLM([_reply([0], ["again"])]) # last reply repeats forever + await _run(_Recaller(seed=[_ep("a", 0.9)]), llm=llm, embed=_embed_empty) + assert len(llm.calls) == llm_multiround._tuning().max_rounds + + +async def test_empty_seed_returns_empty_and_skips_decider() -> None: + llm = _ScriptLLM([_reply([0], [])]) + eps = await _run(_Recaller(seed=[]), llm=llm, embed=_embed_empty) + assert eps == [] + assert llm.calls == [] # decider never called on an empty seed + + +async def test_saturation_stop_after_no_new_core() -> None: + # round 0 cores, round 1 adds no new core -> patience(=1) stops after round 1. + llm = _ScriptLLM([_reply([0], ["again"]), _reply([], ["again"]), _reply([0], [])]) + rec = _Recaller(seed=[_ep("a", 0.9)]) + await _run(rec, llm=llm, embed=_embed_empty) + assert len(llm.calls) == 2 + + +# ── final injection: core-first + guarantee-then-fill ─────────────────────── + + +async def test_core_first_pins_low_scored_core() -> None: + # core the LOWEST-scored candidate; it must still land at rank 0. + seed = [_ep("hi", 0.9), _ep("mid", 0.6), _ep("lo", 0.2)] + llm = _ScriptLLM([_reply([2], [])]) # index 2 == "lo" + eps = await _run(_Recaller(seed=seed), llm=llm, embed=_embed_empty) + assert eps[0].id == "alice__lo" # low-scored core pinned front + assert {e.id for e in eps} == {"alice__hi", "alice__mid", "alice__lo"} + + +async def test_guarantee_gives_subquery_top1_a_slot() -> None: + # top_k=2, round0 cores "a"; a later sub-query's rank-0 "z" must be guaranteed + # a slot over the higher-scored non-core "b" from round 0. + rec = _Recaller( + seed=[_ep("a", 0.9), _ep("b", 0.8)], + per_query={"gap": [_ep("z", 0.4, session="s2")]}, + ) + llm = _ScriptLLM([_reply([0], ["gap"]), _reply([], [])]) + eps = await _run(rec, llm=llm, embed=_embed_empty, top_k=2) + ids = [e.id for e in eps] + assert ids[0] == "alice__a" # core first + assert "alice__z" in ids # sub-query top-1 guaranteed a slot despite low score + + +# ── trace completeness (schema C) ─────────────────────────────────────────── + +_PER_ROUND = { + "dataset", + "owner_id", + "question_id", + "question", + "round_idx", + "round_kind", + "evidence", + "core_indices", + "core_session_ids", + "core_source_subquery", + "core_added", + "core_carried_in", + "stop", + "next_queries", + "recall", + "timing_s", + "decider", +} +_EVI = { + "global_index", + "block_id", + "sub_query", + "id", + "session_id", + "subject", + "summary", + "rrf_score", + "rank_in_block", +} +_BLK = { + "sub_query", + "n_sparse", + "n_dense", + "topk_kept", + "sparse", + "dense", + "rrf_ranked", +} +_RRK = {"id", "session_id", "rrf_score", "rank", "in_sparse", "in_dense"} +_FINAL = { + "dataset", + "owner_id", + "question_id", + "question", + "round_idx", + "injected", + "assembly", +} +_INJ = { + "rank", + "id", + "session_id", + "timestamp", + "is_core", + "slot_source", + "max_rrf_score", + "max_rrf_subquery", + "per_subquery_scores", +} +_ASM = {"n_core", "n_guaranteed", "n_filled", "top_k", "subqueries_seen"} + + +async def test_trace_records_every_schema_field( + tmp_path: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + trace = tmp_path / "trace.jsonl" + monkeypatch.setenv("EVEROS_LLMMR_TRACE_DUMP", str(trace)) + rec = _Recaller( + seed=[_ep("a", 0.9, session="s5"), _ep("b", 0.7, session="s8")], + per_query={"gap": [_ep("z", 0.5, session="s2")]}, + ) + llm = _ScriptLLM([_reply([0], ["gap"]), _reply([0], [])]) + await search_episodes_llm_multiround( + "the question?", + owner_id="longmemeval_0", + where=_WHERE, + episode_recaller=rec, + atomic_fact_recaller=_FactRecaller(), + embed_query_fn=_embed_empty, + llm=llm, + top_k=20, + ) + recs = [json.loads(line) for line in trace.read_text().splitlines() if line.strip()] + per_round = [r for r in recs if r.get("round_idx") is not None] + final = [r for r in recs if r.get("round_idx") is None] + assert len(per_round) == 2 and len(final) == 1 + + for r in per_round: + assert set(r) >= _PER_ROUND, _PER_ROUND - set(r) + assert set(r["decider"]) == {"tokens", "raw"} + for e in r["evidence"]: + assert set(e) >= _EVI, _EVI - set(e) + for b in r["recall"]["blocks"]: + assert set(b) >= _BLK, _BLK - set(b) + for rr in b["rrf_ranked"]: + assert set(rr) >= _RRK, _RRK - set(rr) + + assert per_round[0]["round_kind"] == "seed" + assert per_round[1]["round_kind"] == "subquery" + assert per_round[0]["core_source_subquery"] == ["original question"] + + fin = final[0] + assert set(fin) >= _FINAL + assert set(fin["assembly"]) >= _ASM + for it in fin["injected"]: + assert set(it) >= _INJ, _INJ - set(it) + assert it["slot_source"] in {"core", "guarantee-top1", "maxscore-fill"} + # "z" was surfaced only by the sub-query "gap": max_rrf_subquery reflects it. + z = next(it for it in fin["injected"] if it["id"] == "alice__z") + assert z["max_rrf_subquery"] == "gap" + + +async def test_trace_off_writes_nothing(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("EVEROS_LLMMR_TRACE_DUMP", raising=False) + # No dump env -> the loop must run and return without touching any file. + llm = _ScriptLLM([_reply([0], [])]) + eps = await _run(_Recaller(seed=[_ep("a", 0.9)]), llm=llm, embed=_embed_empty) + assert eps and eps[0].id == "alice__a" + + +def test_decider_text_defaults_to_summary_only(monkeypatch: pytest.MonkeyPatch) -> None: + """Default is the historical summary-only view so an in-flight comparison + cannot silently change behaviour when the code is redeployed.""" + monkeypatch.setenv("EVEROS_LLMMR_DECIDER_FULL_TEXT", "0") + meta = {"summary": "short 200-char preview", "episode": "the full body " * 50} + assert llm_multiround._decider_text(meta) == "short 200-char preview" + + +def test_decider_text_full_mode_picks_the_longer_field( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Stores disagree on which column holds the body: LoCoMo / DeepSeek / Gemini + keep a 200-char prefix in ``summary`` and the body in ``episode``; the 27B and + GPT LongMemEval stores keep the body in ``summary`` with ``episode`` empty. + Picking the longer field is correct on both layouts without store detection. + """ + monkeypatch.setenv("EVEROS_LLMMR_DECIDER_FULL_TEXT", "1") + body = "the full body " * 50 + assert llm_multiround._decider_text({"summary": "prefix", "episode": body}) == body + assert llm_multiround._decider_text({"summary": body, "episode": ""}) == body + assert llm_multiround._decider_text({}) == "" + + +async def test_injection_never_exceeds_top_k_when_core_is_large() -> None: + """Core-first must honour ``top_k``. + + The guarantee and fill stages already stop at ``top_k``; core-first used to be + uncapped, so a decider that accumulated more core than the budget silently + returned more episodes than the caller asked for (measured at 20.8% of + SubtleMemory questions, up to 3.4x the budget), which breaks any same-budget + comparison between retrieval methods. + """ + seed = [_ep(f"e{i}", 1.0 - i / 100) for i in range(12)] + # Round 0 selects 8 core, round 1 selects 8 more -> 16 core for a top_k of 5. + llm = _ScriptLLM( + [_reply(list(range(8)), ["more"]), _reply([*range(8, 12), 0, 1], [])] + ) + out = await _run(_Recaller(seed=seed), llm=llm, top_k=5) + assert len(out) == 5 + + +async def test_core_overflow_env_restores_uncapped_core( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The pre-fix behaviour stays reachable so a prior run can be reproduced.""" + monkeypatch.setenv("EVEROS_LLMMR_CORE_OVERFLOW", "1") + seed = [_ep(f"e{i}", 1.0 - i / 100) for i in range(12)] + llm = _ScriptLLM([_reply(list(range(8)), [])]) + out = await _run(_Recaller(seed=seed), llm=llm, top_k=5) + assert len(out) == 8 # core kept beyond the budget, as before + + +async def test_injected_decider_drives_the_loop_instead_of_the_prompt_llm() -> None: + """An injected decider must drive the loop, and the prompt-LLM must not run. + + The ``decider`` parameter and the RoundDecider protocol exist so a Phase-2 RL policy + can BE the decider and reuse this loop verbatim as its environment. The body used to + ignore the argument and always build LLMRoundDecider, which forced an RL environment + to re-implement retrieval / block rendering / core accumulation / stop conditions -- + and re-implementing diverged: 25/25 sampled sub-queries retrieved a different + candidate set (Jaccard median 0.538) because the HTTP search route dispatches to the + hybrid hierarchy pipeline, not this file's per-sub-query rrf(sparse, dense)[:topk]. + """ + seen: list[dict[str, object]] = [] + + async def policy( + question: str, + core_so_far: str, + evidence: str, + n_candidates: int, + round_idx: int, + ) -> RoundDecision: + seen.append( + { + "round_idx": round_idx, + "n_candidates": n_candidates, + "core_so_far": core_so_far, + "has_blocks": "[block:" in evidence, + } + ) + return RoundDecision(core=[0], queries=[], stop=True) + + llm = _ScriptLLM([_reply([0], [])]) + eps = await _run(_Recaller(seed=[_ep("a", 0.9)]), llm=llm, decider=policy) + + assert seen, "the injected decider was never called" + assert llm.calls == [], "the prompt-LLM decider must not run when one is injected" + assert seen[0]["round_idx"] == 0 + assert seen[0]["core_so_far"] == "(none)", "round 0 starts with an empty core" + assert seen[0]["has_blocks"], "the decider must receive the blocked evidence view" + assert eps, "the loop must still produce an injection" diff --git a/tests/unit/test_memory/test_strategies/test_extract_user_profile_dual_trigger.py b/tests/unit/test_memory/test_strategies/test_extract_user_profile_single_path.py similarity index 100% rename from tests/unit/test_memory/test_strategies/test_extract_user_profile_dual_trigger.py rename to tests/unit/test_memory/test_strategies/test_extract_user_profile_single_path.py From 9b5cd29dc705237c9f3bdef6bbbb33d96f9c0ce5 Mon Sep 17 00:00:00 2001 From: "juwei.yue" Date: Thu, 27 Aug 2026 09:52:58 +0000 Subject: [PATCH 2/8] fix(embedding): stop retrying permanent provider rejections A 400 or 404 from the embedding endpoint was classified the same as a 503 and went through the full retry ladder, turning an unservable model name into minutes of backoff per call and, under concurrency, into an apparent hang. `_classify` now maps 400/401/403/404/413/414/422 to `EmbeddingInputError` -- a domain error the caller can act on -- and leaves 408/429/5xx and transport failures as `EmbeddingServiceError`, which is what retrying is for. --- src/everos/component/embedding/__init__.py | 5 +- .../component/embedding/openai_provider.py | 22 +- .../test_embedding_error_classification.py | 67 ++++++ .../test_llm_timeout_and_extra.py | 193 ++++++++++++++++++ 4 files changed, 285 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_component/test_embedding_error_classification.py create mode 100644 tests/unit/test_component/test_llm_timeout_and_extra.py diff --git a/src/everos/component/embedding/__init__.py b/src/everos/component/embedding/__init__.py index 9a1d4f4e9..fd52206be 100644 --- a/src/everos/component/embedding/__init__.py +++ b/src/everos/component/embedding/__init__.py @@ -4,7 +4,8 @@ Public surface: - :class:`EmbeddingProvider` — Protocol every provider satisfies. -- :class:`EmbeddingServiceError` — provider-side failure. +- :class:`EmbeddingServiceError` — provider-side failure (retryable). +- :class:`EmbeddingInputError` — the provider rejected the input (permanent). - :class:`EmbeddingError` — backward-compat alias for ``EmbeddingServiceError``. - :class:`EmbeddingCapability` — soft-dependency wrapper around an optional :class:`EmbeddingProvider` (``available`` / ``embed_or_none`` @@ -26,6 +27,7 @@ vec = await provider.embed("hello") """ +from everos.core.errors import EmbeddingInputError as EmbeddingInputError from everos.core.errors import EmbeddingServiceError as EmbeddingServiceError from .accessor import get_embedding_capability as get_embedding_capability @@ -38,6 +40,7 @@ __all__ = [ "EmbeddingCapability", "EmbeddingError", + "EmbeddingInputError", "EmbeddingProvider", "EmbeddingServiceError", "OpenAIEmbeddingProvider", diff --git a/src/everos/component/embedding/openai_provider.py b/src/everos/component/embedding/openai_provider.py index b556b94b3..b62ec5d83 100644 --- a/src/everos/component/embedding/openai_provider.py +++ b/src/everos/component/embedding/openai_provider.py @@ -24,10 +24,30 @@ import openai +from everos.core.errors import EmbeddingInputError from everos.core.observability.tracing import memory_span, set_generation_usage from .protocol import EmbeddingServiceError +# Statuses that describe the input, not the service. Retrying sends the identical +# bytes to the identical endpoint, so the answer is identical -- while the cascade +# worker holds its slot through every backoff and the rows behind it wait. +# 408 and 429 are excluded on purpose: both are 4xx and both are transient. +_PERMANENT_STATUSES = frozenset({400, 401, 403, 404, 413, 414, 422}) + + +def _classify(exc: openai.OpenAIError) -> Exception: + """Map a provider error to the retryable or the permanent branch. + + Everything used to become :class:`EmbeddingServiceError`, i.e. retryable. That is + right for a timeout, a 429 or a 5xx, and wrong for the failure actually seen in + production: episodes longer than the model's context come back 400 forever. + """ + status = getattr(exc, "status_code", None) + if status in _PERMANENT_STATUSES: + return EmbeddingInputError(f"embedding input rejected ({status}): {exc}") + return EmbeddingServiceError(str(exc)) + class OpenAIEmbeddingProvider: """OpenAI-compatible embedding provider with batching + concurrency. @@ -111,7 +131,7 @@ async def _embed_chunk(self, chunk: list[str]) -> list[list[float]]: else openai.NOT_GIVEN, ) except openai.OpenAIError as exc: - raise EmbeddingServiceError(str(exc)) from exc + raise _classify(exc) from exc if not response.data: raise EmbeddingServiceError( f"Embedding API returned empty data for {len(chunk)} inputs" diff --git a/tests/unit/test_component/test_embedding_error_classification.py b/tests/unit/test_component/test_embedding_error_classification.py new file mode 100644 index 000000000..58a3965b9 --- /dev/null +++ b/tests/unit/test_component/test_embedding_error_classification.py @@ -0,0 +1,67 @@ +"""An input the embedder will never accept must not be retried. + +Head-of-line blocking, measured: 8 rows carrying episodes longer than the embedding +model's context came back HTTP 400. Every provider error was wrapped as +``EmbeddingServiceError`` -- the retryable branch -- so the cascade worker retried each +one inline, slept its backoff while holding the worker slot, re-enqueued it across +scanner cycles, and the 220 healthy rows queued behind them waited. The retries could +not have helped: identical bytes to an identical endpoint return an identical 400. + +The split pinned here is by status, not by exception type: 4xx is not uniformly +permanent (408 and 429 are transient) and 5xx is not uniformly transient in name only. +""" + +from __future__ import annotations + +import openai +import pytest + +from everos.component.embedding.openai_provider import _classify +from everos.core.errors import ( + EmbeddingInputError, + EmbeddingServiceError, + ExternalServiceError, + InvalidInputError, +) + + +class _StatusError(openai.OpenAIError): + def __init__(self, status: int | None) -> None: + self.status_code = status + super().__init__(f"status {status}") + + +@pytest.mark.parametrize("status", [400, 401, 403, 404, 413, 414, 422]) +def test_input_rejections_are_permanent(status: int) -> None: + """The provider is telling us about the payload; sending it again says the same.""" + err = _classify(_StatusError(status)) + assert isinstance(err, EmbeddingInputError) + assert not isinstance(err, ExternalServiceError) + + +@pytest.mark.parametrize("status", [408, 429, 500, 502, 503, 504, None]) +def test_service_failures_stay_retryable(status: int | None) -> None: + """408 and 429 are 4xx and transient -- a bare `status >= 400` rule breaks both. + + ``None`` covers transport errors (connection reset, DNS), which carry no status and + are the most retryable case of all. + """ + err = _classify(_StatusError(status)) + assert isinstance(err, EmbeddingServiceError) + assert isinstance(err, ExternalServiceError) + + +def test_permanent_branch_is_not_under_external_service_error() -> None: + """This is the property the cascade worker actually switches on. + + ``worker.py`` catches ``ExternalServiceError`` for the retry path and falls through + to ``except Exception`` for permanent failure. Placing the new class anywhere under + ``ExternalServiceError`` would restore the exact bug while looking fixed. + """ + assert issubclass(EmbeddingInputError, InvalidInputError) + assert not issubclass(EmbeddingInputError, ExternalServiceError) + + +def test_status_is_named_in_the_message() -> None: + """`cascade fix` shows this string; without the code it cannot be triaged.""" + assert "413" in str(_classify(_StatusError(413))) diff --git a/tests/unit/test_component/test_llm_timeout_and_extra.py b/tests/unit/test_component/test_llm_timeout_and_extra.py new file mode 100644 index 000000000..fda452bce --- /dev/null +++ b/tests/unit/test_component/test_llm_timeout_and_extra.py @@ -0,0 +1,193 @@ +"""The extraction client's deadline and provider-specific fields are configurable. + +Regression cover for a silent misconfiguration: ``LLMSettings`` had no timeout +and no passthrough, so ``LLMConfig`` was built with three arguments and took the +algo defaults for the rest. A benchmark harness exported +``EVEROS_LLM__TIMEOUT_SECONDS=300`` for months and it did nothing, and there was +no way at all to reach a gateway's own request fields. + +That combination is what made a reasoning model unusable as an extraction +backbone. Served over an OpenAI-compatible endpoint it thinks by default, and +thinking is billed against ``max_tokens``, so an atomic-facts prompt spent the +whole budget reasoning and returned an empty ``content``. Measured through the +real client on one gateway: knob nested correctly = 12.7s / 2264 characters; +knob omitted = 42.4s / **zero** characters. Every attempt then passed the 60s +deadline, so all three retries timed out and the memory was dead-lettered -- +while the run's own logs said "timeout", pointing at the gateway rather than at +the request. Across ten servers that aborted 69 conversations with 0 successes; +with the knob in place the same fleet ran 981 extractions at 0 failures and a +2.3s mean. + +The knob's *shape* is the second trap, and the reason +:func:`test_only_extra_body_survives_the_sdk_signature` exists: everalgo merges +``extra`` into the kwargs it hands the OpenAI SDK, and the SDK rejects unknown +top-level names. Nesting under ``extra_body`` is not a style choice. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from everos.config import load_settings +from everos.config.settings import DeciderSettings, LLMSettings + +NO_THINK = '{"extra_body": {"chat_template_kwargs": {"enable_thinking": false}}}' +NO_THINK_PARSED = {"extra_body": {"chat_template_kwargs": {"enable_thinking": False}}} + + +@pytest.fixture(autouse=True) +def _clear_settings_cache() -> Any: + load_settings.cache_clear() + yield + load_settings.cache_clear() + + +def test_defaults_match_the_algo_defaults() -> None: + """Absent config, behaviour is byte-identical to before these fields existed.""" + assert LLMSettings().timeout_seconds == 60.0 + assert LLMSettings().extra == {} + assert DeciderSettings().timeout_seconds == 60.0 + assert DeciderSettings().extra == {} + + +def test_extra_defaults_are_not_shared_between_instances() -> None: + """A mutable default must not leak across settings objects.""" + first = LLMSettings() + first.extra["chat_template_kwargs"] = {"enable_thinking": False} + assert LLMSettings().extra == {} + + +@pytest.mark.parametrize("section", ["LLM", "DECIDER"]) +def test_env_supplies_timeout_and_extra( + section: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """The env spelling the harness already used has to be the one that works.""" + monkeypatch.setenv(f"EVEROS_{section}__TIMEOUT_SECONDS", "300") + monkeypatch.setenv(f"EVEROS_{section}__EXTRA", NO_THINK) + load_settings.cache_clear() + + cfg = getattr(load_settings(), section.lower()) + assert cfg.timeout_seconds == 300.0 + assert cfg.extra == NO_THINK_PARSED + + +def test_timeout_must_be_positive() -> None: + """A zero deadline fails every call; reject it at load rather than at runtime.""" + with pytest.raises(ValueError): + LLMSettings(timeout_seconds=0) + + +def test_client_passes_both_through_to_the_algo_config( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The settings must reach ``LLMConfig`` -- the step that was missing. + + Asserted against the object handed to ``build_client``, because that is the + boundary where the values were being dropped: the settings were correct and + the algo client honoured what it was given, but nothing carried one to the + other. + """ + import everos.component.llm.client as client_mod + + monkeypatch.setenv("EVEROS_LLM__API_KEY", "k") + monkeypatch.setenv("EVEROS_LLM__BASE_URL", "http://gw.invalid/v1") + monkeypatch.setenv("EVEROS_LLM__TIMEOUT_SECONDS", "300") + monkeypatch.setenv("EVEROS_LLM__EXTRA", NO_THINK) + load_settings.cache_clear() + + seen: list[Any] = [] + monkeypatch.setattr(client_mod, "_llm_client", None) + monkeypatch.setattr( + client_mod, "build_client", lambda cfg: seen.append(cfg) or object() + ) + client_mod.get_llm_client() + + (cfg,) = seen + assert cfg.timeout == 300.0 + assert cfg.extra == NO_THINK_PARSED + + +def test_decider_client_passes_both_through(monkeypatch: pytest.MonkeyPatch) -> None: + """The decider needs it too: it runs the same model inside every search.""" + import everos.component.llm.client as client_mod + + monkeypatch.setenv("EVEROS_LLM__API_KEY", "k") + monkeypatch.setenv("EVEROS_LLM__BASE_URL", "http://gw.invalid/v1") + monkeypatch.setenv("EVEROS_DECIDER__MODEL", "qwen3.8-27b") + monkeypatch.setenv("EVEROS_DECIDER__TIMEOUT_SECONDS", "120") + monkeypatch.setenv("EVEROS_DECIDER__EXTRA", NO_THINK) + load_settings.cache_clear() + + seen: list[Any] = [] + monkeypatch.setattr(client_mod, "_decider_client", None) + monkeypatch.setattr( + client_mod, "build_client", lambda cfg: seen.append(cfg) or object() + ) + client_mod.get_decider_llm_client() + + (cfg,) = seen + assert cfg.timeout == 120.0 + assert cfg.extra == NO_THINK_PARSED + + +def test_config_extra_reaches_the_request_kwargs() -> None: + """``extra`` is only worth configuring if it lands in the outgoing request. + + Calls the algo provider's own kwargs assembly rather than re-implementing + the merge here, so the assertion still means something if that merge + changes: an upstream that stopped forwarding ``config_extra`` would make + every setting above inert while all the tests above still passed. + """ + from everalgo.llm.providers.openai_compat import _build_request_kwargs + from everalgo.llm.types import ChatMessage + + kwargs = _build_request_kwargs( + messages=[ChatMessage(role="user", content="hi")], + model="qwen3.8-27b", + temperature=0.0, + max_tokens=4096, + config_extra=NO_THINK_PARSED, + extra={}, + ) + assert kwargs["extra_body"] == {"chat_template_kwargs": {"enable_thinking": False}} + + +def test_per_call_extra_overrides_the_config() -> None: + """A caller can opt back into what the config disables.""" + from everalgo.llm.providers.openai_compat import _build_request_kwargs + from everalgo.llm.types import ChatMessage + + kwargs = _build_request_kwargs( + messages=[ChatMessage(role="user", content="hi")], + model="qwen3.8-27b", + temperature=0.0, + max_tokens=None, + config_extra=NO_THINK_PARSED, + extra={"extra_body": {"chat_template_kwargs": {"enable_thinking": True}}}, + ) + assert kwargs["extra_body"] == {"chat_template_kwargs": {"enable_thinking": True}} + + +def test_only_extra_body_survives_the_sdk_signature() -> None: + """A gateway-specific name must be nested, not passed at the top level. + + ``_build_request_kwargs`` is a ``dict.update``, so it accepts any key and + cannot catch this -- the rejection happens one layer down, where the kwargs + are splatted into the SDK call. Asserting against the SDK's own signature is + what makes the nesting requirement a checked fact rather than a comment: a + top-level ``chat_template_kwargs`` raised ``TypeError`` on the first call of + a ten-server run. + """ + import inspect + + from openai.resources.chat.completions import AsyncCompletions + + params = inspect.signature(AsyncCompletions.create).parameters + assert not any(p.kind is p.VAR_KEYWORD for p in params.values()), ( + "SDK grew **kwargs; unknown names would now pass silently and this " + "config's nesting requirement needs rechecking" + ) + assert "extra_body" in params + assert "chat_template_kwargs" not in params From 0f91c19afe97714248fcc7f6783dd8b1162c7c56 Mon Sep 17 00:00:00 2001 From: "juwei.yue" Date: Thu, 27 Aug 2026 09:53:18 +0000 Subject: [PATCH 3/8] fix(runtime): bound OME runs, make the pool explicit, allow LLM extras Three defects that only appear under a long benchmark run. An OME strategy with no timeout could hold its slot indefinitely; a stalled extraction then starved every later run behind it. Runs now carry `run_timeout` and raise `TimeoutError`, which the dispatcher records, rather than being cancelled with no trace. The SQLite engine leaked a connection per thread -- the signature was the pool growing from 6 to 58 and the connector wedging. The pool is now explicit and sized, so exhaustion fails loudly instead of degrading. The LLM client had no way to pass provider-specific fields, so the one setting a thinking model needs to answer within a timeout could not be configured at all. `extra` is now plumbed through. --- src/everos/component/llm/__init__.py | 2 + src/everos/component/llm/client.py | 52 ++++++ src/everos/core/errors.py | 17 ++ .../core/persistence/markdown/frontmatter.py | 11 +- src/everos/core/persistence/sqlite/engine.py | 68 +++++++- src/everos/entrypoints/api/app.py | 5 + .../entrypoints/api/lifespans/cascade.py | 13 ++ src/everos/entrypoints/api/lifespans/ome.py | 13 ++ src/everos/entrypoints/api/routes/cascade.py | 121 +++++++++++++ src/everos/infra/ome/_dispatch/runner.py | 13 +- src/everos/infra/ome/config.py | 67 ++++++- .../unit/test_core/test_sqlite_pool_config.py | 164 ++++++++++++++++++ .../test_lifespans/test_readonly_switches.py | 109 ++++++++++++ .../test_routes/test_cascade_quiesce_route.py | 117 +++++++++++++ 14 files changed, 767 insertions(+), 5 deletions(-) create mode 100644 src/everos/entrypoints/api/routes/cascade.py create mode 100644 tests/unit/test_core/test_sqlite_pool_config.py create mode 100644 tests/unit/test_entrypoints/test_api/test_lifespans/test_readonly_switches.py create mode 100644 tests/unit/test_entrypoints/test_api/test_routes/test_cascade_quiesce_route.py diff --git a/src/everos/component/llm/__init__.py b/src/everos/component/llm/__init__.py index c82baf5e9..b6ddad7df 100644 --- a/src/everos/component/llm/__init__.py +++ b/src/everos/component/llm/__init__.py @@ -21,6 +21,7 @@ """ from .client import LLMNotConfiguredError as LLMNotConfiguredError +from .client import get_decider_llm_client as get_decider_llm_client from .client import get_llm_client as get_llm_client from .client import get_multimodal_llm_client as get_multimodal_llm_client from .factory import build_llm_provider as build_llm_provider @@ -40,6 +41,7 @@ "OpenAIProvider", "Usage", "build_llm_provider", + "get_decider_llm_client", "get_llm_client", "get_multimodal_llm_client", ] diff --git a/src/everos/component/llm/client.py b/src/everos/component/llm/client.py index 6d48804fb..e73952300 100644 --- a/src/everos/component/llm/client.py +++ b/src/everos/component/llm/client.py @@ -74,6 +74,7 @@ class LLMNotConfiguredError(RuntimeError): _llm_client: LLMClient | None = None _multimodal_client: LLMClient | None = None +_decider_client: LLMClient | None = None def get_llm_client() -> LLMClient: @@ -101,6 +102,8 @@ def get_llm_client() -> LLMClient: model=llm_cfg.model, api_key=api_key, base_url=llm_cfg.base_url, + timeout=llm_cfg.timeout_seconds, + extra=dict(llm_cfg.extra), ) ) # Wrap for OTel token capture only when tracing is on — keeps the @@ -116,6 +119,55 @@ def get_llm_client() -> LLMClient: return _llm_client +def get_decider_llm_client() -> LLMClient: + """Return the singleton retrieval-decider client. + + Falls back to :func:`get_llm_client` when ``[decider]`` names no model, so a config + that predates the section keeps its current behaviour exactly -- the decider then is + the extraction model, as it always was. + + Raises: + LLMNotConfiguredError: When ``[decider]`` sets a model but leaves ``api_key`` or + ``base_url`` unset, and no usable ``[llm]`` fallback exists. + """ + global _decider_client + if _decider_client is not None: + return _decider_client + settings = load_settings() + cfg = settings.decider + if not cfg.model: + return get_llm_client() + api_key = cfg.api_key.get_secret_value() if cfg.api_key is not None else None + # A local server needs no real key; fall back to the main section's credentials so + # only `model` has to be set to point the decider at a different hosted model. + if not api_key or not cfg.base_url: + llm_cfg = settings.llm + api_key = api_key or ( + llm_cfg.api_key.get_secret_value() if llm_cfg.api_key is not None else None + ) + base_url = cfg.base_url or llm_cfg.base_url + else: + base_url = cfg.base_url + if not api_key or not base_url: + raise LLMNotConfiguredError( + missing_config_error("decider api_key and base_url", "decider") + ) + client: LLMClient = build_client( + LLMConfig( + model=cfg.model, + api_key=api_key, + base_url=base_url, + timeout=cfg.timeout_seconds, + extra=dict(cfg.extra), + ) + ) + if settings.observability.enabled: + client = UsageRecordingClient(client) + _decider_client = _LoggingLLMClient(client) + logger.info("decider_client_built", model=cfg.model, base_url=base_url) + return _decider_client + + def get_multimodal_llm_client() -> LLMClient: """Return the singleton multimodal LLM client (for everalgo.parser). diff --git a/src/everos/core/errors.py b/src/everos/core/errors.py index bd6687b80..f1217970d 100644 --- a/src/everos/core/errors.py +++ b/src/everos/core/errors.py @@ -89,6 +89,23 @@ class FilterError(InvalidInputError): """A caller-supplied filter expression is invalid or malformed.""" +class EmbeddingInputError(InvalidInputError): + """The embedding provider rejected the input itself, not the request. + + Deliberately in the domain branch rather than under + :class:`EmbeddingServiceError`: the cascade worker retries the whole + ``ExternalServiceError`` branch, and a text the provider refuses -- over the + model's token limit, empty, malformed -- is refused identically every time. + Retrying it burns the row's budget and, because the worker holds its slot + across the backoff, blocks the rows queued behind it. Measured once at + 8 unembeddable rows stalling 220 good ones. + + A row that raises this is marked permanently failed and surfaces in + ``cascade fix``, which is the correct destination: the md has to change + before the embedding can ever succeed. + """ + + class PathTraversalError(DomainError): """A write target resolved outside the configured memory root. diff --git a/src/everos/core/persistence/markdown/frontmatter.py b/src/everos/core/persistence/markdown/frontmatter.py index 97eb77f91..349d7f723 100644 --- a/src/everos/core/persistence/markdown/frontmatter.py +++ b/src/everos/core/persistence/markdown/frontmatter.py @@ -348,15 +348,24 @@ class ProfilePathMixin: class UserProfileFrontmatter(ProfilePathMixin, UserScopedFrontmatter): PROFILE_FILENAME: ClassVar[str] = "user.md" ... + + A kind that keeps several files of one schema under the same scope + directory declares ``PROFILE_GLOB`` too, because the cascade scanner + globs each kind exactly once and ``*`` does not span ``/``. Both files + must still sit directly under ``/``: a subdirectory would need + an extra path component the single glob cannot express. """ PROFILE_FILENAME: ClassVar[str] + PROFILE_GLOB: ClassVar[str | None] = None + """Filename glob covering every file of this kind; ``None`` means the + kind has exactly one file and ``PROFILE_FILENAME`` is the pattern.""" SCOPE_DIR: ClassVar[str] @classmethod def path_glob(cls) -> str: # Leading ``*/*/`` matches the / scope prefix. - return f"*/*/{cls.SCOPE_DIR}/*/{cls.PROFILE_FILENAME}" + return f"*/*/{cls.SCOPE_DIR}/*/{cls.PROFILE_GLOB or cls.PROFILE_FILENAME}" class UserScopedFrontmatter(BaseFrontmatter): diff --git a/src/everos/core/persistence/sqlite/engine.py b/src/everos/core/persistence/sqlite/engine.py index 1076c6b79..c1ce15155 100644 --- a/src/everos/core/persistence/sqlite/engine.py +++ b/src/everos/core/persistence/sqlite/engine.py @@ -14,6 +14,9 @@ from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine from everos.config import SqliteSettings +from everos.core.observability.logging import get_logger + +logger = get_logger(__name__) def create_system_engine( @@ -42,12 +45,75 @@ def create_system_engine( # Three slashes = relative path; four slashes = absolute. ``str(db_path)`` # of an absolute Path begins with ``/`` so the f-string yields four. url = f"sqlite+aiosqlite:///{db_path}" - engine = create_async_engine(url, echo=echo, future=True) + # Pool parameters are passed explicitly rather than inherited. They were + # inherited before, and the failure that exposed it is not one the defaults + # can survive: a connection checked out and never returned leaves the pool + # one slot smaller forever, and once every slot is gone each later caller + # waits on a checkout that no longer completes. Two benchmark servers reached + # that state -- aiosqlite connection threads at 20 and 58 against a steady + # 6-10 on their healthy siblings, 7 and 22 of them parked inside aiosqlite's + # connect path -- and every SQLite file simply stopped being written, for + # 2h17m and 3h33m, until they were killed by hand. + # + # What made it expensive was that nothing looked broken. The process answered + # HTTP, the event loop was live, every thread was idle, and CPU was flat. The + # OME queue stopped draining because strategies could not persist their own + # results, so `run_record` rows froze mid-flight in RUNNING -- which reads as + # "still working", not "cannot write". Even the run-timeout backstop was mute: + # it fired, then needed a connection to record the failure. + # + # `pool_pre_ping` and `pool_recycle` reclaim such a connection at its next + # checkout, and `pool_timeout` bounds the wait so exhaustion surfaces as a + # retryable error instead of a silent stall. None of this fixes whatever + # leaks the connection; it stops one leak from taking the process with it. + engine = create_async_engine( + url, + echo=echo, + future=True, + pool_size=sqlite_settings.pool_size, + max_overflow=sqlite_settings.max_overflow, + pool_timeout=sqlite_settings.pool_timeout_seconds, + pool_recycle=sqlite_settings.pool_recycle_seconds, + pool_pre_ping=sqlite_settings.pool_pre_ping, + ) _register_pragma_listener(engine, sqlite_settings) + _register_pool_saturation_listener(engine, sqlite_settings) return engine +def _register_pool_saturation_listener( + engine: AsyncEngine, + sqlite_settings: SqliteSettings, +) -> None: + """Log once per checkout that finds the pool at or near capacity. + + The point is a signal that exists at all. When the pool drained on two + benchmark servers there was nothing to see: no error, no log line, no metric + -- writes simply stopped, and diagnosis came down to counting aiosqlite + threads in a py-spy dump against a healthy sibling process. A warning at the + moment of saturation names the condition while the process is still running, + and its ``checked_out`` count is what distinguishes real concurrency from a + leak: honest load returns connections, so the number oscillates; a leak only + climbs. + """ + capacity = sqlite_settings.pool_size + sqlite_settings.max_overflow + + @event.listens_for(engine.sync_engine, "checkout") + def _warn_when_saturated(_dbapi_conn, _rec, _proxy) -> None: # type: ignore[no-untyped-def] + pool = engine.sync_engine.pool + checked_out = getattr(pool, "checkedout", lambda: -1)() + if checked_out >= capacity: + logger.warning( + "sqlite.pool.saturated", + checked_out=checked_out, + capacity=capacity, + pool_size=sqlite_settings.pool_size, + max_overflow=sqlite_settings.max_overflow, + pool_timeout_seconds=sqlite_settings.pool_timeout_seconds, + ) + + def _register_pragma_listener( engine: AsyncEngine, sqlite_settings: SqliteSettings, diff --git a/src/everos/entrypoints/api/app.py b/src/everos/entrypoints/api/app.py index c174abac1..e28dd7102 100644 --- a/src/everos/entrypoints/api/app.py +++ b/src/everos/entrypoints/api/app.py @@ -38,6 +38,7 @@ SqliteLifespanProvider, ) from .routes import ( + cascade, get, health, knowledge, @@ -139,12 +140,16 @@ def create_app( app.include_router(get.router, prefix="/api/v1") app.include_router(ome.router, prefix="/api/v1") app.include_router(knowledge.router, prefix="/api/v1") + # Operational, not a memory API: it stops a background subsystem. Mounted on + # both prefixes so a caller does not have to know which one this build serves. + app.include_router(cascade.router, prefix="/api/v1") # v2 — cloud-aligned name, same routers. app.include_router(memorize.router, prefix="/api/v2") app.include_router(search.router, prefix="/api/v2") app.include_router(get.router, prefix="/api/v2") app.include_router(ome.router, prefix="/api/v2") app.include_router(knowledge.router, prefix="/api/v2") + app.include_router(cascade.router, prefix="/api/v2") logger.info("app_created", docs_enabled=enable_docs) return app diff --git a/src/everos/entrypoints/api/lifespans/cascade.py b/src/everos/entrypoints/api/lifespans/cascade.py index fc6528920..cbbb8ece0 100644 --- a/src/everos/entrypoints/api/lifespans/cascade.py +++ b/src/everos/entrypoints/api/lifespans/cascade.py @@ -14,6 +14,7 @@ from __future__ import annotations +import os from typing import Any from fastapi import FastAPI @@ -36,6 +37,18 @@ def __init__(self, order: int = 12) -> None: self._orchestrator: CascadeOrchestrator | None = None async def startup(self, app: FastAPI) -> Any: + # A read-only retrieval server does not ingest markdown, so the cascade + # subsystem (watcher / scanner / worker) is pure overhead -- and its periodic + # scan re-enqueues a large store's whole markdown set, which starves search on a + # dense store badly enough to hold it at zero. Off by default (unset), so an + # ingesting daemon is unaffected. + if os.getenv("EVEROS_DISABLE_CASCADE", "").strip().lower() in ( + "1", + "true", + "yes", + ): + logger.info("cascade_lifespan_disabled_by_env") + return None memory_root = MemoryRoot.resolve() memory_root.ensure() diff --git a/src/everos/entrypoints/api/lifespans/ome.py b/src/everos/entrypoints/api/lifespans/ome.py index 4df6cad5a..418fe12c6 100644 --- a/src/everos/entrypoints/api/lifespans/ome.py +++ b/src/everos/entrypoints/api/lifespans/ome.py @@ -9,6 +9,7 @@ from __future__ import annotations import importlib +import os from typing import Any from fastapi import FastAPI @@ -26,6 +27,18 @@ def __init__(self, order: int = 50) -> None: super().__init__(name="ome", order=order) async def startup(self, app: FastAPI) -> Any: + # A read-only retrieval server does not extract, so the OfflineEngine -- and its + # exclusive per-store lock -- is pure overhead. The lock also stops a second + # server from sharing one pre-built store root, which is how a parallel-lane + # evaluation is run. Off by default (unset), so an ingesting daemon that needs + # extraction is unaffected. Mirrors ``EVEROS_DISABLE_CASCADE``. + if os.getenv("EVEROS_DISABLE_OME", "").strip().lower() in ( + "1", + "true", + "yes", + ): + logger.info("ome_lifespan_disabled_by_env") + return None svc = importlib.import_module("everos.service.memorize") engine = svc._get_engine() await engine.start() diff --git a/src/everos/entrypoints/api/routes/cascade.py b/src/everos/entrypoints/api/routes/cascade.py new file mode 100644 index 000000000..7b5d8bf95 --- /dev/null +++ b/src/everos/entrypoints/api/routes/cascade.py @@ -0,0 +1,121 @@ +"""Cascade control — freeze the md → LanceDB projection into a snapshot. + +``POST /api/v1/cascade/quiesce`` drains the projection queue and then stops the +cascade subsystem, so the index stops changing until the process restarts. + +Why a control endpoint exists at all. Cascade keeps LanceDB in step with the +markdown that is the actual source of truth, and paying for that means rewriting +files: ``merge_insert`` lands each write in a fresh fragment, ``optimize()`` +merges the accumulated fragments into one, and ``prune()`` then physically +reclaims the superseded copies -- with a 60s retention window, because the +storage soak measured a 300s window retaining ~24 full-table copies. + +Prune's safety argument is that 60s "comfortably" outlives an in-flight read, +which it documents as "sub-second to a few seconds". A read that takes **longer +than the window** breaks it: the reader resolves a version, works, and comes back +to files that were reclaimed underneath it -- + + LanceError(IO): Object at location .../_indices//tokens.lance not found + +That is not hypothetical. A multi-round retrieval whose decider reads full +episode text spends 56-72s inside one search call (measured), and a benchmark +running ingest and retrieval in the same process lost 45% of one dataset's +questions to exactly this before the endpoint existed: HTTP 500 per search, an +empty context handed to the answer model, and a scored zero. + +The fix is not a longer window -- that only moves the race. It is to notice that +a read-only phase creates no new fragments, so there is nothing for optimize or +prune to do, and the whole subsystem is pure risk. ``EVEROS_DISABLE_CASCADE`` +already covers a process that starts read-only; this covers the process that +writes first and *becomes* read-only, which an env var read once at startup +cannot express. + +Quiesce is deliberately one-way. Restart the process to get the projection back. +""" + +from __future__ import annotations + +from fastapi import APIRouter, HTTPException, Request +from pydantic import BaseModel + +from everos.core.observability.logging import get_logger +from everos.entrypoints.api.utils import cascade_orchestrator + +logger = get_logger(__name__) + +router = APIRouter(prefix="/cascade", tags=["cascade"]) + + +class QuiesceResponse(BaseModel): + """What the final drain accomplished, and what it left behind. + + ``pending_before`` is the queue depth on arrival: a caller that expected the + projection to be up to date can assert it was 0 and learn otherwise. Anything + above 0 in ``pending_after`` means the drain could not finish -- the index is + NOT a complete projection of the markdown, and searching it will silently + under-recall. + """ + + quiesced: bool + """False when the subsystem was already stopped or never started; the call + is idempotent, so this is information rather than an error.""" + drained: int + """Rows the final scan + drain cycle processed.""" + pending_before: int + pending_after: int + failed_permanent: int + """Files that need ``cascade fix``; a data-quality backlog, not a drain + failure. Reported so a caller can see that some markdown never made it.""" + + +@router.post("/quiesce", response_model=QuiesceResponse) +async def quiesce(request: Request) -> QuiesceResponse: + """Drain the projection queue, then stop watcher + scanner + worker. + + Returns when the index is a complete projection of the markdown on disk and + nothing further will rewrite it. That ordering is the point: stopping first + would freeze a *partial* index, which reads as a healthy store that quietly + fails to recall whatever had not been indexed yet. + + Idempotent -- a second call reports ``quiesced=false`` rather than failing. + """ + orch = cascade_orchestrator(request) + if orch is None: + raise HTTPException( + status_code=503, + detail=( + "cascade subsystem is not running; nothing to quiesce " + "(disabled via EVEROS_DISABLE_CASCADE, or this app was built " + "without the cascade lifespan)" + ), + ) + + before = await orch.queue_summary() + # Drain BEFORE stopping. `sync_once` is a full scan + drain, so it also picks + # up markdown the watcher never saw -- which matters here because the watcher + # is the half most likely to be off (inotify watches are a per-user kernel + # resource a shared host can exhaust). + drained = await orch.sync_once() + await orch.stop() + after = await orch.queue_summary() + + logger.info( + "cascade_quiesced", + drained=drained, + pending_before=before.pending, + pending_after=after.pending, + failed_permanent=after.failed_permanent, + ) + if after.pending: + logger.warning( + "cascade_quiesce_left_pending", + pending=after.pending, + reason="index is not a complete projection of the markdown", + ) + return QuiesceResponse( + quiesced=True, + drained=drained, + pending_before=before.pending, + pending_after=after.pending, + failed_permanent=after.failed_permanent, + ) diff --git a/src/everos/infra/ome/_dispatch/runner.py b/src/everos/infra/ome/_dispatch/runner.py index ac71c3393..310a8d5d3 100644 --- a/src/everos/infra/ome/_dispatch/runner.py +++ b/src/everos/infra/ome/_dispatch/runner.py @@ -237,7 +237,18 @@ async def _run_one_attempt( }, ), ): - await meta.func(event, ctx) + # Timeout INSIDE the span so a killed attempt is still + # attributed to the strategy that hung. TimeoutError is an + # Exception, so it lands in the retry / dead-letter handler + # below like any other failure -- the slot is released, the + # record leaves RUNNING, and the work is retried rather than + # silently abandoned. + timeout = self._config.run_timeout_seconds + if timeout is None: + await meta.func(event, ctx) + else: + async with asyncio.timeout(timeout): + await meta.func(event, ctx) finally: _CURRENT_STRATEGY.reset(token) except StrategyContractError as e: diff --git a/src/everos/infra/ome/config.py b/src/everos/infra/ome/config.py index 7786a3efc..658b4153d 100644 --- a/src/everos/infra/ome/config.py +++ b/src/everos/infra/ome/config.py @@ -6,6 +6,7 @@ from __future__ import annotations +import os from pathlib import Path from typing import Annotated, Self @@ -65,6 +66,40 @@ def _check_idle_pair_consistency(self) -> Self: return self +def _env_int(name: str, default: int) -> int: + """Read a positive int from the environment, ignoring anything unparseable. + + A malformed value must not take the process down at import time; the field's own + ``gt=0`` still rejects a parsed non-positive number. + """ + raw = os.getenv(name, "").strip() + if not raw: + return default + try: + return int(raw) + except ValueError: + return default + + +def _env_float(name: str, default: float | None) -> float | None: + """Read a timeout (seconds) from the environment; ``"0"`` / ``"off"`` disable it. + + Same tolerance as :func:`_env_int` -- a malformed value falls back to the + default rather than failing at import. An explicit zero or ``off`` maps to + ``None`` so an operator can turn the ceiling off without editing code. + """ + raw = os.getenv(name, "").strip().lower() + if not raw: + return default + if raw in {"0", "off", "none", "false"}: + return None + try: + parsed = float(raw) + except ValueError: + return default + return parsed if parsed > 0 else None + + class TomlRoot(BaseModel): """Top-level TOML schema for ome.toml.""" @@ -95,11 +130,39 @@ class OMEConfig(BaseModel): max_concurrent_runs: Annotated[ int, Field( + default_factory=lambda: _env_int("EVEROS_OME_MAX_CONCURRENT_RUNS", 20), gt=0, description="Engine-wide cap on concurrent strategy invocations " - "(asyncio.Semaphore in Runner).", + "(asyncio.Semaphore in Runner). Override with " + "EVEROS_OME_MAX_CONCURRENT_RUNS. The default suits an interactive " + "install; a bulk ingest of many conversations is throttled by it long " + "before the machine is, since each slot spends almost all its time " + "waiting on a remote extraction call rather than using CPU.", + ), + ] + run_timeout_seconds: Annotated[ + float | None, + Field( + default_factory=lambda: _env_float( + "EVEROS_OME_RUN_TIMEOUT_SECONDS", 1800.0 + ), + description="Wall-clock ceiling on one strategy attempt; None " + "disables it. Override with EVEROS_OME_RUN_TIMEOUT_SECONDS " + "(0/off disables). Without a ceiling, a coroutine parked on an " + "await that carries no deadline of its own -- an asyncio.Lock held " + "by another stuck coroutine, a connection-pool wait -- keeps its " + "max_concurrent_runs slot forever: it never raises, so it never " + "retries, and its record stays RUNNING. Enough of them and the " + "engine runs nothing at all, observed with 60 of 64 slots parked on " + "one lock, starving every other strategy for 6.7 hours until it was " + "killed by hand. crash_recovery_timeout_seconds does not cover this " + "-- it reclaims orphans from a PREVIOUS process, not live coroutines " + "in this one. The default is deliberately generous (30 min against a " + "measured worst case near 7 min for a 38-subject profile pass): a " + "deadlock backstop, not a latency target, since killing slow-but-" + "healthy work would trade a stall for lost extractions.", ), - ] = 20 + ] max_retries: Annotated[ int, Field( diff --git a/tests/unit/test_core/test_sqlite_pool_config.py b/tests/unit/test_core/test_sqlite_pool_config.py new file mode 100644 index 000000000..5292ca981 --- /dev/null +++ b/tests/unit/test_core/test_sqlite_pool_config.py @@ -0,0 +1,164 @@ +"""The SQLite engine's connection pool is configured, not inherited. + +Regression cover for a stall that took two benchmark servers down for 2h17m and +3h33m. ``create_async_engine`` was called with only ``url``/``echo``/``future``, +so pool behaviour came from library defaults with no timeout, no recycle and no +pre-ping. A connection checked out and never returned shrinks the pool by one +permanently; once every slot is gone, each later caller waits on a checkout that +never completes. + +The symptom is why this needs pinning rather than a comment. Nothing looked +broken: HTTP answered, the event loop ran, every thread was idle, CPU was flat. +Only the write side had stopped -- SQLite file mtimes frozen, ``run_record`` rows +stuck in RUNNING because strategies could not persist their own results, which +reads as "still working" rather than "cannot write". Even the OME run-timeout +backstop was silent: it fired, then needed a connection to record the failure. +Measured signature: aiosqlite connection threads at 20 and 58 versus a steady +6-10 on healthy siblings, with 7 and 22 parked inside aiosqlite's connect path. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from sqlalchemy.pool import AsyncAdaptedQueuePool + +from everos.config.settings import SqliteSettings +from everos.core.persistence.sqlite.engine import create_system_engine + + +@pytest.fixture +def db(tmp_path: Path) -> Path: + return tmp_path / "system.db" + + +def test_defaults_are_explicit_and_bounded() -> None: + """Every knob that governs a stuck checkout has a value we chose.""" + s = SqliteSettings() + assert s.pool_size == 5 + assert s.max_overflow == 10 + assert s.pool_timeout_seconds == 30.0 + assert s.pool_recycle_seconds == 1800 + assert s.pool_pre_ping is True + + +def test_the_engine_actually_receives_them(db: Path) -> None: + """Settings that never reach the engine are decoration. + + Asserted on the live pool object rather than the call arguments: this is the + exact link that was missing before, and reading it back from the engine is + the only way to know it closed. + """ + s = SqliteSettings( + pool_size=3, max_overflow=2, pool_timeout_seconds=7.0, pool_recycle_seconds=60 + ) + engine = create_system_engine(db, s) + pool = engine.sync_engine.pool + assert isinstance(pool, AsyncAdaptedQueuePool) + assert pool.size() == 3 + assert pool._max_overflow == 2 + assert pool._timeout == 7.0 + assert pool._recycle == 60 + assert pool._pre_ping is True + + +def test_a_bounded_wait_is_what_turns_a_hang_into_an_error(db: Path) -> None: + """Exhaustion must raise, not block forever. + + The stall was not that the pool ran dry -- pools do -- but that running dry + had no deadline, so the failure never surfaced anywhere a caller or an + operator could see it. + """ + engine = create_system_engine(db, SqliteSettings(pool_timeout_seconds=0.25)) + assert engine.sync_engine.pool._timeout == 0.25 + + +def test_timeout_must_be_positive() -> None: + """Zero would mean "fail instantly"; negative would mean "wait forever".""" + with pytest.raises(ValueError): + SqliteSettings(pool_timeout_seconds=0) + with pytest.raises(ValueError): + SqliteSettings(pool_timeout_seconds=-1) + + +def test_recycle_can_be_disabled_but_not_arbitrary(db: Path) -> None: + """``-1`` is SQLAlchemy's "never recycle"; keep it reachable, reject below.""" + assert SqliteSettings(pool_recycle_seconds=-1).pool_recycle_seconds == -1 + with pytest.raises(ValueError): + SqliteSettings(pool_recycle_seconds=-2) + + +async def test_pragmas_still_apply_after_the_pool_change(db: Path) -> None: + """The pool rework must not displace the per-connection PRAGMA listener. + + Both listeners hang off the same sync engine, and registering the second is + exactly the kind of edit that quietly drops the first. Verified by reading a + PRAGMA back off a live connection rather than by inspecting the registry: + what matters is that WAL is actually on, not that a callable is attached. + """ + from sqlalchemy import text + + engine = create_system_engine(db, SqliteSettings()) + async with engine.connect() as conn: + mode = (await conn.execute(text("PRAGMA journal_mode"))).scalar() + busy = (await conn.execute(text("PRAGMA busy_timeout"))).scalar() + await engine.dispose() + assert str(mode).upper() == "WAL" + assert busy == SqliteSettings().busy_timeout_ms + + +async def test_saturation_is_logged_so_the_condition_is_visible( + db: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A drained pool must announce itself. + + What made the original failure expensive was the absence of any signal: the + condition had to be reconstructed afterwards by counting threads in a py-spy + dump against a healthy sibling. ``checked_out`` in the log line is the + discriminator -- honest load returns connections so it oscillates, a leak + only climbs. + """ + from sqlalchemy import text + + import everos.core.persistence.sqlite.engine as mod + + seen: list[dict[str, object]] = [] + monkeypatch.setattr( + mod.logger, "warning", lambda _evt, **kw: seen.append(kw), raising=True + ) + + engine = create_system_engine(db, SqliteSettings(pool_size=1, max_overflow=0)) + # Capacity 1: hold the only connection, then force a second checkout of it. + async with engine.connect() as c1: + await c1.execute(text("SELECT 1")) + async with engine.connect() as c2: + await c2.execute(text("SELECT 1")) + await engine.dispose() + + assert seen, "a pool at capacity produced no warning" + assert seen[0]["capacity"] == 1 + assert seen[0]["checked_out"] >= 1 + + +async def test_a_healthy_pool_stays_quiet( + db: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """No warning below capacity -- otherwise the signal is noise. + + A line that fires on every checkout would have been ignored, which is how the + original failure stayed invisible in the first place. + """ + from sqlalchemy import text + + import everos.core.persistence.sqlite.engine as mod + + seen: list[dict[str, object]] = [] + monkeypatch.setattr( + mod.logger, "warning", lambda _evt, **kw: seen.append(kw), raising=True + ) + engine = create_system_engine(db, SqliteSettings(pool_size=5, max_overflow=10)) + async with engine.connect() as conn: + await conn.execute(text("SELECT 1")) + await engine.dispose() + assert seen == [] diff --git a/tests/unit/test_entrypoints/test_api/test_lifespans/test_readonly_switches.py b/tests/unit/test_entrypoints/test_api/test_lifespans/test_readonly_switches.py new file mode 100644 index 000000000..65e4cb8a4 --- /dev/null +++ b/tests/unit/test_entrypoints/test_api/test_lifespans/test_readonly_switches.py @@ -0,0 +1,109 @@ +"""The two switches that let a server run read-only. + +Both subsystems are write-side, and on a retrieval-only server both actively hurt rather +than merely idle: cascade's periodic scan re-enqueues a whole store's markdown, which +starved search on a dense store badly enough to hold it at zero; the OME engine holds an +exclusive per-store lock, which stops a second server from sharing one pre-built store +root -- the shape a parallel-lane evaluation needs. + +Off by default, so an ingesting daemon is unaffected. That default is the part worth +pinning hardest: a switch that defaults to "on" would silently stop extraction. +""" + +from __future__ import annotations + +from typing import Any + +import pytest +from fastapi import FastAPI + +from everos.entrypoints.api.lifespans.cascade import CascadeLifespanProvider +from everos.entrypoints.api.lifespans.ome import OmeLifespanProvider + +TRUTHY = ["1", "true", "TRUE", "yes", "Yes"] +FALSY = ["", " ", "0", "false", "no", "off", "maybe"] + + +@pytest.mark.parametrize("value", TRUTHY) +async def test_cascade_startup_is_skipped_when_disabled( + value: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("EVEROS_DISABLE_CASCADE", value) + provider = CascadeLifespanProvider() + assert await provider.startup(FastAPI()) is None + assert provider._orchestrator is None + + +@pytest.mark.parametrize("value", TRUTHY) +async def test_ome_startup_is_skipped_when_disabled( + value: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("EVEROS_DISABLE_OME", value) + provider = OmeLifespanProvider() + assert await provider.startup(FastAPI()) is None + + +@pytest.mark.parametrize("value", FALSY) +async def test_an_unrecognised_value_does_not_disable_cascade( + value: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """Anything that is not clearly "yes" must leave ingestion working. + + Reaching startup proves the gate did not fire; what it does after that needs a real + store, so the call is expected to get further and fail on that instead. + """ + monkeypatch.setenv("EVEROS_DISABLE_CASCADE", value) + provider = CascadeLifespanProvider() + reached: dict[str, Any] = {} + + def _boom() -> Any: + reached["past_the_gate"] = True + raise RuntimeError("stop here") + + monkeypatch.setattr( + "everos.entrypoints.api.lifespans.cascade.MemoryRoot.resolve", _boom + ) + with pytest.raises(RuntimeError, match="stop here"): + await provider.startup(FastAPI()) + assert reached.get("past_the_gate") is True + + +@pytest.mark.parametrize("value", FALSY) +async def test_an_unrecognised_value_does_not_disable_ome( + value: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("EVEROS_DISABLE_OME", value) + provider = OmeLifespanProvider() + reached: dict[str, Any] = {} + + def _boom(*_a: Any, **_k: Any) -> Any: + reached["past_the_gate"] = True + raise RuntimeError("stop here") + + monkeypatch.setattr("importlib.import_module", _boom) + with pytest.raises(RuntimeError, match="stop here"): + await provider.startup(FastAPI()) + assert reached.get("past_the_gate") is True + + +async def test_unset_leaves_both_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + """The default. A run that ingests must not have to know these exist.""" + monkeypatch.delenv("EVEROS_DISABLE_CASCADE", raising=False) + monkeypatch.delenv("EVEROS_DISABLE_OME", raising=False) + + cascade = CascadeLifespanProvider() + monkeypatch.setattr( + "everos.entrypoints.api.lifespans.cascade.MemoryRoot.resolve", + lambda: (_ for _ in ()).throw(RuntimeError("cascade gate open")), + ) + with pytest.raises(RuntimeError, match="cascade gate open"): + await cascade.startup(FastAPI()) + + # And OME, which the first version of this test named but never constructed. + ome = OmeLifespanProvider() + monkeypatch.setattr( + "importlib.import_module", + lambda *a, **k: (_ for _ in ()).throw(RuntimeError("ome gate open")), + ) + with pytest.raises(RuntimeError, match="ome gate open"): + await ome.startup(FastAPI()) diff --git a/tests/unit/test_entrypoints/test_api/test_routes/test_cascade_quiesce_route.py b/tests/unit/test_entrypoints/test_api/test_routes/test_cascade_quiesce_route.py new file mode 100644 index 000000000..4a945d279 --- /dev/null +++ b/tests/unit/test_entrypoints/test_api/test_routes/test_cascade_quiesce_route.py @@ -0,0 +1,117 @@ +"""``POST /cascade/quiesce`` — freeze the projection before the read stages. + +Cascade rewrites files to keep LanceDB in step with the markdown that is the real +source of truth: ``merge_insert`` lands each write in a fresh fragment, +``optimize()`` merges the accumulation, and ``prune()`` reclaims the superseded +copies after a 60s retention window (short on purpose -- the storage soak measured +a 300s window retaining ~24 full-table copies). + +Prune's safety argument is that 60s outlives an in-flight read, which it documents +as "sub-second to a few seconds". A multi-round retrieval whose decider reads full +episode text spends 56-72s in one search call (measured), so it does not: the +reader resolves a version, works, and comes back to reclaimed files -- + + LanceError(IO): Object at location .../_indices//tokens.lance not found + +The first full LoCoMo run lost 225 of 493 questions to that: HTTP 500 per search, +an empty context handed to the answer model, a scored zero. These tests pin the +endpoint that removes the race -- drain first (else the frozen index is an +incomplete projection), then stop (else prune keeps running). +""" + +from __future__ import annotations + +import unittest.mock as mock +from dataclasses import dataclass + +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient + +from everos.entrypoints.api.routes.cascade import router as cascade_router +from everos.memory.cascade import CascadeOrchestrator + + +@dataclass +class _Summary: + """Shape of ``md_change_state_repo.queue_summary()``.""" + + pending: int = 0 + done: int = 0 + failed_retryable: int = 0 + failed_permanent: int = 0 + + +def _orch(*, before: int, after: int, drained: int) -> mock.MagicMock: + """An autospec orchestrator — ``isinstance(_, CascadeOrchestrator)`` holds.""" + orch = mock.create_autospec(CascadeOrchestrator, instance=True) + orch.queue_summary.side_effect = [ + _Summary(pending=before), + _Summary(pending=after, failed_permanent=2), + ] + orch.sync_once.return_value = drained + return orch + + +def _client(orch: object | None) -> AsyncClient: + app = FastAPI() + app.include_router(cascade_router, prefix="/api/v1") + app.state.lifespan_data = {"cascade": orch} if orch is not None else {} + return AsyncClient(transport=ASGITransport(app=app), base_url="http://test") + + +async def test_drains_before_stopping() -> None: + """The order IS the contract: stopping first freezes a partial index, which + then reads as a healthy store that quietly under-recalls.""" + orch = _orch(before=7, after=0, drained=7) + order: list[str] = [] + orch.sync_once.side_effect = lambda **_: order.append("drain") or 7 + orch.stop.side_effect = lambda: order.append("stop") + + async with _client(orch) as c: + resp = await c.post("/api/v1/cascade/quiesce") + + assert resp.status_code == 200, resp.text + assert order == ["drain", "stop"], "stopped before draining" + + +async def test_reports_the_queue_on_both_sides() -> None: + """``pending_before`` is how a caller discovers the projection was behind -- + which the benchmark's ``add.done`` marker does not tell it, because that + marker tracks the OME extraction queue, not this one.""" + async with _client(_orch(before=42, after=0, drained=42)) as c: + resp = await c.post("/api/v1/cascade/quiesce") + + body = resp.json() + assert body["quiesced"] is True + assert body["pending_before"] == 42 + assert body["pending_after"] == 0 + assert body["drained"] == 42 + assert body["failed_permanent"] == 2 + + +async def test_leftover_pending_is_surfaced_not_swallowed() -> None: + """A drain that could not finish means the index is NOT a full projection of + the markdown; a caller that reads it anyway scores the gap as a miss.""" + async with _client(_orch(before=10, after=3, drained=7)) as c: + resp = await c.post("/api/v1/cascade/quiesce") + + assert resp.status_code == 200 + assert resp.json()["pending_after"] == 3 + + +async def test_no_cascade_is_503_not_a_silent_success() -> None: + """A read-only server has nothing to quiesce. Answering 200 would let the + caller believe it froze a projection that was never running.""" + async with _client(None) as c: + resp = await c.post("/api/v1/cascade/quiesce") + + assert resp.status_code == 503 + assert "quiesce" in resp.text.lower() + + +async def test_stop_is_called_exactly_once() -> None: + """Quiesce is one-way; a restart is what brings the projection back.""" + orch = _orch(before=1, after=0, drained=1) + async with _client(orch) as c: + await c.post("/api/v1/cascade/quiesce") + assert orch.stop.await_count == 1 From cc31dc47251ee088cbe48089e1eb9af9a436a9c6 Mon Sep 17 00:00:00 2001 From: "juwei.yue" Date: Thu, 27 Aug 2026 09:53:18 +0000 Subject: [PATCH 4/8] refactor(memory): decouple profile extraction from clustering Profile extraction listened on two triggers, one of which fired from the clustering path, so a store built without clusters silently produced no profiles and a store with them produced duplicates. It now listens on `EpisodeExtracted` alone. The clustering path itself is untouched -- `agentic` retrieval needs it, and removing it is what left four rebuilt stores with zero clusters and an `agentic` route that early-returned an empty set without erroring. The profile lock is now per subject rather than global, so two subjects no longer serialise behind each other. --- src/everos/config/default.toml | 38 + .../lancedb/tables/user_profile.py | 9 +- .../infra/persistence/markdown/mds/profile.py | 15 +- .../markdown/readers/profile_reader.py | 12 +- .../markdown/writers/profile_writer.py | 24 +- .../memory/cascade/handlers/user_profile.py | 29 +- src/everos/memory/cascade/orchestrator.py | 25 +- src/everos/memory/events.py | 15 - .../memory/strategies/extract_user_profile.py | 791 +++++++++++++----- .../strategies/trigger_profile_clustering.py | 14 +- src/everos/service/search.py | 24 +- .../test_ome_strategies_integration.py | 16 +- .../test_cascade/test_orchestrator.py | 56 +- .../unit/test_memory/test_ome_run_timeout.py | 103 +++ .../test_extract_user_profile.py | 118 +-- .../test_extract_user_profile_single_path.py | 348 ++------ .../test_profile_lock_granularity.py | 115 +++ .../test_profile_subject_sender.py | 375 +++++++++ .../test_trigger_profile_clustering.py | 19 +- 19 files changed, 1533 insertions(+), 613 deletions(-) create mode 100644 tests/unit/test_memory/test_ome_run_timeout.py create mode 100644 tests/unit/test_memory/test_strategies/test_profile_lock_granularity.py create mode 100644 tests/unit/test_memory/test_strategies/test_profile_subject_sender.py diff --git a/src/everos/config/default.toml b/src/everos/config/default.toml index e2137f84a..9b00bfba9 100644 --- a/src/everos/config/default.toml +++ b/src/everos/config/default.toml @@ -58,6 +58,44 @@ model = "openai/gpt-4.1-mini" api_key = "" base_url = "https://openrouter.ai/api/v1" +# LLM for the multi-round retrieval decider. Separate from [llm] because the jobs +# differ: [llm] extracts memories during ingestion, this one decides -- inside the search +# request, round by round -- which episodes are core and what to query next. An empty +# model falls back to [llm], which is how every store built before this section existed +# was scored. api_key / base_url are also inherited from [llm] when left blank, so +# pointing the decider at another hosted model needs only `model`; set base_url to run it +# on a local vLLM / SGLang endpoint instead. +[decider] +# LLM for the multi-round retrieval decider, when it differs from [llm]. Left +# empty the decider runs [llm]'s model, which is what a single-model deployment +# wants. Override via env: EVEROS_DECIDER__MODEL / __API_KEY / __BASE_URL / +# __TIMEOUT_SECONDS / __EXTRA. +# +# ⚠ model and base_url must move together. Renaming the model while leaving the +# endpoint pointed at a server that does not host it returns 404 on every call, +# and the retrieval loop does NOT fail on that -- it falls back to a fixed +# top-ranked core and reports a complete result. See the fallback note in +# everos.memory.search.llm_multiround. +model = "" +api_key = "" +base_url = "" + +# Multi-round loop tuning. Every one of these is a real field, so `everos config` +# prints it and a TOML can set it; the legacy EVEROS_LLMMR_* env names still take +# precedence when exported, so launch scripts keep working unchanged. +max_rounds = 3 # rounds before the loop stops regardless +seed_topk = 50 # candidates the original question contributes +subq_topk = 20 # candidates each follow-up sub-query contributes +max_subqueries = 3 # follow-ups the decider may ask for per round +rrf_k = 60 # RRF smoothing when fusing sparse + dense +no_new_core_patience = 1 # stop after this many rounds add no new core +per_subquery_guarantee = 1 # slots reserved per sub-query in the final fill +retries = 3 # decider retries before falling back +retry_backoff_seconds = 0.5 # doubled per attempt +core_overflow = false # true = let core exceed top_k (pre-2026-08-06) +full_text = false # true = decider reads full episode text, not summary +fallback_core = 3 # core size when every decider attempt fails + [multimodal] # Independent LLM for multimodal parsing (everalgo-parser); must accept # image / pdf / audio image_url parts. Override via env: diff --git a/src/everos/infra/persistence/lancedb/tables/user_profile.py b/src/everos/infra/persistence/lancedb/tables/user_profile.py index 87654b9d7..96098c10e 100644 --- a/src/everos/infra/persistence/lancedb/tables/user_profile.py +++ b/src/everos/infra/persistence/lancedb/tables/user_profile.py @@ -28,7 +28,14 @@ class UserProfile(BaseLanceTable): # No BM25 columns: profile recall is KV-by-owner today. id: str - """PK = ``owner_id`` (one row per user).""" + """PK. ``owner_id`` when the owner is its own subject, else + ``::``. + + The subject rides the PK rather than a column of its own on purpose: a new + column is schema drift, and the guard in + :mod:`everos.infra.persistence.lancedb` refuses to open every store built + before it. Readers recover the subject by stripping the ``owner_id`` prefix + — unambiguous because ``owner_id`` is path-safe and cannot contain ``:``.""" owner_id: str owner_type: str diff --git a/src/everos/infra/persistence/markdown/mds/profile.py b/src/everos/infra/persistence/markdown/mds/profile.py index 283dfb2cd..00b2278bf 100644 --- a/src/everos/infra/persistence/markdown/mds/profile.py +++ b/src/everos/infra/persistence/markdown/mds/profile.py @@ -1,6 +1,8 @@ """UserProfile frontmatter — single-file profile markdown for users. -Path: ``users//user.md``. +Paths: ``users//user.md`` when the owner is the subject, and +``users//user..md`` (one per participant) when the owner is a +group. Both carry this schema; ``subject`` distinguishes them. Carries the LLM-synthesised user profile: a free-form ``summary`` plus the two evidence buckets emitted by :class:`everalgo.user_memory.ProfileExtractor` @@ -21,9 +23,20 @@ class UserProfileFrontmatter(ProfilePathMixin, UserScopedFrontmatter): """Frontmatter for ``users//user.md``.""" PROFILE_FILENAME: ClassVar[str] = "user.md" + PROFILE_GLOB: ClassVar[str | None] = "user*.md" + """Covers ``user.md`` and every ``user..md`` participant file. + ``behaviors.md`` is a different kind, so the prefix is unambiguous.""" type: Literal["user_profile"] = "user_profile" + subject: str = "" + """Who this profile describes, when that is not ``user_id`` itself. + + Empty (the default) is the owner-is-subject shape at + ``users//user.md``. A group owner writes one file per + participant at ``users//user..md`` and names the + participant here; ``user_id`` stays the retrieval partition.""" + summary: str = "" """Free-form one-paragraph summary of the user — the retrieval anchor.""" diff --git a/src/everos/infra/persistence/markdown/readers/profile_reader.py b/src/everos/infra/persistence/markdown/readers/profile_reader.py index 90cb00dbd..cbe0e6d9a 100644 --- a/src/everos/infra/persistence/markdown/readers/profile_reader.py +++ b/src/everos/infra/persistence/markdown/readers/profile_reader.py @@ -37,6 +37,7 @@ async def read( schema: type[T], app_id: str = "default", project_id: str = "default", + filename: str | None = None, ) -> tuple[T, str] | None: """Read the profile file and parse its frontmatter into ``schema``. @@ -47,13 +48,16 @@ async def read( ``SCOPE_DIR`` (via scope mixin) and ``PROFILE_FILENAME``. app_id: App scope segment (defaults to the ``"default"`` space). project_id: Project scope segment (defaults to ``"default"``). + filename: Overrides the schema's ``PROFILE_FILENAME``; mirrors + :meth:`ProfileWriter.write` so a group owner can read back + one participant's file under ``/profiles/``. Returns: ``(frontmatter, body)`` on success; ``None`` if the file is missing. ``body`` is the raw text after the closing ``---`` with the writer-added trailing newline stripped. """ - path = self._resolve_path(scope_id, schema, app_id, project_id) + path = self._resolve_path(scope_id, schema, app_id, project_id, filename) if not await anyio.Path(path).is_file(): return None parsed = await MarkdownReader.read(path) @@ -68,9 +72,10 @@ def path_for( schema: type[BaseFrontmatter], app_id: str = "default", project_id: str = "default", + filename: str | None = None, ) -> Path: """Return the profile path (no IO check).""" - return self._resolve_path(scope_id, schema, app_id, project_id) + return self._resolve_path(scope_id, schema, app_id, project_id, filename) # ── Internals — same shape as ProfileWriter ─────────────────────────── @@ -80,9 +85,10 @@ def _resolve_path( schema: type[BaseFrontmatter], app_id: str, project_id: str, + filename: str | None = None, ) -> Path: scope_dir = getattr(schema, "SCOPE_DIR", "") - filename = getattr(schema, "PROFILE_FILENAME", None) + filename = filename or getattr(schema, "PROFILE_FILENAME", None) if not scope_dir: raise TypeError( f"{schema.__name__} missing ``SCOPE_DIR`` ClassVar — " diff --git a/src/everos/infra/persistence/markdown/writers/profile_writer.py b/src/everos/infra/persistence/markdown/writers/profile_writer.py index 76e8d00c7..42e101d93 100644 --- a/src/everos/infra/persistence/markdown/writers/profile_writer.py +++ b/src/everos/infra/persistence/markdown/writers/profile_writer.py @@ -5,6 +5,7 @@ filename under the agent or user directory:: users//user.md ← user profile + users//user..md ← one participant of a group owner users//behaviors.md ← user behaviour patterns agents//agent.md ← agent playbook agents//soul.md ← agent identity / values @@ -15,9 +16,10 @@ profile writer is the simplest of the three: - **Upsert, not append.** Each ``write`` overwrites the file in full. -- **Fixed path.** Caller passes ``scope_id`` only — no ``name`` - parameter; the filename is fixed by the schema's - ``PROFILE_FILENAME`` ClassVar. +- **Fixed path.** The filename comes from the schema's + ``PROFILE_FILENAME`` ClassVar; callers that hold several profiles under + one scope (a group owner, one file per participant) override it with + the ``filename`` argument. - **No business hooks.** No frontmatter merging, no entry-id generation. The caller hands in a fully-built schema instance. @@ -62,6 +64,7 @@ async def write( body: str, app_id: str = "default", project_id: str = "default", + filename: str | None = None, ) -> Path: """Upsert ``////``. @@ -73,11 +76,18 @@ async def write( body: Profile body text. Trailing newline is normalised. app_id: App scope segment (defaults to the ``"default"`` space). project_id: Project scope segment (defaults to ``"default"``). + filename: Overrides the schema's ``PROFILE_FILENAME`` — how a + group owner keeps one file per participant + (``user..md``) beside its own ``user.md``. Must stay a + single path component so the kind's ``PROFILE_GLOB`` still + finds it. Returns: Absolute path of the written profile file. """ - path = self._resolve_path(scope_id, type(frontmatter), app_id, project_id) + path = self._resolve_path( + scope_id, type(frontmatter), app_id, project_id, filename + ) head_meta = frontmatter.model_dump(exclude_none=False) return await self._writer.write_markdown( path, @@ -92,9 +102,10 @@ def path_for( schema: type[BaseFrontmatter], app_id: str = "default", project_id: str = "default", + filename: str | None = None, ) -> Path: """Return the profile path (no IO check).""" - return self._resolve_path(scope_id, schema, app_id, project_id) + return self._resolve_path(scope_id, schema, app_id, project_id, filename) # ── Internals ───────────────────────────────────────────────────────── @@ -104,9 +115,10 @@ def _resolve_path( schema: type[BaseFrontmatter], app_id: str, project_id: str, + filename: str | None = None, ) -> Path: scope_dir = getattr(schema, "SCOPE_DIR", "") - filename = getattr(schema, "PROFILE_FILENAME", None) + filename = filename or getattr(schema, "PROFILE_FILENAME", None) if not scope_dir: raise TypeError( f"{schema.__name__} missing ``SCOPE_DIR`` ClassVar — " diff --git a/src/everos/memory/cascade/handlers/user_profile.py b/src/everos/memory/cascade/handlers/user_profile.py index 4842986d0..e153853d0 100644 --- a/src/everos/memory/cascade/handlers/user_profile.py +++ b/src/everos/memory/cascade/handlers/user_profile.py @@ -1,14 +1,19 @@ """UserProfile cascade handler — md → LanceDB ``user_profile`` table. -Profile is a single-file kind (mirrors AgentSkill): one -``users//user.md`` per user, replaced wholesale on edit. No -entry markers, no per-entry diff. The LanceDB row carries the typed -projection of the frontmatter so a future query-aware lookup can run -off LanceDB; today the recaller is KV-by-owner. +Profile is a single-file kind (mirrors AgentSkill): each file is replaced +wholesale on edit, with no entry markers and no per-entry diff. The LanceDB +row carries the typed projection of the frontmatter so a future query-aware +lookup can run off LanceDB; today the recaller is KV-by-owner. + +Two layouts share this handler. ``users//user.md`` is the +owner-is-subject shape (``subject`` empty, row id = ``owner_id``), and +``users//user..md`` is the group shape (one file per +participant, row id = ``::``). The handler keys off the +frontmatter's ``subject``, never the path. md contract: -- frontmatter: :class:`UserProfileFrontmatter` (``user_id`` / +- frontmatter: :class:`UserProfileFrontmatter` (``user_id`` / ``subject`` / ``summary`` / ``explicit_info`` / ``implicit_traits`` / ``profile_timestamp_ms``). - body: free-form display text (not indexed; the structured payload @@ -36,7 +41,7 @@ class UserProfileHandler(Handler): - """Cascade handler for ``users//user.md``.""" + """Cascade handler for user-profile md (both layouts above).""" kind = "user_profile" lance_repo: ClassVar[Any] = user_profile_repo @@ -67,6 +72,14 @@ async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome: ) app_id, project_id = resolve_scope(md_path) + # Empty subject = owner-is-subject, the only shape a single-person + # owner can have. A group owner ships one file per participant and + # names the participant here; ``owner_id`` stays the partition, so the + # row id carries the subject or the files overwrite each other. It is + # the id and not a column because a new column is schema drift that + # locks every pre-existing store out of startup. + subject = str(fm.get("subject", "")) + summary = str(fm.get("summary", "")) explicit_info_json = _dump_json(fm.get("explicit_info", [])) implicit_traits_json = _dump_json(fm.get("implicit_traits", [])) @@ -80,7 +93,7 @@ async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome: } ) - row_id = owner_id + row_id = f"{owner_id}::{subject}" if subject else owner_id prior = await user_profile_repo.get_by_id(row_id) if prior is not None and prior.content_sha256 == digest: return HandlerOutcome( diff --git a/src/everos/memory/cascade/orchestrator.py b/src/everos/memory/cascade/orchestrator.py index 7b3cc241d..794fec2d8 100644 --- a/src/everos/memory/cascade/orchestrator.py +++ b/src/everos/memory/cascade/orchestrator.py @@ -16,6 +16,7 @@ import asyncio import dataclasses +import os from everos.component.tokenizer import Tokenizer from everos.config import load_settings @@ -159,15 +160,33 @@ async def start(self) -> None: ``processing`` at boot is leftover from a prior crash that ``claim_pending_batch`` can't re-claim on its own (the WHERE filter is ``status='pending'``). + + ``EVEROS_DISABLE_CASCADE_WATCHER`` skips the inotify half. The scanner + already re-derives the same truth from the filesystem every + ``scan_interval`` seconds, so the only thing lost is latency: an md edit + is picked up within one scan instead of immediately. That trade is + forced on a shared host, where inotify watches are a per-user kernel + resource -- an IDE indexing the workspace can hold 350k of the 524k + ceiling by itself, and every EverOS server then dies at startup with + ``OSError: [Errno 28] inotify watch limit reached``. Unlike + ``EVEROS_DISABLE_CASCADE`` this keeps the worker, so md still reaches + LanceDB -- which an ingesting run cannot do without. """ if self._started: return orphans = await md_change_state_repo.recover_orphan_processing() if orphans: logger.info("cascade_recovered_orphan_processing", count=orphans) - loop = asyncio.get_running_loop() - self._watcher = CascadeWatcher(self._memory_root, loop) - self._watcher.start() + if os.getenv("EVEROS_DISABLE_CASCADE_WATCHER", "").strip().lower() in ( + "1", + "true", + "yes", + ): + logger.info("cascade_watcher_disabled_by_env") + else: + loop = asyncio.get_running_loop() + self._watcher = CascadeWatcher(self._memory_root, loop) + self._watcher.start() await self._scanner.start() await self._worker.start() self._started = True diff --git a/src/everos/memory/events.py b/src/everos/memory/events.py index 6a58972d2..77b88b010 100644 --- a/src/everos/memory/events.py +++ b/src/everos/memory/events.py @@ -92,21 +92,6 @@ class AgentCaseExtracted(BaseEvent): project_id: str = "default" -class ProfileClusterUpdated(BaseEvent): - """Fired after the user-memory cluster strategy has merged a new - memcell into a cluster. - - Drives the profile-extraction strategy; ``cluster_id`` is the new - or merged cluster the source memcell now belongs to. - """ - - memcell_id: str - cluster_id: str - owner_id: str - app_id: str = "default" - project_id: str = "default" - - class SkillClusterUpdated(BaseEvent): """Fired after the agent-case cluster strategy has merged a new case into a cluster. diff --git a/src/everos/memory/strategies/extract_user_profile.py b/src/everos/memory/strategies/extract_user_profile.py index 711eca096..5f25bf392 100644 --- a/src/everos/memory/strategies/extract_user_profile.py +++ b/src/everos/memory/strategies/extract_user_profile.py @@ -1,57 +1,67 @@ -"""extract_user_profile strategy — synthesise the user's profile from clusters. - -Dual-trigger strategy: profile extraction must run whether or not embedding -is configured, so it listens on **two** events and gates itself so exactly -one path fires per memcell (see :func:`_profile_applies`): - -- **Cluster path** (Tier 2+, embedding available): fires on - :class:`ProfileClusterUpdated`, emitted by ``trigger_profile_clustering`` - after it assigns a memcell to a cluster. Pulls the relevant memcells - across all "fresh" clusters (:func:`_select_via_cluster`). -- **Direct path** (Tier 1, no embedding): fires on :class:`EpisodeExtracted` - directly. ``trigger_profile_clustering`` is always registered, but its - per-dispatch body-guard returns early when - :func:`get_embedding_capability` reports unavailable, so no - ``ProfileClusterUpdated`` is ever emitted in Tier 1 — this direct - path is the sole route to profile extraction there. Pulls memcells - via a scalar LanceDB timestamp filter, no embedding required - (:func:`_select_via_timestamp`). - -Both paths converge on the same LLM extraction + markdown persist tail of -``extract_user_profile`` — only memcell *selection* differs. - -Opensource parity (``mem_memorize.py`` Phase 2): - -- **Throttle** (both paths): ``total_count % profile_extraction_interval - == 0``; default interval = 1 (every memcell triggers a re-extraction). - Applied at strategy entry so cluster and direct paths honor the same - gate. ``total_count`` is ``sum(c.count for c in user_clusters)`` on the - cluster path and :meth:`episode_repo.count_by_owner` on the direct - path — both express "cumulative units of source-memory for this owner" - and stay ~1:1 in the extract → memcell → episode → cluster pipeline. -- **Target clusters**: every cluster whose ``last_ts`` is newer than the - user's existing profile timestamp, plus the current cluster (so the - freshly-arrived memcell is always counted even when its cluster's - ``last_ts`` is older than the profile baseline). -- **Input shape**: raw chat messages — algo's ``_render_conversation`` - unwraps the items list. The sqlite ``memcell.payload_json`` column is - the long-term archive that lets us replay this beyond - ``unprocessed_buffer``'s lifetime. - -Single-sender assumption today: ``event.owner_id`` is treated as the -profile subject. Multi-user clusters land their additional sender's -profile in a follow-up turn (each cluster gets re-evaluated on every -``ProfileClusterUpdated`` for any participating user). +"""extract_user_profile strategy — synthesise a profile from the memcell that landed. + +Single trigger: :class:`EpisodeExtracted` with ``source == "pipeline"``, one dispatch +per extracted episode. The profile is a function of the memcell that just arrived and +nothing else, which is the same contract ``extract_episode`` and +``extract_atomic_facts`` already have. + +It used to have a second, cluster-driven path. ``trigger_profile_clustering`` emitted +:class:`ProfileClusterUpdated` per episode, and this strategy selected "every member of +every cluster fresher than the profile". Three things were wrong with it: + +- **Re-reading.** A cluster stays fresh for as long as it keeps receiving memcells, and + a single-project corpus funnels nearly everything into one: measured on EverMemBench + topic 01, 2 of 122 clusters held 441 of 595 members, the largest 321. Every + extraction therefore re-sent hundreds of already-merged memcells -- 8.7x the + necessary volume -- while the UPDATE prompt was already carrying the full current + profile those memcells had been merged into. +- **Cost for nothing.** Reaching the same set cost one embedding call, one read of the + owner's entire cluster list (122-251 rows), and one LanceDB fetch of every fresh + cluster's members, most of which were then discarded. +- **Tier-dependent output.** The cluster path only ran when embedding was available, so + the same data produced a different profile depending on tier. + +Clustering itself still runs: ``agentic`` retrieval and Reflection both read +``cluster_repo``. It simply no longer gates the profile. + +Throttle: ``total_count % PROFILE_EXTRACTION_INTERVAL == 0`` over the owner's +memcell-parented episode count. ``EVEROS_PROFILE_EXTRACTION_INTERVAL=1`` (the default) +means every memcell updates the profile; the counter query is skipped entirely at that +value. + +Input shape: raw chat messages -- algo's ``_render_conversation`` unwraps the items +list. The sqlite ``memcell.payload_json`` column is the long-term archive that lets +this replay beyond ``unprocessed_buffer``'s lifetime. + +Who the profile is about is :data:`PROFILE_SUBJECT`. The default (``owner``) treats +``event.owner_id`` as the subject, which holds whenever an owner is one person: the +ingest fans each Episode out to every ``sender_id`` in the memcell, so a two-person +dialogue already produces one owner -- and one profile -- per participant. + +It stops holding when many people deliberately share one owner, which is how a group +chat keeps retrieval in a single partition. There ``owner`` hands the extractor every +speaker's turns under one name and gets a composite of nobody; ``sender`` writes one +profile per real speaker instead. """ from __future__ import annotations -from everalgo.clustering import Cluster as AlgoCluster +import contextlib +import json +import os +import re +import time +from typing import Any + +import anyio +from everalgo.types import ChatMessage as AlgoChatMessage from everalgo.types import MemCell as AlgoMemCell from everalgo.types import Profile as AlgoProfile from everalgo.user_memory import ProfileExtractor +from everalgo.user_memory.prompts.en.profile import ( + PROFILE_INITIAL_EXTRACTION_PROMPT, +) -from everos.component.embedding import get_embedding_capability from everos.component.llm import get_llm_client from everos.core.observability.logging import get_logger from everos.core.persistence import MemoryRoot @@ -65,21 +75,61 @@ ProfileWriter, UserProfileFrontmatter, ) -from everos.infra.persistence.sqlite import cluster_repo, memcell_repo +from everos.infra.persistence.sqlite import memcell_repo from everos.memory._partition_locks import get_partition_lock -from everos.memory.events import EpisodeExtracted, ProfileClusterUpdated +from everos.memory.events import EpisodeExtracted logger = get_logger(__name__) -PROFILE_EXTRACTION_INTERVAL = 1 +PROFILE_EXTRACTION_INTERVAL = int(os.getenv("EVEROS_PROFILE_EXTRACTION_INTERVAL", "1")) """Opensource parity: re-extract on every Nth clustered memcell. -``N=1`` matches the opensource default; tune via :class:`Settings` once -the storage budget for profile re-extractions becomes a concern.""" + +``N=1`` matches the opensource default and stays the default here. It means the profile +is rewritten once per extracted episode, and the body of this strategy is a read → LLM +merge → overwrite: ingesting 1240 episodes for one owner spends 1240 merge calls, and +the row keeps changing until the last one lands. A run that reads the profile while +ingest is still finishing therefore sees a different profile per question -- measured on +one conversation: 140 distinct versions across 269 searches, with the summary ranging +from 165 to 2445 characters. + +The env var is how a benchmark run raises it without moving the library default away +from +opensource parity.""" PROFILE_MIN_MEMCELLS = 1 """Opensource parity: skip when the candidate cluster set holds fewer than ``N`` memcells across all selected clusters.""" +SUBJECT_OWNER = "owner" +SUBJECT_SENDER = "sender" +PROFILE_SUBJECT = os.getenv("EVEROS_PROFILE_SUBJECT", SUBJECT_OWNER) +"""Who a profile describes: the owner (default) or each real speaker. + +``owner`` treats ``event.owner_id`` as the profile subject. That is correct +whenever the owner is one person -- the ingest fans an Episode out to every +``sender_id`` in a memcell, so a two-person dialogue already yields one owner +(and one profile) per participant. + +``sender`` treats the owner as a **group**: episodes stay in the owner's +partition, and one profile is written per distinct speaker found in the +memcells. This is the only correct shape when many people share an owner -- +otherwise the extractor is handed N people's turns and told they are one +person, and it dutifully synthesises a composite of somebody who does not +exist. Costs one LLM call per speaker per extraction, so raise +``EVEROS_PROFILE_EXTRACTION_INTERVAL`` alongside it.""" + +_SUBJECT_UNSAFE = re.compile(r"[^A-Za-z0-9._-]+") + +PROFILE_TRACE_ENV = "EVEROS_PROFILE_TRACE_DUMP" +"""Env var naming a JSONL path; unset disables the dump. + +Separate from ``EVEROS_LLMMR_TRACE_DUMP`` on purpose. That one is the retrieval +trace, written per server from a single-threaded event loop, so its appends cannot +interleave. Profile extraction is an OME strategy running concurrently across owners, +so its records would interleave into the retrieval file and be unreadable. It also +answers a different question: retrieval trace explains WHICH episodes reached the +answer, this one explains WHERE a profile's content came from.""" + _writer: ProfileWriter | None = None _reader: ProfileReader | None = None @@ -100,69 +150,14 @@ def _get_reader() -> ProfileReader: def _profile_applies(event: BaseEvent) -> bool: - """Route exactly one path per memcell to ``extract_user_profile``. - - - :class:`ProfileClusterUpdated` always applies: it is only ever - emitted by ``trigger_profile_clustering``, whose per-dispatch - body-guard short-circuits (returns before emit) whenever - :func:`get_embedding_capability` reports unavailable. Accepting - this event unconditionally is therefore safe — it can only arrive - when embedding is available, so it never overlaps with the direct - path below. - - :class:`EpisodeExtracted` applies only for pipeline-sourced - episodes (not Reflection's merged episodes) while embedding is - unavailable. Once embedding is available, ``trigger_profile_clustering`` - runs to completion and owns the same memcell via - ``ProfileClusterUpdated`` — so the direct path must stand down - here to avoid a double extraction. This check is a live capability - read (not a registration-time snapshot), so a Tier 3→1 downgrade - mid-process is picked up on the next event. - """ - if isinstance(event, ProfileClusterUpdated): - return True - if isinstance(event, EpisodeExtracted): - return event.source == "pipeline" and not get_embedding_capability().available - return False - - -async def _select_via_cluster( - event: ProfileClusterUpdated, - last_profile_ts: int, - user_clusters: list[AlgoCluster], -) -> list[str]: - """Cluster path (Tier 2+): resolve memcells via the user's cluster set. + """One dispatch per pipeline-extracted episode. Reflection's merged episodes + (``source != "pipeline"``) are excluded: their source memcells were already + merged into the profile when they first arrived. - The caller (:func:`extract_user_profile`) already fetched - ``user_clusters`` for the strategy-entry throttle; we take it as a - parameter to avoid a redundant :meth:`cluster_repo.list_for_owner` - round-trip. Behavior of the selection step itself is unchanged from - the pre-dual-trigger implementation. + No embedding-capability read and no second event: the profile is a function of + the memcell that just landed, exactly like episode and atomic_fact extraction. """ - # Pick clusters fresher than the existing profile (always include - # the one we just updated). - target_clusters = [ - c - for c in user_clusters - if c.last_ts > last_profile_ts or c.id == event.cluster_id - ] - if not target_clusters: - return [] - - # Resolve cluster members (episode entry_ids) → memcell_ids. - # Cluster members store episode entry_ids. To reach the memcell - # payloads we look up each episode's parent_id (= memcell_id) - # in LanceDB, skipping merged episodes (parent_type=cluster) - # whose source memcells were already processed before Reflection. - entry_ids = [m for c in target_clusters for m in c.members] - episodes = await episode_repo.find_by_owner_entries( - event.owner_id, - entry_ids, - app_id=event.app_id, - project_id=event.project_id, - ) - return [ - ep.parent_id for ep in episodes if ep.parent_type == "memcell" and ep.parent_id - ] + return isinstance(event, EpisodeExtracted) and event.source == "pipeline" async def _select_via_timestamp( @@ -211,88 +206,72 @@ async def _select_via_timestamp( @offline_strategy( name="extract_user_profile", - trigger=Immediate(on=[ProfileClusterUpdated, EpisodeExtracted]), + trigger=Immediate(on=[EpisodeExtracted]), applies_to=_profile_applies, emits=[], max_retries=2, ) -async def extract_user_profile( - event: ProfileClusterUpdated | EpisodeExtracted, ctx: StrategyContext -) -> None: +async def extract_user_profile(event: EpisodeExtracted, ctx: StrategyContext) -> None: # Serialise on owner_id: user.md is a single per-user file and the # body is a read → LLM merge → overwrite sequence. Different users # run fully in parallel. + # + # Per-sender mode takes the lock per subject inside the loop instead. Each + # subject owns its own file, so an owner-wide lock buys no extra safety and + # costs the whole group: it serialises every task for the owner across all N + # subjects, so one slow subject blocks the other N-1 AND every queued task. + # Measured on a 38-speaker owner: 60 of the 64 OME slots ended up parked on + # this one lock waiting for a single subject, which starved every other + # strategy on the process and stalled the run for 6.7 hours. partition = f"{event.app_id}:{event.project_id}:{event.owner_id}" - async with get_partition_lock("extract_user_profile", partition): - existing = await _get_reader().read( - event.owner_id, - schema=UserProfileFrontmatter, - app_id=event.app_id, - project_id=event.project_id, - ) - last_profile_ts = existing[0].profile_timestamp_ms if existing else 0 - - # Unified throttle: both paths gate on "cumulative units of - # source-memory for this owner". Cluster path sums cluster - # counts (pre-refactor semantics); direct path counts owner - # episode rows in LanceDB. Applied here so a bumped interval - # (e.g. 10x to cap LLM cost) throttles both Tier 1 and Tier 2+ - # equally instead of only the cluster path. - user_clusters: list[AlgoCluster] | None - if isinstance(event, ProfileClusterUpdated): - user_clusters = await cluster_repo.list_for_owner( + guard: contextlib.AbstractAsyncContextManager[Any] = ( + contextlib.nullcontext() + if PROFILE_SUBJECT == SUBJECT_SENDER + else get_partition_lock("extract_user_profile", partition) + ) + async with guard: + if PROFILE_SUBJECT == SUBJECT_SENDER: + # A group owner has no single profile to watermark against; the + # per-subject files carry the timestamps instead. + existing = None + last_profile_ts = await _subject_baseline_ts( + event.owner_id, event.app_id, event.project_id + ) + else: + existing = await _get_reader().read( event.owner_id, - "user_memory", + schema=UserProfileFrontmatter, app_id=event.app_id, project_id=event.project_id, ) - total_count = sum(c.count for c in user_clusters) - else: - user_clusters = None - # Scope the counter to parent_type='memcell' so it matches - # _select_via_timestamp's selector below — otherwise Reflection- - # merged rows (parent_type='cluster') inflate the throttle count - # without ever being selectable, firing the gate at the wrong cadence. - # TODO(profile-counter): reads LanceDB and therefore races the - # cascade daemon in the same way _select_via_timestamp used to - # (fresh Tier-1 install → count=0 until cascade catches up). - # The throttle only needs a monotonic per-owner integer, so a - # stale-but-monotonic value is acceptable for now; the followup - # is to source this from a sqlite ``memcell`` count-by-owner - # query — see PR #361 review finding M4. + last_profile_ts = existing[0].profile_timestamp_ms if existing else 0 + + # Throttle on "cumulative units of source-memory for this owner", scoped to + # parent_type='memcell' so it matches `_select_via_timestamp`'s selector -- + # otherwise Reflection-merged rows (parent_type='cluster') inflate the count + # without ever being selectable, firing the gate at the wrong cadence. + # TODO(profile-counter): reads LanceDB and therefore races the cascade daemon + # the same way `_select_via_timestamp` used to (fresh install -> count=0 until + # cascade catches up). The throttle only needs a monotonic per-owner integer, + # so a stale-but-monotonic value is acceptable; the followup is to source it + # from a sqlite ``memcell`` count-by-owner query -- PR #361 review finding M4. + if PROFILE_EXTRACTION_INTERVAL > 1: total_count = await episode_repo.count_by_owner( event.owner_id, app_id=event.app_id, project_id=event.project_id, parent_type="memcell", ) + if total_count % PROFILE_EXTRACTION_INTERVAL != 0: + logger.info( + "profile_extraction_throttled", + owner_id=event.owner_id, + total_count=total_count, + interval=PROFILE_EXTRACTION_INTERVAL, + ) + return - if ( - PROFILE_EXTRACTION_INTERVAL > 1 - and total_count % PROFILE_EXTRACTION_INTERVAL != 0 - ): - logger.info( - "profile_extraction_throttled", - owner_id=event.owner_id, - total_count=total_count, - interval=PROFILE_EXTRACTION_INTERVAL, - path="cluster" - if isinstance(event, ProfileClusterUpdated) - else "direct", - ) - return - - # Memcell selection is the only part that differs by trigger; the - # LLM extraction + persist tail below is shared by both paths. - # user_clusters is always populated on the cluster branch above, - # so the fallback to [] here is unreachable — it exists purely to - # satisfy the type checker without an assert. - if isinstance(event, ProfileClusterUpdated): - memcell_ids = await _select_via_cluster( - event, last_profile_ts, user_clusters or [] - ) - else: - memcell_ids = await _select_via_timestamp(event, last_profile_ts) + memcell_ids = await _select_via_timestamp(event, last_profile_ts) if len(memcell_ids) < PROFILE_MIN_MEMCELLS: logger.info( @@ -312,32 +291,448 @@ async def extract_user_profile( if not algo_memcells: return - # Run the LLM extractor — INIT (no prior) or UPDATE (existing). - old_profile = _to_algo_profile(existing[0]) if existing else None extractor = ProfileExtractor(llm=get_llm_client()) - new_profile = await extractor.aextract( - algo_memcells, sender_id=event.owner_id, old_profile=old_profile - ) + if PROFILE_SUBJECT == SUBJECT_SENDER: + subjects = _subjects_of(algo_memcells) + if not subjects: + logger.info( + "profile_extraction_no_subjects", + owner_id=event.owner_id, + memcell_count=len(algo_memcells), + ) + return + # Sequential on purpose: the whole block already holds the owner's + # partition lock, and one LLM call per speaker fanned out at once + # would spike a group of 20+ into the provider's rate limit. + written = 0 + for subject in subjects: + written += await _extract_one_subject( + algo_memcells, + subject=subject, + owner_id=event.owner_id, + app_id=event.app_id, + project_id=event.project_id, + extractor=extractor, + ) + summary_mode = f"{written}/{len(subjects)} subjects" + else: + # Run the LLM extractor — INIT (no prior) or UPDATE (existing). + old_profile = _to_algo_profile(existing[0]) if existing else None + t0 = time.perf_counter() + new_profile, retried = await _aextract_language_checked( + extractor, + algo_memcells, + sender_id=event.owner_id, + old_profile=old_profile, + ) + elapsed = time.perf_counter() - t0 - # Write the fresh profile back to users//user.md. - await _persist_profile( - new_profile, - owner_id=event.owner_id, - app_id=event.app_id, - project_id=event.project_id, - ) + # Write the fresh profile back to users//user.md. + await _persist_profile( + new_profile, + owner_id=event.owner_id, + app_id=event.app_id, + project_id=event.project_id, + ) + summary_mode = "UPDATE" if old_profile is not None else "INIT" + _append_trace( + { + "kind": "profile_extract", + "owner_id": event.owner_id, + # Owner-is-subject: no name, the owner IS who this describes. + "subject": "", + "mode": summary_mode, + "candidates": len(algo_memcells), + "memcells_used": len(algo_memcells), + "memcell_chars": sum( + len(str(getattr(i, "content", ""))) + for mc in algo_memcells + for i in mc.items + ), + "own_profile_ts_ms": last_profile_ts, + "before": _profile_shape(old_profile), + "after": _profile_shape(new_profile), + "summary": str(new_profile.summary or "")[:400], + "cjk_in_summary": len(_CJK.findall(str(new_profile.summary or ""))), + "language_retried": retried, + "elapsed_s": round(elapsed, 3), + } + ) logger.info( "user_profile_extracted", owner_id=event.owner_id, - path="cluster" if isinstance(event, ProfileClusterUpdated) else "direct", memcell_count=len(algo_memcells), - mode="UPDATE" if old_profile is not None else "INIT", + subject=PROFILE_SUBJECT, + mode=summary_mode, ) # ── helpers ────────────────────────────────────────────────────────────── +_CJK = re.compile(r"[\u4e00-\u9fff]") + +LANGUAGE_RETRY = os.getenv("EVEROS_PROFILE_LANGUAGE_RETRY", "1") != "0" +"""Retry an INIT whose output language does not match its input. + +The bundled INIT prompt carries a ``CRITICAL LANGUAGE RULE`` ("output in the SAME +language as the input conversation") and states that this call FIXES the profile's +language -- every later update and compaction preserves it. Measured on EverMemBench +topic 01 with gpt-4.1-mini: **8 of 36 INIT calls (22%) ignored it**, producing Chinese +profiles from an all-English corpus, and all 31 subjects that were extracted more than +once kept their first language with zero exceptions. So one non-compliant coin flip +poisons that person's profile for the rest of the run. + +Input volume does not predict it (Mann-Whitney p=0.458 over the 36 INIT calls), so +waiting for more evidence before the first extraction does not help. Retrying the one +call that decides does. Only INIT is checked -- UPDATE emits index-addressed ops onto +an existing profile and inherits its language by design.""" + +_LANGUAGE_DIRECTIVE = ( + "\n\nThe conversation above is written in {lang}. Your ENTIRE output -- every " + "summary, category, description and trait -- MUST be written in {lang}. Do not " + "translate it into any other language. This overrides any other instruction." +) + + +def _cjk_ratio(text: str) -> float: + """Share of CJK characters, over non-whitespace length.""" + body = "".join(text.split()) + return len(_CJK.findall(body)) / len(body) if body else 0.0 + + +def _language_mismatch(source: str, produced: str) -> bool: + """Whether ``produced`` switched scripts away from ``source``. + + Deliberately coarse: it only fires when one side is essentially free of CJK and + the other is substantially CJK. A profile legitimately quoting a few Chinese + product names out of an English corpus stays under the 5% floor, and a Chinese + corpus answered in English is caught by the same rule in reverse. Anything + subtler than a script switch is not something a ratio can judge, and guessing + would retry calls that were fine. + """ + src, out = _cjk_ratio(source), _cjk_ratio(produced) + return (src < 0.01 and out > 0.05) or (src > 0.05 and out < 0.01) + + +def _profile_text(profile: AlgoProfile) -> str: + """Everything the extractor emitted, for the language check.""" + extras = profile.model_dump(exclude={"owner_id", "timestamp"}) + return json.dumps(extras, ensure_ascii=False, default=str) + + +def _source_language(memcells: list[AlgoMemCell]) -> str: + """Name the input's language for the retry directive.""" + text = "".join(str(getattr(i, "content", "")) for mc in memcells for i in mc.items) + return "Chinese" if _cjk_ratio(text) > 0.05 else "English" + + +async def _aextract_language_checked( + extractor: ProfileExtractor, + memcells: list[AlgoMemCell], + *, + sender_id: str, + old_profile: AlgoProfile | None, +) -> tuple[AlgoProfile, bool]: + """``aextract`` plus one INIT-only language retry. Returns (profile, retried).""" + profile = await extractor.aextract( + memcells, sender_id=sender_id, old_profile=old_profile + ) + if old_profile is not None or not LANGUAGE_RETRY: + return profile, False + source = "".join( + str(getattr(i, "content", "")) for mc in memcells for i in mc.items + ) + if not _language_mismatch(source, _profile_text(profile)): + return profile, False + lang = _source_language(memcells) + logger.warning( + "user_profile_language_retry", + owner_id=getattr(profile, "owner_id", ""), + subject=sender_id, + source_language=lang, + ) + # Append to the bundled prompt rather than replacing it: an override is a whole + # template, and a copy in our config would silently diverge the day everalgo + # edits its own. + retried = await extractor.aextract( + memcells, + sender_id=sender_id, + old_profile=None, + prompt=PROFILE_INITIAL_EXTRACTION_PROMPT + + _LANGUAGE_DIRECTIVE.format(lang=lang), + ) + # Keep the retry even if it also failed: a second sample is no worse than the + # first, and pretending otherwise would need a third call to break the tie. + return retried, True + + +def _trace_path() -> str | None: + """Read per call, so a run can toggle the dump without re-importing.""" + return os.getenv(PROFILE_TRACE_ENV, "").strip() or None + + +def _append_trace(record: dict[str, Any]) -> None: + """Append one profile-extraction record as a JSON line. + + Diagnostic side-channel. Every failure is logged and swallowed: losing a trace + line must never lose a profile. ``default=str`` because algo profile items are + heterogeneous dicts that may carry non-JSON scalars. + """ + path = _trace_path() + if not path: + return + try: + with open(path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(record, ensure_ascii=False, default=str) + "\n") + except Exception as err: + logger.warning("user_profile_trace_dump_error", error=str(err)[:200]) + + +def _profile_shape(profile: AlgoProfile | None) -> dict[str, Any]: + """The measurable shape of a profile: what grew, and how big it got. + + Recorded before and after each merge so a reader can see the delta without + diffing free text -- ``explicit_info`` / ``implicit_traits`` counts are what the + algo's compact threshold (45) and cap (30) act on, and ``chars`` is what lands in + the answer prompt when the profile is injected. + """ + if profile is None: + return {"exists": False} + return { + "exists": True, + "explicit_info": len(list(getattr(profile, "explicit_info", []) or [])), + "implicit_traits": len(list(getattr(profile, "implicit_traits", []) or [])), + "summary_chars": len(str(getattr(profile, "summary", "") or "")), + } + + +def _subject_slug(subject: str) -> str: + """Filename-safe form of a subject name (``"Lan Ye"`` -> ``"Lan_Ye"``). + + The slug only has to locate the file; :attr:`UserProfileFrontmatter.subject` + carries the real name, and the LanceDB row id is keyed on that, so two + names that slugify alike collide on disk but not in the index. The write + path detects that collision rather than silently overwriting. + """ + return _SUBJECT_UNSAFE.sub("_", subject).strip("._-") or "unnamed" + + +def _subject_filename(subject: str) -> str: + """``users//`` filename holding ``subject``'s profile.""" + return f"user.{_subject_slug(subject)}.md" + + +def _subjects_of(memcells: list[AlgoMemCell]) -> list[str]: + """Distinct real speakers across ``memcells``, first-seen order. + + Prefers ``sender_name`` over ``sender_id``: a group ingest pins every + message's ``sender_id`` to the batch owner (that is what keeps retrieval + in one partition) and carries the person's name alongside. Only + ``role == "user"`` turns count -- an assistant is never a profile subject, + and :meth:`ProfileExtractor.aextract` rejects one outright. + """ + seen: list[str] = [] + for cell in memcells: + for item in cell.items: + if not isinstance(item, AlgoChatMessage) or item.role != "user": + continue + subject = (item.sender_name or item.sender_id or "").strip() + if subject and subject not in seen: + seen.append(subject) + return seen + + +def _speaks_in(memcell: AlgoMemCell, subject: str) -> bool: + """Whether ``subject`` has a user turn in ``memcell``. + + A memcell is a whole slice of conversation, so keeping only the ones a subject + spoke in still hands the extractor everyone else's surrounding turns -- what it + drops is the meetings that person never attended, which is not evidence about + them in the first place. + """ + return any( + isinstance(item, AlgoChatMessage) + and item.role == "user" + and (item.sender_name or item.sender_id or "").strip() == subject + for item in memcell.items + ) + + +def _retarget(memcells: list[AlgoMemCell], subject: str) -> list[AlgoMemCell]: + """Copy ``memcells`` with user turns re-keyed from name to ``sender_id``. + + :meth:`ProfileExtractor.aextract` validates ``sender_id`` against the + memcells' own user senders and will not accept a name that only appears in + ``sender_name``. Rewriting the copy is what lets a real person be the + target while the persisted memcell keeps the owner as its sender. It also + stops the rendered transcript from claiming ``Lan Ye(user_id:01)``. + """ + out: list[AlgoMemCell] = [] + for cell in memcells: + clone = cell.model_copy(deep=True) + for item in clone.items: + if isinstance(item, AlgoChatMessage) and item.role == "user": + item.sender_id = (item.sender_name or item.sender_id or "").strip() + out.append(clone) + return out + + +async def _subject_baseline_ts(owner_id: str, app_id: str, project_id: str) -> int: + """Oldest participant-profile timestamp under ``owner_id`` (0 when none). + + The **minimum**, not the maximum: memcell selection runs once and feeds + every subject from the same set, so watermarking on the freshest subject + would starve the ones that lag behind. + """ + own = _get_reader().path_for( + owner_id, + schema=UserProfileFrontmatter, + app_id=app_id, + project_id=project_id, + ) + oldest: int | None = None + async for path in anyio.Path(own.parent).glob("user.*.md"): + parsed = await _get_reader().read( + owner_id, + schema=UserProfileFrontmatter, + app_id=app_id, + project_id=project_id, + filename=path.name, + ) + if parsed is None: + continue + ts = parsed[0].profile_timestamp_ms + oldest = ts if oldest is None else min(oldest, ts) + return oldest or 0 + + +async def _extract_one_subject( + memcells: list[AlgoMemCell], + *, + subject: str, + owner_id: str, + app_id: str, + project_id: str, + extractor: ProfileExtractor, +) -> bool: + """Synthesise and persist one subject's profile. False = skipped. + + Serialised per ``(owner, subject)`` -- the granularity of the file actually + being rewritten. Two tasks for the same owner now block each other only when + they reach the *same* speaker; previously they contended on the owner for the + entire N-subject pass, which is how one stuck subject took a whole process + down (see the caller). + """ + async with get_partition_lock( + "extract_user_profile", f"{app_id}:{project_id}:{owner_id}::{subject}" + ): + return await _extract_one_subject_locked( + memcells, + subject=subject, + owner_id=owner_id, + app_id=app_id, + project_id=project_id, + extractor=extractor, + ) + + +async def _extract_one_subject_locked( + memcells: list[AlgoMemCell], + *, + subject: str, + owner_id: str, + app_id: str, + project_id: str, + extractor: ProfileExtractor, +) -> bool: + """Body of :func:`_extract_one_subject`; caller holds the subject's lock.""" + filename = _subject_filename(subject) + prior = await _get_reader().read( + owner_id, + schema=UserProfileFrontmatter, + app_id=app_id, + project_id=project_id, + filename=filename, + ) + if prior is not None and prior[0].subject and prior[0].subject != subject: + # Two names slugified onto one file. Writing would destroy the other + # person's profile, so refuse and say whose. + logger.error( + "user_profile_subject_slug_collision", + owner_id=owner_id, + subject=subject, + occupied_by=prior[0].subject, + filename=filename, + ) + return False + # Per-subject watermark, not the owner-wide one. Participants advance at wildly + # different rates -- a rarely-speaking member's profile stays old, and the + # owner-wide baseline is the MINIMUM across all of them, so it lags behind by + # however long the quietest member has been silent. Filtering on that baseline + # alone would hand a regular speaker every memcell since the quietest member last + # spoke instead of the one that just arrived. + own_ts = prior[0].profile_timestamp_ms if prior is not None else 0 + mine = [mc for mc in memcells if mc.timestamp > own_ts and _speaks_in(mc, subject)] + if not mine: + # Present in the candidate set only because somebody else spoke, or already + # merged. Nothing to re-read: the profile already encodes it. + _append_trace( + { + "kind": "profile_extract", + "owner_id": owner_id, + "subject": subject, + "skipped": "no_new_memcells", + "candidates": len(memcells), + "own_profile_ts_ms": own_ts, + } + ) + return False + + old_profile = _to_algo_profile(prior[0]) if prior is not None else None + t0 = time.perf_counter() + new_profile, retried = await _aextract_language_checked( + extractor, + _retarget(mine, subject), + sender_id=subject, + old_profile=old_profile, + ) + elapsed = time.perf_counter() - t0 + await _persist_profile( + new_profile, + owner_id=owner_id, + app_id=app_id, + project_id=project_id, + subject=subject, + ) + _append_trace( + { + "kind": "profile_extract", + "owner_id": owner_id, + "subject": subject, + # INIT fixes the profile's language and writes it whole; UPDATE emits + # index-addressed ops onto it. Which one ran explains both the cost of the + # call and whether a language choice was made here. + "mode": "UPDATE" if old_profile is not None else "INIT", + # Candidates the owner-level selector produced vs what this subject + # actually read: the gap is the per-subject filter doing its job. + "candidates": len(memcells), + "memcells_used": len(mine), + "memcell_chars": sum( + len(str(getattr(i, "content", ""))) for mc in mine for i in mc.items + ), + "own_profile_ts_ms": own_ts, + "before": _profile_shape(old_profile), + "after": _profile_shape(new_profile), + "summary": str(new_profile.summary or "")[:400], + "cjk_in_summary": len(_CJK.findall(str(new_profile.summary or ""))), + "language_retried": retried, + "elapsed_s": round(elapsed, 3), + } + ) + return True + + def _to_algo_profile(fm: UserProfileFrontmatter) -> AlgoProfile: """Rehydrate an algo :class:`Profile` from the markdown frontmatter.""" return AlgoProfile.model_validate( @@ -352,15 +747,26 @@ def _to_algo_profile(fm: UserProfileFrontmatter) -> AlgoProfile: async def _persist_profile( - profile: AlgoProfile, *, owner_id: str, app_id: str, project_id: str + profile: AlgoProfile, + *, + owner_id: str, + app_id: str, + project_id: str, + subject: str = "", ) -> None: - """Write the freshly extracted profile to ``users//user.md``.""" + """Write the freshly extracted profile under ``users//``. + + ``subject`` empty writes the owner's own ``user.md``; a subject writes + ``user..md`` and records the real name in the frontmatter, which is + what the cascade keys the LanceDB row id on. + """ extras = profile.model_dump(exclude={"owner_id", "summary", "timestamp"}) explicit_info = extras.get("explicit_info") or [] implicit_traits = extras.get("implicit_traits") or [] frontmatter = UserProfileFrontmatter( - id=f"profile_{owner_id}", + id=f"profile_{owner_id}::{subject}" if subject else f"profile_{owner_id}", user_id=owner_id, + subject=subject, summary=profile.summary, explicit_info=list(explicit_info), implicit_traits=list(implicit_traits), @@ -372,4 +778,5 @@ async def _persist_profile( body=profile.summary, app_id=app_id, project_id=project_id, + filename=_subject_filename(subject) if subject else None, ) diff --git a/src/everos/memory/strategies/trigger_profile_clustering.py b/src/everos/memory/strategies/trigger_profile_clustering.py index 3bcc305fc..9efb178db 100644 --- a/src/everos/memory/strategies/trigger_profile_clustering.py +++ b/src/everos/memory/strategies/trigger_profile_clustering.py @@ -22,7 +22,7 @@ from everos.infra.ome.triggers import Immediate from everos.infra.persistence.sqlite import cluster_repo, mint_cluster_id from everos.memory._partition_locks import get_partition_lock -from everos.memory.events import EpisodeExtracted, ProfileClusterUpdated +from everos.memory.events import EpisodeExtracted logger = get_logger(__name__) @@ -30,7 +30,7 @@ @offline_strategy( name="trigger_profile_clustering", trigger=Immediate(on=[EpisodeExtracted]), - emits=[ProfileClusterUpdated], + emits=[], applies_to=lambda e: e.source == "pipeline", max_retries=2, ) @@ -127,17 +127,7 @@ async def trigger_profile_clustering( project_id=event.project_id, ) - # 6. Emit ProfileClusterUpdated → downstream extract_user_profile. assert to_save.id is not None # both branches above set id - await ctx.emit( - ProfileClusterUpdated( - memcell_id=event.memcell_id, - cluster_id=to_save.id, - owner_id=event.owner_id, - app_id=event.app_id, - project_id=event.project_id, - ) - ) logger.info( "profile_cluster_updated", memcell_id=event.memcell_id, diff --git a/src/everos/service/search.py b/src/everos/service/search.py index 8cb6ee85d..2d7e53a26 100644 --- a/src/everos/service/search.py +++ b/src/everos/service/search.py @@ -23,7 +23,11 @@ from typing import TYPE_CHECKING from everos.component.embedding import get_embedding_capability -from everos.component.llm import LLMNotConfiguredError, get_llm_client +from everos.component.llm import ( + LLMNotConfiguredError, + get_decider_llm_client, + get_llm_client, +) from everos.component.rerank import get_rerank_capability from everos.component.tokenizer import build_tokenizer from everos.core.observability.logging import get_logger @@ -71,6 +75,23 @@ def _get_llm_client() -> LLMClient | None: return None +def _get_decider_client() -> LLMClient | None: + """Client for the multi-round decider; ``None`` when no LLM is configured at all. + + Resolves ``[decider]`` and falls back to ``[llm]`` when that section names no model, + so search behaves exactly as before unless a decider is explicitly configured. + """ + try: + return get_decider_llm_client() + except LLMNotConfiguredError: + logger.warning( + "decider_not_configured", + hint="set [decider] model (and api_key / base_url, or inherit them from " + "[llm]) to give multi-round retrieval its own model", + ) + return None + + def _get_manager() -> SearchManager: global _manager if _manager is None: @@ -84,6 +105,7 @@ def _get_manager() -> SearchManager: embedding=get_embedding_capability().provider, reranker=get_rerank_capability().provider, llm_client=_get_llm_client(), + decider_client=_get_decider_client(), ) return _manager diff --git a/tests/integration/test_ome_strategies_integration.py b/tests/integration/test_ome_strategies_integration.py index 88d9c8811..d198310a1 100644 --- a/tests/integration/test_ome_strategies_integration.py +++ b/tests/integration/test_ome_strategies_integration.py @@ -527,7 +527,8 @@ async def test_profile_chain_e2e( monkeypatch: pytest.MonkeyPatch, ) -> None: """Chain: EpisodeExtracted → trigger_profile_clustering (sqlite) → - ProfileClusterUpdated → extract_user_profile → SUCCESS. + EpisodeExtracted → extract_user_profile → SUCCESS (single path; the cluster + event it used to travel on is gone). Real ``cluster_by_geometry`` (cosine + time-window) with a hash-based deterministic embedder so the geometry stage operates on well-spread @@ -623,6 +624,19 @@ async def test_profile_chain_e2e( ), capture_logs() as logs, ): + # The selector this strategy actually calls. It used to be + # `find_by_owner_entries`, and when the profile refactor moved selection to a + # timestamp window the stub was not moved with it -- leaving + # `list_by_owner_after_ts` as a bare MagicMock, which the strategy awaited and + # died on. The run then went FAILED -> FAILED -> DEAD_LETTER while the + # assertion only said "expected SUCCESS", naming the symptom and nothing else. + # + # `columns=["parent_id"]` makes the return contractually a list of raw dicts, + # not ORM rows, so the stub has to hand back dicts or the strategy's + # `row["parent_id"]` reads a Mock attribute and the id set fills with junk. + mock_episode_repo.list_by_owner_after_ts = AsyncMock( + return_value=[{"parent_id": "mc_20260517_0001"}] + ) mock_episode_repo.find_by_owner_entries = AsyncMock( return_value=[fake_episode_row] ) diff --git a/tests/unit/test_memory/test_cascade/test_orchestrator.py b/tests/unit/test_memory/test_cascade/test_orchestrator.py index 0c57152a3..d67bbf6a8 100644 --- a/tests/unit/test_memory/test_cascade/test_orchestrator.py +++ b/tests/unit/test_memory/test_cascade/test_orchestrator.py @@ -4,6 +4,7 @@ from collections.abc import AsyncIterator from pathlib import Path +from unittest.mock import patch import pytest from sqlmodel import SQLModel @@ -54,14 +55,21 @@ def _make_orchestrator(memory_root: MemoryRoot) -> CascadeOrchestrator: async def test_double_start_is_idempotent(runtime: MemoryRoot) -> None: - """Calling start twice does not relaunch tasks.""" - orch = _make_orchestrator(runtime) - await orch.start() - # Capture watcher identity to verify the second start doesn't replace it. - first_watcher = orch._watcher - await orch.start() - assert orch._watcher is first_watcher - await orch.stop() + """Calling start twice does not relaunch tasks. + + The watcher is stubbed because a real one takes an inotify watch, a per-user + kernel resource this host runs out of (the IDE's indexer holds 350k of the + 524k ceiling). Idempotency is the subject here; the observer thread is not. + """ + with patch("everos.memory.cascade.orchestrator.CascadeWatcher") as watcher_cls: + orch = _make_orchestrator(runtime) + await orch.start() + # Capture watcher identity to verify the second start doesn't replace it. + first_watcher = orch._watcher + await orch.start() + assert orch._watcher is first_watcher + assert watcher_cls.call_count == 1 + await orch.stop() async def test_stop_before_start_is_noop(runtime: MemoryRoot) -> None: @@ -70,10 +78,11 @@ async def test_stop_before_start_is_noop(runtime: MemoryRoot) -> None: async def test_double_stop_is_idempotent(runtime: MemoryRoot) -> None: - orch = _make_orchestrator(runtime) - await orch.start() - await orch.stop() - await orch.stop() # second stop is a no-op + with patch("everos.memory.cascade.orchestrator.CascadeWatcher"): + orch = _make_orchestrator(runtime) + await orch.start() + await orch.stop() + await orch.stop() # second stop is a no-op async def test_queue_summary_returns_empty_on_fresh_runtime( @@ -218,3 +227,26 @@ def test_deadlines_are_deliberately_not_configurable() -> None: "optimize_rebuild_interval_seconds", } assert not any("timeout" in f or "deadline" in f for f in exposed) + + +async def test_watcher_can_be_disabled_without_losing_the_worker( + runtime: MemoryRoot, monkeypatch: pytest.MonkeyPatch +) -> None: + """``EVEROS_DISABLE_CASCADE_WATCHER`` drops inotify, keeps md -> LanceDB. + + Forced by a shared host: inotify watches are a per-user kernel resource, the + ceiling lives in read-only ``/proc/sys``, and an IDE indexer can hold 350k of + 524k on its own -- every server then dies at startup with ``Errno 28``. The + scanner re-derives the same truth every 30s, so what is lost is latency. The + worker must survive, or an ingesting run writes md that never gets indexed. + """ + monkeypatch.setenv("EVEROS_DISABLE_CASCADE_WATCHER", "1") + with patch("everos.memory.cascade.orchestrator.CascadeWatcher") as watcher_cls: + orch = _make_orchestrator(runtime) + await orch.start() + assert watcher_cls.call_count == 0, "watcher was constructed anyway" + assert orch._watcher is None + assert orch._started is True + assert orch._worker is not None + assert orch._scanner is not None + await orch.stop() diff --git a/tests/unit/test_memory/test_ome_run_timeout.py b/tests/unit/test_memory/test_ome_run_timeout.py new file mode 100644 index 000000000..d5f7dc93f --- /dev/null +++ b/tests/unit/test_memory/test_ome_run_timeout.py @@ -0,0 +1,103 @@ +"""A strategy that hangs must lose its slot, not the whole engine. + +Regression cover for a full-process stall. ``OMEConfig`` bounded retries and +recovered orphans left by a *previous* process, but nothing bounded a live +attempt. A coroutine parked on an await with no deadline of its own -- an +``asyncio.Lock`` held by another stuck coroutine, a connection-pool wait -- kept +its ``max_concurrent_runs`` slot indefinitely: it never raised, so it never +retried, and its record stayed RUNNING forever. + +Observed in a benchmark run: 60 of 64 slots parked on one lock, which starved +every other strategy in the process. Extraction stopped for 6.7 hours while the +server still answered HTTP and every liveness signal read healthy -- the queue +depth was the only thing that moved, and it moved the wrong way. +""" + +from __future__ import annotations + +import asyncio + +import pytest + +from everos.infra.ome.config import OMEConfig, _env_float + + +def _cfg(**kw: object) -> OMEConfig: + return OMEConfig(jobstore_path="/tmp/x.db", max_concurrent_runs=4, **kw) # type: ignore[arg-type] + + +def test_a_ceiling_is_on_by_default() -> None: + """Opt-out, not opt-in: the failure it prevents is silent and total.""" + assert _cfg().run_timeout_seconds == 1800.0 + + +def test_the_ceiling_leaves_room_for_slow_but_healthy_work() -> None: + """Measured worst case is a ~7-minute 38-subject profile pass. + + Pinned as an inequality rather than a constant: the point is the margin, so + tuning the default stays free while shrinking it below the known-good + workload does not. + """ + assert (_cfg().run_timeout_seconds or 0) >= 4 * 7 * 60 + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("300", 300.0), + ("0", None), + ("off", None), + ("none", None), + ("false", None), + ("", 1800.0), + ("garbage", 1800.0), + ("-5", None), + ], +) +def test_env_parsing( + raw: str, expected: float | None, monkeypatch: pytest.MonkeyPatch +) -> None: + """Operators need an off switch, and a typo must not take the process down.""" + monkeypatch.setenv("EVEROS_OME_RUN_TIMEOUT_SECONDS", raw) + assert _env_float("EVEROS_OME_RUN_TIMEOUT_SECONDS", 1800.0) == expected + + +def test_disabled_ceiling_is_representable() -> None: + assert _cfg(run_timeout_seconds=None).run_timeout_seconds is None + + +async def test_a_hung_attempt_raises_timeout_error_and_frees_its_slot() -> None: + """The mechanism, end to end: hang -> cancel -> Exception -> slot released. + + ``asyncio.timeout`` cancels the body, and it is ``TimeoutError`` (an + ``Exception``, unlike ``CancelledError``) that escapes -- which is what lets + the runner's existing ``except Exception`` path retry or dead-letter the + attempt instead of leaking the slot. Written against the primitives so it + pins the property the runner depends on. + """ + sem = asyncio.Semaphore(1) + never = asyncio.Event() # stands in for a lock nobody will release + + async def hangs() -> None: + await never.wait() + + with pytest.raises(TimeoutError): + async with sem: + async with asyncio.timeout(0.05): + await hangs() + + # The slot is back: without the ceiling this acquire would block forever. + async with asyncio.timeout(1): + async with sem: + pass + + +async def test_cancelled_error_alone_would_not_have_been_caught() -> None: + """Why the fix is a timeout and not a bare cancel. + + ``CancelledError`` derives from ``BaseException``, so cancelling the attempt + directly would slip past ``except Exception`` and abandon the record in + RUNNING -- the same leak, with extra steps. + """ + assert not issubclass(asyncio.CancelledError, Exception) + assert issubclass(TimeoutError, Exception) diff --git a/tests/unit/test_memory/test_strategies/test_extract_user_profile.py b/tests/unit/test_memory/test_strategies/test_extract_user_profile.py index 3b756b1c4..8b7d65d0c 100644 --- a/tests/unit/test_memory/test_strategies/test_extract_user_profile.py +++ b/tests/unit/test_memory/test_strategies/test_extract_user_profile.py @@ -1,6 +1,6 @@ """Tests for :func:`extract_user_profile`. -Heavy mocking — the strategy threads through ``cluster_repo`` (sqlite), +Heavy mocking — the strategy threads through ``episode_repo`` (LanceDB), ``memcell_repo`` (sqlite, payload deserialise), ``ProfileReader`` / ``ProfileWriter`` (md), and ``ProfileExtractor`` (algo). We mock all seams so the test exercises the orchestration only. @@ -12,16 +12,14 @@ import importlib from unittest.mock import AsyncMock, MagicMock, patch -import numpy as np import pytest -from everalgo.clustering import Cluster as AlgoCluster from everalgo.types import ChatMessage, MemCell from everalgo.types import Profile as AlgoProfile from everos.infra.ome.testing import FakeStrategyContext from everos.infra.persistence.markdown import UserProfileFrontmatter from everos.memory._partition_locks import _reset_for_tests -from everos.memory.events import ProfileClusterUpdated +from everos.memory.events import EpisodeExtracted from everos.memory.strategies.extract_user_profile import extract_user_profile @@ -34,26 +32,18 @@ def _event( *, owner_id: str = "u_alice", memcell_id: str = "mc_aaaaaaaaaaa1", - cluster_id: str = "cl_user00000001", -) -> ProfileClusterUpdated: - return ProfileClusterUpdated( + episode_timestamp_ms: int = 1_700_000_000_000, +) -> EpisodeExtracted: + """The strategy's only trigger. ``cluster_id`` is gone with the cluster path.""" + return EpisodeExtracted( memcell_id=memcell_id, - cluster_id=cluster_id, + episode_entry_id=f"ep_{memcell_id}", + episode_text="episode narrative", + episode_timestamp_ms=episode_timestamp_ms, owner_id=owner_id, ) -def _algo_cluster(*, cluster_id: str, members: list[str], last_ts: int) -> AlgoCluster: - return AlgoCluster( - id=cluster_id, - centroid=np.zeros(1024, dtype=np.float32), - count=len(members), - last_ts=last_ts, - preview=[], - members=members, - ) - - def _episode_row(entry_id: str, parent_id: str) -> MagicMock: """Stand-in for a LanceDB Episode row with parent_type=memcell.""" row = MagicMock() @@ -86,7 +76,7 @@ def _memcell_row(memcell_id: str, *, sender_id: str, ts_ms: int) -> MagicMock: async def test_strategy_meta_is_attached() -> None: meta = extract_user_profile.meta assert meta.name == "extract_user_profile" - assert ProfileClusterUpdated in meta.trigger.on + assert list(meta.trigger.on) == [EpisodeExtracted] assert meta.emits == frozenset() assert meta.max_retries == 2 @@ -96,11 +86,6 @@ async def test_init_mode_writes_profile_when_no_existing( monkeypatch: pytest.MonkeyPatch, ) -> None: """No prior profile → ProfileExtractor invoked without ``old_profile``.""" - cluster = _algo_cluster( - cluster_id="cl_user00000001", - members=["ep_20260101_0001"], - last_ts=1_700_000_001_000, - ) ep_rows = [_episode_row("ep_20260101_0001", "mc_aaaaaaaaaaa1")] mc_rows = [ _memcell_row("mc_aaaaaaaaaaa1", sender_id="u_alice", ts_ms=1_700_000_001_000) @@ -116,9 +101,6 @@ async def test_init_mode_writes_profile_when_no_existing( ) with ( - patch( - "everos.memory.strategies.extract_user_profile.cluster_repo" - ) as mock_cluster_repo, patch( "everos.memory.strategies.extract_user_profile.episode_repo" ) as mock_episode_repo, @@ -139,8 +121,12 @@ async def test_init_mode_writes_profile_when_no_existing( "everos.memory.strategies.extract_user_profile.ProfileWriter" ) as mock_writer_cls, ): - mock_cluster_repo.list_for_owner = AsyncMock(return_value=[cluster]) - mock_episode_repo.find_by_owner_entries = AsyncMock(return_value=ep_rows) + # Single path: the strategy asks LanceDB for episodes newer than the + # profile and takes their parent memcells. `columns=["parent_id"]` makes + # the return raw dicts, not model rows. + mock_episode_repo.list_by_owner_after_ts = AsyncMock( + return_value=[{"parent_id": r.parent_id} for r in ep_rows] + ) mock_memcell_repo.find_by_ids = AsyncMock(return_value=mc_rows) mock_reader_cls.return_value.read = AsyncMock(return_value=None) mock_writer_cls.return_value.write = AsyncMock(return_value=None) @@ -174,11 +160,6 @@ async def test_update_mode_rehydrates_old_profile( monkeypatch: pytest.MonkeyPatch, ) -> None: """Existing profile → algo Profile rehydrated and passed as old_profile.""" - cluster = _algo_cluster( - cluster_id="cl_user00000001", - members=["ep_20260101_0001"], - last_ts=1_700_000_002_000, - ) ep_rows = [_episode_row("ep_20260101_0001", "mc_aaaaaaaaaaa1")] mc_rows = [ _memcell_row("mc_aaaaaaaaaaa1", sender_id="u_alice", ts_ms=1_700_000_002_000) @@ -202,9 +183,6 @@ async def test_update_mode_rehydrates_old_profile( ) with ( - patch( - "everos.memory.strategies.extract_user_profile.cluster_repo" - ) as mock_cluster_repo, patch( "everos.memory.strategies.extract_user_profile.episode_repo" ) as mock_episode_repo, @@ -225,8 +203,12 @@ async def test_update_mode_rehydrates_old_profile( "everos.memory.strategies.extract_user_profile.ProfileWriter" ) as mock_writer_cls, ): - mock_cluster_repo.list_for_owner = AsyncMock(return_value=[cluster]) - mock_episode_repo.find_by_owner_entries = AsyncMock(return_value=ep_rows) + # Single path: the strategy asks LanceDB for episodes newer than the + # profile and takes their parent memcells. `columns=["parent_id"]` makes + # the return raw dicts, not model rows. + mock_episode_repo.list_by_owner_after_ts = AsyncMock( + return_value=[{"parent_id": r.parent_id} for r in ep_rows] + ) mock_memcell_repo.find_by_ids = AsyncMock(return_value=mc_rows) mock_reader_cls.return_value.read = AsyncMock( return_value=(existing_fm, "prior summary") @@ -250,16 +232,14 @@ async def test_update_mode_rehydrates_old_profile( @pytest.mark.asyncio async def test_skips_when_no_members(monkeypatch: pytest.MonkeyPatch) -> None: - """An empty target cluster set (no fresh clusters) → no extractor call.""" - # Existing profile timestamp newer than every cluster's last_ts → no - # target_cluster matches `last_ts > last_profile_ts`, but the current - # cluster_id should still force inclusion. Set the current cluster id + """Nothing newer than the profile → no extractor call. + + `_select_via_timestamp` always includes the event's own memcell, so an empty + LanceDB supplement still yields one candidate id; the skip happens downstream + when the memcell rows come back empty. + """ + # Kept from the cluster-path era: the profile timestamp is ahead of the # to a non-existent value to drop everything. - stale_cluster = _algo_cluster( - cluster_id="cl_other000001", - members=["ep_other00000"], - last_ts=1_600_000_000_000, - ) existing_fm = UserProfileFrontmatter( id="profile_u_alice", user_id="u_alice", @@ -271,8 +251,8 @@ async def test_skips_when_no_members(monkeypatch: pytest.MonkeyPatch) -> None: with ( patch( - "everos.memory.strategies.extract_user_profile.cluster_repo" - ) as mock_cluster_repo, + "everos.memory.strategies.extract_user_profile.episode_repo" + ) as mock_episode_repo, patch( "everos.memory.strategies.extract_user_profile.memcell_repo" ) as mock_memcell_repo, @@ -286,7 +266,7 @@ async def test_skips_when_no_members(monkeypatch: pytest.MonkeyPatch) -> None: "everos.memory.strategies.extract_user_profile.ProfileWriter" ) as mock_writer_cls, ): - mock_cluster_repo.list_for_owner = AsyncMock(return_value=[stale_cluster]) + mock_episode_repo.list_by_owner_after_ts = AsyncMock(return_value=[]) mock_memcell_repo.find_by_ids = AsyncMock(return_value=[]) mock_reader_cls.return_value.read = AsyncMock( return_value=(existing_fm, "prior") @@ -297,9 +277,7 @@ async def test_skips_when_no_members(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(mod, "_writer", None, raising=False) monkeypatch.setattr(mod, "_reader", None, raising=False) - await extract_user_profile( - _event(cluster_id="cl_unknown00000"), FakeStrategyContext() - ) + await extract_user_profile(_event(), FakeStrategyContext()) mock_extractor_cls.return_value.aextract.assert_not_called() mock_writer_cls.return_value.write.assert_not_called() @@ -326,17 +304,7 @@ async def mock_aextract(_memcells, *, sender_id, **_kwargs): implicit_traits=[], ) - cluster_a = _algo_cluster( - cluster_id="cl_a", members=["ep_a"], last_ts=1_700_000_000_000 - ) - cluster_b = _algo_cluster( - cluster_id="cl_b", members=["ep_b"], last_ts=1_700_000_000_000 - ) - with ( - patch( - "everos.memory.strategies.extract_user_profile.cluster_repo" - ) as mock_cluster_repo, patch( "everos.memory.strategies.extract_user_profile.episode_repo" ) as mock_episode_repo, @@ -357,15 +325,9 @@ async def mock_aextract(_memcells, *, sender_id, **_kwargs): "everos.memory.strategies.extract_user_profile.ProfileExtractor" ) as mock_extractor_cls, ): - mock_cluster_repo.list_for_owner = AsyncMock( - side_effect=lambda owner, _kind, **_kw: ( - [cluster_a] if owner == owner_a else [cluster_b] - ) - ) - mock_episode_repo.find_by_owner_entries = AsyncMock( - side_effect=lambda _owner, ids, **_kw: [ - _episode_row(ids[0], f"mc_{ids[0]}") - ] + # Single path: one episode per owner, its parent memcell is what gets read. + mock_episode_repo.list_by_owner_after_ts = AsyncMock( + side_effect=lambda **kw: [{"parent_id": f"mc_ep_{kw['owner_id']}"}] ) mock_memcell_repo.find_by_ids = AsyncMock( side_effect=lambda ids: [ @@ -381,12 +343,8 @@ async def mock_aextract(_memcells, *, sender_id, **_kwargs): monkeypatch.setattr(mod, "_writer", None, raising=False) await asyncio.gather( - extract_user_profile( - _event(owner_id=owner_a, cluster_id="cl_a"), FakeStrategyContext() - ), - extract_user_profile( - _event(owner_id=owner_b, cluster_id="cl_b"), FakeStrategyContext() - ), + extract_user_profile(_event(owner_id=owner_a), FakeStrategyContext()), + extract_user_profile(_event(owner_id=owner_b), FakeStrategyContext()), ) return log diff --git a/tests/unit/test_memory/test_strategies/test_extract_user_profile_single_path.py b/tests/unit/test_memory/test_strategies/test_extract_user_profile_single_path.py index 5c5a11577..be94a8943 100644 --- a/tests/unit/test_memory/test_strategies/test_extract_user_profile_single_path.py +++ b/tests/unit/test_memory/test_strategies/test_extract_user_profile_single_path.py @@ -1,15 +1,24 @@ -"""Dual-trigger contract for :func:`extract_user_profile`. - -Pre-refactor, the strategy only listened for ``ProfileClusterUpdated`` — a -cluster-path-only event that ``trigger_profile_clustering`` never emits when -embedding is unavailable (Tier 1, see Task 14's registration gate). This -file pins the fix: the strategy now also listens for ``EpisodeExtracted`` -(direct path) and gates each event type via ``_profile_applies`` so exactly -one path fires per memcell, regardless of tier. +"""Single-path contract for :func:`extract_user_profile` — decoupled from clusters. + +The strategy used to have two paths. ``trigger_profile_clustering`` emitted +``ProfileClusterUpdated`` per episode and the strategy selected "every member of every +cluster fresher than the profile"; a second, direct path on :class:`EpisodeExtracted` +existed only for the no-embedding tier, and ``_profile_applies`` gated the two so +exactly one fired per memcell. + +That is gone. There is one trigger (:class:`EpisodeExtracted`, ``source="pipeline"``) +and one selector (:func:`_select_via_timestamp`), so the profile is a function of the +memcell that just landed -- the same contract ``extract_episode`` and +``extract_atomic_facts`` have. This file pins that: the tests that used to say "direct +path" now describe the only path, and the ones below them assert the decoupling itself. + +Clustering still runs; ``agentic`` retrieval and Reflection read ``cluster_repo``. It +just no longer gates the profile. """ from __future__ import annotations +import contextlib import importlib from unittest.mock import AsyncMock, MagicMock, patch @@ -18,7 +27,7 @@ from everos.infra.ome.testing import FakeStrategyContext from everos.memory._partition_locks import _reset_for_tests -from everos.memory.events import EpisodeExtracted, ProfileClusterUpdated +from everos.memory.events import EpisodeExtracted from everos.memory.strategies.extract_user_profile import ( _profile_applies, _select_via_timestamp, @@ -31,17 +40,6 @@ def _isolate_partition_locks() -> None: _reset_for_tests() -def _cluster_event( - *, - owner_id: str = "u_alice", - memcell_id: str = "mc_aaaaaaaaaaa1", - cluster_id: str = "cl_user00000001", -) -> ProfileClusterUpdated: - return ProfileClusterUpdated( - memcell_id=memcell_id, cluster_id=cluster_id, owner_id=owner_id - ) - - def _episode_event( *, owner_id: str = "u_alice", @@ -60,73 +58,22 @@ def _episode_event( def _mock_capability(*, available: bool): - return patch( - "everos.memory.strategies.extract_user_profile.get_embedding_capability", - return_value=MagicMock(available=available), - ) - - -# ── _profile_applies gate ───────────────────────────────────────────────── - - -def test_applies_to_cluster_event_always_true() -> None: - """ProfileClusterUpdated only fires when trigger_profile_clustering ran - (embed available), so accepting it unconditionally cannot double-fire.""" - event = _cluster_event() - for cap_available in (True, False): - with _mock_capability(available=cap_available): - assert _profile_applies(event) is True - - -def test_applies_to_episode_event_only_when_no_embed() -> None: - """Direct path fires only for pipeline-sourced episodes while embedding - is unavailable; the cluster path owns the memcell once embed is on.""" - pipeline_event = _episode_event(source="pipeline") - reflection_event = _episode_event(source="reflection") - - with _mock_capability(available=False): - assert _profile_applies(pipeline_event) is True - assert _profile_applies(reflection_event) is False - - with _mock_capability(available=True): - assert _profile_applies(pipeline_event) is False - assert _profile_applies(reflection_event) is False - - -@pytest.mark.parametrize( - ("event_factory", "embed_available", "expected"), - [ - (_episode_event, False, True), - (_episode_event, True, False), - (_cluster_event, True, True), - (_cluster_event, False, True), - ], - ids=[ - "episode+no_embed->direct_path", - "episode+embed->skipped", - "cluster+embed->cluster_path", - "cluster+no_embed->cluster_path", - ], -) -def test_applies_to_matrix(event_factory, embed_available, expected) -> None: - with _mock_capability(available=embed_available): - assert _profile_applies(event_factory()) is expected - - -def test_meta_registers_both_event_types() -> None: - meta = extract_user_profile.meta - assert set(meta.trigger.on) == {ProfileClusterUpdated, EpisodeExtracted} - assert meta.applies_to is _profile_applies + """No-op. The strategy no longer reads embedding capability -- kept so the + existing tests still read as "this holds on either tier", which is now the + point rather than a precondition. + """ + del available + return contextlib.nullcontext() -# ── direct path (Tier 1, EpisodeExtracted) ──────────────────────────────── +# ── the single path ────────────────────────────────────────────────────── @pytest.mark.asyncio -async def test_direct_path_fetches_via_timestamp_and_writes_profile( +async def test_fetches_via_timestamp_and_writes_profile( monkeypatch: pytest.MonkeyPatch, ) -> None: - """EpisodeExtracted in Tier 1 pulls episodes via list_by_owner_after_ts, + """EpisodeExtracted pulls episodes via list_by_owner_after_ts, feeds them to the LLM extractor, and persists the resulting profile.""" # `_select_via_timestamp` calls `list_by_owner_after_ts(..., columns=[...])` # so the repo returns raw dicts (projection contract) — not full Episode @@ -217,111 +164,7 @@ async def test_direct_path_fetches_via_timestamp_and_writes_profile( @pytest.mark.asyncio -async def test_cluster_path_unaffected_by_dual_trigger_refactor( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Cluster path (ProfileClusterUpdated) keeps its pre-refactor behavior: - _select_via_cluster still drives memcell selection from cluster members.""" - import numpy as np - from everalgo.clustering import Cluster as AlgoCluster - - cluster = AlgoCluster( - id="cl_user00000001", - centroid=np.zeros(1024, dtype=np.float32), - count=1, - last_ts=1_700_000_001_000, - preview=[], - members=["ep_20260101_0001"], - ) - ep_row = MagicMock() - ep_row.entry_id = "ep_20260101_0001" - ep_row.parent_type = "memcell" - ep_row.parent_id = "mc_aaaaaaaaaaa1" - - from everalgo.types import ChatMessage - from everalgo.types import MemCell as AlgoMemCell - - cell = AlgoMemCell( - items=[ - ChatMessage( - id="m1", - role="user", - content="hi", - timestamp=1_700_000_001_000, - sender_id="u_alice", - ) - ], - timestamp=1_700_000_001_000, - ) - mc_row = MagicMock() - mc_row.memcell_id = "mc_aaaaaaaaaaa1" - mc_row.payload_json = cell.model_dump_json() - - new_profile = AlgoProfile.model_validate( - { - "owner_id": "u_alice", - "summary": "Alice is a hiker.", - "timestamp": 1_700_000_001_000, - "explicit_info": [], - "implicit_traits": [], - } - ) - - with ( - patch( - "everos.memory.strategies.extract_user_profile.cluster_repo" - ) as mock_cluster_repo, - patch( - "everos.memory.strategies.extract_user_profile.episode_repo" - ) as mock_episode_repo, - patch( - "everos.memory.strategies.extract_user_profile.memcell_repo" - ) as mock_memcell_repo, - patch( - "everos.memory.strategies.extract_user_profile.get_llm_client", - return_value=object(), - ), - patch( - "everos.memory.strategies.extract_user_profile.ProfileExtractor" - ) as mock_extractor_cls, - patch( - "everos.memory.strategies.extract_user_profile.ProfileReader" - ) as mock_reader_cls, - patch( - "everos.memory.strategies.extract_user_profile.ProfileWriter" - ) as mock_writer_cls, - ): - mock_cluster_repo.list_for_owner = AsyncMock(return_value=[cluster]) - mock_episode_repo.find_by_owner_entries = AsyncMock(return_value=[ep_row]) - mock_memcell_repo.find_by_ids = AsyncMock(return_value=[mc_row]) - mock_reader_cls.return_value.read = AsyncMock(return_value=None) - mock_writer_cls.return_value.write = AsyncMock(return_value=None) - mock_extractor_cls.return_value.aextract = AsyncMock(return_value=new_profile) - mod = importlib.import_module("everos.memory.strategies.extract_user_profile") - monkeypatch.setattr(mod, "_writer", None, raising=False) - monkeypatch.setattr(mod, "_reader", None, raising=False) - - await extract_user_profile(_cluster_event(), FakeStrategyContext()) - - mock_episode_repo.find_by_owner_entries.assert_awaited_once() - mock_episode_repo.list_by_owner_after_ts.assert_not_called() - write_call = mock_writer_cls.return_value.write.call_args - assert write_call.kwargs["frontmatter"].summary == "Alice is a hiker." - - -# ── _select_via_timestamp (event-first, cascade-race-proof) ────────────── -# -# Round-2 review found the direct-path selector previously read LanceDB -# exclusively — on a fresh Tier-1 install the cascade may not have indexed -# the just-arrived memcell yet, so ``list_by_owner_after_ts`` returned -# ``[]``, the MIN_MEMCELLS guard early-returned, and the first memory's -# profile was permanently lost. The selector now always seeds the set -# with ``event.memcell_id`` (event-first, matches ``EpisodeExtracted``'s -# contract at ``events.py:40-51``); LanceDB is a best-effort supplement. - - -@pytest.mark.asyncio -async def test_direct_path_returns_event_memcell_when_lancedb_empty() -> None: +async def test_returns_event_memcell_when_lancedb_empty() -> None: """M4: cascade race — LanceDB returns [] but the event's memcell is still emitted, so the strategy never early-returns on the first memory.""" event = _episode_event(memcell_id="mc_fresh_install") @@ -334,7 +177,7 @@ async def test_direct_path_returns_event_memcell_when_lancedb_empty() -> None: @pytest.mark.asyncio -async def test_direct_path_returns_event_memcell_plus_supplement() -> None: +async def test_returns_event_memcell_plus_supplement() -> None: """Union: event's memcell merged with the LanceDB supplement, deduped.""" event = _episode_event(memcell_id="mc_current") # Projection contract: repo returns raw dicts when caller passes `columns`. @@ -352,7 +195,7 @@ async def test_direct_path_returns_event_memcell_plus_supplement() -> None: @pytest.mark.asyncio -async def test_direct_path_dedupes_when_supplement_overlaps_event() -> None: +async def test_dedupes_when_supplement_overlaps_event() -> None: """No duplicate when the LanceDB supplement returns the same memcell.""" event = _episode_event(memcell_id="mc_shared") # Projection contract: repo returns raw dicts when caller passes `columns`. @@ -366,9 +209,7 @@ async def test_direct_path_dedupes_when_supplement_overlaps_event() -> None: @pytest.mark.asyncio -async def test_direct_path_includes_event_memcell_even_when_timestamp_le_last_profile() -> ( # noqa: E501 - None -): +async def test_includes_event_memcell_even_when_timestamp_le_last_profile() -> None: """M5: historical-timestamp import — the event's own memcell has a timestamp <= ``last_profile_ts``, so LanceDB legitimately returns []; the selector must still include the event's memcell (matches the @@ -385,25 +226,6 @@ async def test_direct_path_includes_event_memcell_even_when_timestamp_le_last_pr # ── grep-style confirmation: Tier 2+ EpisodeExtracted never double-fires ── -def test_tier2_episode_created_does_not_fire_direct_path() -> None: - """For a memcell created under Tier 2+ (embed available), the direct - path's gate must return False — the cluster path (via the later - ProfileClusterUpdated emitted by trigger_profile_clustering) is the - only one that proceeds.""" - with _mock_capability(available=True): - assert _profile_applies(_episode_event(source="pipeline")) is False - - -# ── strategy-entry throttle (unified across both paths) ─────────────────── -# -# The pre-refactor throttle sat inside ``_select_via_cluster`` so the -# Tier-1 direct path silently skipped it. Bumping ``PROFILE_EXTRACTION_INTERVAL`` -# to cap LLM cost therefore only slowed the cluster path down. These tests -# pin the lifted-throttle contract: both paths gate on the same modulo -# check at strategy entry, and the direct path derives its count from -# ``episode_repo.count_by_owner`` (~1:1 with cluster.count in normal usage). - - def _cluster_with_count(cluster_id: str, count: int) -> object: """Minimal AlgoCluster stand-in — only ``count``/``last_ts``/``id`` are read.""" import numpy as np @@ -420,7 +242,7 @@ def _cluster_with_count(cluster_id: str, count: int) -> object: @pytest.mark.asyncio -async def test_direct_path_throttles_by_episode_count( +async def test_throttles_by_episode_count( monkeypatch: pytest.MonkeyPatch, ) -> None: """Tier 1: episode_count % interval != 0 skips extraction and logs. @@ -471,7 +293,7 @@ async def test_direct_path_throttles_by_episode_count( @pytest.mark.asyncio -async def test_direct_path_does_not_throttle_at_default_interval_1( +async def test_does_not_throttle_at_default_interval_1( monkeypatch: pytest.MonkeyPatch, ) -> None: """Interval=1 disables the gate outright (``interval > 1`` guard).""" @@ -546,7 +368,7 @@ async def test_direct_path_does_not_throttle_at_default_interval_1( @pytest.mark.asyncio -async def test_direct_path_fires_when_count_is_multiple_of_interval( +async def test_fires_when_count_is_multiple_of_interval( monkeypatch: pytest.MonkeyPatch, ) -> None: """Count=5, interval=5 → gate passes → LLM extractor is invoked.""" @@ -620,61 +442,7 @@ async def test_direct_path_fires_when_count_is_multiple_of_interval( @pytest.mark.asyncio -async def test_cluster_path_still_throttles_after_lift( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Tier 2+: sum(c.count) % interval != 0 still throttles. - - Preserves pre-refactor cluster-path semantics after lifting the - throttle to strategy entry. Two clusters with counts 4+2 = 6, - ``PROFILE_EXTRACTION_INTERVAL`` = 5 → 6 % 5 == 1 → throttled. - """ - clusters = [ - _cluster_with_count("cl_a", 4), - _cluster_with_count("cl_b", 2), - ] - - with ( - patch( - "everos.memory.strategies.extract_user_profile.cluster_repo" - ) as mock_cluster_repo, - patch( - "everos.memory.strategies.extract_user_profile.episode_repo" - ) as mock_episode_repo, - patch( - "everos.memory.strategies.extract_user_profile.memcell_repo" - ) as mock_memcell_repo, - patch( - "everos.memory.strategies.extract_user_profile.ProfileExtractor" - ) as mock_extractor_cls, - patch( - "everos.memory.strategies.extract_user_profile.ProfileReader" - ) as mock_reader_cls, - patch( - "everos.memory.strategies.extract_user_profile.ProfileWriter" - ) as mock_writer_cls, - ): - mock_cluster_repo.list_for_owner = AsyncMock(return_value=clusters) - mock_episode_repo.find_by_owner_entries = AsyncMock(return_value=[]) - mock_memcell_repo.find_by_ids = AsyncMock(return_value=[]) - mock_reader_cls.return_value.read = AsyncMock(return_value=None) - mock_writer_cls.return_value.write = AsyncMock(return_value=None) - mock_extractor_cls.return_value.aextract = AsyncMock() - mod = importlib.import_module("everos.memory.strategies.extract_user_profile") - monkeypatch.setattr(mod, "_writer", None, raising=False) - monkeypatch.setattr(mod, "_reader", None, raising=False) - monkeypatch.setattr(mod, "PROFILE_EXTRACTION_INTERVAL", 5) - - await extract_user_profile(_cluster_event(), FakeStrategyContext()) - - mock_cluster_repo.list_for_owner.assert_awaited_once() - mock_episode_repo.find_by_owner_entries.assert_not_called() - mock_extractor_cls.return_value.aextract.assert_not_called() - mock_writer_cls.return_value.write.assert_not_called() - - -@pytest.mark.asyncio -async def test_direct_path_throttle_ignores_reflection_merged_episodes( +async def test_throttle_ignores_reflection_merged_episodes( monkeypatch: pytest.MonkeyPatch, ) -> None: """Tier 1: throttle counter must exclude Reflection-merged rows. @@ -768,3 +536,49 @@ async def _fake_count(owner_id: str, **kwargs: object) -> int: assert kwargs["parent_type"] == "memcell" mock_extractor_cls.return_value.aextract.assert_awaited_once() mock_writer_cls.return_value.write.assert_awaited_once() + + +# ── the decoupling itself ──────────────────────────────────────────────── + + +def test_only_episode_extracted_is_registered() -> None: + """One trigger. A second event type is what let the cluster path exist.""" + meta = extract_user_profile.meta + assert list(meta.trigger.on) == [EpisodeExtracted] + + +def test_applies_regardless_of_embedding_availability() -> None: + """The old gate stood the direct path down whenever embedding was available, + handing the memcell to the cluster path instead. Nothing reads capability now, + so the same data yields the same profile on every tier.""" + event = _episode_event() + for available in (True, False): + with patch( + "everos.component.embedding.get_embedding_capability", + return_value=MagicMock(available=available), + ): + assert _profile_applies(event) is True + + +def test_reflection_merged_episodes_are_still_excluded() -> None: + """Their source memcells were merged into the profile when they first arrived.""" + assert _profile_applies(_episode_event(source="reflection")) is False + + +def test_strategy_module_no_longer_touches_the_cluster_repo() -> None: + """A structural assertion on purpose: the cost of the old path was a read of the + owner's entire cluster list plus a LanceDB fetch of every fresh cluster's members, + per memcell. A reintroduced import brings that back silently.""" + import everos.memory.strategies.extract_user_profile as mod + + assert not hasattr(mod, "cluster_repo") + assert not hasattr(mod, "_select_via_cluster") + + +def test_clustering_strategy_emits_nothing() -> None: + """``extract_user_profile`` was its only consumer.""" + from everos.memory.strategies.trigger_profile_clustering import ( + trigger_profile_clustering, + ) + + assert trigger_profile_clustering.meta.emits == frozenset() diff --git a/tests/unit/test_memory/test_strategies/test_profile_lock_granularity.py b/tests/unit/test_memory/test_strategies/test_profile_lock_granularity.py new file mode 100644 index 000000000..fad1a99b8 --- /dev/null +++ b/tests/unit/test_memory/test_strategies/test_profile_lock_granularity.py @@ -0,0 +1,115 @@ +"""Per-sender profiles lock per subject, not per owner. + +The partition lock was owner-wide from when every owner had exactly one +``user.md``. Per-sender mode broke that assumption: each subject owns its own +file, so an owner-wide lock buys no extra safety while serialising every task +for the owner across all N subjects. + +Cost of getting it wrong, measured on a 38-speaker owner: one subject stuck +inside the loop held the owner lock, 60 of the engine's 64 slots piled up behind +it, and the process stopped running any strategy -- including the extractors that +had nothing to do with profiles -- for 6.7 hours. + +The fix is granularity, not removal: two tasks that reach the *same* speaker must +still serialise, because that path is read -> LLM merge -> overwrite on one file. +""" + +from __future__ import annotations + +import asyncio +import importlib + +import pytest + +from everos.memory import _partition_locks + +# import_module, not `from ... import extract_user_profile`: the decorator binds +# a Strategy object to that name in the package namespace, which shadows the +# module the source assertions below need to reach. +mod = importlib.import_module("everos.memory.strategies.extract_user_profile") + + +@pytest.fixture(autouse=True) +def _clean_locks() -> None: + _partition_locks._reset_for_tests() + + +def _lock(owner: str, subject: str) -> asyncio.Lock: + return _partition_locks.get_partition_lock( + "extract_user_profile", f"app:proj:{owner}::{subject}" + ) + + +def test_different_subjects_of_one_owner_get_different_locks() -> None: + """The property that keeps one stuck speaker from freezing the group.""" + assert _lock("group", "Lan Ye") is not _lock("group", "Bo Chen") + + +def test_the_same_subject_still_serialises() -> None: + """Granularity, not removal -- the file still needs a single writer.""" + assert _lock("group", "Lan Ye") is _lock("group", "Lan Ye") + + +def test_the_owner_lock_and_a_subject_lock_are_distinct() -> None: + """Owner mode and sender mode must not collide on one key. + + ```` and ``::`` are different partitions; a subject + named such that the two collide would reintroduce the stall on one owner. + """ + owner_key = _partition_locks.get_partition_lock( + "extract_user_profile", "app:proj:group" + ) + assert owner_key is not _lock("group", "Lan Ye") + + +def test_sender_mode_takes_no_owner_wide_lock() -> None: + """Per-sender runs must not hold an owner lock around the whole pass. + + Asserted on the source rather than by running the strategy because the stall + was structural: the ``async with`` spanned the subject loop, so any owner-wide + acquire there is the defect regardless of what the loop body does. + """ + # The decorator returns a Strategy; the original coroutine is on `.meta.func`. + code = mod.extract_user_profile.meta.func.__code__ + # The guard is chosen by mode; nullcontext is what makes sender mode lock-free + # at the owner level. + assert "nullcontext" in set(code.co_names) + # And the acquire must be conditional, not unconditional-then-ignored. + assert "PROFILE_SUBJECT" in set(code.co_names) + + +def test_per_subject_lock_wraps_the_body() -> None: + """``_extract_one_subject`` must acquire before doing any of the work. + + The body lives in ``_extract_one_subject_locked`` precisely so the lock is + unmissable at the boundary; if someone inlines it back, this fails. + """ + assert asyncio.iscoroutinefunction(mod._extract_one_subject) + assert asyncio.iscoroutinefunction(mod._extract_one_subject_locked) + names = set(mod._extract_one_subject.__code__.co_names) + assert "get_partition_lock" in names + assert "_extract_one_subject_locked" in names + # The wrapper must not do the work itself -- reading or writing before the + # acquire is the race the lock exists to prevent. + assert "_get_reader" not in names + assert "_persist_profile" not in names + + +async def test_two_subjects_make_progress_while_a_third_is_stuck() -> None: + """The end state the fix buys, expressed as a scheduling property.""" + stuck = asyncio.Event() + + async def work(subject: str) -> str: + async with _lock("group", subject): + if subject == "Lan Ye": + await stuck.wait() # never set: this one hangs forever + return subject + + hung = asyncio.create_task(work("Lan Ye")) + await asyncio.sleep(0) + done = await asyncio.gather(work("Bo Chen"), work("Mei Zheng")) + assert done == ["Bo Chen", "Mei Zheng"] + assert not hung.done() + hung.cancel() + with pytest.raises(asyncio.CancelledError): + await hung diff --git a/tests/unit/test_memory/test_strategies/test_profile_subject_sender.py b/tests/unit/test_memory/test_strategies/test_profile_subject_sender.py new file mode 100644 index 000000000..06f95a38e --- /dev/null +++ b/tests/unit/test_memory/test_strategies/test_profile_subject_sender.py @@ -0,0 +1,375 @@ +"""Per-sender profile subjects — ``EVEROS_PROFILE_SUBJECT=sender``. + +The default (``owner``) is correct whenever an owner is one person, and the +existing suites cover it. These tests pin the group shape: many people share +one owner (that is what keeps a group chat's retrieval in one partition), so +the subject has to come from the speaker rather than the owner, or the +extractor is handed N people's turns under one name and synthesises a +composite of nobody. + +What is asserted here is the mechanism, not the LLM: subject discovery, the +sender re-keying that lets ``aextract`` accept a name, the filename/row-id +split, and the fact that turning the switch off changes nothing. +""" + +from __future__ import annotations + +import importlib +import itertools +import json +from pathlib import Path + +import pytest +from everalgo.types import ChatMessage as AlgoChatMessage +from everalgo.types import MemCell as AlgoMemCell +from everalgo.types import Profile as AlgoProfile + +from everos.memory.search.recall.profile import _subject_of + +# The module, not the package's re-export: ``@offline_strategy`` replaces the +# module-level name with a ``Strategy`` object, so ``from ... import +# extract_user_profile`` hands back the strategy and none of the helpers. +eup = importlib.import_module("everos.memory.strategies.extract_user_profile") + + +_SEQ = itertools.count(1) + + +def _msg( + *, sender_id: str, sender_name: str | None, role: str = "user", text: str = "hi" +) -> AlgoChatMessage: + return AlgoChatMessage.model_validate( + { + "id": f"m{next(_SEQ)}", + "role": role, + "content": text, + "timestamp": 1_700_000_000_000, + "sender_id": sender_id, + "sender_name": sender_name, + } + ) + + +def _cell(*messages: AlgoChatMessage) -> AlgoMemCell: + return AlgoMemCell.model_validate( + {"items": list(messages), "timestamp": 1_700_000_000_000} + ) + + +# ── subject discovery ──────────────────────────────────────────────────── + + +def test_subjects_prefer_sender_name_over_the_shared_owner() -> None: + """A group ingest pins every sender_id to the owner; names carry the person.""" + cell = _cell( + _msg(sender_id="01", sender_name="Weihua Zhang"), + _msg(sender_id="01", sender_name="Mingzhi Li"), + _msg(sender_id="01", sender_name="Weihua Zhang"), + ) + assert eup._subjects_of([cell]) == ["Weihua Zhang", "Mingzhi Li"] + + +def test_subjects_skip_assistant_turns() -> None: + """``aextract`` rejects an assistant outright, so it never becomes a subject.""" + cell = _cell( + _msg(sender_id="01", sender_name="Lan Ye"), + _msg(sender_id="01", sender_name="Helper", role="assistant"), + ) + assert eup._subjects_of([cell]) == ["Lan Ye"] + + +def test_subjects_fall_back_to_sender_id_when_unnamed() -> None: + cell = _cell(_msg(sender_id="caroline_conv0", sender_name=None)) + assert eup._subjects_of([cell]) == ["caroline_conv0"] + + +def test_subjects_span_memcells_in_first_seen_order() -> None: + cells = [ + _cell(_msg(sender_id="01", sender_name="B")), + _cell( + _msg(sender_id="01", sender_name="A"), _msg(sender_id="01", sender_name="B") + ), + ] + assert eup._subjects_of(cells) == ["B", "A"] + + +# ── the re-keying that makes ``aextract`` accept a name ────────────────── + + +def test_retarget_rekeys_user_turns_and_leaves_the_source_untouched() -> None: + """``aextract`` validates sender_id against the memcells' own user senders.""" + cell = _cell( + _msg(sender_id="01", sender_name="Lan Ye"), + _msg(sender_id="01", sender_name="Bot", role="assistant"), + ) + out = eup._retarget([cell], "Lan Ye") + + assert [m.sender_id for m in out[0].items] == ["Lan Ye", "01"] + # Assistant turns keep the owner id -- only a user turn can be a subject. + assert [m.sender_id for m in cell.items] == ["01", "01"], "source was mutated" + + +def test_retarget_output_satisfies_the_extractor_validation() -> None: + """The exact predicate ``ProfileExtractor.aextract`` raises on.""" + from everalgo.user_memory.profile import _user_senders + + cell = _cell(_msg(sender_id="01", sender_name="Lan Ye")) + assert "Lan Ye" not in _user_senders([cell]), "precondition" + assert "Lan Ye" in _user_senders(eup._retarget([cell], "Lan Ye")) + + +# ── filename / row-id split ────────────────────────────────────────────── + + +def test_slug_is_filename_safe_and_stable() -> None: + assert eup._subject_slug("Lan Ye") == "Lan_Ye" + assert eup._subject_slug("Zhang/Wei..") == "Zhang_Wei" + assert eup._subject_slug("///") == "unnamed" + assert eup._subject_slug("Lan Ye") == eup._subject_slug("Lan Ye") + + +def test_subject_filename_sits_under_the_kind_glob() -> None: + """The cascade globs each kind once, so the file must match ``user*.md``.""" + from pathlib import PurePosixPath + + from everos.infra.persistence.markdown import UserProfileFrontmatter + + glob = UserProfileFrontmatter.path_glob() + for name in ("user.md", eup._subject_filename("Lan Ye")): + rel = f"default_app/default_project/users/01/{name}" + assert PurePosixPath(rel).match(glob), (rel, glob) + + +def test_row_id_round_trips_through_the_recaller() -> None: + """The subject rides the PK because a new column locks old stores out.""" + assert _subject_of("01::Lan Ye", "01") == "Lan Ye" + assert _subject_of("01", "01") == "" + # A name containing the delimiter still survives: the prefix is stripped, not split. + assert _subject_of("01::a::b", "01") == "a::b" + + +# ── the switch is off by default ───────────────────────────────────────── + + +def test_owner_is_the_default_subject_mode() -> None: + assert eup.PROFILE_SUBJECT == eup.SUBJECT_OWNER + + +# ── per-subject watermark + memcell filtering ──────────────────────────── + + +def test_speaks_in_matches_only_the_subject_s_own_user_turns() -> None: + cell = _cell( + _msg(sender_id="01", sender_name="Lan Ye"), + _msg(sender_id="01", sender_name="Bot", role="assistant"), + ) + assert eup._speaks_in(cell, "Lan Ye") + assert not eup._speaks_in(cell, "Jing Lv") + # An assistant turn is never a subject's evidence, even by name. + assert not eup._speaks_in(cell, "Bot") + + +def test_filtering_keeps_the_surrounding_turns_of_a_kept_memcell() -> None: + """Dropping a memcell drops a meeting, not the other people in it.""" + cell = _cell( + _msg(sender_id="01", sender_name="Lan Ye", text="ops view"), + _msg(sender_id="01", sender_name="Weihua Zhang", text="director view"), + ) + assert eup._speaks_in(cell, "Lan Ye") + kept = eup._retarget([cell], "Lan Ye") + texts = [str(m.content) for m in kept[0].items] + assert any("director view" in t for t in texts), "context was stripped" + + +# ── extraction trace ───────────────────────────────────────────────────── + + +def test_trace_is_off_unless_the_env_names_a_path( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Whitespace is not a path: a blank env value must not create a file.""" + monkeypatch.delenv(eup.PROFILE_TRACE_ENV, raising=False) + assert eup._trace_path() is None + monkeypatch.setenv(eup.PROFILE_TRACE_ENV, " ") + assert eup._trace_path() is None + target = tmp_path / "p.jsonl" + monkeypatch.setenv(eup.PROFILE_TRACE_ENV, str(target)) + assert eup._trace_path() == str(target) + eup._append_trace({"kind": "profile_extract", "owner_id": "01"}) + assert json.loads(target.read_text())["owner_id"] == "01" + + +def test_trace_write_failure_never_propagates( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Losing a trace line must not lose a profile.""" + monkeypatch.setenv(eup.PROFILE_TRACE_ENV, str(tmp_path / "nope" / "p.jsonl")) + eup._append_trace({"kind": "profile_extract"}) # parent dir absent -> OSError + + +def test_trace_record_is_json_serialisable_with_algo_objects( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Algo profile items are heterogeneous dicts; ``default=str`` covers them.""" + target = tmp_path / "p.jsonl" + monkeypatch.setenv(eup.PROFILE_TRACE_ENV, str(target)) + eup._append_trace({"kind": "profile_extract", "obj": object(), "n": {1, 2}}) + assert "profile_extract" in json.loads(target.read_text())["kind"] + + +def test_profile_shape_reports_what_the_compact_threshold_acts_on() -> None: + """Counts, not text: 45 items triggers compaction, 30 is the cap.""" + assert eup._profile_shape(None) == {"exists": False} + prof = AlgoProfile.model_validate( + { + "owner_id": "01", + "summary": "abc", + "timestamp": 1, + "explicit_info": [{"category": "a", "description": "b"}], + "implicit_traits": [{"trait": "x"}, {"trait": "y"}], + } + ) + assert eup._profile_shape(prof) == { + "exists": True, + "explicit_info": 1, + "implicit_traits": 2, + "summary_chars": 3, + } + + +# ── INIT language retry ────────────────────────────────────────────────── +# +# Measured on EverMemBench topic 01 (gpt-4.1-mini, 36 INIT calls): 8 of 36 (22%) +# produced a Chinese profile from an all-English corpus, and all 31 subjects +# extracted more than once kept their first language with ZERO exceptions. INIT +# fixes the language, so that one call is what has to be retried. + + +@pytest.mark.parametrize( + ("source", "produced", "mismatch"), + [ + # The real failure: English corpus, Chinese profile. Verbatim from the trace. + ( + "Good morning everyone. Today we launch the Carbon Emission platform.", + "Mingzhi Li 是技术部门成员，负责技术架构和数据集成相关工作。", + True, + ), + # The same rule in reverse, so a Chinese corpus is not left unguarded. + ( + "大家早上好，今天我们启动碳排放平台项目。", + "Mingzhi Li leads the tech team.", + True, + ), + # Compliant: both English. + ("Good morning everyone.", "Weihua Zhang is leading the launch.", False), + # Compliant: both Chinese. + ("大家早上好。", "张伟华负责启动该项目。", False), + # A few borrowed product names must NOT trip it -- under the 5% floor. + ( + "We evaluated the 碳核算 module against ISO 14064 for the whole quarter " + "and agreed the reporting pipeline needs a second pass before launch.", + "The team evaluated the accounting module against ISO 14064.", + False, + ), + ("", "", False), + ], +) +def test_language_mismatch_only_fires_on_a_script_switch( + source: str, produced: str, mismatch: bool +) -> None: + assert eup._language_mismatch(source, produced) is mismatch + + +def test_source_language_names_the_input_for_the_directive() -> None: + zh = _cell(_msg(sender_id="01", sender_name="A", text="大家早上好，今天启动项目。")) + en = _cell(_msg(sender_id="01", sender_name="A", text="Good morning everyone.")) + assert eup._source_language([zh]) == "Chinese" + assert eup._source_language([en]) == "English" + + +def _profile(summary: str) -> AlgoProfile: + return AlgoProfile.model_validate( + { + "owner_id": "01", + "summary": summary, + "timestamp": 1, + "explicit_info": [], + "implicit_traits": [], + } + ) + + +class _Extractor: + """Records each call so the test can assert what the retry sent.""" + + def __init__(self, *outputs: str) -> None: + self._outputs = list(outputs) + self.calls: list[dict] = [] + + async def aextract(self, memcells, **kwargs): + self.calls.append(kwargs) + return _profile(self._outputs[len(self.calls) - 1]) + + +async def test_init_retries_once_with_the_language_pinned() -> None: + """The retry appends to the bundled prompt; it must not replace it.""" + cell = _cell(_msg(sender_id="01", sender_name="A", text="Good morning everyone.")) + ex = _Extractor("张三是技术负责人。", "A leads the tech team.") + + profile, retried = await eup._aextract_language_checked( + ex, [cell], sender_id="A", old_profile=None + ) + + assert retried is True + assert profile.summary == "A leads the tech team." + assert len(ex.calls) == 2 + assert ex.calls[0].get("prompt") is None, "first call must use the bundled prompt" + sent = ex.calls[1]["prompt"] + assert sent.startswith(eup.PROFILE_INITIAL_EXTRACTION_PROMPT), "prompt was replaced" + assert "English" in sent + # A retry is a fresh INIT, not an update onto the rejected profile. + assert ex.calls[1]["old_profile"] is None + + +async def test_compliant_init_does_not_retry() -> None: + cell = _cell(_msg(sender_id="01", sender_name="A", text="Good morning everyone.")) + ex = _Extractor("A leads the tech team.") + profile, retried = await eup._aextract_language_checked( + ex, [cell], sender_id="A", old_profile=None + ) + assert retried is False + assert len(ex.calls) == 1 + assert profile.summary == "A leads the tech team." + + +async def test_update_is_never_retried() -> None: + """UPDATE emits ops onto an existing profile and inherits its language.""" + cell = _cell(_msg(sender_id="01", sender_name="A", text="Good morning everyone.")) + ex = _Extractor("张三是技术负责人。") + _profile_out, retried = await eup._aextract_language_checked( + ex, [cell], sender_id="A", old_profile=_profile("张三是技术负责人。") + ) + assert retried is False + assert len(ex.calls) == 1 + + +async def test_retry_is_kept_even_when_it_also_fails() -> None: + """A second sample is no worse than the first; breaking a tie needs a third call.""" + cell = _cell(_msg(sender_id="01", sender_name="A", text="Good morning everyone.")) + ex = _Extractor("张三是技术负责人。", "李四也是中文的。") + profile, retried = await eup._aextract_language_checked( + ex, [cell], sender_id="A", old_profile=None + ) + assert retried is True + assert profile.summary == "李四也是中文的。" + + +async def test_retry_can_be_switched_off(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(eup, "LANGUAGE_RETRY", False) + cell = _cell(_msg(sender_id="01", sender_name="A", text="Good morning everyone.")) + ex = _Extractor("张三是技术负责人。") + _p, retried = await eup._aextract_language_checked( + ex, [cell], sender_id="A", old_profile=None + ) + assert retried is False + assert len(ex.calls) == 1 diff --git a/tests/unit/test_memory/test_strategies/test_trigger_profile_clustering.py b/tests/unit/test_memory/test_strategies/test_trigger_profile_clustering.py index b4d28622b..fa45aa45d 100644 --- a/tests/unit/test_memory/test_strategies/test_trigger_profile_clustering.py +++ b/tests/unit/test_memory/test_strategies/test_trigger_profile_clustering.py @@ -2,7 +2,8 @@ Mirrors the skill-side test layout: mock embedder + cluster_repo + cluster_by_geometry, drive the strategy via :class:`FakeStrategyContext`, -verify a single :class:`ProfileClusterUpdated` event is emitted. +verify it emits nothing: the profile no longer keys off clustering, and the +clusters it writes are read by ``agentic`` retrieval and Reflection instead. """ from __future__ import annotations @@ -18,7 +19,7 @@ from everos.component.embedding import EmbeddingCapability, EmbeddingProvider from everos.infra.ome.testing import FakeStrategyContext from everos.memory._partition_locks import _reset_for_tests -from everos.memory.events import EpisodeExtracted, ProfileClusterUpdated +from everos.memory.events import EpisodeExtracted from everos.memory.strategies.trigger_profile_clustering import ( trigger_profile_clustering, ) @@ -69,7 +70,9 @@ async def test_strategy_meta_is_attached() -> None: meta = trigger_profile_clustering.meta assert meta.name == "trigger_profile_clustering" assert EpisodeExtracted in meta.trigger.on - assert meta.emits == frozenset({ProfileClusterUpdated}) + # Emits nothing: `extract_user_profile` was the only consumer and it now + # triggers directly off `EpisodeExtracted`. + assert meta.emits == frozenset() assert meta.max_retries == 2 assert meta.applies_to is not None @@ -125,11 +128,7 @@ async def test_creates_new_cluster_when_no_existing( "project_id": "default", } - emitted = [e for e in ctx.emitted if isinstance(e, ProfileClusterUpdated)] - assert len(emitted) == 1 - assert emitted[0].memcell_id == "mc_aaaaaaaaaaa1" - assert emitted[0].cluster_id == "cl_newuser00001" - assert emitted[0].owner_id == "u_alice" + assert ctx.emitted == [] matching = [r for r in captured if r.get("event") == "profile_cluster_updated"] assert matching, "expected profile_cluster_updated log line" @@ -180,9 +179,7 @@ async def test_merges_into_existing_cluster_when_algo_matches( assert persisted.id == "cl_existing0001" assert persisted.count == 2 - emitted = [e for e in ctx.emitted if isinstance(e, ProfileClusterUpdated)] - assert len(emitted) == 1 - assert emitted[0].cluster_id == "cl_existing0001" + assert ctx.emitted == [] # ── partition lock (owner_id-level serialisation) ──────────────────────── From d8ee0741aa42a772eed8390499569820c7d016a5 Mon Sep 17 00:00:00 2001 From: "juwei.yue" Date: Thu, 27 Aug 2026 09:53:34 +0000 Subject: [PATCH 5/8] feat(benchmarks): one reproducible runner for all four benchmarks The four benchmarks each had their own driver, so a fix to one never reached the others and no two numbers were produced by the same code. They now share `run.py` and differ only in `adapters/.py`, with every knob that decides a number living in `configs/.toml` -- a run is reproducible from its config, and `reproduce.sh` passes no model overrides. Defects this closes, each of which produced a complete-looking wrong number: Profile injection was implemented only in the EverMemBench adapter; the other three accepted `include_profile` and discarded the profiles. The rendering is now shared, and returns the memories unchanged when there is no profile so existing prompts stay byte-identical. A decider model name with no endpoint went to the extraction endpoint, 404ed on every call, and fell back silently. `run.py` now probes the decider with a real call before the first question and refuses to start if it does not answer; `--decider-base-url` exists so the pair can be set together, and both are folded into `run_spec.json` so it records what actually ran. A store whose owners were built under different partition keys returned zero episodes and scored ~0.5% with no error. SEARCH now asserts the owner exists in the store first, and names what the store does hold when it does not. `.env.example` shipped path placeholders that looked configured, which beat the config defaults and sent runs at `/01/dialogue.json`. Unresolved `${VAR}` is now detected and reported by variable name. --- .gitignore | 67 + Makefile | 21 +- benchmarks/.env.example | 66 + benchmarks/README.md | 452 +-- benchmarks/README.zh.md | 254 ++ benchmarks/adapters/__init__.py | 36 + benchmarks/adapters/_profile.py | 66 + benchmarks/adapters/base.py | 160 + benchmarks/adapters/evermembench.py | 489 +++ benchmarks/adapters/locomo.py | 197 ++ benchmarks/adapters/longmemeval.py | 369 ++ benchmarks/adapters/subtlememory.py | 725 ++++ benchmarks/config.py | 326 +- benchmarks/config.toml | 33 - benchmarks/configs/default.toml | 32 + benchmarks/configs/evermembench.toml | 113 + benchmarks/configs/locomo.toml | 67 + benchmarks/configs/longmemeval.toml | 64 + benchmarks/configs/subtlememory.toml | 74 + benchmarks/data/.gitkeep | 0 benchmarks/metrics/__init__.py | 10 + benchmarks/metrics/core.py | 131 + benchmarks/metrics/ir.py | 75 + benchmarks/reproduce.sh | 77 + benchmarks/run.py | 3058 +++++++++++++++-- docs/openapi.json | 100 +- pyproject.toml | 22 +- tests/unit/test_benchmark_answer_route.py | 152 + tests/unit/test_benchmark_config.py | 76 + tests/unit/test_benchmark_decider_endpoint.py | 215 ++ .../unit/test_benchmark_fleet_shard_guard.py | 142 + tests/unit/test_benchmark_harness_own.py | 652 ++++ tests/unit/test_benchmark_last_seventeen.py | 428 +++ tests/unit/test_benchmark_no_exemptions.py | 353 ++ tests/unit/test_benchmark_operational.py | 540 +++ tests/unit/test_benchmark_owner_in_store.py | 106 + tests/unit/test_benchmark_pipeline.py | 336 ++ .../unit/test_benchmark_profile_injection.py | 144 + tests/unit/test_benchmark_profile_subject.py | 136 + .../test_benchmark_profile_trace_config.py | 139 + tests/unit/test_benchmark_prompts.py | 98 + .../test_benchmark_quiesce_between_passes.py | 122 + tests/unit/test_benchmark_store_default.py | 67 + uv.lock | 2958 ++++++++-------- 44 files changed, 11639 insertions(+), 2109 deletions(-) create mode 100644 benchmarks/README.zh.md create mode 100644 benchmarks/adapters/__init__.py create mode 100644 benchmarks/adapters/_profile.py create mode 100644 benchmarks/adapters/base.py create mode 100644 benchmarks/adapters/evermembench.py create mode 100644 benchmarks/adapters/locomo.py create mode 100644 benchmarks/adapters/longmemeval.py create mode 100644 benchmarks/adapters/subtlememory.py delete mode 100644 benchmarks/config.toml create mode 100644 benchmarks/configs/default.toml create mode 100644 benchmarks/configs/evermembench.toml create mode 100644 benchmarks/configs/locomo.toml create mode 100644 benchmarks/configs/longmemeval.toml create mode 100644 benchmarks/configs/subtlememory.toml create mode 100644 benchmarks/data/.gitkeep create mode 100644 benchmarks/metrics/__init__.py create mode 100644 benchmarks/metrics/core.py create mode 100644 benchmarks/metrics/ir.py create mode 100755 benchmarks/reproduce.sh create mode 100644 tests/unit/test_benchmark_answer_route.py create mode 100644 tests/unit/test_benchmark_decider_endpoint.py create mode 100644 tests/unit/test_benchmark_fleet_shard_guard.py create mode 100644 tests/unit/test_benchmark_harness_own.py create mode 100644 tests/unit/test_benchmark_last_seventeen.py create mode 100644 tests/unit/test_benchmark_no_exemptions.py create mode 100644 tests/unit/test_benchmark_operational.py create mode 100644 tests/unit/test_benchmark_owner_in_store.py create mode 100644 tests/unit/test_benchmark_pipeline.py create mode 100644 tests/unit/test_benchmark_profile_injection.py create mode 100644 tests/unit/test_benchmark_profile_subject.py create mode 100644 tests/unit/test_benchmark_profile_trace_config.py create mode 100644 tests/unit/test_benchmark_prompts.py create mode 100644 tests/unit/test_benchmark_quiesce_between_passes.py create mode 100644 tests/unit/test_benchmark_store_default.py diff --git a/.gitignore b/.gitignore index 120b2b24a..33d5ce36f 100755 --- a/.gitignore +++ b/.gitignore @@ -260,8 +260,16 @@ evaluation/src/adapters/*/prompts/profile/*.json # Locomo source dataset (downloadable, not source code) data/locomo10.json +# Benchmark inputs live here (benchmarks/configs/*.toml default to it), and they are +# large, redistributable only by their own authors, and not ours to ship. The directory +# itself is kept so a fresh clone has somewhere obvious to put them. +benchmarks/data/* +!benchmarks/data/.gitkeep evaluation/data/locomo/locomo10.json evaluation/locomo_evaluation/data/locomo10.json +# Benchmark inputs live here (benchmarks/configs/*.toml default to it), and they are +# large, redistributable only by their own authors, and not ours to ship. The directory +# itself is kept so a fresh clone has somewhere obvious to put them. # Legacy src kept locally for migration reference; not under version control. src_old/ @@ -280,3 +288,62 @@ benchmarks/.env # Local everos runtime data (memory root, indexes, OME state) .everos*/ + +# --------------------------------------------------------------------------- +# Internal-only benchmark scaffolding — kept in the working tree, never shipped +# --------------------------------------------------------------------------- +# These exist to run ablations on this machine, not to let anyone reproduce a +# published number. Two things make them unshippable rather than merely +# unpolished: they hardcode absolute paths into a private workspace +# (/Evermind/..., /root/fullrun, /root/.everos_bench), and the audit set compares +# against reference harnesses that live outside this repository and are not +# published — so a user who cloned this would get code that cannot run at all. +# +# The shippable harness is what remains: run.py + config.py + adapters/ + +# configs/ + metrics/ + scripts/reproduce.sh + README.md + .env.example. + +# Everything that measures something OTHER than a benchmark's published number: +# extraction-backbone arms, retrieval-policy arms, and the two store helpers that +# only those flows use (rewriting an existing store's episodes with a different +# extractor, and waiting for the index to catch up afterwards). Both sweep drivers +# read checkpoints out of a private path, and one needs a train/test split file +# from another repository, so neither can run anywhere else. +# +# The shipped surface is the reproduction and nothing else: reproduce.sh + run.py + +# config.py + adapters/ + configs/ + metrics/ + README.md + .env.example. +benchmarks/ablations/ + +# Migration audit. Every file here resolves paths under Evaluation/ to diff this +# harness against the reference implementations it was ported from; without that +# checkout none of it executes. MIGRATION*.md are the internal record of that +# port, not user documentation. +benchmarks/audit/ + +# Tests that require the same absent checkout. benchmark_parity_env.py says it +# plainly: 269 of the 395 benchmark tests skip when the references are missing, +# and `make test` still reports success — so shipping them hands users a suite +# that is permanently green and permanently vacuous. +tests/unit/benchmark_parity_env.py +tests/unit/test_benchmark_differential.py +tests/unit/test_benchmark_differential_audit.py +tests/unit/test_benchmark_differential_lme_emb.py +tests/unit/test_benchmark_differential_phases.py +tests/unit/test_benchmark_differential_pool.py +tests/unit/test_benchmark_differential_surface.py +tests/unit/test_benchmark_parity.py +tests/unit/test_benchmark_parity_gate.py +tests/unit/test_benchmark_parity_reported.py +# Also diffs against LoCoMo/EverOS/test_locomo.py (line 27), so it belongs here +# rather than in the shipped suite. +tests/unit/test_benchmark_client_and_gate.py + +# Pins the ablation tooling above, so it goes with it. A test whose subject is not +# shipped would fail at import on a fresh clone, not skip. +tests/unit/test_benchmark_reextract_sampling.py +tests/unit/test_benchmark_metrics_and_reextract.py +tests/unit/test_benchmark_store_tools.py +tests/unit/test_benchmark_ablation_tools.py + +# Hardcodes four store paths on this machine (/root/smoke_*). Marked `slow` so +# it is deselected by default, but the paths would still be published. +tests/unit/test_multiround_per_store.py diff --git a/Makefile b/Makefile index 139f16cec..768c09ad4 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install install-deps lint docs-check check-commits check-pr-title check-assets check-file-sizes check-deprecated-names check-github-docs check-cjk check-datetime openapi check-openapi format test integration package cov ci clean +.PHONY: help install install-deps lint docs-check check-commits check-pr-title check-assets check-file-sizes check-deprecated-names check-github-docs check-cjk check-datetime openapi check-openapi format test integration package cov ci verify-parity clean help: @echo "Targets:" @@ -36,8 +36,8 @@ install: install-deps uv run pre-commit install --hook-type commit-msg lint: - uv run ruff check src tests - uv run ruff format --check src tests + uv run ruff check src tests benchmarks + uv run ruff format --check src tests benchmarks uv run lint-imports uv run python scripts/check_repo_assets.py uv run python scripts/check_file_sizes.py @@ -105,12 +105,23 @@ check-openapi: uv run python scripts/dump_openapi.py --check format: - uv run ruff check --fix src tests - uv run ruff format src tests + uv run ruff check --fix src tests benchmarks + uv run ruff format src tests benchmarks test: uv run pytest tests/unit -v +# The differential tests compare against a checkout outside this repository, so all of +# them skip when it is absent -- 269 of the 395 benchmark tests, and `make test` still +# reported success. This target refuses to skip: run it before any evaluation, because a +# number produced by an unverified harness is worse than no number. +verify-parity: + BENCHMARK_PARITY_STRICT=1 uv run pytest tests/unit -q -k benchmark + uv run python benchmarks/audit/check_parity.py + uv run python benchmarks/audit/check_protocol.py + uv run python benchmarks/audit/coverage.py + uv run python benchmarks/audit/partition.py + integration: uv run pytest tests/integration -v diff --git a/benchmarks/.env.example b/benchmarks/.env.example index b8af07bb6..150e816a2 100644 --- a/benchmarks/.env.example +++ b/benchmarks/.env.example @@ -4,3 +4,69 @@ ANSWER_API_KEY=sk-... ANSWER_BASE_URL=https://openrouter.ai/api/v1 JUDGE_API_KEY=sk-... JUDGE_BASE_URL=https://openrouter.ai/api/v1 + +# --- Infrastructure the servers need. Model choice does NOT belong here: +# it lives in configs/.toml, which is the single source of the +# experiment's identity. A model name in this file was silently overridden by +# the config at launch while still being what the run record reported. --- +# Required when using --servers: the fleet inherits this process's environment, and a +# server whose LLM is unconfigured aborts at startup ("LLM api_key and base_url is not +# configured"). `everos init` only writes a blank template, so these belong here rather +# than in each generated everos.toml. +EVEROS_LLM__BASE_URL=https://openrouter.ai/api/v1 +EVEROS_LLM__API_KEY= +EVEROS_LLM__TIMEOUT_SECONDS=300 +EVEROS_EMBEDDING__BASE_URL=http://127.0.0.1:9200/v1 +EVEROS_EMBEDDING__MODEL=Qwen3-Embedding-4B +EVEROS_EMBEDDING__API_KEY= +# Engine-wide cap on concurrent extractions. The default of 20 throttles a bulk ingest +# long before the machine does -- each slot spends its time waiting on a remote call. +EVEROS_OME_MAX_CONCURRENT_RUNS=64 + +# --------------------------------------------------------------------------- +# Paths (all optional — every one has a working default) +# --------------------------------------------------------------------------- +# LEFT EMPTY ON PURPOSE. Each of these has a default that works out of the box, and +# an empty value falls through to it. A placeholder like `/path/to/Evaluation` would +# NOT: it is a set variable, so it wins over the default, and the run then writes to +# a directory literally called `/path/to/...`. A value that looks configured but is +# not is worse than no value at all. +# +# Fill one in only when the real path is somewhere else. + +# Datasets. Default: benchmarks/data/. +# locomo -> benchmarks/data/locomo10.json +# longmemeval -> benchmarks/data/longmemeval_s.json +# subtlememory -> benchmarks/data/subtlememory/ (a directory) +# evermembench -> benchmarks/data/evermembench.json +BENCH_DATA_LOCOMO= +BENCH_DATA_LONGMEMEVAL= +BENCH_DATA_SUBTLEMEMORY= +BENCH_DATA_EVERMEMBENCH= + +# EverMemBench only: its raw release, read to recover session names the converted +# file does not carry. Default: benchmarks/data/raw/EverMemBench-Dynamic. +EVERMEMBENCH_RAW_ROOT= + +# Where runs are written. Default: benchmarks/results//. Point this at a +# directory outside the repository when the filesystem holding it is near capacity — +# a full disk surfaces as [Errno 28] on every answer write, which is retried and then +# recorded as a failed question rather than as an outage. +BENCH_EVAL_ROOT= + +# --------------------------------------------------------------------------- +# The multi-round retrieval decider (optional) +# --------------------------------------------------------------------------- +# Leave BOTH empty and the decider runs the model from [llm] — the single-model +# setup, which needs no extra configuration and is what a first run should use. +# +# Set them TOGETHER to give the decider its own model. A name without its endpoint +# returns 404 on every call, and the retrieval loop then falls back to a fixed core +# and still reports a complete result. run.py probes the decider at startup and +# refuses to run rather than let that happen quietly. +# +# Our published numbers used BENCH_DECIDER_MODEL=qwen3.6-27B. +BENCH_DECIDER_MODEL= +BENCH_DECIDER_BASE_URL= +# Any non-empty value for a self-hosted server; a hosted API needs its real key. +BENCH_DECIDER_API_KEY=EMPTY diff --git a/benchmarks/README.md b/benchmarks/README.md index 416f683bf..5201f38ac 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,338 +1,268 @@ -# Running the LoCoMo Benchmark +# Running the benchmarks -EverOS ships a self-contained runner for the -[LoCoMo](https://github.com/snap-research/locomo) (Long Conversation Memory) -benchmark ([Maharana et al., 2024](https://arxiv.org/abs/2402.17753)). -LoCoMo evaluates how well a memory system retrieves facts from long -multi-session dialogues across four question categories: **single-hop**, -**multi-hop**, **open-domain**, and **temporal**. This guide walks through -reproducing EverOS's LoCoMo retrieval scores locally. +EverOS ships a self-contained runner for four long-term-memory benchmarks: +[LoCoMo](https://github.com/snap-research/locomo) +([Maharana et al., 2024](https://arxiv.org/abs/2402.17753)), +[LongMemEval](https://github.com/xiaowu0162/LongMemEval), SubtleMemory, and +EverMemBench. -## Pipeline at a glance +All four run through one pipeline and differ only in a per-dataset adapter. +`benchmarks/adapters/.py` is where each benchmark's own rules live — how it +names an owner, what counts as gold evidence, how its judge grades. Nothing else +in the runner knows which benchmark it is running. -Each conversation runs through a four-stage pipeline: - -``` -ADD ──► wait_ready ──► SEARCH ──► ANSWER ──► JUDGE - │ │ │ │ │ - │ ingest msgs & query EverOS generate LLM-as-judge - │ flush per-session per QA pair answers majority vote - │ into EverOS from (judge_runs×) - │ context - ▼ - cascade + OME drain - (per-conv polling) -``` - -- **ADD** — sends LoCoMo sessions to EverOS (`/add` + `/flush`), then polls - cascade and OME queues until the conversation's data is fully indexed. -- **SEARCH** — queries EverOS `/search` for each QA question. -- **ANSWER** — feeds retrieved episodes to an LLM to generate an answer. -- **JUDGE** — an LLM judge scores each answer as CORRECT or WRONG against - the gold answer; runs `judge_runs` times per question and majority-votes. - -Stages are **independently re-runnable** — each reads from and writes to -JSONL files, so you can re-judge with a different model without re-ingesting -or re-searching. - -Multiple conversations run **in parallel** via `conversations_concurrency`. -Within each conversation, search and eval questions run concurrently via -`search_concurrency` and `eval_concurrency`. - -## Contents - -- [Prerequisites](#prerequisites) -- [Configuration](#configuration) -- [1. Prepare the dataset](#1-prepare-the-dataset) -- [2. Start the server](#2-start-the-server) -- [3. Run the benchmark](#3-run-the-benchmark) -- [4. Output](#4-output) -- [CLI reference](#cli-reference) -- [Notes](#notes) +> 中文版见 [README.zh.md](README.zh.md)。 --- -## Prerequisites +## Quickstart -- A working EverOS installation — complete **all steps** in - [QUICKSTART.md](../QUICKSTART.md) (configure providers, start server, - verify search works — not just `/health`) -- Python 3.12+ with `tqdm` installed (`pip install tqdm`) -- EverOS configured for chat-only extraction — in your `everos.toml`: +```bash +# 1. install +uv sync - ```toml - [memorize] - mode = "chat" - ``` +# 2. configure +cp benchmarks/.env.example benchmarks/.env +$EDITOR benchmarks/.env # answer/judge API keys, extraction backbone - And in `ome.toml`, disable strategies the benchmark does not use: +# 3. put the dataset where the config expects it +curl -o benchmarks/data/locomo10.json \ + https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json - ```toml - [strategies.extract_foresight] - enabled = false +# 4. reproduce +DATASET=locomo bash benchmarks/reproduce.sh +``` - [strategies.extract_user_profile] - enabled = false - ``` +That last command runs the full pipeline — ADD, SEARCH, ANSWER, JUDGE — at the +configuration the published number was produced with, and writes a report under +`benchmarks/results/LoCoMo/locomo/`. - This keeps episode extraction, `extract_atomic_facts`, and - `trigger_profile_clustering` (agentic search relies on clusters), - while cutting unnecessary LLM calls from foresight and profile - extraction. +--- -- Copy `benchmarks/.env.example` → `benchmarks/.env` and fill in your API - keys: +## Layout -```bash -cp benchmarks/.env.example benchmarks/.env -# Edit benchmarks/.env: -ANSWER_API_KEY=sk-... # LLM for generating answers -ANSWER_BASE_URL=https://openrouter.ai/api/v1 -JUDGE_API_KEY=sk-... # LLM for judging answers -JUDGE_BASE_URL=https://openrouter.ai/api/v1 +``` +benchmarks/ +├── reproduce.sh the entry point; runs one benchmark end to end +├── run.py the pipeline (ADD → SEARCH → ANSWER → JUDGE) +├── config.py the frozen config model +├── configs/ +│ ├── default.toml shared defaults +│ └── .toml one per benchmark: models, top_k, concurrency +├── adapters/ +│ ├── base.py the four questions an adapter answers +│ └── .py one per benchmark: owners, gold, prompts, judge +├── metrics/ ranked-retrieval and core-selection metrics +├── data/ ← put the datasets here (git-ignored) +├── results/ ← runs are written here (git-ignored) +└── .env.example copy to .env ``` -Keys are comma-separated for round-robin failover (e.g. -`ANSWER_API_KEY=sk-aaa,sk-bbb`). More keys raise the effective RPM -ceiling, which lets you increase `eval_concurrency` in `config.toml` for -faster answer/judge throughput. - -## Configuration - -The only required configuration is provider credentials — copy -`benchmarks/.env.example` → `benchmarks/.env` and fill in your API keys -(already done in [Prerequisites](#prerequisites)). +`data/` and `results/` are the only two directories you write to. Both are +git-ignored; the datasets are redistributable only by their own authors, and a +run's output is large and regenerable. -Everything else has sensible defaults in `benchmarks/config.toml` — see -the comments in that file for tunable parameters. +--- ## 1. Prepare the dataset -LoCoMo 10 contains 10 multi-session conversations (~50 sessions each, -~150 QA pairs per conversation across 4 categories, adversarial -category excluded). +Each benchmark's input path comes from its config, which defaults to +`benchmarks/data/`. **Putting the file there is all that is required** — no +environment variable, no flag: -```bash -mkdir -p data -curl -o data/locomo10.json \ - https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json -``` +| Benchmark | Put the file at | Environment override | +|---|---|---| +| `locomo` | `benchmarks/data/locomo10.json` | `BENCH_DATA_LOCOMO` | +| `longmemeval` | `benchmarks/data/longmemeval_s.json` | `BENCH_DATA_LONGMEMEVAL` | +| `subtlememory` | `benchmarks/data/subtlememory/` (a directory of `persona_0..9/`) | `BENCH_DATA_SUBTLEMEMORY` | +| `evermembench` | `benchmarks/data/evermembench.json` | `BENCH_DATA_EVERMEMBENCH` | + +Set the environment variable only when the data lives somewhere else — a shared +mount, another disk. It is read from `benchmarks/.env`. -## 2. Start the server +### Getting each dataset -Raise the file descriptor limit **before** starting — concurrent agentic -searches open many LanceDB segment files simultaneously (EverOS compacts -segments automatically, but burst concurrency during benchmark can exceed -the default macOS limit of 256): +**LoCoMo** — 10 multi-session conversations, ~50 sessions each, ~150 QA pairs per +conversation across four categories (the adversarial category is excluded). ```bash -ulimit -n 10240 -everos server start [--root ] +curl -o benchmarks/data/locomo10.json \ + https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json ``` -> **Important:** if you use a custom `--root`, pass the same path to the -> benchmark runner via `--everos-root` — the runner polls the cascade and -> OME databases under that root to know when data is ready. A mismatch -> causes silent readiness false-positives. +**LongMemEval** — the `longmemeval_s` split from +[xiaowu0162/LongMemEval](https://github.com/xiaowu0162/LongMemEval). Place it as +`benchmarks/data/longmemeval_s.json`. -## 3. Run the benchmark +**SubtleMemory** and **EverMemBench** — released by their own authors; follow each +project's instructions. EverMemBench needs its raw release converted once; point +`EVERMEMBENCH_RAW_ROOT` at the raw directory, which the adapter reads to recover +session names the converted file does not carry. -All runs require `--run-name`, which becomes the `project_id` used for data -isolation (see [Run isolation](#run-isolation) below). +### When a path is wrong -**Smoke test first** — verify end-to-end connectivity before a full run: +A missing or misconfigured path **fails the run**; it does not load zero rows and +score 0%. An unset variable with no default reaches the loader as the literal +`${BENCH_DATA_LOCOMO}` and is reported by name, so the error says which variable +to set rather than describing a file called `${...}`. -```bash -python benchmarks/run.py --run-name smoke --smoke [--everos-root ] -``` +--- -**Full run (all 10 conversations):** +## 2. Configure ```bash -python benchmarks/run.py --run-name locomo-agentic [--everos-root ] +cp benchmarks/.env.example benchmarks/.env ``` -**Single conversation:** +The minimum for a first run: -```bash -python benchmarks/run.py --run-name locomo-agentic --conv 0 [--everos-root ] -``` +| Key | What it is | +|---|---| +| `ANSWER_API_KEY` / `ANSWER_BASE_URL` | the model that answers questions from retrieved memories | +| `JUDGE_API_KEY` / `JUDGE_BASE_URL` | the model that grades those answers | +| `EVEROS_LLM__MODEL` / `__API_KEY` / `__BASE_URL` | the extraction backbone EverOS runs during ADD | -**Skip ingest, re-run search + answer + judge:** +Everything that decides the number — models, `top_k`, retrieval knobs, concurrency +— lives in `benchmarks/configs/.toml`, not in the command line. That is +deliberate: a run is reproducible from its config, and `reproduce.sh` passes no +model overrides. -```bash -python benchmarks/run.py --run-name locomo-agentic --stages search answer judge -``` +### The multi-round decider (optional) + +`llm_multiround` retrieval runs a *decider* that reads candidate memories, picks +the core set, and asks follow-up sub-queries. By default it runs the same model as +extraction, which needs no extra configuration. -**Re-judge only (reuse existing answer JSONL):** +To give it its own model, set **both** together: ```bash -python benchmarks/run.py --run-name locomo-agentic --stages judge +BENCH_DECIDER_MODEL= +BENCH_DECIDER_BASE_URL= ``` -## 4. Output - -Output root is `benchmarks/results//`: +Setting only one is the failure this runner checks for at startup: a model name +sent to an endpoint that does not serve it returns 404 on every call, and the +retrieval loop then falls back to a fixed top-ranked core **and still reports a +complete result**. `run.py` probes the decider before the first question and +refuses to start if it does not answer. -``` -benchmarks/results// -├── run_spec.json # reproducibility snapshot (git hash, config, stages) -├── conv0/ -│ ├── search_.jsonl # per-question search results -│ ├── answer_.jsonl # per-question generated answers -│ ├── judge_.jsonl # per-question judge verdicts -│ └── error.log # only on failure — full traceback -├── conv1/ … conv9/ -├── report.json # aggregate accuracy by method + category -└── report.txt # human-readable accuracy table -``` +--- -`report.json` and `report.txt` are written after all conversations finish -(only when the `judge` stage is included). +## 3. Start the server (only if you want to manage it yourself) -**Sample `report.txt`:** +`reproduce.sh` starts and stops its own servers. Start one manually only when you +want to reuse a store across runs: -``` -================================================================ - EverOS LoCoMo Benchmark Report -================================================================ - -Run Info - Run name: locomo-agentic - Generated: 2026-06-28T14:30:00+00:00 - Git hash: abc1234 - EverOS version: 1.1.0 - Python: 3.12.11 - Conversations: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - Stages: ['add', 'search', 'answer', 'judge'] - -Configuration - Answer model: gpt-4.1-mini - Judge model: gpt-4o-mini - Judge runs: 3 - Top-k: 10 - Eval owner: speaker_a - ----------------------------------------------------------------- - Method: agentic ----------------------------------------------------------------- - - Max accuracy: 93.4% (best of 3 judge runs / mean / majority) - Majority: 93.3% (1437/1540) - Mean accuracy: 93.3% (avg across 3 judge runs) - - Per category: - 1. single-hop 94.0% (265/282) - 2. multi-hop 91.0% (292/321) - 3. open-domain 80.2% (77/96) - 4. temporal 95.5% (803/841) - - Per conversation: - conv0 93.4% (142/152) - conv1 96.3% (78/81) - ... - - Search: 1540 queries, avg 23.1s, p50 19.4s, max 142.3s - Answer: 1540 questions, avg 4.7s, 7,224,168 tokens - Judge: 1540 questions × 3 runs, 2,335,683 tokens, unanimous 95.2% - - Total tokens: 9,559,851 +```bash +ulimit -n 10240 # concurrent searches open many LanceDB segment files +everos server start [--root ] ``` -## CLI reference +Pass the same path to the runner with `--everos-root`. The runner polls the +cascade and OME databases under that root to know when data is ready, so a +mismatch causes silent readiness false-positives. -| Flag | Default | Description | -|---|---|---| -| `--run-name` | *(required)* | Run name — maps to `project_id` for data isolation | -| `--conv` | `0 1 2 … 9` | Conversation indices to run | -| `--stages` | `add search answer judge` | Pipeline stages to execute | -| `--config` | `config` | TOML config name (without `.toml` extension) | -| `--base-url` | `http://localhost:8000` | EverOS server address | -| `--everos-root` | `~/.everos` | EverOS root path (for cascade/OME queue polling) | -| `--data-path` | `data/locomo10.json` | Path to LoCoMo dataset JSON | -| `--smoke` | off | Smoke mode: 2 convs, first 50 msgs each, 10 QA (stratified), `judge_runs=1` | +--- -## Notes +## 4. Run -### Evaluation methodology +```bash +# full reproduction, all conversations, all four stages +DATASET=locomo bash benchmarks/reproduce.sh -The runner uses an **LLM-as-Judge** approach: a judge LLM receives the -question, the gold answer, and the generated answer, then outputs `CORRECT` -or `WRONG`. Each question is judged `judge_runs` times (default 3); the -final verdict is a **majority vote**. Accuracy = correct / total per method -and per category. +# a slice, for a smoke test +DATASET=locomo CONV=0 STAGES=search bash benchmarks/reproduce.sh \ + --everos-root -The four LoCoMo question categories test different retrieval capabilities: +# any run.py flag is forwarded +DATASET=longmemeval bash benchmarks/reproduce.sh --servers 4 +``` -| Category | Name | Tests | +| Variable | Default | Meaning | |---|---|---| -| 1 | single-hop | Direct fact retrieval from one episode | -| 2 | multi-hop | Reasoning across multiple episodes | -| 3 | open-domain | General knowledge grounded in conversation | -| 4 | temporal | Time-sensitive questions requiring date reasoning | +| `DATASET` | `locomo` | `locomo` \| `longmemeval` \| `subtlememory` \| `evermembench` | +| `CONV` | `all` | conversation indices; a slice is not a reproduction | +| `STAGES` | `add search answer judge` | which stages to run | +| `RUN` | derived | result directory name | -Category 5 (adversarial — questions with no answer in the conversation) is -excluded from evaluation. +### The four stages -### Run isolation +| Stage | What it does | +|---|---| +| **ADD** | streams each conversation into EverOS, which extracts memories | +| **SEARCH** | one retrieval per question; writes the episodes the answer model will see | +| **ANSWER** | asks the answer model each question against those episodes | +| **JUDGE** | grades each answer against the reference | -Each benchmark run is scoped by three identifiers: +Stages are resumable: each writes a JSONL artefact per conversation and skips +what is already in it. A crash at question 190 of 199 loses one question, not +190. Re-running with `--stages answer judge` reuses the existing search results. -| Scope | Value | Purpose | -|---|---|---| -| `app_id` | `locomo_benchmark` | Fixed; separates benchmark data from production | -| `project_id` | `--run-name` value | Per-experiment isolation | -| `owner_id` | `_conv` | Per-conversation memory partition | - -Two runs with the **same** `--run-name` share the same memory corpus — -useful when re-running later stages, but problematic if you want a clean -ingest. Use distinct names (e.g. `locomo-agentic`, `locomo-hybrid`) for -independent experiments. +--- -### Stage independence +## 5. Output -Each stage reads from and writes to JSONL files in `conv/`. This means: +``` +benchmarks/results/// +├── report.txt human-readable summary +├── report.json the same numbers, machine-readable +├── run_spec.json every model, endpoint, package version and knob used +├── conv/ +│ ├── search_.jsonl +│ ├── answer_.jsonl +│ └── judge_.jsonl +└── traces/ per-round retrieval traces, when enabled +``` -- `--stages search` reads from the EverOS server (requires prior `add`). -- `--stages answer` reads `search_.jsonl` (requires prior `search`). -- `--stages judge` reads `answer_.jsonl` (requires prior `answer`). +`run_spec.json` is what makes a number reproducible later. It records the models +*actually served* — not what the config asked for — along with package versions, +because the extraction algorithm's version changes what a store contains. -You can swap the judge model and re-run `--stages judge` without touching -ingest or search. +--- -### Smoke mode +## Reproducibility -`--smoke` is a **pipeline sanity check**, not a scored run. It forces: +Two things decide whether a number can be compared to a published one: -- 2 conversations (conv 0, 1) running in parallel -- First 50 messages each (across however many sessions that covers) -- 10 QA pairs per conversation, stratified-sampled to cover all categories -- `judge_runs=1` (no majority vote) +**The store's extraction backbone.** A store built with a different extraction +model is a different store. `run_spec.json` records which one built it, and +`store_spec.json` inside a shared store records the `everalgo` versions that +produced it. Retrieval reproduces across those versions; extraction does not. -Use it to verify end-to-end connectivity before committing to a full run. +**The judge.** Each benchmark's judge is its own — LongMemEval adds per-category +leniency clauses, EverMemBench grades multiple-choice by letter with no LLM call +at all. The adapters carry each benchmark's judge verbatim from its reference +harness, and a parity check compares them byte for byte. -### Runtime estimates +--- -Rough estimates with default settings (varies by provider latency): +## CLI reference -| Scope | Time | Token cost (approx.) | -|---|---|---| -| Smoke | 2–5 min | ~80k tokens | -| Single conv (full) | 15–30 min | ~1M tokens | -| Full 10-conv run | 2–4 hours | ~10M tokens | +`reproduce.sh` covers the normal path. `run.py` takes these directly: + +| Flag | Meaning | +|---|---| +| `--conv` | conversation indices, or `all` | +| `--stages` | any of `add search answer judge` | +| `--everos-root` | store to use; defaults to `//store` | +| `--servers` | how many EverOS servers to run in parallel | +| `--results-root` | where to write; defaults to `benchmarks/results/` | +| `--data-path` | overrides the dataset path for one run | +| `--methods` | `llm_multiround` \| `hybrid` \| `agentic` | +| `--answer-model` / `--judge-model` | override for one run | +| `--decider-model` / `--decider-base-url` | the multi-round decider; set both | +| `--smoke` | 10 sampled questions per conversation | -The `add` + `wait_ready` phase dominates wall-clock time; LLM calls -(answer + judge) dominate token cost. +--- -### Troubleshooting +## Troubleshooting | Symptom | Cause | Fix | |---|---|---| -| `Connection refused` on run | EverOS server not running | `everos server start` | -| `ANSWER_API_KEY not set` | Missing `.env` | Copy `.env.example` → `.env`, fill keys | -| `Timeout after 1800s` in wait_ready | Cascade/OME still processing | Increase `cascade_timeout` in config.toml or check server logs | -| `OME task(s) failed` warning | OME strategy crashed | Check `everos cascade status`; data may be incomplete | -| `Missing search_*.jsonl` | Running `answer` without prior `search` | Add `search` to `--stages` or run it first | -| `Too many open files (os error 24)` | LanceDB FD exhaustion from concurrent searches | Lower `search_concurrency` in config.toml (agentic needs more FDs per query) or raise `ulimit -n` | -| Low accuracy across all categories | Embedding/rerank not configured | Verify `everos.toml` has working embedding + rerank providers | -| `conv/error.log` exists | Unhandled exception in that conversation | Read the traceback; other conversations are unaffected | +| `benchmarks/.env not found` | first run | `cp benchmarks/.env.example benchmarks/.env` | +| `data directory not found: benchmarks/data/...` | dataset not downloaded | see [1. Prepare the dataset](#1-prepare-the-dataset) | +| `decider ... did not answer` | `BENCH_DECIDER_MODEL` set without its endpoint | set both, or neither | +| `decider ... produced no usable content` | a reasoning model spent its budget thinking | `EVEROS_DECIDER__EXTRA='{"extra_body": {"chat_template_kwargs": {"enable_thinking": false}}}'` | +| `Timeout after 1800s` in wait_ready | extraction still running | raise `cascade_timeout` in the config, or check the server log | +| `Too many open files` | LanceDB FD exhaustion under concurrency | lower `search_concurrency`, or raise `ulimit -n` | +| 0 episodes retrieved, no error | store built under different partition keys | `app_id` / `project_id` must match how the store was built | diff --git a/benchmarks/README.zh.md b/benchmarks/README.zh.md new file mode 100644 index 000000000..40b4bf783 --- /dev/null +++ b/benchmarks/README.zh.md @@ -0,0 +1,254 @@ +# 评测运行指南 + +EverOS 自带一套四个长期记忆评测基准的运行器: +[LoCoMo](https://github.com/snap-research/locomo) +([Maharana et al., 2024](https://arxiv.org/abs/2402.17753))、 +[LongMemEval](https://github.com/xiaowu0162/LongMemEval)、SubtleMemory、 +EverMemBench。 + +四者走同一条流水线,唯一的差别是各自的 adapter。每个基准自己的规则 —— 如何命名 +owner、什么算 gold 证据、judge 如何判分 —— 都在 +`benchmarks/adapters/.py` 里。运行器的其余部分不知道自己在跑哪个基准。 + +> English version: [README.md](README.md) + +--- + +## 快速开始 + +```bash +# 1. 安装 +uv sync + +# 2. 配置 +cp benchmarks/.env.example benchmarks/.env +$EDITOR benchmarks/.env # answer/judge 的 API key、抽取 backbone + +# 3. 把数据集放到配置期望的位置 +curl -o benchmarks/data/locomo10.json \ + https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json + +# 4. 复现 +DATASET=locomo bash benchmarks/reproduce.sh +``` + +最后这条命令会按**已发布数字所用的配置**跑完整流水线(ADD → SEARCH → ANSWER → +JUDGE),报告写到 `benchmarks/results/LoCoMo/locomo/`。 + +--- + +## 目录结构 + +``` +benchmarks/ +├── reproduce.sh 入口:端到端跑一个基准 +├── run.py 流水线(ADD → SEARCH → ANSWER → JUDGE) +├── config.py 冻结的配置模型 +├── configs/ +│ ├── default.toml 共享默认值 +│ └── .toml 每个基准一份:模型、top_k、并发 +├── adapters/ +│ ├── base.py 一个 adapter 要回答的四个问题 +│ └── .py 每个基准一份:owner、gold、prompt、judge +├── metrics/ 排序检索指标 + core 选择指标 +├── data/ ← 数据集放这里(已 git-ignore) +├── results/ ← 跑批产物写这里(已 git-ignore) +└── .env.example 复制成 .env +``` + +`data/` 和 `results/` 是你唯二需要写入的目录,两个都已 git-ignore —— +数据集只有其原作者有权再分发,而跑批产物又大又可重新生成。 + +--- + +## 1. 准备数据集 + +每个基准的输入路径来自它自己的配置,默认就指向 `benchmarks/data/`。 +**把文件放进去就够了** —— 不需要环境变量,不需要命令行参数: + +| 基准 | 文件放在 | 环境变量(可选覆盖) | +|---|---|---| +| `locomo` | `benchmarks/data/locomo10.json` | `BENCH_DATA_LOCOMO` | +| `longmemeval` | `benchmarks/data/longmemeval_s.json` | `BENCH_DATA_LONGMEMEVAL` | +| `subtlememory` | `benchmarks/data/subtlememory/`(内含 `persona_0..9/` 的目录) | `BENCH_DATA_SUBTLEMEMORY` | +| `evermembench` | `benchmarks/data/evermembench.json` | `BENCH_DATA_EVERMEMBENCH` | + +只有当数据放在别处(共享挂载、另一块盘)时才需要设环境变量,写在 +`benchmarks/.env` 里。 + +### 各数据集怎么拿 + +**LoCoMo** —— 10 段多 session 对话,每段约 50 个 session,每段约 150 个 QA, +覆盖四个类别(adversarial 类别被排除)。 + +```bash +curl -o benchmarks/data/locomo10.json \ + https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json +``` + +**LongMemEval** —— 取 `longmemeval_s` 划分,来自 +[xiaowu0162/LongMemEval](https://github.com/xiaowu0162/LongMemEval), +放成 `benchmarks/data/longmemeval_s.json`。 + +**SubtleMemory** 和 **EverMemBench** —— 由各自作者发布,按其项目说明获取。 +EverMemBench 需要先把原始发布版转换一次;把 `EVERMEMBENCH_RAW_ROOT` 指向原始目录, +adapter 会从中恢复转换后文件里没有的 session 名。 + +### 路径不对时会怎样 + +路径缺失或配错会**让这次跑失败**,而不是载入 0 行然后打出 0%。 +未设置且没有默认值的变量会以字面量 `${BENCH_DATA_LOCOMO}` 到达 loader 并被点名报出, +所以报错说的是"该设哪个变量",而不是"找不到一个叫 `${...}` 的文件"。 + +--- + +## 2. 配置 + +```bash +cp benchmarks/.env.example benchmarks/.env +``` + +第一次跑至少要填: + +| 键 | 是什么 | +|---|---| +| `ANSWER_API_KEY` / `ANSWER_BASE_URL` | 根据检索到的记忆回答问题的模型 | +| `JUDGE_API_KEY` / `JUDGE_BASE_URL` | 给上面这些回答判分的模型 | +| `EVEROS_LLM__MODEL` / `__API_KEY` / `__BASE_URL` | ADD 阶段 EverOS 跑的抽取 backbone | + +**决定数字的一切** —— 模型、`top_k`、检索旋钮、并发 —— 都在 +`benchmarks/configs/.toml` 里,不在命令行上。这是刻意的: +一次跑要能从它的配置复现,所以 `reproduce.sh` 不传任何模型覆盖参数。 + +### 多轮 decider(可选) + +`llm_multiround` 检索会跑一个 **decider**:读候选记忆、挑出 core 集合、 +提出补充子查询。默认它跑和抽取相同的模型,无需额外配置。 + +要给它单独的模型,**两个必须成对设置**: + +```bash +BENCH_DECIDER_MODEL=<模型名> +BENCH_DECIDER_BASE_URL=<提供该模型的端点> +``` + +只设一个,正是这个运行器在启动时专门检查的那种故障: +把模型名发到不提供该模型的端点,每次调用都返回 404,而检索循环会**退回到固定的 +top-N core 并照常报出一个完整结果**。`run.py` 会在第一个问题之前探测 decider, +不应答就拒绝启动。 + +--- + +## 3. 启动 server(只在你想自己管理时) + +`reproduce.sh` 会自己起停 server。只有当你想跨多次跑复用同一个库时才手动启: + +```bash +ulimit -n 10240 # 并发检索会同时打开大量 LanceDB 分片文件 +everos server start [--root ] +``` + +把同一个路径用 `--everos-root` 传给运行器。运行器要靠轮询该 root 下的 cascade 和 +OME 数据库来判断数据是否就绪,路径不一致会造成**静默的假就绪**。 + +--- + +## 4. 运行 + +```bash +# 完整复现:全部对话、四个阶段 +DATASET=locomo bash benchmarks/reproduce.sh + +# 只跑一个切片做冒烟 +DATASET=locomo CONV=0 STAGES=search bash benchmarks/reproduce.sh \ + --everos-root <一个已有的库> + +# run.py 的任何参数都会被转发 +DATASET=longmemeval bash benchmarks/reproduce.sh --servers 4 +``` + +| 变量 | 默认 | 含义 | +|---|---|---| +| `DATASET` | `locomo` | `locomo` \| `longmemeval` \| `subtlememory` \| `evermembench` | +| `CONV` | `all` | 对话下标;**切片不算复现** | +| `STAGES` | `add search answer judge` | 跑哪些阶段 | +| `RUN` | 自动推导 | 结果目录名 | + +### 四个阶段 + +| 阶段 | 做什么 | +|---|---| +| **ADD** | 把每段对话流式送入 EverOS,由它抽取记忆 | +| **SEARCH** | 每题一次检索;写下答题模型将会看到的 episodes | +| **ANSWER** | 拿这些 episodes 让答题模型逐题作答 | +| **JUDGE** | 对照参考答案给每个回答判分 | + +阶段可断点续跑:每个阶段按对话写 JSONL,已在文件里的会跳过。 +在第 199 题里的第 190 题崩溃,只丢一题,不是 190 题。 +用 `--stages answer judge` 重跑会直接复用已有的检索结果。 + +--- + +## 5. 输出 + +``` +benchmarks/results/// +├── report.txt 人读的汇总 +├── report.json 同样的数字,机器可读 +├── run_spec.json 用到的每个模型、端点、包版本、旋钮 +├── conv/ +│ ├── search_.jsonl +│ ├── answer_.jsonl +│ └── judge_.jsonl +└── traces/ 开启时:逐轮检索轨迹 +``` + +`run_spec.json` 是一个数字日后还能复现的依据。它记录的是**实际被服务的模型** +(不是配置里写的那个),外加包版本 —— 因为抽取算法的版本会改变一个库里装的是什么。 + +--- + +## 可复现性 + +一个数字能不能和已发布的比,取决于两件事: + +**库的抽取 backbone。** 用不同抽取模型建的库是不同的库。 +`run_spec.json` 记录了是谁建的,共享库里的 `store_spec.json` 记录了产出它的 +`everalgo` 版本。**检索跨版本可复现,抽取不行。** + +**judge。** 每个基准的 judge 是它自己的 —— LongMemEval 会按类别追加宽容条款, +EverMemBench 的选择题按字母判分、完全不调 LLM。adapter 逐字搬运了各基准参考实现里的 +judge,并有一道 parity 门禁逐字节比对。 + +--- + +## 命令行参考 + +`reproduce.sh` 覆盖了常规路径。`run.py` 直接接受这些参数: + +| 参数 | 含义 | +|---|---| +| `--conv` | 对话下标,或 `all` | +| `--stages` | `add search answer judge` 的任意组合 | +| `--everos-root` | 用哪个库;默认 `//store` | +| `--servers` | 并行起几个 EverOS server | +| `--results-root` | 写到哪;默认 `benchmarks/results/` | +| `--data-path` | 单次覆盖数据集路径 | +| `--methods` | `llm_multiround` \| `hybrid` \| `agentic` | +| `--answer-model` / `--judge-model` | 单次覆盖 | +| `--decider-model` / `--decider-base-url` | 多轮 decider,**两个一起设** | +| `--smoke` | 每段对话抽 10 题 | + +--- + +## 排障 + +| 现象 | 原因 | 处理 | +|---|---|---| +| `benchmarks/.env not found` | 第一次跑 | `cp benchmarks/.env.example benchmarks/.env` | +| `data directory not found: benchmarks/data/...` | 数据集没下载 | 见[「1. 准备数据集」](#1-准备数据集) | +| `decider ... did not answer` | 设了 `BENCH_DECIDER_MODEL` 却没设端点 | 两个都设,或都不设 | +| `decider ... produced no usable content` | 推理型模型把预算全花在思考上 | `EVEROS_DECIDER__EXTRA='{"extra_body": {"chat_template_kwargs": {"enable_thinking": false}}}'` | +| wait_ready 里 `Timeout after 1800s` | 抽取还没跑完 | 调大配置里的 `cascade_timeout`,或去看 server 日志 | +| `Too many open files` | 并发下 LanceDB 文件描述符耗尽 | 调低 `search_concurrency`,或调大 `ulimit -n` | +| 检索到 0 条 episode 且不报错 | 库是用不同的分区键建的 | `app_id` / `project_id` 必须和建库时一致 | diff --git a/benchmarks/adapters/__init__.py b/benchmarks/adapters/__init__.py new file mode 100644 index 000000000..5e374a68e --- /dev/null +++ b/benchmarks/adapters/__init__.py @@ -0,0 +1,36 @@ +"""Dataset adapters, resolved by name. + +``run.py`` never imports a specific adapter; it asks for one by ``--benchmark``. Adding +a benchmark means adding a module here and one line to the registry -- no change to the +pipeline. +""" + +from __future__ import annotations + +from types import ModuleType + +from . import evermembench, locomo, longmemeval, subtlememory + +_REGISTRY: dict[str, ModuleType] = { + "locomo": locomo, + "longmemeval": longmemeval, + "evermembench": evermembench, + "subtlememory": subtlememory, +} + + +def get(name: str) -> ModuleType: + """Return the adapter module for ``name``, or raise with the valid choices.""" + try: + return _REGISTRY[name] + except KeyError: + raise KeyError( + f"unknown benchmark {name!r}; choices: {sorted(_REGISTRY)}" + ) from None + + +def names() -> list[str]: + return sorted(_REGISTRY) + + +__all__ = ["get", "names"] diff --git a/benchmarks/adapters/_profile.py b/benchmarks/adapters/_profile.py new file mode 100644 index 000000000..d0b0d739c --- /dev/null +++ b/benchmarks/adapters/_profile.py @@ -0,0 +1,66 @@ +"""Rendering the owner's profile into an answer prompt. + +Shared because it was not: the renderer lived inside ``evermembench.py`` and every other +benchmark's context builder simply dropped the ``profiles`` argument. That made +``include_profile`` a no-op on three of the four benchmarks -- the server fetched the +profile, the harness threw it away, and two "profile on vs off" runs on LoCoMo and +LongMemEval therefore compared a configuration against itself. Their whole measured +difference (0.91 pp and 1.20 pp, McNemar p=0.10 and p=0.15) was decider nondeterminism. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +PROFILE_HEADING = "## User profile" +MEMORY_HEADING = "## Retrieved memories" + + +def render_profile_lines(profiles: Sequence[dict]) -> list[str]: + """The profile as dash-prefixed lines, or empty when there is nothing to show. + + Reads both shapes the search response uses: a row wrapping ``profile_data``, and the + profile object itself. + """ + out: list[str] = [] + for prof in profiles or (): + data = prof.get("profile_data") or prof + summary = str(data.get("summary") or "").strip() + if summary: + out.append(f"- {summary}") + for item in data.get("explicit_info") or []: + if not isinstance(item, dict): + continue + cat = str(item.get("category") or "").strip() + desc = str(item.get("description") or "").strip() + if desc: + out.append(f"- [{cat}] {desc}" if cat else f"- {desc}") + for item in data.get("implicit_traits") or []: + if not isinstance(item, dict): + continue + trait = str(item.get("trait") or item.get("name") or "").strip() + basis = str(item.get("basis") or "").strip() + if trait and basis: + out.append(f"- [trait] {trait} -- {basis}") + elif basis: + out.append(f"- [trait] {basis}") + return out + + +def with_profile_block(memories: str, profiles: Sequence[dict]) -> str: + """Put the profile ahead of the rendered memories, as its own labelled section. + + Kept out of the memory list rather than prepended to it: a profile is standing + context, with no timestamp to reason about and no session it belongs to. A model + told to weigh recency would otherwise treat it as one more dated memory. + + Returns ``memories`` unchanged when there is no profile, which is what keeps every + benchmark's prompt byte-identical to its reference harness while the flag is off. + """ + lines = render_profile_lines(profiles) + if not lines: + return memories + block = PROFILE_HEADING + "\n" + "\n".join(lines) + if not memories: + return block + return f"{block}\n\n{MEMORY_HEADING}\n{memories}" diff --git a/benchmarks/adapters/base.py b/benchmarks/adapters/base.py new file mode 100644 index 000000000..8c0074a68 --- /dev/null +++ b/benchmarks/adapters/base.py @@ -0,0 +1,160 @@ +"""Dataset adapters: the one place a benchmark's own shape is allowed to live. + +`run.py` drives ADD -> SEARCH -> ANSWER -> JUDGE and knows nothing about any particular +benchmark. Everything that differs between benchmarks answers four questions, and an +adapter is exactly those four answers: + + 1. load_units() -- how to read the conversations and questions off disk + 2. owner_of() -- what memory owner a question's answer lives under. Owner naming is + decided when the store is BUILT, so this must reproduce the + builder's convention rather than invent one. Getting it wrong + returns zero episodes and scores 0% with no error anywhere. + 3. gold_of() -- gold session ids, in the form THE STORE uses. Every benchmark + cites + evidence differently (haystack positions, D: dia + ids, original session names) and none of them matches the store + directly. + 4. judge_spec() -- which judge, and the answer prompt it grades against. The hybrid + judge's leniency clauses are keyed to LongMemEval's categories and + would mis-fire on any other benchmark, so the judge belongs to the + adapter, not to a shared scoring layer. + +Anything that is NOT one of those four belongs in run.py. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any, Protocol, runtime_checkable + + +@runtime_checkable +class DatasetAdapter(Protocol): + """Four questions, one benchmark.""" + + name: str + + def load_units(self, data_path: str) -> list[dict[str, Any]]: + """Return one entry per conversation/topic, each carrying its own questions. + + The shape run.py consumes is ``{"index": int, "sessions": [...], "qa": [...]}``; + an adapter is free to derive that however its source data is organised. + """ + ... + + def owner_of(self, unit: dict[str, Any], eval_owner: str) -> str: + """Memory owner id to query for this unit. + + ``eval_owner`` carries the config's preference where a benchmark has more than + one candidate (LoCoMo has two speakers); benchmarks with a single owner per unit + ignore it. + """ + ... + + def gold_of(self, unit: dict[str, Any], qa: dict[str, Any]) -> set[str]: + """Gold session ids for one question, already translated into store ids.""" + ... + + def sessions_of(self, unit: dict[str, Any]) -> list[dict[str, Any]]: + """Ingestible sessions for the ADD stage. + + Each session is ``{"session_idx": int, "messages": [...], "timestamp_ms": int}`` + and each message carries ``speaker`` / ``text`` / ``dia_id``. Only the ADD stage + needs this; a benchmark scored against a pre-built store never calls it. + + Splitting this out is what lets ADD run for anything other than LoCoMo: the + loader used to read ``unit["conversation"]`` directly, so every other dataset + died with ``KeyError: 'conversation'`` the moment ingestion started. + """ + raise NotImplementedError + + def judge_spec(self) -> dict[str, Any]: + """``{"judge": , "answer_prompt":