fix(memory): close the rating loop and stop diluting what gets served

better_memory/cli/install_hooks.py

@@ -55,7 +55,13 @@ class HookSpec:
 _OUR_HOOKS: tuple[HookSpec, ...] = (
     HookSpec("better_memory.hooks.session_bootstrap", "SessionStart", None,              False, True),
     HookSpec("better_memory.hooks.observer",          "PostToolUse",  "Write|Edit|Bash", True,  False),
-    HookSpec("better_memory.hooks.session_close",     "Stop",         None,              True,  False),
+    # Stop MUST be synchronous with stdout attached. The rating sweep replies with
+    # a `decision: "block"` payload (hooks/session_close.py) — a control-flow
+    # response that Claude Code only honours from a blocking hook. Registered
+    # async, the block is dropped and RATE_MEMORIES never forces a rating turn.
+    # Measured A/B on identical tasks: async => 0% of exposures rated;
+    # sync => 100% rated. See docs/decisions/stop-hook-must-be-sync.md.
+    HookSpec("better_memory.hooks.session_close",     "Stop",         None,              False, True),
     # Contextual injection: surface curated memories relevant to the current
     # prompt / tool-input. Sync + needs_stdout (reads additionalContext from
     # stdout). Gated at runtime by BETTER_MEMORY_CONTEXT_INJECT_MODE; both

better_memory/db/migrations/0013_ignored_counters.sql

@@ -0,0 +1,62 @@
+-- Migration 0013: per-memory `ignored` counters, and a backfill from history.
+--
+-- Ratings were write-only for the `ignored` class: memory_rating._apply_one
+-- bumps useful_count / times_misled / times_overlooked but does nothing at all
+-- when a memory is classified `ignored`. Nothing in the system therefore ever
+-- learns that a memory keeps being shown and keeps not mattering.
+--
+-- Measured on a live DB with 4670 rated exposures: 55 memories had been rated
+-- 10+ times each and were useful ZERO times. Between them they accounted for
+-- 1284 rated exposures — 27.5% of everything served — while contributing
+-- nothing. They were never demoted because the ranking key
+-- (useful_count + 3*times_overlooked) has no negative term.
+--
+-- These counters give the ranker that term. The backfill matters as much as
+-- the schema: it turns the rating history that already exists into signal
+-- immediately, instead of waiting to re-learn what the DB already knows.
+
+ALTER TABLE reflections       ADD COLUMN times_ignored   INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE reflections       ADD COLUMN last_ignored_at TEXT;
+ALTER TABLE semantic_memories ADD COLUMN times_ignored   INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE semantic_memories ADD COLUMN last_ignored_at TEXT;
+
+-- Backfill from session_memory_exposure. Counted per distinct
+-- (session_id, memory_id): a memory retrieved several times in one session
+-- writes one exposure row per retrieval, and _apply_one stamps all of them
+-- with the same classification, so counting rows would score retrieval
+-- chattiness rather than sessions in which the memory failed to land.
+
+UPDATE reflections SET
+    times_ignored = COALESCE((
+        SELECT COUNT(DISTINCT e.session_id)
+        FROM session_memory_exposure e
+        WHERE e.memory_kind = 'reflection'
+          AND e.memory_id = reflections.id
+          AND e.classification = 'ignored'
+    ), 0),
+    last_ignored_at = (
+        SELECT MAX(e.rated_at)
+        FROM session_memory_exposure e
+        WHERE e.memory_kind = 'reflection'
+          AND e.memory_id = reflections.id
+          AND e.classification = 'ignored'
+    );
+
+UPDATE semantic_memories SET
+    times_ignored = COALESCE((
+        SELECT COUNT(DISTINCT e.session_id)
+        FROM session_memory_exposure e
+        WHERE e.memory_kind = 'semantic'
+          AND e.memory_id = semantic_memories.id
+          AND e.classification = 'ignored'
+    ), 0),
+    last_ignored_at = (
+        SELECT MAX(e.rated_at)
+        FROM session_memory_exposure e
+        WHERE e.memory_kind = 'semantic'
+          AND e.memory_id = semantic_memories.id
+          AND e.classification = 'ignored'
+    );
+
+CREATE INDEX IF NOT EXISTS idx_reflections_ignored
+    ON reflections(times_ignored) WHERE useful_count = 0;

better_memory/mcp/handlers/reflections.py

@@ -71,6 +71,7 @@ async def retrieve(self, args: dict[str, Any]) -> list[TextContent]:
             tech=args.get("tech"),
             phase=args.get("phase"),
             polarity=args.get("polarity"),
+            query=args.get("query"),
         ):
             diag_cid: str | None = None
             t_retrieve = 0.0
@@ -110,7 +111,12 @@ def _retention_step() -> None:
                 _diag.step("mcp.memory.retrieve", "after_retention_scheduler")
 
             project = args.get("project") or project_name()
-            limit_per_bucket = args.get("limit_per_bucket", 20)
+            # Default 5, not 20. An LLM working one task draws on ~5 memories
+            # regardless of how many it is handed (measured: 5.3 +/- 2.4 useful
+            # per session against 42.7 exposed). Everything past that dilutes
+            # the set and burns context, so the default returns a shortlist and
+            # callers that genuinely want the long tail ask for it.
+            limit_per_bucket = args.get("limit_per_bucket", 5)
             t_reflections = time.monotonic() if diag_cid else 0.0
             _diag.step(
                 "mcp.memory.retrieve",
@@ -123,6 +129,7 @@ def _retention_step() -> None:
                 phase=args.get("phase"),
                 polarity=args.get("polarity"),
                 limit_per_bucket=limit_per_bucket,
+                query=args.get("query"),
             )
             _diag.step(
                 "mcp.memory.retrieve",

better_memory/mcp/tools.py

@@ -154,14 +154,26 @@ def tool_definitions(
             name="memory.retrieve",
             description=(
                 "Retrieve reflections (do / dont / neutral lessons distilled "
-                "from prior observations) bucketed by polarity. Filter by "
-                "project, tech, phase, and polarity. For raw observation "
-                "lookup, use memory.retrieve_observations."
+                "from prior observations) bucketed by polarity. ALWAYS pass "
+                "`query` — a plain-language description of the task you are "
+                "about to do — or you get the same generic top-ranked lessons "
+                "every session regardless of what you are working on. Filter "
+                "further by project, tech, phase, and polarity. For raw "
+                "observation lookup, use memory.retrieve_observations."
             ),
             inputSchema={
                 "type": "object",
                 "additionalProperties": False,
                 "properties": {
+                    "query": {
+                        "type": "string",
+                        "description": (
+                            "Plain-language description of the task at hand, "
+                            "e.g. 'changing how retention archives reflections'. "
+                            "Ranks results by relevance to this text fused with "
+                            "the usefulness prior."
+                        ),
+                    },
                     "project": {"type": "string"},
                     "tech": {"type": "string"},
                     "phase": {
@@ -174,6 +186,13 @@ def tool_definitions(
                     },
                     "limit_per_bucket": {"type": "integer"},
                 },
+                # `query` is deliberately NOT required. Making it mandatory
+                # would break every existing caller (and the start_episode /
+                # bootstrap internal paths) for a param whose absence degrades
+                # gracefully to the popularity order. The description urges it
+                # instead; the real usefulness gains come from the shortlist
+                # default and the ignored-history demotion, not from coercing
+                # a query on every call.
             },
         ),
         Tool(

better_memory/services/memory_rating.py

@@ -67,6 +67,17 @@ class ApplySessionRatingsResult(TypedDict):
 # rank harder than a cited one. Imported by reflection.py and semantic.py.
 OVERLOOKED_RANKING_WEIGHT = 3
 
+# Demotion for memories that keep being served and keep not mattering.
+# A memory is only demoted once it has been ignored in at least
+# IGNORED_DEMOTION_FLOOR distinct sessions AND has never been useful — one or
+# two ignores say nothing (the task simply wasn't relevant that day), but a
+# double-digit run of them with no hits is the memory telling you it does not
+# belong in the default set. Memories with any useful history are never
+# demoted, however often they are also ignored: the best-performing memory on
+# the live DB is ignored in 125 of the 192 sessions it appears in.
+IGNORED_DEMOTION_FLOOR = 10
+IGNORED_DEMOTION_WEIGHT = 1
+
 
 class MemoryRatingService:
     """Writes useful_count / times_misled on reflections + semantic memories,
@@ -214,8 +225,17 @@ def _apply_one(
                 f"WHERE id = ?",
                 (now, memory_id),
             )
-        # 'ignored' is a no-op on the memory row; reached only via
-        # apply_session_ratings, not credit_one.
+        elif classification == "ignored":
+            # Reached only via apply_session_ratings, not credit_one. Counted
+            # once per session, matching the migration-0013 backfill: a memory
+            # retrieved five times in one session that lands nowhere failed
+            # once, not five times.
+            self._conn.execute(
+                f"UPDATE {table} "
+                f"SET times_ignored = times_ignored + 1, last_ignored_at = ? "
+                f"WHERE id = ?",
+                (now, memory_id),
+            )
 
         # 4. Stamp the exposure row.
         self._conn.execute(

better_memory/services/reflection.py

@@ -33,7 +33,25 @@
 
 from better_memory import _diag
 from better_memory._common import default_clock, env_session_id
-from better_memory.services.memory_rating import OVERLOOKED_RANKING_WEIGHT
+from better_memory.search.query import sanitize_fts5_query
+from better_memory.services.memory_rating import (
+    IGNORED_DEMOTION_FLOOR,
+    IGNORED_DEMOTION_WEIGHT,
+    OVERLOOKED_RANKING_WEIGHT,
+)
+
+# Dropped from relevance queries. These survive `sanitize_fts5_query` and the
+# >2-char filter, but appear in so many reflections that OR-matching on them
+# promotes generic rows over the ones that actually describe the task.
+_QUERY_STOPWORDS: frozenset[str] = frozenset({
+    "the", "and", "for", "with", "that", "this", "from", "into", "when",
+    "what", "how", "why", "does", "did", "are", "was", "were", "have", "has",
+    "had", "not", "any", "all", "can", "should", "would", "could", "will",
+    "about", "before", "after", "then", "than", "there", "their", "them",
+    "you", "your", "its", "our", "out", "use", "using", "used", "get",
+    "make", "made", "want", "need", "like", "just", "some", "which", "who",
+    "where", "here", "over", "under", "each", "other", "same", "such",
+})
 
 
 def _later_ts(a: str | None, b: str | None) -> str | None:
@@ -1168,6 +1186,73 @@ def apply_decision(
         )
 
     # --------------------------------------------------------- retrieve_reflections
+    def _fuse_by_relevance(
+        self, rows: list, *, query: str, rrf_k: int = 60,
+    ) -> list:
+        """Re-order ``rows`` by RRF fusion of popularity rank and BM25 rank.
+
+        ``rows`` arrives in popularity order (``useful_count + 3*times_overlooked``,
+        then confidence, then recency). We compute a second ranking over the same
+        ids by BM25 relevance against ``reflection_fts`` (title / use_cases /
+        hints) and fuse the two with reciprocal rank fusion:
+
+            score(d) = 1/(k + pop_rank(d)) + 1/(k + rel_rank(d))
+
+        matching :mod:`better_memory.search.hybrid`. Rows the query does not
+        match keep only the popularity term, so relevance *promotes* without
+        ever discarding — a query that matches nothing degrades exactly to the
+        previous behaviour.
+
+        Tokens are OR-ed rather than AND-ed: ``sanitize_fts5_query`` joins bare
+        terms, which FTS5 reads as implicit AND, and a natural-language task
+        description almost never has every token present in one reflection.
+        """
+        ids = [r["id"] for r in rows]
+        if not ids:
+            return rows
+
+        sanitized = sanitize_fts5_query(query)
+        tokens = [
+            t for t in sanitized.split()
+            if len(t) > 2 and t.lower() not in _QUERY_STOPWORDS
+        ]
+        if not tokens:
+            return rows
+        match_expr = " OR ".join(tokens)
+
+        placeholders = ",".join("?" for _ in ids)
+        try:
+            rel_rows = self._conn.execute(
+                f"""
+                SELECT r.id AS id, bm25(reflection_fts) AS bm
+                FROM reflection_fts
+                JOIN reflections r ON r.rowid = reflection_fts.rowid
+                WHERE reflection_fts MATCH ? AND r.id IN ({placeholders})
+                ORDER BY bm ASC
+                """,
+                [match_expr, *ids],
+            ).fetchall()
+        except sqlite3.OperationalError:
+            # Malformed MATCH expression despite sanitising, or the FTS table
+            # is absent on an old DB. Relevance is an enhancement, never a
+            # hard dependency — fall back to the popularity order.
+            return rows
+
+        if not rel_rows:
+            return rows
+
+        rel_rank = {r["id"]: i for i, r in enumerate(rel_rows)}
+        scored = []
+        for pop_rank, row in enumerate(rows):
+            score = 1.0 / (rrf_k + pop_rank)
+            rr = rel_rank.get(row["id"])
+            if rr is not None:
+                score += 1.0 / (rrf_k + rr)
+            scored.append((score, pop_rank, row))
+        # pop_rank breaks ties deterministically and keeps the fusion stable.
+        scored.sort(key=lambda t: (-t[0], t[1]))
+        return [row for _, _, row in scored]
+
     def retrieve_reflections(
         self,
         *,
@@ -1177,6 +1262,7 @@ def retrieve_reflections(
         polarity: str | None = None,
         limit_per_bucket: int | None = 20,
         track_exposure: bool = True,
+        query: str | None = None,
     ) -> dict[str, list[dict]]:
         """Return reflections bucketed by polarity, ordered by confidence DESC.
 
@@ -1185,6 +1271,12 @@ def retrieve_reflections(
         - ``tech``: matches same-tech rows OR cross-tech (tech IS NULL) rows.
         - ``phase``: optional exact match.
         - ``polarity``: optional exact match; non-matching buckets remain empty.
+        - ``query``: optional natural-language description of the task at hand.
+          When given, the filtered set is re-ordered by RRF fusion of the
+          popularity prior with a BM25 relevance ranking over
+          ``title / use_cases / hints`` (see :meth:`_fuse_by_relevance`).
+          Without it, ordering is the popularity prior alone — which returns
+          the same rows to every caller regardless of what they are doing.
         - ``limit_per_bucket``: cap each polarity bucket. Default 20 per spec §7.
           Pass ``None`` to disable the cap (returns every matching row per
           bucket); used by SessionBootstrapService which injects all
@@ -1221,21 +1313,35 @@ def retrieve_reflections(
 
             where = " AND ".join(clauses)
             params.append(OVERLOOKED_RANKING_WEIGHT)
+            params.append(IGNORED_DEMOTION_WEIGHT)
+            params.append(IGNORED_DEMOTION_FLOOR)
             _diag.step(fn, "executing_select")
+            # The demotion term fires only for memories with no useful history
+            # at all, and only past the floor — see IGNORED_DEMOTION_FLOOR.
+            # Without it the ranking key has no negative term, so a memory that
+            # has been served 142 times and helped zero times sorts level with
+            # one that has never been served.
             rows = self._conn.execute(
                 f"""
                 SELECT id, title, phase, polarity, use_cases, hints,
                        confidence, tech, evidence_count, useful_count,
                        times_misled, updated_at
                 FROM reflections
                 WHERE {where}
-                ORDER BY (useful_count + ? * times_overlooked) DESC,
+                ORDER BY (useful_count + ? * times_overlooked
+                          - ? * CASE WHEN useful_count = 0
+                                     THEN MAX(0, times_ignored - ?)
+                                     ELSE 0 END) DESC,
                          confidence DESC, updated_at DESC
                 """,
                 params,
             ).fetchall()
             _diag.step(fn, "select_done", n_rows=len(rows))
 
+            if query:
+                rows = self._fuse_by_relevance(rows, query=query)
+                _diag.step(fn, "relevance_fused", n_rows=len(rows))
+
             # Convert None (unlimited) to sys.maxsize so the loop body has a definite int.
             cap = limit_per_bucket if limit_per_bucket is not None else sys.maxsize
             buckets: dict[str, list[dict]] = {"do": [], "dont": [], "neutral": []}
@@ -1291,11 +1397,18 @@ def retrieve_reflections(
                         _diag.step(
                             fn, "exposure_insert", n_ids=len(all_ids)
                         )
+                        # One row per (session, memory) — see
+                        # SessionBootstrapService.record_exposures for why
+                        # re-serves must not add rows.
                         self._conn.executemany(
-                            "INSERT OR IGNORE INTO session_memory_exposure "
+                            "INSERT INTO session_memory_exposure "
                             "(session_id, memory_kind, memory_id, exposed_at, source) "
-                            "VALUES (?, 'reflection', ?, ?, 'retrieve')",
-                            [(sid, rid, now) for rid in all_ids],
+                            "SELECT ?, 'reflection', ?, ?, 'retrieve' "
+                            "WHERE NOT EXISTS ("
+                            "  SELECT 1 FROM session_memory_exposure "
+                            "  WHERE session_id = ? AND memory_kind = 'reflection' "
+                            "    AND memory_id = ?)",
+                            [(sid, rid, now, sid, rid) for rid in all_ids],
                         )
                         _diag.step(fn, "exposure_commit")
                         self._conn.commit()