From 54104034b5c92dc074bf9adbf3132d8f04258a23 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Tue, 1 Sep 2026 21:23:42 +0800 Subject: [PATCH 01/35] fix(retrieval): enable production node filter --- packages/shared-python/shared/services/retrieval/nav_config.py | 2 +- packages/shared-python/shared/tests/test_nav_bridge_config.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/shared-python/shared/services/retrieval/nav_config.py b/packages/shared-python/shared/services/retrieval/nav_config.py index b533a37f..54049e39 100644 --- a/packages/shared-python/shared/services/retrieval/nav_config.py +++ b/packages/shared-python/shared/services/retrieval/nav_config.py @@ -61,7 +61,7 @@ "max_waves": 0, "max_harvest_depth": 3, "plan_control_digest_chars": 600, - "enable_node_filter": False, + "enable_node_filter": True, "filter_max_rounds": 3, "filter_min_hits": 1, "filter_max_hits": 40, diff --git a/packages/shared-python/shared/tests/test_nav_bridge_config.py b/packages/shared-python/shared/tests/test_nav_bridge_config.py index bfd42c33..076b66d3 100644 --- a/packages/shared-python/shared/tests/test_nav_bridge_config.py +++ b/packages/shared-python/shared/tests/test_nav_bridge_config.py @@ -41,6 +41,7 @@ def test_build_nav_config_is_checklist_map_trim_stack() -> None: assert cfg.planner_thinking == "enabled" assert cfg.planner_think_max_tokens == 16_384 assert cfg.token_limit == 100_000 + assert cfg.enable_node_filter is True assert not hasattr(cfg, "llm_model_env") From 05e1aaf0b1531a62f704939e91a7d941ffc1fd6e Mon Sep 17 00:00:00 2001 From: suguanYang Date: Tue, 1 Sep 2026 21:44:23 +0800 Subject: [PATCH 02/35] perf: optimize map-nav snapshot and scoring --- apps/api/scripts/backfill_map_unit_indexes.py | 4 +- ...est_retrieval_serving_manifest_contract.py | 55 ++++ .../shared/services/redis/redis_service.py | 43 +++ .../services/retrieval/execution/routes.py | 11 + .../retrieval/namespace_map_snapshot.py | 8 +- .../retrieval/namespace_map_snapshot_cache.py | 43 --- .../retrieval/namespace_map_snapshot_redis.py | 38 +++ .../services/retrieval/nav/nav_map_scores.py | 46 ++-- .../shared/services/retrieval/nav_snapshot.py | 256 ++++++++++++++++-- .../services/retrieval/serving_manifest.py | 121 ++++++++- 10 files changed, 522 insertions(+), 103 deletions(-) delete mode 100644 packages/shared-python/shared/services/retrieval/namespace_map_snapshot_cache.py create mode 100644 packages/shared-python/shared/services/retrieval/namespace_map_snapshot_redis.py diff --git a/apps/api/scripts/backfill_map_unit_indexes.py b/apps/api/scripts/backfill_map_unit_indexes.py index ebf1a04d..6d3a64e5 100644 --- a/apps/api/scripts/backfill_map_unit_indexes.py +++ b/apps/api/scripts/backfill_map_unit_indexes.py @@ -67,7 +67,7 @@ def _bootstrap_python_path() -> None: lock_namespace_generation, ) from shared.services.retrieval.serving_manifest import ( - decode_serving_manifest, + decode_namespace_map_snapshot, persist_revision_serving_state, ) @@ -160,7 +160,7 @@ def check_fallback_readiness(*, document_id: str = "") -> list[NamespaceFallback snapshot_status = "missing" else: try: - payload = decode_serving_manifest( + payload = decode_namespace_map_snapshot( bytes(snapshot.payload_zlib), checksum=str(snapshot.checksum), format_version=int(snapshot.format_version), diff --git a/apps/api/tests/contract/test_retrieval_serving_manifest_contract.py b/apps/api/tests/contract/test_retrieval_serving_manifest_contract.py index 5bb68c06..d80c688d 100644 --- a/apps/api/tests/contract/test_retrieval_serving_manifest_contract.py +++ b/apps/api/tests/contract/test_retrieval_serving_manifest_contract.py @@ -5,8 +5,11 @@ import pytest from shared.services.retrieval.serving_manifest import ( + NAMESPACE_MAP_SNAPSHOT_FORMAT_VERSION, SERVING_MANIFEST_FORMAT_VERSION, + decode_namespace_map_snapshot, decode_serving_manifest, + encode_namespace_map_snapshot, encode_serving_manifest, ) @@ -49,3 +52,55 @@ def test_serving_manifest_rejects_unknown_version() -> None: checksum=checksum, format_version=SERVING_MANIFEST_FORMAT_VERSION + 1, ) + + +def test_namespace_snapshot_uses_routing_only_v2_and_reads_legacy_v1() -> None: + payload = { + "documents": { + "doc_1": { + "job_result_id": "result_1", + "job_id": "job_1", + "source_file_name": "private.pdf", + "sections": [ + { + "section_id": "sec_1", + "section_path": "Root", + "section_title": "Root", + "section_level": 0, + "summary": "summary", + "sort_order": 0, + "unused": "drop", + } + ], + "chunks": [ + { + "chunk_id": "chunk_1", + "section_id": "sec_1", + "chunk_type": "text", + "sort_order": 0, + "connect_to": [], + "content": "drop", + } + ], + } + } + } + compressed, checksum, version = encode_namespace_map_snapshot(payload) + + assert version == NAMESPACE_MAP_SNAPSHOT_FORMAT_VERSION + decoded = decode_namespace_map_snapshot( + compressed, checksum=checksum, format_version=version + ) + document = decoded["documents"]["doc_1"] + assert "source_file_name" not in document + assert "unused" not in document["sections"][0] + assert "content" not in document["chunks"][0] + + legacy_compressed, legacy_checksum, legacy_version = encode_serving_manifest( + payload + ) + assert decode_namespace_map_snapshot( + legacy_compressed, + checksum=legacy_checksum, + format_version=legacy_version, + ) == payload diff --git a/packages/shared-python/shared/services/redis/redis_service.py b/packages/shared-python/shared/services/redis/redis_service.py index e83e3a7b..185467ec 100644 --- a/packages/shared-python/shared/services/redis/redis_service.py +++ b/packages/shared-python/shared/services/redis/redis_service.py @@ -160,6 +160,49 @@ async def _operation(): original_exception=e, ) + async def get_bytes(self, key: str) -> bytes | None: + """Get a value without JSON decoding (for compressed binary blobs).""" + try: + client = await self._get_client() + full_key = self._build_key(key) + + async def _operation() -> bytes | None: + result = await client.get(full_key) + if result is None: + return None + if isinstance(result, bytes): + return result + if isinstance(result, bytearray): + return bytes(result) + return str(result).encode("utf-8") + + return await self._execute_with_retry(_operation) + except Exception as e: + logger.error(f"Redis GET_BYTES operation failed: {e}") + raise RedisOperationError( + internal_message=f"GET_BYTES operation failed: {str(e)}", + operation="GET_BYTES", + original_exception=e, + ) + + async def set_bytes(self, key: str, value: bytes, *, ex: int) -> bool: + """Set a compressed binary value with an explicit TTL.""" + try: + client = await self._get_client() + full_key = self._build_key(key) + + async def _operation() -> bool: + return bool(await client.set(full_key, value, ex=ex)) + + return await self._execute_with_retry(_operation) + except Exception as e: + logger.error(f"Redis SET_BYTES operation failed: {e}") + raise RedisOperationError( + internal_message=f"SET_BYTES operation failed: {str(e)}", + operation="SET_BYTES", + original_exception=e, + ) + async def delete(self, *keys: str) -> int: """Delete keys.""" try: diff --git a/packages/shared-python/shared/services/retrieval/execution/routes.py b/packages/shared-python/shared/services/retrieval/execution/routes.py index 5dbf4f7c..ca9809a4 100644 --- a/packages/shared-python/shared/services/retrieval/execution/routes.py +++ b/packages/shared-python/shared/services/retrieval/execution/routes.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import resource import time from contextlib import AbstractAsyncContextManager @@ -173,6 +174,7 @@ async def _run_mapnav_route( context: RetrievalRouteContext, ) -> RetrievalRouteOutcome: """Default agentic path: PLANNER + HARVEST + CONTROL (checklist map-nav).""" + process_started = resource.getrusage(resource.RUSAGE_SELF) from shared.services.retrieval import nav_llm_backend # noqa: F401 from shared.services.retrieval.nav import run_nav_episode from shared.services.retrieval.nav.nav_hierarchy import ProviderToolSpace @@ -202,6 +204,7 @@ async def _run_mapnav_route( exclude_sections=context.exclude_sections, lazy=True, revision_pins=snapshot_pins, + generation=(snapshot_pins.generation if snapshot_pins is not None else None), ) if snapshot_pins is not None and not await is_revision_generation_stable( context.db, @@ -223,6 +226,7 @@ async def _run_mapnav_route( exclude_sections=context.exclude_sections, lazy=True, revision_pins=snapshot_pins, + generation=(snapshot_pins.generation if snapshot_pins is not None else None), ) snapshot_seconds = time.perf_counter() - snapshot_started logger.info( @@ -338,6 +342,13 @@ async def _run_mapnav_route( } completion_detail = f"chunks | evidence={len(evidence_text)} chars | router=mapnav" + process_finished = resource.getrusage(resource.RUSAGE_SELF) + logger.info( + "retrieval mapnav stage=process_resources cpu_seconds=%.3f max_rss_kb=%d", + (process_finished.ru_utime + process_finished.ru_stime) + - (process_started.ru_utime + process_started.ru_stime), + int(process_finished.ru_maxrss), + ) return RetrievalRouteOutcome( response=response, hit_stats_results=resolved.refs, diff --git a/packages/shared-python/shared/services/retrieval/namespace_map_snapshot.py b/packages/shared-python/shared/services/retrieval/namespace_map_snapshot.py index 41fa6c6b..2c31b296 100644 --- a/packages/shared-python/shared/services/retrieval/namespace_map_snapshot.py +++ b/packages/shared-python/shared/services/retrieval/namespace_map_snapshot.py @@ -19,8 +19,8 @@ ) from shared.services.retrieval.publication_models import DocumentPublicationScope from shared.services.retrieval.serving_manifest import ( - decode_serving_manifest, - encode_serving_manifest, + decode_namespace_map_snapshot, + encode_namespace_map_snapshot, ) @@ -107,7 +107,7 @@ def _decode_documents( if row is None: return {} try: - payload = decode_serving_manifest( + payload = decode_namespace_map_snapshot( row.payload_zlib, checksum=row.checksum, format_version=row.format_version, @@ -127,7 +127,7 @@ def _write_snapshot( documents: dict[str, dict[str, Any]], target_generation: int, ) -> None: - encoded, checksum, format_version = encode_serving_manifest( + encoded, checksum, format_version = encode_namespace_map_snapshot( {"documents": documents} ) if row is None: diff --git a/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_cache.py b/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_cache.py deleted file mode 100644 index 6cc5cab2..00000000 --- a/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_cache.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Process-local cache for decoded namespace MAP snapshot documents. - -Keyed by ``(user_id, namespace, generation)`` so a publish/archive that bumps -the namespace generation invalidates stale entries automatically -- no manual -invalidation call is needed. Bounded LRU keeps memory use predictable across -many namespaces sharing one worker process. -""" - -from __future__ import annotations - -import threading -from collections import OrderedDict -from typing import Any - -_MAX_ENTRIES = 64 -_lock = threading.Lock() -_cache: "OrderedDict[tuple[str, str, int], dict[str, dict[str, Any]]]" = OrderedDict() - - -def get_cached_namespace_documents( - *, user_id: str, namespace: str, generation: int -) -> dict[str, dict[str, Any]] | None: - key = (user_id, namespace, generation) - with _lock: - documents = _cache.get(key) - if documents is not None: - _cache.move_to_end(key) - return documents - - -def cache_namespace_documents( - *, - user_id: str, - namespace: str, - generation: int, - documents: dict[str, dict[str, Any]], -) -> None: - key = (user_id, namespace, generation) - with _lock: - _cache[key] = documents - _cache.move_to_end(key) - while len(_cache) > _MAX_ENTRIES: - _cache.popitem(last=False) diff --git a/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_redis.py b/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_redis.py new file mode 100644 index 00000000..fc8447c6 --- /dev/null +++ b/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_redis.py @@ -0,0 +1,38 @@ +"""Redis cache for compressed namespace MAP routing snapshots.""" + +from __future__ import annotations + +from shared.models.schemas.retrieval_namespace import normalize_retrieval_namespace +from shared.services.redis import RedisServiceFactory + +_CACHE_TTL_SECONDS = 3600 +_KEY_PREFIX = "retrieval:snapshot:v2" + + +def snapshot_cache_key(*, user_id: str, namespace: str, generation: int) -> str: + normalized_namespace = normalize_retrieval_namespace(namespace) + return f"{_KEY_PREFIX}:{user_id}:{normalized_namespace}:g{int(generation)}" + + +async def get_snapshot_blob( + *, user_id: str, namespace: str, generation: int +) -> bytes | None: + service = RedisServiceFactory.get_service() + return await service.get_bytes( + snapshot_cache_key( + user_id=user_id, namespace=namespace, generation=generation + ) + ) + + +async def set_snapshot_blob( + *, user_id: str, namespace: str, generation: int, payload_zlib: bytes +) -> bool: + service = RedisServiceFactory.get_service() + return await service.set_bytes( + snapshot_cache_key( + user_id=user_id, namespace=namespace, generation=generation + ), + payload_zlib, + ex=_CACHE_TTL_SECONDS, + ) diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py b/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py index 28c04600..c3637464 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py @@ -227,31 +227,31 @@ def _pool_unit_scores_to_tree( unit_scores: Dict[str, float], ) -> Dict[str, float]: """MAX-pool globally comparable unit scores onto one document tree.""" - map_scores = { - leaf_id: float(unit_scores.get(leaf_id, 0.0) or 0.0) for leaf_id in leaves - } - - def score_node(section_id: str) -> float: - if section_id in map_scores: - return map_scores[section_id] + # ``_walk_tree`` inserts every parent before its children. Reversing that + # order is therefore a postorder traversal without allocating descendant + # lists or revisiting nodes. + map_scores: Dict[str, float] = {} + descendant_leaf_max: Dict[str, float] = {} + for section_id in reversed(children_map): kids = children_map.get(section_id) or [] if not kids: - score = float(unit_scores.get(section_id, 0.0) or 0.0) - map_scores[section_id] = score - return score - descendant_leaves = _collect_descendant_leaves(section_id, children_map, leaves) - parts = [ - float(unit_scores.get(leaf_id, 0.0) or 0.0) for leaf_id in descendant_leaves - ] - self_key = f"{section_id}__self" - if self_key in unit_scores: - parts.append(float(unit_scores[self_key])) - score = float(max(parts)) if parts else 0.0 - map_scores[section_id] = score - return score - - for section_id in children_map: - score_node(section_id) + leaf_score = float(unit_scores.get(section_id, 0.0) or 0.0) + descendant_leaf_max[section_id] = leaf_score + score = leaf_score + else: + child_leaf_scores = [ + descendant_leaf_max.get(kid, float(unit_scores.get(kid, 0.0) or 0.0)) + for kid in kids + ] + leaf_score = max(child_leaf_scores, default=0.0) + descendant_leaf_max[section_id] = leaf_score + self_score = float(unit_scores.get(f"{section_id}__self", 0.0) or 0.0) + score = max(leaf_score, self_score) + map_scores[section_id] = float(score) + # Preserve the legacy behavior for leaf ids that are not present in the + # children map (defensive support for sparse providers). + for leaf_id in leaves: + map_scores.setdefault(leaf_id, float(unit_scores.get(leaf_id, 0.0) or 0.0)) return map_scores diff --git a/packages/shared-python/shared/services/retrieval/nav_snapshot.py b/packages/shared-python/shared/services/retrieval/nav_snapshot.py index ea502c54..50a4b4f8 100644 --- a/packages/shared-python/shared/services/retrieval/nav_snapshot.py +++ b/packages/shared-python/shared/services/retrieval/nav_snapshot.py @@ -9,7 +9,9 @@ import json import logging +import os import time +import zlib from dataclasses import dataclass from collections.abc import Callable, Iterator, Mapping from typing import Any, Protocol @@ -33,6 +35,7 @@ DocumentChunk, DocumentSection, RetrievalNamespaceMapSnapshot, + RetrievalNamespaceGeneration, RetrievalServingRevisionManifest, ) from shared.models.database.job_result import JobResult @@ -46,11 +49,14 @@ knowhere_database_url, ) from shared.services.retrieval.search.section_filters import is_excluded_section -from shared.services.retrieval.serving_manifest import decode_serving_manifest +from shared.services.retrieval.serving_manifest import ( + decode_namespace_map_snapshot, + decode_serving_manifest, +) from shared.services.retrieval.manifest_cache import get_cached_manifest_payloads -from shared.services.retrieval.namespace_map_snapshot_cache import ( - cache_namespace_documents, - get_cached_namespace_documents, +from shared.services.retrieval.namespace_map_snapshot_redis import ( + get_snapshot_blob, + set_snapshot_blob, ) @@ -70,6 +76,16 @@ _logger = logging.getLogger(__name__) +def _snapshot_timing_enabled() -> bool: + """Enable detailed snapshot timings for local performance diagnosis.""" + return os.environ.get("RETRIEVAL_SNAPSHOT_TIMING", "0").strip().lower() in { + "1", + "true", + "yes", + "on", + } + + class SnapshotSession(Protocol): """Minimal database interface required by the snapshot loader.""" @@ -149,12 +165,14 @@ async def load_nav_snapshot( exclude_sections: list[dict[str, str]] | None = None, lazy: bool = False, revision_pins: Mapping[str, str] | None = None, + generation: int | None = None, ) -> NavSnapshot: """Preload namespace current revision into a sync map-nav snapshot.""" excluded_docs = [ str(x).strip() for x in (exclude_document_ids or ()) if str(x).strip() ] excluded_secs = list(exclude_sections or ()) + snapshot_started = time.perf_counter() if revision_pins is None: doc_stmt = ( @@ -182,7 +200,9 @@ async def load_nav_snapshot( ) if excluded_docs: doc_stmt = doc_stmt.where(Document.document_id.notin_(excluded_docs)) + doc_query_started = time.perf_counter() doc_rows = list((await db.execute(doc_stmt)).all()) + doc_query_seconds = time.perf_counter() - doc_query_started if not doc_rows: raise ValueError( f"no active documents with current revision for " @@ -207,11 +227,13 @@ async def load_nav_snapshot( current_job_result_ids.add(job_result_id) document_revisions.append((did, job_result_id)) + job_query_started = time.perf_counter() job_result_rows = await db.execute( select(JobResult.id, JobResult.job_id).where( JobResult.id.in_(list(current_job_result_ids)) ) ) + job_query_seconds = time.perf_counter() - job_query_started job_id_by_result_id = { str(job_result_id): str(job_id) for job_result_id, job_id in job_result_rows.all() @@ -223,13 +245,16 @@ async def load_nav_snapshot( user_id=user_id, namespace=namespace, document_revisions=document_revisions, + expected_generation=generation, ) if snapshot_entries is not None: + parse_started = time.perf_counter() manifest_sections = _parse_manifest_entries( snapshot_entries, exclude_sections=excluded_secs, job_id_by_result_id=job_id_by_result_id, ) + parse_seconds = time.perf_counter() - parse_started else: _logger.warning( "retrieval snapshot fallback=manifest_merge user_id=%s namespace=%s documents=%d", @@ -245,6 +270,7 @@ async def load_nav_snapshot( exclude_sections=excluded_secs, job_id_by_result_id=job_id_by_result_id, ) + parse_seconds = 0.0 if manifest_sections is None: _logger.warning( "retrieval snapshot fallback=table_scan user_id=%s namespace=%s documents=%d", @@ -324,21 +350,39 @@ async def load_nav_snapshot( for chunk_id in chunk_ids }, ) + ref_index_started = time.perf_counter() + lazy_ref_index = LazyChunkRefIndex( + chunk_ref_index, + resolver=store.load_chunk_reference_metadata, + ) + ref_index_seconds = time.perf_counter() - ref_index_started except Exception: store.close() raise - return NavSnapshot( + snapshot = NavSnapshot( provider=provider, - chunk_ref_index=LazyChunkRefIndex( - chunk_ref_index, - resolver=store.load_chunk_reference_metadata, - ), + chunk_ref_index=lazy_ref_index, document_ids=list(provider.document_ids()), document_titles={ did: kept_titles.get(did, did) for did in provider.document_ids() }, document_revisions=dict(revisions), ) + if _snapshot_timing_enabled(): + _logger.info( + "retrieval snapshot timing total_seconds=%.3f parse_seconds=%.3f " + "ref_index_copy_seconds=%.3f doc_query_seconds=%.3f " + "job_query_seconds=%.3f documents=%d sections=%d refs=%d mode=lazy", + time.perf_counter() - snapshot_started, + parse_seconds, + ref_index_seconds, + doc_query_seconds, + job_query_seconds, + len(snapshot.document_ids), + sum(len(rows) for rows in sections_by_doc.values()), + len(snapshot.chunk_ref_index), + ) + return snapshot units_by_doc, chunk_ref_index = await _load_chunks( db, @@ -358,7 +402,7 @@ async def load_nav_snapshot( f"user_id={user_id!r} namespace={namespace!r}" ) - return build_nav_snapshot( + snapshot = build_nav_snapshot( document_titles=kept_titles, sections_by_doc={did: sections_by_doc.get(did, []) for did in kept_titles}, units_by_doc={did: units_by_doc.get(did, []) for did in kept_titles}, @@ -369,6 +413,20 @@ async def load_nav_snapshot( if document_id in kept_titles }, ) + if _snapshot_timing_enabled(): + _logger.info( + "retrieval snapshot timing total_seconds=%.3f parse_seconds=%.3f " + "doc_query_seconds=%.3f job_query_seconds=%.3f documents=%d " + "sections=%d refs=%d mode=eager", + time.perf_counter() - snapshot_started, + parse_seconds, + doc_query_seconds, + job_query_seconds, + len(snapshot.document_ids), + sum(len(rows) for rows in sections_by_doc.values()), + len(snapshot.chunk_ref_index), + ) + return snapshot async def _resolve_namespace_snapshot_entries( @@ -377,21 +435,39 @@ async def _resolve_namespace_snapshot_entries( user_id: str, namespace: str, document_revisions: list[tuple[str, str]], + expected_generation: int | None = None, ) -> list[tuple[object, ...]] | None: """Return manifest-shaped entries from the persisted namespace snapshot. Returns ``None`` (triggering the exact per-revision fallback) when the snapshot row is missing, corrupt, or stale for any requested revision. """ + generation_statement = select(RetrievalNamespaceGeneration.generation).where( + RetrievalNamespaceGeneration.user_id == user_id, + RetrievalNamespaceGeneration.namespace == namespace, + ) + try: + generation_result = await db.execute(generation_statement) + _current_generation = generation_result.scalar_one_or_none() + except SQLAlchemyError as exc: + await db.rollback() + _logger.warning("retrieval snapshot generation lookup failed error=%s", exc) + _current_generation = None + if _snapshot_timing_enabled(): + _logger.info( + "retrieval snapshot generation generation=%s", + _current_generation, + ) + generation_value = int(expected_generation) if expected_generation is not None else None statement = select( RetrievalNamespaceMapSnapshot.generation, - RetrievalNamespaceMapSnapshot.payload_zlib, RetrievalNamespaceMapSnapshot.checksum, RetrievalNamespaceMapSnapshot.format_version, ).where( RetrievalNamespaceMapSnapshot.user_id == user_id, RetrievalNamespaceMapSnapshot.namespace == namespace, ) + lookup_started = time.perf_counter() try: row = (await db.execute(statement)).first() except SQLAlchemyError as exc: @@ -403,30 +479,124 @@ async def _resolve_namespace_snapshot_entries( exc, ) return None + lookup_seconds = time.perf_counter() - lookup_started + if _snapshot_timing_enabled(): + _logger.info( + "retrieval snapshot lookup seconds=%.3f found=%s", + lookup_seconds, + row is not None, + ) if row is None: return None - generation, payload_zlib, checksum, format_version = row - documents = get_cached_namespace_documents( - user_id=user_id, namespace=namespace, generation=int(generation) - ) - if documents is None: - try: - payload = decode_serving_manifest( - bytes(payload_zlib), - checksum=str(checksum), - format_version=int(format_version), + generation, checksum, format_version = row + row_generation = int(generation) + if generation_value is not None and row_generation != generation_value: + _logger.info( + "retrieval snapshot generation mismatch row=%d expected=%d", + row_generation, + generation_value, + ) + return None + cached_blob: bytes | None = None + try: + cached_blob = await get_snapshot_blob( + user_id=user_id, namespace=namespace, generation=row_generation + ) + except Exception as exc: + _logger.warning("retrieval snapshot redis get failed error=%s", exc) + database_blob: bytes | None = None + if cached_blob is None: + payload_result = await db.execute( + select(RetrievalNamespaceMapSnapshot.payload_zlib).where( + RetrievalNamespaceMapSnapshot.user_id == user_id, + RetrievalNamespaceMapSnapshot.namespace == namespace, ) - except (ValueError, TypeError): + ) + payload_row = payload_result.first() + if payload_row is None or payload_row[0] is None: return None - decoded_documents = payload.get("documents") - if not isinstance(decoded_documents, dict): + database_blob = bytes(payload_row[0]) + blob: bytes = database_blob + else: + blob = cached_blob + timing_enabled = _snapshot_timing_enabled() + decode_timings: dict[str, float] | None = {} if timing_enabled else None + decode_started = time.perf_counter() + try: + payload = decode_namespace_map_snapshot( + blob, + checksum=str(checksum), + format_version=int(format_version), + timings=decode_timings, + ) + except (ValueError, TypeError, zlib.error): + if cached_blob is not None: + # A stale/corrupt cache entry must never shadow the PostgreSQL source. + try: + if database_blob is None: + fallback_result = await db.execute( + select(RetrievalNamespaceMapSnapshot.payload_zlib).where( + RetrievalNamespaceMapSnapshot.user_id == user_id, + RetrievalNamespaceMapSnapshot.namespace == namespace, + ) + ) + fallback_row = fallback_result.first() + if fallback_row is None or fallback_row[0] is None: + return None + database_blob = bytes(fallback_row[0]) + assert database_blob is not None + payload = decode_namespace_map_snapshot( + database_blob, + checksum=str(checksum), + format_version=int(format_version), + timings=decode_timings, + ) + blob = database_blob + cached_blob = None + except (ValueError, TypeError, zlib.error): + return None + else: return None - documents = decoded_documents - cache_namespace_documents( - user_id=user_id, - namespace=namespace, - generation=int(generation), - documents=documents, + decoded_documents = payload.get("documents") + if not isinstance(decoded_documents, dict): + return None + documents = decoded_documents + if cached_blob is None: + try: + await set_snapshot_blob( + user_id=user_id, + namespace=namespace, + generation=row_generation, + payload_zlib=blob, + ) + except Exception as exc: + _logger.warning("retrieval snapshot redis set failed error=%s", exc) + if timing_enabled: + _logger.info( + "retrieval snapshot decode cache_hit=false seconds=%.3f " + "compressed_bytes=%d decompressed_bytes=%d " + "decompress_seconds=%.3f checksum_seconds=%.3f " + "json_decode_seconds=%.3f documents=%d", + time.perf_counter() - decode_started, + int((decode_timings or {}).get("compressed_bytes", 0.0)), + int((decode_timings or {}).get("decompressed_bytes", 0.0)), + (decode_timings or {}).get("decompress_seconds", 0.0), + (decode_timings or {}).get("checksum_seconds", 0.0), + (decode_timings or {}).get("json_decode_seconds", 0.0), + len(documents), + ) + elif timing_enabled: + _logger.info( + "retrieval snapshot decode cache_hit=true seconds=%.3f " + "compressed_bytes=%d decompressed_bytes=%d decompress_seconds=%.3f " + "checksum_seconds=%.3f json_decode_seconds=%.3f documents=%d", + time.perf_counter() - decode_started, + int((decode_timings or {}).get("compressed_bytes", 0.0)), + int((decode_timings or {}).get("decompressed_bytes", 0.0)), + (decode_timings or {}).get("decompress_seconds", 0.0), + (decode_timings or {}).get("checksum_seconds", 0.0), + (decode_timings or {}).get("json_decode_seconds", 0.0), + len(documents), ) entries: list[tuple[object, ...]] = [] for document_id, job_result_id in document_revisions: @@ -532,6 +702,11 @@ def _parse_manifest_entries( ref_index: dict[str, dict[str, Any]] = {} root_assets_by_doc: dict[str, set[str]] = {} text_connections_by_doc: dict[str, list[tuple[str, str]]] = {} + section_seconds = 0.0 + chunk_seconds = 0.0 + section_count = 0 + chunk_count = 0 + connection_count = 0 try: for document_id, _job_result_id, payload_zlib, checksum, format_version in manifest_entries: if isinstance(payload_zlib, dict): @@ -546,6 +721,7 @@ def _parse_manifest_entries( checksum=str(checksum), format_version=int(str(format_version)), ) + section_started = time.perf_counter() raw_sections = payload.get("sections") if not isinstance(raw_sections, list): return None @@ -577,6 +753,9 @@ def _parse_manifest_entries( ) by_doc.setdefault(str(document_id), []).append(section) path_by_id[section_id] = section_path + section_count += 1 + section_seconds += time.perf_counter() - section_started + chunk_started = time.perf_counter() raw_chunks = payload.get("chunks") if not isinstance(raw_chunks, list): return None @@ -628,15 +807,32 @@ def _parse_manifest_entries( text_connections_by_doc.setdefault(document_key, []).append( (section_id or "", target) ) + connection_count += 1 + chunk_count += 1 + chunk_seconds += time.perf_counter() - chunk_started except (TypeError, ValueError, KeyError): return None remounted: dict[str, dict[str, Any]] = {} + remount_started = time.perf_counter() for document_id, asset_ids in root_assets_by_doc.items(): owners: dict[str, list[str]] = {} for section_id, target in text_connections_by_doc.get(document_id, ()): if target in asset_ids: owners.setdefault(section_id, []).append(target) remounted[document_id] = {"root": sorted(asset_ids), "owners": owners} + if _snapshot_timing_enabled(): + _logger.info( + "retrieval snapshot parse sections=%d chunks=%d connections=%d " + "section_seconds=%.3f chunk_seconds=%.3f remount_seconds=%.3f " + "ref_index_keys=%d", + section_count, + chunk_count, + connection_count, + section_seconds, + chunk_seconds, + time.perf_counter() - remount_started, + len(ref_index), + ) return by_doc, path_by_id, (ids_by_doc, ref_index, remounted) diff --git a/packages/shared-python/shared/services/retrieval/serving_manifest.py b/packages/shared-python/shared/services/retrieval/serving_manifest.py index c66ae579..9895ca43 100644 --- a/packages/shared-python/shared/services/retrieval/serving_manifest.py +++ b/packages/shared-python/shared/services/retrieval/serving_manifest.py @@ -4,8 +4,9 @@ import hashlib import json +import time import zlib -from typing import Any +from typing import Any, MutableMapping from sqlalchemy import delete, select from sqlalchemy.orm import Session @@ -20,6 +21,7 @@ from shared.services.retrieval.publication_models import DocumentPublicationScope SERVING_MANIFEST_FORMAT_VERSION = 1 +NAMESPACE_MAP_SNAPSHOT_FORMAT_VERSION = 2 def build_revision_serving_payload( @@ -181,24 +183,141 @@ def decode_serving_manifest( *, checksum: str, format_version: int, + timings: MutableMapping[str, float] | None = None, ) -> dict[str, Any]: """Validate and decode one persisted serving manifest.""" if format_version != SERVING_MANIFEST_FORMAT_VERSION: raise ValueError(f"unsupported serving manifest version: {format_version}") + started = time.perf_counter() try: canonical_payload = zlib.decompress(payload_zlib) except zlib.error as exc: raise ValueError("invalid serving manifest compression") from exc + if timings is not None: + timings["decompress_seconds"] = time.perf_counter() - started + checksum_started = time.perf_counter() actual_checksum = hashlib.sha256(canonical_payload).hexdigest() if actual_checksum != checksum: raise ValueError("serving manifest checksum mismatch") + if timings is not None: + timings["checksum_seconds"] = time.perf_counter() - checksum_started + json_started = time.perf_counter() try: decoded = json.loads(canonical_payload.decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise ValueError("invalid serving manifest JSON") from exc + if timings is not None: + timings["json_decode_seconds"] = time.perf_counter() - json_started + timings["compressed_bytes"] = float(len(payload_zlib)) + timings["decompressed_bytes"] = float(len(canonical_payload)) + timings["decode_seconds"] = time.perf_counter() - started if not isinstance(decoded, dict): raise ValueError("serving manifest payload must be an object") return decoded + + +def encode_namespace_map_snapshot( + payload: dict[str, Any], +) -> tuple[bytes, str, int]: + """Encode the routing-only namespace snapshot using its own format version.""" + documents = payload.get("documents") + routing_documents: dict[str, Any] = {} + if isinstance(documents, dict): + for document_id, raw_document in documents.items(): + if not isinstance(raw_document, dict): + continue + sections = [ + { + key: section[key] + for key in ( + "section_id", + "parent_section_id", + "section_path", + "section_title", + "section_level", + "summary", + "sort_order", + ) + if key in section + } + for section in (raw_document.get("sections") or []) + if isinstance(section, dict) + ] + chunks = [ + { + key: chunk[key] + for key in ( + "chunk_id", + "section_id", + "chunk_type", + "sort_order", + "connect_to", + ) + if key in chunk + } + for chunk in (raw_document.get("chunks") or []) + if isinstance(chunk, dict) + ] + routing_documents[str(document_id)] = { + "job_result_id": raw_document.get("job_result_id"), + "job_id": raw_document.get("job_id"), + "sections": sections, + "chunks": chunks, + "root_asset_ids": raw_document.get("root_asset_ids") or [], + "remounted_assets_by_section": raw_document.get( + "remounted_assets_by_section" + ) + or {}, + } + compressed, checksum, _ = encode_serving_manifest({"documents": routing_documents}) + return compressed, checksum, NAMESPACE_MAP_SNAPSHOT_FORMAT_VERSION + + +def decode_namespace_map_snapshot( + payload_zlib: bytes, + *, + checksum: str, + format_version: int, + timings: MutableMapping[str, float] | None = None, +) -> dict[str, Any]: + """Decode namespace snapshots, retaining compatibility with legacy v1 rows.""" + if format_version == SERVING_MANIFEST_FORMAT_VERSION: + return decode_serving_manifest( + payload_zlib, + checksum=checksum, + format_version=SERVING_MANIFEST_FORMAT_VERSION, + timings=timings, + ) + if format_version != NAMESPACE_MAP_SNAPSHOT_FORMAT_VERSION: + raise ValueError(f"unsupported namespace snapshot version: {format_version}") + # The compression/checksum contract is identical to serving manifests; only + # the payload shape/version is different. + started = time.perf_counter() + try: + canonical_payload = zlib.decompress(payload_zlib) + except zlib.error as exc: + raise ValueError("invalid namespace snapshot compression") from exc + if timings is not None: + timings["decompress_seconds"] = time.perf_counter() - started + timings["compressed_bytes"] = float(len(payload_zlib)) + timings["decompressed_bytes"] = float(len(canonical_payload)) + checksum_started = time.perf_counter() + actual_checksum = hashlib.sha256(canonical_payload).hexdigest() + if actual_checksum != checksum: + raise ValueError("namespace snapshot checksum mismatch") + if timings is not None: + timings["checksum_seconds"] = time.perf_counter() - checksum_started + json_started = time.perf_counter() + try: + decoded = json.loads(canonical_payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("invalid namespace snapshot JSON") from exc + if timings is not None: + timings["json_decode_seconds"] = time.perf_counter() - json_started + timings["decode_seconds"] = time.perf_counter() - started + if not isinstance(decoded, dict): + raise ValueError("namespace snapshot payload must be an object") + return decoded From 25c074c70e247ac398055681b669ffcb5bc3d07e Mon Sep 17 00:00:00 2001 From: suguanYang Date: Tue, 1 Sep 2026 21:55:21 +0800 Subject: [PATCH 03/35] fix: enforce snapshot generation and validation --- .../retrieval/namespace_map_snapshot_redis.py | 6 +- .../shared/services/retrieval/nav_snapshot.py | 37 +++-- .../services/retrieval/serving_manifest.py | 140 +++++++++--------- 3 files changed, 99 insertions(+), 84 deletions(-) diff --git a/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_redis.py b/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_redis.py index fc8447c6..b977e5f4 100644 --- a/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_redis.py +++ b/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_redis.py @@ -9,7 +9,7 @@ _KEY_PREFIX = "retrieval:snapshot:v2" -def snapshot_cache_key(*, user_id: str, namespace: str, generation: int) -> str: +def build_snapshot_cache_key(*, user_id: str, namespace: str, generation: int) -> str: normalized_namespace = normalize_retrieval_namespace(namespace) return f"{_KEY_PREFIX}:{user_id}:{normalized_namespace}:g{int(generation)}" @@ -19,7 +19,7 @@ async def get_snapshot_blob( ) -> bytes | None: service = RedisServiceFactory.get_service() return await service.get_bytes( - snapshot_cache_key( + build_snapshot_cache_key( user_id=user_id, namespace=namespace, generation=generation ) ) @@ -30,7 +30,7 @@ async def set_snapshot_blob( ) -> bool: service = RedisServiceFactory.get_service() return await service.set_bytes( - snapshot_cache_key( + build_snapshot_cache_key( user_id=user_id, namespace=namespace, generation=generation ), payload_zlib, diff --git a/packages/shared-python/shared/services/retrieval/nav_snapshot.py b/packages/shared-python/shared/services/retrieval/nav_snapshot.py index 50a4b4f8..3210b55f 100644 --- a/packages/shared-python/shared/services/retrieval/nav_snapshot.py +++ b/packages/shared-python/shared/services/retrieval/nav_snapshot.py @@ -58,6 +58,7 @@ get_snapshot_blob, set_snapshot_blob, ) +from shared.core.exceptions.redis_exceptions import RedisOperationError # Keep each payload query bounded under the API's 30-second statement timeout. @@ -76,7 +77,7 @@ _logger = logging.getLogger(__name__) -def _snapshot_timing_enabled() -> bool: +def _is_snapshot_timing_enabled() -> bool: """Enable detailed snapshot timings for local performance diagnosis.""" return os.environ.get("RETRIEVAL_SNAPSHOT_TIMING", "0").strip().lower() in { "1", @@ -368,7 +369,7 @@ async def load_nav_snapshot( }, document_revisions=dict(revisions), ) - if _snapshot_timing_enabled(): + if _is_snapshot_timing_enabled(): _logger.info( "retrieval snapshot timing total_seconds=%.3f parse_seconds=%.3f " "ref_index_copy_seconds=%.3f doc_query_seconds=%.3f " @@ -413,7 +414,7 @@ async def load_nav_snapshot( if document_id in kept_titles }, ) - if _snapshot_timing_enabled(): + if _is_snapshot_timing_enabled(): _logger.info( "retrieval snapshot timing total_seconds=%.3f parse_seconds=%.3f " "doc_query_seconds=%.3f job_query_seconds=%.3f documents=%d " @@ -453,7 +454,7 @@ async def _resolve_namespace_snapshot_entries( await db.rollback() _logger.warning("retrieval snapshot generation lookup failed error=%s", exc) _current_generation = None - if _snapshot_timing_enabled(): + if _is_snapshot_timing_enabled(): _logger.info( "retrieval snapshot generation generation=%s", _current_generation, @@ -480,7 +481,7 @@ async def _resolve_namespace_snapshot_entries( ) return None lookup_seconds = time.perf_counter() - lookup_started - if _snapshot_timing_enabled(): + if _is_snapshot_timing_enabled(): _logger.info( "retrieval snapshot lookup seconds=%.3f found=%s", lookup_seconds, @@ -497,12 +498,24 @@ async def _resolve_namespace_snapshot_entries( generation_value, ) return None + if ( + generation_value is None + and _current_generation is not None + and int(_current_generation) > 0 + and row_generation != int(_current_generation) + ): + _logger.info( + "retrieval snapshot generation mismatch row=%d current=%d", + row_generation, + int(_current_generation), + ) + return None cached_blob: bytes | None = None try: cached_blob = await get_snapshot_blob( user_id=user_id, namespace=namespace, generation=row_generation ) - except Exception as exc: + except RedisOperationError as exc: _logger.warning("retrieval snapshot redis get failed error=%s", exc) database_blob: bytes | None = None if cached_blob is None: @@ -519,8 +532,8 @@ async def _resolve_namespace_snapshot_entries( blob: bytes = database_blob else: blob = cached_blob - timing_enabled = _snapshot_timing_enabled() - decode_timings: dict[str, float] | None = {} if timing_enabled else None + is_timing_enabled = _is_snapshot_timing_enabled() + decode_timings: dict[str, float] | None = {} if is_timing_enabled else None decode_started = time.perf_counter() try: payload = decode_namespace_map_snapshot( @@ -569,9 +582,9 @@ async def _resolve_namespace_snapshot_entries( generation=row_generation, payload_zlib=blob, ) - except Exception as exc: + except RedisOperationError as exc: _logger.warning("retrieval snapshot redis set failed error=%s", exc) - if timing_enabled: + if is_timing_enabled: _logger.info( "retrieval snapshot decode cache_hit=false seconds=%.3f " "compressed_bytes=%d decompressed_bytes=%d " @@ -585,7 +598,7 @@ async def _resolve_namespace_snapshot_entries( (decode_timings or {}).get("json_decode_seconds", 0.0), len(documents), ) - elif timing_enabled: + elif is_timing_enabled: _logger.info( "retrieval snapshot decode cache_hit=true seconds=%.3f " "compressed_bytes=%d decompressed_bytes=%d decompress_seconds=%.3f " @@ -820,7 +833,7 @@ def _parse_manifest_entries( if target in asset_ids: owners.setdefault(section_id, []).append(target) remounted[document_id] = {"root": sorted(asset_ids), "owners": owners} - if _snapshot_timing_enabled(): + if _is_snapshot_timing_enabled(): _logger.info( "retrieval snapshot parse sections=%d chunks=%d connections=%d " "section_seconds=%.3f chunk_seconds=%.3f remount_seconds=%.3f " diff --git a/packages/shared-python/shared/services/retrieval/serving_manifest.py b/packages/shared-python/shared/services/retrieval/serving_manifest.py index 9895ca43..8480b8f4 100644 --- a/packages/shared-python/shared/services/retrieval/serving_manifest.py +++ b/packages/shared-python/shared/services/retrieval/serving_manifest.py @@ -189,18 +189,41 @@ def decode_serving_manifest( if format_version != SERVING_MANIFEST_FORMAT_VERSION: raise ValueError(f"unsupported serving manifest version: {format_version}") + return _decode_compressed_json( + payload_zlib, + checksum=checksum, + timings=timings, + compression_error="invalid serving manifest compression", + checksum_error="serving manifest checksum mismatch", + json_error="invalid serving manifest JSON", + object_error="serving manifest payload must be an object", + ) + + +def _decode_compressed_json( + payload_zlib: bytes, + *, + checksum: str, + timings: MutableMapping[str, float] | None, + compression_error: str, + checksum_error: str, + json_error: str, + object_error: str, +) -> dict[str, Any]: + """Decompress, validate, and decode a canonical JSON payload.""" + started = time.perf_counter() try: canonical_payload = zlib.decompress(payload_zlib) except zlib.error as exc: - raise ValueError("invalid serving manifest compression") from exc + raise ValueError(compression_error) from exc if timings is not None: timings["decompress_seconds"] = time.perf_counter() - started checksum_started = time.perf_counter() actual_checksum = hashlib.sha256(canonical_payload).hexdigest() if actual_checksum != checksum: - raise ValueError("serving manifest checksum mismatch") + raise ValueError(checksum_error) if timings is not None: timings["checksum_seconds"] = time.perf_counter() - checksum_started @@ -208,14 +231,14 @@ def decode_serving_manifest( try: decoded = json.loads(canonical_payload.decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise ValueError("invalid serving manifest JSON") from exc + raise ValueError(json_error) from exc if timings is not None: timings["json_decode_seconds"] = time.perf_counter() - json_started timings["compressed_bytes"] = float(len(payload_zlib)) timings["decompressed_bytes"] = float(len(canonical_payload)) timings["decode_seconds"] = time.perf_counter() - started if not isinstance(decoded, dict): - raise ValueError("serving manifest payload must be an object") + raise ValueError(object_error) return decoded @@ -224,54 +247,52 @@ def encode_namespace_map_snapshot( ) -> tuple[bytes, str, int]: """Encode the routing-only namespace snapshot using its own format version.""" documents = payload.get("documents") + if not isinstance(documents, dict): + raise ValueError("namespace snapshot documents must be an object") routing_documents: dict[str, Any] = {} - if isinstance(documents, dict): - for document_id, raw_document in documents.items(): - if not isinstance(raw_document, dict): - continue - sections = [ + for document_id, raw_document in documents.items(): + if not isinstance(raw_document, dict): + raise ValueError(f"namespace snapshot document is not an object: {document_id}") + raw_sections = raw_document.get("sections") + raw_chunks = raw_document.get("chunks") + if not isinstance(raw_sections, list) or not isinstance(raw_chunks, list): + raise ValueError(f"namespace snapshot records are invalid: {document_id}") + sections = [] + for section in raw_sections: + if not isinstance(section, dict) or not str(section.get("section_id") or ""): + raise ValueError(f"namespace snapshot section is invalid: {document_id}") + sections.append( { key: section[key] for key in ( - "section_id", - "parent_section_id", - "section_path", - "section_title", - "section_level", - "summary", - "sort_order", + "section_id", "parent_section_id", "section_path", + "section_title", "section_level", "summary", "sort_order", ) if key in section } - for section in (raw_document.get("sections") or []) - if isinstance(section, dict) - ] - chunks = [ + ) + chunks = [] + for chunk in raw_chunks: + if not isinstance(chunk, dict) or not str(chunk.get("chunk_id") or ""): + raise ValueError(f"namespace snapshot chunk is invalid: {document_id}") + chunks.append( { key: chunk[key] - for key in ( - "chunk_id", - "section_id", - "chunk_type", - "sort_order", - "connect_to", - ) + for key in ("chunk_id", "section_id", "chunk_type", "sort_order", "connect_to") if key in chunk } - for chunk in (raw_document.get("chunks") or []) - if isinstance(chunk, dict) - ] - routing_documents[str(document_id)] = { - "job_result_id": raw_document.get("job_result_id"), - "job_id": raw_document.get("job_id"), - "sections": sections, - "chunks": chunks, - "root_asset_ids": raw_document.get("root_asset_ids") or [], - "remounted_assets_by_section": raw_document.get( - "remounted_assets_by_section" - ) - or {}, - } + ) + routing_documents[str(document_id)] = { + "job_result_id": raw_document.get("job_result_id"), + "job_id": raw_document.get("job_id"), + "sections": sections, + "chunks": chunks, + "root_asset_ids": raw_document.get("root_asset_ids") or [], + "remounted_assets_by_section": raw_document.get( + "remounted_assets_by_section" + ) + or {}, + } compressed, checksum, _ = encode_serving_manifest({"documents": routing_documents}) return compressed, checksum, NAMESPACE_MAP_SNAPSHOT_FORMAT_VERSION @@ -293,31 +314,12 @@ def decode_namespace_map_snapshot( ) if format_version != NAMESPACE_MAP_SNAPSHOT_FORMAT_VERSION: raise ValueError(f"unsupported namespace snapshot version: {format_version}") - # The compression/checksum contract is identical to serving manifests; only - # the payload shape/version is different. - started = time.perf_counter() - try: - canonical_payload = zlib.decompress(payload_zlib) - except zlib.error as exc: - raise ValueError("invalid namespace snapshot compression") from exc - if timings is not None: - timings["decompress_seconds"] = time.perf_counter() - started - timings["compressed_bytes"] = float(len(payload_zlib)) - timings["decompressed_bytes"] = float(len(canonical_payload)) - checksum_started = time.perf_counter() - actual_checksum = hashlib.sha256(canonical_payload).hexdigest() - if actual_checksum != checksum: - raise ValueError("namespace snapshot checksum mismatch") - if timings is not None: - timings["checksum_seconds"] = time.perf_counter() - checksum_started - json_started = time.perf_counter() - try: - decoded = json.loads(canonical_payload.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise ValueError("invalid namespace snapshot JSON") from exc - if timings is not None: - timings["json_decode_seconds"] = time.perf_counter() - json_started - timings["decode_seconds"] = time.perf_counter() - started - if not isinstance(decoded, dict): - raise ValueError("namespace snapshot payload must be an object") - return decoded + return _decode_compressed_json( + payload_zlib, + checksum=checksum, + timings=timings, + compression_error="invalid namespace snapshot compression", + checksum_error="namespace snapshot checksum mismatch", + json_error="invalid namespace snapshot JSON", + object_error="namespace snapshot payload must be an object", + ) From 1223551ffeabf73fe569df5080e9f69d12792b31 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Tue, 1 Sep 2026 22:38:24 +0800 Subject: [PATCH 04/35] fix: harden map-nav snapshot cache --- ...est_retrieval_map_score_parity_contract.py | 88 +++++++++ ...retrieval_snapshot_consistency_contract.py | 28 +++ .../test_retrieval_snapshot_redis_contract.py | 181 ++++++++++++++++++ .../retrieval-serving-snapshot-cache.md | 48 +++++ .../shared/services/redis/redis_service.py | 34 +++- .../services/retrieval/execution/routes.py | 3 +- .../retrieval/namespace_map_snapshot_redis.py | 56 +++--- .../shared/services/retrieval/nav_snapshot.py | 14 +- .../services/retrieval/serving_manifest.py | 6 +- 9 files changed, 417 insertions(+), 41 deletions(-) create mode 100644 apps/api/tests/contract/test_retrieval_map_score_parity_contract.py create mode 100644 apps/api/tests/contract/test_retrieval_snapshot_redis_contract.py create mode 100644 docs/design/retrieval-serving-snapshot-cache.md diff --git a/apps/api/tests/contract/test_retrieval_map_score_parity_contract.py b/apps/api/tests/contract/test_retrieval_map_score_parity_contract.py new file mode 100644 index 00000000..c7d651c0 --- /dev/null +++ b/apps/api/tests/contract/test_retrieval_map_score_parity_contract.py @@ -0,0 +1,88 @@ +"""Contract tests for map score pooling semantics.""" + +from __future__ import annotations + +from typing import Final + +import pytest + +from shared.services.retrieval.nav.nav_map_scores import _pool_unit_scores_to_tree + + +_CASES: Final[tuple[tuple[dict[str, list[str]], set[str], dict[str, float]], ...]] = ( + ( + {"root-a": ["section-a", "section-b"], "section-a": [], "section-b": []}, + {"section-a", "section-b"}, + {"section-a": 0.4, "section-b": 0.8, "root-a__self": 0.2}, + ), + ( + { + "root-a": ["parent-a"], + "parent-a": ["leaf-a", "leaf-b"], + "leaf-a": [], + "leaf-b": [], + "root-b": ["leaf-c"], + "leaf-c": [], + }, + {"leaf-a", "leaf-b", "leaf-c"}, + { + "leaf-a": 0.9, + "leaf-b": 0.3, + "leaf-c": 0.7, + "parent-a__self": 0.95, + }, + ), +) + + +def _legacy_pool( + children_map: dict[str, list[str]], + leaves: set[str], + unit_scores: dict[str, float], +) -> dict[str, float]: + map_scores = { + leaf_id: float(unit_scores.get(leaf_id, 0.0) or 0.0) for leaf_id in leaves + } + + def score_node(section_id: str) -> float: + if section_id in map_scores: + return map_scores[section_id] + children = children_map.get(section_id) or [] + if not children: + score = float(unit_scores.get(section_id, 0.0) or 0.0) + map_scores[section_id] = score + return score + descendants: list[str] = [] + + def collect(section: str) -> None: + nested = children_map.get(section) or [] + if not nested: + if section in leaves: + descendants.append(section) + return + for child in nested: + collect(child) + + collect(section_id) + parts = [float(unit_scores.get(leaf_id, 0.0) or 0.0) for leaf_id in descendants] + self_key = f"{section_id}__self" + if self_key in unit_scores: + parts.append(float(unit_scores[self_key])) + score = float(max(parts)) if parts else 0.0 + map_scores[section_id] = score + return score + + for section_id in children_map: + score_node(section_id) + return map_scores + + +@pytest.mark.parametrize("children_map, leaves, unit_scores", _CASES) +def test_map_score_pooling_preserves_legacy_semantics( + children_map: dict[str, list[str]], + leaves: set[str], + unit_scores: dict[str, float], +) -> None: + assert _pool_unit_scores_to_tree(children_map, leaves, unit_scores) == _legacy_pool( + children_map, leaves, unit_scores + ) diff --git a/apps/api/tests/contract/test_retrieval_snapshot_consistency_contract.py b/apps/api/tests/contract/test_retrieval_snapshot_consistency_contract.py index bbe03e05..a30af3c3 100644 --- a/apps/api/tests/contract/test_retrieval_snapshot_consistency_contract.py +++ b/apps/api/tests/contract/test_retrieval_snapshot_consistency_contract.py @@ -8,11 +8,13 @@ from httpx import AsyncClient import pytest from sqlalchemy import Executable, Result +from sqlalchemy.exc import SQLAlchemyError from shared.services.retrieval.execution.reference_resolver import ( resolve_workflow_references, ) from shared.services.retrieval.nav_snapshot import SnapshotSession, load_nav_snapshot +from shared.services.retrieval.nav_snapshot import _resolve_namespace_snapshot_entries from tests.support.retrieval_snapshot_support import contract_db_session from tests.support.contract_database import ContractDatabase @@ -20,6 +22,32 @@ _USER_ID = "local-dev-user" +class _GenerationUnavailableSession: + def __init__(self) -> None: + self.rollback_count = 0 + + async def execute(self, _statement: Executable) -> Result[tuple[object, ...]]: + raise SQLAlchemyError("generation table unavailable") + + async def rollback(self) -> None: + self.rollback_count += 1 + + +@pytest.mark.asyncio +async def test_snapshot_loader_falls_back_when_generation_cannot_be_verified() -> None: + session = _GenerationUnavailableSession() + + result = await _resolve_namespace_snapshot_entries( + session, + user_id=_USER_ID, + namespace="default", + document_revisions=[("doc-a", "result-a")], + ) + + assert result is None + assert session.rollback_count == 1 + + class _PublishingSession: def __init__( self, diff --git a/apps/api/tests/contract/test_retrieval_snapshot_redis_contract.py b/apps/api/tests/contract/test_retrieval_snapshot_redis_contract.py new file mode 100644 index 00000000..29934133 --- /dev/null +++ b/apps/api/tests/contract/test_retrieval_snapshot_redis_contract.py @@ -0,0 +1,181 @@ +"""Contract tests for binary namespace snapshot caching.""" + +from __future__ import annotations + +import pytest +from sqlalchemy import Executable + +from shared.core.config.redis import RedisConfig, RedisConfigManager +from shared.services.redis.redis_service_factory import RedisServiceFactory +from shared.services.redis.redis_service import RedisService +from shared.services.retrieval.nav_snapshot import _resolve_namespace_snapshot_entries +from shared.services.retrieval.namespace_map_snapshot_redis import ( + NamespaceMapSnapshotRedisCache, +) +from shared.services.retrieval.serving_manifest import encode_namespace_map_snapshot + + +class _FakeRedisClient: + def __init__(self, *, decode_responses: bool) -> None: + self.decode_responses = decode_responses + self.values: dict[str, bytes] = {} + self.ttls: dict[str, int] = {} + + async def get(self, key: str) -> bytes | None: + return self.values.get(key) + + async def set(self, key: str, value: bytes, *, ex: int) -> bool: + self.values[key] = value + self.ttls[key] = ex + return True + + async def aclose(self) -> None: + return None + + +@pytest.mark.asyncio +async def test_binary_redis_operations_preserve_compressed_snapshot_bytes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clients: list[_FakeRedisClient] = [] + + def create_client(*args: object, **kwargs: object) -> _FakeRedisClient: + client = _FakeRedisClient( + decode_responses=bool(kwargs.get("decode_responses")) + ) + clients.append(client) + return client + + monkeypatch.setattr( + "shared.services.redis.redis_service.redis.from_url", create_client + ) + service = RedisService(RedisConfigManager(RedisConfig())) + payload = b"\x78\x9c\x00\xffcompressed-snapshot" + + assert await service.set_bytes("contract:snapshot", payload, ex=3600) + assert await service.get_bytes("contract:snapshot") == payload + assert len(clients) == 1 + assert clients[0].decode_responses is False + assert clients[0].ttls["knowhere-api:contract:snapshot"] == 3600 + + await service.close() + + +@pytest.mark.asyncio +async def test_snapshot_cache_scopes_reads_and_writes_by_generation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _FakeSnapshotService: + def __init__(self) -> None: + self.get_keys: list[str] = [] + self.set_calls: list[tuple[str, bytes, int]] = [] + + async def get_bytes(self, key: str) -> bytes | None: + self.get_keys.append(key) + return b"snapshot" + + async def set_bytes(self, key: str, value: bytes, *, ex: int) -> bool: + self.set_calls.append((key, value, ex)) + return True + + fake_service = _FakeSnapshotService() + monkeypatch.setattr( + RedisServiceFactory, + "get_service", + classmethod(lambda cls: fake_service), + ) + + assert ( + await NamespaceMapSnapshotRedisCache.get( + user_id="user", + namespace=" ", + generation=7, + ) + == b"snapshot" + ) + assert await NamespaceMapSnapshotRedisCache.set( + user_id="user", + namespace="default", + generation=8, + payload_zlib=b"compressed", + ) + + assert fake_service.get_keys == ["retrieval:snapshot:v2:user:default:g7"] + assert fake_service.set_calls == [ + ( + "retrieval:snapshot:v2:user:default:g8", + b"compressed", + 3600, + ) + ] + + +class _SnapshotResult: + def __init__(self, *, scalar: object = None, row: tuple[object, ...] | None = None): + self._scalar = scalar + self._row = row + + def scalar_one_or_none(self) -> object: + return self._scalar + + def first(self) -> tuple[object, ...] | None: + return self._row + + +class _SnapshotSequenceSession: + def __init__(self, results: list[_SnapshotResult]) -> None: + self._results = iter(results) + + async def execute(self, _statement: Executable) -> _SnapshotResult: + return next(self._results) + + async def rollback(self) -> None: + return None + + +@pytest.mark.asyncio +async def test_corrupt_redis_blob_falls_back_to_postgres_snapshot( + monkeypatch: pytest.MonkeyPatch, +) -> None: + payload_zlib, checksum, format_version = encode_namespace_map_snapshot( + { + "documents": { + "doc-a": { + "job_result_id": "result-a", + "job_id": "job-a", + "sections": [], + "chunks": [], + } + } + } + ) + set_calls: list[bytes] = [] + + async def get_corrupt_blob(**_: object) -> bytes: + return b"corrupt" + + async def record_repaired_blob(**kwargs: object) -> bool: + set_calls.append(bytes(kwargs["payload_zlib"])) + return True + + monkeypatch.setattr(NamespaceMapSnapshotRedisCache, "get", get_corrupt_blob) + monkeypatch.setattr(NamespaceMapSnapshotRedisCache, "set", record_repaired_blob) + session = _SnapshotSequenceSession( + [ + _SnapshotResult(scalar=3), + _SnapshotResult(row=(3, checksum, format_version)), + _SnapshotResult(row=(payload_zlib,)), + ] + ) + + entries = await _resolve_namespace_snapshot_entries( + session, + user_id="user", + namespace="default", + document_revisions=[("doc-a", "result-a")], + expected_generation=3, + ) + + assert entries is not None + assert entries[0][0:2] == ("doc-a", "result-a") + assert set_calls == [payload_zlib] diff --git a/docs/design/retrieval-serving-snapshot-cache.md b/docs/design/retrieval-serving-snapshot-cache.md new file mode 100644 index 00000000..36f23f9d --- /dev/null +++ b/docs/design/retrieval-serving-snapshot-cache.md @@ -0,0 +1,48 @@ +# Retrieval serving snapshot cache + +Map-nav reads a namespace routing snapshot that contains section/chunk +relationships, ordering, chunk types, connection references, and remount +ownership. It intentionally does not contain chunk body text; final hydration +still reads pinned revision content from PostgreSQL. + +## Persisted formats + +- Namespace snapshots written by current publication code use format version 2 + and contain routing metadata only. +- The reader remains compatible with version 1 snapshots so existing rows can + be served during rollout. No unconditional namespace rebuild is required + solely because the reader was upgraded. +- A missing, corrupt, stale, or generation-mismatched snapshot falls back to + the exact manifest/table loading path. + +## Redis cache + +The map-nav reader caches the compressed snapshot bytes, not a decoded Python +dictionary. Keys are scoped by user, normalized namespace, and serving +generation: + +```text +retrieval:snapshot:v2:{user_id}:{namespace}:g{generation} +``` + +The cache TTL is one hour. Redis errors, misses, and invalid blobs are +non-fatal: PostgreSQL remains the source of truth and the reader repopulates +Redis after a successful database read. Binary snapshot operations use a Redis +connection with response decoding disabled; normal JSON Redis operations keep +their existing text-decoding connection. + +## Generation coherence + +Retrieval carries one captured revision set and namespace generation through +snapshot loading, map-nav, reference resolution, and final hydration. A +generation lookup failure is treated as inability to establish coherence and +uses the fallback path. A generation mismatch is never served as a valid +snapshot. + +## Diagnostics + +Set `RETRIEVAL_SNAPSHOT_TIMING=1` for detailed snapshot decode timings during +local or staging diagnosis. The route CPU metric is process-level CPU time for +the map-nav route. `ru_maxrss` is a process high-water mark, not a request-level +memory peak, and must not be interpreted as one. + diff --git a/packages/shared-python/shared/services/redis/redis_service.py b/packages/shared-python/shared/services/redis/redis_service.py index 185467ec..c25a3e1d 100644 --- a/packages/shared-python/shared/services/redis/redis_service.py +++ b/packages/shared-python/shared/services/redis/redis_service.py @@ -38,6 +38,7 @@ def __init__(self, config_manager: Optional[RedisConfigManager] = None): config_manager = RedisConfigManager(settings) self.config_manager = config_manager self._client: Optional[redis.Redis] = None + self._binary_client: Optional[redis.Redis] = None self._health_checker: Optional[RedisHealthChecker] = None self._lock = asyncio.Lock() @@ -60,6 +61,26 @@ async def _get_client(self) -> redis.Redis: ) return self._client + async def _get_binary_client(self) -> redis.Redis: + """Get a Redis client that preserves arbitrary binary responses.""" + if self._binary_client is None: + async with self._lock: + if self._binary_client is None: + try: + connection_params = self.config_manager.get_connection_params() + connection_params["decode_responses"] = False + self._binary_client = redis.from_url( + self.config_manager.get_connection_url(), + **connection_params, + ) + logger.debug("Redis binary client initialized") + except Exception as e: + raise RedisConnectionError( + internal_message=f"Redis binary client initialization failed: {str(e)}", + original_exception=e, + ) + return self._binary_client + async def _execute_with_retry( self, operation: Callable[[], Awaitable[ResponseT]] ) -> ResponseT: @@ -163,7 +184,7 @@ async def _operation(): async def get_bytes(self, key: str) -> bytes | None: """Get a value without JSON decoding (for compressed binary blobs).""" try: - client = await self._get_client() + client = await self._get_binary_client() full_key = self._build_key(key) async def _operation() -> bytes | None: @@ -188,7 +209,7 @@ async def _operation() -> bytes | None: async def set_bytes(self, key: str, value: bytes, *, ex: int) -> bool: """Set a compressed binary value with an explicit TTL.""" try: - client = await self._get_client() + client = await self._get_binary_client() full_key = self._build_key(key) async def _operation() -> bool: @@ -651,18 +672,21 @@ async def is_healthy(self) -> bool: async def close(self): """Close the Redis connection.""" - if self._client: + clients = [client for client in (self._client, self._binary_client) if client] + for client in clients: close_client = cast( Callable[[], Awaitable[None]] | None, - getattr(self._client, "aclose", None), + getattr(client, "aclose", None), ) if close_client is not None: await close_client() else: - await self._client.close() + await client.close() + if clients: self._client = None + self._binary_client = None self._health_checker = None logger.info("Redis connection closed") diff --git a/packages/shared-python/shared/services/retrieval/execution/routes.py b/packages/shared-python/shared/services/retrieval/execution/routes.py index ca9809a4..dc16c0f1 100644 --- a/packages/shared-python/shared/services/retrieval/execution/routes.py +++ b/packages/shared-python/shared/services/retrieval/execution/routes.py @@ -344,7 +344,8 @@ async def _run_mapnav_route( completion_detail = f"chunks | evidence={len(evidence_text)} chars | router=mapnav" process_finished = resource.getrusage(resource.RUSAGE_SELF) logger.info( - "retrieval mapnav stage=process_resources cpu_seconds=%.3f max_rss_kb=%d", + "retrieval mapnav stage=process_resources cpu_seconds=%.3f " + "process_max_rss_kb=%d", (process_finished.ru_utime + process_finished.ru_stime) - (process_started.ru_utime + process_started.ru_stime), int(process_finished.ru_maxrss), diff --git a/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_redis.py b/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_redis.py index b977e5f4..162b85c2 100644 --- a/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_redis.py +++ b/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_redis.py @@ -9,30 +9,34 @@ _KEY_PREFIX = "retrieval:snapshot:v2" -def build_snapshot_cache_key(*, user_id: str, namespace: str, generation: int) -> str: - normalized_namespace = normalize_retrieval_namespace(namespace) - return f"{_KEY_PREFIX}:{user_id}:{normalized_namespace}:g{int(generation)}" - - -async def get_snapshot_blob( - *, user_id: str, namespace: str, generation: int -) -> bytes | None: - service = RedisServiceFactory.get_service() - return await service.get_bytes( - build_snapshot_cache_key( - user_id=user_id, namespace=namespace, generation=generation +class NamespaceMapSnapshotRedisCache: + """Access generation-scoped compressed namespace snapshots in Redis.""" + + @staticmethod + def _build_key(*, user_id: str, namespace: str, generation: int) -> str: + normalized_namespace = normalize_retrieval_namespace(namespace) + return f"{_KEY_PREFIX}:{user_id}:{normalized_namespace}:g{int(generation)}" + + @classmethod + async def get( + cls, *, user_id: str, namespace: str, generation: int + ) -> bytes | None: + service = RedisServiceFactory.get_service() + return await service.get_bytes( + cls._build_key( + user_id=user_id, namespace=namespace, generation=generation + ) + ) + + @classmethod + async def set( + cls, *, user_id: str, namespace: str, generation: int, payload_zlib: bytes + ) -> bool: + service = RedisServiceFactory.get_service() + return await service.set_bytes( + cls._build_key( + user_id=user_id, namespace=namespace, generation=generation + ), + payload_zlib, + ex=_CACHE_TTL_SECONDS, ) - ) - - -async def set_snapshot_blob( - *, user_id: str, namespace: str, generation: int, payload_zlib: bytes -) -> bool: - service = RedisServiceFactory.get_service() - return await service.set_bytes( - build_snapshot_cache_key( - user_id=user_id, namespace=namespace, generation=generation - ), - payload_zlib, - ex=_CACHE_TTL_SECONDS, - ) diff --git a/packages/shared-python/shared/services/retrieval/nav_snapshot.py b/packages/shared-python/shared/services/retrieval/nav_snapshot.py index 3210b55f..b15425e3 100644 --- a/packages/shared-python/shared/services/retrieval/nav_snapshot.py +++ b/packages/shared-python/shared/services/retrieval/nav_snapshot.py @@ -55,8 +55,7 @@ ) from shared.services.retrieval.manifest_cache import get_cached_manifest_payloads from shared.services.retrieval.namespace_map_snapshot_redis import ( - get_snapshot_blob, - set_snapshot_blob, + NamespaceMapSnapshotRedisCache, ) from shared.core.exceptions.redis_exceptions import RedisOperationError @@ -452,8 +451,11 @@ async def _resolve_namespace_snapshot_entries( _current_generation = generation_result.scalar_one_or_none() except SQLAlchemyError as exc: await db.rollback() - _logger.warning("retrieval snapshot generation lookup failed error=%s", exc) - _current_generation = None + _logger.warning( + "retrieval snapshot generation lookup failed; using fallback error=%s", + exc, + ) + return None if _is_snapshot_timing_enabled(): _logger.info( "retrieval snapshot generation generation=%s", @@ -512,7 +514,7 @@ async def _resolve_namespace_snapshot_entries( return None cached_blob: bytes | None = None try: - cached_blob = await get_snapshot_blob( + cached_blob = await NamespaceMapSnapshotRedisCache.get( user_id=user_id, namespace=namespace, generation=row_generation ) except RedisOperationError as exc: @@ -576,7 +578,7 @@ async def _resolve_namespace_snapshot_entries( documents = decoded_documents if cached_blob is None: try: - await set_snapshot_blob( + await NamespaceMapSnapshotRedisCache.set( user_id=user_id, namespace=namespace, generation=row_generation, diff --git a/packages/shared-python/shared/services/retrieval/serving_manifest.py b/packages/shared-python/shared/services/retrieval/serving_manifest.py index 8480b8f4..77a33e1c 100644 --- a/packages/shared-python/shared/services/retrieval/serving_manifest.py +++ b/packages/shared-python/shared/services/retrieval/serving_manifest.py @@ -249,7 +249,7 @@ def encode_namespace_map_snapshot( documents = payload.get("documents") if not isinstance(documents, dict): raise ValueError("namespace snapshot documents must be an object") - routing_documents: dict[str, Any] = {} + routing_documents: dict[str, dict[str, object]] = {} for document_id, raw_document in documents.items(): if not isinstance(raw_document, dict): raise ValueError(f"namespace snapshot document is not an object: {document_id}") @@ -257,7 +257,7 @@ def encode_namespace_map_snapshot( raw_chunks = raw_document.get("chunks") if not isinstance(raw_sections, list) or not isinstance(raw_chunks, list): raise ValueError(f"namespace snapshot records are invalid: {document_id}") - sections = [] + sections: list[dict[str, object]] = [] for section in raw_sections: if not isinstance(section, dict) or not str(section.get("section_id") or ""): raise ValueError(f"namespace snapshot section is invalid: {document_id}") @@ -271,7 +271,7 @@ def encode_namespace_map_snapshot( if key in section } ) - chunks = [] + chunks: list[dict[str, object]] = [] for chunk in raw_chunks: if not isinstance(chunk, dict) or not str(chunk.get("chunk_id") or ""): raise ValueError(f"namespace snapshot chunk is invalid: {document_id}") From 6ed286d33f4d4e40bfc683ac4ce4214bda4dde3c Mon Sep 17 00:00:00 2001 From: suguanYang Date: Tue, 1 Sep 2026 23:06:12 +0800 Subject: [PATCH 05/35] perf: narrow map index frequency lookup --- .../test_retrieval_map_unit_index_contract.py | 8 +++++-- .../services/retrieval/nav/nav_knowhere.py | 23 +++++++++++++------ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py b/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py index b4b047d8..8b0214d9 100644 --- a/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py +++ b/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py @@ -105,11 +105,15 @@ def close(self) -> None: ] assert len(frequency_executions) == 1 statement, parameters = frequency_executions[0] - assert "map_unit_id = ANY" in statement + assert "scoped_units AS MATERIALIZED" in statement + assert "JOIN scoped_units" in statement + assert "channel = ANY" in statement assert "token_hash = ANY" in statement + assert "map_unit_id = ANY" not in statement assert isinstance(parameters, list) assert parameters[0] == ["unit-frequency"] - assert parameters[1] == [ + assert parameters[1] == ["path", "content"] + assert parameters[2] == [ "6e51d6a3d90b6a3243d38e6da6b3f31f49867c1360beba83da8ca9630f9672c7" ] diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py b/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py index bd4e24f7..debfb3c8 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py @@ -435,18 +435,27 @@ def load_persisted_score_corpus( frequencies: Dict[Tuple[str, str], Dict[str, int]] = {} if unit_rows and query_tokens: stage_started = time.perf_counter() - # Restrict the token scan to this episode's map units instead of - # matching token_hash across the whole table then filtering. + # Keep the episode scope explicit without materializing every + # token row matching a common query term. PostgreSQL can choose + # either the scoped-unit side or the channel/token_hash-leading + # index, while the scope still limits results to the pinned + # revision and allowed sections. allowed_map_unit_ids = [str(row["map_unit_id"]) for row in unit_rows] cur.execute( - "SELECT map_unit_id, channel, token, frequency " - "FROM document_map_unit_tokens " - "WHERE map_unit_id = ANY(%s) " - "AND token_hash = ANY(%s) AND channel = ANY(%s)", + "WITH scoped_units AS MATERIALIZED (" + "SELECT unnest(%s::text[]) AS map_unit_id" + ") " + "SELECT tokens.map_unit_id, tokens.channel, tokens.token, " + "tokens.frequency " + "FROM document_map_unit_tokens AS tokens " + "JOIN scoped_units " + "ON scoped_units.map_unit_id = tokens.map_unit_id " + "WHERE tokens.channel = ANY(%s) " + "AND tokens.token_hash = ANY(%s)", [ allowed_map_unit_ids, - list(query_token_hashes), list(_MAP_SCORE_CHANNELS), + list(query_token_hashes), ], ) for map_unit_id, channel, token, frequency in cur.fetchall(): From a3ad93a5b3a5dd042d774a51639eef8c143e32bd Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 2 Sep 2026 00:11:16 +0800 Subject: [PATCH 06/35] perf: avoid legacy job chunk hydration --- .../test_retrieval_map_unit_index_contract.py | 97 ++++++++++++++++++- .../services/retrieval/hydration/connected.py | 10 +- 2 files changed, 103 insertions(+), 4 deletions(-) diff --git a/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py b/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py index 8b0214d9..d571ad43 100644 --- a/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py +++ b/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py @@ -6,7 +6,7 @@ from uuid import uuid4 from httpx import AsyncClient -from sqlalchemy import delete, select, text +from sqlalchemy import Engine, delete, event, select, text from shared.models.database.document import ( DocumentMapUnit, @@ -21,6 +21,7 @@ compute_corpus_map_and_unit_scores, select_map_highlights, ) +from shared.services.retrieval.hydration.connected import hydrate_connected_target_rows from shared.services.retrieval.nav.nav_knowhere import ( KnowhereProvider, LazyKnowhereProvider, @@ -497,6 +498,100 @@ def record_reference_load( snapshot.close() +async def test_connected_hydration_does_not_load_legacy_job_chunks( + developer_api_client_factory: Callable[ + [], AbstractAsyncContextManager[AsyncClient] + ], +) -> None: + identifier = uuid4().hex[:8] + namespace = f"connected-job-{identifier}" + document_id = f"doc_connected_{identifier}" + job_id = f"job_connected_{identifier}" + job_result_id = f"result_connected_{identifier}" + statements: list[str] = [] + + def capture_job_chunk_query( + _connection: Any, + _cursor: Any, + statement: str, + _parameters: Any, + _context: Any, + _executemany: bool, + ) -> None: + if "job_chunks" in statement.lower(): + statements.append(statement) + + async with developer_api_client_factory(): + await _seed_revision( + namespace=namespace, + document_id=document_id, + job_id=job_id, + job_result_id=job_result_id, + ) + scope = DocumentPublicationScope( + user_id=_USER_ID, + namespace=namespace, + document_id=document_id, + job_result_id=job_result_id, + source_file_name="connected.pdf", + ) + chunks = [ + { + "chunk_id": "body-connected", + "type": "text", + "content": "body connected evidence", + "path": "connected.pdf/Root/Section/body", + "order": 1, + "metadata": {"connect_to": [{"target": "asset-connected"}]}, + }, + { + "chunk_id": "asset-connected", + "type": "image", + "content": "asset connected summary", + "path": "images/asset-connected.png", + "order": 2, + "file_path": "images/asset-connected.png", + "metadata": {}, + }, + ] + async with contract_db_session() as db: + await db.run_sync( + lambda sync_db: _publish_revision_with_generation_lock( + sync_db, + scope=scope, + chunks=chunks, + ) + ) + await db.commit() + + event.listen(Engine, "before_cursor_execute", capture_job_chunk_query) + try: + async with contract_db_session() as db: + hydrated = await hydrate_connected_target_rows( + db=db, + rows=[ + { + "document_id": document_id, + "job_result_id": job_result_id, + "chunk_id": "body-connected", + "chunk_type": "text", + "chunk_metadata": { + "connect_to": [{"target": "asset-connected"}] + }, + } + ], + exclude_document_ids=[], + exclude_sections=[], + revision_pins={document_id: job_result_id}, + ) + finally: + event.remove(Engine, "before_cursor_execute", capture_job_chunk_query) + + assert [row["chunk_id"] for row in hydrated] == ["asset-connected"] + assert hydrated[0]["job_id"] == job_id + assert statements == [] + + def test_incomplete_index_returns_empty_scores() -> None: first_sections = [ SectionRow("root-a", None, "Root A", "Root A", 0, "", 0), diff --git a/packages/shared-python/shared/services/retrieval/hydration/connected.py b/packages/shared-python/shared/services/retrieval/hydration/connected.py index 2a1f3127..8e637957 100644 --- a/packages/shared-python/shared/services/retrieval/hydration/connected.py +++ b/packages/shared-python/shared/services/retrieval/hydration/connected.py @@ -62,7 +62,11 @@ async def hydrate_connected_target_rows( return [] stmt = ( - select(Document, DocumentChunk, DocumentSection, JobResult) + # Select only the job identifier needed for the public projection. + # Selecting the JobResult entity triggers its ``chunks`` selectin + # relationship, loading the entire legacy job-chunk collection for + # every connected revision during final hydration. + select(Document, DocumentChunk, DocumentSection, JobResult.job_id) .join( DocumentChunk, ( @@ -90,7 +94,7 @@ async def hydrate_connected_target_rows( result = await db.execute(stmt) hydrated_rows: list[dict[str, Any]] = [] - for document, chunk, section, job_result in result.all(): + for document, chunk, section, job_id in result.all(): section_path = section.section_path if section else None hydrated_rows.append( { @@ -105,7 +109,7 @@ async def hydrate_connected_target_rows( 'file_path': chunk.file_path, 'chunk_metadata': chunk.chunk_metadata or {}, 'job_result_id': chunk.job_result_id, - 'job_id': job_result.job_id if job_result else None, + 'job_id': job_id, 'sort_order': chunk.sort_order, } ) From 79f169b03634e269f8f22abe9f2f216f7bc4087e Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Wed, 2 Sep 2026 10:09:15 +0800 Subject: [PATCH 07/35] feat(retrieval): switch map-unit BM25 to word-level tokens Replace character-level CJK tokenization with the shared jieba path and bump map-unit index format to v2 so query hashes match stored tokens. Add --tokens-only/--skip-current-format for lightweight reindex after tokenizer changes without rebuilding manifests or namespace snapshots. Co-authored-by: Cursor --- apps/api/scripts/backfill_map_unit_indexes.py | 132 +++++++++++++++--- .../test_retrieval_map_unit_index_contract.py | 2 +- .../services/retrieval/map_unit_index.py | 4 +- .../services/retrieval/nav/knowhere_hybrid.py | 30 ++-- .../services/retrieval/nav/nav_knowhere.py | 4 +- .../retrieval/search/map_unit_discovery.py | 9 +- .../tests/test_knowhere_hybrid_tokenize.py | 47 +++++++ 7 files changed, 187 insertions(+), 41 deletions(-) create mode 100644 packages/shared-python/shared/tests/test_knowhere_hybrid_tokenize.py diff --git a/apps/api/scripts/backfill_map_unit_indexes.py b/apps/api/scripts/backfill_map_unit_indexes.py index ebf1a04d..43a9b5c4 100644 --- a/apps/api/scripts/backfill_map_unit_indexes.py +++ b/apps/api/scripts/backfill_map_unit_indexes.py @@ -7,9 +7,20 @@ deployment with ``--apply`` so each revision is rebuilt and committed independently; without ``--apply`` it is a read-only inventory. +Re-run after map-unit tokenizer / ``MAP_UNIT_INDEX_FORMAT_VERSION`` changes +(e.g. v1 character-level → v2 word-level, or loading a domain userdict) so +query hashes match stored tokens. Prefer:: + + python /app/scripts/backfill_map_unit_indexes.py --apply --tokens-only + python /app/scripts/backfill_map_unit_indexes.py --apply --tokens-only --skip-current-format + +``--tokens-only`` rebuilds map-unit tokens/index only (skips serving manifest +and namespace MAP snapshot). Use full ``--apply`` only when those artifacts +are missing. + Use ``--check`` after backfill to verify whether query-time snapshot fallbacks (manifest_merge / table_scan) would still fire, and whether -map-unit indexes are complete for scoring. +map-unit indexes are complete for scoring at the current format version. """ # ruff: noqa: E402 @@ -57,7 +68,10 @@ def _bootstrap_python_path() -> None: RetrievalNamespaceMapSnapshot, RetrievalServingRevisionManifest, ) -from shared.services.retrieval.map_unit_index import replace_document_map_units +from shared.services.retrieval.map_unit_index import ( + MAP_UNIT_INDEX_FORMAT_VERSION, + replace_document_map_units, +) from shared.services.retrieval.namespace_map_snapshot import ( patch_namespace_map_snapshot, ) @@ -92,6 +106,23 @@ def _build_parser() -> argparse.ArgumentParser: parser.add_argument( "--document-id", default="", help="Limit the backfill to one document." ) + parser.add_argument( + "--tokens-only", + action="store_true", + help=( + "With --apply: rebuild only document_map_units / tokens / index " + "(skip serving manifest, namespace MAP snapshot, and generation bump). " + "Use after tokenizer / format_version changes when manifests are already ready." + ), + ) + parser.add_argument( + "--skip-current-format", + action="store_true", + help=( + "With --apply: skip revisions whose map-unit index already has " + f"format_version={MAP_UNIT_INDEX_FORMAT_VERSION}." + ), + ) return parser @@ -196,6 +227,7 @@ def check_fallback_readiness(*, document_id: str = "") -> list[NamespaceFallback DocumentMapUnitIndex.unit_count, DocumentMapUnitIndex.average_idf_path, DocumentMapUnitIndex.average_idf_content, + DocumentMapUnitIndex.format_version, ).where( DocumentMapUnitIndex.document_id.in_( [document_id_value for document_id_value, _ in revisions] @@ -208,8 +240,16 @@ def check_fallback_readiness(*, document_id: str = "") -> list[NamespaceFallback int(unit_count or 0), float(average_idf_path or 0.0), float(average_idf_content or 0.0), + int(format_version or 0), ) - for document_id_value, job_result_id, unit_count, average_idf_path, average_idf_content in index_rows + for ( + document_id_value, + job_result_id, + unit_count, + average_idf_path, + average_idf_content, + format_version, + ) in index_rows } missing_map_index = 0 suspicious_zero_idf = 0 @@ -218,7 +258,10 @@ def check_fallback_readiness(*, document_id: str = "") -> list[NamespaceFallback if stats is None: missing_map_index += 1 continue - unit_count, average_idf_path, average_idf_content = stats + unit_count, average_idf_path, average_idf_content, format_version = stats + if format_version != MAP_UNIT_INDEX_FORMAT_VERSION: + missing_map_index += 1 + continue if ( unit_count > 0 and average_idf_path == 0.0 @@ -295,26 +338,36 @@ def print_fallback_check(reports: list[NamespaceFallbackReport]) -> int: return 1 if failed else 0 -def backfill_map_unit_indexes(*, apply: bool, document_id: str = "") -> int: +def backfill_map_unit_indexes( + *, + apply: bool, + document_id: str = "", + tokens_only: bool = False, + skip_current_format: bool = False, +) -> int: documents = _load_documents(document_id) if not apply: for document in documents: print( f"would backfill document={document.document_id} revision={document.current_job_result_id}" + + (" tokens_only" if tokens_only else "") ) return len(documents) session_factory = get_sync_session_factory() + backfilled = 0 + skipped_current = 0 for document in documents: job_result_id = document.current_job_result_id if not job_result_id: continue with session_factory() as db: - lock_namespace_generation( - db, - user_id=document.user_id, - namespace=document.namespace, - ) + if not tokens_only: + lock_namespace_generation( + db, + user_id=document.user_id, + namespace=document.namespace, + ) locked_document = db.execute( select(Document) .where(Document.document_id == document.document_id) @@ -333,6 +386,26 @@ def backfill_map_unit_indexes(*, apply: bool, document_id: str = "") -> int: f"revision={job_result_id}" ) continue + if skip_current_format: + existing = db.execute( + select(DocumentMapUnitIndex.format_version) + .where( + DocumentMapUnitIndex.document_id + == locked_document.document_id + ) + .where(DocumentMapUnitIndex.job_result_id == job_result_id) + ).scalar_one_or_none() + if ( + existing is not None + and int(existing) == MAP_UNIT_INDEX_FORMAT_VERSION + ): + db.rollback() + skipped_current += 1 + print( + f"skipped current-format document={document.document_id} " + f"revision={job_result_id} format_version={existing}" + ) + continue scope = DocumentPublicationScope( user_id=locked_document.user_id, namespace=locked_document.namespace, @@ -341,18 +414,25 @@ def backfill_map_unit_indexes(*, apply: bool, document_id: str = "") -> int: source_file_name=str(locked_document.source_file_name or ""), ) replace_document_map_units(db, scope=scope) - manifest_payload = persist_revision_serving_state(db, scope=scope) - patch_namespace_map_snapshot( - db, scope=scope, manifest_payload=manifest_payload - ) - advance_namespace_generation( - db, - user_id=scope.user_id, - namespace=scope.namespace, - ) + if not tokens_only: + manifest_payload = persist_revision_serving_state(db, scope=scope) + patch_namespace_map_snapshot( + db, scope=scope, manifest_payload=manifest_payload + ) + advance_namespace_generation( + db, + user_id=scope.user_id, + namespace=scope.namespace, + ) db.commit() - print(f"backfilled document={document.document_id} revision={job_result_id}") - return len(documents) + backfilled += 1 + mode = "tokens_only" if tokens_only else "full" + print( + f"backfilled document={document.document_id} revision={job_result_id} mode={mode}" + ) + if skip_current_format: + print(f"skipped_current_format={skipped_current}") + return backfilled def main() -> None: @@ -360,11 +440,19 @@ def main() -> None: if arguments.check: if arguments.apply: raise SystemExit("use either --check or --apply, not both") + if arguments.tokens_only or arguments.skip_current_format: + raise SystemExit("--tokens-only/--skip-current-format require --apply") reports = check_fallback_readiness(document_id=str(arguments.document_id)) raise SystemExit(print_fallback_check(reports)) + if (arguments.tokens_only or arguments.skip_current_format) and not arguments.apply: + raise SystemExit("--tokens-only/--skip-current-format require --apply") + count = backfill_map_unit_indexes( - apply=bool(arguments.apply), document_id=str(arguments.document_id) + apply=bool(arguments.apply), + document_id=str(arguments.document_id), + tokens_only=bool(arguments.tokens_only), + skip_current_format=bool(arguments.skip_current_format), ) action = "backfilled" if arguments.apply else "found" print(f"{action} revisions={count}") diff --git a/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py b/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py index b4b047d8..e71dd6b0 100644 --- a/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py +++ b/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py @@ -56,7 +56,7 @@ def __init__(self) -> None: def execute(self, statement: str, parameters: object = None) -> None: executions.append((statement, parameters)) if "document_map_unit_indexes" in statement: - self.rows = [(document_id, job_result_id, 1, 1, 0.0, 0.0)] + self.rows = [(document_id, job_result_id, 2, 1, 0.0, 0.0)] elif "FROM document_map_units AS units" in statement: self.rows = [("unit-frequency", document_id, "chunk-frequency", "section-frequency", 1, 1)] elif "FROM document_map_unit_tokens" in statement: diff --git a/packages/shared-python/shared/services/retrieval/map_unit_index.py b/packages/shared-python/shared/services/retrieval/map_unit_index.py index 4fff8206..ee18434d 100644 --- a/packages/shared-python/shared/services/retrieval/map_unit_index.py +++ b/packages/shared-python/shared/services/retrieval/map_unit_index.py @@ -23,11 +23,11 @@ UnitRow, ) from shared.services.retrieval.nav.nav_map_scores import build_score_units +from shared.services.retrieval.nav.knowhere_hybrid import MAP_UNIT_INDEX_FORMAT_VERSION from shared.services.retrieval.nav.persisted_score_load import average_idf_from_unit_dfs from shared.services.retrieval.publication_models import DocumentPublicationScope - -MAP_UNIT_INDEX_FORMAT_VERSION = 1 +__all__ = ["MAP_UNIT_INDEX_FORMAT_VERSION", "replace_document_map_units"] def replace_document_map_units( diff --git a/packages/shared-python/shared/services/retrieval/nav/knowhere_hybrid.py b/packages/shared-python/shared/services/retrieval/nav/knowhere_hybrid.py index 2c4bbb48..6e5a95f8 100644 --- a/packages/shared-python/shared/services/retrieval/nav/knowhere_hybrid.py +++ b/packages/shared-python/shared/services/retrieval/nav/knowhere_hybrid.py @@ -8,28 +8,34 @@ from __future__ import annotations -import os -import re import math +import os from dataclasses import dataclass from typing import Dict, List, Mapping, Optional, Sequence, Tuple +from shared.utils.text_utils import tokenize_for_retrieval as _tokenize_word_level + RRF_K = 60 CHANNEL_WEIGHT_PATH = 1.0 CHANNEL_WEIGHT_CONTENT = 2.0 +# Persisted map-unit token format. Bump when tokenizer semantics change. +# v1: character-level CJK regex. v2: word-level jieba (text_utils). +MAP_UNIT_INDEX_FORMAT_VERSION = 2 def tokenize_for_retrieval(text: str, *, dedupe: bool = True) -> List[str]: - tokens = re.findall(r"[a-z0-9_]+|[\u4e00-\u9fff]", str(text or "").lower()) - if not dedupe: - return [t for t in tokens if t] - seen: set[str] = set() - out: List[str] = [] - for t in tokens: - if t and t not in seen: - seen.add(t) - out.append(t) - return out + """Word-level retrieval tokens (jieba + English), aligned with chunk publication. + + Uses the same knobs as ``search.lexical_text``: no stopword filtering and + ``min_token_length=2`` so map-unit indexes match ``document_chunks`` search + text. Character-level regex tokenization was removed. + """ + return _tokenize_word_level( + text, + stopwords=[], + dedupe=dedupe, + min_token_length=2, + ) def tokenize_query_for_ranker(query: str) -> List[str]: diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py b/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py index bd4e24f7..3dadd57e 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py @@ -42,6 +42,7 @@ from .nav_address import NavLevel from .nav_hierarchy import NodeMeta from .knowhere_hybrid import ( + MAP_UNIT_INDEX_FORMAT_VERSION, PersistedScoreCorpus, PersistedScoreUnit, tokenize_query_for_ranker, @@ -51,7 +52,6 @@ # Knowhere sentinel path for the virtual document container (not a collectable leaf). ROOT_SECTION_PATH = "Root" _DEFAULT_DSN = "postgresql://root:root123@127.0.0.1:5433/Knowhere" -_MAP_UNIT_INDEX_FORMAT_VERSION = 1 _MAP_SCORE_CHANNELS: Tuple[str, str] = ("path", "content") _logger = logging.getLogger(__name__) @@ -353,7 +353,7 @@ def load_persisted_score_corpus( len(index_rows), ) if len(index_rows) != len(revisions) or any( - len(row) < 6 or int(row[2]) != _MAP_UNIT_INDEX_FORMAT_VERSION + len(row) < 6 or int(row[2]) != MAP_UNIT_INDEX_FORMAT_VERSION for row in index_rows ): return None diff --git a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py index 95436458..4e7d1a4a 100644 --- a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py +++ b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py @@ -32,6 +32,7 @@ normalize_chunk_type, ) from shared.services.retrieval.nav.knowhere_hybrid import ( + MAP_UNIT_INDEX_FORMAT_VERSION, PersistedScoreCorpus, PersistedScoreUnit, score_persisted_corpus_many, @@ -282,7 +283,7 @@ async def map_unit_discovery( ) + """ SELECT indexes.average_idf_path, indexes.average_idf_content, - indexes.unit_count + indexes.unit_count, indexes.format_version FROM document_map_unit_indexes AS indexes JOIN ( SELECT DISTINCT document_id, job_result_id FROM scoped_units @@ -293,9 +294,13 @@ async def map_unit_discovery( ), params, ) + # Only count indexes written with the current tokenizer (v2 = word-level). + # Stale char-level (v1) rows look complete by unit_count but cannot match + # word-level query hashes — treat them as missing so readiness fails closed. index_parts = [ (float(path_idf or 0.0), float(content_idf or 0.0), int(unit_count or 0)) - for path_idf, content_idf, unit_count in index_result.all() + for path_idf, content_idf, unit_count, format_version in index_result.all() + if int(format_version or 0) == MAP_UNIT_INDEX_FORMAT_VERSION ] expected_revisions = { (str(row["document_id"]), str(row["job_result_id"])) for row in unit_rows diff --git a/packages/shared-python/shared/tests/test_knowhere_hybrid_tokenize.py b/packages/shared-python/shared/tests/test_knowhere_hybrid_tokenize.py new file mode 100644 index 00000000..2d4b96e1 --- /dev/null +++ b/packages/shared-python/shared/tests/test_knowhere_hybrid_tokenize.py @@ -0,0 +1,47 @@ +"""Word-level map-unit tokenization (replaces character-level CJK regex).""" + +from __future__ import annotations + +from shared.services.retrieval.nav.knowhere_hybrid import ( + MAP_UNIT_INDEX_FORMAT_VERSION, + build_content_search_text, + build_path_search_text, + tokenize_for_retrieval, + tokenize_query_for_ranker, +) +from shared.utils.text_utils import tokenize_for_retrieval as tokenize_publication + + +def test_map_unit_index_format_is_word_level_v2() -> None: + assert MAP_UNIT_INDEX_FORMAT_VERSION == 2 + + +def test_query_uses_chinese_word_tokens_not_characters() -> None: + tokens = tokenize_query_for_ranker("冠心病的诊断标准") + assert "冠心病" in tokens + assert "诊断" in tokens + assert "标准" in tokens + assert "冠" not in tokens + assert "诊" not in tokens + + +def test_index_text_matches_publication_tokenizer() -> None: + text = "冠心病患者的诊断标准与心肌炎鉴别" + map_tokens = tokenize_for_retrieval(text, dedupe=False) + pub_tokens = tokenize_publication( + text, stopwords=[], dedupe=False, min_token_length=2 + ) + assert map_tokens == pub_tokens + assert "冠心病" in map_tokens + assert "心肌炎" in map_tokens + + +def test_build_search_text_joins_word_tokens() -> None: + content = build_content_search_text("冠心病诊断标准") + path = build_path_search_text(section_path="指南 / 冠心病 / 诊断") + assert "冠心病" in content.split() + assert "诊断" in content.split() + assert "冠心病" in path.split() + # Single CJK characters must not appear as standalone tokens. + assert "冠" not in content.split() + assert "冠" not in path.split() From 0e374bccb82a4f47bcd1fd7c4cc2591fee7b6a7b Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Wed, 2 Sep 2026 12:18:53 +0800 Subject: [PATCH 08/35] refactor(retrieval): remove legacy evidence rendering and implement new evidence text rendering Deleted the legacy evidence rendering functionality and tests. Introduced a new method for rendering evidence text that groups documents and sections more effectively. Updated related components to utilize the new rendering approach. --- .../test_evidence_renderer_contract.py | 45 +++++ .../test_legacy_evidence_renderer_contract.py | 43 ----- .../test_retrieval_lazy_tree_contract.py | 4 +- .../test_page_memory_retrieval_contract.py | 2 + .../services/retrieval/execution/routes.py | 30 +++- .../retrieval/hydration/asset_inline.py | 85 ++++++++++ .../retrieval/hydration/evidence_text.py | 45 +++++ .../retrieval/hydration/legacy_evidence.py | 55 ------- .../retrieval/hydration/result_assembly.py | 47 ++++-- .../services/retrieval/hydration/row_utils.py | 10 -- .../services/retrieval/nav/nav_compose.py | 46 ++---- .../services/retrieval/nav/nav_hierarchy.py | 69 +++++++- .../shared/testing/contract_runtime.py | 7 + .../shared/tests/test_asset_inline.py | 154 ++++++++++++++++++ 14 files changed, 475 insertions(+), 167 deletions(-) create mode 100644 apps/api/tests/contract/test_evidence_renderer_contract.py delete mode 100644 apps/api/tests/contract/test_legacy_evidence_renderer_contract.py create mode 100644 packages/shared-python/shared/services/retrieval/hydration/asset_inline.py create mode 100644 packages/shared-python/shared/services/retrieval/hydration/evidence_text.py delete mode 100644 packages/shared-python/shared/services/retrieval/hydration/legacy_evidence.py create mode 100644 packages/shared-python/shared/tests/test_asset_inline.py diff --git a/apps/api/tests/contract/test_evidence_renderer_contract.py b/apps/api/tests/contract/test_evidence_renderer_contract.py new file mode 100644 index 00000000..8188408b --- /dev/null +++ b/apps/api/tests/contract/test_evidence_renderer_contract.py @@ -0,0 +1,45 @@ +from shared.services.retrieval.execution.routes import _render_rows_evidence + + +def test_render_rows_evidence_should_group_by_traceable_path() -> None: + rows = [ + { + "chunk_id": "c2", + "content": "second section content", + "sort_order": 2, + "source": { + "source_file_name": "alpha.pdf", + "section_path": "Alpha / Two", + }, + }, + { + "chunk_id": "c1", + "content": "first section content\nwith more detail", + "sort_order": 1, + "source": { + "source_file_name": "alpha.pdf", + "section_path": "Alpha / One", + }, + }, + { + "chunk_id": "c3", + "content": "
metric
", + "source_file_name": "beta.pdf", + "section_path": "Beta / Table", + }, + ] + + evidence_text = _render_rows_evidence(rows) + + assert "[E1]" in evidence_text + assert "[E2]" in evidence_text + assert "[E3]" in evidence_text + assert "[§ alpha.pdf / Alpha / One]" in evidence_text + assert "[§ alpha.pdf / Alpha / Two]" in evidence_text + assert "[§ beta.pdf / Beta / Table]" in evidence_text + assert "first section content" in evidence_text + assert "second section content" in evidence_text + assert "
metric
" in evidence_text + assert "[Document]" not in evidence_text + assert "▸" not in evidence_text + assert "┈" not in evidence_text diff --git a/apps/api/tests/contract/test_legacy_evidence_renderer_contract.py b/apps/api/tests/contract/test_legacy_evidence_renderer_contract.py deleted file mode 100644 index dcf4c086..00000000 --- a/apps/api/tests/contract/test_legacy_evidence_renderer_contract.py +++ /dev/null @@ -1,43 +0,0 @@ -from shared.services.retrieval.hydration.legacy_evidence import render_legacy_evidence_text - - -def test_render_legacy_evidence_text_should_group_documents_and_sections() -> None: - rows = [ - { - "chunk_id": "c2", - "content": "second section content", - "sort_order": 2, - "source": { - "source_file_name": "alpha.pdf", - "section_path": "Alpha / Two", - }, - }, - { - "chunk_id": "c1", - "content": "first section content\nwith more detail", - "sort_order": 1, - "source": { - "source_file_name": "alpha.pdf", - "section_path": "Alpha / One", - }, - }, - { - "chunk_id": "c3", - "content": "
metric
", - "source_file_name": "beta.pdf", - "section_path": "Beta / Table", - }, - ] - - evidence_text = render_legacy_evidence_text(rows) - - assert "[Document] alpha.pdf" in evidence_text - assert "[Document] beta.pdf" in evidence_text - assert "▸ Alpha / One" in evidence_text - assert "▸ Alpha / Two" in evidence_text - assert " ┈ first section content" in evidence_text - assert " ┈ with more detail" in evidence_text - assert " ┈
metric
" in evidence_text - assert "\u3010\u6587\u6863\u3011" not in evidence_text - assert "[\u8868\u683c\u5185\u5bb9]" not in evidence_text - assert "[\u56fe\u7247" not in evidence_text diff --git a/apps/api/tests/contract/test_retrieval_lazy_tree_contract.py b/apps/api/tests/contract/test_retrieval_lazy_tree_contract.py index a920bb39..9571f151 100644 --- a/apps/api/tests/contract/test_retrieval_lazy_tree_contract.py +++ b/apps/api/tests/contract/test_retrieval_lazy_tree_contract.py @@ -70,7 +70,7 @@ def node_meta(self, section_id: str) -> NodeMeta: budget_chars=100, ) - assert result.evidence_text == "[E1]\n[§ Child]\nevidence" + assert result.evidence_text == "[E1]\n[§ Root / Child]\nevidence" assert provider.metadata_calls == 0 @@ -102,4 +102,4 @@ def test_evidence_pack_identifies_header_owners_from_parent_chain() -> None: ) assert result.kept_chunks == [chunks[1]] - assert result.evidence_text == "[E1]\n[§ Child]\nchild evidence" + assert result.evidence_text == "[E1]\n[§ Root / Parent / Child]\nchild evidence" diff --git a/apps/worker/tests/contract/test_page_memory_retrieval_contract.py b/apps/worker/tests/contract/test_page_memory_retrieval_contract.py index 922aff4c..1319db3f 100644 --- a/apps/worker/tests/contract/test_page_memory_retrieval_contract.py +++ b/apps/worker/tests/contract/test_page_memory_retrieval_contract.py @@ -144,6 +144,8 @@ async def test_table_result_assembly_uses_summary_not_html() -> None: assert "企业名称;统一社会信用代码" in content assert "SHOULD NOT LEAK" not in content assert " AbstractAsyncContextManager[AsyncSession]: return get_db_context() +def _evidence_path_header(row: dict) -> str: + source = row.get("source") + if not isinstance(source, dict): + source = row + file_name = str(source.get("source_file_name") or "").strip() + section_path = str(source.get("section_path") or "").strip() + if file_name and section_path: + return f"{file_name} / {section_path}" + return file_name or section_path + + +def _render_rows_evidence(rows: list[dict]) -> str: + groups: dict[str, list[str]] = {} + for row in rows: + header = _evidence_path_header(row) + content = str(row.get("content") or "").strip() + if not content: + continue + groups.setdefault(header, []).append(content) + return render_evidence_blocks(list(groups.items())) + + async def run_retrieval_route( context: RetrievalRouteContext, ) -> RetrievalRouteOutcome: @@ -102,7 +122,7 @@ async def _try_run_small_corpus_route( "namespace": context.namespace, "query": context.query, "router_used": "small_corpus_all", - "evidence_text": render_legacy_evidence_text(results), + "evidence_text": _render_rows_evidence(results), "answer_text": "", "results": results, } @@ -157,7 +177,7 @@ async def _run_classic_topk_route( "namespace": context.namespace, "query": context.query, "router_used": "classic_topk", - "evidence_text": render_legacy_evidence_text(results), + "evidence_text": _render_rows_evidence(results), "answer_text": "", "results": results, } diff --git a/packages/shared-python/shared/services/retrieval/hydration/asset_inline.py b/packages/shared-python/shared/services/retrieval/hydration/asset_inline.py new file mode 100644 index 00000000..4c49f00d --- /dev/null +++ b/packages/shared-python/shared/services/retrieval/hydration/asset_inline.py @@ -0,0 +1,85 @@ +"""Insert connected image/table bodies at text placeholders. + +Replaces ``[images/...]`` / ``[tables/...]`` (or ``connect_to.ref``) with the +asset display body, with a newline before and after. Targets not found at a +placeholder are appended once. Leftover path placeholders are stripped. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping, Sequence +from typing import Any + +_PATH_REF_RE = re.compile(r"\[(?:images|tables)/[^\]\n]+\]") +_SAME_AS_RE = re.compile(r"\[SAME-AS [^\]]+\]") + + +def strip_path_placeholders(content: str) -> str: + text = _PATH_REF_RE.sub("", content) + text = _SAME_AS_RE.sub("", text) + return text.strip() + + +def inline_assets_at_placeholders( + host_text: str, + *, + connections: Sequence[Mapping[str, Any]] | Sequence[Any], + display_by_target: Mapping[str, str], +) -> tuple[str, set[str]]: + """Return (body, embedded_target_ids). + + ``display_by_target`` maps chunk_id → display body. Only targets present in + that map are inserted. Each target is inserted at most once. + """ + text = str(host_text or "") + embedded: set[str] = set() + pending_append: list[tuple[str, str]] = [] + + for item in connections or (): + if not isinstance(item, Mapping): + continue + target_id = str(item.get("target") or "").strip() + if not target_id or target_id in embedded: + continue + body = str(display_by_target.get(target_id) or "").strip() + if not body: + continue + + ref = str(item.get("ref") or "").strip() + placed = False + for candidate in _ref_candidates(ref): + if candidate and candidate in text: + text = text.replace(candidate, f"\n{body}\n", 1) + embedded.add(target_id) + placed = True + break + if not placed: + pending_append.append((target_id, body)) + + for target_id, body in pending_append: + if target_id in embedded: + continue + if text.strip(): + text = f"{text.rstrip()}\n\n{body}" + else: + text = body + embedded.add(target_id) + + return strip_path_placeholders(text), embedded + + +def _ref_candidates(ref: str) -> list[str]: + raw = str(ref or "").strip() + if not raw: + return [] + out: list[str] = [raw] + if raw.startswith("[") and raw.endswith("]"): + inner = raw[1:-1].strip() + if inner and inner not in out: + out.append(inner) + else: + bracketed = f"[{raw}]" + if bracketed not in out: + out.append(bracketed) + return out diff --git a/packages/shared-python/shared/services/retrieval/hydration/evidence_text.py b/packages/shared-python/shared/services/retrieval/hydration/evidence_text.py new file mode 100644 index 00000000..a868302e --- /dev/null +++ b/packages/shared-python/shared/services/retrieval/hydration/evidence_text.py @@ -0,0 +1,45 @@ +"""Shared evidence_text rendering: one ``[E#]`` block per group. + +Each block is ``[E#]`` + ``[§ path]`` (full traceable path) + body lines. +Groups are caller-provided; bodies in one group stay in that group. +""" + +from __future__ import annotations + +from typing import Sequence + + +def render_evidence_blocks( + groups: Sequence[tuple[str, Sequence[str]]], + *, + start_index: int = 1, +) -> str: + """Render (path, bodies) groups into evidence_text. + + ``path`` is the full traceable header (e.g. file / section chain). + Bodies are joined with newlines; multiple bodies in one group are indented. + ``start_index`` sets the first ``[E#]`` number (default 1). + """ + parts: list[str] = [] + index = max(1, int(start_index or 1)) + for path, bodies in groups: + texts = [str(t or "").strip() for t in bodies] + texts = [t for t in texts if t] + if not texts: + continue + block: list[str] = [f"[E{index + len(parts)}]"] + header = str(path or "").strip() + if header: + block.append(f"[§ {header}]") + indent = len(texts) >= 2 + for text in texts: + if indent: + block.append( + "\n".join( + (" " + ln if ln.strip() else ln) for ln in text.splitlines() + ) + ) + else: + block.append(text) + parts.append("\n".join(block).strip()) + return "\n\n".join(parts) diff --git a/packages/shared-python/shared/services/retrieval/hydration/legacy_evidence.py b/packages/shared-python/shared/services/retrieval/hydration/legacy_evidence.py deleted file mode 100644 index 63b4ea35..00000000 --- a/packages/shared-python/shared/services/retrieval/hydration/legacy_evidence.py +++ /dev/null @@ -1,55 +0,0 @@ -from __future__ import annotations - -from collections import defaultdict -from typing import Any - - -def render_legacy_evidence_text(rows: list[dict[str, Any]]) -> str: - """Render assembled retrieval rows into evidence-only context.""" - grouped_rows: dict[str, list[dict[str, Any]]] = defaultdict(list) - for row in rows: - doc_name = _source_value(row, "source_file_name") or "Unknown document" - grouped_rows[doc_name].append(row) - - parts: list[str] = [] - for doc_name in sorted(grouped_rows): - parts.append(f"[Document] {doc_name}") - last_section = object() - for row in sorted(grouped_rows[doc_name], key=_row_sort_key): - section_path = _source_value(row, "section_path") or doc_name - if section_path != last_section: - parts.append(f"▸ {section_path}") - last_section = section_path - _append_content_lines(parts, row.get("content")) - - return "\n".join(parts) - - -def _source_value(row: dict[str, Any], key: str) -> str: - source = row.get("source") - if isinstance(source, dict): - value = source.get(key) - if value: - return str(value) - value = row.get(key) - return str(value) if value else "" - - -def _row_sort_key(row: dict[str, Any]) -> tuple[str, int, str]: - section_path = _source_value(row, "section_path") - try: - sort_order = int(row.get("sort_order") or 0) - except (TypeError, ValueError): - sort_order = 0 - chunk_id = str(row.get("chunk_id") or "") - return section_path, sort_order, chunk_id - - -def _append_content_lines(parts: list[str], content: object) -> None: - text = str(content or "").strip() - if not text: - return - for line in text.splitlines(): - stripped = line.strip() - if stripped: - parts.append(f" ┈ {stripped}") diff --git a/packages/shared-python/shared/services/retrieval/hydration/result_assembly.py b/packages/shared-python/shared/services/retrieval/hydration/result_assembly.py index 863bfadb..2f6aa54a 100644 --- a/packages/shared-python/shared/services/retrieval/hydration/result_assembly.py +++ b/packages/shared-python/shared/services/retrieval/hydration/result_assembly.py @@ -5,9 +5,12 @@ from sqlalchemy.ext.asyncio import AsyncSession +from shared.services.retrieval.hydration.asset_inline import ( + inline_assets_at_placeholders, + strip_path_placeholders, +) from shared.services.retrieval.hydration.connected import hydrate_connected_target_rows from shared.services.retrieval.hydration.row_utils import ( - clean_content, extract_page_nums, filter_excluded_rows, iter_connected_target_ids, @@ -70,18 +73,12 @@ async def assemble_retrieval_results( assembled_row['content'] = _compose_table_content(row, rows_by_chunk_id) assembled_row['content_source'] = 'summary' elif chunk_type == 'text': - related_parts = _connected_media_parts(row, rows_by_chunk_id) - if base_content and related_parts: - assembled_row['content'] = '\n\n'.join([base_content, *related_parts]) - elif related_parts: - assembled_row['content'] = '\n\n'.join(related_parts) - else: - assembled_row['content'] = base_content + assembled_row['content'] = _compose_text_content(row, rows_by_chunk_id) assembled_row['content_source'] = 'content' else: assembled_row['content'] = base_content assembled_row['content_source'] = 'content' - assembled_row['content'] = clean_content(assembled_row['content']) + assembled_row['content'] = strip_path_placeholders(assembled_row['content']) assembled.append(assembled_row) return assembled @@ -93,6 +90,26 @@ def _page_summary(row: dict[str, Any]) -> str: return str(metadata.get('summary') or '').strip() +def _compose_text_content( + row: dict[str, Any], + rows_by_chunk_id: dict[str, dict[str, Any]], +) -> str: + base_content = str(row.get('content') or '') + display_by_target = _connected_display_by_target(row, rows_by_chunk_id) + if not display_by_target: + return base_content + metadata = row.get('chunk_metadata') or row.get('metadata') or {} + connections = ( + metadata.get('connect_to') if isinstance(metadata, dict) else None + ) or [] + content, _embedded = inline_assets_at_placeholders( + base_content, + connections=connections if isinstance(connections, list) else [], + display_by_target=display_by_target, + ) + return content + + def _compose_table_content( row: dict[str, Any], rows_by_chunk_id: dict[str, dict[str, Any]], @@ -102,11 +119,11 @@ def _compose_table_content( return '\n\n'.join(part for part in parts if part) -def _connected_media_parts( +def _connected_display_by_target( row: dict[str, Any], rows_by_chunk_id: dict[str, dict[str, Any]], -) -> list[str]: - connected_targets: list[tuple[int, str]] = [] +) -> dict[str, str]: + display: dict[str, str] = {} for target_id in iter_connected_target_ids(row): target_row = rows_by_chunk_id.get(target_id) if not target_row: @@ -119,10 +136,8 @@ def _connected_media_parts( else: continue if target_content: - sort_key = int(target_row.get('sort_order', 0) or 0) - connected_targets.append((sort_key, target_content)) - connected_targets.sort(key=lambda item: item[0]) - return [content for _, content in connected_targets] + display[target_id] = target_content + return display def _connected_image_parts( diff --git a/packages/shared-python/shared/services/retrieval/hydration/row_utils.py b/packages/shared-python/shared/services/retrieval/hydration/row_utils.py index 32bcb2ba..74fa382e 100644 --- a/packages/shared-python/shared/services/retrieval/hydration/row_utils.py +++ b/packages/shared-python/shared/services/retrieval/hydration/row_utils.py @@ -1,6 +1,5 @@ from __future__ import annotations -import re from typing import Any from shared.services.retrieval.search.section_filters import is_excluded_section @@ -22,15 +21,6 @@ ReferenceLookupKey = tuple[str, str, str, str] -_PATH_REF_RE = re.compile(r'\[(?:images|tables)/[^\]\n]+\]') -_SAME_AS_RE = re.compile(r'\[SAME-AS [^\]]+\]') - - -def clean_content(content: str) -> str: - text = _PATH_REF_RE.sub('', content) - text = _SAME_AS_RE.sub('', text) - return text.strip() - def normalize_chunk_type(raw: object) -> str: return str(raw or '').strip().split('\n', 1)[0].lower() diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_compose.py b/packages/shared-python/shared/services/retrieval/nav/nav_compose.py index 3951a70c..e23a5f82 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_compose.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_compose.py @@ -12,6 +12,8 @@ from dataclasses import dataclass from typing import Any, Dict, List, Optional, Sequence, Set, Tuple +from shared.services.retrieval.hydration.evidence_text import render_evidence_blocks + from ._compat import Chunk from ._compat import line_node_id from ._compat import ToolSpace @@ -100,17 +102,11 @@ def direct_parent_id(ts: ToolSpace, section_id: str, doc_id: str) -> Optional[st return line_node_id(resolved, b.lines[p].line_id) -def _section_title(ts: ToolSpace, section_id: str, doc_id: str, *, max_chars: int = 40) -> str: +def _section_title(ts: ToolSpace, section_id: str, doc_id: str) -> str: sid = str(section_id or "").strip() if not sid: return "" - def _clip(text: str) -> str: - t = (text or "").strip() - if len(t) > max_chars: - return t[:max_chars].rstrip() - return t - # ``path_titles`` is a lightweight title lookup. Prefer it over # ``get_structure`` because the latter may calculate the complete subtree # chunk count, which is unnecessary while rendering evidence headers. @@ -121,7 +117,7 @@ def _clip(text: str) -> str: except TypeError: path = str(path_titles(sid) or "").strip() if path: - return _clip(path.rsplit(" / ", 1)[-1]) + return path # Prefer structure title (Knowhere / ProviderToolSpace); never parse ids. try: @@ -131,7 +127,7 @@ def _clip(text: str) -> str: if isinstance(st, dict): raw = st.get("preview") or st.get("title") or "" if isinstance(raw, str) and raw.strip(): - return _clip(raw.strip()) + return raw.strip() resolved = _section_doc_id(ts, sid, doc_id) idx = getattr(ts, "_idx", None) @@ -145,13 +141,13 @@ def _clip(text: str) -> str: bb = getattr(idx, "_bundles", {}).get(resolved) if bb and bb.lines: title = (bb.lines[0].content or "").strip() - return _clip(title) if title else sid + return title if title else sid return sid _, j = loc if j < 0 or j >= len(b.lines): return sid title = (b.lines[j].content or "").strip() - return _clip(title) if title else sid + return title if title else sid def _chunk_body(chunk: Chunk) -> str: @@ -302,7 +298,7 @@ def _build_groups( if parent_id is None: parent_id = owner if parent_id not in groups: - title = _section_title(ts, parent_id, owner_doc, max_chars=40) + title = _section_title(ts, parent_id, owner_doc) groups[parent_id] = _ParentGroup( parent_id=parent_id, parent_title=title, @@ -325,24 +321,13 @@ def _render_group( selected: Sequence[_ChildItem], *, evidence_index: int, - indent: bool, ) -> str: - """Render one evidence block (full text only).""" - parts: List[str] = [f"[E{evidence_index}]"] - if group.parent_title: - parts.append(f"[§ {group.parent_title}]") - for child in selected: - body = _chunk_body(child.chunk) - if not body: - continue - if indent: - indented = "\n".join( - (" " + ln if ln.strip() else ln) for ln in body.splitlines() - ) - parts.append(indented) - else: - parts.append(body) - return "\n".join(parts).strip() + """Render one evidence block via the shared evidence renderer.""" + bodies = [_chunk_body(child.chunk) for child in selected] + return render_evidence_blocks( + [(group.parent_title or "", bodies)], + start_index=evidence_index, + ) def _scored_flat(groups: Sequence[_ParentGroup]) -> List[Tuple[Chunk, float]]: @@ -399,9 +384,8 @@ def _render_kept( ] if not entries: continue - indent = len(entries) >= 2 block = _render_group( - g, entries, evidence_index=len(parts) + 1, indent=indent + g, entries, evidence_index=len(parts) + 1 ) add = len(block) + (len(sep) if parts else 0) if used + add <= budget_chars: diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_hierarchy.py b/packages/shared-python/shared/services/retrieval/nav/nav_hierarchy.py index 80b9cde0..ead3dfc6 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_hierarchy.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_hierarchy.py @@ -36,6 +36,10 @@ runtime_checkable, ) +from shared.services.retrieval.hydration.asset_inline import ( + inline_assets_at_placeholders, +) + if TYPE_CHECKING: from .knowhere_hybrid import PersistedScoreCorpus @@ -179,17 +183,72 @@ def parent_id(self, section_id: str) -> Optional[str]: return str(parent) if parent else None def _node_unit_span(self, section_id: str) -> Tuple[str, int, int]: - """(joined text, first sort_order, unit count) for one node's own units.""" + """(joined text, first sort_order, unit count) for one node's own units. + + Evidence display only: text units insert connected assets at placeholders. + Scoring still uses ``materialize_self_only_chunks`` / raw ``unit_text``. + """ self_units = getattr(self._provider, "self_units", None) unit_text = getattr(self._provider, "unit_text", None) if not callable(self_units) or not callable(unit_text): return "", 0, 0 units = list(self_units(section_id) or ()) - texts = [t for t in (str(unit_text(u) or "").strip() for u in units) if t] - if not texts: - return "", 0, len(units) + if not units: + return "", 0, 0 first_order = int(getattr(units[0], "sort_order", 0) or 0) - return "\n".join(texts), first_order, len(units) + + asset_types = {"image", "table"} + text_units: List[Any] = [] + asset_by_id: Dict[str, str] = {} + for unit in units: + chunk_type = str(getattr(unit, "chunk_type", "") or "").strip().lower() + chunk_id = str(getattr(unit, "chunk_id", "") or "").strip() + body = str(unit_text(unit) or "").strip() + if chunk_type in asset_types: + if chunk_id and body: + asset_by_id[chunk_id] = body + continue + text_units.append(unit) + + if not text_units: + texts = [body for body in asset_by_id.values() if body] + return "\n".join(texts), first_order, len(units) + + parts: List[str] = [] + used_assets: Set[str] = set() + for unit in text_units: + content = str(unit_text(unit) or "").strip() + meta = getattr(unit, "metadata", None) or {} + connections = ( + meta.get("connect_to") if isinstance(meta, dict) else None + ) or [] + if not isinstance(connections, list): + connections = [] + wanted = { + str(item.get("target") or "").strip() + for item in connections + if isinstance(item, dict) + } + display = { + target_id: asset_by_id[target_id] + for target_id in wanted + if target_id in asset_by_id + } + content, embedded = inline_assets_at_placeholders( + content, + connections=connections, + display_by_target=display, + ) + used_assets.update(embedded) + if content: + parts.append(content) + + for target_id, body in asset_by_id.items(): + if target_id in used_assets or not body: + continue + parts.append(body) + + return "\n".join(parts), first_order, len(units) def _make_chunk( self, node_id: str, doc_id: str, text: str, order: int, section_id: str diff --git a/packages/shared-python/shared/testing/contract_runtime.py b/packages/shared-python/shared/testing/contract_runtime.py index 27a3c7de..f5fbbd5f 100644 --- a/packages/shared-python/shared/testing/contract_runtime.py +++ b/packages/shared-python/shared/testing/contract_runtime.py @@ -350,6 +350,13 @@ def configure_contract_environment( "QSTASH_CURRENT_SIGNING_KEY": "qstash-current-test-key", "QSTASH_NEXT_SIGNING_KEY": "qstash-next-test-key", "QSTASH_CALLBACK_BASE_URL": "http://localhost:5005/api/v1", + # Pin product defaults so a developer's local apps/api/.env cannot + # change contract expectations (credits seed, upload allow-list). + "FREE_PLAN_INITIAL_CREDITS": "5", + "SUPPORTED_EXTENSIONS": ( + ".doc,.docx,.pdf,.txt,.xls,.xlsx,.csv,.pptx," + ".jpg,.jpeg,.png,.md,.html,.htm" + ), } if "BILLING_ENABLED" not in os.environ: diff --git a/packages/shared-python/shared/tests/test_asset_inline.py b/packages/shared-python/shared/tests/test_asset_inline.py new file mode 100644 index 00000000..a59652f0 --- /dev/null +++ b/packages/shared-python/shared/tests/test_asset_inline.py @@ -0,0 +1,154 @@ +"""Unit tests for placeholder-based asset inlining.""" + +from __future__ import annotations + +import pytest + +from shared.services.retrieval.hydration.asset_inline import ( + inline_assets_at_placeholders, +) +from shared.services.retrieval.hydration.result_assembly import ( + assemble_retrieval_results, +) +from shared.services.retrieval.nav.nav_hierarchy import ProviderToolSpace +from shared.services.retrieval.nav.nav_knowhere import ( + KnowhereProvider, + SectionRow, + UnitRow, +) + + +def test_inline_replaces_placeholder_with_newlines() -> None: + body, embedded = inline_assets_at_placeholders( + "see [images/a.png] here", + connections=[ + { + "target": "img-1", + "relation": "embeds", + "ref": "[images/a.png]", + } + ], + display_by_target={"img-1": "[Image: images/a.png]\nsummary"}, + ) + assert body == "see \n[Image: images/a.png]\nsummary\n here" + assert embedded == {"img-1"} + assert "[images/" not in body + + +def test_inline_appends_when_placeholder_missing() -> None: + body, embedded = inline_assets_at_placeholders( + "plain text", + connections=[{"target": "img-1", "ref": "[images/a.png]"}], + display_by_target={"img-1": "[Image: images/a.png]"}, + ) + assert body == "plain text\n\n[Image: images/a.png]" + assert embedded == {"img-1"} + + +def test_inline_does_not_duplicate_target() -> None: + body, embedded = inline_assets_at_placeholders( + "x [images/a.png] y", + connections=[ + {"target": "img-1", "ref": "[images/a.png]"}, + {"target": "img-1", "ref": "[images/a.png]"}, + ], + display_by_target={"img-1": "[Image: images/a.png]"}, + ) + assert body.count("[Image: images/a.png]") == 1 + assert embedded == {"img-1"} + + +@pytest.mark.asyncio +async def test_assemble_inserts_table_at_placeholder() -> None: + rows = [ + { + "chunk_id": "text-1", + "chunk_type": "text", + "content": "见表 [tables/table-1.html] 结束", + "chunk_metadata": { + "connect_to": [ + { + "target": "table-1", + "relation": "embeds", + "ref": "[tables/table-1.html]", + } + ] + }, + }, + { + "chunk_id": "table-1", + "chunk_type": "table", + "content": "
SHOULD NOT LEAK
", + "file_path": "tables/table-1.html", + "asset_url": "https://assets.example.com/job-1/tables/table-1.html", + "chunk_metadata": { + "summary": "企业入驻信息登记模板", + "keywords": ["企业名称"], + }, + }, + ] + assembled = await assemble_retrieval_results( + rows=rows, + exclude_document_ids=[], + exclude_sections=[], + ) + assert len(assembled) == 1 + content = assembled[0]["content"] + assert "[tables/" not in content + assert content.index("见表") < content.index("[Table:") + assert content.index("[Table:") < content.index("结束") + assert "企业入驻信息登记模板" in content + assert "SHOULD NOT LEAK" not in content + + +def test_node_unit_span_inlines_section_assets() -> None: + provider = KnowhereProvider( + doc_id="doc-1", + sections=[ + SectionRow( + section_id="sec-1", + parent_section_id=None, + section_path="One", + section_title="One", + section_level=1, + summary="", + sort_order=0, + ) + ], + units=[ + UnitRow( + chunk_id="text-1", + section_id="sec-1", + chunk_type="text", + content="see [images/a.png] end", + sort_order=0, + metadata={ + "connect_to": [ + { + "target": "img-1", + "relation": "embeds", + "ref": "[images/a.png]", + } + ] + }, + ), + UnitRow( + chunk_id="img-1", + section_id="sec-1", + chunk_type="image", + content="images/a.png", + sort_order=1, + file_path="images/a.png", + metadata={"summary": "chart summary"}, + ), + ], + ) + ts = ProviderToolSpace(provider) + text, _order, count = ts._node_unit_span("sec-1") + assert count == 2 + assert "[images/" not in text + assert text.index("see") < text.index("[Image:") + assert text.index("[Image:") < text.index("end") + assert "chart summary" in text + # Asset must not also appear as a trailing standalone copy. + assert text.count("[Image:") == 1 From 05afe687507f01742865ed9d2d252d22c6c9f0c6 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 2 Sep 2026 17:37:53 +0800 Subject: [PATCH 09/35] perf: persist per-channel map index statistics --- CONTEXT.md | 5 +- ...d_token_leading_map_unit_covering_index.py | 44 ++ ...1c2d3e4f5a6_add_channel_bm25_statistics.py | 42 ++ apps/api/scripts/backfill_map_unit_indexes.py | 37 +- .../test_retrieval_map_unit_index_contract.py | 26 +- .../tests/migrations/test_schema_contract.py | 51 ++ ...ding-covering-index-for-map-unit-lookup.md | 11 + ...kfill-serving-index-statistics-in-place.md | 12 + docs/adr/README.md | 2 + ...al-heart-vessel-trace-optimization-plan.md | 543 ++++++++++++++++++ .../shared/models/database/document.py | 19 +- .../services/retrieval/map_unit_index.py | 14 + .../services/retrieval/nav/nav_knowhere.py | 96 +++- .../services/retrieval/nav/nav_map_scores.py | 31 +- .../retrieval/nav/persisted_score_load.py | 11 +- .../shared/services/retrieval/nav_snapshot.py | 43 +- .../retrieval/search/map_unit_discovery.py | 136 ++++- 17 files changed, 1074 insertions(+), 49 deletions(-) create mode 100644 apps/api/alembic/versions/a0b1c2d3e4f5_add_token_leading_map_unit_covering_index.py create mode 100644 apps/api/alembic/versions/b1c2d3e4f5a6_add_channel_bm25_statistics.py create mode 100644 docs/adr/0009-use-token-leading-covering-index-for-map-unit-lookup.md create mode 100644 docs/adr/0010-backfill-serving-index-statistics-in-place.md create mode 100644 docs/design/retrieval-heart-vessel-trace-optimization-plan.md diff --git a/CONTEXT.md b/CONTEXT.md index 8609579f..029d2c64 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -228,7 +228,10 @@ generation and retries or falls back if publication changes it during capture. The compatibility requirement that a serving-index retrieval returns the same selected chunk IDs, ordering, rounded scores, citations, and asset references -as the legacy retrieval path for the same request. +as the legacy retrieval path for the same request. Every retrieval optimization +must preserve this quality contract; a latency improvement without validated +semantic parity is not shippable. Validation also compares source sections, +evidence content, router, and stop reason for each pinned request. ### Retrieval Revision Pin diff --git a/apps/api/alembic/versions/a0b1c2d3e4f5_add_token_leading_map_unit_covering_index.py b/apps/api/alembic/versions/a0b1c2d3e4f5_add_token_leading_map_unit_covering_index.py new file mode 100644 index 00000000..b051b8f0 --- /dev/null +++ b/apps/api/alembic/versions/a0b1c2d3e4f5_add_token_leading_map_unit_covering_index.py @@ -0,0 +1,44 @@ +"""Add the token-leading covering index for map-unit lookups.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + + +revision: str = "a0b1c2d3e4f5" +down_revision: str | None = "9f0a1b2c3d4e" +branch_labels: Sequence[str] | None = None +depends_on: Sequence[str] | None = None + +_INDEX_NAME = "idx_document_map_unit_tokens_token_lookup" + + +def upgrade() -> None: + """Create the additive index without taking a table-wide write lock.""" + uses_external_transaction: bool = bool( + op.get_context().opts.get("knowhere_external_transaction", False) + ) + statement: str = ( + f"CREATE INDEX {{concurrently}}IF NOT EXISTS {_INDEX_NAME} " + "ON document_map_unit_tokens (channel, token_hash, map_unit_id) " + "INCLUDE (token, frequency)" + ) + if uses_external_transaction: + op.execute(statement.format(concurrently="")) + return + with op.get_context().autocommit_block(): + op.execute(statement.format(concurrently="CONCURRENTLY ")) + + +def downgrade() -> None: + """Remove only the index introduced by this migration.""" + uses_external_transaction: bool = bool( + op.get_context().opts.get("knowhere_external_transaction", False) + ) + if uses_external_transaction: + op.execute(f"DROP INDEX IF EXISTS {_INDEX_NAME}") + return + with op.get_context().autocommit_block(): + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {_INDEX_NAME}") diff --git a/apps/api/alembic/versions/b1c2d3e4f5a6_add_channel_bm25_statistics.py b/apps/api/alembic/versions/b1c2d3e4f5a6_add_channel_bm25_statistics.py new file mode 100644 index 00000000..bb826334 --- /dev/null +++ b/apps/api/alembic/versions/b1c2d3e4f5a6_add_channel_bm25_statistics.py @@ -0,0 +1,42 @@ +"""Add persisted per-channel BM25 corpus statistics.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + + +revision: str = "b1c2d3e4f5a6" +down_revision: str | None = "a0b1c2d3e4f5" +branch_labels: Sequence[str] | None = None +depends_on: Sequence[str] | None = None + + +def upgrade() -> None: + """Add nullable statistics so incomplete legacy rows keep the fallback.""" + column_names: tuple[str, ...] = ( + "path_document_count", + "path_total_length", + "content_document_count", + "content_total_length", + ) + for column_name in column_names: + op.execute( + f"ALTER TABLE document_map_unit_indexes " + f"ADD COLUMN IF NOT EXISTS {column_name} INTEGER" + ) + + +def downgrade() -> None: + """Remove the additive statistics columns.""" + column_names: tuple[str, ...] = ( + "content_total_length", + "content_document_count", + "path_total_length", + "path_document_count", + ) + for column_name in column_names: + op.execute( + f"ALTER TABLE document_map_unit_indexes DROP COLUMN IF EXISTS {column_name}" + ) diff --git a/apps/api/scripts/backfill_map_unit_indexes.py b/apps/api/scripts/backfill_map_unit_indexes.py index f37afb66..98678c7f 100644 --- a/apps/api/scripts/backfill_map_unit_indexes.py +++ b/apps/api/scripts/backfill_map_unit_indexes.py @@ -224,10 +224,14 @@ def check_fallback_readiness(*, document_id: str = "") -> list[NamespaceFallback select( DocumentMapUnitIndex.document_id, DocumentMapUnitIndex.job_result_id, + DocumentMapUnitIndex.format_version, DocumentMapUnitIndex.unit_count, DocumentMapUnitIndex.average_idf_path, DocumentMapUnitIndex.average_idf_content, - DocumentMapUnitIndex.format_version, + DocumentMapUnitIndex.path_document_count, + DocumentMapUnitIndex.path_total_length, + DocumentMapUnitIndex.content_document_count, + DocumentMapUnitIndex.content_total_length, ).where( DocumentMapUnitIndex.document_id.in_( [document_id_value for document_id_value, _ in revisions] @@ -237,18 +241,26 @@ def check_fallback_readiness(*, document_id: str = "") -> list[NamespaceFallback ) index_by_revision = { (str(document_id_value), str(job_result_id)): ( + int(format_version or 0), int(unit_count or 0), float(average_idf_path or 0.0), float(average_idf_content or 0.0), - int(format_version or 0), + path_document_count, + path_total_length, + content_document_count, + content_total_length, ) for ( document_id_value, job_result_id, + format_version, unit_count, average_idf_path, average_idf_content, - format_version, + path_document_count, + path_total_length, + content_document_count, + content_total_length, ) in index_rows } missing_map_index = 0 @@ -258,8 +270,23 @@ def check_fallback_readiness(*, document_id: str = "") -> list[NamespaceFallback if stats is None: missing_map_index += 1 continue - unit_count, average_idf_path, average_idf_content, format_version = stats - if format_version != MAP_UNIT_INDEX_FORMAT_VERSION: + ( + format_version, + unit_count, + average_idf_path, + average_idf_content, + path_document_count, + path_total_length, + content_document_count, + content_total_length, + ) = stats + if ( + format_version != MAP_UNIT_INDEX_FORMAT_VERSION + or path_document_count is None + or path_total_length is None + or content_document_count is None + or content_total_length is None + ): missing_map_index += 1 continue if ( diff --git a/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py b/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py index 287a6b4c..f7829045 100644 --- a/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py +++ b/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py @@ -57,7 +57,11 @@ def __init__(self) -> None: def execute(self, statement: str, parameters: object = None) -> None: executions.append((statement, parameters)) if "document_map_unit_indexes" in statement: - self.rows = [(document_id, job_result_id, 2, 1, 0.0, 0.0)] + self.rows = [ + (document_id, job_result_id, 2, 1, 0.0, 0.0, 1, 1, 1, 1) + ] + elif "FROM document_sections" in statement: + self.rows = [(document_id, 1)] elif "FROM document_map_units AS units" in statement: self.rows = [("unit-frequency", document_id, "chunk-frequency", "section-frequency", 1, 1)] elif "FROM document_map_unit_tokens" in statement: @@ -99,10 +103,18 @@ def close(self) -> None: ) assert corpus is not None + selective_executions = [ + statement + for statement, _parameters in executions + if "SELECT DISTINCT map_unit_id" in statement + ] + assert len(selective_executions) == 1 + assert "JOIN (VALUES" in selective_executions[0] frequency_executions = [ (statement, parameters) for statement, parameters in executions if "FROM document_map_unit_tokens" in statement + and "SELECT DISTINCT map_unit_id" not in statement ] assert len(frequency_executions) == 1 statement, parameters = frequency_executions[0] @@ -286,6 +298,18 @@ async def test_published_map_units_preserve_scores_without_chunk_payload_reads( ) assert index.unit_count == len(expected_units) + assert index.path_document_count == sum( + unit.path_token_count > 0 for unit in persisted_units + ) + assert index.path_total_length == sum( + unit.path_token_count for unit in persisted_units + ) + assert index.content_document_count == sum( + unit.content_token_count > 0 for unit in persisted_units + ) + assert index.content_total_length == sum( + unit.content_token_count for unit in persisted_units + ) assert [unit.unit_id for unit in persisted_units] == [ str(unit["chunk_id"]) for unit in expected_units ] diff --git a/apps/api/tests/migrations/test_schema_contract.py b/apps/api/tests/migrations/test_schema_contract.py index 3ead1217..37ea8d65 100644 --- a/apps/api/tests/migrations/test_schema_contract.py +++ b/apps/api/tests/migrations/test_schema_contract.py @@ -255,6 +255,57 @@ def test_should_index_document_chunks_in_lazy_section_order( ) +def test_should_create_token_leading_map_unit_covering_index( + migrated_head_engine: Engine, +) -> None: + with migrated_head_engine.begin() as connection: + index_definition = connection.execute( + text( + """ + SELECT indexdef + FROM pg_indexes + WHERE schemaname = current_schema() + AND tablename = 'document_map_unit_tokens' + AND indexname = 'idx_document_map_unit_tokens_token_lookup' + """ + ) + ).scalar_one() + + definition = str(index_definition) + assert "(channel, token_hash, map_unit_id)" in definition + assert "INCLUDE (token, frequency)" in definition + + +def test_should_add_per_channel_map_unit_bm25_statistics( + migrated_head_engine: Engine, +) -> None: + with migrated_head_engine.begin() as connection: + columns = { + str(row[0]): str(row[1]) + for row in connection.execute( + text( + """ + SELECT column_name, is_nullable + FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'document_map_unit_indexes' + AND column_name IN ( + 'path_document_count', 'path_total_length', + 'content_document_count', 'content_total_length' + ) + """ + ) + ).all() + } + + assert columns == { + "path_document_count": "YES", + "path_total_length": "YES", + "content_document_count": "YES", + "content_total_length": "YES", + } + + def test_should_upgrade_with_a_caller_owned_connection( alembic_engine: Engine, ) -> None: diff --git a/docs/adr/0009-use-token-leading-covering-index-for-map-unit-lookup.md b/docs/adr/0009-use-token-leading-covering-index-for-map-unit-lookup.md new file mode 100644 index 00000000..da5d9c99 --- /dev/null +++ b/docs/adr/0009-use-token-leading-covering-index-for-map-unit-lookup.md @@ -0,0 +1,11 @@ +# Use a token-leading covering index for map-unit lookup + +**Status: accepted.** Token-selective retrieval will use an additive, +idempotent PostgreSQL covering index led by `(channel, token_hash, map_unit_id)` +and including `(token, frequency)`. This preserves the token-index-driven query +shape and allows index-only plans where PostgreSQL visibility permits them. The +migration is additive and keeps the existing lookup index until a separate +production-plan and load review proves it redundant. Requests fall back to the +exact legacy reader when the index or serving data is missing or incomplete. +The additional storage and write-maintenance cost is accepted in exchange for +lower first-request retrieval latency and row transfer. diff --git a/docs/adr/0010-backfill-serving-index-statistics-in-place.md b/docs/adr/0010-backfill-serving-index-statistics-in-place.md new file mode 100644 index 00000000..ec499205 --- /dev/null +++ b/docs/adr/0010-backfill-serving-index-statistics-in-place.md @@ -0,0 +1,12 @@ +# Backfill serving-index statistics in place + +**Status: accepted.** Existing retrieval-visible data will be maintained by +backfilling only the current revision of each active Document. Complete serving +indexes receive the new per-channel BM25 statistics through the existing +per-document `apps/api/scripts/backfill_map_unit_indexes.py --apply` rebuild; +missing or incomplete indexes use the existing full rebuild. v1 and v2 are +internal read paths under the same Retrieval contract, so each revision commits +independently and the optimized reader requires the new serving-index format +marker before it can be selected. An interrupted or failed backfill therefore +continues using the exact legacy reader, limiting database load and avoiding +unnecessary regeneration of correct token rows and snapshots. diff --git a/docs/adr/README.md b/docs/adr/README.md index 00d7b9da..4d535f15 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -23,3 +23,5 @@ Use this shape: | [0006](0006-atomically-publish-retrieval-serving-index.md) | Atomically publish the retrieval-serving index | | [0007](0007-use-coherent-retrieval-serving-generations.md) | Use coherent retrieval-serving generations | | [0008](0008-use-a-maintenance-window-for-serving-index-rollout.md) | Roll out the serving index online | +| [0009](0009-use-token-leading-covering-index-for-map-unit-lookup.md) | Use a token-leading covering index for map-unit lookup | +| [0010](0010-backfill-serving-index-statistics-in-place.md) | Backfill serving-index statistics in place | diff --git a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md new file mode 100644 index 00000000..4762f69a --- /dev/null +++ b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md @@ -0,0 +1,543 @@ +# Retrieval optimization plan from the fresh `心血管` trace + +## Scope and evidence + +The detailed stage breakdown below is based on a newly issued production +request, not a historical request: + +```json +{ + "namespace": "default", + "query": "心血管", + "top_k": 3, + "use_agentic": null +} +``` + +- Trace: `01a05e530f7787333c0e32ca16633b85` +- Route: `/api/v1/retrieval/query` +- HTTP status: `200` +- Router: `mapnav` +- Stop reason: `completed` +- Server span: `42.365 s` +- Client wall time: `44.690 s` +- Result: `29` referenced chunks, approximately `11,998` evidence characters + +The current production baseline is restricted to the last-night and today +window in Beijing time (`2026-09-01 20:00` through `2026-09-02 11:00`, or +`2026-09-01 12:00Z` through `2026-09-02 03:00Z`). Logfire recorded 203 v1 +retrieval roots and one v2 root in that window; all successful roots were from +`deployment.environment=production`. The v1 traffic includes both the direct +`/v1/retrieval/query` path and the externally prefixed `/api/v1/retrieval/query` +path. The FastAPI route is `/v1/retrieval/query` in both cases. + +Logfire does not currently record a git commit in `service_version` or another +resource attribute. These traces are therefore the latest observed production +behavior, but they are not proof that production matches a particular local +HEAD. The runtime reports Python `3.12.14`, built on `2026-09-01`, which is +useful deployment context but not a source revision identifier. + +v1 and v2 share the same retrieval execution plan. v2 only adds the optional +`llm_config` request field and passes it into the shared plan; when it is absent, +the retrieval SQL, route selection, scoring, hydration, and public projection +are the same. The one v2 request in this baseline had `llm_config=null`. + +The timings below are parent/child timings. Parent stages include their child +stages and must not be summed with those children. + +## Non-negotiable quality invariant + +Every optimization in this plan must preserve retrieval semantic parity for the +same pinned request: selected chunk IDs, ordering, rounded scores, source +sections, citations, evidence content, and asset references must remain +unchanged. A latency improvement that has not passed the parity checks is not +shippable, and the exact legacy retrieval path remains the fallback. + +## Current optimization boundary + +This phase covers non-LLM work only: SQL plans, indexes, serving-index row +transfer, cache behavior, snapshot decoding, map scoring, and instrumentation. +Planner, Harvest, and Control prompts, models, thinking settings, and model-call +orchestration are deferred until this phase reaches semantic-parity and latency +gates. + +The optimized reader is an internal implementation shared by v1 and v2. Public +API versioning is not a performance variant: every non-LLM change must preserve +the same Retrieval contract for v1 and v2. v2's optional `llm_config` may change +model settings, but it must not change lexical retrieval semantics. + +Implement and validate slices sequentially on the immutable local production +copy. Instrumentation is measurement-only and must first prove that the +baseline result is unchanged. Each subsequent SQL, index, cache, or local +scoring change is evaluated independently against the temporary parity set +before the next change begins; several individually validated slices may be +deployed together afterward. + +## Stage breakdown + +| Stage | Observed time | Evidence | +| --- | ---: | --- | +| Scope threshold probe | 1.838 s | `count_scoped_chunks` applies `LIMIT top_k + 1` before the outer `count(*)`; this is a bounded small-corpus gate, not an exact count | +| Snapshot load | 3.064 s | 644 documents, 115,892 references | +| Planner | 19.996 s | One subgoal; Planner LLM 19.995 s | +| Tree build | 0.966 s | 50,112 sections | +| Map-index load | 8.523 s | 644 index rows; 42,794 unit rows; 3 query tokens | +| Unit scoring | 0.190 s | 16,662 scored units | +| Map scoring aggregate | 9.802 s | Parent aggregate around scoring/pooling work | +| Harvest | 5.260 s | Harvest LLM 2.762 s; one wave/subgoal | +| Control | 1.256 s | Control LLM 1.248 s | +| Orchestration | 6.518 s | Parent around Harvest and Control | +| Evidence pack | 0.262 s | 23 chunks | +| Final hydration | 0.444 s | 23 results | +| Episode | 36.633 s | Parent from episode start through evidence pack | + +The map-index sub-stages were: + +- index metadata: `0.028 s`, 644 rows; +- map-unit rows: `4.288 s`, 42,794 rows, `cache_hit=false`; +- frequency rows: `3.866 s`, 42,794 units and 3 tokens. + +The `sections=50112` value in the current trace is now accounted for at the +source boundary: both the `tree_build` and `map_pooling` log sites computed it +as `sum(len(tree_by_doc[doc_id][0]) ...)`, where the first tuple member is the +reachable section-node map. It therefore means **section nodes**, not +section-child edges, leaf sections, or chunks. The local instrumentation now +also records `section_edges` and `leaf_sections` for the map scorer, and +`section_rows`, `section_paths`, `root_sections`, and `leaf_sections` for the +snapshot loader. A local isolated database now contains the target namespace's +644 documents, 50,112 sections, 60,187 chunks, and 42,794 map units. The +namespace snapshot payload was not copied after CSV transfer caused excessive +local write amplification, so this copy is not a complete end-to-end retrieval +parity corpus. + +For planner work, the local token table contains the 41,105 real rows for the +three-token probe plus 1,390,687 non-matching filler rows. Its 1,431,792 total +rows preserve the observed production channel ratio (content 1,275,676; path +156,116) and keep the real target-token distribution. Production has +13,948,605 rows (content 12,524,037; path 1,423,950), so this is a scaled +production-shaped copy rather than an exact row-count clone; the scale limit is +documented because a full synthetic INSERT caused severe local overlay +filesystem write amplification. + +On this copy, with all 42,794 local map units in scope, the existing +scope-first frequency query returned 36,478 rows in every run and measured +`103–121 ms` across five warm repetitions. The materialized token-first +candidate also returned 36,478 rows (symmetric difference `0`) but measured +`323–378 ms`; PostgreSQL used the existing non-covering lookup index by +default. A token-selective unit projection driven by a materialized +matching-token CTE returned 13,573 units and measured `205–223 ms`; PostgreSQL +still chose a unit-leading lookup for the join after an index-only token scan. +The scaled copy has a much higher target-hash fraction than production, so +this does not predict the full-scale join order. These results validate row +parity but do not yet demonstrate an end-to-end latency benefit; the reader +must not change until rows/bytes transferred and retrieval quality are measured +through the complete caller path. + +The SQL-output proxy confirms the potential transfer reduction: the existing +unit projection emitted 42,794 rows / 4.05 MB, while the token-selective +projection emitted 13,573 rows / 1.29 MB. This is a database-client output +measurement, not an application latency claim; Python decoding, filtering, +scoring, and cache behavior still need to be measured together. + +Using the existing persisted BM25 scorer with the same full-corpus channel +denominators, the complete and token-selective projections both produced +13,573 positive-scoring units for `心血管`; the maximum score delta was `0.0` +and their top-100 ordering was identical. This confirms that dropping +zero-frequency units is score-safe for this probe, but it is not yet a public +API retrieval-quality gate. + +The actual `ReadOnlyChunkStore.load_persisted_score_corpus` caller was also run +against the 508 local revisions whose serving index has format version `1`. +It loaded 44,894 section rows, returned the same 13,573 score units, and took +`1.089 s` for loading plus `0.168 s` for scoring. Revisions with incomplete +serving indexes correctly returned the existing fallback (`None`); they were +not silently included in the optimized sample. + +A temporary scorer baseline was frozen at +`/tmp/knowhere-retrieval-parity-baseline.json.gz` (SHA-256 of the uncompressed +payload `19436c6ee96a352d2000c52c85327bd2183e44c042dbea862824ba6ce10178c7`). +It covers `心血管`, the single-token `心`, and the no-match `肺` cases, with +all per-unit scores retained for exact comparison. The artifact is local-only +and must be regenerated only after an explicit decision to accept a retrieval +behavior change. + +The classic-route SQL shape was then measured against the same namespace and +revision scope. After one cold run, five repetitions of the legacy projection +measured `716–763 ms`; the token-selective projection measured `781–844 ms` +(one outlier at `1.73 s`). Output fell from 42,794 rows / 9.53 MB to 13,573 +rows / 2.93 MB. This is a substantial transfer reduction with no stable SQL +latency win yet, so the next gate is application-level p95 rather than a claim +that the query itself is faster. + +The reference trace above is not representative of request mix. In the current +baseline window, 196 successful v1 roots used `use_agentic=false` (classic), +three used the default map-nav route (`use_agentic=null`), and one explicitly +used `use_agentic=true`. The classic roots had P50 `10.99 s`, P90 `25.46 s`, +and a maximum of `36.65 s`. There were also three unauthenticated v1 attempts +(`401`) and one successful v2 root. The explicit `use_agentic=true` v1 request +took `41.31 s`, and the single v2 request took `109.36 s`; neither is combined +with the classic latency distribution because they exercise different route +and/or model settings. +Across their SQL children, `WITH knowhere` spans had P50 `1.15 s`, P90 +`8.24 s`, P95 `10.46 s`, and a maximum of `18.90 s`. In a representative +27.91-second classic request, map-unit discovery took `4.02 s` and the +token-frequency query took `18.90 s`. These current-window classic measurements +are part of the baseline for the shared v1/v2 reader. + +The resource log still renders literal format placeholders +(`cpu_seconds=%.3f process_max_rss_kb=%d`), so this trace cannot prove CPU +saturation or request-level memory usage. + +## Findings + +### Confirmed bottlenecks + +1. In the reference map-nav trace, the Planner LLM consumes approximately 20 + seconds. Outbound HTTP spans are only a few hundred milliseconds, so most of + the elapsed time is provider generation/thinking or uninstrumented client + wait, not network transfer. +2. The first request transfers all 42,794 map-unit rows before applying the + three query tokens. This costs 4.288 seconds and creates unnecessary Python + allocations. +3. The frequency lookup still costs 3.866 seconds even for only three tokens. + The current reader already filters `document_map_unit_tokens` by + `token_hash` before joining the pinned scope; the remaining question is + whether a token-leading covering index or a different join shape helps on a + full-scale corpus. The local production-shaped benchmark above does not + show a benefit yet. +4. The map scoring aggregate is 9.802 seconds, while the explicitly measured + unit scoring and pooling work is only 0.190 and 0.094 seconds. The gap is an + instrumentation boundary and/or additional map-scoring work that must be + measured before CPU tuning. +5. The bounded scope threshold probe costs 1.838 seconds. The default map-nav + route does not need an exact chunk count for ranking, but the probe still + distinguishes corpora at or below `top_k` from larger corpora before route + selection. The trace alone does not establish that an `EXISTS` rewrite or + snapshot metadata would be faster. +6. The current production request mix is primarily classic retrieval. The + token-selective reader must therefore be exercised through both the classic + and map-nav callers; a map-nav-only benchmark would not represent the + dominant workload. + +### Already healthy or lower priority + +- Snapshot load is material but not the largest stage at 3.064 seconds. +- Evidence pack and final hydration together are below one second in this + request. They are not the current optimization priority. +- The previous connected-hydration change is active: no full `JobResult` + chunk graph load appears in the trace. +- No prompt or retrieval-quality behavior should be changed as part of the + first implementation slices. + +## Prioritized optimization plan + +### Validation gate: bounded scope threshold probe (not an implementation P0) + +The current implementation already uses `LIMIT top_k + 1` inside +`count_scoped_chunks`, then counts that bounded subquery. Do not describe this +as an exact count or assume a 1.5–1.8 second saving from an `EXISTS` rewrite. +Benchmark the current probe against any equivalent early-exit query or trusted, +generation-pinned snapshot metadata. Retain this slice only if +`EXPLAIN (ANALYZE, BUFFERS)` and production-shaped repetitions show a meaningful +improvement; otherwise leave the current implementation unchanged and keep +this out of the optimization queue. + +Acceptance criteria: + +- route selection is unchanged for corpora below, equal to, and above + `top_k`; +- no stale snapshot can incorrectly classify a small corpus; +- the baseline and candidate plans make the `top_k + 1` bound and early-exit + behavior explicit; +- any candidate replacement is retained only when a latency improvement is + demonstrated; otherwise this remains a validation note, not an optimization + slice; +- selected chunk IDs, scores, ordering, and evidence remain unchanged; +- benchmark results record the route family (`classic`, `mapnav`, or + `small_corpus`) and separate cold, warm, and response-cache-hit requests; + cache-hit timings are not mixed into cold-request latency claims. + +### P0: Make map-unit projection token-selective + +The current map-nav reader already makes its frequency lookup token-selective, +but it still loads every revision-scoped map unit before applying the query +tokens. Change only the unit projection: start from +`document_map_unit_tokens` filtered by `channel` and `token_hash`, then join the +pinned revision/unit scope, and return only units matching at least one query +token. Keep the existing frequency SQL shape until a production-shaped plan +proves that a different join order is better. + +Roll this out in two stages. The first stage may enable the token-selective +reader only when the request has no section exclusions, where revision-scoped +channel statistics are sufficient. Requests with section filters continue to +use the exact legacy reader until section-scoped denominator statistics are +implemented and pass semantic-parity checks. A missing, incomplete, or +incompatible serving index always uses the same fallback. + +BM25 denominators must remain corpus-wide. Obtain exact corpus statistics using +persisted serving-index statistics while preserving the current per-channel +semantics. Extend `DocumentMapUnitIndex` with +`path_document_count`, `path_total_length`, `content_document_count`, and +`content_total_length`, calculated atomically when publishing each revision. +Units with zero path length must not contribute to path `document_count` or +`total_length`, and the same rule applies independently to content. Do not +reuse `DocumentMapUnitIndex.unit_count` for both channels, and never calculate +average length from only the matching subset. + +Hypothesis to validate: this should reduce map-index latency, rows transferred, +and temporary memory for this workload; no fixed seconds-saving claim is made +before a full-scale or statistically equivalent local benchmark. The scaled +copy above is a negative/shape-control result, not an approval to change the +reader. + +The current implementation slice only adds the per-channel statistics fields, +publication-time population, and the DevOps backfill/readiness contract. It +now enables token-selective projection in both persisted readers when the scope +is unfiltered and serving statistics are present. Section-filtered or +incomplete revisions continue through the legacy path. Both v1 and v2 call the +same shared readers; no route-specific lexical algorithm was introduced. + +Acceptance criteria: + +- a new idempotent migration adds the token-leading covering index required by + this query, for example `(channel, token_hash, map_unit_id) INCLUDE + (token, frequency)`. Keep the existing lookup index during this rollout; + evaluate removal separately only after `EXPLAIN (ANALYZE, BUFFERS)` and + production load confirm the new index is safe (or document the visibility + conditions that prevent an index-only scan); +- serving-index statistics include positive-length `document_count` and + `total_length` separately for path and content, aggregated over the exact + pinned revision and section scope; +- frequency maps and scores are identical to the current implementation on a + fixed production-data copy; +- duplicate token matches do not duplicate units; +- rows and bytes transferred are measured before and after; +- a safe fallback remains available if the persisted index is incomplete; +- v1 and v2 requests with equivalent retrieval fields produce identical public + retrieval results; v2's optional `llm_config` is recorded as a model-setting + difference, not as a separate lexical retrieval algorithm; +- the benchmark uses current-window classic requests as the primary sample and + includes at least one map-nav and one v2 request as shared-plan regression + samples; +- revision-scope cardinality and the size of the `(document_id, + job_result_id)` parameter set are recorded, so large `IN` lists are not hidden + inside token-query timing. + +### P1: Optimize the frequency SQL plan + +Remove or avoid a materialized full `scoped_units` side when the planner picks +the wrong join order. Drive from the token index, then apply channel, +token-hash, pinned revision, active-document, and namespace predicates. + +Hypothesis to validate: the revised join order may reduce frequency-stage +latency; claim no fixed saving until production-shaped `EXPLAIN (ANALYZE, +BUFFERS)` and repeated local runs confirm it. + +Acceptance criteria: + +- returned `(map_unit_id, channel, token, frequency)` rows are identical; +- no archived or wrong-generation revision can enter the result; +- statement timeout is not approached on the production read-only database; +- the query remains correct for one token, many tokens, and no matching token; +- repeated measurements cover the observed frequency-latency range (warm and + cold cache, one and many query tokens, and small and large revision scopes) + before any saving claim is made. + +### P1: Measure and then reduce the unaccounted map-scoring time + +Add structured timers around map-pooling, section aggregation, ranking, and +projection so the `9.802 s` parent stage can be reconciled with its children. +Only after that measurement should we change data structures or algorithms. + +Likely follow-up: precompute section owners, children, and postorder arrays +while decoding the snapshot, then reuse them during scoring instead of +rebuilding dictionaries and sets. + +Hypothesis to validate: precomputation should reduce local map-scoring time; +the magnitude is intentionally left to measurement. + +### Deferred: Planner and other LLM latency + +The Planner is the largest single stage, but this phase makes no changes to +Planner, Harvest, or Control model calls. Keep their existing behavior and +fallbacks intact. Revisit model timing, thinking, prompts, and call +orchestration only after the non-LLM slices below have passed their semantic- +parity and latency gates. + +### Deferred: overlap planner and query-independent map scoring (experimental) + +The current route waits for the Planner before starting map scoring. For a +planner result whose `retrieval_query` is exactly the user query and whose +scope/filters are unchanged, map scoring can be started concurrently on a +separate read-only connection. The planner result is then used only to decide +whether the already-computed candidate set is sufficient. + +This could hide part of the approximately `9.8 s` map-scoring aggregate behind +the approximately `20 s` Planner wait, but it changes LLM-boundary +orchestration. It is deferred until the Planner phase is explicitly in scope; +it must not run when the planner rewrites the query, adds a node filter, or +changes the revision/scope. + +Potential saving: up to the overlapping map-scoring time for eligible simple +queries. This is an experimental concurrency change with risks around +connection-pool pressure, duplicate work, cancellation, and revision +coherence. + +Acceptance criteria: + +- the candidate scores are byte-for-byte equal to the serial path; +- the concurrent read-only store is isolated from mutable navigation state; +- planner rewrites and filters correctly disable the overlap; +- cancellation and database connection cleanup are covered by contract tests; +- concurrency load tests show no increase in statement timeouts or RSS. + +### P2: Reduce snapshot parse allocations + +Redis stores compressed snapshot bytes, but every request still decompresses +and JSON-decodes 115,892 references. The current parsing path creates both +canonical and owner-qualified reference keys and then copies the mapping. + +Use one canonical representation with owner-aware fallback lookup and avoid +the second full mapping copy. Consider a versioned binary format only after a +benchmark; it is a larger migration and is not required for the first slices. + +Hypothesis to validate: the representation change should reduce decode time and +temporary RSS; the magnitude is intentionally left to measurement. + +Acceptance criteria: + +- both existing lookup forms remain valid; +- revision pinning and selected references are unchanged; +- checksum/version validation and generation invalidation remain intact; +- cold Redis-miss and Redis-hit memory are measured separately. + +### P2: Redis map-unit projection cache + +The current unit cache is episode/process-local; the snapshot Redis cache does +not eliminate the map-unit SQL transfer. Add a generation- and revision-scoped +Redis cache for a compact map-unit projection and index statistics, with a +bounded TTL and size limit. Bind each key to `user_id`, `namespace`, serving +generation, `document_id`, `job_result_id`, and index format version. A +generation change naturally invalidates old entries; Redis misses, version +mismatches, and decode failures fall back to PostgreSQL. + +This is not a guaranteed cold-first-request optimization. It helps only when a +previous publisher or request has populated the generation key. PostgreSQL +remains the source of truth and is the fallback on misses or Redis failures. + +## Existing-data maintenance + +Backfill only the currently retrieval-visible revision for each active +Document. Add the channel-stat columns with an additive migration and introduce +serving-index format version 2. v1 and v2 remain two internal read paths under +the same Retrieval contract and must produce identical results. For a complete +existing index, a resumable per-revision statistics backfill aggregates the existing +`document_map_units` rows, writes the four path/content values, and marks the +revision v2 in the same transaction. It must not regenerate token rows or +snapshots unnecessarily. Revisions with a missing or incomplete index use the +existing full `backfill_map_unit_indexes --apply` path. + +Each revision is committed independently under the existing generation and +active-document locks. The job is idempotent and safe to interrupt: revisions +without a completed v2 marker remain on the exact legacy reader, and the +optimized internal reader is selected only after the backfill check reports +complete, coherent active revisions; otherwise the same Retrieval request uses +the exact legacy reader. + +### DevOps handoff + +DevOps runs the maintenance explicitly; API startup and user requests never +trigger it. The runbook should: + +1. deploy the additive migration and verify the reader still falls back safely; + Build the new index without dropping the existing lookup index, and monitor + lock waits, build duration, and disk headroom; +2. deploy the application code that writes v2 for new publications and selects + the fast reader only for complete v2 revisions; v1 revisions continue on the + legacy reader; +3. run the read-only inventory/check command and record active/current revision + counts; +4. run the resumable statistics backfill in bounded batches (with optional + document selection), monitoring database load and generation changes; +5. rerun the check until every retrieval-visible revision has a coherent v2 + marker and no serving fallback is reported; +6. monitor semantic-parity probes, latency, errors, and timeouts. The reader's + version/completeness checks automatically retain the legacy path when a + revision is not ready. + +The handoff must document the exact command, batch/concurrency limits, +pause/resume procedure, application-version rollback procedure, and the final +check output. A partial +backfill is an expected intermediate state, not a failed deployment, provided +that incomplete revisions remain on the legacy reader. + +### Deferred: LLM orchestration improvements + +- Do not add a deterministic shortcut around Control. `plan_control` remains + the sole checklist-reconciliation authority; non-empty or seemingly related + evidence is not sufficient to replace its accept/widen/drop/replan decision. +- Do not change subgoal parallelism or Harvest/Control materialization in this + phase. These remain candidates only after the LLM boundary is explicitly + revisited. + +These changes must not alter prompt text or citation selection semantics. + +## Validation protocol + +Every implementation slice must be validated against a local copy of the +production namespace before deployment. Production rollout then relies on the +automatic legacy fallback for any incomplete or incompatible revision. + +Validation has two layers. First, maintain a small temporary retrieval-parity +set (two or three representative queries, including the current `心血管` trace) +containing current legacy-reader outputs, and run deterministic non-LLM parity +checks on an immutable production-data copy with fixed revision pins, section +scope, and queries. Compare loader rows, frequency maps, channel statistics, +per-unit scores, and ordering against that set. This set is a quality guard for +the optimization work, not a new product behavior contract; do not refresh it +just to hide a regression. Generate it once from the current legacy reader and +freeze it for the whole optimization series; refresh it only after an explicit +decision to accept a retrieval behavior change. Second, run end-to-end +Retrieval requests to measure latency, rows/bytes, and memory. Do not use +nondeterministic Planner/Harvest timing as evidence for this non-LLM change. + +1. Capture a baseline with the exact request, a cold process, and an empty or + generation-scoped Redis key. Run at least 10 cold repetitions; measure p50, + p95, stage timings, peak RSS, rows, and bytes transferred. +2. Run a separate warm-cache series. Never present warm-cache results as the + first-request improvement. +3. For every SQL change, compare `EXPLAIN (ANALYZE, BUFFERS)` and exact returned + rows against the current query using the read-only production database or + an immutable local copy. +4. Compare retrieval quality per request, requiring zero public-result + differences: selected chunk IDs, order, rounded scores, source sections, + evidence content/hash, asset references, router, stop reason, and citation + validity. Aggregate quality metrics cannot replace this exact parity check. +5. Exercise edge cases: empty result, one token, multiple tokens, small + corpus, incomplete index, archived revision, namespace generation change, + and Redis miss/failure. +6. Fix the malformed CPU/RSS log fields before making any CPU or memory claim. + Use per-stage CPU deltas and cgroup metrics rather than process high-water + RSS alone. +7. Roll out one optimization at a time, with automatic legacy fallback and + production p50/p95/error/timeout monitoring. If rollback is required, + redeploy the previous application version; leave additive schema/data in + place for a later retry. + +For each slice, retain it only when repeated local runs show a stable +improvement in the targeted stage and no regression in total non-LLM p95. The +temporary parity set is a hard gate regardless of the measured speedup. + +## Recommended implementation order + +1. Add missing structured timing and CPU/RSS instrumentation. +2. Benchmark the existing bounded `top_k + 1` scope probe; keep a replacement + only if it demonstrates a measured improvement. +3. Implement one token-selective serving-index slice: add the covering index, + persist per-channel BM25 statistics, and make unfiltered map-unit discovery + token-selective with the exact legacy fallback for filtered/incomplete + scopes. +4. Optimize the frequency join order and verify the production query plan. +5. Reconcile and optimize the unaccounted map-scoring work. +6. Consider snapshot allocation and Redis map-unit projection work as follow-up + improvements. diff --git a/packages/shared-python/shared/models/database/document.py b/packages/shared-python/shared/models/database/document.py index 3d630b91..6a46fdd2 100644 --- a/packages/shared-python/shared/models/database/document.py +++ b/packages/shared-python/shared/models/database/document.py @@ -328,6 +328,13 @@ class DocumentMapUnitToken(Base): "token_hash", "map_unit_id", ), + Index( + "idx_document_map_unit_tokens_token_lookup", + "channel", + "token_hash", + "map_unit_id", + postgresql_include=["token", "frequency"], + ), Index( "idx_document_map_unit_tokens_unit_lookup", "map_unit_id", @@ -340,7 +347,7 @@ class DocumentMapUnitToken(Base): class DocumentMapUnitIndex(Base): - """Completeness marker for a revision's materialized map-unit index.""" + """Completeness marker and corpus statistics for a materialized index.""" __tablename__ = "document_map_unit_indexes" @@ -362,6 +369,16 @@ class DocumentMapUnitIndex(Base): average_idf_content: Mapped[float] = mapped_column( Float, nullable=False, default=0.0 ) + path_document_count: Mapped[Optional[int]] = mapped_column( + Integer, nullable=True + ) + path_total_length: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + content_document_count: Mapped[Optional[int]] = mapped_column( + Integer, nullable=True + ) + content_total_length: Mapped[Optional[int]] = mapped_column( + Integer, nullable=True + ) created_at: Mapped[datetime] = mapped_column( DateTime, default=utc_now_naive, nullable=False ) diff --git a/packages/shared-python/shared/services/retrieval/map_unit_index.py b/packages/shared-python/shared/services/retrieval/map_unit_index.py index ee18434d..ab7a0523 100644 --- a/packages/shared-python/shared/services/retrieval/map_unit_index.py +++ b/packages/shared-python/shared/services/retrieval/map_unit_index.py @@ -88,6 +88,10 @@ def replace_document_map_units( token_count = 0 path_unit_df: Counter[str] = Counter() content_unit_df: Counter[str] = Counter() + path_document_count: int = 0 + path_total_length: int = 0 + content_document_count: int = 0 + content_total_length: int = 0 for sort_order, unit in enumerate(score_units): unit_id = str(unit.get("chunk_id") or "").strip() section_id = str(unit.get("section_id") or "").strip() @@ -96,6 +100,12 @@ def replace_document_map_units( map_unit_id = f"dmu_{uuid4().hex}" path_tokens = str(unit.get("path_search_text") or "").split() content_tokens = str(unit.get("content_search_text") or "").split() + if path_tokens: + path_document_count += 1 + path_total_length += len(path_tokens) + if content_tokens: + content_document_count += 1 + content_total_length += len(content_tokens) path_unit_df.update(set(path_tokens)) content_unit_df.update(set(content_tokens)) # ``provider.self_units`` already reflects root-asset remount (assets @@ -152,6 +162,10 @@ def replace_document_map_units( unit_count=persisted_count, token_document_frequency=content_unit_df, ), + path_document_count=path_document_count, + path_total_length=path_total_length, + content_document_count=content_document_count, + content_total_length=content_total_length, ) ) diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py b/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py index 81179b25..1a8d3ba3 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py @@ -335,7 +335,9 @@ def load_persisted_score_corpus( cur.execute( "SELECT indexes.document_id, indexes.job_result_id, " "indexes.format_version, indexes.unit_count, " - "indexes.average_idf_path, indexes.average_idf_content " + "indexes.average_idf_path, indexes.average_idf_content, " + "indexes.path_document_count, indexes.path_total_length, " + "indexes.content_document_count, indexes.content_total_length " "FROM document_map_unit_indexes AS indexes " f"JOIN (VALUES {values_sql}) AS revisions(document_id, job_result_id) " "ON indexes.document_id = revisions.document_id " @@ -353,7 +355,8 @@ def load_persisted_score_corpus( len(index_rows), ) if len(index_rows) != len(revisions) or any( - len(row) < 6 or int(row[2]) != MAP_UNIT_INDEX_FORMAT_VERSION + len(row) < 10 or int(row[2]) != MAP_UNIT_INDEX_FORMAT_VERSION + or any(value is None for value in row[6:10]) for row in index_rows ): return None @@ -386,8 +389,63 @@ def load_persisted_score_corpus( for document_id, section_ids in allowed_by_document.items() for section_id in section_ids } + cur.execute( + "SELECT document_id, count(*) FROM document_sections " + "WHERE document_id = ANY(%s) GROUP BY document_id", + [[document_id for document_id, _ in revisions]], + ) + section_counts = { + str(document_id): int(count) + for document_id, count in cur.fetchall() + } + has_complete_section_scope: bool = all( + len(allowed_by_document.get(document_id, set())) + == section_counts.get(document_id, 0) + for document_id, _ in revisions + ) all_unit_rows = self._score_unit_rows_cache.get(revision_key) - if all_unit_rows is None: + if has_complete_section_scope and query_token_hashes: + stage_started = time.perf_counter() + cur.execute( + "WITH matching_tokens AS MATERIALIZED (" + "SELECT DISTINCT map_unit_id FROM document_map_unit_tokens " + "WHERE channel = ANY(%s) AND token_hash = ANY(%s)" + "), scoped_units AS MATERIALIZED (" + f"SELECT units.id, units.document_id, units.unit_id, units.section_id, " + "units.path_token_count, units.content_token_count " + "FROM document_map_units AS units " + f"JOIN (VALUES {values_sql}) AS revisions(document_id, job_result_id) " + "ON units.document_id = revisions.document_id " + "AND units.job_result_id = revisions.job_result_id" + ") SELECT scoped_units.id, scoped_units.document_id, " + "scoped_units.unit_id, scoped_units.section_id, " + "scoped_units.path_token_count, scoped_units.content_token_count " + "FROM matching_tokens JOIN scoped_units " + "ON scoped_units.id = matching_tokens.map_unit_id", + [ + list(_MAP_SCORE_CHANNELS), + list(query_token_hashes), + *revision_params, + ], + ) + unit_rows = [ + { + "map_unit_id": str(row[0]), + "document_id": str(row[1]), + "unit_id": str(row[2]), + "section_id": str(row[3] or ""), + "path_token_count": int(row[4] or 0), + "content_token_count": int(row[5] or 0), + } + for row in cur.fetchall() + ] + _logger.info( + "retrieval map-index load stage=units-selective seconds=%.3f rows=%d", + time.perf_counter() - stage_started, + len(unit_rows), + ) + all_unit_rows = None + elif all_unit_rows is None: stage_started = time.perf_counter() cur.execute( "SELECT units.id, units.document_id, units.unit_id, " @@ -427,11 +485,13 @@ def load_persisted_score_corpus( True, ) - unit_rows = [ - row - for row in all_unit_rows - if (str(row["document_id"]), str(row["section_id"])) in allowed_pairs_set - ] + if not has_complete_section_scope or not query_token_hashes: + unit_rows = [ + row + for row in all_unit_rows or [] + if (str(row["document_id"]), str(row["section_id"])) + in allowed_pairs_set + ] frequencies: Dict[Tuple[str, str], Dict[str, int]] = {} if unit_rows and query_tokens: stage_started = time.perf_counter() @@ -482,6 +542,16 @@ def load_persisted_score_corpus( query_tokens=query_tokens, frequencies=frequencies, average_idf=average_idf_path, + document_count_override=( + sum(int(row[6] or 0) for row in index_rows) + if has_complete_section_scope and query_token_hashes + else None + ), + total_length_override=( + sum(int(row[7] or 0) for row in index_rows) + if has_complete_section_scope and query_token_hashes + else None + ), ) content_stats = build_channel_bm25_stats( unit_rows=unit_rows, @@ -491,6 +561,16 @@ def load_persisted_score_corpus( query_tokens=query_tokens, frequencies=frequencies, average_idf=average_idf_content, + document_count_override=( + sum(int(row[8] or 0) for row in index_rows) + if has_complete_section_scope and query_token_hashes + else None + ), + total_length_override=( + sum(int(row[9] or 0) for row in index_rows) + if has_complete_section_scope and query_token_hashes + else None + ), ) return PersistedScoreCorpus( units=[ diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py b/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py index c3637464..94883b19 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py @@ -20,6 +20,22 @@ _logger = logging.getLogger(__name__) +def _tree_shape_counts( + tree_by_doc: Dict[ + str, + Tuple[Dict[str, List[str]], Set[str], Dict[str, str]], + ], +) -> Tuple[int, int, int]: + """Return reachable section nodes, parent-child edges, and leaves.""" + section_nodes: int = sum(len(value[0]) for value in tree_by_doc.values()) + section_edges: int = sum( + sum(len(children) for children in value[0].values()) + for value in tree_by_doc.values() + ) + leaf_sections: int = sum(len(value[1]) for value in tree_by_doc.values()) + return section_nodes, section_edges, leaf_sections + + def _build_legacy_score_corpus(ts: Any, doc_ids: Sequence[str]) -> PersistedScoreCorpus: """Build the retired in-memory scorer input when persisted indexes are absent.""" raw_units: List[dict] = [] @@ -414,11 +430,15 @@ def compute_corpus_map_and_unit_scores_many( cached = _walk_tree(ts, doc_id, root_ids) tree_cache[doc_id] = cached tree_by_doc[doc_id] = cached + section_nodes, section_edges, leaf_sections = _tree_shape_counts(tree_by_doc) _logger.info( - "retrieval mapnav phase=tree_build seconds=%.3f documents=%d sections=%d", + "retrieval mapnav phase=tree_build seconds=%.3f documents=%d " + "section_nodes=%d section_edges=%d leaf_sections=%d", time.perf_counter() - tree_started, len(valid_doc_ids), - sum(len(value[0]) for value in tree_by_doc.values()), + section_nodes, + section_edges, + leaf_sections, ) persisted_loader = getattr(ts, "load_persisted_score_corpus", None) @@ -471,10 +491,13 @@ def compute_corpus_map_and_unit_scores_many( map_scores[doc_id] = doc_max results[query] = (map_scores, unit_scores) _logger.info( - "retrieval mapnav phase=map_pooling seconds=%.3f documents=%d sections=%d", + "retrieval mapnav phase=map_pooling seconds=%.3f documents=%d " + "section_nodes=%d section_edges=%d leaf_sections=%d", time.perf_counter() - pooling_started, len(valid_doc_ids), - sum(len(value[0]) for value in tree_by_doc.values()), + section_nodes, + section_edges, + leaf_sections, ) return results diff --git a/packages/shared-python/shared/services/retrieval/nav/persisted_score_load.py b/packages/shared-python/shared/services/retrieval/nav/persisted_score_load.py index 9ce10c2e..712f23b5 100644 --- a/packages/shared-python/shared/services/retrieval/nav/persisted_score_load.py +++ b/packages/shared-python/shared/services/retrieval/nav/persisted_score_load.py @@ -47,12 +47,19 @@ def build_channel_bm25_stats( query_tokens: Sequence[str], frequencies: Mapping[tuple[str, str], Mapping[str, int]], average_idf: float, + document_count_override: int | None = None, + total_length_override: int | None = None, ) -> PersistedBm25Stats: """Build channel stats from already-fetched unit rows and query-token freqs.""" lengths = [ int(row[length_field]) for row in unit_rows if int(row[length_field]) > 0 ] - document_count = len(lengths) + document_count = ( + len(lengths) if document_count_override is None else document_count_override + ) + total_length = ( + sum(lengths) if total_length_override is None else total_length_override + ) document_frequency = { token: sum( 1 @@ -66,7 +73,7 @@ def build_channel_bm25_stats( } return PersistedBm25Stats( document_count=document_count, - total_length=sum(lengths), + total_length=total_length, document_frequency=document_frequency, average_idf=float(average_idf), ) diff --git a/packages/shared-python/shared/services/retrieval/nav_snapshot.py b/packages/shared-python/shared/services/retrieval/nav_snapshot.py index b15425e3..abdd28b1 100644 --- a/packages/shared-python/shared/services/retrieval/nav_snapshot.py +++ b/packages/shared-python/shared/services/retrieval/nav_snapshot.py @@ -86,6 +86,26 @@ def _is_snapshot_timing_enabled() -> bool: } +def _section_shape_counts( + sections_by_doc: Mapping[str, list[SectionRow]], +) -> tuple[int, int, int, int]: + """Return section rows, distinct paths, roots, and leaves for diagnostics.""" + rows: list[SectionRow] = [ + row for document_rows in sections_by_doc.values() for row in document_rows + ] + parent_ids: set[str] = { + str(row.parent_section_id).strip() + for row in rows + if str(row.parent_section_id or "").strip() + } + return ( + len(rows), + len({str(row.section_path) for row in rows}), + sum(1 for row in rows if not str(row.parent_section_id or "").strip()), + sum(1 for row in rows if str(row.section_id) not in parent_ids), + ) + + class SnapshotSession(Protocol): """Minimal database interface required by the snapshot loader.""" @@ -369,17 +389,25 @@ async def load_nav_snapshot( document_revisions=dict(revisions), ) if _is_snapshot_timing_enabled(): + section_rows, section_paths, root_sections, leaf_sections = ( + _section_shape_counts(sections_by_doc) + ) _logger.info( "retrieval snapshot timing total_seconds=%.3f parse_seconds=%.3f " "ref_index_copy_seconds=%.3f doc_query_seconds=%.3f " - "job_query_seconds=%.3f documents=%d sections=%d refs=%d mode=lazy", + "job_query_seconds=%.3f documents=%d section_rows=%d " + "section_paths=%d root_sections=%d leaf_sections=%d refs=%d " + "mode=lazy", time.perf_counter() - snapshot_started, parse_seconds, ref_index_seconds, doc_query_seconds, job_query_seconds, len(snapshot.document_ids), - sum(len(rows) for rows in sections_by_doc.values()), + section_rows, + section_paths, + root_sections, + leaf_sections, len(snapshot.chunk_ref_index), ) return snapshot @@ -414,16 +442,23 @@ async def load_nav_snapshot( }, ) if _is_snapshot_timing_enabled(): + section_rows, section_paths, root_sections, leaf_sections = ( + _section_shape_counts(sections_by_doc) + ) _logger.info( "retrieval snapshot timing total_seconds=%.3f parse_seconds=%.3f " "doc_query_seconds=%.3f job_query_seconds=%.3f documents=%d " - "sections=%d refs=%d mode=eager", + "section_rows=%d section_paths=%d root_sections=%d " + "leaf_sections=%d refs=%d mode=eager", time.perf_counter() - snapshot_started, parse_seconds, doc_query_seconds, job_query_seconds, len(snapshot.document_ids), - sum(len(rows) for rows in sections_by_doc.values()), + section_rows, + section_paths, + root_sections, + leaf_sections, len(snapshot.chunk_ref_index), ) return snapshot diff --git a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py index 4e7d1a4a..a4f15f04 100644 --- a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py +++ b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py @@ -211,6 +211,15 @@ async def map_unit_discovery( params.update(type_params) params.update(signal_params) + unfiltered_scope: bool = not any( + ( + chunk_types, + signal_paths, + exclude_sections, + exclude_document_ids, + ) + ) + cte = _SCOPED_UNITS_CTE.format( revision_join=revision_join, revision_clause=revision_clause, @@ -218,7 +227,24 @@ async def map_unit_discovery( type_clause=type_clause, signal_clause=signal_clause, ) - unit_result = await db.execute(text(cte + "SELECT * FROM scoped_units"), params) + unit_statement = cte + "SELECT * FROM scoped_units" + if unfiltered_scope: + unit_statement = ( + "WITH matching_tokens AS MATERIALIZED (" + "SELECT DISTINCT map_unit_id FROM document_map_unit_tokens " + "WHERE channel = ANY(:channels) AND token_hash = ANY(:token_hashes)" + "), " + + cte.lstrip().removeprefix("WITH ") + + " SELECT DISTINCT scoped_units.* " + "FROM matching_tokens JOIN scoped_units " + "ON scoped_units.map_unit_id = matching_tokens.map_unit_id" + ) + params = { + **params, + "channels": list(_MAP_SCORE_CHANNELS), + "token_hashes": query_token_hashes, + } + unit_result = await db.execute(text(unit_statement), params) unit_rows = [dict(row._mapping) for row in unit_result.all()] unit_rows = [ row @@ -283,7 +309,9 @@ async def map_unit_discovery( ) + """ SELECT indexes.average_idf_path, indexes.average_idf_content, - indexes.unit_count, indexes.format_version + indexes.unit_count, indexes.format_version, + indexes.path_document_count, indexes.path_total_length, + indexes.content_document_count, indexes.content_total_length FROM document_map_unit_indexes AS indexes JOIN ( SELECT DISTINCT document_id, job_result_id FROM scoped_units @@ -294,29 +322,71 @@ async def map_unit_discovery( ), params, ) - # Only count indexes written with the current tokenizer (v2 = word-level). - # Stale char-level (v1) rows look complete by unit_count but cannot match - # word-level query hashes — treat them as missing so readiness fails closed. index_parts = [ - (float(path_idf or 0.0), float(content_idf or 0.0), int(unit_count or 0)) - for path_idf, content_idf, unit_count, format_version in index_result.all() - if int(format_version or 0) == MAP_UNIT_INDEX_FORMAT_VERSION - ] - expected_revisions = { - (str(row["document_id"]), str(row["job_result_id"])) for row in unit_rows - } - unfiltered_scope = not any( ( - chunk_types, - signal_paths, - exclude_sections, - exclude_document_ids, + float(path_idf or 0.0), + float(content_idf or 0.0), + int(unit_count or 0), + int(format_version or 0), + path_document_count, + path_total_length, + content_document_count, + content_total_length, ) + for ( + path_idf, + content_idf, + unit_count, + format_version, + path_document_count, + path_total_length, + content_document_count, + content_total_length, + ) in index_result.all() + ] + if unfiltered_scope: + revision_result = await db.execute( + text(cte + "SELECT DISTINCT document_id, job_result_id FROM scoped_units"), + params, + ) + expected_revisions = { + (str(row[0]), str(row[1])) for row in revision_result.all() + } + else: + expected_revisions = { + (str(row["document_id"]), str(row["job_result_id"])) + for row in unit_rows + } + indexed_unit_count = sum( + unit_count for _path_idf, _content_idf, unit_count, *_rest in index_parts ) - index_unit_count_mismatch = unfiltered_scope and sum( - unit_count for _path_idf, _content_idf, unit_count in index_parts - ) != len(unit_rows) - if len(index_parts) != len(expected_revisions) or index_unit_count_mismatch: + index_unit_count_mismatch = ( + indexed_unit_count < len(unit_rows) + if unfiltered_scope + else indexed_unit_count != len(unit_rows) + ) + index_statistics_incomplete = unfiltered_scope and any( + format_version != MAP_UNIT_INDEX_FORMAT_VERSION + or path_document_count is None + or path_total_length is None + or content_document_count is None + or content_total_length is None + for ( + _path_idf, + _content_idf, + _unit_count, + format_version, + path_document_count, + path_total_length, + content_document_count, + content_total_length, + ) in index_parts + ) + if ( + len(index_parts) != len(expected_revisions) + or index_unit_count_mismatch + or index_statistics_incomplete + ): try: await record_retrieval_index_readiness( user_id=user_id, @@ -361,13 +431,13 @@ async def map_unit_discovery( average_idf_path = combine_average_idf( [ (path_idf, unit_count) - for path_idf, _content_idf, unit_count in index_parts + for path_idf, _content_idf, unit_count, *_rest in index_parts ] ) average_idf_content = combine_average_idf( [ (content_idf, unit_count) - for _path_idf, content_idf, unit_count in index_parts + for _path_idf, content_idf, unit_count, *_rest in index_parts ] ) @@ -379,6 +449,16 @@ async def map_unit_discovery( query_tokens=query_tokens, frequencies=frequencies, average_idf=average_idf_path, + document_count_override=( + sum(int(part[4] or 0) for part in index_parts) + if unfiltered_scope + else None + ), + total_length_override=( + sum(int(part[5] or 0) for part in index_parts) + if unfiltered_scope + else None + ), ) content_stats = build_channel_bm25_stats( unit_rows=unit_rows, @@ -388,6 +468,16 @@ async def map_unit_discovery( query_tokens=query_tokens, frequencies=frequencies, average_idf=average_idf_content, + document_count_override=( + sum(int(part[6] or 0) for part in index_parts) + if unfiltered_scope + else None + ), + total_length_override=( + sum(int(part[7] or 0) for part in index_parts) + if unfiltered_scope + else None + ), ) corpus = PersistedScoreCorpus( From daefd1df636d33b5b2e10eab334878ef8742c1e6 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 2 Sep 2026 19:56:48 +0800 Subject: [PATCH 10/35] fix: preserve map index compatibility across filtered scopes --- ...est_retrieval_classic_map_unit_contract.py | 187 ++++++++++++++++++ .../test_retrieval_map_unit_index_contract.py | 2 +- .../services/retrieval/nav/nav_knowhere.py | 18 +- .../retrieval/search/map_unit_discovery.py | 14 ++ 4 files changed, 213 insertions(+), 8 deletions(-) diff --git a/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py b/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py index 63941a1f..45fd8d17 100644 --- a/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py +++ b/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py @@ -173,6 +173,193 @@ def capture_frequency_query( assert "FROM matching_tokens" in statements[-1] +async def test_classic_route_only_uses_token_selective_projection_for_unfiltered_scope( + developer_api_client_factory: Callable[ + [], AbstractAsyncContextManager[AsyncClient] + ], +) -> None: + identifier = uuid4().hex[:8] + namespace = f"classic-projection-{identifier}" + projection_statements: list[str] = [] + + def capture_unit_projection( + _connection: Any, + _cursor: Any, + statement: str, + _parameters: Any, + _context: Any, + _executemany: bool, + ) -> None: + if "SELECT DISTINCT scoped_units.*" in statement: + projection_statements.append("token_selective") + elif "SELECT * FROM scoped_units" in statement: + projection_statements.append("legacy") + + event.listen(Engine, "before_cursor_execute", capture_unit_projection) + try: + async with developer_api_client_factory() as api_client: + await _publish_document( + namespace=namespace, + source_file_name="projection.pdf", + chunks=[ + { + "chunk_id": f"projection-hit-{identifier}", + "type": "text", + "content": "projection token marker", + "path": "projection.pdf/Root/Section/hit", + "order": 1, + "metadata": {}, + }, + { + "chunk_id": f"projection-filler-a-{identifier}", + "type": "text", + "content": "unrelated filler a", + "path": "projection.pdf/Root/Section/a", + "order": 2, + "metadata": {}, + }, + { + "chunk_id": f"projection-filler-b-{identifier}", + "type": "text", + "content": "unrelated filler b", + "path": "projection.pdf/Root/Section/b", + "order": 3, + "metadata": {}, + }, + { + "chunk_id": f"projection-filler-c-{identifier}", + "type": "text", + "content": "unrelated filler c", + "path": "projection.pdf/Root/Section/c", + "order": 4, + "metadata": {}, + }, + ], + ) + unfiltered_response = await api_client.post( + "/api/v1/retrieval/query", + json={ + "namespace": namespace, + "query": "projection token marker", + "top_k": 1, + "use_agentic": False, + }, + ) + filtered_response = await api_client.post( + "/api/v1/retrieval/query", + json={ + "namespace": namespace, + "query": "projection token marker", + "top_k": 1, + "use_agentic": False, + "signal_paths": ["Section"], + "filter_mode": "keep", + }, + ) + finally: + event.remove(Engine, "before_cursor_execute", capture_unit_projection) + + assert unfiltered_response.status_code == 200 + assert filtered_response.status_code == 200 + assert projection_statements[:2] == ["token_selective", "legacy"] + + +async def test_classic_route_falls_back_for_v1_index_with_excluded_document( + developer_api_client_factory: Callable[ + [], AbstractAsyncContextManager[AsyncClient] + ], +) -> None: + identifier = uuid4().hex[:8] + namespace = f"classic-v1-fallback-{identifier}" + legacy_queries: list[str] = [] + + def capture_legacy_query( + _connection: Any, + _cursor: Any, + statement: str, + _parameters: Any, + _context: Any, + _executemany: bool, + ) -> None: + if "plainto_tsquery('simple'" in statement: + legacy_queries.append(statement) + + event.listen(Engine, "before_cursor_execute", capture_legacy_query) + try: + async with developer_api_client_factory() as api_client: + first = await _publish_document( + namespace=namespace, + source_file_name="legacy-fallback.pdf", + chunks=[ + { + "chunk_id": f"legacy-hit-{identifier}", + "type": "text", + "content": "legacy fallback marker", + "path": "legacy-fallback.pdf/Root/Section/body", + "order": 1, + "metadata": {}, + }, + { + "chunk_id": f"legacy-filler-a-{identifier}", + "type": "text", + "content": "unrelated legacy filler a", + "path": "legacy-fallback.pdf/Root/Section/a", + "order": 2, + "metadata": {}, + }, + { + "chunk_id": f"legacy-filler-b-{identifier}", + "type": "text", + "content": "unrelated legacy filler b", + "path": "legacy-fallback.pdf/Root/Section/b", + "order": 3, + "metadata": {}, + }, + ], + ) + excluded = await _publish_document( + namespace=namespace, + source_file_name="excluded.pdf", + chunks=[ + { + "chunk_id": f"excluded-{identifier}", + "type": "text", + "content": "unrelated filler", + "path": "excluded.pdf/Root/Section/body", + "order": 1, + "metadata": {}, + } + ], + ) + await ContractDatabase.execute( + """ + UPDATE document_map_unit_indexes + SET format_version = 1 + WHERE document_id = :document_id + """, + {"document_id": first["document_id"]}, + ) + response = await api_client.post( + "/api/v1/retrieval/query", + json={ + "namespace": namespace, + "query": "legacy fallback marker", + "top_k": 1, + "use_agentic": False, + "exclude_document_ids": [excluded["document_id"]], + }, + ) + finally: + event.remove(Engine, "before_cursor_execute", capture_legacy_query) + + assert response.status_code == 200 + body = cast(dict[str, object], response.json()) + results = cast(list[dict[str, object]], body["results"]) + assert len(results) == 1 + assert results[0]["chunk_id"] == f"legacy-hit-{identifier}" + assert legacy_queries + + async def test_classic_route_image_filter_scores_only_units_with_images( developer_api_client_factory: Callable[ [], AbstractAsyncContextManager[AsyncClient] diff --git a/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py b/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py index f7829045..5128382f 100644 --- a/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py +++ b/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py @@ -61,7 +61,7 @@ def execute(self, statement: str, parameters: object = None) -> None: (document_id, job_result_id, 2, 1, 0.0, 0.0, 1, 1, 1, 1) ] elif "FROM document_sections" in statement: - self.rows = [(document_id, 1)] + self.rows = [(document_id, job_result_id, 1)] elif "FROM document_map_units AS units" in statement: self.rows = [("unit-frequency", document_id, "chunk-frequency", "section-frequency", 1, 1)] elif "FROM document_map_unit_tokens" in statement: diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py b/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py index 1a8d3ba3..3c1365d8 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py @@ -390,18 +390,22 @@ def load_persisted_score_corpus( for section_id in section_ids } cur.execute( - "SELECT document_id, count(*) FROM document_sections " - "WHERE document_id = ANY(%s) GROUP BY document_id", - [[document_id for document_id, _ in revisions]], + "SELECT sections.document_id, sections.job_result_id, count(*) " + "FROM document_sections AS sections " + f"JOIN (VALUES {values_sql}) AS revisions(document_id, job_result_id) " + "ON sections.document_id = revisions.document_id " + "AND sections.job_result_id = revisions.job_result_id " + "GROUP BY sections.document_id, sections.job_result_id", + revision_params, ) section_counts = { - str(document_id): int(count) - for document_id, count in cur.fetchall() + (str(document_id), str(job_result_id)): int(count) + for document_id, job_result_id, count in cur.fetchall() } has_complete_section_scope: bool = all( len(allowed_by_document.get(document_id, set())) - == section_counts.get(document_id, 0) - for document_id, _ in revisions + == section_counts.get((document_id, job_result_id), 0) + for document_id, job_result_id in revisions ) all_unit_rows = self._score_unit_rows_cache.get(revision_key) if has_complete_section_scope and query_token_hashes: diff --git a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py index a4f15f04..65c2fc44 100644 --- a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py +++ b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py @@ -365,6 +365,19 @@ async def map_unit_discovery( if unfiltered_scope else indexed_unit_count != len(unit_rows) ) + index_format_incompatible = any( + format_version != MAP_UNIT_INDEX_FORMAT_VERSION + for ( + _path_idf, + _content_idf, + _unit_count, + format_version, + _path_document_count, + _path_total_length, + _content_document_count, + _content_total_length, + ) in index_parts + ) index_statistics_incomplete = unfiltered_scope and any( format_version != MAP_UNIT_INDEX_FORMAT_VERSION or path_document_count is None @@ -385,6 +398,7 @@ async def map_unit_discovery( if ( len(index_parts) != len(expected_revisions) or index_unit_count_mismatch + or index_format_incompatible or index_statistics_incomplete ): try: From 9615a23ac3dab1cf4b6c2c33d37d547b52c42023 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 2 Sep 2026 19:56:58 +0800 Subject: [PATCH 11/35] docs: reset parity benchmark after main rebase --- ...al-heart-vessel-trace-optimization-plan.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md index 4762f69a..a7936636 100644 --- a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md +++ b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md @@ -153,13 +153,12 @@ It loaded 44,894 section rows, returned the same 13,573 score units, and took serving indexes correctly returned the existing fallback (`None`); they were not silently included in the optimized sample. -A temporary scorer baseline was frozen at -`/tmp/knowhere-retrieval-parity-baseline.json.gz` (SHA-256 of the uncompressed -payload `19436c6ee96a352d2000c52c85327bd2183e44c042dbea862824ba6ce10178c7`). -It covers `心血管`, the single-token `心`, and the no-match `肺` cases, with -all per-unit scores retained for exact comparison. The artifact is local-only -and must be regenerated only after an explicit decision to accept a retrieval -behavior change. +A temporary scorer baseline was previously frozen at +`/tmp/knowhere-retrieval-parity-baseline.json.gz`. That artifact was generated +before rebasing onto the current main tokenizer and is now historical evidence +only; it must not be used as the acceptance baseline for this branch. Generate +a new baseline from the current `origin/main` behavior after the local copy has +been rebuilt with v2 tokens. The classic-route SQL shape was then measured against the same namespace and revision scope. After one cold run, five repetitions of the legacy projection @@ -169,6 +168,12 @@ rows / 2.93 MB. This is a substantial transfer reduction with no stable SQL latency win yet, so the next gate is application-level p95 rather than a claim that the query itself is faster. +The earlier discovery-level comparison and three-query digests were also +captured before the rebase, against the old tokenizer/data state. They are +retained only as historical investigation notes and do not establish current +latency or retrieval parity. Re-run both the baseline and optimized discovery +paths after the v2 local backfill, using the current main code as the baseline. + The reference trace above is not representative of request mix. In the current baseline window, 196 successful v1 roots used `use_agentic=false` (classic), three used the default map-nav route (`use_agentic=null`), and one explicitly From 13c5735328bbc9fb2dd8e887cf5168c157c82c34 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 2 Sep 2026 21:28:55 +0800 Subject: [PATCH 12/35] chore: align retrieval helper naming --- .../services/retrieval/nav/nav_map_scores.py | 4 +-- .../shared/services/retrieval/nav_snapshot.py | 6 ++-- .../retrieval/search/map_unit_discovery.py | 28 +++++++++---------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py b/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py index 94883b19..eb207ce3 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py @@ -20,7 +20,7 @@ _logger = logging.getLogger(__name__) -def _tree_shape_counts( +def _count_tree_shape( tree_by_doc: Dict[ str, Tuple[Dict[str, List[str]], Set[str], Dict[str, str]], @@ -430,7 +430,7 @@ def compute_corpus_map_and_unit_scores_many( cached = _walk_tree(ts, doc_id, root_ids) tree_cache[doc_id] = cached tree_by_doc[doc_id] = cached - section_nodes, section_edges, leaf_sections = _tree_shape_counts(tree_by_doc) + section_nodes, section_edges, leaf_sections = _count_tree_shape(tree_by_doc) _logger.info( "retrieval mapnav phase=tree_build seconds=%.3f documents=%d " "section_nodes=%d section_edges=%d leaf_sections=%d", diff --git a/packages/shared-python/shared/services/retrieval/nav_snapshot.py b/packages/shared-python/shared/services/retrieval/nav_snapshot.py index abdd28b1..9f92617c 100644 --- a/packages/shared-python/shared/services/retrieval/nav_snapshot.py +++ b/packages/shared-python/shared/services/retrieval/nav_snapshot.py @@ -86,7 +86,7 @@ def _is_snapshot_timing_enabled() -> bool: } -def _section_shape_counts( +def _count_section_shape( sections_by_doc: Mapping[str, list[SectionRow]], ) -> tuple[int, int, int, int]: """Return section rows, distinct paths, roots, and leaves for diagnostics.""" @@ -390,7 +390,7 @@ async def load_nav_snapshot( ) if _is_snapshot_timing_enabled(): section_rows, section_paths, root_sections, leaf_sections = ( - _section_shape_counts(sections_by_doc) + _count_section_shape(sections_by_doc) ) _logger.info( "retrieval snapshot timing total_seconds=%.3f parse_seconds=%.3f " @@ -443,7 +443,7 @@ async def load_nav_snapshot( ) if _is_snapshot_timing_enabled(): section_rows, section_paths, root_sections, leaf_sections = ( - _section_shape_counts(sections_by_doc) + _count_section_shape(sections_by_doc) ) _logger.info( "retrieval snapshot timing total_seconds=%.3f parse_seconds=%.3f " diff --git a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py index 65c2fc44..20427477 100644 --- a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py +++ b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py @@ -211,7 +211,7 @@ async def map_unit_discovery( params.update(type_params) params.update(signal_params) - unfiltered_scope: bool = not any( + is_unfiltered_scope: bool = not any( ( chunk_types, signal_paths, @@ -228,7 +228,7 @@ async def map_unit_discovery( signal_clause=signal_clause, ) unit_statement = cte + "SELECT * FROM scoped_units" - if unfiltered_scope: + if is_unfiltered_scope: unit_statement = ( "WITH matching_tokens AS MATERIALIZED (" "SELECT DISTINCT map_unit_id FROM document_map_unit_tokens " @@ -344,7 +344,7 @@ async def map_unit_discovery( content_total_length, ) in index_result.all() ] - if unfiltered_scope: + if is_unfiltered_scope: revision_result = await db.execute( text(cte + "SELECT DISTINCT document_id, job_result_id FROM scoped_units"), params, @@ -360,12 +360,12 @@ async def map_unit_discovery( indexed_unit_count = sum( unit_count for _path_idf, _content_idf, unit_count, *_rest in index_parts ) - index_unit_count_mismatch = ( + has_index_unit_count_mismatch = ( indexed_unit_count < len(unit_rows) - if unfiltered_scope + if is_unfiltered_scope else indexed_unit_count != len(unit_rows) ) - index_format_incompatible = any( + is_index_format_incompatible = any( format_version != MAP_UNIT_INDEX_FORMAT_VERSION for ( _path_idf, @@ -378,7 +378,7 @@ async def map_unit_discovery( _content_total_length, ) in index_parts ) - index_statistics_incomplete = unfiltered_scope and any( + has_incomplete_index_statistics = is_unfiltered_scope and any( format_version != MAP_UNIT_INDEX_FORMAT_VERSION or path_document_count is None or path_total_length is None @@ -397,9 +397,9 @@ async def map_unit_discovery( ) if ( len(index_parts) != len(expected_revisions) - or index_unit_count_mismatch - or index_format_incompatible - or index_statistics_incomplete + or has_index_unit_count_mismatch + or is_index_format_incompatible + or has_incomplete_index_statistics ): try: await record_retrieval_index_readiness( @@ -465,12 +465,12 @@ async def map_unit_discovery( average_idf=average_idf_path, document_count_override=( sum(int(part[4] or 0) for part in index_parts) - if unfiltered_scope + if is_unfiltered_scope else None ), total_length_override=( sum(int(part[5] or 0) for part in index_parts) - if unfiltered_scope + if is_unfiltered_scope else None ), ) @@ -484,12 +484,12 @@ async def map_unit_discovery( average_idf=average_idf_content, document_count_override=( sum(int(part[6] or 0) for part in index_parts) - if unfiltered_scope + if is_unfiltered_scope else None ), total_length_override=( sum(int(part[7] or 0) for part in index_parts) - if unfiltered_scope + if is_unfiltered_scope else None ), ) From 5ec729556b69eca382a3164b2296364d4fbb0263 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 2 Sep 2026 22:24:30 +0800 Subject: [PATCH 13/35] docs: mark discarded local benchmark historical --- ...al-heart-vessel-trace-optimization-plan.md | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md index a7936636..2ba7420d 100644 --- a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md +++ b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md @@ -104,22 +104,22 @@ reachable section-node map. It therefore means **section nodes**, not section-child edges, leaf sections, or chunks. The local instrumentation now also records `section_edges` and `leaf_sections` for the map scorer, and `section_rows`, `section_paths`, `root_sections`, and `leaf_sections` for the -snapshot loader. A local isolated database now contains the target namespace's -644 documents, 50,112 sections, 60,187 chunks, and 42,794 map units. The -namespace snapshot payload was not copied after CSV transfer caused excessive -local write amplification, so this copy is not a complete end-to-end retrieval -parity corpus. +snapshot loader. A previous isolated fixture contained the target namespace's +644 documents, 50,112 sections, 60,187 chunks, and 42,794 map units. That +fixture has been discarded after CSV transfer caused excessive local write +amplification; the replacement DevOps dump is pending and must be treated as +the only current end-to-end parity corpus. -For planner work, the local token table contains the 41,105 real rows for the +For planner work, the discarded fixture contained 41,105 real rows for the three-token probe plus 1,390,687 non-matching filler rows. Its 1,431,792 total -rows preserve the observed production channel ratio (content 1,275,676; path -156,116) and keep the real target-token distribution. Production has -13,948,605 rows (content 12,524,037; path 1,423,950), so this is a scaled -production-shaped copy rather than an exact row-count clone; the scale limit is -documented because a full synthetic INSERT caused severe local overlay -filesystem write amplification. - -On this copy, with all 42,794 local map units in scope, the existing +rows preserved the observed production channel ratio (content 1,275,676; path +156,116) and kept the real target-token distribution. Production had +13,948,605 rows (content 12,524,037; path 1,423,950), so those measurements were +from a scaled production-shaped copy rather than an exact row-count clone. +They remain historical investigation evidence only; repeat them on the +replacement dump before using them for an implementation decision. + +On that discarded copy, with all 42,794 map units in scope, the existing scope-first frequency query returned 36,478 rows in every run and measured `103–121 ms` across five warm repetitions. The materialized token-first candidate also returned 36,478 rows (symmetric difference `0`) but measured @@ -133,7 +133,8 @@ parity but do not yet demonstrate an end-to-end latency benefit; the reader must not change until rows/bytes transferred and retrieval quality are measured through the complete caller path. -The SQL-output proxy confirms the potential transfer reduction: the existing +The historical SQL-output proxy confirms the potential transfer reduction: the +existing unit projection emitted 42,794 rows / 4.05 MB, while the token-selective projection emitted 13,573 rows / 1.29 MB. This is a database-client output measurement, not an application latency claim; Python decoding, filtering, @@ -147,7 +148,8 @@ zero-frequency units is score-safe for this probe, but it is not yet a public API retrieval-quality gate. The actual `ReadOnlyChunkStore.load_persisted_score_corpus` caller was also run -against the 508 local revisions whose serving index has format version `1`. +against 508 revisions in that discarded fixture whose serving index had format +version `1`. It loaded 44,894 section rows, returned the same 13,573 score units, and took `1.089 s` for loading plus `0.168 s` for scoring. Revisions with incomplete serving indexes correctly returned the existing fallback (`None`); they were @@ -160,8 +162,8 @@ only; it must not be used as the acceptance baseline for this branch. Generate a new baseline from the current `origin/main` behavior after the local copy has been rebuilt with v2 tokens. -The classic-route SQL shape was then measured against the same namespace and -revision scope. After one cold run, five repetitions of the legacy projection +The classic-route SQL shape was then measured against the same historical +namespace and revision scope. After one cold run, five repetitions of the legacy projection measured `716–763 ms`; the token-selective projection measured `781–844 ms` (one outlier at `1.73 s`). Output fell from 42,794 rows / 9.53 MB to 13,573 rows / 2.93 MB. This is a substantial transfer reduction with no stable SQL From 3566fc8ec7e7d368d42225f4d6922213369f9752 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 03:21:04 +0800 Subject: [PATCH 14/35] fix: stabilize persisted index aggregation order --- .../shared/services/retrieval/search/map_unit_discovery.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py index 20427477..cecf28d5 100644 --- a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py +++ b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py @@ -318,6 +318,7 @@ async def map_unit_discovery( ) AS scoped_revisions ON indexes.document_id = scoped_revisions.document_id AND indexes.job_result_id = scoped_revisions.job_result_id + ORDER BY indexes.document_id, indexes.job_result_id """ ), params, From 060613a475501c1e4e345b9e5a95639e8bbae6e0 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 03:29:33 +0800 Subject: [PATCH 15/35] docs: define retrieval score parity tolerance --- .../retrieval-heart-vessel-trace-optimization-plan.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md index 2ba7420d..c61b5a8d 100644 --- a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md +++ b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md @@ -50,8 +50,12 @@ stages and must not be summed with those children. Every optimization in this plan must preserve retrieval semantic parity for the same pinned request: selected chunk IDs, ordering, rounded scores, source sections, citations, evidence content, and asset references must remain -unchanged. A latency improvement that has not passed the parity checks is not -shippable, and the exact legacy retrieval path remains the fallback. +unchanged. Score comparisons use a maximum absolute tolerance of `1e-4` to +avoid treating harmless floating-point accumulation-order differences as a +retrieval change; IDs, ordering, source sections, citations, evidence content, +and asset references have no tolerance. A latency improvement that has not +passed the parity checks is not shippable, and the exact legacy retrieval path +remains the fallback. ## Current optimization boundary From f25827081bdbb2a169b4a7669ee5773f97dc88bb Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 03:34:16 +0800 Subject: [PATCH 16/35] revert: remove ineffective index ordering --- .../shared/services/retrieval/search/map_unit_discovery.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py index cecf28d5..20427477 100644 --- a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py +++ b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py @@ -318,7 +318,6 @@ async def map_unit_discovery( ) AS scoped_revisions ON indexes.document_id = scoped_revisions.document_id AND indexes.job_result_id = scoped_revisions.job_result_id - ORDER BY indexes.document_id, indexes.job_result_id """ ), params, From 54a848bfb15ab37c39cdefaac10c9f26fce440a4 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 03:48:09 +0800 Subject: [PATCH 17/35] docs: record restored dump parity benchmark --- ...trieval-heart-vessel-trace-optimization-plan.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md index c61b5a8d..28bbc0ab 100644 --- a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md +++ b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md @@ -180,6 +180,20 @@ retained only as historical investigation notes and do not establish current latency or retrieval parity. Re-run both the baseline and optimized discovery paths after the v2 local backfill, using the current main code as the baseline. +The replacement local dump is now available on PostgreSQL port `55433`. +After applying the additive schema migration, populating the four channel +statistics from the existing map units, and running `VACUUM (ANALYZE)`, the +token-leading covering index uses an index-only scan (`Heap Fetches=0`). For +the `心血管` probe, the isolated SQL projection measured approximately +`150 ms` for scope-first versus `77 ms` for token-selective, and the frequency +lookup measured `2.7 ms`. Ten repeated classic application calls across +`心血管`, `心脏`, and `肺` preserved chunk IDs, ordering, sources, and evidence; +the maximum score delta was `1.7e-5`. Warm p50 improved by about `24 ms` for +`心血管`, was effectively unchanged for `心脏`, and regressed by less than +`1 ms` for the empty `肺` probe. Treat this as a SQL/transfer improvement with +no yet-established broad end-to-end latency win; production rollout still +requires the same migration, backfill, and rollback checks below. + The reference trace above is not representative of request mix. In the current baseline window, 196 successful v1 roots used `use_agentic=false` (classic), three used the default map-nav route (`use_agentic=null`), and one explicitly From 882d5e1ebba92602a07f9c4ec7d456b002e0d9d4 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 04:04:02 +0800 Subject: [PATCH 18/35] feat: add map unit statistics backfill command --- .../scripts/backfill_map_unit_statistics.py | 197 ++++++++++++++++++ ...al-heart-vessel-trace-optimization-plan.md | 14 +- 2 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 apps/api/scripts/backfill_map_unit_statistics.py diff --git a/apps/api/scripts/backfill_map_unit_statistics.py b/apps/api/scripts/backfill_map_unit_statistics.py new file mode 100644 index 00000000..94c7aff1 --- /dev/null +++ b/apps/api/scripts/backfill_map_unit_statistics.py @@ -0,0 +1,197 @@ +# ruff: noqa: E402 + +"""Backfill persisted per-channel BM25 statistics without rebuilding tokens. + +This maintenance command aggregates existing ``document_map_units`` rows and +updates the four nullable statistics columns on the current active revision. +It never rewrites map-unit tokens, serving manifests, or namespace snapshots. +Each revision is committed independently so interruption is safe. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from dataclasses import dataclass +from pathlib import Path + + +def _bootstrap_python_path() -> None: + api_root = Path(__file__).resolve().parents[1] + shared_root = api_root.parents[1] / "packages" / "shared-python" + for path in (api_root, shared_root): + value = os.fspath(path) + if value not in sys.path: + sys.path.insert(0, value) + + +_bootstrap_python_path() + +from sqlalchemy import func, select, update +from sqlalchemy.orm import Session + +from shared.core.database_sync import get_sync_session_factory +from shared.models.database.document import ( + Document, + DocumentMapUnit, + DocumentMapUnitIndex, +) +from shared.services.retrieval.nav.knowhere_hybrid import MAP_UNIT_INDEX_FORMAT_VERSION + + +@dataclass(frozen=True) +class RevisionStatistics: + path_document_count: int + path_total_length: int + content_document_count: int + content_total_length: int + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="Write statistics.") + parser.add_argument( + "--check", action="store_true", help="Report missing or mismatched statistics." + ) + parser.add_argument("--document-id", default="") + parser.add_argument("--user-id", default="") + parser.add_argument("--namespace", default="") + parser.add_argument("--batch-size", type=int, default=100) + return parser + + +def _load_documents( + *, document_id: str, user_id: str, namespace: str +) -> list[Document]: + session_factory = get_sync_session_factory() + with session_factory() as db: + statement = ( + select(Document) + .where(Document.status == "active") + .where(Document.current_job_result_id.is_not(None)) + .order_by(Document.document_id) + ) + if document_id: + statement = statement.where(Document.document_id == document_id) + if user_id: + statement = statement.where(Document.user_id == user_id) + if namespace: + statement = statement.where(Document.namespace == namespace) + return list(db.scalars(statement).all()) + + +def _aggregate_statistics( + db: Session, *, document_id: str, job_result_id: str +) -> RevisionStatistics: + statement = ( + select( + func.count() + .filter(DocumentMapUnit.path_token_count > 0) + .label("path_document_count"), + func.coalesce(func.sum(DocumentMapUnit.path_token_count), 0).label( + "path_total_length" + ), + func.count() + .filter(DocumentMapUnit.content_token_count > 0) + .label("content_document_count"), + func.coalesce(func.sum(DocumentMapUnit.content_token_count), 0).label( + "content_total_length" + ), + ) + .where(DocumentMapUnit.document_id == document_id) + .where(DocumentMapUnit.job_result_id == job_result_id) + ) + row = db.execute(statement).one() + return RevisionStatistics( + path_document_count=int(row.path_document_count or 0), + path_total_length=int(row.path_total_length or 0), + content_document_count=int(row.content_document_count or 0), + content_total_length=int(row.content_total_length or 0), + ) + + +def _is_complete(index: DocumentMapUnitIndex | None, stats: RevisionStatistics) -> bool: + return bool( + index + and index.format_version == MAP_UNIT_INDEX_FORMAT_VERSION + and index.path_document_count == stats.path_document_count + and index.path_total_length == stats.path_total_length + and index.content_document_count == stats.content_document_count + and index.content_total_length == stats.content_total_length + ) + + +def _process_batch( + documents: list[Document], *, apply_changes: bool +) -> tuple[int, int, int]: + session_factory = get_sync_session_factory() + updated = 0 + complete = 0 + skipped = 0 + with session_factory() as db: + for document in documents: + job_result_id = str(document.current_job_result_id or "") + stats = _aggregate_statistics( + db, document_id=document.document_id, job_result_id=job_result_id + ) + index = db.scalar( + select(DocumentMapUnitIndex) + .where(DocumentMapUnitIndex.document_id == document.document_id) + .where(DocumentMapUnitIndex.job_result_id == job_result_id) + .with_for_update() + ) + if index is None or index.format_version != MAP_UNIT_INDEX_FORMAT_VERSION: + skipped += 1 + print(f"skip document={document.document_id} reason=missing_or_legacy_index") + db.rollback() + continue + if _is_complete(index, stats): + complete += 1 + db.rollback() + continue + if not apply_changes: + updated += 1 + db.rollback() + continue + db.execute( + update(DocumentMapUnitIndex) + .where(DocumentMapUnitIndex.id == index.id) + .values( + path_document_count=stats.path_document_count, + path_total_length=stats.path_total_length, + content_document_count=stats.content_document_count, + content_total_length=stats.content_total_length, + ) + ) + db.commit() + updated += 1 + return updated, complete, skipped + + +def main() -> None: + args = _build_parser().parse_args() + if args.apply == args.check: + raise SystemExit("choose exactly one of --apply or --check") + if args.batch_size <= 0: + raise SystemExit("--batch-size must be positive") + documents = _load_documents( + document_id=args.document_id.strip(), + user_id=args.user_id.strip(), + namespace=args.namespace.strip(), + ) + totals = [0, 0, 0] + for offset in range(0, len(documents), args.batch_size): + batch_totals = _process_batch( + documents[offset : offset + args.batch_size], apply_changes=args.apply + ) + totals = [left + right for left, right in zip(totals, batch_totals)] + action = "applied" if args.apply else "would_update" + print( + f"{action}={totals[0]} complete={totals[1]} skipped={totals[2]} " + f"documents={len(documents)}" + ) + + +if __name__ == "__main__": + main() diff --git a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md index 28bbc0ab..5ed4c662 100644 --- a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md +++ b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md @@ -483,7 +483,19 @@ trigger it. The runbook should: 3. run the read-only inventory/check command and record active/current revision counts; 4. run the resumable statistics backfill in bounded batches (with optional - document selection), monitoring database load and generation changes; + document selection), monitoring database load and generation changes. The + command is: + + ```bash + python /app/scripts/backfill_map_unit_statistics.py \ + --apply --batch-size 100 + ``` + + Use `--user-id`, `--namespace`, or `--document-id` to narrow a rehearsal. + `--check` is read-only and should return `would_update=0` before rollout. + This command only aggregates existing `document_map_units`; it does not + regenerate token rows or snapshots. Revisions with a missing or legacy + index remain on the existing full backfill path. 5. rerun the check until every retrieval-visible revision has a coherent v2 marker and no serving fallback is reported; 6. monitor semantic-parity probes, latency, errors, and timeouts. The reader's From aad2604be1de422658afe123645cb6e643c79b1c Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 04:04:53 +0800 Subject: [PATCH 19/35] fix: support runtime backfill script paths --- apps/api/scripts/backfill_map_unit_statistics.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/api/scripts/backfill_map_unit_statistics.py b/apps/api/scripts/backfill_map_unit_statistics.py index 94c7aff1..5dbcd6e2 100644 --- a/apps/api/scripts/backfill_map_unit_statistics.py +++ b/apps/api/scripts/backfill_map_unit_statistics.py @@ -19,7 +19,15 @@ def _bootstrap_python_path() -> None: api_root = Path(__file__).resolve().parents[1] - shared_root = api_root.parents[1] / "packages" / "shared-python" + candidate_roots = ( + api_root / "packages" / "shared-python", + api_root.parents[1] / "packages" / "shared-python", + ) + shared_root = next( + (path for path in candidate_roots if path.is_dir()), None + ) + if shared_root is None: + raise RuntimeError("Could not locate shared-python package") for path in (api_root, shared_root): value = os.fspath(path) if value not in sys.path: From 462e64bd717582365877c4df8b3e4de8c5fa0452 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 10:06:56 +0800 Subject: [PATCH 20/35] perf: reuse pinned revisions during discovery --- .../retrieval/search/map_unit_discovery.py | 67 +++++++++++++++++-- 1 file changed, 61 insertions(+), 6 deletions(-) diff --git a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py index 20427477..e8d3d6dd 100644 --- a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py +++ b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py @@ -244,6 +244,7 @@ async def map_unit_discovery( "channels": list(_MAP_SCORE_CHANNELS), "token_hashes": query_token_hashes, } + stage_started = time.monotonic() unit_result = await db.execute(text(unit_statement), params) unit_rows = [dict(row._mapping) for row in unit_result.all()] unit_rows = [ @@ -255,6 +256,11 @@ async def map_unit_discovery( exclude_sections=exclude_sections, ) ] + logger.info( + "retrieval map-unit stage=units seconds={:.3f} rows={}", + time.monotonic() - stage_started, + len(unit_rows), + ) if not unit_rows: return DiscoveryResult(status="discovery_done", payload={"fused_rows": []}) @@ -281,6 +287,7 @@ async def map_unit_discovery( ON scoped_units.map_unit_id = matching_tokens.map_unit_id """ ) + stage_started = time.monotonic() frequency_result = await db.execute( frequency_query, { @@ -294,7 +301,14 @@ async def map_unit_discovery( frequencies.setdefault((str(map_unit_id), str(channel)), {})[str(token)] = ( int(frequency) ) + logger.info( + "retrieval map-unit stage=frequencies seconds={:.3f} rows={} units={}", + time.monotonic() - stage_started, + sum(len(values) for values in frequencies.values()), + len(frequencies), + ) + stage_started = time.monotonic() index_result = await db.execute( text( ( @@ -344,14 +358,37 @@ async def map_unit_discovery( content_total_length, ) in index_result.all() ] + logger.info( + "retrieval map-unit stage=indexes seconds={:.3f} rows={}", + time.monotonic() - stage_started, + len(index_parts), + ) if is_unfiltered_scope: - revision_result = await db.execute( - text(cte + "SELECT DISTINCT document_id, job_result_id FROM scoped_units"), - params, + if revision_pins is not None: + # The route captures the active revision set before discovery. Reuse + # that immutable set instead of scanning scoped map units a second + # time just to derive the same revision keys. Missing index rows + # still fail the existing completeness check below. + expected_revisions = { + (str(document_id), str(job_result_id)) + for document_id, job_result_id in revision_pins.items() + } + revision_check_seconds = 0.0 + else: + stage_started = time.monotonic() + revision_result = await db.execute( + text(cte + "SELECT DISTINCT document_id, job_result_id FROM scoped_units"), + params, + ) + expected_revisions = { + (str(row[0]), str(row[1])) for row in revision_result.all() + } + revision_check_seconds = time.monotonic() - stage_started + logger.info( + "retrieval map-unit stage=revision-check seconds={:.3f} rows={}", + revision_check_seconds, + len(expected_revisions), ) - expected_revisions = { - (str(row[0]), str(row[1])) for row in revision_result.all() - } else: expected_revisions = { (str(row["document_id"]), str(row["job_result_id"])) @@ -455,6 +492,7 @@ async def map_unit_discovery( ] ) + stage_started = time.monotonic() path_stats = build_channel_bm25_stats( unit_rows=unit_rows, map_unit_id_field="map_unit_id", @@ -493,6 +531,11 @@ async def map_unit_discovery( else None ), ) + logger.info( + "retrieval map-unit stage=stats seconds={:.3f} units={}", + time.monotonic() - stage_started, + len(unit_rows), + ) corpus = PersistedScoreCorpus( units=[ @@ -510,7 +553,13 @@ async def map_unit_discovery( path_stats=path_stats, content_stats=content_stats, ) + stage_started = time.monotonic() scores_by_unit = score_persisted_corpus_many(corpus, [query]).get(query, {}) + logger.info( + "retrieval map-unit stage=scoring seconds={:.3f} units={}", + time.monotonic() - stage_started, + len(scores_by_unit), + ) rows_by_unit_id = {row["map_unit_id"]: row for row in unit_rows} ranked_unit_ids = sorted( @@ -519,6 +568,7 @@ async def map_unit_discovery( reverse=True, )[:top_k] + stage_started = time.monotonic() fused_rows = await _hydrate_winning_units( db, ranked_unit_ids=ranked_unit_ids, @@ -529,6 +579,11 @@ async def map_unit_discovery( exclude_sections=exclude_sections, revision_pins=revision_pins, ) + logger.info( + "retrieval map-unit stage=hydration seconds={:.3f} rows={}", + time.monotonic() - stage_started, + len(fused_rows), + ) if fused_rows: normalize_row_scores( fused_rows, From ff064550e7989ae303c8de7069f29ec7b382e80d Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 10:13:04 +0800 Subject: [PATCH 21/35] docs: record revision pin reuse benchmark --- ...rieval-heart-vessel-trace-optimization-plan.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md index 5ed4c662..fc509d88 100644 --- a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md +++ b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md @@ -194,6 +194,15 @@ the maximum score delta was `1.7e-5`. Warm p50 improved by about `24 ms` for no yet-established broad end-to-end latency win; production rollout still requires the same migration, backfill, and rollback checks below. +The current classic reader now reuses the immutable revision pins captured at +request start when validating an unfiltered scope. This removes a redundant +`SELECT DISTINCT` over scoped map units; requests without pins retain the +original query. On the restored dump, the removed validation query measured +approximately `0.35–0.6 s` before the change. Stage instrumentation shows the +remaining warm classic work is dominated by database reads and hydration, not +BM25 statistics or Python scoring. Result IDs, ordering, sources, evidence, +and score deltas remain within the existing `1e-4` parity tolerance. + The reference trace above is not representative of request mix. In the current baseline window, 196 successful v1 roots used `use_agentic=false` (classic), three used the default map-nav route (`use_agentic=null`), and one explicitly @@ -239,7 +248,11 @@ saturation or request-level memory usage. distinguishes corpora at or below `top_k` from larger corpora before route selection. The trace alone does not establish that an `EXISTS` rewrite or snapshot metadata would be faster. -6. The current production request mix is primarily classic retrieval. The +6. Classic discovery previously re-scanned scoped map units solely to derive + revision keys after the request had already captured revision pins. Reusing + those pins removes that duplicate read without changing the completeness + check; the no-pin path remains unchanged. +7. The current production request mix is primarily classic retrieval. The token-selective reader must therefore be exercised through both the classic and map-nav callers; a map-nav-only benchmark would not represent the dominant workload. From 32c3e09c81244156d620736e4dac429fb838e4fc Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 10:16:13 +0800 Subject: [PATCH 22/35] perf: read index metadata from revision pins --- ...al-heart-vessel-trace-optimization-plan.md | 5 +++ .../retrieval/search/map_unit_discovery.py | 32 +++++++++++++++---- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md index fc509d88..8301c4d5 100644 --- a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md +++ b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md @@ -252,6 +252,11 @@ saturation or request-level memory usage. revision keys after the request had already captured revision pins. Reusing those pins removes that duplicate read without changing the completeness check; the no-pin path remains unchanged. + The same pins now drive the unfiltered index-metadata lookup directly, + avoiding another scoped-unit CTE. On the restored dump this reduced the + warm index stage from roughly `0.16–0.8 s` to `0.02–0.04 s` for the sampled + namespace. The classic result parity gate still passes for the temporary + query set. 7. The current production request mix is primarily classic retrieval. The token-selective reader must therefore be exercised through both the classic and map-nav callers; a map-nav-only benchmark would not represent the diff --git a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py index e8d3d6dd..a39844bb 100644 --- a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py +++ b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py @@ -308,9 +308,29 @@ async def map_unit_discovery( len(frequencies), ) - stage_started = time.monotonic() - index_result = await db.execute( - text( + if is_unfiltered_scope and revision_pins is not None: + pinned_pairs = [ + (str(document_id).strip(), str(job_result_id).strip()) + for document_id, job_result_id in revision_pins.items() + if str(document_id).strip() and str(job_result_id).strip() + ] + pinned_values_sql = ", ".join( + f"(:_pin_document_{index}, :_pin_revision_{index})" + for index, _pair in enumerate(pinned_pairs) + ) + index_statement = f""" + SELECT indexes.average_idf_path, indexes.average_idf_content, + indexes.unit_count, indexes.format_version, + indexes.path_document_count, indexes.path_total_length, + indexes.content_document_count, indexes.content_total_length + FROM document_map_unit_indexes AS indexes + JOIN (VALUES {pinned_values_sql}) + AS scoped_revisions(document_id, job_result_id) + ON indexes.document_id = scoped_revisions.document_id + AND indexes.job_result_id = scoped_revisions.job_result_id + """ + else: + index_statement = ( ( cte if signal_paths @@ -333,9 +353,9 @@ async def map_unit_discovery( ON indexes.document_id = scoped_revisions.document_id AND indexes.job_result_id = scoped_revisions.job_result_id """ - ), - params, - ) + ) + stage_started = time.monotonic() + index_result = await db.execute(text(index_statement), params) index_parts = [ ( float(path_idf or 0.0), From d7f41fb83f000db2d6fb71afb540403d55b77730 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 10:33:00 +0800 Subject: [PATCH 23/35] test: cover pinned index metadata parity --- ...est_retrieval_classic_map_unit_contract.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py b/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py index 45fd8d17..388b8198 100644 --- a/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py +++ b/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py @@ -13,6 +13,7 @@ replace_document_revision_content, ) from shared.services.retrieval.publication_models import DocumentPublicationScope +from shared.services.retrieval.search.map_unit_discovery import map_unit_discovery from shared.services.retrieval.serving_generation import lock_namespace_generation from tests.support.contract_database import ContractDatabase from tests.support.retrieval_snapshot_support import contract_db_session @@ -264,6 +265,110 @@ def capture_unit_projection( assert projection_statements[:2] == ["token_selective", "legacy"] +async def test_unfiltered_revision_pins_reuse_index_metadata_without_scoped_revision_cte( + developer_api_client_factory: Callable[ + [], AbstractAsyncContextManager[AsyncClient] + ], +) -> None: + identifier = uuid4().hex[:8] + namespace = f"classic-pinned-index-{identifier}" + index_statements: list[str] = [] + + def capture_index_query( + _connection: Any, + _cursor: Any, + statement: str, + _parameters: Any, + _context: Any, + _executemany: bool, + ) -> None: + if ( + "document_map_unit_indexes" in statement + and "average_idf_path" in statement + ): + index_statements.append(statement) + + async with developer_api_client_factory(): + document = await _publish_document( + namespace=namespace, + source_file_name="pinned-index.pdf", + chunks=[ + { + "chunk_id": f"pinned-hit-{identifier}", + "type": "text", + "content": "pinned revision semantic marker", + "path": "pinned-index.pdf/Root/Section/hit", + "order": 1, + "metadata": {}, + }, + { + "chunk_id": f"pinned-filler-{identifier}", + "type": "text", + "content": "unrelated filler", + "path": "pinned-index.pdf/Root/Section/filler", + "order": 2, + "metadata": {}, + }, + ], + ) + + def result_signature(result: Any) -> list[tuple[Any, ...]]: + rows = list(result.payload.get("fused_rows") or []) + return [ + ( + row.get("chunk_id"), + row.get("document_id"), + row.get("job_result_id"), + row.get("section_path"), + row.get("score"), + row.get("discovery_score"), + row.get("content"), + ) + for row in rows + ] + + event.listen(Engine, "before_cursor_execute", capture_index_query) + try: + async with contract_db_session() as db: + unpinned = await map_unit_discovery( + db, + user_id=_USER_ID, + namespace=namespace, + query="pinned revision semantic marker", + top_k=10, + exclude_document_ids=[], + exclude_sections=[], + revision_pins=None, + ) + + index_statements.clear() + + async with contract_db_session() as db: + pinned = await map_unit_discovery( + db, + user_id=_USER_ID, + namespace=namespace, + query="pinned revision semantic marker", + top_k=10, + exclude_document_ids=[], + exclude_sections=[], + revision_pins={ + document["document_id"]: document["job_result_id"] + }, + ) + finally: + event.remove(Engine, "before_cursor_execute", capture_index_query) + + assert result_signature(pinned) == result_signature(unpinned) + assert index_statements + assert all("JOIN (VALUES" in statement for statement in index_statements) + assert all("scoped_units AS" not in statement for statement in index_statements) + assert all( + "SELECT DISTINCT document_id, job_result_id" not in statement + for statement in index_statements + ) + + async def test_classic_route_falls_back_for_v1_index_with_excluded_document( developer_api_client_factory: Callable[ [], AbstractAsyncContextManager[AsyncClient] From 8dc0b482d31eb43e5dca8364304b125bdff3816f Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 10:38:52 +0800 Subject: [PATCH 24/35] docs: document local agentic database ssl settings --- .../retrieval-heart-vessel-trace-optimization-plan.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md index 8301c4d5..6c3fbcfa 100644 --- a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md +++ b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md @@ -203,6 +203,12 @@ remaining warm classic work is dominated by database reads and hydration, not BM25 statistics or Python scoring. Result IDs, ordering, sources, evidence, and score deltas remain within the existing `1e-4` parity tolerance. +For the local restored PostgreSQL (which does not enable SSL), agentic smoke +must set `DB_SSL_MODE=disable` for the global async engine and provide the +plain libpq form through `KNOWHERE_DATABASE_URL` for the synchronous map-nav +reader. Without these local-only settings, final reference hydration attempts +an SSL upgrade and fails even though the database is healthy. + The reference trace above is not representative of request mix. In the current baseline window, 196 successful v1 roots used `use_agentic=false` (classic), three used the default map-nav route (`use_agentic=null`), and one explicitly From 93dac29a210570e3b45d1de495863acfa6a14e21 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 10:52:34 +0800 Subject: [PATCH 25/35] perf: skip redundant map section scope scan --- .../services/retrieval/nav/nav_knowhere.py | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py b/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py index 3c1365d8..8ea5b5c2 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py @@ -389,24 +389,32 @@ def load_persisted_score_corpus( for document_id, section_ids in allowed_by_document.items() for section_id in section_ids } - cur.execute( - "SELECT sections.document_id, sections.job_result_id, count(*) " - "FROM document_sections AS sections " - f"JOIN (VALUES {values_sql}) AS revisions(document_id, job_result_id) " - "ON sections.document_id = revisions.document_id " - "AND sections.job_result_id = revisions.job_result_id " - "GROUP BY sections.document_id, sections.job_result_id", - revision_params, - ) - section_counts = { - (str(document_id), str(job_result_id)): int(count) - for document_id, job_result_id, count in cur.fetchall() - } - has_complete_section_scope: bool = all( - len(allowed_by_document.get(document_id, set())) - == section_counts.get((document_id, job_result_id), 0) - for document_id, job_result_id in revisions - ) + if not self._excluded_sections: + # The lazy snapshot was built from the complete pinned + # namespace and no section filters were requested. Avoid a + # second GROUP BY over document_sections just to prove the + # same fact; filtered snapshots retain the exact validation + # query below. + has_complete_section_scope = True + else: + cur.execute( + "SELECT sections.document_id, sections.job_result_id, count(*) " + "FROM document_sections AS sections " + f"JOIN (VALUES {values_sql}) AS revisions(document_id, job_result_id) " + "ON sections.document_id = revisions.document_id " + "AND sections.job_result_id = revisions.job_result_id " + "GROUP BY sections.document_id, sections.job_result_id", + revision_params, + ) + section_counts = { + (str(document_id), str(job_result_id)): int(count) + for document_id, job_result_id, count in cur.fetchall() + } + has_complete_section_scope = all( + len(allowed_by_document.get(document_id, set())) + == section_counts.get((document_id, job_result_id), 0) + for document_id, job_result_id in revisions + ) all_unit_rows = self._score_unit_rows_cache.get(revision_key) if has_complete_section_scope and query_token_hashes: stage_started = time.perf_counter() From db6ae648507323e21e38f5ab1bde5e1d53ed9044 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 10:56:25 +0800 Subject: [PATCH 26/35] perf: defer filtered map scope allocation --- .../services/retrieval/nav/nav_knowhere.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py b/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py index 8ea5b5c2..25d5fb5f 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py @@ -380,15 +380,7 @@ def load_persisted_score_corpus( average_idf_content, ) - allowed_by_document = { - str(document_id): {str(section_id) for section_id in section_ids} - for document_id, section_ids in allowed_section_ids_by_document.items() - } - allowed_pairs_set = { - (document_id, section_id) - for document_id, section_ids in allowed_by_document.items() - for section_id in section_ids - } + allowed_pairs_set: set[tuple[str, str]] = set() if not self._excluded_sections: # The lazy snapshot was built from the complete pinned # namespace and no section filters were requested. Avoid a @@ -397,6 +389,15 @@ def load_persisted_score_corpus( # query below. has_complete_section_scope = True else: + allowed_by_document = { + str(document_id): {str(section_id) for section_id in section_ids} + for document_id, section_ids in allowed_section_ids_by_document.items() + } + allowed_pairs_set = { + (document_id, section_id) + for document_id, section_ids in allowed_by_document.items() + for section_id in section_ids + } cur.execute( "SELECT sections.document_id, sections.job_result_id, count(*) " "FROM document_sections AS sections " From ff33ba58baaf2e0cfd3c7b851c40fecc8ea8efee Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 10:59:52 +0800 Subject: [PATCH 27/35] fix: preserve filtered map scope semantics --- .../services/retrieval/nav/nav_knowhere.py | 63 ++++++++----------- 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py b/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py index 25d5fb5f..cd569631 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py @@ -380,42 +380,33 @@ def load_persisted_score_corpus( average_idf_content, ) - allowed_pairs_set: set[tuple[str, str]] = set() - if not self._excluded_sections: - # The lazy snapshot was built from the complete pinned - # namespace and no section filters were requested. Avoid a - # second GROUP BY over document_sections just to prove the - # same fact; filtered snapshots retain the exact validation - # query below. - has_complete_section_scope = True - else: - allowed_by_document = { - str(document_id): {str(section_id) for section_id in section_ids} - for document_id, section_ids in allowed_section_ids_by_document.items() - } - allowed_pairs_set = { - (document_id, section_id) - for document_id, section_ids in allowed_by_document.items() - for section_id in section_ids - } - cur.execute( - "SELECT sections.document_id, sections.job_result_id, count(*) " - "FROM document_sections AS sections " - f"JOIN (VALUES {values_sql}) AS revisions(document_id, job_result_id) " - "ON sections.document_id = revisions.document_id " - "AND sections.job_result_id = revisions.job_result_id " - "GROUP BY sections.document_id, sections.job_result_id", - revision_params, - ) - section_counts = { - (str(document_id), str(job_result_id)): int(count) - for document_id, job_result_id, count in cur.fetchall() - } - has_complete_section_scope = all( - len(allowed_by_document.get(document_id, set())) - == section_counts.get((document_id, job_result_id), 0) - for document_id, job_result_id in revisions - ) + allowed_by_document = { + str(document_id): {str(section_id) for section_id in section_ids} + for document_id, section_ids in allowed_section_ids_by_document.items() + } + allowed_pairs_set = { + (document_id, section_id) + for document_id, section_ids in allowed_by_document.items() + for section_id in section_ids + } + cur.execute( + "SELECT sections.document_id, sections.job_result_id, count(*) " + "FROM document_sections AS sections " + f"JOIN (VALUES {values_sql}) AS revisions(document_id, job_result_id) " + "ON sections.document_id = revisions.document_id " + "AND sections.job_result_id = revisions.job_result_id " + "GROUP BY sections.document_id, sections.job_result_id", + revision_params, + ) + section_counts = { + (str(document_id), str(job_result_id)): int(count) + for document_id, job_result_id, count in cur.fetchall() + } + has_complete_section_scope = all( + len(allowed_by_document.get(document_id, set())) + == section_counts.get((document_id, job_result_id), 0) + for document_id, job_result_id in revisions + ) all_unit_rows = self._score_unit_rows_cache.get(revision_key) if has_complete_section_scope and query_token_hashes: stage_started = time.perf_counter() From 689dde7cafea782b3c90913eaa4ddcc9e64af90d Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 11:03:18 +0800 Subject: [PATCH 28/35] fix: format mapnav resource metrics --- .../shared/services/retrieval/execution/routes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/shared-python/shared/services/retrieval/execution/routes.py b/packages/shared-python/shared/services/retrieval/execution/routes.py index 6e0bee4a..6ed99293 100644 --- a/packages/shared-python/shared/services/retrieval/execution/routes.py +++ b/packages/shared-python/shared/services/retrieval/execution/routes.py @@ -364,8 +364,8 @@ async def _run_mapnav_route( completion_detail = f"chunks | evidence={len(evidence_text)} chars | router=mapnav" process_finished = resource.getrusage(resource.RUSAGE_SELF) logger.info( - "retrieval mapnav stage=process_resources cpu_seconds=%.3f " - "process_max_rss_kb=%d", + "retrieval mapnav stage=process_resources cpu_seconds={:.3f} " + "process_max_rss_kb={}", (process_finished.ru_utime + process_finished.ru_stime) - (process_started.ru_utime + process_started.ru_stime), int(process_finished.ru_maxrss), From abf0d2836a034318b060e167d10f91b65e5fceef Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 14:23:58 +0800 Subject: [PATCH 29/35] fix: harden retrieval backfill readiness checks --- ...r_token_leading_map_unit_covering_index.py | 62 +++++++++ apps/api/scripts/backfill_map_unit_indexes.py | 12 +- .../scripts/backfill_map_unit_statistics.py | 20 ++- ...test_backfill_map_unit_indexes_contract.py | 127 ++++++++++++++++++ .../tests/migrations/test_schema_contract.py | 125 +++++++++++++++-- 5 files changed, 334 insertions(+), 12 deletions(-) create mode 100644 apps/api/alembic/versions/c2d3e4f5a6b7_repair_token_leading_map_unit_covering_index.py diff --git a/apps/api/alembic/versions/c2d3e4f5a6b7_repair_token_leading_map_unit_covering_index.py b/apps/api/alembic/versions/c2d3e4f5a6b7_repair_token_leading_map_unit_covering_index.py new file mode 100644 index 00000000..241f1193 --- /dev/null +++ b/apps/api/alembic/versions/c2d3e4f5a6b7_repair_token_leading_map_unit_covering_index.py @@ -0,0 +1,62 @@ +"""Repair a missing or invalid token-leading map-unit covering index.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op +from sqlalchemy import text + + +revision: str = "c2d3e4f5a6b7" +down_revision: str | None = "b1c2d3e4f5a6" +branch_labels: Sequence[str] | None = None +depends_on: Sequence[str] | None = None + +_INDEX_NAME = "idx_document_map_unit_tokens_token_lookup" +_INDEX_COLUMNS = "(channel, token_hash, map_unit_id) INCLUDE (token, frequency)" + + +def _is_index_ready() -> bool: + """Return whether the current schema contains a usable covering index.""" + is_ready = op.get_bind().execute( + text( + "SELECT indexes.indisvalid AND indexes.indisready " + "FROM pg_index AS indexes " + "JOIN pg_class AS classes ON classes.oid = indexes.indexrelid " + "JOIN pg_namespace AS namespaces " + "ON namespaces.oid = classes.relnamespace " + "WHERE namespaces.nspname = current_schema() " + "AND classes.relname = :index_name" + ), + {"index_name": _INDEX_NAME}, + ).scalar_one_or_none() + return bool(is_ready) + + +def _repair_index(*, concurrently: bool) -> None: + """Replace a missing or unusable index using the allowed DDL mode.""" + if _is_index_ready(): + return + concurrent_clause: str = "CONCURRENTLY " if concurrently else "" + op.execute(f"DROP INDEX {concurrent_clause}IF EXISTS {_INDEX_NAME}") + op.execute( + f"CREATE INDEX {concurrent_clause}{_INDEX_NAME} " + f"ON document_map_unit_tokens {_INDEX_COLUMNS}" + ) + + +def upgrade() -> None: + """Ensure the additive covering index exists and is usable.""" + uses_external_transaction: bool = bool( + op.get_context().opts.get("knowhere_external_transaction", False) + ) + if uses_external_transaction: + _repair_index(concurrently=False) + return + with op.get_context().autocommit_block(): + _repair_index(concurrently=True) + + +def downgrade() -> None: + """Keep the index owned by the preceding additive migration.""" diff --git a/apps/api/scripts/backfill_map_unit_indexes.py b/apps/api/scripts/backfill_map_unit_indexes.py index 98678c7f..48d9bef1 100644 --- a/apps/api/scripts/backfill_map_unit_indexes.py +++ b/apps/api/scripts/backfill_map_unit_indexes.py @@ -155,7 +155,11 @@ class NamespaceFallbackReport: @property def ready(self) -> bool: - return not self.would_hit_snapshot_fallback and not self.scoring_incomplete + return ( + not self.would_hit_snapshot_fallback + and not self.scoring_incomplete + and self.missing_revision_manifest == 0 + ) def check_fallback_readiness(*, document_id: str = "") -> list[NamespaceFallbackReport]: @@ -321,7 +325,11 @@ def check_fallback_readiness(*, document_id: str = "") -> list[NamespaceFallback would_hit_snapshot_fallback = ( snapshot_status != "ok" or missing_from_snapshot > 0 ) - scoring_incomplete = missing_map_index > 0 or suspicious_zero_idf > 0 + # A zero average IDF is mathematically valid, notably for a + # two-unit corpus where every token appears in exactly one unit. + # Keep the count as diagnostic output, but readiness is determined + # by the format marker and required persisted statistics above. + scoring_incomplete = missing_map_index > 0 reports.append( NamespaceFallbackReport( user_id=user_id, diff --git a/apps/api/scripts/backfill_map_unit_statistics.py b/apps/api/scripts/backfill_map_unit_statistics.py index 5dbcd6e2..a4ec4d28 100644 --- a/apps/api/scripts/backfill_map_unit_statistics.py +++ b/apps/api/scripts/backfill_map_unit_statistics.py @@ -130,6 +130,13 @@ def _is_complete(index: DocumentMapUnitIndex | None, stats: RevisionStatistics) ) +def _is_check_ready( + *, would_update: int, complete: int, skipped: int, documents: int +) -> bool: + """Return whether a read-only inventory proves every document is ready.""" + return would_update == 0 and skipped == 0 and complete == documents + + def _process_batch( documents: list[Document], *, apply_changes: bool ) -> tuple[int, int, int]: @@ -143,12 +150,14 @@ def _process_batch( stats = _aggregate_statistics( db, document_id=document.document_id, job_result_id=job_result_id ) - index = db.scalar( + index_statement = ( select(DocumentMapUnitIndex) .where(DocumentMapUnitIndex.document_id == document.document_id) .where(DocumentMapUnitIndex.job_result_id == job_result_id) - .with_for_update() ) + if apply_changes: + index_statement = index_statement.with_for_update() + index = db.scalar(index_statement) if index is None or index.format_version != MAP_UNIT_INDEX_FORMAT_VERSION: skipped += 1 print(f"skip document={document.document_id} reason=missing_or_legacy_index") @@ -199,6 +208,13 @@ def main() -> None: f"{action}={totals[0]} complete={totals[1]} skipped={totals[2]} " f"documents={len(documents)}" ) + if args.check and not _is_check_ready( + would_update=totals[0], + complete=totals[1], + skipped=totals[2], + documents=len(documents), + ): + raise SystemExit(1) if __name__ == "__main__": diff --git a/apps/api/tests/contract/test_backfill_map_unit_indexes_contract.py b/apps/api/tests/contract/test_backfill_map_unit_indexes_contract.py index 72c61f91..3f61f463 100644 --- a/apps/api/tests/contract/test_backfill_map_unit_indexes_contract.py +++ b/apps/api/tests/contract/test_backfill_map_unit_indexes_contract.py @@ -1,6 +1,10 @@ from __future__ import annotations from pathlib import Path +from types import SimpleNamespace +from typing import Any, cast + +from sqlalchemy.dialects import postgresql def test_backfill_script_resolves_shared_package_from_runtime_image_layout( @@ -26,3 +30,126 @@ def test_backfill_script_resolves_shared_package_from_source_checkout_layout( shared_root.mkdir(parents=True) assert _resolve_shared_root(api_root) == shared_root + + +def test_statistics_backfill_aggregates_positive_lengths_per_channel() -> None: + from scripts.backfill_map_unit_statistics import ( + RevisionStatistics, + _aggregate_statistics, + ) + + class AggregateResult: + def one(self) -> SimpleNamespace: + return SimpleNamespace( + path_document_count=2, + path_total_length=9, + content_document_count=1, + content_total_length=7, + ) + + class AggregateSession: + statement_sql: str = "" + + def execute(self, statement: Any) -> AggregateResult: + self.statement_sql = str( + statement.compile( + dialect=postgresql.dialect(), + compile_kwargs={"literal_binds": True}, + ) + ) + return AggregateResult() + + session = AggregateSession() + statistics = _aggregate_statistics( + cast(Any, session), document_id="doc_1", job_result_id="result_1" + ) + + assert statistics == RevisionStatistics( + path_document_count=2, + path_total_length=9, + content_document_count=1, + content_total_length=7, + ) + assert "path_token_count > 0" in session.statement_sql + assert "content_token_count > 0" in session.statement_sql + + +def test_statistics_backfill_readiness_requires_every_document_complete() -> None: + from scripts.backfill_map_unit_statistics import _is_check_ready + + assert _is_check_ready( + would_update=0, complete=4, skipped=0, documents=4 + ) + assert not _is_check_ready( + would_update=1, complete=3, skipped=0, documents=4 + ) + assert not _is_check_ready( + would_update=0, complete=3, skipped=1, documents=4 + ) + + +def test_statistics_backfill_completion_rejects_missing_or_legacy_indexes() -> None: + from scripts.backfill_map_unit_statistics import RevisionStatistics, _is_complete + + statistics = RevisionStatistics( + path_document_count=1, + path_total_length=2, + content_document_count=1, + content_total_length=3, + ) + legacy_index = SimpleNamespace( + format_version=1, + path_document_count=1, + path_total_length=2, + content_document_count=1, + content_total_length=3, + ) + complete_index = SimpleNamespace( + format_version=2, + path_document_count=1, + path_total_length=2, + content_document_count=1, + content_total_length=3, + ) + + assert not _is_complete(None, statistics) + assert not _is_complete(cast(Any, legacy_index), statistics) + assert _is_complete(cast(Any, complete_index), statistics) + + +def test_full_backfill_readiness_requires_revision_manifests() -> None: + from scripts.backfill_map_unit_indexes import NamespaceFallbackReport + + report = NamespaceFallbackReport( + user_id="user_1", + namespace="default", + active_docs=1, + snapshot_status="ok", + missing_from_snapshot=0, + missing_map_index=0, + missing_revision_manifest=1, + suspicious_zero_idf=0, + would_hit_snapshot_fallback=False, + scoring_incomplete=False, + ) + + assert not report.ready + + +def test_full_backfill_readiness_allows_mathematically_valid_zero_idf() -> None: + from scripts.backfill_map_unit_indexes import NamespaceFallbackReport + + report = NamespaceFallbackReport( + user_id="user_1", + namespace="default", + active_docs=1, + snapshot_status="ok", + missing_from_snapshot=0, + missing_map_index=0, + missing_revision_manifest=0, + suspicious_zero_idf=1, + would_hit_snapshot_fallback=False, + scoring_incomplete=False, + ) + + assert report.ready diff --git a/apps/api/tests/migrations/test_schema_contract.py b/apps/api/tests/migrations/test_schema_contract.py index 37ea8d65..8b4dc85d 100644 --- a/apps/api/tests/migrations/test_schema_contract.py +++ b/apps/api/tests/migrations/test_schema_contract.py @@ -49,6 +49,11 @@ def _upgrade_to_snapshot_parents(*, engine: Engine) -> None: command.upgrade(config, "fbe1c2d3e4f5") +def _upgrade_to_channel_statistics(*, engine: Engine) -> None: + config = _build_alembic_command_config(engine=engine) + command.upgrade(config, "b1c2d3e4f5a6") + + def _insert_job( connection: Connection, *, @@ -259,21 +264,125 @@ def test_should_create_token_leading_map_unit_covering_index( migrated_head_engine: Engine, ) -> None: with migrated_head_engine.begin() as connection: - index_definition = connection.execute( + index_row = connection.execute( text( """ - SELECT indexdef - FROM pg_indexes - WHERE schemaname = current_schema() - AND tablename = 'document_map_unit_tokens' - AND indexname = 'idx_document_map_unit_tokens_token_lookup' + SELECT pg_get_indexdef(indexes.indexrelid), + indexes.indisvalid, + indexes.indisready + FROM pg_index AS indexes + JOIN pg_class AS classes ON classes.oid = indexes.indexrelid + JOIN pg_namespace AS namespaces + ON namespaces.oid = classes.relnamespace + WHERE namespaces.nspname = current_schema() + AND classes.relname = 'idx_document_map_unit_tokens_token_lookup' """ ) - ).scalar_one() + ).one() - definition = str(index_definition) + definition = str(index_row[0]) assert "(channel, token_hash, map_unit_id)" in definition assert "INCLUDE (token, frequency)" in definition + assert index_row[1] is True + assert index_row[2] is True + + +def test_should_repair_a_missing_token_leading_map_unit_covering_index( + alembic_engine: Engine, +) -> None: + _upgrade_to_channel_statistics(engine=alembic_engine) + with alembic_engine.begin() as connection: + connection.execute( + text("DROP INDEX idx_document_map_unit_tokens_token_lookup") + ) + + _upgrade_to_heads(engine=alembic_engine) + + with alembic_engine.begin() as connection: + index_state = connection.execute( + text( + """ + SELECT indexes.indisvalid, indexes.indisready + FROM pg_index AS indexes + JOIN pg_class AS classes ON classes.oid = indexes.indexrelid + JOIN pg_namespace AS namespaces + ON namespaces.oid = classes.relnamespace + WHERE namespaces.nspname = current_schema() + AND classes.relname = 'idx_document_map_unit_tokens_token_lookup' + """ + ) + ).one() + + assert index_state[0] is True + assert index_state[1] is True + + +def test_should_repair_a_missing_covering_index_with_a_caller_owned_connection( + alembic_engine: Engine, +) -> None: + _upgrade_to_channel_statistics(engine=alembic_engine) + with alembic_engine.begin() as connection: + connection.execute( + text("DROP INDEX idx_document_map_unit_tokens_token_lookup") + ) + + _upgrade_to_heads_with_external_connection(engine=alembic_engine) + + with alembic_engine.begin() as connection: + index_state = connection.execute( + text( + """ + SELECT indexes.indisvalid, indexes.indisready + FROM pg_index AS indexes + JOIN pg_class AS classes ON classes.oid = indexes.indexrelid + JOIN pg_namespace AS namespaces + ON namespaces.oid = classes.relnamespace + WHERE namespaces.nspname = current_schema() + AND classes.relname = 'idx_document_map_unit_tokens_token_lookup' + """ + ) + ).one() + + assert index_state[0] is True + assert index_state[1] is True + + +def test_should_repair_an_invalid_token_leading_map_unit_covering_index( + alembic_engine: Engine, +) -> None: + _upgrade_to_channel_statistics(engine=alembic_engine) + with alembic_engine.begin() as connection: + connection.execute( + text( + """ + UPDATE pg_index + SET indisvalid = FALSE, + indisready = FALSE + WHERE indexrelid = + 'idx_document_map_unit_tokens_token_lookup'::regclass + """ + ) + ) + + _upgrade_to_heads(engine=alembic_engine) + + with alembic_engine.begin() as connection: + index_state = connection.execute( + text( + """ + SELECT indexes.indisvalid, indexes.indisready + FROM pg_index AS indexes + JOIN pg_class AS classes ON classes.oid = indexes.indexrelid + JOIN pg_namespace AS namespaces + ON namespaces.oid = classes.relnamespace + WHERE namespaces.nspname = current_schema() + AND classes.relname = 'idx_document_map_unit_tokens_token_lookup' + """ + ) + ).one() + + assert index_state[0] is True + assert index_state[1] is True def test_should_add_per_channel_map_unit_bm25_statistics( From b32c4a55e42f3184d82b070f4721d41153eccd95 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 14:24:09 +0800 Subject: [PATCH 30/35] docs: add retrieval serving index rollout runbook --- deploy/ecs/README.md | 39 +- ...kfill-serving-index-statistics-in-place.md | 18 +- ...al-heart-vessel-trace-optimization-plan.md | 4 +- ...retrieval-serving-index-rollout-runbook.md | 403 ++++++++++++++++++ 4 files changed, 442 insertions(+), 22 deletions(-) create mode 100644 docs/design/retrieval-serving-index-rollout-runbook.md diff --git a/deploy/ecs/README.md b/deploy/ecs/README.md index 7b7d6170..37a8c572 100644 --- a/deploy/ecs/README.md +++ b/deploy/ecs/README.md @@ -77,31 +77,44 @@ services. It does not create or delete AWS resources. ## Required post-deploy backfill -The map-nav lexical-index migration creates the derived index tables, but it does -not rebuild indexes for revisions that already exist. Until those revisions are -backfilled, retrieval remains quality-preserving but uses the legacy scoring -path. Every release containing the map-nav index change must include the -following DevOps action in its release notification. +Follow the complete +[`retrieval-serving-index-rollout-runbook.md`](../../docs/design/retrieval-serving-index-rollout-runbook.md) +for schema verification, statistics maintenance, readiness gates, parity, +monitoring, pause/resume, and rollback. + +The additive migration does not populate the four per-channel statistics for +existing revisions. Until those revisions are ready, retrieval remains +quality-preserving but uses the legacy scoring path. Every release containing +this change must include the following DevOps action in its release +notification. Run the commands as a one-off container using the newly deployed API image and the production database secret. Do not run them inside the long-lived API task. ```bash -# Read-only inventory -python /app/scripts/backfill_map_unit_indexes.py +# Read-only statistics inventory, after migration +python /app/scripts/backfill_map_unit_statistics.py --check --batch-size 100 # Optional canary: apply one affected document first -python /app/scripts/backfill_map_unit_indexes.py \ +python /app/scripts/backfill_map_unit_statistics.py \ --document-id \ + --batch-size 100 \ --apply -# Apply to all current document revisions -python /app/scripts/backfill_map_unit_indexes.py --apply +# Apply statistics to complete format-v2 indexes +python /app/scripts/backfill_map_unit_statistics.py --apply --batch-size 100 + +# Final statistics and full serving-readiness checks +python /app/scripts/backfill_map_unit_statistics.py --check --batch-size 100 +python /app/scripts/backfill_map_unit_indexes.py --check ``` -The script commits each document revision independently and is safe to rerun. -Verify the canary retrieval before starting the full apply. New or republished -documents build their index automatically during publication. +The statistics script commits each document revision independently and is safe +to rerun. Missing or legacy indexes reported as `skipped` require the existing +full `backfill_map_unit_indexes.py --apply --document-id ` path. +Run only one maintenance process at a time. Verify the canary retrieval before +starting the full apply. New or republished documents build their index +automatically during publication. ## Manual staging availability diff --git a/docs/adr/0010-backfill-serving-index-statistics-in-place.md b/docs/adr/0010-backfill-serving-index-statistics-in-place.md index ec499205..afac7749 100644 --- a/docs/adr/0010-backfill-serving-index-statistics-in-place.md +++ b/docs/adr/0010-backfill-serving-index-statistics-in-place.md @@ -2,11 +2,13 @@ **Status: accepted.** Existing retrieval-visible data will be maintained by backfilling only the current revision of each active Document. Complete serving -indexes receive the new per-channel BM25 statistics through the existing -per-document `apps/api/scripts/backfill_map_unit_indexes.py --apply` rebuild; -missing or incomplete indexes use the existing full rebuild. v1 and v2 are -internal read paths under the same Retrieval contract, so each revision commits -independently and the optimized reader requires the new serving-index format -marker before it can be selected. An interrupted or failed backfill therefore -continues using the exact legacy reader, limiting database load and avoiding -unnecessary regeneration of correct token rows and snapshots. +indexes receive the new per-channel BM25 statistics through the statistics-only +`apps/api/scripts/backfill_map_unit_statistics.py --apply` command. It aggregates +existing map units and does not regenerate token rows, manifests, or namespace +snapshots. Missing or legacy indexes use the existing full +`apps/api/scripts/backfill_map_unit_indexes.py --apply --document-id ...` +rebuild. v1 and v2 are internal read paths under the same Retrieval contract, +so each revision commits independently and the optimized reader requires the +new serving-index format marker before it can be selected. An interrupted or +failed backfill therefore continues using the exact legacy reader, limiting +database load and avoiding unnecessary regeneration of correct data. diff --git a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md index 6c3fbcfa..b6c521d7 100644 --- a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md +++ b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md @@ -496,7 +496,9 @@ the exact legacy reader. ### DevOps handoff DevOps runs the maintenance explicitly; API startup and user requests never -trigger it. The runbook should: +trigger it. The executable rollout procedure is documented in +[`retrieval-serving-index-rollout-runbook.md`](retrieval-serving-index-rollout-runbook.md). +The runbook: 1. deploy the additive migration and verify the reader still falls back safely; Build the new index without dropping the existing lookup index, and monitor diff --git a/docs/design/retrieval-serving-index-rollout-runbook.md b/docs/design/retrieval-serving-index-rollout-runbook.md new file mode 100644 index 00000000..04112a18 --- /dev/null +++ b/docs/design/retrieval-serving-index-rollout-runbook.md @@ -0,0 +1,403 @@ +# Retrieval Serving Index Rollout Runbook + +## Scope + +This runbook rolls out the retrieval serving-index performance changes without +changing retrieval semantics. It covers the additive schema migration, the +existing-data statistics backfill, readiness checks, retrieval parity probes, +monitoring, pause/resume, and application rollback. + +This rollout does not use a runtime feature flag. The reader selects the +optimized path only when every revision required by a request has a coherent +format-v2 index and all four channel statistics. Otherwise it uses the existing +legacy reader automatically. + +Planner, Harvest, Control, prompts, BM25 formulas, and citation-selection +semantics are outside this rollout. + +## Preconditions + +- Deploy only a build produced from the reviewed retrieval optimization branch. +- Confirm the target is the intended production environment before every + command. +- Use the API image for Alembic and maintenance commands. It contains: + - `/app/alembic` + - `/app/scripts/backfill_map_unit_statistics.py` + - `/app/scripts/backfill_map_unit_indexes.py` +- Confirm database backups and the normal application-version rollback path are + available. +- Record current retrieval p50/p95, errors, timeouts, and the output of the + temporary two-or-three-query quality set before deployment. +- Check free database disk space. The token-leading index is additive and the + existing indexes must remain in place. + +## Local acceptance evidence + +The production-shaped local restore used for final verification contained: + +- 2,996 documents; +- 125,835 sections; +- 170,280 chunks; +- 107,032 map units; +- 12,423,317 map-unit token rows; +- 644 active documents in the benchmark namespace. + +Final readiness on that namespace: + +```text +alembic head: c2d3e4f5a6b7 +covering index: idx_document_map_unit_tokens_token_lookup +statistics check: would_update=0 complete=644 skipped=0 documents=644 +coherent current indexes: 644/644 +``` + +Classic parity queries preserved chunk IDs, ordering, sources, and evidence. +The maximum observed score delta was below the accepted `1e-4` tolerance. +The v1 and v2 classic entry points returned the same chunk IDs and evidence +hash. A complete map-nav smoke returned `stop_reason=completed`, 23 result rows, +and 27 referenced chunks. + +These figures describe the local restored copy. They are not substitutes for +the production checks below. + +## Phase 1: Preflight inventory on the existing schema + +Before migration, record active/current revision counts using only the existing +schema. Do not run the statistics checker yet because it reads columns added by +revision `b1c2d3e4f5a6`. + +```sql +SELECT count(*) AS active_current_documents +FROM documents +WHERE status = 'active' + AND current_job_result_id IS NOT NULL; + +SELECT count(*) AS current_map_indexes +FROM documents +JOIN document_map_unit_indexes AS indexes + ON indexes.document_id = documents.document_id + AND indexes.job_result_id = documents.current_job_result_id +WHERE documents.status = 'active' + AND documents.current_job_result_id IS NOT NULL; +``` + +Record both counts and investigate any pre-existing difference. This inventory +does not determine format-v2 readiness. + +## Phase 2: Apply the additive migrations + +The production release workflow runs migrations before updating ECS services. +Record the workflow job URL and output. For a manual rehearsal, run Alembic from +the API image: + +```bash +cd /app +python -m alembic upgrade heads +python -m alembic current +``` + +The expected head for this rollout is: + +```text +c2d3e4f5a6b7 +``` + +The three relevant additive migrations are: + +1. `a0b1c2d3e4f5`: creates + `idx_document_map_unit_tokens_token_lookup` on + `(channel, token_hash, map_unit_id) INCLUDE (token, frequency)`; +2. `b1c2d3e4f5a6`: adds nullable path/content document-count and total-length + columns to `document_map_unit_indexes`; +3. `c2d3e4f5a6b7`: repairs the covering index if it is missing or PostgreSQL + reports `indisvalid=false` or `indisready=false` after an interrupted build. + +The index migration uses `CREATE INDEX CONCURRENTLY` in the normal Alembic +execution path. Monitor lock waits, database CPU, I/O, replication lag, and +free disk space while it runs. Do not drop either pre-existing map-unit-token +index during this rollout. + +Verify the schema: + +```sql +SELECT version_num FROM alembic_version; + +SELECT + classes.relname AS index_name, + indexes.indisvalid, + indexes.indisready, + pg_get_indexdef(indexes.indexrelid) AS index_definition +FROM pg_index AS indexes +JOIN pg_class AS classes ON classes.oid = indexes.indexrelid +WHERE classes.relname = 'idx_document_map_unit_tokens_token_lookup'; + +SELECT column_name +FROM information_schema.columns +WHERE table_name = 'document_map_unit_indexes' + AND column_name IN ( + 'path_document_count', + 'path_total_length', + 'content_document_count', + 'content_total_length' + ) +ORDER BY column_name; +``` + +The covering index must exist with `indisvalid=true`, `indisready=true`, and +the expected key/include columns. A failed concurrent build can leave an +invalid index. If either flag is false, stop the rollout, drop only that invalid +index with `DROP INDEX CONCURRENTLY`, and rerun the migration. + +## Phase 3: Deploy the application build + +Deploy the application after the additive migrations finish. New publications +will write coherent format-v2 statistics. Existing incomplete revisions remain +on the legacy reader until maintenance completes. + +Immediately verify: + +- API health checks pass; +- no migration or model-loading error appears in API logs; +- classic and map-nav requests still complete; +- incomplete-index warnings show `fallback=legacy_fts`, rather than returning + partial or empty results; +- no increase appears in retrieval errors or timeouts. + +## Phase 4: Backfill existing format-v2 indexes + +After the migration, run the statistics inventory: + +```bash +python /app/scripts/backfill_map_unit_statistics.py \ + --check \ + --batch-size 100 +``` + +Record the final summary line: + +```text +would_update= complete= skipped= documents= +``` + +Interpretation: + +- `complete`: already coherent format-v2 revisions; +- `would_update`: existing format-v2 indexes whose four statistics need an + in-place update; +- `skipped`: missing or legacy indexes that require the full index backfill. + +Run exactly one maintenance process against the database. `--batch-size` is a +serial session grouping, not a concurrency setting. + +First rehearse one namespace or document: + +```bash +python /app/scripts/backfill_map_unit_statistics.py \ + --apply \ + --batch-size 100 \ + --user-id \ + --namespace +``` + +Then run the full statistics backfill: + +```bash +python /app/scripts/backfill_map_unit_statistics.py \ + --apply \ + --batch-size 100 +``` + +The command: + +- reads existing `document_map_units`; +- computes positive-length document counts and total token lengths separately + for the path and content channels; +- commits each revision independently; +- does not regenerate token rows, manifests, or namespace snapshots; +- is idempotent and safe to restart. + +If `skipped` is non-zero, list and process those documents separately with the +existing full index command: + +```bash +python /app/scripts/backfill_map_unit_indexes.py \ + --apply \ + --document-id +``` + +Do not use `--tokens-only` unless investigation proves that only map-unit token +data is missing and the serving manifest and namespace snapshot are already +coherent. + +## Phase 5: Final readiness gate + +Repeat the read-only check until it exits successfully and reports: + +```text +would_update=0 skipped=0 complete= documents= +``` + +```bash +python /app/scripts/backfill_map_unit_statistics.py \ + --check \ + --batch-size 100 +``` + +Also verify that all retrieval-visible active revisions are coherent: + +```sql +SELECT + count(*) FILTER ( + WHERE indexes.format_version = 2 + AND indexes.path_document_count IS NOT NULL + AND indexes.path_total_length IS NOT NULL + AND indexes.content_document_count IS NOT NULL + AND indexes.content_total_length IS NOT NULL + ) AS ready_revisions, + count(*) AS current_revisions +FROM documents +LEFT JOIN document_map_unit_indexes AS indexes + ON indexes.document_id = documents.document_id + AND indexes.job_result_id = documents.current_job_result_id +WHERE documents.status = 'active' + AND documents.current_job_result_id IS NOT NULL; +``` + +With the `LEFT JOIN`, `current_revisions` includes active documents with no +index. `ready_revisions` must equal `current_revisions`. Also require: + +```sql +SELECT count(*) AS missing_index_rows +FROM documents +LEFT JOIN document_map_unit_indexes AS indexes + ON indexes.document_id = documents.document_id + AND indexes.job_result_id = documents.current_job_result_id +WHERE documents.status = 'active' + AND documents.current_job_result_id IS NOT NULL + AND indexes.id IS NULL; +``` + +`missing_index_rows` must be zero. + +Run the complete serving-index readiness checker: + +```bash +python /app/scripts/backfill_map_unit_indexes.py --check +``` + +For every namespace, require `status=READY` and inspect the report rather than +only its exit code. Require: + +```text +missing_from_snapshot=0 +missing_map_index=0 +missing_revision_manifest=0 +``` + +`suspicious_zero_idf` is diagnostic only. Zero average IDF is valid for some +small corpora, including a two-unit corpus where each token occurs in exactly +one unit; it must not independently block readiness. + +## Phase 6: Retrieval-quality gate + +Run the frozen temporary quality set. Use the same two or three queries, +namespace, top-k, filters, and revision generation captured before deployment. + +For classic retrieval, require no change in: + +- router; +- selected chunk IDs and order; +- source document and section; +- evidence content/hash; +- asset references; +- rounded scores, with an absolute tolerance of `1e-4`. + +Exercise at least: + +1. v1 `use_agentic=false`; +2. v2 `use_agentic=false` with equivalent retrieval fields; +3. one `use_agentic=true` map-nav smoke; +4. one request with `use_agentic` omitted, confirming it routes to map-nav; +5. one filtered request, confirming filtered-scope semantics and the safe + fallback where required. + +Map-nav LLM output is nondeterministic. For production smoke, require successful +completion, valid citations, expected namespace isolation, and relevant +evidence. Do not require byte-identical ordering between independent Planner +runs. Deterministic map-score parity remains covered by the contract suite. + +Stop the rollout if classic result parity fails. Do not refresh the baseline to +hide a difference. + +## Phase 7: Performance and reliability observation + +Monitor at least one normal traffic window after the backfill. Classic public +result parity must have zero differences, total non-LLM p95 must not exceed the +recorded baseline, and retrieval error/timeout rates must not regress. + +- retrieval request p50/p95 and maximum latency, separated by `router_used`; +- classic `search.map_unit_discovery` stages: units, frequencies, indexes, + statistics, scoring, and hydration; +- map-nav snapshot, episode, and hydration stages; +- PostgreSQL statement timeouts, lock waits, CPU, I/O, and connection usage; +- Redis errors and namespace snapshot cache misses; +- retrieval errors, incomplete-index fallbacks, and response timeouts; +- process CPU and maximum RSS from the corrected map-nav resource log. + +Do not mix classic and map-nav latency distributions. Do not treat Redis-warm +snapshot measurements as cold-request performance. + +## Pause and resume + +To pause, stop launching new maintenance command processes. Wait for the +current revision to finish if database health permits, then send `SIGINT` or +`SIGTERM` to the one-off task. Each completed revision has already committed; +the interrupted transaction rolls back and remains on the safe reader path. + +To resume, rerun the same `--apply` command. Completed revisions are detected +and skipped. Follow it with `--check` and retain both summary outputs in the +deployment record. + +## Rollback + +If application errors, timeouts, or quality regressions occur: + +1. stop the backfill process; +2. redeploy the previous application version; +3. verify classic and map-nav requests using the frozen quality set; +4. retain the additive columns, index, and already-computed statistics unless + database health specifically requires their removal. + +Application rollback is sufficient because the previous application ignores +the additive schema. Avoid running Alembic downgrade during an incident: a +concurrent index drop or table alteration adds operational risk and is not +required to restore the previous behavior. + +## Completion record + +Attach the following to the deployment ticket: + +- deployed application image digest and Git commit; +- pre-migration active/current inventory; +- post-migration statistics `--check` output; +- Alembic `current` output; +- covering-index and column verification output; +- rehearsal and full `--apply` summaries; +- final `--check` output; +- ready/current revision counts; +- frozen-query parity results; +- classic and map-nav latency summaries; +- observed fallback, error, and timeout counts; +- rollback decision or explicit confirmation that rollback was not required. + +## ECS one-off task requirements + +Run maintenance as a one-off task created from the newly registered production +API task definition. Reuse its task role, execution role, VPC subnets, security +groups, Secrets Manager injection, and CloudWatch log configuration. Override +only the container command with one of the commands in this runbook. + +Do not execute maintenance inside a long-lived API task. Do not copy, export, or +place the production database URL in the ECS command, shell history, workflow +input, deployment ticket, or logs. The one-off task must receive it through the +same production secret as the API task. From 3c1248f8407aa9c0b705c43c053847f899b88ea8 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 15:52:42 +0800 Subject: [PATCH 31/35] fix: preserve legacy retrieval on empty map-unit matches --- ...est_retrieval_classic_map_unit_contract.py | 162 +++++++++++++++++- .../retrieval/search/map_unit_discovery.py | 52 +++++- 2 files changed, 206 insertions(+), 8 deletions(-) diff --git a/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py b/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py index 388b8198..1117c231 100644 --- a/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py +++ b/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py @@ -5,7 +5,8 @@ from typing import Any, cast from uuid import uuid4 -from httpx import AsyncClient +import pytest +from httpx import AsyncClient, Response from sqlalchemy import Engine, event, select from shared.models.database.document import DocumentMapUnit @@ -13,7 +14,10 @@ replace_document_revision_content, ) from shared.services.retrieval.publication_models import DocumentPublicationScope -from shared.services.retrieval.search.map_unit_discovery import map_unit_discovery +from shared.services.retrieval.search.map_unit_discovery import ( + DiscoveryResult, + map_unit_discovery, +) from shared.services.retrieval.serving_generation import lock_namespace_generation from tests.support.contract_database import ContractDatabase from tests.support.retrieval_snapshot_support import contract_db_session @@ -369,6 +373,28 @@ def result_signature(result: Any) -> list[tuple[Any, ...]]: ) +async def test_classic_discovery_returns_empty_for_an_empty_revision_pin( + developer_api_client_factory: Callable[ + [], AbstractAsyncContextManager[AsyncClient] + ], +) -> None: + namespace: str = f"classic-empty-pins-{uuid4().hex[:8]}" + async with developer_api_client_factory(): + async with contract_db_session() as db: + result: DiscoveryResult = await map_unit_discovery( + db, + user_id=_USER_ID, + namespace=namespace, + query="empty revision marker", + top_k=1, + exclude_document_ids=[], + exclude_sections=[], + revision_pins={}, + ) + + assert result.payload["fused_rows"] == [] + + async def test_classic_route_falls_back_for_v1_index_with_excluded_document( developer_api_client_factory: Callable[ [], AbstractAsyncContextManager[AsyncClient] @@ -465,6 +491,138 @@ def capture_legacy_query( assert legacy_queries +@pytest.mark.parametrize( + "incomplete_index_kind", ["legacy_format", "missing_index", "missing_tokens"] +) +async def test_unfiltered_classic_route_falls_back_when_selective_rows_are_unavailable( + developer_api_client_factory: Callable[ + [], AbstractAsyncContextManager[AsyncClient] + ], + incomplete_index_kind: str, +) -> None: + identifier: str = uuid4().hex[:8] + namespace: str = f"classic-token-fallback-{incomplete_index_kind}-{identifier}" + legacy_queries: list[str] = [] + + def capture_legacy_query( + _connection: object, + _cursor: object, + statement: str, + _parameters: object, + _context: object, + _executemany: bool, + ) -> None: + if "plainto_tsquery('simple'" in statement: + legacy_queries.append(statement) + + event.listen(Engine, "before_cursor_execute", capture_legacy_query) + try: + async with developer_api_client_factory() as api_client: + document: dict[str, str] = await _publish_document( + namespace=namespace, + source_file_name="legacy-token-hash.pdf", + chunks=[ + { + "chunk_id": f"legacy-token-hit-{identifier}", + "type": "text", + "content": "legacy token fallback marker", + "path": "legacy-token-hash.pdf/Root/Section/body", + "order": 1, + "metadata": {}, + }, + { + "chunk_id": f"legacy-token-filler-a-{identifier}", + "type": "text", + "content": "unrelated legacy filler a", + "path": "legacy-token-hash.pdf/Root/Section/a", + "order": 2, + "metadata": {}, + }, + { + "chunk_id": f"legacy-token-filler-b-{identifier}", + "type": "text", + "content": "unrelated legacy filler b", + "path": "legacy-token-hash.pdf/Root/Section/b", + "order": 3, + "metadata": {}, + }, + ], + ) + if incomplete_index_kind == "legacy_format": + await ContractDatabase.execute( + """ + UPDATE document_map_unit_indexes + SET format_version = 1 + WHERE document_id = :document_id + """, + {"document_id": document["document_id"]}, + ) + await ContractDatabase.execute( + """ + UPDATE document_map_unit_tokens + SET token_hash = :legacy_token_hash + WHERE map_unit_id IN ( + SELECT id + FROM document_map_units + WHERE document_id = :document_id + ) + """, + { + "document_id": document["document_id"], + "legacy_token_hash": "legacy-token-hash", + }, + ) + elif incomplete_index_kind == "missing_tokens": + await ContractDatabase.execute( + """ + DELETE FROM document_map_unit_tokens + WHERE map_unit_id IN ( + SELECT id + FROM document_map_units + WHERE document_id = :document_id + ) + """, + {"document_id": document["document_id"]}, + ) + else: + await ContractDatabase.execute( + """ + DELETE FROM document_map_unit_indexes + WHERE document_id = :document_id + """, + {"document_id": document["document_id"]}, + ) + await ContractDatabase.execute( + """ + DELETE FROM document_map_unit_tokens + WHERE map_unit_id IN ( + SELECT id + FROM document_map_units + WHERE document_id = :document_id + ) + """, + {"document_id": document["document_id"]}, + ) + response: Response = await api_client.post( + "/api/v1/retrieval/query", + json={ + "namespace": namespace, + "query": "legacy token fallback marker", + "top_k": 1, + "use_agentic": False, + }, + ) + finally: + event.remove(Engine, "before_cursor_execute", capture_legacy_query) + + assert response.status_code == 200 + body = cast(dict[str, object], response.json()) + results = cast(list[dict[str, object]], body["results"]) + assert len(results) == 1 + assert results[0]["chunk_id"] == f"legacy-token-hit-{identifier}" + assert legacy_queries + + async def test_classic_route_image_filter_scores_only_units_with_images( developer_api_client_factory: Callable[ [], AbstractAsyncContextManager[AsyncClient] diff --git a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py index a39844bb..b14d8a97 100644 --- a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py +++ b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py @@ -20,7 +20,7 @@ from collections.abc import Mapping from dataclasses import dataclass, field from hashlib import sha256 -from typing import Any +from typing import Any, cast from loguru import logger from sqlalchemy import text @@ -193,6 +193,8 @@ async def map_unit_discovery( query_tokens = tokenize_query_for_ranker(query) if not query_tokens: return DiscoveryResult(status="discovery_done", payload={"fused_rows": []}) + if revision_pins is not None and not revision_pins: + return DiscoveryResult(status="discovery_done", payload={"fused_rows": []}) query_token_hashes = [ sha256(token.encode("utf-8")).hexdigest() for token in query_tokens ] @@ -262,9 +264,6 @@ async def map_unit_discovery( len(unit_rows), ) - if not unit_rows: - return DiscoveryResult(status="discovery_done", payload={"fused_rows": []}) - frequency_scope_cte = cte if signal_paths else _SCOPED_UNIT_IDS_CTE.format( revision_join=revision_join, revision_clause=revision_clause, @@ -322,7 +321,8 @@ async def map_unit_discovery( SELECT indexes.average_idf_path, indexes.average_idf_content, indexes.unit_count, indexes.format_version, indexes.path_document_count, indexes.path_total_length, - indexes.content_document_count, indexes.content_total_length + indexes.content_document_count, indexes.content_total_length, + indexes.token_count FROM document_map_unit_indexes AS indexes JOIN (VALUES {pinned_values_sql}) AS scoped_revisions(document_id, job_result_id) @@ -345,7 +345,8 @@ async def map_unit_discovery( SELECT indexes.average_idf_path, indexes.average_idf_content, indexes.unit_count, indexes.format_version, indexes.path_document_count, indexes.path_total_length, - indexes.content_document_count, indexes.content_total_length + indexes.content_document_count, indexes.content_total_length, + indexes.token_count FROM document_map_unit_indexes AS indexes JOIN ( SELECT DISTINCT document_id, job_result_id FROM scoped_units @@ -366,6 +367,7 @@ async def map_unit_discovery( path_total_length, content_document_count, content_total_length, + int(token_count or 0), ) for ( path_idf, @@ -376,6 +378,7 @@ async def map_unit_discovery( path_total_length, content_document_count, content_total_length, + token_count, ) in index_result.all() ] logger.info( @@ -417,6 +420,40 @@ async def map_unit_discovery( indexed_unit_count = sum( unit_count for _path_idf, _content_idf, unit_count, *_rest in index_parts ) + has_index_storage_mismatch: bool = False + if is_unfiltered_scope and not unit_rows and expected_revisions: + storage_counts: tuple[int | None, int | None] = cast( + tuple[int | None, int | None], + ( + await db.execute( + text( + _SCOPED_UNIT_IDS_CTE.format( + revision_join=revision_join, + revision_clause=revision_clause, + exclude_clause=exclude_clause, + type_clause=type_clause, + ) + + """ + SELECT + (SELECT COUNT(*) FROM scoped_units) AS unit_count, + ( + SELECT COUNT(*) + FROM document_map_unit_tokens AS tokens + JOIN scoped_units + ON scoped_units.map_unit_id = tokens.map_unit_id + ) AS token_count + """ + ), + params, + ) + ).one(), + ) + actual_unit_count: int | None = storage_counts[0] + actual_token_count: int | None = storage_counts[1] + indexed_token_count: int = sum(int(part[8]) for part in index_parts) + has_index_storage_mismatch = indexed_unit_count != int( + actual_unit_count or 0 + ) or indexed_token_count != int(actual_token_count or 0) has_index_unit_count_mismatch = ( indexed_unit_count < len(unit_rows) if is_unfiltered_scope @@ -433,6 +470,7 @@ async def map_unit_discovery( _path_total_length, _content_document_count, _content_total_length, + _token_count, ) in index_parts ) has_incomplete_index_statistics = is_unfiltered_scope and any( @@ -450,11 +488,13 @@ async def map_unit_discovery( path_total_length, content_document_count, content_total_length, + _token_count, ) in index_parts ) if ( len(index_parts) != len(expected_revisions) or has_index_unit_count_mismatch + or has_index_storage_mismatch or is_index_format_incompatible or has_incomplete_index_statistics ): From e59111b3aa2ece01ec6c21032e86292b0fc465f3 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 17:53:13 +0800 Subject: [PATCH 32/35] fix: preserve retrieval semantics during statistics backfill --- ...est_retrieval_classic_map_unit_contract.py | 110 ++++++++++++++++++ ...kfill-serving-index-statistics-in-place.md | 12 +- ...al-heart-vessel-trace-optimization-plan.md | 47 +++++--- ...retrieval-serving-index-rollout-runbook.md | 9 +- .../retrieval/search/map_unit_discovery.py | 51 ++++++-- 5 files changed, 195 insertions(+), 34 deletions(-) diff --git a/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py b/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py index 1117c231..fe5b732c 100644 --- a/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py +++ b/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py @@ -491,6 +491,116 @@ def capture_legacy_query( assert legacy_queries +async def test_classic_discovery_preserves_results_before_statistics_backfill( + developer_api_client_factory: Callable[ + [], AbstractAsyncContextManager[AsyncClient] + ], +) -> None: + identifier = uuid4().hex[:8] + namespace = f"classic-statistics-parity-{identifier}" + legacy_queries: list[str] = [] + + def capture_legacy_query( + _connection: Any, + _cursor: Any, + statement: str, + _parameters: Any, + _context: Any, + _executemany: bool, + ) -> None: + if "plainto_tsquery('simple'" in statement: + legacy_queries.append(statement) + + def result_signature(result: DiscoveryResult) -> list[tuple[Any, ...]]: + return [ + ( + row.get("chunk_id"), + row.get("document_id"), + row.get("job_result_id"), + row.get("section_path"), + row.get("source_file_name"), + row.get("score"), + row.get("discovery_score"), + row.get("content"), + row.get("chunk_metadata"), + ) + for row in list(result.payload.get("fused_rows") or []) + ] + + event.listen(Engine, "before_cursor_execute", capture_legacy_query) + try: + async with developer_api_client_factory(): + document = await _publish_document( + namespace=namespace, + source_file_name="statistics-parity.pdf", + chunks=[ + { + "chunk_id": f"statistics-hit-{identifier}", + "type": "text", + "content": "statistics parity retrieval marker", + "path": "statistics-parity.pdf/Root/Section/hit", + "order": 1, + "metadata": {}, + }, + { + "chunk_id": f"statistics-filler-a-{identifier}", + "type": "text", + "content": "unrelated statistics filler a", + "path": "statistics-parity.pdf/Root/Section/a", + "order": 2, + "metadata": {}, + }, + { + "chunk_id": f"statistics-filler-b-{identifier}", + "type": "text", + "content": "unrelated statistics filler b", + "path": "statistics-parity.pdf/Root/Section/b", + "order": 3, + "metadata": {}, + }, + ], + ) + query = "statistics parity retrieval marker" + async with contract_db_session() as db: + post_backfill = await map_unit_discovery( + db, + user_id=_USER_ID, + namespace=namespace, + query=query, + top_k=3, + exclude_document_ids=[], + exclude_sections=[], + ) + + await ContractDatabase.execute( + """ + UPDATE document_map_unit_indexes + SET path_document_count = NULL, + path_total_length = NULL, + content_document_count = NULL, + content_total_length = NULL + WHERE document_id = :document_id + """, + {"document_id": document["document_id"]}, + ) + + async with contract_db_session() as db: + pre_backfill = await map_unit_discovery( + db, + user_id=_USER_ID, + namespace=namespace, + query=query, + top_k=3, + exclude_document_ids=[], + exclude_sections=[], + ) + finally: + event.remove(Engine, "before_cursor_execute", capture_legacy_query) + + assert result_signature(pre_backfill) == result_signature(post_backfill) + assert legacy_queries == [] + + @pytest.mark.parametrize( "incomplete_index_kind", ["legacy_format", "missing_index", "missing_tokens"] ) diff --git a/docs/adr/0010-backfill-serving-index-statistics-in-place.md b/docs/adr/0010-backfill-serving-index-statistics-in-place.md index afac7749..75e2ac4c 100644 --- a/docs/adr/0010-backfill-serving-index-statistics-in-place.md +++ b/docs/adr/0010-backfill-serving-index-statistics-in-place.md @@ -8,7 +8,11 @@ existing map units and does not regenerate token rows, manifests, or namespace snapshots. Missing or legacy indexes use the existing full `apps/api/scripts/backfill_map_unit_indexes.py --apply --document-id ...` rebuild. v1 and v2 are internal read paths under the same Retrieval contract, -so each revision commits independently and the optimized reader requires the -new serving-index format marker before it can be selected. An interrupted or -failed backfill therefore continues using the exact legacy reader, limiting -database load and avoiding unnecessary regeneration of correct data. +so each revision commits independently. While the four statistics are NULL, +the reader keeps the full scope-first map-unit projection and derives the +existing per-channel denominators from those rows; it does not switch to a +semantically different FTS query. Once statistics are complete, the +token-selective projection may be used. Missing, legacy, or storage- +inconsistent index data still uses the exact legacy reader. An interrupted or +failed backfill therefore preserves retrieval semantics and avoids unnecessary +regeneration of correct data. diff --git a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md index b6c521d7..ed62a57b 100644 --- a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md +++ b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md @@ -319,9 +319,12 @@ proves that a different join order is better. Roll this out in two stages. The first stage may enable the token-selective reader only when the request has no section exclusions, where revision-scoped channel statistics are sufficient. Requests with section filters continue to -use the exact legacy reader until section-scoped denominator statistics are -implemented and pass semantic-parity checks. A missing, incomplete, or -incompatible serving index always uses the same fallback. +use the existing scope-first map-unit reader until section-scoped denominator +statistics are implemented and pass semantic-parity checks. A missing, +incompatible, or storage-inconsistent serving index uses the exact legacy FTS +fallback. A complete index with NULL channel statistics keeps the full scope +map-unit rows and derives the two channel denominators from those rows until +the statistics backfill completes. BM25 denominators must remain corpus-wide. Obtain exact corpus statistics using persisted serving-index statistics while preserving the current per-channel @@ -341,10 +344,14 @@ reader. The current implementation slice only adds the per-channel statistics fields, publication-time population, and the DevOps backfill/readiness contract. It -now enables token-selective projection in both persisted readers when the scope -is unfiltered and serving statistics are present. Section-filtered or -incomplete revisions continue through the legacy path. Both v1 and v2 call the -same shared readers; no route-specific lexical algorithm was introduced. +enables token-selective projection when the scope is unfiltered and serving +statistics are present. Before statistics backfill, an unfiltered revision +uses the full scope map-unit rows with row-derived per-channel denominators, so +the lexical result remains unchanged; section-filtered requests retain their +existing scope-first map-unit reader. Only a missing, incompatible, or +storage-inconsistent index uses the exact legacy FTS fallback. Both v1 and v2 +call the same shared readers; no route-specific lexical algorithm was +introduced. Acceptance criteria: @@ -483,15 +490,17 @@ the same Retrieval contract and must produce identical results. For a complete existing index, a resumable per-revision statistics backfill aggregates the existing `document_map_units` rows, writes the four path/content values, and marks the revision v2 in the same transaction. It must not regenerate token rows or -snapshots unnecessarily. Revisions with a missing or incomplete index use the +snapshots unnecessarily. Revisions with a missing or legacy index use the existing full `backfill_map_unit_indexes --apply` path. Each revision is committed independently under the existing generation and -active-document locks. The job is idempotent and safe to interrupt: revisions -without a completed v2 marker remain on the exact legacy reader, and the -optimized internal reader is selected only after the backfill check reports -complete, coherent active revisions; otherwise the same Retrieval request uses -the exact legacy reader. +active-document locks. The job is idempotent and safe to interrupt. While the +four statistics are NULL, retrieval keeps the full scope-first map-unit reader +and derives the existing per-channel denominators from its rows. Once the +backfill check reports complete, coherent active revisions, the unfiltered +reader can use token-selective projection and persisted denominators. Missing, +incompatible, or storage-inconsistent index data still uses the exact legacy +FTS fallback. ### DevOps handoff @@ -504,8 +513,9 @@ The runbook: Build the new index without dropping the existing lookup index, and monitor lock waits, build duration, and disk headroom; 2. deploy the application code that writes v2 for new publications and selects - the fast reader only for complete v2 revisions; v1 revisions continue on the - legacy reader; + the token-selective reader only for complete v2 revisions; revisions with + NULL statistics continue on the full scope-first map-unit reader, while v1 + or otherwise unusable index data continues on the legacy FTS reader; 3. run the read-only inventory/check command and record active/current revision counts; 4. run the resumable statistics backfill in bounded batches (with optional @@ -525,14 +535,15 @@ The runbook: 5. rerun the check until every retrieval-visible revision has a coherent v2 marker and no serving fallback is reported; 6. monitor semantic-parity probes, latency, errors, and timeouts. The reader's - version/completeness checks automatically retain the legacy path when a - revision is not ready. + checks retain the full scope-first map-unit path while only statistics are + incomplete, and use legacy FTS when index data is unusable. The handoff must document the exact command, batch/concurrency limits, pause/resume procedure, application-version rollback procedure, and the final check output. A partial backfill is an expected intermediate state, not a failed deployment, provided -that incomplete revisions remain on the legacy reader. +that incomplete statistics do not select token-only projection and unusable +indexes remain on the legacy reader. ### Deferred: LLM orchestration improvements diff --git a/docs/design/retrieval-serving-index-rollout-runbook.md b/docs/design/retrieval-serving-index-rollout-runbook.md index 04112a18..56d2c3cc 100644 --- a/docs/design/retrieval-serving-index-rollout-runbook.md +++ b/docs/design/retrieval-serving-index-rollout-runbook.md @@ -151,16 +151,17 @@ index with `DROP INDEX CONCURRENTLY`, and rerun the migration. ## Phase 3: Deploy the application build Deploy the application after the additive migrations finish. New publications -will write coherent format-v2 statistics. Existing incomplete revisions remain -on the legacy reader until maintenance completes. +will write coherent format-v2 statistics. Existing revisions with NULL channel +statistics remain on the full scope-first map-unit reader until maintenance +completes; missing, legacy, or unusable indexes remain on the legacy reader. Immediately verify: - API health checks pass; - no migration or model-loading error appears in API logs; - classic and map-nav requests still complete; -- incomplete-index warnings show `fallback=legacy_fts`, rather than returning - partial or empty results; +- incomplete-index warnings distinguish statistics-incomplete map-unit serving + from `fallback=legacy_fts`; neither case may return partial or empty results; - no increase appears in retrieval errors or timeouts. ## Phase 4: Backfill existing format-v2 indexes diff --git a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py index b14d8a97..68e27402 100644 --- a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py +++ b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py @@ -491,13 +491,13 @@ async def map_unit_discovery( _token_count, ) in index_parts ) - if ( + has_unusable_index = ( len(index_parts) != len(expected_revisions) or has_index_unit_count_mismatch or has_index_storage_mismatch or is_index_format_incompatible - or has_incomplete_index_statistics - ): + ) + if has_unusable_index: try: await record_retrieval_index_readiness( user_id=user_id, @@ -529,11 +529,46 @@ async def map_unit_discovery( filter_mode=filter_mode, revision_pins=revision_pins, ) + if has_incomplete_index_statistics: + logger.warning( + "retrieval map index statistics incomplete user_id=%s namespace=%s " + "using row-derived BM25 denominators", + user_id, + namespace, + ) + # Token-selective projection is only sufficient when persisted channel + # denominators are available. Before the statistics backfill, restore + # the old full-scope unit projection so row-derived BM25 statistics use + # the same corpus as the legacy map-unit reader. + stage_started = time.monotonic() + full_unit_params = { + key: value + for key, value in params.items() + if key not in {"channels", "token_hashes"} + } + full_unit_result = await db.execute( + text(cte + "SELECT * FROM scoped_units"), full_unit_params + ) + unit_rows = [dict(row._mapping) for row in full_unit_result.all()] + unit_rows = [ + row + for row in unit_rows + if not is_excluded_section( + document_id=row.get("document_id"), + section_path=row.get("section_path"), + exclude_sections=exclude_sections, + ) + ] + logger.info( + "retrieval map-unit stage=full-units-for-stats seconds={:.3f} rows={}", + time.monotonic() - stage_started, + len(unit_rows), + ) try: await record_retrieval_index_readiness( user_id=user_id, namespace=namespace, - ready=True, + ready=not has_incomplete_index_statistics, expected_revisions=len(expected_revisions), indexed_revisions=len(index_parts), ) @@ -563,12 +598,12 @@ async def map_unit_discovery( average_idf=average_idf_path, document_count_override=( sum(int(part[4] or 0) for part in index_parts) - if is_unfiltered_scope + if is_unfiltered_scope and not has_incomplete_index_statistics else None ), total_length_override=( sum(int(part[5] or 0) for part in index_parts) - if is_unfiltered_scope + if is_unfiltered_scope and not has_incomplete_index_statistics else None ), ) @@ -582,12 +617,12 @@ async def map_unit_discovery( average_idf=average_idf_content, document_count_override=( sum(int(part[6] or 0) for part in index_parts) - if is_unfiltered_scope + if is_unfiltered_scope and not has_incomplete_index_statistics else None ), total_length_override=( sum(int(part[7] or 0) for part in index_parts) - if is_unfiltered_scope + if is_unfiltered_scope and not has_incomplete_index_statistics else None ), ) From 6a1f8258be94edae6070ae0de7cc704fcb407bd5 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 3 Sep 2026 19:30:45 +0800 Subject: [PATCH 33/35] fix: resolve statistics backfill runtime path --- .../scripts/backfill_map_unit_statistics.py | 24 +++++++++++------- ...test_backfill_map_unit_indexes_contract.py | 25 +++++++++++++++++++ 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/apps/api/scripts/backfill_map_unit_statistics.py b/apps/api/scripts/backfill_map_unit_statistics.py index a4ec4d28..86214ef7 100644 --- a/apps/api/scripts/backfill_map_unit_statistics.py +++ b/apps/api/scripts/backfill_map_unit_statistics.py @@ -17,17 +17,23 @@ from pathlib import Path +def _resolve_shared_root(api_root: Path) -> Path: + """Resolve shared-python in source checkouts and runtime images.""" + runtime_shared_root = api_root / "packages" / "shared-python" + if runtime_shared_root.is_dir(): + return runtime_shared_root + + repository_root = api_root.parent.parent + repository_shared_root = repository_root / "packages" / "shared-python" + if repository_shared_root.is_dir(): + return repository_shared_root + + raise RuntimeError(f"Could not locate shared-python package from {api_root}") + + def _bootstrap_python_path() -> None: api_root = Path(__file__).resolve().parents[1] - candidate_roots = ( - api_root / "packages" / "shared-python", - api_root.parents[1] / "packages" / "shared-python", - ) - shared_root = next( - (path for path in candidate_roots if path.is_dir()), None - ) - if shared_root is None: - raise RuntimeError("Could not locate shared-python package") + shared_root = _resolve_shared_root(api_root) for path in (api_root, shared_root): value = os.fspath(path) if value not in sys.path: diff --git a/apps/api/tests/contract/test_backfill_map_unit_indexes_contract.py b/apps/api/tests/contract/test_backfill_map_unit_indexes_contract.py index 3f61f463..18f68dc1 100644 --- a/apps/api/tests/contract/test_backfill_map_unit_indexes_contract.py +++ b/apps/api/tests/contract/test_backfill_map_unit_indexes_contract.py @@ -32,6 +32,31 @@ def test_backfill_script_resolves_shared_package_from_source_checkout_layout( assert _resolve_shared_root(api_root) == shared_root +def test_statistics_backfill_resolves_shared_package_from_runtime_image_layout( + tmp_path: Path, +) -> None: + from scripts.backfill_map_unit_statistics import _resolve_shared_root + + api_root = tmp_path / "app" + shared_root = api_root / "packages" / "shared-python" + shared_root.mkdir(parents=True) + + assert _resolve_shared_root(api_root) == shared_root + + +def test_statistics_backfill_resolves_shared_package_from_source_checkout_layout( + tmp_path: Path, +) -> None: + from scripts.backfill_map_unit_statistics import _resolve_shared_root + + repository_root = tmp_path / "repository" + api_root = repository_root / "apps" / "api" + shared_root = repository_root / "packages" / "shared-python" + shared_root.mkdir(parents=True) + + assert _resolve_shared_root(api_root) == shared_root + + def test_statistics_backfill_aggregates_positive_lengths_per_channel() -> None: from scripts.backfill_map_unit_statistics import ( RevisionStatistics, From 3207a67dcb33dd845cffe806a4ef4493e1917cee Mon Sep 17 00:00:00 2001 From: suguanYang Date: Mon, 7 Sep 2026 10:30:43 +0800 Subject: [PATCH 34/35] fix(worker): normalize encoded document filenames --- .../document_ingestion/creation_service.py | 4 ++-- .../unit/test_document_ingestion_filename.py | 18 ++++++++++++++++++ .../support/internal_parse_name.py | 2 ++ .../tests/unit/test_internal_parse_name.py | 12 ++++++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 apps/api/tests/unit/test_document_ingestion_filename.py diff --git a/apps/api/app/services/document_ingestion/creation_service.py b/apps/api/app/services/document_ingestion/creation_service.py index 597aac79..ec88c65c 100644 --- a/apps/api/app/services/document_ingestion/creation_service.py +++ b/apps/api/app/services/document_ingestion/creation_service.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from typing import cast -from urllib.parse import urlparse +from urllib.parse import unquote, urlparse from app.repositories.job_repository import JobRepository from app.services.document_ingestion.command import DocumentIngestionCommand @@ -308,7 +308,7 @@ def _build_job_response( def _resolve_url_source_file_name(*, source_url: str, file_extension: str) -> str: parsed_url = urlparse(source_url) - url_basename = str(os.path.basename(parsed_url.path)) + url_basename = unquote(str(os.path.basename(parsed_url.path))) if url_basename and os.path.splitext(url_basename)[1].lower() == file_extension: return url_basename if url_basename: diff --git a/apps/api/tests/unit/test_document_ingestion_filename.py b/apps/api/tests/unit/test_document_ingestion_filename.py new file mode 100644 index 00000000..703e455d --- /dev/null +++ b/apps/api/tests/unit/test_document_ingestion_filename.py @@ -0,0 +1,18 @@ +from app.services.document_ingestion.creation_service import ( + _resolve_url_source_file_name, +) + + +def test_resolve_url_source_file_name_decodes_percent_encoded_path() -> None: + source_url = ( + "https://files.example.test/" + "%E4%B8%AD%E6%96%87%E7%AE%80%E5%8E%86-%E9%9B%B7%E7%BF%94-" + "%E4%B8%AD%E7%A7%91%E9%99%A2%285%29.doc" + ) + + source_file_name = _resolve_url_source_file_name( + source_url=source_url, + file_extension=".doc", + ) + + assert source_file_name == "中文简历-雷翔-中科院(5).doc" diff --git a/apps/worker/app/services/document_parser/support/internal_parse_name.py b/apps/worker/app/services/document_parser/support/internal_parse_name.py index 7a59ed8c..f4ecee2d 100644 --- a/apps/worker/app/services/document_parser/support/internal_parse_name.py +++ b/apps/worker/app/services/document_parser/support/internal_parse_name.py @@ -3,6 +3,7 @@ import os from dataclasses import dataclass +from urllib.parse import unquote from app.services.common.file_utils import path_handle from app.services.document_parser.support.filename_limits import ( @@ -27,6 +28,7 @@ def normalize_internal_parse_name( candidate_name = ( os.path.basename(filename) if isinstance(filename, str) and filename else "" ) + candidate_name = unquote(candidate_name) cleaned_name = ( path_handle(candidate_name, mode="clean_single") if candidate_name else "" ) diff --git a/apps/worker/tests/unit/test_internal_parse_name.py b/apps/worker/tests/unit/test_internal_parse_name.py index c43dcabb..cabcb912 100644 --- a/apps/worker/tests/unit/test_internal_parse_name.py +++ b/apps/worker/tests/unit/test_internal_parse_name.py @@ -4,10 +4,22 @@ from pathlib import Path from app.services.document_parser.support.internal_parse_name import ( + normalize_internal_parse_name, prepare_internal_parse_input, ) +def test_normalize_internal_parse_name_decodes_url_encoded_filename() -> None: + encoded_filename = ( + "%E4%B8%AD%E6%96%87%E7%AE%80%E5%8E%86-%E9%9B%B7%E7%BF%94-" + "%E4%B8%AD%E7%A7%91%E9%99%A2%285%29.doc" + ) + + normalized_name = normalize_internal_parse_name(encoded_filename) + + assert normalized_name == "中文简历-雷翔-中科院-5-.doc" + + def test_prepare_internal_parse_input_handles_long_encoded_filename( tmp_path, ) -> None: From 2ad20358bb8e851f36c028327706ff2f70a8e621 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Mon, 7 Sep 2026 16:50:53 +0800 Subject: [PATCH 35/35] fix: remove unused merge migration import --- .../versions/d3e4f5a6b7c8_merge_retrieval_index_heads.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/apps/api/alembic/versions/d3e4f5a6b7c8_merge_retrieval_index_heads.py b/apps/api/alembic/versions/d3e4f5a6b7c8_merge_retrieval_index_heads.py index 70e3e095..347a1dfb 100644 --- a/apps/api/alembic/versions/d3e4f5a6b7c8_merge_retrieval_index_heads.py +++ b/apps/api/alembic/versions/d3e4f5a6b7c8_merge_retrieval_index_heads.py @@ -4,9 +4,6 @@ from collections.abc import Sequence -from alembic import op - - revision: str = "d3e4f5a6b7c8" down_revision: tuple[str, str] = ( "0a1b2c3d4e5f",