diff --git a/CONTEXT.md b/CONTEXT.md
index 8609579f8..029d2c64c 100644
--- a/CONTEXT.md
+++ b/CONTEXT.md
@@ -228,7 +228,10 @@ generation and retries or falls back if publication changes it during capture.
The compatibility requirement that a serving-index retrieval returns the same
selected chunk IDs, ordering, rounded scores, citations, and asset references
-as the legacy retrieval path for the same request.
+as the legacy retrieval path for the same request. Every retrieval optimization
+must preserve this quality contract; a latency improvement without validated
+semantic parity is not shippable. Validation also compares source sections,
+evidence content, router, and stop reason for each pinned request.
### Retrieval Revision Pin
diff --git a/apps/api/alembic/versions/a0b1c2d3e4f5_add_token_leading_map_unit_covering_index.py b/apps/api/alembic/versions/a0b1c2d3e4f5_add_token_leading_map_unit_covering_index.py
new file mode 100644
index 000000000..b051b8f0e
--- /dev/null
+++ b/apps/api/alembic/versions/a0b1c2d3e4f5_add_token_leading_map_unit_covering_index.py
@@ -0,0 +1,44 @@
+"""Add the token-leading covering index for map-unit lookups."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+
+revision: str = "a0b1c2d3e4f5"
+down_revision: str | None = "9f0a1b2c3d4e"
+branch_labels: Sequence[str] | None = None
+depends_on: Sequence[str] | None = None
+
+_INDEX_NAME = "idx_document_map_unit_tokens_token_lookup"
+
+
+def upgrade() -> None:
+ """Create the additive index without taking a table-wide write lock."""
+ uses_external_transaction: bool = bool(
+ op.get_context().opts.get("knowhere_external_transaction", False)
+ )
+ statement: str = (
+ f"CREATE INDEX {{concurrently}}IF NOT EXISTS {_INDEX_NAME} "
+ "ON document_map_unit_tokens (channel, token_hash, map_unit_id) "
+ "INCLUDE (token, frequency)"
+ )
+ if uses_external_transaction:
+ op.execute(statement.format(concurrently=""))
+ return
+ with op.get_context().autocommit_block():
+ op.execute(statement.format(concurrently="CONCURRENTLY "))
+
+
+def downgrade() -> None:
+ """Remove only the index introduced by this migration."""
+ uses_external_transaction: bool = bool(
+ op.get_context().opts.get("knowhere_external_transaction", False)
+ )
+ if uses_external_transaction:
+ op.execute(f"DROP INDEX IF EXISTS {_INDEX_NAME}")
+ return
+ with op.get_context().autocommit_block():
+ op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {_INDEX_NAME}")
diff --git a/apps/api/alembic/versions/b1c2d3e4f5a6_add_channel_bm25_statistics.py b/apps/api/alembic/versions/b1c2d3e4f5a6_add_channel_bm25_statistics.py
new file mode 100644
index 000000000..bb826334d
--- /dev/null
+++ b/apps/api/alembic/versions/b1c2d3e4f5a6_add_channel_bm25_statistics.py
@@ -0,0 +1,42 @@
+"""Add persisted per-channel BM25 corpus statistics."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+
+revision: str = "b1c2d3e4f5a6"
+down_revision: str | None = "a0b1c2d3e4f5"
+branch_labels: Sequence[str] | None = None
+depends_on: Sequence[str] | None = None
+
+
+def upgrade() -> None:
+ """Add nullable statistics so incomplete legacy rows keep the fallback."""
+ column_names: tuple[str, ...] = (
+ "path_document_count",
+ "path_total_length",
+ "content_document_count",
+ "content_total_length",
+ )
+ for column_name in column_names:
+ op.execute(
+ f"ALTER TABLE document_map_unit_indexes "
+ f"ADD COLUMN IF NOT EXISTS {column_name} INTEGER"
+ )
+
+
+def downgrade() -> None:
+ """Remove the additive statistics columns."""
+ column_names: tuple[str, ...] = (
+ "content_total_length",
+ "content_document_count",
+ "path_total_length",
+ "path_document_count",
+ )
+ for column_name in column_names:
+ op.execute(
+ f"ALTER TABLE document_map_unit_indexes DROP COLUMN IF EXISTS {column_name}"
+ )
diff --git a/apps/api/alembic/versions/c2d3e4f5a6b7_repair_token_leading_map_unit_covering_index.py b/apps/api/alembic/versions/c2d3e4f5a6b7_repair_token_leading_map_unit_covering_index.py
new file mode 100644
index 000000000..241f11930
--- /dev/null
+++ b/apps/api/alembic/versions/c2d3e4f5a6b7_repair_token_leading_map_unit_covering_index.py
@@ -0,0 +1,62 @@
+"""Repair a missing or invalid token-leading map-unit covering index."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+from sqlalchemy import text
+
+
+revision: str = "c2d3e4f5a6b7"
+down_revision: str | None = "b1c2d3e4f5a6"
+branch_labels: Sequence[str] | None = None
+depends_on: Sequence[str] | None = None
+
+_INDEX_NAME = "idx_document_map_unit_tokens_token_lookup"
+_INDEX_COLUMNS = "(channel, token_hash, map_unit_id) INCLUDE (token, frequency)"
+
+
+def _is_index_ready() -> bool:
+ """Return whether the current schema contains a usable covering index."""
+ is_ready = op.get_bind().execute(
+ text(
+ "SELECT indexes.indisvalid AND indexes.indisready "
+ "FROM pg_index AS indexes "
+ "JOIN pg_class AS classes ON classes.oid = indexes.indexrelid "
+ "JOIN pg_namespace AS namespaces "
+ "ON namespaces.oid = classes.relnamespace "
+ "WHERE namespaces.nspname = current_schema() "
+ "AND classes.relname = :index_name"
+ ),
+ {"index_name": _INDEX_NAME},
+ ).scalar_one_or_none()
+ return bool(is_ready)
+
+
+def _repair_index(*, concurrently: bool) -> None:
+ """Replace a missing or unusable index using the allowed DDL mode."""
+ if _is_index_ready():
+ return
+ concurrent_clause: str = "CONCURRENTLY " if concurrently else ""
+ op.execute(f"DROP INDEX {concurrent_clause}IF EXISTS {_INDEX_NAME}")
+ op.execute(
+ f"CREATE INDEX {concurrent_clause}{_INDEX_NAME} "
+ f"ON document_map_unit_tokens {_INDEX_COLUMNS}"
+ )
+
+
+def upgrade() -> None:
+ """Ensure the additive covering index exists and is usable."""
+ uses_external_transaction: bool = bool(
+ op.get_context().opts.get("knowhere_external_transaction", False)
+ )
+ if uses_external_transaction:
+ _repair_index(concurrently=False)
+ return
+ with op.get_context().autocommit_block():
+ _repair_index(concurrently=True)
+
+
+def downgrade() -> None:
+ """Keep the index owned by the preceding additive migration."""
diff --git a/apps/api/alembic/versions/d3e4f5a6b7c8_merge_retrieval_index_heads.py b/apps/api/alembic/versions/d3e4f5a6b7c8_merge_retrieval_index_heads.py
new file mode 100644
index 000000000..347a1dfb4
--- /dev/null
+++ b/apps/api/alembic/versions/d3e4f5a6b7c8_merge_retrieval_index_heads.py
@@ -0,0 +1,21 @@
+"""Merge the retrieval index migration branches."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+revision: str = "d3e4f5a6b7c8"
+down_revision: tuple[str, str] = (
+ "0a1b2c3d4e5f",
+ "c2d3e4f5a6b7",
+)
+branch_labels: Sequence[str] | None = None
+depends_on: Sequence[str] | None = None
+
+
+def upgrade() -> None:
+ """Merge migration heads without applying additional schema changes."""
+
+
+def downgrade() -> None:
+ """Split the migration graph back into its two parent heads."""
diff --git a/apps/api/app/services/document_ingestion/creation_service.py b/apps/api/app/services/document_ingestion/creation_service.py
index 597aac796..ec88c65ce 100644
--- a/apps/api/app/services/document_ingestion/creation_service.py
+++ b/apps/api/app/services/document_ingestion/creation_service.py
@@ -5,7 +5,7 @@
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import cast
-from urllib.parse import urlparse
+from urllib.parse import unquote, urlparse
from app.repositories.job_repository import JobRepository
from app.services.document_ingestion.command import DocumentIngestionCommand
@@ -308,7 +308,7 @@ def _build_job_response(
def _resolve_url_source_file_name(*, source_url: str, file_extension: str) -> str:
parsed_url = urlparse(source_url)
- url_basename = str(os.path.basename(parsed_url.path))
+ url_basename = unquote(str(os.path.basename(parsed_url.path)))
if url_basename and os.path.splitext(url_basename)[1].lower() == file_extension:
return url_basename
if url_basename:
diff --git a/apps/api/scripts/backfill_map_unit_indexes.py b/apps/api/scripts/backfill_map_unit_indexes.py
index ebf1a04d2..48d9bef16 100644
--- a/apps/api/scripts/backfill_map_unit_indexes.py
+++ b/apps/api/scripts/backfill_map_unit_indexes.py
@@ -7,9 +7,20 @@
deployment with ``--apply`` so each revision is rebuilt and committed
independently; without ``--apply`` it is a read-only inventory.
+Re-run after map-unit tokenizer / ``MAP_UNIT_INDEX_FORMAT_VERSION`` changes
+(e.g. v1 character-level → v2 word-level, or loading a domain userdict) so
+query hashes match stored tokens. Prefer::
+
+ python /app/scripts/backfill_map_unit_indexes.py --apply --tokens-only
+ python /app/scripts/backfill_map_unit_indexes.py --apply --tokens-only --skip-current-format
+
+``--tokens-only`` rebuilds map-unit tokens/index only (skips serving manifest
+and namespace MAP snapshot). Use full ``--apply`` only when those artifacts
+are missing.
+
Use ``--check`` after backfill to verify whether query-time snapshot
fallbacks (manifest_merge / table_scan) would still fire, and whether
-map-unit indexes are complete for scoring.
+map-unit indexes are complete for scoring at the current format version.
"""
# ruff: noqa: E402
@@ -57,7 +68,10 @@ def _bootstrap_python_path() -> None:
RetrievalNamespaceMapSnapshot,
RetrievalServingRevisionManifest,
)
-from shared.services.retrieval.map_unit_index import replace_document_map_units
+from shared.services.retrieval.map_unit_index import (
+ MAP_UNIT_INDEX_FORMAT_VERSION,
+ replace_document_map_units,
+)
from shared.services.retrieval.namespace_map_snapshot import (
patch_namespace_map_snapshot,
)
@@ -67,7 +81,7 @@ def _bootstrap_python_path() -> None:
lock_namespace_generation,
)
from shared.services.retrieval.serving_manifest import (
- decode_serving_manifest,
+ decode_namespace_map_snapshot,
persist_revision_serving_state,
)
@@ -92,6 +106,23 @@ def _build_parser() -> argparse.ArgumentParser:
parser.add_argument(
"--document-id", default="", help="Limit the backfill to one document."
)
+ parser.add_argument(
+ "--tokens-only",
+ action="store_true",
+ help=(
+ "With --apply: rebuild only document_map_units / tokens / index "
+ "(skip serving manifest, namespace MAP snapshot, and generation bump). "
+ "Use after tokenizer / format_version changes when manifests are already ready."
+ ),
+ )
+ parser.add_argument(
+ "--skip-current-format",
+ action="store_true",
+ help=(
+ "With --apply: skip revisions whose map-unit index already has "
+ f"format_version={MAP_UNIT_INDEX_FORMAT_VERSION}."
+ ),
+ )
return parser
@@ -124,7 +155,11 @@ class NamespaceFallbackReport:
@property
def ready(self) -> bool:
- return not self.would_hit_snapshot_fallback and not self.scoring_incomplete
+ return (
+ not self.would_hit_snapshot_fallback
+ and not self.scoring_incomplete
+ and self.missing_revision_manifest == 0
+ )
def check_fallback_readiness(*, document_id: str = "") -> list[NamespaceFallbackReport]:
@@ -160,7 +195,7 @@ def check_fallback_readiness(*, document_id: str = "") -> list[NamespaceFallback
snapshot_status = "missing"
else:
try:
- payload = decode_serving_manifest(
+ payload = decode_namespace_map_snapshot(
bytes(snapshot.payload_zlib),
checksum=str(snapshot.checksum),
format_version=int(snapshot.format_version),
@@ -193,9 +228,14 @@ def check_fallback_readiness(*, document_id: str = "") -> list[NamespaceFallback
select(
DocumentMapUnitIndex.document_id,
DocumentMapUnitIndex.job_result_id,
+ DocumentMapUnitIndex.format_version,
DocumentMapUnitIndex.unit_count,
DocumentMapUnitIndex.average_idf_path,
DocumentMapUnitIndex.average_idf_content,
+ DocumentMapUnitIndex.path_document_count,
+ DocumentMapUnitIndex.path_total_length,
+ DocumentMapUnitIndex.content_document_count,
+ DocumentMapUnitIndex.content_total_length,
).where(
DocumentMapUnitIndex.document_id.in_(
[document_id_value for document_id_value, _ in revisions]
@@ -205,11 +245,27 @@ def check_fallback_readiness(*, document_id: str = "") -> list[NamespaceFallback
)
index_by_revision = {
(str(document_id_value), str(job_result_id)): (
+ int(format_version or 0),
int(unit_count or 0),
float(average_idf_path or 0.0),
float(average_idf_content or 0.0),
+ path_document_count,
+ path_total_length,
+ content_document_count,
+ content_total_length,
)
- for document_id_value, job_result_id, unit_count, average_idf_path, average_idf_content in index_rows
+ for (
+ document_id_value,
+ job_result_id,
+ format_version,
+ unit_count,
+ average_idf_path,
+ average_idf_content,
+ path_document_count,
+ path_total_length,
+ content_document_count,
+ content_total_length,
+ ) in index_rows
}
missing_map_index = 0
suspicious_zero_idf = 0
@@ -218,7 +274,25 @@ def check_fallback_readiness(*, document_id: str = "") -> list[NamespaceFallback
if stats is None:
missing_map_index += 1
continue
- unit_count, average_idf_path, average_idf_content = stats
+ (
+ format_version,
+ unit_count,
+ average_idf_path,
+ average_idf_content,
+ path_document_count,
+ path_total_length,
+ content_document_count,
+ content_total_length,
+ ) = stats
+ if (
+ format_version != MAP_UNIT_INDEX_FORMAT_VERSION
+ or path_document_count is None
+ or path_total_length is None
+ or content_document_count is None
+ or content_total_length is None
+ ):
+ missing_map_index += 1
+ continue
if (
unit_count > 0
and average_idf_path == 0.0
@@ -251,7 +325,11 @@ def check_fallback_readiness(*, document_id: str = "") -> list[NamespaceFallback
would_hit_snapshot_fallback = (
snapshot_status != "ok" or missing_from_snapshot > 0
)
- scoring_incomplete = missing_map_index > 0 or suspicious_zero_idf > 0
+ # A zero average IDF is mathematically valid, notably for a
+ # two-unit corpus where every token appears in exactly one unit.
+ # Keep the count as diagnostic output, but readiness is determined
+ # by the format marker and required persisted statistics above.
+ scoring_incomplete = missing_map_index > 0
reports.append(
NamespaceFallbackReport(
user_id=user_id,
@@ -295,26 +373,36 @@ def print_fallback_check(reports: list[NamespaceFallbackReport]) -> int:
return 1 if failed else 0
-def backfill_map_unit_indexes(*, apply: bool, document_id: str = "") -> int:
+def backfill_map_unit_indexes(
+ *,
+ apply: bool,
+ document_id: str = "",
+ tokens_only: bool = False,
+ skip_current_format: bool = False,
+) -> int:
documents = _load_documents(document_id)
if not apply:
for document in documents:
print(
f"would backfill document={document.document_id} revision={document.current_job_result_id}"
+ + (" tokens_only" if tokens_only else "")
)
return len(documents)
session_factory = get_sync_session_factory()
+ backfilled = 0
+ skipped_current = 0
for document in documents:
job_result_id = document.current_job_result_id
if not job_result_id:
continue
with session_factory() as db:
- lock_namespace_generation(
- db,
- user_id=document.user_id,
- namespace=document.namespace,
- )
+ if not tokens_only:
+ lock_namespace_generation(
+ db,
+ user_id=document.user_id,
+ namespace=document.namespace,
+ )
locked_document = db.execute(
select(Document)
.where(Document.document_id == document.document_id)
@@ -333,6 +421,26 @@ def backfill_map_unit_indexes(*, apply: bool, document_id: str = "") -> int:
f"revision={job_result_id}"
)
continue
+ if skip_current_format:
+ existing = db.execute(
+ select(DocumentMapUnitIndex.format_version)
+ .where(
+ DocumentMapUnitIndex.document_id
+ == locked_document.document_id
+ )
+ .where(DocumentMapUnitIndex.job_result_id == job_result_id)
+ ).scalar_one_or_none()
+ if (
+ existing is not None
+ and int(existing) == MAP_UNIT_INDEX_FORMAT_VERSION
+ ):
+ db.rollback()
+ skipped_current += 1
+ print(
+ f"skipped current-format document={document.document_id} "
+ f"revision={job_result_id} format_version={existing}"
+ )
+ continue
scope = DocumentPublicationScope(
user_id=locked_document.user_id,
namespace=locked_document.namespace,
@@ -341,18 +449,25 @@ def backfill_map_unit_indexes(*, apply: bool, document_id: str = "") -> int:
source_file_name=str(locked_document.source_file_name or ""),
)
replace_document_map_units(db, scope=scope)
- manifest_payload = persist_revision_serving_state(db, scope=scope)
- patch_namespace_map_snapshot(
- db, scope=scope, manifest_payload=manifest_payload
- )
- advance_namespace_generation(
- db,
- user_id=scope.user_id,
- namespace=scope.namespace,
- )
+ if not tokens_only:
+ manifest_payload = persist_revision_serving_state(db, scope=scope)
+ patch_namespace_map_snapshot(
+ db, scope=scope, manifest_payload=manifest_payload
+ )
+ advance_namespace_generation(
+ db,
+ user_id=scope.user_id,
+ namespace=scope.namespace,
+ )
db.commit()
- print(f"backfilled document={document.document_id} revision={job_result_id}")
- return len(documents)
+ backfilled += 1
+ mode = "tokens_only" if tokens_only else "full"
+ print(
+ f"backfilled document={document.document_id} revision={job_result_id} mode={mode}"
+ )
+ if skip_current_format:
+ print(f"skipped_current_format={skipped_current}")
+ return backfilled
def main() -> None:
@@ -360,11 +475,19 @@ def main() -> None:
if arguments.check:
if arguments.apply:
raise SystemExit("use either --check or --apply, not both")
+ if arguments.tokens_only or arguments.skip_current_format:
+ raise SystemExit("--tokens-only/--skip-current-format require --apply")
reports = check_fallback_readiness(document_id=str(arguments.document_id))
raise SystemExit(print_fallback_check(reports))
+ if (arguments.tokens_only or arguments.skip_current_format) and not arguments.apply:
+ raise SystemExit("--tokens-only/--skip-current-format require --apply")
+
count = backfill_map_unit_indexes(
- apply=bool(arguments.apply), document_id=str(arguments.document_id)
+ apply=bool(arguments.apply),
+ document_id=str(arguments.document_id),
+ tokens_only=bool(arguments.tokens_only),
+ skip_current_format=bool(arguments.skip_current_format),
)
action = "backfilled" if arguments.apply else "found"
print(f"{action} revisions={count}")
diff --git a/apps/api/scripts/backfill_map_unit_statistics.py b/apps/api/scripts/backfill_map_unit_statistics.py
new file mode 100644
index 000000000..86214ef7b
--- /dev/null
+++ b/apps/api/scripts/backfill_map_unit_statistics.py
@@ -0,0 +1,227 @@
+# ruff: noqa: E402
+
+"""Backfill persisted per-channel BM25 statistics without rebuilding tokens.
+
+This maintenance command aggregates existing ``document_map_units`` rows and
+updates the four nullable statistics columns on the current active revision.
+It never rewrites map-unit tokens, serving manifests, or namespace snapshots.
+Each revision is committed independently so interruption is safe.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+
+def _resolve_shared_root(api_root: Path) -> Path:
+ """Resolve shared-python in source checkouts and runtime images."""
+ runtime_shared_root = api_root / "packages" / "shared-python"
+ if runtime_shared_root.is_dir():
+ return runtime_shared_root
+
+ repository_root = api_root.parent.parent
+ repository_shared_root = repository_root / "packages" / "shared-python"
+ if repository_shared_root.is_dir():
+ return repository_shared_root
+
+ raise RuntimeError(f"Could not locate shared-python package from {api_root}")
+
+
+def _bootstrap_python_path() -> None:
+ api_root = Path(__file__).resolve().parents[1]
+ shared_root = _resolve_shared_root(api_root)
+ for path in (api_root, shared_root):
+ value = os.fspath(path)
+ if value not in sys.path:
+ sys.path.insert(0, value)
+
+
+_bootstrap_python_path()
+
+from sqlalchemy import func, select, update
+from sqlalchemy.orm import Session
+
+from shared.core.database_sync import get_sync_session_factory
+from shared.models.database.document import (
+ Document,
+ DocumentMapUnit,
+ DocumentMapUnitIndex,
+)
+from shared.services.retrieval.nav.knowhere_hybrid import MAP_UNIT_INDEX_FORMAT_VERSION
+
+
+@dataclass(frozen=True)
+class RevisionStatistics:
+ path_document_count: int
+ path_total_length: int
+ content_document_count: int
+ content_total_length: int
+
+
+def _build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--apply", action="store_true", help="Write statistics.")
+ parser.add_argument(
+ "--check", action="store_true", help="Report missing or mismatched statistics."
+ )
+ parser.add_argument("--document-id", default="")
+ parser.add_argument("--user-id", default="")
+ parser.add_argument("--namespace", default="")
+ parser.add_argument("--batch-size", type=int, default=100)
+ return parser
+
+
+def _load_documents(
+ *, document_id: str, user_id: str, namespace: str
+) -> list[Document]:
+ session_factory = get_sync_session_factory()
+ with session_factory() as db:
+ statement = (
+ select(Document)
+ .where(Document.status == "active")
+ .where(Document.current_job_result_id.is_not(None))
+ .order_by(Document.document_id)
+ )
+ if document_id:
+ statement = statement.where(Document.document_id == document_id)
+ if user_id:
+ statement = statement.where(Document.user_id == user_id)
+ if namespace:
+ statement = statement.where(Document.namespace == namespace)
+ return list(db.scalars(statement).all())
+
+
+def _aggregate_statistics(
+ db: Session, *, document_id: str, job_result_id: str
+) -> RevisionStatistics:
+ statement = (
+ select(
+ func.count()
+ .filter(DocumentMapUnit.path_token_count > 0)
+ .label("path_document_count"),
+ func.coalesce(func.sum(DocumentMapUnit.path_token_count), 0).label(
+ "path_total_length"
+ ),
+ func.count()
+ .filter(DocumentMapUnit.content_token_count > 0)
+ .label("content_document_count"),
+ func.coalesce(func.sum(DocumentMapUnit.content_token_count), 0).label(
+ "content_total_length"
+ ),
+ )
+ .where(DocumentMapUnit.document_id == document_id)
+ .where(DocumentMapUnit.job_result_id == job_result_id)
+ )
+ row = db.execute(statement).one()
+ return RevisionStatistics(
+ path_document_count=int(row.path_document_count or 0),
+ path_total_length=int(row.path_total_length or 0),
+ content_document_count=int(row.content_document_count or 0),
+ content_total_length=int(row.content_total_length or 0),
+ )
+
+
+def _is_complete(index: DocumentMapUnitIndex | None, stats: RevisionStatistics) -> bool:
+ return bool(
+ index
+ and index.format_version == MAP_UNIT_INDEX_FORMAT_VERSION
+ and index.path_document_count == stats.path_document_count
+ and index.path_total_length == stats.path_total_length
+ and index.content_document_count == stats.content_document_count
+ and index.content_total_length == stats.content_total_length
+ )
+
+
+def _is_check_ready(
+ *, would_update: int, complete: int, skipped: int, documents: int
+) -> bool:
+ """Return whether a read-only inventory proves every document is ready."""
+ return would_update == 0 and skipped == 0 and complete == documents
+
+
+def _process_batch(
+ documents: list[Document], *, apply_changes: bool
+) -> tuple[int, int, int]:
+ session_factory = get_sync_session_factory()
+ updated = 0
+ complete = 0
+ skipped = 0
+ with session_factory() as db:
+ for document in documents:
+ job_result_id = str(document.current_job_result_id or "")
+ stats = _aggregate_statistics(
+ db, document_id=document.document_id, job_result_id=job_result_id
+ )
+ index_statement = (
+ select(DocumentMapUnitIndex)
+ .where(DocumentMapUnitIndex.document_id == document.document_id)
+ .where(DocumentMapUnitIndex.job_result_id == job_result_id)
+ )
+ if apply_changes:
+ index_statement = index_statement.with_for_update()
+ index = db.scalar(index_statement)
+ if index is None or index.format_version != MAP_UNIT_INDEX_FORMAT_VERSION:
+ skipped += 1
+ print(f"skip document={document.document_id} reason=missing_or_legacy_index")
+ db.rollback()
+ continue
+ if _is_complete(index, stats):
+ complete += 1
+ db.rollback()
+ continue
+ if not apply_changes:
+ updated += 1
+ db.rollback()
+ continue
+ db.execute(
+ update(DocumentMapUnitIndex)
+ .where(DocumentMapUnitIndex.id == index.id)
+ .values(
+ path_document_count=stats.path_document_count,
+ path_total_length=stats.path_total_length,
+ content_document_count=stats.content_document_count,
+ content_total_length=stats.content_total_length,
+ )
+ )
+ db.commit()
+ updated += 1
+ return updated, complete, skipped
+
+
+def main() -> None:
+ args = _build_parser().parse_args()
+ if args.apply == args.check:
+ raise SystemExit("choose exactly one of --apply or --check")
+ if args.batch_size <= 0:
+ raise SystemExit("--batch-size must be positive")
+ documents = _load_documents(
+ document_id=args.document_id.strip(),
+ user_id=args.user_id.strip(),
+ namespace=args.namespace.strip(),
+ )
+ totals = [0, 0, 0]
+ for offset in range(0, len(documents), args.batch_size):
+ batch_totals = _process_batch(
+ documents[offset : offset + args.batch_size], apply_changes=args.apply
+ )
+ totals = [left + right for left, right in zip(totals, batch_totals)]
+ action = "applied" if args.apply else "would_update"
+ print(
+ f"{action}={totals[0]} complete={totals[1]} skipped={totals[2]} "
+ f"documents={len(documents)}"
+ )
+ if args.check and not _is_check_ready(
+ would_update=totals[0],
+ complete=totals[1],
+ skipped=totals[2],
+ documents=len(documents),
+ ):
+ raise SystemExit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/apps/api/tests/contract/test_backfill_map_unit_indexes_contract.py b/apps/api/tests/contract/test_backfill_map_unit_indexes_contract.py
index 72c61f919..18f68dc13 100644
--- a/apps/api/tests/contract/test_backfill_map_unit_indexes_contract.py
+++ b/apps/api/tests/contract/test_backfill_map_unit_indexes_contract.py
@@ -1,6 +1,10 @@
from __future__ import annotations
from pathlib import Path
+from types import SimpleNamespace
+from typing import Any, cast
+
+from sqlalchemy.dialects import postgresql
def test_backfill_script_resolves_shared_package_from_runtime_image_layout(
@@ -26,3 +30,151 @@ def test_backfill_script_resolves_shared_package_from_source_checkout_layout(
shared_root.mkdir(parents=True)
assert _resolve_shared_root(api_root) == shared_root
+
+
+def test_statistics_backfill_resolves_shared_package_from_runtime_image_layout(
+ tmp_path: Path,
+) -> None:
+ from scripts.backfill_map_unit_statistics import _resolve_shared_root
+
+ api_root = tmp_path / "app"
+ shared_root = api_root / "packages" / "shared-python"
+ shared_root.mkdir(parents=True)
+
+ assert _resolve_shared_root(api_root) == shared_root
+
+
+def test_statistics_backfill_resolves_shared_package_from_source_checkout_layout(
+ tmp_path: Path,
+) -> None:
+ from scripts.backfill_map_unit_statistics import _resolve_shared_root
+
+ repository_root = tmp_path / "repository"
+ api_root = repository_root / "apps" / "api"
+ shared_root = repository_root / "packages" / "shared-python"
+ shared_root.mkdir(parents=True)
+
+ assert _resolve_shared_root(api_root) == shared_root
+
+
+def test_statistics_backfill_aggregates_positive_lengths_per_channel() -> None:
+ from scripts.backfill_map_unit_statistics import (
+ RevisionStatistics,
+ _aggregate_statistics,
+ )
+
+ class AggregateResult:
+ def one(self) -> SimpleNamespace:
+ return SimpleNamespace(
+ path_document_count=2,
+ path_total_length=9,
+ content_document_count=1,
+ content_total_length=7,
+ )
+
+ class AggregateSession:
+ statement_sql: str = ""
+
+ def execute(self, statement: Any) -> AggregateResult:
+ self.statement_sql = str(
+ statement.compile(
+ dialect=postgresql.dialect(),
+ compile_kwargs={"literal_binds": True},
+ )
+ )
+ return AggregateResult()
+
+ session = AggregateSession()
+ statistics = _aggregate_statistics(
+ cast(Any, session), document_id="doc_1", job_result_id="result_1"
+ )
+
+ assert statistics == RevisionStatistics(
+ path_document_count=2,
+ path_total_length=9,
+ content_document_count=1,
+ content_total_length=7,
+ )
+ assert "path_token_count > 0" in session.statement_sql
+ assert "content_token_count > 0" in session.statement_sql
+
+
+def test_statistics_backfill_readiness_requires_every_document_complete() -> None:
+ from scripts.backfill_map_unit_statistics import _is_check_ready
+
+ assert _is_check_ready(
+ would_update=0, complete=4, skipped=0, documents=4
+ )
+ assert not _is_check_ready(
+ would_update=1, complete=3, skipped=0, documents=4
+ )
+ assert not _is_check_ready(
+ would_update=0, complete=3, skipped=1, documents=4
+ )
+
+
+def test_statistics_backfill_completion_rejects_missing_or_legacy_indexes() -> None:
+ from scripts.backfill_map_unit_statistics import RevisionStatistics, _is_complete
+
+ statistics = RevisionStatistics(
+ path_document_count=1,
+ path_total_length=2,
+ content_document_count=1,
+ content_total_length=3,
+ )
+ legacy_index = SimpleNamespace(
+ format_version=1,
+ path_document_count=1,
+ path_total_length=2,
+ content_document_count=1,
+ content_total_length=3,
+ )
+ complete_index = SimpleNamespace(
+ format_version=2,
+ path_document_count=1,
+ path_total_length=2,
+ content_document_count=1,
+ content_total_length=3,
+ )
+
+ assert not _is_complete(None, statistics)
+ assert not _is_complete(cast(Any, legacy_index), statistics)
+ assert _is_complete(cast(Any, complete_index), statistics)
+
+
+def test_full_backfill_readiness_requires_revision_manifests() -> None:
+ from scripts.backfill_map_unit_indexes import NamespaceFallbackReport
+
+ report = NamespaceFallbackReport(
+ user_id="user_1",
+ namespace="default",
+ active_docs=1,
+ snapshot_status="ok",
+ missing_from_snapshot=0,
+ missing_map_index=0,
+ missing_revision_manifest=1,
+ suspicious_zero_idf=0,
+ would_hit_snapshot_fallback=False,
+ scoring_incomplete=False,
+ )
+
+ assert not report.ready
+
+
+def test_full_backfill_readiness_allows_mathematically_valid_zero_idf() -> None:
+ from scripts.backfill_map_unit_indexes import NamespaceFallbackReport
+
+ report = NamespaceFallbackReport(
+ user_id="user_1",
+ namespace="default",
+ active_docs=1,
+ snapshot_status="ok",
+ missing_from_snapshot=0,
+ missing_map_index=0,
+ missing_revision_manifest=0,
+ suspicious_zero_idf=1,
+ would_hit_snapshot_fallback=False,
+ scoring_incomplete=False,
+ )
+
+ assert report.ready
diff --git a/apps/api/tests/contract/test_evidence_renderer_contract.py b/apps/api/tests/contract/test_evidence_renderer_contract.py
new file mode 100644
index 000000000..8188408b9
--- /dev/null
+++ b/apps/api/tests/contract/test_evidence_renderer_contract.py
@@ -0,0 +1,45 @@
+from shared.services.retrieval.execution.routes import _render_rows_evidence
+
+
+def test_render_rows_evidence_should_group_by_traceable_path() -> None:
+ rows = [
+ {
+ "chunk_id": "c2",
+ "content": "second section content",
+ "sort_order": 2,
+ "source": {
+ "source_file_name": "alpha.pdf",
+ "section_path": "Alpha / Two",
+ },
+ },
+ {
+ "chunk_id": "c1",
+ "content": "first section content\nwith more detail",
+ "sort_order": 1,
+ "source": {
+ "source_file_name": "alpha.pdf",
+ "section_path": "Alpha / One",
+ },
+ },
+ {
+ "chunk_id": "c3",
+ "content": "",
+ "source_file_name": "beta.pdf",
+ "section_path": "Beta / Table",
+ },
+ ]
+
+ evidence_text = _render_rows_evidence(rows)
+
+ assert "[E1]" in evidence_text
+ assert "[E2]" in evidence_text
+ assert "[E3]" in evidence_text
+ assert "[§ alpha.pdf / Alpha / One]" in evidence_text
+ assert "[§ alpha.pdf / Alpha / Two]" in evidence_text
+ assert "[§ beta.pdf / Beta / Table]" in evidence_text
+ assert "first section content" in evidence_text
+ assert "second section content" in evidence_text
+ assert "" in evidence_text
+ assert "[Document]" not in evidence_text
+ assert "▸" not in evidence_text
+ assert "┈" not in evidence_text
diff --git a/apps/api/tests/contract/test_legacy_evidence_renderer_contract.py b/apps/api/tests/contract/test_legacy_evidence_renderer_contract.py
deleted file mode 100644
index dcf4c086f..000000000
--- a/apps/api/tests/contract/test_legacy_evidence_renderer_contract.py
+++ /dev/null
@@ -1,43 +0,0 @@
-from shared.services.retrieval.hydration.legacy_evidence import render_legacy_evidence_text
-
-
-def test_render_legacy_evidence_text_should_group_documents_and_sections() -> None:
- rows = [
- {
- "chunk_id": "c2",
- "content": "second section content",
- "sort_order": 2,
- "source": {
- "source_file_name": "alpha.pdf",
- "section_path": "Alpha / Two",
- },
- },
- {
- "chunk_id": "c1",
- "content": "first section content\nwith more detail",
- "sort_order": 1,
- "source": {
- "source_file_name": "alpha.pdf",
- "section_path": "Alpha / One",
- },
- },
- {
- "chunk_id": "c3",
- "content": "",
- "source_file_name": "beta.pdf",
- "section_path": "Beta / Table",
- },
- ]
-
- evidence_text = render_legacy_evidence_text(rows)
-
- assert "[Document] alpha.pdf" in evidence_text
- assert "[Document] beta.pdf" in evidence_text
- assert "▸ Alpha / One" in evidence_text
- assert "▸ Alpha / Two" in evidence_text
- assert " ┈ first section content" in evidence_text
- assert " ┈ with more detail" in evidence_text
- assert " ┈ " in evidence_text
- assert "\u3010\u6587\u6863\u3011" not in evidence_text
- assert "[\u8868\u683c\u5185\u5bb9]" not in evidence_text
- assert "[\u56fe\u7247" not in evidence_text
diff --git a/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py b/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py
index 63941a1f9..fe5b732ca 100644
--- a/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py
+++ b/apps/api/tests/contract/test_retrieval_classic_map_unit_contract.py
@@ -5,7 +5,8 @@
from typing import Any, cast
from uuid import uuid4
-from httpx import AsyncClient
+import pytest
+from httpx import AsyncClient, Response
from sqlalchemy import Engine, event, select
from shared.models.database.document import DocumentMapUnit
@@ -13,6 +14,10 @@
replace_document_revision_content,
)
from shared.services.retrieval.publication_models import DocumentPublicationScope
+from shared.services.retrieval.search.map_unit_discovery import (
+ DiscoveryResult,
+ map_unit_discovery,
+)
from shared.services.retrieval.serving_generation import lock_namespace_generation
from tests.support.contract_database import ContractDatabase
from tests.support.retrieval_snapshot_support import contract_db_session
@@ -173,6 +178,561 @@ def capture_frequency_query(
assert "FROM matching_tokens" in statements[-1]
+async def test_classic_route_only_uses_token_selective_projection_for_unfiltered_scope(
+ developer_api_client_factory: Callable[
+ [], AbstractAsyncContextManager[AsyncClient]
+ ],
+) -> None:
+ identifier = uuid4().hex[:8]
+ namespace = f"classic-projection-{identifier}"
+ projection_statements: list[str] = []
+
+ def capture_unit_projection(
+ _connection: Any,
+ _cursor: Any,
+ statement: str,
+ _parameters: Any,
+ _context: Any,
+ _executemany: bool,
+ ) -> None:
+ if "SELECT DISTINCT scoped_units.*" in statement:
+ projection_statements.append("token_selective")
+ elif "SELECT * FROM scoped_units" in statement:
+ projection_statements.append("legacy")
+
+ event.listen(Engine, "before_cursor_execute", capture_unit_projection)
+ try:
+ async with developer_api_client_factory() as api_client:
+ await _publish_document(
+ namespace=namespace,
+ source_file_name="projection.pdf",
+ chunks=[
+ {
+ "chunk_id": f"projection-hit-{identifier}",
+ "type": "text",
+ "content": "projection token marker",
+ "path": "projection.pdf/Root/Section/hit",
+ "order": 1,
+ "metadata": {},
+ },
+ {
+ "chunk_id": f"projection-filler-a-{identifier}",
+ "type": "text",
+ "content": "unrelated filler a",
+ "path": "projection.pdf/Root/Section/a",
+ "order": 2,
+ "metadata": {},
+ },
+ {
+ "chunk_id": f"projection-filler-b-{identifier}",
+ "type": "text",
+ "content": "unrelated filler b",
+ "path": "projection.pdf/Root/Section/b",
+ "order": 3,
+ "metadata": {},
+ },
+ {
+ "chunk_id": f"projection-filler-c-{identifier}",
+ "type": "text",
+ "content": "unrelated filler c",
+ "path": "projection.pdf/Root/Section/c",
+ "order": 4,
+ "metadata": {},
+ },
+ ],
+ )
+ unfiltered_response = await api_client.post(
+ "/api/v1/retrieval/query",
+ json={
+ "namespace": namespace,
+ "query": "projection token marker",
+ "top_k": 1,
+ "use_agentic": False,
+ },
+ )
+ filtered_response = await api_client.post(
+ "/api/v1/retrieval/query",
+ json={
+ "namespace": namespace,
+ "query": "projection token marker",
+ "top_k": 1,
+ "use_agentic": False,
+ "signal_paths": ["Section"],
+ "filter_mode": "keep",
+ },
+ )
+ finally:
+ event.remove(Engine, "before_cursor_execute", capture_unit_projection)
+
+ assert unfiltered_response.status_code == 200
+ assert filtered_response.status_code == 200
+ assert projection_statements[:2] == ["token_selective", "legacy"]
+
+
+async def test_unfiltered_revision_pins_reuse_index_metadata_without_scoped_revision_cte(
+ developer_api_client_factory: Callable[
+ [], AbstractAsyncContextManager[AsyncClient]
+ ],
+) -> None:
+ identifier = uuid4().hex[:8]
+ namespace = f"classic-pinned-index-{identifier}"
+ index_statements: list[str] = []
+
+ def capture_index_query(
+ _connection: Any,
+ _cursor: Any,
+ statement: str,
+ _parameters: Any,
+ _context: Any,
+ _executemany: bool,
+ ) -> None:
+ if (
+ "document_map_unit_indexes" in statement
+ and "average_idf_path" in statement
+ ):
+ index_statements.append(statement)
+
+ async with developer_api_client_factory():
+ document = await _publish_document(
+ namespace=namespace,
+ source_file_name="pinned-index.pdf",
+ chunks=[
+ {
+ "chunk_id": f"pinned-hit-{identifier}",
+ "type": "text",
+ "content": "pinned revision semantic marker",
+ "path": "pinned-index.pdf/Root/Section/hit",
+ "order": 1,
+ "metadata": {},
+ },
+ {
+ "chunk_id": f"pinned-filler-{identifier}",
+ "type": "text",
+ "content": "unrelated filler",
+ "path": "pinned-index.pdf/Root/Section/filler",
+ "order": 2,
+ "metadata": {},
+ },
+ ],
+ )
+
+ def result_signature(result: Any) -> list[tuple[Any, ...]]:
+ rows = list(result.payload.get("fused_rows") or [])
+ return [
+ (
+ row.get("chunk_id"),
+ row.get("document_id"),
+ row.get("job_result_id"),
+ row.get("section_path"),
+ row.get("score"),
+ row.get("discovery_score"),
+ row.get("content"),
+ )
+ for row in rows
+ ]
+
+ event.listen(Engine, "before_cursor_execute", capture_index_query)
+ try:
+ async with contract_db_session() as db:
+ unpinned = await map_unit_discovery(
+ db,
+ user_id=_USER_ID,
+ namespace=namespace,
+ query="pinned revision semantic marker",
+ top_k=10,
+ exclude_document_ids=[],
+ exclude_sections=[],
+ revision_pins=None,
+ )
+
+ index_statements.clear()
+
+ async with contract_db_session() as db:
+ pinned = await map_unit_discovery(
+ db,
+ user_id=_USER_ID,
+ namespace=namespace,
+ query="pinned revision semantic marker",
+ top_k=10,
+ exclude_document_ids=[],
+ exclude_sections=[],
+ revision_pins={
+ document["document_id"]: document["job_result_id"]
+ },
+ )
+ finally:
+ event.remove(Engine, "before_cursor_execute", capture_index_query)
+
+ assert result_signature(pinned) == result_signature(unpinned)
+ assert index_statements
+ assert all("JOIN (VALUES" in statement for statement in index_statements)
+ assert all("scoped_units AS" not in statement for statement in index_statements)
+ assert all(
+ "SELECT DISTINCT document_id, job_result_id" not in statement
+ for statement in index_statements
+ )
+
+
+async def test_classic_discovery_returns_empty_for_an_empty_revision_pin(
+ developer_api_client_factory: Callable[
+ [], AbstractAsyncContextManager[AsyncClient]
+ ],
+) -> None:
+ namespace: str = f"classic-empty-pins-{uuid4().hex[:8]}"
+ async with developer_api_client_factory():
+ async with contract_db_session() as db:
+ result: DiscoveryResult = await map_unit_discovery(
+ db,
+ user_id=_USER_ID,
+ namespace=namespace,
+ query="empty revision marker",
+ top_k=1,
+ exclude_document_ids=[],
+ exclude_sections=[],
+ revision_pins={},
+ )
+
+ assert result.payload["fused_rows"] == []
+
+
+async def test_classic_route_falls_back_for_v1_index_with_excluded_document(
+ developer_api_client_factory: Callable[
+ [], AbstractAsyncContextManager[AsyncClient]
+ ],
+) -> None:
+ identifier = uuid4().hex[:8]
+ namespace = f"classic-v1-fallback-{identifier}"
+ legacy_queries: list[str] = []
+
+ def capture_legacy_query(
+ _connection: Any,
+ _cursor: Any,
+ statement: str,
+ _parameters: Any,
+ _context: Any,
+ _executemany: bool,
+ ) -> None:
+ if "plainto_tsquery('simple'" in statement:
+ legacy_queries.append(statement)
+
+ event.listen(Engine, "before_cursor_execute", capture_legacy_query)
+ try:
+ async with developer_api_client_factory() as api_client:
+ first = await _publish_document(
+ namespace=namespace,
+ source_file_name="legacy-fallback.pdf",
+ chunks=[
+ {
+ "chunk_id": f"legacy-hit-{identifier}",
+ "type": "text",
+ "content": "legacy fallback marker",
+ "path": "legacy-fallback.pdf/Root/Section/body",
+ "order": 1,
+ "metadata": {},
+ },
+ {
+ "chunk_id": f"legacy-filler-a-{identifier}",
+ "type": "text",
+ "content": "unrelated legacy filler a",
+ "path": "legacy-fallback.pdf/Root/Section/a",
+ "order": 2,
+ "metadata": {},
+ },
+ {
+ "chunk_id": f"legacy-filler-b-{identifier}",
+ "type": "text",
+ "content": "unrelated legacy filler b",
+ "path": "legacy-fallback.pdf/Root/Section/b",
+ "order": 3,
+ "metadata": {},
+ },
+ ],
+ )
+ excluded = await _publish_document(
+ namespace=namespace,
+ source_file_name="excluded.pdf",
+ chunks=[
+ {
+ "chunk_id": f"excluded-{identifier}",
+ "type": "text",
+ "content": "unrelated filler",
+ "path": "excluded.pdf/Root/Section/body",
+ "order": 1,
+ "metadata": {},
+ }
+ ],
+ )
+ await ContractDatabase.execute(
+ """
+ UPDATE document_map_unit_indexes
+ SET format_version = 1
+ WHERE document_id = :document_id
+ """,
+ {"document_id": first["document_id"]},
+ )
+ response = await api_client.post(
+ "/api/v1/retrieval/query",
+ json={
+ "namespace": namespace,
+ "query": "legacy fallback marker",
+ "top_k": 1,
+ "use_agentic": False,
+ "exclude_document_ids": [excluded["document_id"]],
+ },
+ )
+ finally:
+ event.remove(Engine, "before_cursor_execute", capture_legacy_query)
+
+ assert response.status_code == 200
+ body = cast(dict[str, object], response.json())
+ results = cast(list[dict[str, object]], body["results"])
+ assert len(results) == 1
+ assert results[0]["chunk_id"] == f"legacy-hit-{identifier}"
+ assert legacy_queries
+
+
+async def test_classic_discovery_preserves_results_before_statistics_backfill(
+ developer_api_client_factory: Callable[
+ [], AbstractAsyncContextManager[AsyncClient]
+ ],
+) -> None:
+ identifier = uuid4().hex[:8]
+ namespace = f"classic-statistics-parity-{identifier}"
+ legacy_queries: list[str] = []
+
+ def capture_legacy_query(
+ _connection: Any,
+ _cursor: Any,
+ statement: str,
+ _parameters: Any,
+ _context: Any,
+ _executemany: bool,
+ ) -> None:
+ if "plainto_tsquery('simple'" in statement:
+ legacy_queries.append(statement)
+
+ def result_signature(result: DiscoveryResult) -> list[tuple[Any, ...]]:
+ return [
+ (
+ row.get("chunk_id"),
+ row.get("document_id"),
+ row.get("job_result_id"),
+ row.get("section_path"),
+ row.get("source_file_name"),
+ row.get("score"),
+ row.get("discovery_score"),
+ row.get("content"),
+ row.get("chunk_metadata"),
+ )
+ for row in list(result.payload.get("fused_rows") or [])
+ ]
+
+ event.listen(Engine, "before_cursor_execute", capture_legacy_query)
+ try:
+ async with developer_api_client_factory():
+ document = await _publish_document(
+ namespace=namespace,
+ source_file_name="statistics-parity.pdf",
+ chunks=[
+ {
+ "chunk_id": f"statistics-hit-{identifier}",
+ "type": "text",
+ "content": "statistics parity retrieval marker",
+ "path": "statistics-parity.pdf/Root/Section/hit",
+ "order": 1,
+ "metadata": {},
+ },
+ {
+ "chunk_id": f"statistics-filler-a-{identifier}",
+ "type": "text",
+ "content": "unrelated statistics filler a",
+ "path": "statistics-parity.pdf/Root/Section/a",
+ "order": 2,
+ "metadata": {},
+ },
+ {
+ "chunk_id": f"statistics-filler-b-{identifier}",
+ "type": "text",
+ "content": "unrelated statistics filler b",
+ "path": "statistics-parity.pdf/Root/Section/b",
+ "order": 3,
+ "metadata": {},
+ },
+ ],
+ )
+ query = "statistics parity retrieval marker"
+ async with contract_db_session() as db:
+ post_backfill = await map_unit_discovery(
+ db,
+ user_id=_USER_ID,
+ namespace=namespace,
+ query=query,
+ top_k=3,
+ exclude_document_ids=[],
+ exclude_sections=[],
+ )
+
+ await ContractDatabase.execute(
+ """
+ UPDATE document_map_unit_indexes
+ SET path_document_count = NULL,
+ path_total_length = NULL,
+ content_document_count = NULL,
+ content_total_length = NULL
+ WHERE document_id = :document_id
+ """,
+ {"document_id": document["document_id"]},
+ )
+
+ async with contract_db_session() as db:
+ pre_backfill = await map_unit_discovery(
+ db,
+ user_id=_USER_ID,
+ namespace=namespace,
+ query=query,
+ top_k=3,
+ exclude_document_ids=[],
+ exclude_sections=[],
+ )
+ finally:
+ event.remove(Engine, "before_cursor_execute", capture_legacy_query)
+
+ assert result_signature(pre_backfill) == result_signature(post_backfill)
+ assert legacy_queries == []
+
+
+@pytest.mark.parametrize(
+ "incomplete_index_kind", ["legacy_format", "missing_index", "missing_tokens"]
+)
+async def test_unfiltered_classic_route_falls_back_when_selective_rows_are_unavailable(
+ developer_api_client_factory: Callable[
+ [], AbstractAsyncContextManager[AsyncClient]
+ ],
+ incomplete_index_kind: str,
+) -> None:
+ identifier: str = uuid4().hex[:8]
+ namespace: str = f"classic-token-fallback-{incomplete_index_kind}-{identifier}"
+ legacy_queries: list[str] = []
+
+ def capture_legacy_query(
+ _connection: object,
+ _cursor: object,
+ statement: str,
+ _parameters: object,
+ _context: object,
+ _executemany: bool,
+ ) -> None:
+ if "plainto_tsquery('simple'" in statement:
+ legacy_queries.append(statement)
+
+ event.listen(Engine, "before_cursor_execute", capture_legacy_query)
+ try:
+ async with developer_api_client_factory() as api_client:
+ document: dict[str, str] = await _publish_document(
+ namespace=namespace,
+ source_file_name="legacy-token-hash.pdf",
+ chunks=[
+ {
+ "chunk_id": f"legacy-token-hit-{identifier}",
+ "type": "text",
+ "content": "legacy token fallback marker",
+ "path": "legacy-token-hash.pdf/Root/Section/body",
+ "order": 1,
+ "metadata": {},
+ },
+ {
+ "chunk_id": f"legacy-token-filler-a-{identifier}",
+ "type": "text",
+ "content": "unrelated legacy filler a",
+ "path": "legacy-token-hash.pdf/Root/Section/a",
+ "order": 2,
+ "metadata": {},
+ },
+ {
+ "chunk_id": f"legacy-token-filler-b-{identifier}",
+ "type": "text",
+ "content": "unrelated legacy filler b",
+ "path": "legacy-token-hash.pdf/Root/Section/b",
+ "order": 3,
+ "metadata": {},
+ },
+ ],
+ )
+ if incomplete_index_kind == "legacy_format":
+ await ContractDatabase.execute(
+ """
+ UPDATE document_map_unit_indexes
+ SET format_version = 1
+ WHERE document_id = :document_id
+ """,
+ {"document_id": document["document_id"]},
+ )
+ await ContractDatabase.execute(
+ """
+ UPDATE document_map_unit_tokens
+ SET token_hash = :legacy_token_hash
+ WHERE map_unit_id IN (
+ SELECT id
+ FROM document_map_units
+ WHERE document_id = :document_id
+ )
+ """,
+ {
+ "document_id": document["document_id"],
+ "legacy_token_hash": "legacy-token-hash",
+ },
+ )
+ elif incomplete_index_kind == "missing_tokens":
+ await ContractDatabase.execute(
+ """
+ DELETE FROM document_map_unit_tokens
+ WHERE map_unit_id IN (
+ SELECT id
+ FROM document_map_units
+ WHERE document_id = :document_id
+ )
+ """,
+ {"document_id": document["document_id"]},
+ )
+ else:
+ await ContractDatabase.execute(
+ """
+ DELETE FROM document_map_unit_indexes
+ WHERE document_id = :document_id
+ """,
+ {"document_id": document["document_id"]},
+ )
+ await ContractDatabase.execute(
+ """
+ DELETE FROM document_map_unit_tokens
+ WHERE map_unit_id IN (
+ SELECT id
+ FROM document_map_units
+ WHERE document_id = :document_id
+ )
+ """,
+ {"document_id": document["document_id"]},
+ )
+ response: Response = await api_client.post(
+ "/api/v1/retrieval/query",
+ json={
+ "namespace": namespace,
+ "query": "legacy token fallback marker",
+ "top_k": 1,
+ "use_agentic": False,
+ },
+ )
+ finally:
+ event.remove(Engine, "before_cursor_execute", capture_legacy_query)
+
+ assert response.status_code == 200
+ body = cast(dict[str, object], response.json())
+ results = cast(list[dict[str, object]], body["results"])
+ assert len(results) == 1
+ assert results[0]["chunk_id"] == f"legacy-token-hit-{identifier}"
+ assert legacy_queries
+
+
async def test_classic_route_image_filter_scores_only_units_with_images(
developer_api_client_factory: Callable[
[], AbstractAsyncContextManager[AsyncClient]
diff --git a/apps/api/tests/contract/test_retrieval_lazy_tree_contract.py b/apps/api/tests/contract/test_retrieval_lazy_tree_contract.py
index a920bb396..9571f151f 100644
--- a/apps/api/tests/contract/test_retrieval_lazy_tree_contract.py
+++ b/apps/api/tests/contract/test_retrieval_lazy_tree_contract.py
@@ -70,7 +70,7 @@ def node_meta(self, section_id: str) -> NodeMeta:
budget_chars=100,
)
- assert result.evidence_text == "[E1]\n[§ Child]\nevidence"
+ assert result.evidence_text == "[E1]\n[§ Root / Child]\nevidence"
assert provider.metadata_calls == 0
@@ -102,4 +102,4 @@ def test_evidence_pack_identifies_header_owners_from_parent_chain() -> None:
)
assert result.kept_chunks == [chunks[1]]
- assert result.evidence_text == "[E1]\n[§ Child]\nchild evidence"
+ assert result.evidence_text == "[E1]\n[§ Root / Parent / Child]\nchild evidence"
diff --git a/apps/api/tests/contract/test_retrieval_map_score_parity_contract.py b/apps/api/tests/contract/test_retrieval_map_score_parity_contract.py
new file mode 100644
index 000000000..c7d651c07
--- /dev/null
+++ b/apps/api/tests/contract/test_retrieval_map_score_parity_contract.py
@@ -0,0 +1,88 @@
+"""Contract tests for map score pooling semantics."""
+
+from __future__ import annotations
+
+from typing import Final
+
+import pytest
+
+from shared.services.retrieval.nav.nav_map_scores import _pool_unit_scores_to_tree
+
+
+_CASES: Final[tuple[tuple[dict[str, list[str]], set[str], dict[str, float]], ...]] = (
+ (
+ {"root-a": ["section-a", "section-b"], "section-a": [], "section-b": []},
+ {"section-a", "section-b"},
+ {"section-a": 0.4, "section-b": 0.8, "root-a__self": 0.2},
+ ),
+ (
+ {
+ "root-a": ["parent-a"],
+ "parent-a": ["leaf-a", "leaf-b"],
+ "leaf-a": [],
+ "leaf-b": [],
+ "root-b": ["leaf-c"],
+ "leaf-c": [],
+ },
+ {"leaf-a", "leaf-b", "leaf-c"},
+ {
+ "leaf-a": 0.9,
+ "leaf-b": 0.3,
+ "leaf-c": 0.7,
+ "parent-a__self": 0.95,
+ },
+ ),
+)
+
+
+def _legacy_pool(
+ children_map: dict[str, list[str]],
+ leaves: set[str],
+ unit_scores: dict[str, float],
+) -> dict[str, float]:
+ map_scores = {
+ leaf_id: float(unit_scores.get(leaf_id, 0.0) or 0.0) for leaf_id in leaves
+ }
+
+ def score_node(section_id: str) -> float:
+ if section_id in map_scores:
+ return map_scores[section_id]
+ children = children_map.get(section_id) or []
+ if not children:
+ score = float(unit_scores.get(section_id, 0.0) or 0.0)
+ map_scores[section_id] = score
+ return score
+ descendants: list[str] = []
+
+ def collect(section: str) -> None:
+ nested = children_map.get(section) or []
+ if not nested:
+ if section in leaves:
+ descendants.append(section)
+ return
+ for child in nested:
+ collect(child)
+
+ collect(section_id)
+ parts = [float(unit_scores.get(leaf_id, 0.0) or 0.0) for leaf_id in descendants]
+ self_key = f"{section_id}__self"
+ if self_key in unit_scores:
+ parts.append(float(unit_scores[self_key]))
+ score = float(max(parts)) if parts else 0.0
+ map_scores[section_id] = score
+ return score
+
+ for section_id in children_map:
+ score_node(section_id)
+ return map_scores
+
+
+@pytest.mark.parametrize("children_map, leaves, unit_scores", _CASES)
+def test_map_score_pooling_preserves_legacy_semantics(
+ children_map: dict[str, list[str]],
+ leaves: set[str],
+ unit_scores: dict[str, float],
+) -> None:
+ assert _pool_unit_scores_to_tree(children_map, leaves, unit_scores) == _legacy_pool(
+ children_map, leaves, unit_scores
+ )
diff --git a/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py b/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py
index b4b047d86..5128382fc 100644
--- a/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py
+++ b/apps/api/tests/contract/test_retrieval_map_unit_index_contract.py
@@ -6,7 +6,7 @@
from uuid import uuid4
from httpx import AsyncClient
-from sqlalchemy import delete, select, text
+from sqlalchemy import Engine, delete, event, select, text
from shared.models.database.document import (
DocumentMapUnit,
@@ -21,6 +21,7 @@
compute_corpus_map_and_unit_scores,
select_map_highlights,
)
+from shared.services.retrieval.hydration.connected import hydrate_connected_target_rows
from shared.services.retrieval.nav.nav_knowhere import (
KnowhereProvider,
LazyKnowhereProvider,
@@ -56,7 +57,11 @@ def __init__(self) -> None:
def execute(self, statement: str, parameters: object = None) -> None:
executions.append((statement, parameters))
if "document_map_unit_indexes" in statement:
- self.rows = [(document_id, job_result_id, 1, 1, 0.0, 0.0)]
+ self.rows = [
+ (document_id, job_result_id, 2, 1, 0.0, 0.0, 1, 1, 1, 1)
+ ]
+ elif "FROM document_sections" in statement:
+ self.rows = [(document_id, job_result_id, 1)]
elif "FROM document_map_units AS units" in statement:
self.rows = [("unit-frequency", document_id, "chunk-frequency", "section-frequency", 1, 1)]
elif "FROM document_map_unit_tokens" in statement:
@@ -98,18 +103,30 @@ def close(self) -> None:
)
assert corpus is not None
+ selective_executions = [
+ statement
+ for statement, _parameters in executions
+ if "SELECT DISTINCT map_unit_id" in statement
+ ]
+ assert len(selective_executions) == 1
+ assert "JOIN (VALUES" in selective_executions[0]
frequency_executions = [
(statement, parameters)
for statement, parameters in executions
if "FROM document_map_unit_tokens" in statement
+ and "SELECT DISTINCT map_unit_id" not in statement
]
assert len(frequency_executions) == 1
statement, parameters = frequency_executions[0]
- assert "map_unit_id = ANY" in statement
+ assert "scoped_units AS MATERIALIZED" in statement
+ assert "JOIN scoped_units" in statement
+ assert "channel = ANY" in statement
assert "token_hash = ANY" in statement
+ assert "map_unit_id = ANY" not in statement
assert isinstance(parameters, list)
assert parameters[0] == ["unit-frequency"]
- assert parameters[1] == [
+ assert parameters[1] == ["path", "content"]
+ assert parameters[2] == [
"6e51d6a3d90b6a3243d38e6da6b3f31f49867c1360beba83da8ca9630f9672c7"
]
@@ -281,6 +298,18 @@ async def test_published_map_units_preserve_scores_without_chunk_payload_reads(
)
assert index.unit_count == len(expected_units)
+ assert index.path_document_count == sum(
+ unit.path_token_count > 0 for unit in persisted_units
+ )
+ assert index.path_total_length == sum(
+ unit.path_token_count for unit in persisted_units
+ )
+ assert index.content_document_count == sum(
+ unit.content_token_count > 0 for unit in persisted_units
+ )
+ assert index.content_total_length == sum(
+ unit.content_token_count for unit in persisted_units
+ )
assert [unit.unit_id for unit in persisted_units] == [
str(unit["chunk_id"]) for unit in expected_units
]
@@ -493,6 +522,100 @@ def record_reference_load(
snapshot.close()
+async def test_connected_hydration_does_not_load_legacy_job_chunks(
+ developer_api_client_factory: Callable[
+ [], AbstractAsyncContextManager[AsyncClient]
+ ],
+) -> None:
+ identifier = uuid4().hex[:8]
+ namespace = f"connected-job-{identifier}"
+ document_id = f"doc_connected_{identifier}"
+ job_id = f"job_connected_{identifier}"
+ job_result_id = f"result_connected_{identifier}"
+ statements: list[str] = []
+
+ def capture_job_chunk_query(
+ _connection: Any,
+ _cursor: Any,
+ statement: str,
+ _parameters: Any,
+ _context: Any,
+ _executemany: bool,
+ ) -> None:
+ if "job_chunks" in statement.lower():
+ statements.append(statement)
+
+ async with developer_api_client_factory():
+ await _seed_revision(
+ namespace=namespace,
+ document_id=document_id,
+ job_id=job_id,
+ job_result_id=job_result_id,
+ )
+ scope = DocumentPublicationScope(
+ user_id=_USER_ID,
+ namespace=namespace,
+ document_id=document_id,
+ job_result_id=job_result_id,
+ source_file_name="connected.pdf",
+ )
+ chunks = [
+ {
+ "chunk_id": "body-connected",
+ "type": "text",
+ "content": "body connected evidence",
+ "path": "connected.pdf/Root/Section/body",
+ "order": 1,
+ "metadata": {"connect_to": [{"target": "asset-connected"}]},
+ },
+ {
+ "chunk_id": "asset-connected",
+ "type": "image",
+ "content": "asset connected summary",
+ "path": "images/asset-connected.png",
+ "order": 2,
+ "file_path": "images/asset-connected.png",
+ "metadata": {},
+ },
+ ]
+ async with contract_db_session() as db:
+ await db.run_sync(
+ lambda sync_db: _publish_revision_with_generation_lock(
+ sync_db,
+ scope=scope,
+ chunks=chunks,
+ )
+ )
+ await db.commit()
+
+ event.listen(Engine, "before_cursor_execute", capture_job_chunk_query)
+ try:
+ async with contract_db_session() as db:
+ hydrated = await hydrate_connected_target_rows(
+ db=db,
+ rows=[
+ {
+ "document_id": document_id,
+ "job_result_id": job_result_id,
+ "chunk_id": "body-connected",
+ "chunk_type": "text",
+ "chunk_metadata": {
+ "connect_to": [{"target": "asset-connected"}]
+ },
+ }
+ ],
+ exclude_document_ids=[],
+ exclude_sections=[],
+ revision_pins={document_id: job_result_id},
+ )
+ finally:
+ event.remove(Engine, "before_cursor_execute", capture_job_chunk_query)
+
+ assert [row["chunk_id"] for row in hydrated] == ["asset-connected"]
+ assert hydrated[0]["job_id"] == job_id
+ assert statements == []
+
+
def test_incomplete_index_returns_empty_scores() -> None:
first_sections = [
SectionRow("root-a", None, "Root A", "Root A", 0, "", 0),
diff --git a/apps/api/tests/contract/test_retrieval_serving_manifest_contract.py b/apps/api/tests/contract/test_retrieval_serving_manifest_contract.py
index 5bb68c063..d80c688da 100644
--- a/apps/api/tests/contract/test_retrieval_serving_manifest_contract.py
+++ b/apps/api/tests/contract/test_retrieval_serving_manifest_contract.py
@@ -5,8 +5,11 @@
import pytest
from shared.services.retrieval.serving_manifest import (
+ NAMESPACE_MAP_SNAPSHOT_FORMAT_VERSION,
SERVING_MANIFEST_FORMAT_VERSION,
+ decode_namespace_map_snapshot,
decode_serving_manifest,
+ encode_namespace_map_snapshot,
encode_serving_manifest,
)
@@ -49,3 +52,55 @@ def test_serving_manifest_rejects_unknown_version() -> None:
checksum=checksum,
format_version=SERVING_MANIFEST_FORMAT_VERSION + 1,
)
+
+
+def test_namespace_snapshot_uses_routing_only_v2_and_reads_legacy_v1() -> None:
+ payload = {
+ "documents": {
+ "doc_1": {
+ "job_result_id": "result_1",
+ "job_id": "job_1",
+ "source_file_name": "private.pdf",
+ "sections": [
+ {
+ "section_id": "sec_1",
+ "section_path": "Root",
+ "section_title": "Root",
+ "section_level": 0,
+ "summary": "summary",
+ "sort_order": 0,
+ "unused": "drop",
+ }
+ ],
+ "chunks": [
+ {
+ "chunk_id": "chunk_1",
+ "section_id": "sec_1",
+ "chunk_type": "text",
+ "sort_order": 0,
+ "connect_to": [],
+ "content": "drop",
+ }
+ ],
+ }
+ }
+ }
+ compressed, checksum, version = encode_namespace_map_snapshot(payload)
+
+ assert version == NAMESPACE_MAP_SNAPSHOT_FORMAT_VERSION
+ decoded = decode_namespace_map_snapshot(
+ compressed, checksum=checksum, format_version=version
+ )
+ document = decoded["documents"]["doc_1"]
+ assert "source_file_name" not in document
+ assert "unused" not in document["sections"][0]
+ assert "content" not in document["chunks"][0]
+
+ legacy_compressed, legacy_checksum, legacy_version = encode_serving_manifest(
+ payload
+ )
+ assert decode_namespace_map_snapshot(
+ legacy_compressed,
+ checksum=legacy_checksum,
+ format_version=legacy_version,
+ ) == payload
diff --git a/apps/api/tests/contract/test_retrieval_snapshot_consistency_contract.py b/apps/api/tests/contract/test_retrieval_snapshot_consistency_contract.py
index bbe03e059..a30af3c31 100644
--- a/apps/api/tests/contract/test_retrieval_snapshot_consistency_contract.py
+++ b/apps/api/tests/contract/test_retrieval_snapshot_consistency_contract.py
@@ -8,11 +8,13 @@
from httpx import AsyncClient
import pytest
from sqlalchemy import Executable, Result
+from sqlalchemy.exc import SQLAlchemyError
from shared.services.retrieval.execution.reference_resolver import (
resolve_workflow_references,
)
from shared.services.retrieval.nav_snapshot import SnapshotSession, load_nav_snapshot
+from shared.services.retrieval.nav_snapshot import _resolve_namespace_snapshot_entries
from tests.support.retrieval_snapshot_support import contract_db_session
from tests.support.contract_database import ContractDatabase
@@ -20,6 +22,32 @@
_USER_ID = "local-dev-user"
+class _GenerationUnavailableSession:
+ def __init__(self) -> None:
+ self.rollback_count = 0
+
+ async def execute(self, _statement: Executable) -> Result[tuple[object, ...]]:
+ raise SQLAlchemyError("generation table unavailable")
+
+ async def rollback(self) -> None:
+ self.rollback_count += 1
+
+
+@pytest.mark.asyncio
+async def test_snapshot_loader_falls_back_when_generation_cannot_be_verified() -> None:
+ session = _GenerationUnavailableSession()
+
+ result = await _resolve_namespace_snapshot_entries(
+ session,
+ user_id=_USER_ID,
+ namespace="default",
+ document_revisions=[("doc-a", "result-a")],
+ )
+
+ assert result is None
+ assert session.rollback_count == 1
+
+
class _PublishingSession:
def __init__(
self,
diff --git a/apps/api/tests/contract/test_retrieval_snapshot_redis_contract.py b/apps/api/tests/contract/test_retrieval_snapshot_redis_contract.py
new file mode 100644
index 000000000..299341336
--- /dev/null
+++ b/apps/api/tests/contract/test_retrieval_snapshot_redis_contract.py
@@ -0,0 +1,181 @@
+"""Contract tests for binary namespace snapshot caching."""
+
+from __future__ import annotations
+
+import pytest
+from sqlalchemy import Executable
+
+from shared.core.config.redis import RedisConfig, RedisConfigManager
+from shared.services.redis.redis_service_factory import RedisServiceFactory
+from shared.services.redis.redis_service import RedisService
+from shared.services.retrieval.nav_snapshot import _resolve_namespace_snapshot_entries
+from shared.services.retrieval.namespace_map_snapshot_redis import (
+ NamespaceMapSnapshotRedisCache,
+)
+from shared.services.retrieval.serving_manifest import encode_namespace_map_snapshot
+
+
+class _FakeRedisClient:
+ def __init__(self, *, decode_responses: bool) -> None:
+ self.decode_responses = decode_responses
+ self.values: dict[str, bytes] = {}
+ self.ttls: dict[str, int] = {}
+
+ async def get(self, key: str) -> bytes | None:
+ return self.values.get(key)
+
+ async def set(self, key: str, value: bytes, *, ex: int) -> bool:
+ self.values[key] = value
+ self.ttls[key] = ex
+ return True
+
+ async def aclose(self) -> None:
+ return None
+
+
+@pytest.mark.asyncio
+async def test_binary_redis_operations_preserve_compressed_snapshot_bytes(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ clients: list[_FakeRedisClient] = []
+
+ def create_client(*args: object, **kwargs: object) -> _FakeRedisClient:
+ client = _FakeRedisClient(
+ decode_responses=bool(kwargs.get("decode_responses"))
+ )
+ clients.append(client)
+ return client
+
+ monkeypatch.setattr(
+ "shared.services.redis.redis_service.redis.from_url", create_client
+ )
+ service = RedisService(RedisConfigManager(RedisConfig()))
+ payload = b"\x78\x9c\x00\xffcompressed-snapshot"
+
+ assert await service.set_bytes("contract:snapshot", payload, ex=3600)
+ assert await service.get_bytes("contract:snapshot") == payload
+ assert len(clients) == 1
+ assert clients[0].decode_responses is False
+ assert clients[0].ttls["knowhere-api:contract:snapshot"] == 3600
+
+ await service.close()
+
+
+@pytest.mark.asyncio
+async def test_snapshot_cache_scopes_reads_and_writes_by_generation(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ class _FakeSnapshotService:
+ def __init__(self) -> None:
+ self.get_keys: list[str] = []
+ self.set_calls: list[tuple[str, bytes, int]] = []
+
+ async def get_bytes(self, key: str) -> bytes | None:
+ self.get_keys.append(key)
+ return b"snapshot"
+
+ async def set_bytes(self, key: str, value: bytes, *, ex: int) -> bool:
+ self.set_calls.append((key, value, ex))
+ return True
+
+ fake_service = _FakeSnapshotService()
+ monkeypatch.setattr(
+ RedisServiceFactory,
+ "get_service",
+ classmethod(lambda cls: fake_service),
+ )
+
+ assert (
+ await NamespaceMapSnapshotRedisCache.get(
+ user_id="user",
+ namespace=" ",
+ generation=7,
+ )
+ == b"snapshot"
+ )
+ assert await NamespaceMapSnapshotRedisCache.set(
+ user_id="user",
+ namespace="default",
+ generation=8,
+ payload_zlib=b"compressed",
+ )
+
+ assert fake_service.get_keys == ["retrieval:snapshot:v2:user:default:g7"]
+ assert fake_service.set_calls == [
+ (
+ "retrieval:snapshot:v2:user:default:g8",
+ b"compressed",
+ 3600,
+ )
+ ]
+
+
+class _SnapshotResult:
+ def __init__(self, *, scalar: object = None, row: tuple[object, ...] | None = None):
+ self._scalar = scalar
+ self._row = row
+
+ def scalar_one_or_none(self) -> object:
+ return self._scalar
+
+ def first(self) -> tuple[object, ...] | None:
+ return self._row
+
+
+class _SnapshotSequenceSession:
+ def __init__(self, results: list[_SnapshotResult]) -> None:
+ self._results = iter(results)
+
+ async def execute(self, _statement: Executable) -> _SnapshotResult:
+ return next(self._results)
+
+ async def rollback(self) -> None:
+ return None
+
+
+@pytest.mark.asyncio
+async def test_corrupt_redis_blob_falls_back_to_postgres_snapshot(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ payload_zlib, checksum, format_version = encode_namespace_map_snapshot(
+ {
+ "documents": {
+ "doc-a": {
+ "job_result_id": "result-a",
+ "job_id": "job-a",
+ "sections": [],
+ "chunks": [],
+ }
+ }
+ }
+ )
+ set_calls: list[bytes] = []
+
+ async def get_corrupt_blob(**_: object) -> bytes:
+ return b"corrupt"
+
+ async def record_repaired_blob(**kwargs: object) -> bool:
+ set_calls.append(bytes(kwargs["payload_zlib"]))
+ return True
+
+ monkeypatch.setattr(NamespaceMapSnapshotRedisCache, "get", get_corrupt_blob)
+ monkeypatch.setattr(NamespaceMapSnapshotRedisCache, "set", record_repaired_blob)
+ session = _SnapshotSequenceSession(
+ [
+ _SnapshotResult(scalar=3),
+ _SnapshotResult(row=(3, checksum, format_version)),
+ _SnapshotResult(row=(payload_zlib,)),
+ ]
+ )
+
+ entries = await _resolve_namespace_snapshot_entries(
+ session,
+ user_id="user",
+ namespace="default",
+ document_revisions=[("doc-a", "result-a")],
+ expected_generation=3,
+ )
+
+ assert entries is not None
+ assert entries[0][0:2] == ("doc-a", "result-a")
+ assert set_calls == [payload_zlib]
diff --git a/apps/api/tests/migrations/test_schema_contract.py b/apps/api/tests/migrations/test_schema_contract.py
index 2625b5ef2..735ed05c2 100644
--- a/apps/api/tests/migrations/test_schema_contract.py
+++ b/apps/api/tests/migrations/test_schema_contract.py
@@ -49,6 +49,11 @@ def _upgrade_to_snapshot_parents(*, engine: Engine) -> None:
command.upgrade(config, "fbe1c2d3e4f5")
+def _upgrade_to_channel_statistics(*, engine: Engine) -> None:
+ config = _build_alembic_command_config(engine=engine)
+ command.upgrade(config, "b1c2d3e4f5a6")
+
+
def _insert_job(
connection: Connection,
*,
@@ -277,6 +282,161 @@ def test_should_create_content_trigram_index_for_regex_search(
assert "WHERE (content IS NOT NULL)" in definition
+def test_should_create_token_leading_map_unit_covering_index(
+ migrated_head_engine: Engine,
+) -> None:
+ with migrated_head_engine.begin() as connection:
+ index_row = connection.execute(
+ text(
+ """
+ SELECT pg_get_indexdef(indexes.indexrelid),
+ indexes.indisvalid,
+ indexes.indisready
+ FROM pg_index AS indexes
+ JOIN pg_class AS classes ON classes.oid = indexes.indexrelid
+ JOIN pg_namespace AS namespaces
+ ON namespaces.oid = classes.relnamespace
+ WHERE namespaces.nspname = current_schema()
+ AND classes.relname = 'idx_document_map_unit_tokens_token_lookup'
+ """
+ )
+ ).one()
+
+ definition = str(index_row[0])
+ assert "(channel, token_hash, map_unit_id)" in definition
+ assert "INCLUDE (token, frequency)" in definition
+ assert index_row[1] is True
+ assert index_row[2] is True
+
+
+def test_should_repair_a_missing_token_leading_map_unit_covering_index(
+ alembic_engine: Engine,
+) -> None:
+ _upgrade_to_channel_statistics(engine=alembic_engine)
+ with alembic_engine.begin() as connection:
+ connection.execute(
+ text("DROP INDEX idx_document_map_unit_tokens_token_lookup")
+ )
+
+ _upgrade_to_heads(engine=alembic_engine)
+
+ with alembic_engine.begin() as connection:
+ index_state = connection.execute(
+ text(
+ """
+ SELECT indexes.indisvalid, indexes.indisready
+ FROM pg_index AS indexes
+ JOIN pg_class AS classes ON classes.oid = indexes.indexrelid
+ JOIN pg_namespace AS namespaces
+ ON namespaces.oid = classes.relnamespace
+ WHERE namespaces.nspname = current_schema()
+ AND classes.relname = 'idx_document_map_unit_tokens_token_lookup'
+ """
+ )
+ ).one()
+
+ assert index_state[0] is True
+ assert index_state[1] is True
+
+
+def test_should_repair_a_missing_covering_index_with_a_caller_owned_connection(
+ alembic_engine: Engine,
+) -> None:
+ _upgrade_to_channel_statistics(engine=alembic_engine)
+ with alembic_engine.begin() as connection:
+ connection.execute(
+ text("DROP INDEX idx_document_map_unit_tokens_token_lookup")
+ )
+
+ _upgrade_to_heads_with_external_connection(engine=alembic_engine)
+
+ with alembic_engine.begin() as connection:
+ index_state = connection.execute(
+ text(
+ """
+ SELECT indexes.indisvalid, indexes.indisready
+ FROM pg_index AS indexes
+ JOIN pg_class AS classes ON classes.oid = indexes.indexrelid
+ JOIN pg_namespace AS namespaces
+ ON namespaces.oid = classes.relnamespace
+ WHERE namespaces.nspname = current_schema()
+ AND classes.relname = 'idx_document_map_unit_tokens_token_lookup'
+ """
+ )
+ ).one()
+
+ assert index_state[0] is True
+ assert index_state[1] is True
+
+
+def test_should_repair_an_invalid_token_leading_map_unit_covering_index(
+ alembic_engine: Engine,
+) -> None:
+ _upgrade_to_channel_statistics(engine=alembic_engine)
+ with alembic_engine.begin() as connection:
+ connection.execute(
+ text(
+ """
+ UPDATE pg_index
+ SET indisvalid = FALSE,
+ indisready = FALSE
+ WHERE indexrelid =
+ 'idx_document_map_unit_tokens_token_lookup'::regclass
+ """
+ )
+ )
+
+ _upgrade_to_heads(engine=alembic_engine)
+
+ with alembic_engine.begin() as connection:
+ index_state = connection.execute(
+ text(
+ """
+ SELECT indexes.indisvalid, indexes.indisready
+ FROM pg_index AS indexes
+ JOIN pg_class AS classes ON classes.oid = indexes.indexrelid
+ JOIN pg_namespace AS namespaces
+ ON namespaces.oid = classes.relnamespace
+ WHERE namespaces.nspname = current_schema()
+ AND classes.relname = 'idx_document_map_unit_tokens_token_lookup'
+ """
+ )
+ ).one()
+
+ assert index_state[0] is True
+ assert index_state[1] is True
+
+
+def test_should_add_per_channel_map_unit_bm25_statistics(
+ migrated_head_engine: Engine,
+) -> None:
+ with migrated_head_engine.begin() as connection:
+ columns = {
+ str(row[0]): str(row[1])
+ for row in connection.execute(
+ text(
+ """
+ SELECT column_name, is_nullable
+ FROM information_schema.columns
+ WHERE table_schema = current_schema()
+ AND table_name = 'document_map_unit_indexes'
+ AND column_name IN (
+ 'path_document_count', 'path_total_length',
+ 'content_document_count', 'content_total_length'
+ )
+ """
+ )
+ ).all()
+ }
+
+ assert columns == {
+ "path_document_count": "YES",
+ "path_total_length": "YES",
+ "content_document_count": "YES",
+ "content_total_length": "YES",
+ }
+
+
def test_should_upgrade_with_a_caller_owned_connection(
alembic_engine: Engine,
) -> None:
diff --git a/apps/api/tests/unit/test_document_ingestion_filename.py b/apps/api/tests/unit/test_document_ingestion_filename.py
new file mode 100644
index 000000000..703e455de
--- /dev/null
+++ b/apps/api/tests/unit/test_document_ingestion_filename.py
@@ -0,0 +1,18 @@
+from app.services.document_ingestion.creation_service import (
+ _resolve_url_source_file_name,
+)
+
+
+def test_resolve_url_source_file_name_decodes_percent_encoded_path() -> None:
+ source_url = (
+ "https://files.example.test/"
+ "%E4%B8%AD%E6%96%87%E7%AE%80%E5%8E%86-%E9%9B%B7%E7%BF%94-"
+ "%E4%B8%AD%E7%A7%91%E9%99%A2%285%29.doc"
+ )
+
+ source_file_name = _resolve_url_source_file_name(
+ source_url=source_url,
+ file_extension=".doc",
+ )
+
+ assert source_file_name == "中文简历-雷翔-中科院(5).doc"
diff --git a/apps/worker/app/services/document_parser/support/internal_parse_name.py b/apps/worker/app/services/document_parser/support/internal_parse_name.py
index 7a59ed8c7..f4ecee2db 100644
--- a/apps/worker/app/services/document_parser/support/internal_parse_name.py
+++ b/apps/worker/app/services/document_parser/support/internal_parse_name.py
@@ -3,6 +3,7 @@
import os
from dataclasses import dataclass
+from urllib.parse import unquote
from app.services.common.file_utils import path_handle
from app.services.document_parser.support.filename_limits import (
@@ -27,6 +28,7 @@ def normalize_internal_parse_name(
candidate_name = (
os.path.basename(filename) if isinstance(filename, str) and filename else ""
)
+ candidate_name = unquote(candidate_name)
cleaned_name = (
path_handle(candidate_name, mode="clean_single") if candidate_name else ""
)
diff --git a/apps/worker/tests/contract/test_page_memory_retrieval_contract.py b/apps/worker/tests/contract/test_page_memory_retrieval_contract.py
index 922aff4c5..1319db3fb 100644
--- a/apps/worker/tests/contract/test_page_memory_retrieval_contract.py
+++ b/apps/worker/tests/contract/test_page_memory_retrieval_contract.py
@@ -144,6 +144,8 @@ async def test_table_result_assembly_uses_summary_not_html() -> None:
assert "企业名称;统一社会信用代码" in content
assert "SHOULD NOT LEAK" not in content
assert " None:
+ encoded_filename = (
+ "%E4%B8%AD%E6%96%87%E7%AE%80%E5%8E%86-%E9%9B%B7%E7%BF%94-"
+ "%E4%B8%AD%E7%A7%91%E9%99%A2%285%29.doc"
+ )
+
+ normalized_name = normalize_internal_parse_name(encoded_filename)
+
+ assert normalized_name == "中文简历-雷翔-中科院-5-.doc"
+
+
def test_prepare_internal_parse_input_handles_long_encoded_filename(
tmp_path,
) -> None:
diff --git a/deploy/ecs/README.md b/deploy/ecs/README.md
index 7b7d61705..37a8c5729 100644
--- a/deploy/ecs/README.md
+++ b/deploy/ecs/README.md
@@ -77,31 +77,44 @@ services. It does not create or delete AWS resources.
## Required post-deploy backfill
-The map-nav lexical-index migration creates the derived index tables, but it does
-not rebuild indexes for revisions that already exist. Until those revisions are
-backfilled, retrieval remains quality-preserving but uses the legacy scoring
-path. Every release containing the map-nav index change must include the
-following DevOps action in its release notification.
+Follow the complete
+[`retrieval-serving-index-rollout-runbook.md`](../../docs/design/retrieval-serving-index-rollout-runbook.md)
+for schema verification, statistics maintenance, readiness gates, parity,
+monitoring, pause/resume, and rollback.
+
+The additive migration does not populate the four per-channel statistics for
+existing revisions. Until those revisions are ready, retrieval remains
+quality-preserving but uses the legacy scoring path. Every release containing
+this change must include the following DevOps action in its release
+notification.
Run the commands as a one-off container using the newly deployed API image and
the production database secret. Do not run them inside the long-lived API task.
```bash
-# Read-only inventory
-python /app/scripts/backfill_map_unit_indexes.py
+# Read-only statistics inventory, after migration
+python /app/scripts/backfill_map_unit_statistics.py --check --batch-size 100
# Optional canary: apply one affected document first
-python /app/scripts/backfill_map_unit_indexes.py \
+python /app/scripts/backfill_map_unit_statistics.py \
--document-id \
+ --batch-size 100 \
--apply
-# Apply to all current document revisions
-python /app/scripts/backfill_map_unit_indexes.py --apply
+# Apply statistics to complete format-v2 indexes
+python /app/scripts/backfill_map_unit_statistics.py --apply --batch-size 100
+
+# Final statistics and full serving-readiness checks
+python /app/scripts/backfill_map_unit_statistics.py --check --batch-size 100
+python /app/scripts/backfill_map_unit_indexes.py --check
```
-The script commits each document revision independently and is safe to rerun.
-Verify the canary retrieval before starting the full apply. New or republished
-documents build their index automatically during publication.
+The statistics script commits each document revision independently and is safe
+to rerun. Missing or legacy indexes reported as `skipped` require the existing
+full `backfill_map_unit_indexes.py --apply --document-id ` path.
+Run only one maintenance process at a time. Verify the canary retrieval before
+starting the full apply. New or republished documents build their index
+automatically during publication.
## Manual staging availability
diff --git a/docs/adr/0009-use-token-leading-covering-index-for-map-unit-lookup.md b/docs/adr/0009-use-token-leading-covering-index-for-map-unit-lookup.md
new file mode 100644
index 000000000..da5d9c990
--- /dev/null
+++ b/docs/adr/0009-use-token-leading-covering-index-for-map-unit-lookup.md
@@ -0,0 +1,11 @@
+# Use a token-leading covering index for map-unit lookup
+
+**Status: accepted.** Token-selective retrieval will use an additive,
+idempotent PostgreSQL covering index led by `(channel, token_hash, map_unit_id)`
+and including `(token, frequency)`. This preserves the token-index-driven query
+shape and allows index-only plans where PostgreSQL visibility permits them. The
+migration is additive and keeps the existing lookup index until a separate
+production-plan and load review proves it redundant. Requests fall back to the
+exact legacy reader when the index or serving data is missing or incomplete.
+The additional storage and write-maintenance cost is accepted in exchange for
+lower first-request retrieval latency and row transfer.
diff --git a/docs/adr/0010-backfill-serving-index-statistics-in-place.md b/docs/adr/0010-backfill-serving-index-statistics-in-place.md
new file mode 100644
index 000000000..75e2ac4c8
--- /dev/null
+++ b/docs/adr/0010-backfill-serving-index-statistics-in-place.md
@@ -0,0 +1,18 @@
+# Backfill serving-index statistics in place
+
+**Status: accepted.** Existing retrieval-visible data will be maintained by
+backfilling only the current revision of each active Document. Complete serving
+indexes receive the new per-channel BM25 statistics through the statistics-only
+`apps/api/scripts/backfill_map_unit_statistics.py --apply` command. It aggregates
+existing map units and does not regenerate token rows, manifests, or namespace
+snapshots. Missing or legacy indexes use the existing full
+`apps/api/scripts/backfill_map_unit_indexes.py --apply --document-id ...`
+rebuild. v1 and v2 are internal read paths under the same Retrieval contract,
+so each revision commits independently. While the four statistics are NULL,
+the reader keeps the full scope-first map-unit projection and derives the
+existing per-channel denominators from those rows; it does not switch to a
+semantically different FTS query. Once statistics are complete, the
+token-selective projection may be used. Missing, legacy, or storage-
+inconsistent index data still uses the exact legacy reader. An interrupted or
+failed backfill therefore preserves retrieval semantics and avoids unnecessary
+regeneration of correct data.
diff --git a/docs/adr/README.md b/docs/adr/README.md
index 00d7b9daa..4d535f151 100644
--- a/docs/adr/README.md
+++ b/docs/adr/README.md
@@ -23,3 +23,5 @@ Use this shape:
| [0006](0006-atomically-publish-retrieval-serving-index.md) | Atomically publish the retrieval-serving index |
| [0007](0007-use-coherent-retrieval-serving-generations.md) | Use coherent retrieval-serving generations |
| [0008](0008-use-a-maintenance-window-for-serving-index-rollout.md) | Roll out the serving index online |
+| [0009](0009-use-token-leading-covering-index-for-map-unit-lookup.md) | Use a token-leading covering index for map-unit lookup |
+| [0010](0010-backfill-serving-index-statistics-in-place.md) | Backfill serving-index statistics in place |
diff --git a/docs/design/retrieval-heart-vessel-trace-optimization-plan.md b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md
new file mode 100644
index 000000000..ed62a57b7
--- /dev/null
+++ b/docs/design/retrieval-heart-vessel-trace-optimization-plan.md
@@ -0,0 +1,617 @@
+# Retrieval optimization plan from the fresh `心血管` trace
+
+## Scope and evidence
+
+The detailed stage breakdown below is based on a newly issued production
+request, not a historical request:
+
+```json
+{
+ "namespace": "default",
+ "query": "心血管",
+ "top_k": 3,
+ "use_agentic": null
+}
+```
+
+- Trace: `01a05e530f7787333c0e32ca16633b85`
+- Route: `/api/v1/retrieval/query`
+- HTTP status: `200`
+- Router: `mapnav`
+- Stop reason: `completed`
+- Server span: `42.365 s`
+- Client wall time: `44.690 s`
+- Result: `29` referenced chunks, approximately `11,998` evidence characters
+
+The current production baseline is restricted to the last-night and today
+window in Beijing time (`2026-09-01 20:00` through `2026-09-02 11:00`, or
+`2026-09-01 12:00Z` through `2026-09-02 03:00Z`). Logfire recorded 203 v1
+retrieval roots and one v2 root in that window; all successful roots were from
+`deployment.environment=production`. The v1 traffic includes both the direct
+`/v1/retrieval/query` path and the externally prefixed `/api/v1/retrieval/query`
+path. The FastAPI route is `/v1/retrieval/query` in both cases.
+
+Logfire does not currently record a git commit in `service_version` or another
+resource attribute. These traces are therefore the latest observed production
+behavior, but they are not proof that production matches a particular local
+HEAD. The runtime reports Python `3.12.14`, built on `2026-09-01`, which is
+useful deployment context but not a source revision identifier.
+
+v1 and v2 share the same retrieval execution plan. v2 only adds the optional
+`llm_config` request field and passes it into the shared plan; when it is absent,
+the retrieval SQL, route selection, scoring, hydration, and public projection
+are the same. The one v2 request in this baseline had `llm_config=null`.
+
+The timings below are parent/child timings. Parent stages include their child
+stages and must not be summed with those children.
+
+## Non-negotiable quality invariant
+
+Every optimization in this plan must preserve retrieval semantic parity for the
+same pinned request: selected chunk IDs, ordering, rounded scores, source
+sections, citations, evidence content, and asset references must remain
+unchanged. Score comparisons use a maximum absolute tolerance of `1e-4` to
+avoid treating harmless floating-point accumulation-order differences as a
+retrieval change; IDs, ordering, source sections, citations, evidence content,
+and asset references have no tolerance. A latency improvement that has not
+passed the parity checks is not shippable, and the exact legacy retrieval path
+remains the fallback.
+
+## Current optimization boundary
+
+This phase covers non-LLM work only: SQL plans, indexes, serving-index row
+transfer, cache behavior, snapshot decoding, map scoring, and instrumentation.
+Planner, Harvest, and Control prompts, models, thinking settings, and model-call
+orchestration are deferred until this phase reaches semantic-parity and latency
+gates.
+
+The optimized reader is an internal implementation shared by v1 and v2. Public
+API versioning is not a performance variant: every non-LLM change must preserve
+the same Retrieval contract for v1 and v2. v2's optional `llm_config` may change
+model settings, but it must not change lexical retrieval semantics.
+
+Implement and validate slices sequentially on the immutable local production
+copy. Instrumentation is measurement-only and must first prove that the
+baseline result is unchanged. Each subsequent SQL, index, cache, or local
+scoring change is evaluated independently against the temporary parity set
+before the next change begins; several individually validated slices may be
+deployed together afterward.
+
+## Stage breakdown
+
+| Stage | Observed time | Evidence |
+| --- | ---: | --- |
+| Scope threshold probe | 1.838 s | `count_scoped_chunks` applies `LIMIT top_k + 1` before the outer `count(*)`; this is a bounded small-corpus gate, not an exact count |
+| Snapshot load | 3.064 s | 644 documents, 115,892 references |
+| Planner | 19.996 s | One subgoal; Planner LLM 19.995 s |
+| Tree build | 0.966 s | 50,112 sections |
+| Map-index load | 8.523 s | 644 index rows; 42,794 unit rows; 3 query tokens |
+| Unit scoring | 0.190 s | 16,662 scored units |
+| Map scoring aggregate | 9.802 s | Parent aggregate around scoring/pooling work |
+| Harvest | 5.260 s | Harvest LLM 2.762 s; one wave/subgoal |
+| Control | 1.256 s | Control LLM 1.248 s |
+| Orchestration | 6.518 s | Parent around Harvest and Control |
+| Evidence pack | 0.262 s | 23 chunks |
+| Final hydration | 0.444 s | 23 results |
+| Episode | 36.633 s | Parent from episode start through evidence pack |
+
+The map-index sub-stages were:
+
+- index metadata: `0.028 s`, 644 rows;
+- map-unit rows: `4.288 s`, 42,794 rows, `cache_hit=false`;
+- frequency rows: `3.866 s`, 42,794 units and 3 tokens.
+
+The `sections=50112` value in the current trace is now accounted for at the
+source boundary: both the `tree_build` and `map_pooling` log sites computed it
+as `sum(len(tree_by_doc[doc_id][0]) ...)`, where the first tuple member is the
+reachable section-node map. It therefore means **section nodes**, not
+section-child edges, leaf sections, or chunks. The local instrumentation now
+also records `section_edges` and `leaf_sections` for the map scorer, and
+`section_rows`, `section_paths`, `root_sections`, and `leaf_sections` for the
+snapshot loader. A previous isolated fixture contained the target namespace's
+644 documents, 50,112 sections, 60,187 chunks, and 42,794 map units. That
+fixture has been discarded after CSV transfer caused excessive local write
+amplification; the replacement DevOps dump is pending and must be treated as
+the only current end-to-end parity corpus.
+
+For planner work, the discarded fixture contained 41,105 real rows for the
+three-token probe plus 1,390,687 non-matching filler rows. Its 1,431,792 total
+rows preserved the observed production channel ratio (content 1,275,676; path
+156,116) and kept the real target-token distribution. Production had
+13,948,605 rows (content 12,524,037; path 1,423,950), so those measurements were
+from a scaled production-shaped copy rather than an exact row-count clone.
+They remain historical investigation evidence only; repeat them on the
+replacement dump before using them for an implementation decision.
+
+On that discarded copy, with all 42,794 map units in scope, the existing
+scope-first frequency query returned 36,478 rows in every run and measured
+`103–121 ms` across five warm repetitions. The materialized token-first
+candidate also returned 36,478 rows (symmetric difference `0`) but measured
+`323–378 ms`; PostgreSQL used the existing non-covering lookup index by
+default. A token-selective unit projection driven by a materialized
+matching-token CTE returned 13,573 units and measured `205–223 ms`; PostgreSQL
+still chose a unit-leading lookup for the join after an index-only token scan.
+The scaled copy has a much higher target-hash fraction than production, so
+this does not predict the full-scale join order. These results validate row
+parity but do not yet demonstrate an end-to-end latency benefit; the reader
+must not change until rows/bytes transferred and retrieval quality are measured
+through the complete caller path.
+
+The historical SQL-output proxy confirms the potential transfer reduction: the
+existing
+unit projection emitted 42,794 rows / 4.05 MB, while the token-selective
+projection emitted 13,573 rows / 1.29 MB. This is a database-client output
+measurement, not an application latency claim; Python decoding, filtering,
+scoring, and cache behavior still need to be measured together.
+
+Using the existing persisted BM25 scorer with the same full-corpus channel
+denominators, the complete and token-selective projections both produced
+13,573 positive-scoring units for `心血管`; the maximum score delta was `0.0`
+and their top-100 ordering was identical. This confirms that dropping
+zero-frequency units is score-safe for this probe, but it is not yet a public
+API retrieval-quality gate.
+
+The actual `ReadOnlyChunkStore.load_persisted_score_corpus` caller was also run
+against 508 revisions in that discarded fixture whose serving index had format
+version `1`.
+It loaded 44,894 section rows, returned the same 13,573 score units, and took
+`1.089 s` for loading plus `0.168 s` for scoring. Revisions with incomplete
+serving indexes correctly returned the existing fallback (`None`); they were
+not silently included in the optimized sample.
+
+A temporary scorer baseline was previously frozen at
+`/tmp/knowhere-retrieval-parity-baseline.json.gz`. That artifact was generated
+before rebasing onto the current main tokenizer and is now historical evidence
+only; it must not be used as the acceptance baseline for this branch. Generate
+a new baseline from the current `origin/main` behavior after the local copy has
+been rebuilt with v2 tokens.
+
+The classic-route SQL shape was then measured against the same historical
+namespace and revision scope. After one cold run, five repetitions of the legacy projection
+measured `716–763 ms`; the token-selective projection measured `781–844 ms`
+(one outlier at `1.73 s`). Output fell from 42,794 rows / 9.53 MB to 13,573
+rows / 2.93 MB. This is a substantial transfer reduction with no stable SQL
+latency win yet, so the next gate is application-level p95 rather than a claim
+that the query itself is faster.
+
+The earlier discovery-level comparison and three-query digests were also
+captured before the rebase, against the old tokenizer/data state. They are
+retained only as historical investigation notes and do not establish current
+latency or retrieval parity. Re-run both the baseline and optimized discovery
+paths after the v2 local backfill, using the current main code as the baseline.
+
+The replacement local dump is now available on PostgreSQL port `55433`.
+After applying the additive schema migration, populating the four channel
+statistics from the existing map units, and running `VACUUM (ANALYZE)`, the
+token-leading covering index uses an index-only scan (`Heap Fetches=0`). For
+the `心血管` probe, the isolated SQL projection measured approximately
+`150 ms` for scope-first versus `77 ms` for token-selective, and the frequency
+lookup measured `2.7 ms`. Ten repeated classic application calls across
+`心血管`, `心脏`, and `肺` preserved chunk IDs, ordering, sources, and evidence;
+the maximum score delta was `1.7e-5`. Warm p50 improved by about `24 ms` for
+`心血管`, was effectively unchanged for `心脏`, and regressed by less than
+`1 ms` for the empty `肺` probe. Treat this as a SQL/transfer improvement with
+no yet-established broad end-to-end latency win; production rollout still
+requires the same migration, backfill, and rollback checks below.
+
+The current classic reader now reuses the immutable revision pins captured at
+request start when validating an unfiltered scope. This removes a redundant
+`SELECT DISTINCT` over scoped map units; requests without pins retain the
+original query. On the restored dump, the removed validation query measured
+approximately `0.35–0.6 s` before the change. Stage instrumentation shows the
+remaining warm classic work is dominated by database reads and hydration, not
+BM25 statistics or Python scoring. Result IDs, ordering, sources, evidence,
+and score deltas remain within the existing `1e-4` parity tolerance.
+
+For the local restored PostgreSQL (which does not enable SSL), agentic smoke
+must set `DB_SSL_MODE=disable` for the global async engine and provide the
+plain libpq form through `KNOWHERE_DATABASE_URL` for the synchronous map-nav
+reader. Without these local-only settings, final reference hydration attempts
+an SSL upgrade and fails even though the database is healthy.
+
+The reference trace above is not representative of request mix. In the current
+baseline window, 196 successful v1 roots used `use_agentic=false` (classic),
+three used the default map-nav route (`use_agentic=null`), and one explicitly
+used `use_agentic=true`. The classic roots had P50 `10.99 s`, P90 `25.46 s`,
+and a maximum of `36.65 s`. There were also three unauthenticated v1 attempts
+(`401`) and one successful v2 root. The explicit `use_agentic=true` v1 request
+took `41.31 s`, and the single v2 request took `109.36 s`; neither is combined
+with the classic latency distribution because they exercise different route
+and/or model settings.
+Across their SQL children, `WITH knowhere` spans had P50 `1.15 s`, P90
+`8.24 s`, P95 `10.46 s`, and a maximum of `18.90 s`. In a representative
+27.91-second classic request, map-unit discovery took `4.02 s` and the
+token-frequency query took `18.90 s`. These current-window classic measurements
+are part of the baseline for the shared v1/v2 reader.
+
+The resource log still renders literal format placeholders
+(`cpu_seconds=%.3f process_max_rss_kb=%d`), so this trace cannot prove CPU
+saturation or request-level memory usage.
+
+## Findings
+
+### Confirmed bottlenecks
+
+1. In the reference map-nav trace, the Planner LLM consumes approximately 20
+ seconds. Outbound HTTP spans are only a few hundred milliseconds, so most of
+ the elapsed time is provider generation/thinking or uninstrumented client
+ wait, not network transfer.
+2. The first request transfers all 42,794 map-unit rows before applying the
+ three query tokens. This costs 4.288 seconds and creates unnecessary Python
+ allocations.
+3. The frequency lookup still costs 3.866 seconds even for only three tokens.
+ The current reader already filters `document_map_unit_tokens` by
+ `token_hash` before joining the pinned scope; the remaining question is
+ whether a token-leading covering index or a different join shape helps on a
+ full-scale corpus. The local production-shaped benchmark above does not
+ show a benefit yet.
+4. The map scoring aggregate is 9.802 seconds, while the explicitly measured
+ unit scoring and pooling work is only 0.190 and 0.094 seconds. The gap is an
+ instrumentation boundary and/or additional map-scoring work that must be
+ measured before CPU tuning.
+5. The bounded scope threshold probe costs 1.838 seconds. The default map-nav
+ route does not need an exact chunk count for ranking, but the probe still
+ distinguishes corpora at or below `top_k` from larger corpora before route
+ selection. The trace alone does not establish that an `EXISTS` rewrite or
+ snapshot metadata would be faster.
+6. Classic discovery previously re-scanned scoped map units solely to derive
+ revision keys after the request had already captured revision pins. Reusing
+ those pins removes that duplicate read without changing the completeness
+ check; the no-pin path remains unchanged.
+ The same pins now drive the unfiltered index-metadata lookup directly,
+ avoiding another scoped-unit CTE. On the restored dump this reduced the
+ warm index stage from roughly `0.16–0.8 s` to `0.02–0.04 s` for the sampled
+ namespace. The classic result parity gate still passes for the temporary
+ query set.
+7. The current production request mix is primarily classic retrieval. The
+ token-selective reader must therefore be exercised through both the classic
+ and map-nav callers; a map-nav-only benchmark would not represent the
+ dominant workload.
+
+### Already healthy or lower priority
+
+- Snapshot load is material but not the largest stage at 3.064 seconds.
+- Evidence pack and final hydration together are below one second in this
+ request. They are not the current optimization priority.
+- The previous connected-hydration change is active: no full `JobResult`
+ chunk graph load appears in the trace.
+- No prompt or retrieval-quality behavior should be changed as part of the
+ first implementation slices.
+
+## Prioritized optimization plan
+
+### Validation gate: bounded scope threshold probe (not an implementation P0)
+
+The current implementation already uses `LIMIT top_k + 1` inside
+`count_scoped_chunks`, then counts that bounded subquery. Do not describe this
+as an exact count or assume a 1.5–1.8 second saving from an `EXISTS` rewrite.
+Benchmark the current probe against any equivalent early-exit query or trusted,
+generation-pinned snapshot metadata. Retain this slice only if
+`EXPLAIN (ANALYZE, BUFFERS)` and production-shaped repetitions show a meaningful
+improvement; otherwise leave the current implementation unchanged and keep
+this out of the optimization queue.
+
+Acceptance criteria:
+
+- route selection is unchanged for corpora below, equal to, and above
+ `top_k`;
+- no stale snapshot can incorrectly classify a small corpus;
+- the baseline and candidate plans make the `top_k + 1` bound and early-exit
+ behavior explicit;
+- any candidate replacement is retained only when a latency improvement is
+ demonstrated; otherwise this remains a validation note, not an optimization
+ slice;
+- selected chunk IDs, scores, ordering, and evidence remain unchanged;
+- benchmark results record the route family (`classic`, `mapnav`, or
+ `small_corpus`) and separate cold, warm, and response-cache-hit requests;
+ cache-hit timings are not mixed into cold-request latency claims.
+
+### P0: Make map-unit projection token-selective
+
+The current map-nav reader already makes its frequency lookup token-selective,
+but it still loads every revision-scoped map unit before applying the query
+tokens. Change only the unit projection: start from
+`document_map_unit_tokens` filtered by `channel` and `token_hash`, then join the
+pinned revision/unit scope, and return only units matching at least one query
+token. Keep the existing frequency SQL shape until a production-shaped plan
+proves that a different join order is better.
+
+Roll this out in two stages. The first stage may enable the token-selective
+reader only when the request has no section exclusions, where revision-scoped
+channel statistics are sufficient. Requests with section filters continue to
+use the existing scope-first map-unit reader until section-scoped denominator
+statistics are implemented and pass semantic-parity checks. A missing,
+incompatible, or storage-inconsistent serving index uses the exact legacy FTS
+fallback. A complete index with NULL channel statistics keeps the full scope
+map-unit rows and derives the two channel denominators from those rows until
+the statistics backfill completes.
+
+BM25 denominators must remain corpus-wide. Obtain exact corpus statistics using
+persisted serving-index statistics while preserving the current per-channel
+semantics. Extend `DocumentMapUnitIndex` with
+`path_document_count`, `path_total_length`, `content_document_count`, and
+`content_total_length`, calculated atomically when publishing each revision.
+Units with zero path length must not contribute to path `document_count` or
+`total_length`, and the same rule applies independently to content. Do not
+reuse `DocumentMapUnitIndex.unit_count` for both channels, and never calculate
+average length from only the matching subset.
+
+Hypothesis to validate: this should reduce map-index latency, rows transferred,
+and temporary memory for this workload; no fixed seconds-saving claim is made
+before a full-scale or statistically equivalent local benchmark. The scaled
+copy above is a negative/shape-control result, not an approval to change the
+reader.
+
+The current implementation slice only adds the per-channel statistics fields,
+publication-time population, and the DevOps backfill/readiness contract. It
+enables token-selective projection when the scope is unfiltered and serving
+statistics are present. Before statistics backfill, an unfiltered revision
+uses the full scope map-unit rows with row-derived per-channel denominators, so
+the lexical result remains unchanged; section-filtered requests retain their
+existing scope-first map-unit reader. Only a missing, incompatible, or
+storage-inconsistent index uses the exact legacy FTS fallback. Both v1 and v2
+call the same shared readers; no route-specific lexical algorithm was
+introduced.
+
+Acceptance criteria:
+
+- a new idempotent migration adds the token-leading covering index required by
+ this query, for example `(channel, token_hash, map_unit_id) INCLUDE
+ (token, frequency)`. Keep the existing lookup index during this rollout;
+ evaluate removal separately only after `EXPLAIN (ANALYZE, BUFFERS)` and
+ production load confirm the new index is safe (or document the visibility
+ conditions that prevent an index-only scan);
+- serving-index statistics include positive-length `document_count` and
+ `total_length` separately for path and content, aggregated over the exact
+ pinned revision and section scope;
+- frequency maps and scores are identical to the current implementation on a
+ fixed production-data copy;
+- duplicate token matches do not duplicate units;
+- rows and bytes transferred are measured before and after;
+- a safe fallback remains available if the persisted index is incomplete;
+- v1 and v2 requests with equivalent retrieval fields produce identical public
+ retrieval results; v2's optional `llm_config` is recorded as a model-setting
+ difference, not as a separate lexical retrieval algorithm;
+- the benchmark uses current-window classic requests as the primary sample and
+ includes at least one map-nav and one v2 request as shared-plan regression
+ samples;
+- revision-scope cardinality and the size of the `(document_id,
+ job_result_id)` parameter set are recorded, so large `IN` lists are not hidden
+ inside token-query timing.
+
+### P1: Optimize the frequency SQL plan
+
+Remove or avoid a materialized full `scoped_units` side when the planner picks
+the wrong join order. Drive from the token index, then apply channel,
+token-hash, pinned revision, active-document, and namespace predicates.
+
+Hypothesis to validate: the revised join order may reduce frequency-stage
+latency; claim no fixed saving until production-shaped `EXPLAIN (ANALYZE,
+BUFFERS)` and repeated local runs confirm it.
+
+Acceptance criteria:
+
+- returned `(map_unit_id, channel, token, frequency)` rows are identical;
+- no archived or wrong-generation revision can enter the result;
+- statement timeout is not approached on the production read-only database;
+- the query remains correct for one token, many tokens, and no matching token;
+- repeated measurements cover the observed frequency-latency range (warm and
+ cold cache, one and many query tokens, and small and large revision scopes)
+ before any saving claim is made.
+
+### P1: Measure and then reduce the unaccounted map-scoring time
+
+Add structured timers around map-pooling, section aggregation, ranking, and
+projection so the `9.802 s` parent stage can be reconciled with its children.
+Only after that measurement should we change data structures or algorithms.
+
+Likely follow-up: precompute section owners, children, and postorder arrays
+while decoding the snapshot, then reuse them during scoring instead of
+rebuilding dictionaries and sets.
+
+Hypothesis to validate: precomputation should reduce local map-scoring time;
+the magnitude is intentionally left to measurement.
+
+### Deferred: Planner and other LLM latency
+
+The Planner is the largest single stage, but this phase makes no changes to
+Planner, Harvest, or Control model calls. Keep their existing behavior and
+fallbacks intact. Revisit model timing, thinking, prompts, and call
+orchestration only after the non-LLM slices below have passed their semantic-
+parity and latency gates.
+
+### Deferred: overlap planner and query-independent map scoring (experimental)
+
+The current route waits for the Planner before starting map scoring. For a
+planner result whose `retrieval_query` is exactly the user query and whose
+scope/filters are unchanged, map scoring can be started concurrently on a
+separate read-only connection. The planner result is then used only to decide
+whether the already-computed candidate set is sufficient.
+
+This could hide part of the approximately `9.8 s` map-scoring aggregate behind
+the approximately `20 s` Planner wait, but it changes LLM-boundary
+orchestration. It is deferred until the Planner phase is explicitly in scope;
+it must not run when the planner rewrites the query, adds a node filter, or
+changes the revision/scope.
+
+Potential saving: up to the overlapping map-scoring time for eligible simple
+queries. This is an experimental concurrency change with risks around
+connection-pool pressure, duplicate work, cancellation, and revision
+coherence.
+
+Acceptance criteria:
+
+- the candidate scores are byte-for-byte equal to the serial path;
+- the concurrent read-only store is isolated from mutable navigation state;
+- planner rewrites and filters correctly disable the overlap;
+- cancellation and database connection cleanup are covered by contract tests;
+- concurrency load tests show no increase in statement timeouts or RSS.
+
+### P2: Reduce snapshot parse allocations
+
+Redis stores compressed snapshot bytes, but every request still decompresses
+and JSON-decodes 115,892 references. The current parsing path creates both
+canonical and owner-qualified reference keys and then copies the mapping.
+
+Use one canonical representation with owner-aware fallback lookup and avoid
+the second full mapping copy. Consider a versioned binary format only after a
+benchmark; it is a larger migration and is not required for the first slices.
+
+Hypothesis to validate: the representation change should reduce decode time and
+temporary RSS; the magnitude is intentionally left to measurement.
+
+Acceptance criteria:
+
+- both existing lookup forms remain valid;
+- revision pinning and selected references are unchanged;
+- checksum/version validation and generation invalidation remain intact;
+- cold Redis-miss and Redis-hit memory are measured separately.
+
+### P2: Redis map-unit projection cache
+
+The current unit cache is episode/process-local; the snapshot Redis cache does
+not eliminate the map-unit SQL transfer. Add a generation- and revision-scoped
+Redis cache for a compact map-unit projection and index statistics, with a
+bounded TTL and size limit. Bind each key to `user_id`, `namespace`, serving
+generation, `document_id`, `job_result_id`, and index format version. A
+generation change naturally invalidates old entries; Redis misses, version
+mismatches, and decode failures fall back to PostgreSQL.
+
+This is not a guaranteed cold-first-request optimization. It helps only when a
+previous publisher or request has populated the generation key. PostgreSQL
+remains the source of truth and is the fallback on misses or Redis failures.
+
+## Existing-data maintenance
+
+Backfill only the currently retrieval-visible revision for each active
+Document. Add the channel-stat columns with an additive migration and introduce
+serving-index format version 2. v1 and v2 remain two internal read paths under
+the same Retrieval contract and must produce identical results. For a complete
+existing index, a resumable per-revision statistics backfill aggregates the existing
+`document_map_units` rows, writes the four path/content values, and marks the
+revision v2 in the same transaction. It must not regenerate token rows or
+snapshots unnecessarily. Revisions with a missing or legacy index use the
+existing full `backfill_map_unit_indexes --apply` path.
+
+Each revision is committed independently under the existing generation and
+active-document locks. The job is idempotent and safe to interrupt. While the
+four statistics are NULL, retrieval keeps the full scope-first map-unit reader
+and derives the existing per-channel denominators from its rows. Once the
+backfill check reports complete, coherent active revisions, the unfiltered
+reader can use token-selective projection and persisted denominators. Missing,
+incompatible, or storage-inconsistent index data still uses the exact legacy
+FTS fallback.
+
+### DevOps handoff
+
+DevOps runs the maintenance explicitly; API startup and user requests never
+trigger it. The executable rollout procedure is documented in
+[`retrieval-serving-index-rollout-runbook.md`](retrieval-serving-index-rollout-runbook.md).
+The runbook:
+
+1. deploy the additive migration and verify the reader still falls back safely;
+ Build the new index without dropping the existing lookup index, and monitor
+ lock waits, build duration, and disk headroom;
+2. deploy the application code that writes v2 for new publications and selects
+ the token-selective reader only for complete v2 revisions; revisions with
+ NULL statistics continue on the full scope-first map-unit reader, while v1
+ or otherwise unusable index data continues on the legacy FTS reader;
+3. run the read-only inventory/check command and record active/current revision
+ counts;
+4. run the resumable statistics backfill in bounded batches (with optional
+ document selection), monitoring database load and generation changes. The
+ command is:
+
+ ```bash
+ python /app/scripts/backfill_map_unit_statistics.py \
+ --apply --batch-size 100
+ ```
+
+ Use `--user-id`, `--namespace`, or `--document-id` to narrow a rehearsal.
+ `--check` is read-only and should return `would_update=0` before rollout.
+ This command only aggregates existing `document_map_units`; it does not
+ regenerate token rows or snapshots. Revisions with a missing or legacy
+ index remain on the existing full backfill path.
+5. rerun the check until every retrieval-visible revision has a coherent v2
+ marker and no serving fallback is reported;
+6. monitor semantic-parity probes, latency, errors, and timeouts. The reader's
+ checks retain the full scope-first map-unit path while only statistics are
+ incomplete, and use legacy FTS when index data is unusable.
+
+The handoff must document the exact command, batch/concurrency limits,
+pause/resume procedure, application-version rollback procedure, and the final
+check output. A partial
+backfill is an expected intermediate state, not a failed deployment, provided
+that incomplete statistics do not select token-only projection and unusable
+indexes remain on the legacy reader.
+
+### Deferred: LLM orchestration improvements
+
+- Do not add a deterministic shortcut around Control. `plan_control` remains
+ the sole checklist-reconciliation authority; non-empty or seemingly related
+ evidence is not sufficient to replace its accept/widen/drop/replan decision.
+- Do not change subgoal parallelism or Harvest/Control materialization in this
+ phase. These remain candidates only after the LLM boundary is explicitly
+ revisited.
+
+These changes must not alter prompt text or citation selection semantics.
+
+## Validation protocol
+
+Every implementation slice must be validated against a local copy of the
+production namespace before deployment. Production rollout then relies on the
+automatic legacy fallback for any incomplete or incompatible revision.
+
+Validation has two layers. First, maintain a small temporary retrieval-parity
+set (two or three representative queries, including the current `心血管` trace)
+containing current legacy-reader outputs, and run deterministic non-LLM parity
+checks on an immutable production-data copy with fixed revision pins, section
+scope, and queries. Compare loader rows, frequency maps, channel statistics,
+per-unit scores, and ordering against that set. This set is a quality guard for
+the optimization work, not a new product behavior contract; do not refresh it
+just to hide a regression. Generate it once from the current legacy reader and
+freeze it for the whole optimization series; refresh it only after an explicit
+decision to accept a retrieval behavior change. Second, run end-to-end
+Retrieval requests to measure latency, rows/bytes, and memory. Do not use
+nondeterministic Planner/Harvest timing as evidence for this non-LLM change.
+
+1. Capture a baseline with the exact request, a cold process, and an empty or
+ generation-scoped Redis key. Run at least 10 cold repetitions; measure p50,
+ p95, stage timings, peak RSS, rows, and bytes transferred.
+2. Run a separate warm-cache series. Never present warm-cache results as the
+ first-request improvement.
+3. For every SQL change, compare `EXPLAIN (ANALYZE, BUFFERS)` and exact returned
+ rows against the current query using the read-only production database or
+ an immutable local copy.
+4. Compare retrieval quality per request, requiring zero public-result
+ differences: selected chunk IDs, order, rounded scores, source sections,
+ evidence content/hash, asset references, router, stop reason, and citation
+ validity. Aggregate quality metrics cannot replace this exact parity check.
+5. Exercise edge cases: empty result, one token, multiple tokens, small
+ corpus, incomplete index, archived revision, namespace generation change,
+ and Redis miss/failure.
+6. Fix the malformed CPU/RSS log fields before making any CPU or memory claim.
+ Use per-stage CPU deltas and cgroup metrics rather than process high-water
+ RSS alone.
+7. Roll out one optimization at a time, with automatic legacy fallback and
+ production p50/p95/error/timeout monitoring. If rollback is required,
+ redeploy the previous application version; leave additive schema/data in
+ place for a later retry.
+
+For each slice, retain it only when repeated local runs show a stable
+improvement in the targeted stage and no regression in total non-LLM p95. The
+temporary parity set is a hard gate regardless of the measured speedup.
+
+## Recommended implementation order
+
+1. Add missing structured timing and CPU/RSS instrumentation.
+2. Benchmark the existing bounded `top_k + 1` scope probe; keep a replacement
+ only if it demonstrates a measured improvement.
+3. Implement one token-selective serving-index slice: add the covering index,
+ persist per-channel BM25 statistics, and make unfiltered map-unit discovery
+ token-selective with the exact legacy fallback for filtered/incomplete
+ scopes.
+4. Optimize the frequency join order and verify the production query plan.
+5. Reconcile and optimize the unaccounted map-scoring work.
+6. Consider snapshot allocation and Redis map-unit projection work as follow-up
+ improvements.
diff --git a/docs/design/retrieval-serving-index-rollout-runbook.md b/docs/design/retrieval-serving-index-rollout-runbook.md
new file mode 100644
index 000000000..56d2c3cca
--- /dev/null
+++ b/docs/design/retrieval-serving-index-rollout-runbook.md
@@ -0,0 +1,404 @@
+# Retrieval Serving Index Rollout Runbook
+
+## Scope
+
+This runbook rolls out the retrieval serving-index performance changes without
+changing retrieval semantics. It covers the additive schema migration, the
+existing-data statistics backfill, readiness checks, retrieval parity probes,
+monitoring, pause/resume, and application rollback.
+
+This rollout does not use a runtime feature flag. The reader selects the
+optimized path only when every revision required by a request has a coherent
+format-v2 index and all four channel statistics. Otherwise it uses the existing
+legacy reader automatically.
+
+Planner, Harvest, Control, prompts, BM25 formulas, and citation-selection
+semantics are outside this rollout.
+
+## Preconditions
+
+- Deploy only a build produced from the reviewed retrieval optimization branch.
+- Confirm the target is the intended production environment before every
+ command.
+- Use the API image for Alembic and maintenance commands. It contains:
+ - `/app/alembic`
+ - `/app/scripts/backfill_map_unit_statistics.py`
+ - `/app/scripts/backfill_map_unit_indexes.py`
+- Confirm database backups and the normal application-version rollback path are
+ available.
+- Record current retrieval p50/p95, errors, timeouts, and the output of the
+ temporary two-or-three-query quality set before deployment.
+- Check free database disk space. The token-leading index is additive and the
+ existing indexes must remain in place.
+
+## Local acceptance evidence
+
+The production-shaped local restore used for final verification contained:
+
+- 2,996 documents;
+- 125,835 sections;
+- 170,280 chunks;
+- 107,032 map units;
+- 12,423,317 map-unit token rows;
+- 644 active documents in the benchmark namespace.
+
+Final readiness on that namespace:
+
+```text
+alembic head: c2d3e4f5a6b7
+covering index: idx_document_map_unit_tokens_token_lookup
+statistics check: would_update=0 complete=644 skipped=0 documents=644
+coherent current indexes: 644/644
+```
+
+Classic parity queries preserved chunk IDs, ordering, sources, and evidence.
+The maximum observed score delta was below the accepted `1e-4` tolerance.
+The v1 and v2 classic entry points returned the same chunk IDs and evidence
+hash. A complete map-nav smoke returned `stop_reason=completed`, 23 result rows,
+and 27 referenced chunks.
+
+These figures describe the local restored copy. They are not substitutes for
+the production checks below.
+
+## Phase 1: Preflight inventory on the existing schema
+
+Before migration, record active/current revision counts using only the existing
+schema. Do not run the statistics checker yet because it reads columns added by
+revision `b1c2d3e4f5a6`.
+
+```sql
+SELECT count(*) AS active_current_documents
+FROM documents
+WHERE status = 'active'
+ AND current_job_result_id IS NOT NULL;
+
+SELECT count(*) AS current_map_indexes
+FROM documents
+JOIN document_map_unit_indexes AS indexes
+ ON indexes.document_id = documents.document_id
+ AND indexes.job_result_id = documents.current_job_result_id
+WHERE documents.status = 'active'
+ AND documents.current_job_result_id IS NOT NULL;
+```
+
+Record both counts and investigate any pre-existing difference. This inventory
+does not determine format-v2 readiness.
+
+## Phase 2: Apply the additive migrations
+
+The production release workflow runs migrations before updating ECS services.
+Record the workflow job URL and output. For a manual rehearsal, run Alembic from
+the API image:
+
+```bash
+cd /app
+python -m alembic upgrade heads
+python -m alembic current
+```
+
+The expected head for this rollout is:
+
+```text
+c2d3e4f5a6b7
+```
+
+The three relevant additive migrations are:
+
+1. `a0b1c2d3e4f5`: creates
+ `idx_document_map_unit_tokens_token_lookup` on
+ `(channel, token_hash, map_unit_id) INCLUDE (token, frequency)`;
+2. `b1c2d3e4f5a6`: adds nullable path/content document-count and total-length
+ columns to `document_map_unit_indexes`;
+3. `c2d3e4f5a6b7`: repairs the covering index if it is missing or PostgreSQL
+ reports `indisvalid=false` or `indisready=false` after an interrupted build.
+
+The index migration uses `CREATE INDEX CONCURRENTLY` in the normal Alembic
+execution path. Monitor lock waits, database CPU, I/O, replication lag, and
+free disk space while it runs. Do not drop either pre-existing map-unit-token
+index during this rollout.
+
+Verify the schema:
+
+```sql
+SELECT version_num FROM alembic_version;
+
+SELECT
+ classes.relname AS index_name,
+ indexes.indisvalid,
+ indexes.indisready,
+ pg_get_indexdef(indexes.indexrelid) AS index_definition
+FROM pg_index AS indexes
+JOIN pg_class AS classes ON classes.oid = indexes.indexrelid
+WHERE classes.relname = 'idx_document_map_unit_tokens_token_lookup';
+
+SELECT column_name
+FROM information_schema.columns
+WHERE table_name = 'document_map_unit_indexes'
+ AND column_name IN (
+ 'path_document_count',
+ 'path_total_length',
+ 'content_document_count',
+ 'content_total_length'
+ )
+ORDER BY column_name;
+```
+
+The covering index must exist with `indisvalid=true`, `indisready=true`, and
+the expected key/include columns. A failed concurrent build can leave an
+invalid index. If either flag is false, stop the rollout, drop only that invalid
+index with `DROP INDEX CONCURRENTLY`, and rerun the migration.
+
+## Phase 3: Deploy the application build
+
+Deploy the application after the additive migrations finish. New publications
+will write coherent format-v2 statistics. Existing revisions with NULL channel
+statistics remain on the full scope-first map-unit reader until maintenance
+completes; missing, legacy, or unusable indexes remain on the legacy reader.
+
+Immediately verify:
+
+- API health checks pass;
+- no migration or model-loading error appears in API logs;
+- classic and map-nav requests still complete;
+- incomplete-index warnings distinguish statistics-incomplete map-unit serving
+ from `fallback=legacy_fts`; neither case may return partial or empty results;
+- no increase appears in retrieval errors or timeouts.
+
+## Phase 4: Backfill existing format-v2 indexes
+
+After the migration, run the statistics inventory:
+
+```bash
+python /app/scripts/backfill_map_unit_statistics.py \
+ --check \
+ --batch-size 100
+```
+
+Record the final summary line:
+
+```text
+would_update= complete= skipped= documents=
+```
+
+Interpretation:
+
+- `complete`: already coherent format-v2 revisions;
+- `would_update`: existing format-v2 indexes whose four statistics need an
+ in-place update;
+- `skipped`: missing or legacy indexes that require the full index backfill.
+
+Run exactly one maintenance process against the database. `--batch-size` is a
+serial session grouping, not a concurrency setting.
+
+First rehearse one namespace or document:
+
+```bash
+python /app/scripts/backfill_map_unit_statistics.py \
+ --apply \
+ --batch-size 100 \
+ --user-id \
+ --namespace
+```
+
+Then run the full statistics backfill:
+
+```bash
+python /app/scripts/backfill_map_unit_statistics.py \
+ --apply \
+ --batch-size 100
+```
+
+The command:
+
+- reads existing `document_map_units`;
+- computes positive-length document counts and total token lengths separately
+ for the path and content channels;
+- commits each revision independently;
+- does not regenerate token rows, manifests, or namespace snapshots;
+- is idempotent and safe to restart.
+
+If `skipped` is non-zero, list and process those documents separately with the
+existing full index command:
+
+```bash
+python /app/scripts/backfill_map_unit_indexes.py \
+ --apply \
+ --document-id
+```
+
+Do not use `--tokens-only` unless investigation proves that only map-unit token
+data is missing and the serving manifest and namespace snapshot are already
+coherent.
+
+## Phase 5: Final readiness gate
+
+Repeat the read-only check until it exits successfully and reports:
+
+```text
+would_update=0 skipped=0 complete= documents=
+```
+
+```bash
+python /app/scripts/backfill_map_unit_statistics.py \
+ --check \
+ --batch-size 100
+```
+
+Also verify that all retrieval-visible active revisions are coherent:
+
+```sql
+SELECT
+ count(*) FILTER (
+ WHERE indexes.format_version = 2
+ AND indexes.path_document_count IS NOT NULL
+ AND indexes.path_total_length IS NOT NULL
+ AND indexes.content_document_count IS NOT NULL
+ AND indexes.content_total_length IS NOT NULL
+ ) AS ready_revisions,
+ count(*) AS current_revisions
+FROM documents
+LEFT JOIN document_map_unit_indexes AS indexes
+ ON indexes.document_id = documents.document_id
+ AND indexes.job_result_id = documents.current_job_result_id
+WHERE documents.status = 'active'
+ AND documents.current_job_result_id IS NOT NULL;
+```
+
+With the `LEFT JOIN`, `current_revisions` includes active documents with no
+index. `ready_revisions` must equal `current_revisions`. Also require:
+
+```sql
+SELECT count(*) AS missing_index_rows
+FROM documents
+LEFT JOIN document_map_unit_indexes AS indexes
+ ON indexes.document_id = documents.document_id
+ AND indexes.job_result_id = documents.current_job_result_id
+WHERE documents.status = 'active'
+ AND documents.current_job_result_id IS NOT NULL
+ AND indexes.id IS NULL;
+```
+
+`missing_index_rows` must be zero.
+
+Run the complete serving-index readiness checker:
+
+```bash
+python /app/scripts/backfill_map_unit_indexes.py --check
+```
+
+For every namespace, require `status=READY` and inspect the report rather than
+only its exit code. Require:
+
+```text
+missing_from_snapshot=0
+missing_map_index=0
+missing_revision_manifest=0
+```
+
+`suspicious_zero_idf` is diagnostic only. Zero average IDF is valid for some
+small corpora, including a two-unit corpus where each token occurs in exactly
+one unit; it must not independently block readiness.
+
+## Phase 6: Retrieval-quality gate
+
+Run the frozen temporary quality set. Use the same two or three queries,
+namespace, top-k, filters, and revision generation captured before deployment.
+
+For classic retrieval, require no change in:
+
+- router;
+- selected chunk IDs and order;
+- source document and section;
+- evidence content/hash;
+- asset references;
+- rounded scores, with an absolute tolerance of `1e-4`.
+
+Exercise at least:
+
+1. v1 `use_agentic=false`;
+2. v2 `use_agentic=false` with equivalent retrieval fields;
+3. one `use_agentic=true` map-nav smoke;
+4. one request with `use_agentic` omitted, confirming it routes to map-nav;
+5. one filtered request, confirming filtered-scope semantics and the safe
+ fallback where required.
+
+Map-nav LLM output is nondeterministic. For production smoke, require successful
+completion, valid citations, expected namespace isolation, and relevant
+evidence. Do not require byte-identical ordering between independent Planner
+runs. Deterministic map-score parity remains covered by the contract suite.
+
+Stop the rollout if classic result parity fails. Do not refresh the baseline to
+hide a difference.
+
+## Phase 7: Performance and reliability observation
+
+Monitor at least one normal traffic window after the backfill. Classic public
+result parity must have zero differences, total non-LLM p95 must not exceed the
+recorded baseline, and retrieval error/timeout rates must not regress.
+
+- retrieval request p50/p95 and maximum latency, separated by `router_used`;
+- classic `search.map_unit_discovery` stages: units, frequencies, indexes,
+ statistics, scoring, and hydration;
+- map-nav snapshot, episode, and hydration stages;
+- PostgreSQL statement timeouts, lock waits, CPU, I/O, and connection usage;
+- Redis errors and namespace snapshot cache misses;
+- retrieval errors, incomplete-index fallbacks, and response timeouts;
+- process CPU and maximum RSS from the corrected map-nav resource log.
+
+Do not mix classic and map-nav latency distributions. Do not treat Redis-warm
+snapshot measurements as cold-request performance.
+
+## Pause and resume
+
+To pause, stop launching new maintenance command processes. Wait for the
+current revision to finish if database health permits, then send `SIGINT` or
+`SIGTERM` to the one-off task. Each completed revision has already committed;
+the interrupted transaction rolls back and remains on the safe reader path.
+
+To resume, rerun the same `--apply` command. Completed revisions are detected
+and skipped. Follow it with `--check` and retain both summary outputs in the
+deployment record.
+
+## Rollback
+
+If application errors, timeouts, or quality regressions occur:
+
+1. stop the backfill process;
+2. redeploy the previous application version;
+3. verify classic and map-nav requests using the frozen quality set;
+4. retain the additive columns, index, and already-computed statistics unless
+ database health specifically requires their removal.
+
+Application rollback is sufficient because the previous application ignores
+the additive schema. Avoid running Alembic downgrade during an incident: a
+concurrent index drop or table alteration adds operational risk and is not
+required to restore the previous behavior.
+
+## Completion record
+
+Attach the following to the deployment ticket:
+
+- deployed application image digest and Git commit;
+- pre-migration active/current inventory;
+- post-migration statistics `--check` output;
+- Alembic `current` output;
+- covering-index and column verification output;
+- rehearsal and full `--apply` summaries;
+- final `--check` output;
+- ready/current revision counts;
+- frozen-query parity results;
+- classic and map-nav latency summaries;
+- observed fallback, error, and timeout counts;
+- rollback decision or explicit confirmation that rollback was not required.
+
+## ECS one-off task requirements
+
+Run maintenance as a one-off task created from the newly registered production
+API task definition. Reuse its task role, execution role, VPC subnets, security
+groups, Secrets Manager injection, and CloudWatch log configuration. Override
+only the container command with one of the commands in this runbook.
+
+Do not execute maintenance inside a long-lived API task. Do not copy, export, or
+place the production database URL in the ECS command, shell history, workflow
+input, deployment ticket, or logs. The one-off task must receive it through the
+same production secret as the API task.
diff --git a/docs/design/retrieval-serving-snapshot-cache.md b/docs/design/retrieval-serving-snapshot-cache.md
new file mode 100644
index 000000000..36f23f9df
--- /dev/null
+++ b/docs/design/retrieval-serving-snapshot-cache.md
@@ -0,0 +1,48 @@
+# Retrieval serving snapshot cache
+
+Map-nav reads a namespace routing snapshot that contains section/chunk
+relationships, ordering, chunk types, connection references, and remount
+ownership. It intentionally does not contain chunk body text; final hydration
+still reads pinned revision content from PostgreSQL.
+
+## Persisted formats
+
+- Namespace snapshots written by current publication code use format version 2
+ and contain routing metadata only.
+- The reader remains compatible with version 1 snapshots so existing rows can
+ be served during rollout. No unconditional namespace rebuild is required
+ solely because the reader was upgraded.
+- A missing, corrupt, stale, or generation-mismatched snapshot falls back to
+ the exact manifest/table loading path.
+
+## Redis cache
+
+The map-nav reader caches the compressed snapshot bytes, not a decoded Python
+dictionary. Keys are scoped by user, normalized namespace, and serving
+generation:
+
+```text
+retrieval:snapshot:v2:{user_id}:{namespace}:g{generation}
+```
+
+The cache TTL is one hour. Redis errors, misses, and invalid blobs are
+non-fatal: PostgreSQL remains the source of truth and the reader repopulates
+Redis after a successful database read. Binary snapshot operations use a Redis
+connection with response decoding disabled; normal JSON Redis operations keep
+their existing text-decoding connection.
+
+## Generation coherence
+
+Retrieval carries one captured revision set and namespace generation through
+snapshot loading, map-nav, reference resolution, and final hydration. A
+generation lookup failure is treated as inability to establish coherence and
+uses the fallback path. A generation mismatch is never served as a valid
+snapshot.
+
+## Diagnostics
+
+Set `RETRIEVAL_SNAPSHOT_TIMING=1` for detailed snapshot decode timings during
+local or staging diagnosis. The route CPU metric is process-level CPU time for
+the map-nav route. `ru_maxrss` is a process high-water mark, not a request-level
+memory peak, and must not be interpreted as one.
+
diff --git a/packages/shared-python/shared/models/database/document.py b/packages/shared-python/shared/models/database/document.py
index 3d630b917..6a46fdd26 100644
--- a/packages/shared-python/shared/models/database/document.py
+++ b/packages/shared-python/shared/models/database/document.py
@@ -328,6 +328,13 @@ class DocumentMapUnitToken(Base):
"token_hash",
"map_unit_id",
),
+ Index(
+ "idx_document_map_unit_tokens_token_lookup",
+ "channel",
+ "token_hash",
+ "map_unit_id",
+ postgresql_include=["token", "frequency"],
+ ),
Index(
"idx_document_map_unit_tokens_unit_lookup",
"map_unit_id",
@@ -340,7 +347,7 @@ class DocumentMapUnitToken(Base):
class DocumentMapUnitIndex(Base):
- """Completeness marker for a revision's materialized map-unit index."""
+ """Completeness marker and corpus statistics for a materialized index."""
__tablename__ = "document_map_unit_indexes"
@@ -362,6 +369,16 @@ class DocumentMapUnitIndex(Base):
average_idf_content: Mapped[float] = mapped_column(
Float, nullable=False, default=0.0
)
+ path_document_count: Mapped[Optional[int]] = mapped_column(
+ Integer, nullable=True
+ )
+ path_total_length: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
+ content_document_count: Mapped[Optional[int]] = mapped_column(
+ Integer, nullable=True
+ )
+ content_total_length: Mapped[Optional[int]] = mapped_column(
+ Integer, nullable=True
+ )
created_at: Mapped[datetime] = mapped_column(
DateTime, default=utc_now_naive, nullable=False
)
diff --git a/packages/shared-python/shared/services/redis/redis_service.py b/packages/shared-python/shared/services/redis/redis_service.py
index e83e3a7bc..c25a3e1d6 100644
--- a/packages/shared-python/shared/services/redis/redis_service.py
+++ b/packages/shared-python/shared/services/redis/redis_service.py
@@ -38,6 +38,7 @@ def __init__(self, config_manager: Optional[RedisConfigManager] = None):
config_manager = RedisConfigManager(settings)
self.config_manager = config_manager
self._client: Optional[redis.Redis] = None
+ self._binary_client: Optional[redis.Redis] = None
self._health_checker: Optional[RedisHealthChecker] = None
self._lock = asyncio.Lock()
@@ -60,6 +61,26 @@ async def _get_client(self) -> redis.Redis:
)
return self._client
+ async def _get_binary_client(self) -> redis.Redis:
+ """Get a Redis client that preserves arbitrary binary responses."""
+ if self._binary_client is None:
+ async with self._lock:
+ if self._binary_client is None:
+ try:
+ connection_params = self.config_manager.get_connection_params()
+ connection_params["decode_responses"] = False
+ self._binary_client = redis.from_url(
+ self.config_manager.get_connection_url(),
+ **connection_params,
+ )
+ logger.debug("Redis binary client initialized")
+ except Exception as e:
+ raise RedisConnectionError(
+ internal_message=f"Redis binary client initialization failed: {str(e)}",
+ original_exception=e,
+ )
+ return self._binary_client
+
async def _execute_with_retry(
self, operation: Callable[[], Awaitable[ResponseT]]
) -> ResponseT:
@@ -160,6 +181,49 @@ async def _operation():
original_exception=e,
)
+ async def get_bytes(self, key: str) -> bytes | None:
+ """Get a value without JSON decoding (for compressed binary blobs)."""
+ try:
+ client = await self._get_binary_client()
+ full_key = self._build_key(key)
+
+ async def _operation() -> bytes | None:
+ result = await client.get(full_key)
+ if result is None:
+ return None
+ if isinstance(result, bytes):
+ return result
+ if isinstance(result, bytearray):
+ return bytes(result)
+ return str(result).encode("utf-8")
+
+ return await self._execute_with_retry(_operation)
+ except Exception as e:
+ logger.error(f"Redis GET_BYTES operation failed: {e}")
+ raise RedisOperationError(
+ internal_message=f"GET_BYTES operation failed: {str(e)}",
+ operation="GET_BYTES",
+ original_exception=e,
+ )
+
+ async def set_bytes(self, key: str, value: bytes, *, ex: int) -> bool:
+ """Set a compressed binary value with an explicit TTL."""
+ try:
+ client = await self._get_binary_client()
+ full_key = self._build_key(key)
+
+ async def _operation() -> bool:
+ return bool(await client.set(full_key, value, ex=ex))
+
+ return await self._execute_with_retry(_operation)
+ except Exception as e:
+ logger.error(f"Redis SET_BYTES operation failed: {e}")
+ raise RedisOperationError(
+ internal_message=f"SET_BYTES operation failed: {str(e)}",
+ operation="SET_BYTES",
+ original_exception=e,
+ )
+
async def delete(self, *keys: str) -> int:
"""Delete keys."""
try:
@@ -608,18 +672,21 @@ async def is_healthy(self) -> bool:
async def close(self):
"""Close the Redis connection."""
- if self._client:
+ clients = [client for client in (self._client, self._binary_client) if client]
+ for client in clients:
close_client = cast(
Callable[[], Awaitable[None]] | None,
- getattr(self._client, "aclose", None),
+ getattr(client, "aclose", None),
)
if close_client is not None:
await close_client()
else:
- await self._client.close()
+ await client.close()
+ if clients:
self._client = None
+ self._binary_client = None
self._health_checker = None
logger.info("Redis connection closed")
diff --git a/packages/shared-python/shared/services/retrieval/execution/routes.py b/packages/shared-python/shared/services/retrieval/execution/routes.py
index 5dbf4f7c5..6ed992937 100644
--- a/packages/shared-python/shared/services/retrieval/execution/routes.py
+++ b/packages/shared-python/shared/services/retrieval/execution/routes.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import resource
import time
from contextlib import AbstractAsyncContextManager
@@ -14,9 +15,7 @@
from shared.services.retrieval.hydration.result_assembly import (
assemble_retrieval_results,
)
-from shared.services.retrieval.hydration.legacy_evidence import (
- render_legacy_evidence_text,
-)
+from shared.services.retrieval.hydration.evidence_text import render_evidence_blocks
from shared.services.retrieval.execution.route_types import (
RetrievalRouteContext,
RetrievalRouteOutcome,
@@ -39,6 +38,28 @@ def open_fresh_database_context() -> AbstractAsyncContextManager[AsyncSession]:
return get_db_context()
+def _evidence_path_header(row: dict) -> str:
+ source = row.get("source")
+ if not isinstance(source, dict):
+ source = row
+ file_name = str(source.get("source_file_name") or "").strip()
+ section_path = str(source.get("section_path") or "").strip()
+ if file_name and section_path:
+ return f"{file_name} / {section_path}"
+ return file_name or section_path
+
+
+def _render_rows_evidence(rows: list[dict]) -> str:
+ groups: dict[str, list[str]] = {}
+ for row in rows:
+ header = _evidence_path_header(row)
+ content = str(row.get("content") or "").strip()
+ if not content:
+ continue
+ groups.setdefault(header, []).append(content)
+ return render_evidence_blocks(list(groups.items()))
+
+
async def run_retrieval_route(
context: RetrievalRouteContext,
) -> RetrievalRouteOutcome:
@@ -101,7 +122,7 @@ async def _try_run_small_corpus_route(
"namespace": context.namespace,
"query": context.query,
"router_used": "small_corpus_all",
- "evidence_text": render_legacy_evidence_text(results),
+ "evidence_text": _render_rows_evidence(results),
"answer_text": "",
"results": results,
}
@@ -156,7 +177,7 @@ async def _run_classic_topk_route(
"namespace": context.namespace,
"query": context.query,
"router_used": "classic_topk",
- "evidence_text": render_legacy_evidence_text(results),
+ "evidence_text": _render_rows_evidence(results),
"answer_text": "",
"results": results,
}
@@ -173,6 +194,7 @@ async def _run_mapnav_route(
context: RetrievalRouteContext,
) -> RetrievalRouteOutcome:
"""Default agentic path: PLANNER + HARVEST + CONTROL (checklist map-nav)."""
+ process_started = resource.getrusage(resource.RUSAGE_SELF)
from shared.services.retrieval import nav_llm_backend # noqa: F401
from shared.services.retrieval.nav import run_nav_episode
from shared.services.retrieval.nav.nav_hierarchy import ProviderToolSpace
@@ -202,6 +224,7 @@ async def _run_mapnav_route(
exclude_sections=context.exclude_sections,
lazy=True,
revision_pins=snapshot_pins,
+ generation=(snapshot_pins.generation if snapshot_pins is not None else None),
)
if snapshot_pins is not None and not await is_revision_generation_stable(
context.db,
@@ -223,6 +246,7 @@ async def _run_mapnav_route(
exclude_sections=context.exclude_sections,
lazy=True,
revision_pins=snapshot_pins,
+ generation=(snapshot_pins.generation if snapshot_pins is not None else None),
)
snapshot_seconds = time.perf_counter() - snapshot_started
logger.info(
@@ -338,6 +362,14 @@ async def _run_mapnav_route(
}
completion_detail = f"chunks | evidence={len(evidence_text)} chars | router=mapnav"
+ process_finished = resource.getrusage(resource.RUSAGE_SELF)
+ logger.info(
+ "retrieval mapnav stage=process_resources cpu_seconds={:.3f} "
+ "process_max_rss_kb={}",
+ (process_finished.ru_utime + process_finished.ru_stime)
+ - (process_started.ru_utime + process_started.ru_stime),
+ int(process_finished.ru_maxrss),
+ )
return RetrievalRouteOutcome(
response=response,
hit_stats_results=resolved.refs,
diff --git a/packages/shared-python/shared/services/retrieval/hydration/asset_inline.py b/packages/shared-python/shared/services/retrieval/hydration/asset_inline.py
new file mode 100644
index 000000000..4c49f00d8
--- /dev/null
+++ b/packages/shared-python/shared/services/retrieval/hydration/asset_inline.py
@@ -0,0 +1,85 @@
+"""Insert connected image/table bodies at text placeholders.
+
+Replaces ``[images/...]`` / ``[tables/...]`` (or ``connect_to.ref``) with the
+asset display body, with a newline before and after. Targets not found at a
+placeholder are appended once. Leftover path placeholders are stripped.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+_PATH_REF_RE = re.compile(r"\[(?:images|tables)/[^\]\n]+\]")
+_SAME_AS_RE = re.compile(r"\[SAME-AS [^\]]+\]")
+
+
+def strip_path_placeholders(content: str) -> str:
+ text = _PATH_REF_RE.sub("", content)
+ text = _SAME_AS_RE.sub("", text)
+ return text.strip()
+
+
+def inline_assets_at_placeholders(
+ host_text: str,
+ *,
+ connections: Sequence[Mapping[str, Any]] | Sequence[Any],
+ display_by_target: Mapping[str, str],
+) -> tuple[str, set[str]]:
+ """Return (body, embedded_target_ids).
+
+ ``display_by_target`` maps chunk_id → display body. Only targets present in
+ that map are inserted. Each target is inserted at most once.
+ """
+ text = str(host_text or "")
+ embedded: set[str] = set()
+ pending_append: list[tuple[str, str]] = []
+
+ for item in connections or ():
+ if not isinstance(item, Mapping):
+ continue
+ target_id = str(item.get("target") or "").strip()
+ if not target_id or target_id in embedded:
+ continue
+ body = str(display_by_target.get(target_id) or "").strip()
+ if not body:
+ continue
+
+ ref = str(item.get("ref") or "").strip()
+ placed = False
+ for candidate in _ref_candidates(ref):
+ if candidate and candidate in text:
+ text = text.replace(candidate, f"\n{body}\n", 1)
+ embedded.add(target_id)
+ placed = True
+ break
+ if not placed:
+ pending_append.append((target_id, body))
+
+ for target_id, body in pending_append:
+ if target_id in embedded:
+ continue
+ if text.strip():
+ text = f"{text.rstrip()}\n\n{body}"
+ else:
+ text = body
+ embedded.add(target_id)
+
+ return strip_path_placeholders(text), embedded
+
+
+def _ref_candidates(ref: str) -> list[str]:
+ raw = str(ref or "").strip()
+ if not raw:
+ return []
+ out: list[str] = [raw]
+ if raw.startswith("[") and raw.endswith("]"):
+ inner = raw[1:-1].strip()
+ if inner and inner not in out:
+ out.append(inner)
+ else:
+ bracketed = f"[{raw}]"
+ if bracketed not in out:
+ out.append(bracketed)
+ return out
diff --git a/packages/shared-python/shared/services/retrieval/hydration/connected.py b/packages/shared-python/shared/services/retrieval/hydration/connected.py
index 2a1f31276..8e6379573 100644
--- a/packages/shared-python/shared/services/retrieval/hydration/connected.py
+++ b/packages/shared-python/shared/services/retrieval/hydration/connected.py
@@ -62,7 +62,11 @@ async def hydrate_connected_target_rows(
return []
stmt = (
- select(Document, DocumentChunk, DocumentSection, JobResult)
+ # Select only the job identifier needed for the public projection.
+ # Selecting the JobResult entity triggers its ``chunks`` selectin
+ # relationship, loading the entire legacy job-chunk collection for
+ # every connected revision during final hydration.
+ select(Document, DocumentChunk, DocumentSection, JobResult.job_id)
.join(
DocumentChunk,
(
@@ -90,7 +94,7 @@ async def hydrate_connected_target_rows(
result = await db.execute(stmt)
hydrated_rows: list[dict[str, Any]] = []
- for document, chunk, section, job_result in result.all():
+ for document, chunk, section, job_id in result.all():
section_path = section.section_path if section else None
hydrated_rows.append(
{
@@ -105,7 +109,7 @@ async def hydrate_connected_target_rows(
'file_path': chunk.file_path,
'chunk_metadata': chunk.chunk_metadata or {},
'job_result_id': chunk.job_result_id,
- 'job_id': job_result.job_id if job_result else None,
+ 'job_id': job_id,
'sort_order': chunk.sort_order,
}
)
diff --git a/packages/shared-python/shared/services/retrieval/hydration/evidence_text.py b/packages/shared-python/shared/services/retrieval/hydration/evidence_text.py
new file mode 100644
index 000000000..a868302e7
--- /dev/null
+++ b/packages/shared-python/shared/services/retrieval/hydration/evidence_text.py
@@ -0,0 +1,45 @@
+"""Shared evidence_text rendering: one ``[E#]`` block per group.
+
+Each block is ``[E#]`` + ``[§ path]`` (full traceable path) + body lines.
+Groups are caller-provided; bodies in one group stay in that group.
+"""
+
+from __future__ import annotations
+
+from typing import Sequence
+
+
+def render_evidence_blocks(
+ groups: Sequence[tuple[str, Sequence[str]]],
+ *,
+ start_index: int = 1,
+) -> str:
+ """Render (path, bodies) groups into evidence_text.
+
+ ``path`` is the full traceable header (e.g. file / section chain).
+ Bodies are joined with newlines; multiple bodies in one group are indented.
+ ``start_index`` sets the first ``[E#]`` number (default 1).
+ """
+ parts: list[str] = []
+ index = max(1, int(start_index or 1))
+ for path, bodies in groups:
+ texts = [str(t or "").strip() for t in bodies]
+ texts = [t for t in texts if t]
+ if not texts:
+ continue
+ block: list[str] = [f"[E{index + len(parts)}]"]
+ header = str(path or "").strip()
+ if header:
+ block.append(f"[§ {header}]")
+ indent = len(texts) >= 2
+ for text in texts:
+ if indent:
+ block.append(
+ "\n".join(
+ (" " + ln if ln.strip() else ln) for ln in text.splitlines()
+ )
+ )
+ else:
+ block.append(text)
+ parts.append("\n".join(block).strip())
+ return "\n\n".join(parts)
diff --git a/packages/shared-python/shared/services/retrieval/hydration/legacy_evidence.py b/packages/shared-python/shared/services/retrieval/hydration/legacy_evidence.py
deleted file mode 100644
index 63b4ea358..000000000
--- a/packages/shared-python/shared/services/retrieval/hydration/legacy_evidence.py
+++ /dev/null
@@ -1,55 +0,0 @@
-from __future__ import annotations
-
-from collections import defaultdict
-from typing import Any
-
-
-def render_legacy_evidence_text(rows: list[dict[str, Any]]) -> str:
- """Render assembled retrieval rows into evidence-only context."""
- grouped_rows: dict[str, list[dict[str, Any]]] = defaultdict(list)
- for row in rows:
- doc_name = _source_value(row, "source_file_name") or "Unknown document"
- grouped_rows[doc_name].append(row)
-
- parts: list[str] = []
- for doc_name in sorted(grouped_rows):
- parts.append(f"[Document] {doc_name}")
- last_section = object()
- for row in sorted(grouped_rows[doc_name], key=_row_sort_key):
- section_path = _source_value(row, "section_path") or doc_name
- if section_path != last_section:
- parts.append(f"▸ {section_path}")
- last_section = section_path
- _append_content_lines(parts, row.get("content"))
-
- return "\n".join(parts)
-
-
-def _source_value(row: dict[str, Any], key: str) -> str:
- source = row.get("source")
- if isinstance(source, dict):
- value = source.get(key)
- if value:
- return str(value)
- value = row.get(key)
- return str(value) if value else ""
-
-
-def _row_sort_key(row: dict[str, Any]) -> tuple[str, int, str]:
- section_path = _source_value(row, "section_path")
- try:
- sort_order = int(row.get("sort_order") or 0)
- except (TypeError, ValueError):
- sort_order = 0
- chunk_id = str(row.get("chunk_id") or "")
- return section_path, sort_order, chunk_id
-
-
-def _append_content_lines(parts: list[str], content: object) -> None:
- text = str(content or "").strip()
- if not text:
- return
- for line in text.splitlines():
- stripped = line.strip()
- if stripped:
- parts.append(f" ┈ {stripped}")
diff --git a/packages/shared-python/shared/services/retrieval/hydration/result_assembly.py b/packages/shared-python/shared/services/retrieval/hydration/result_assembly.py
index 863bfadba..2f6aa54ad 100644
--- a/packages/shared-python/shared/services/retrieval/hydration/result_assembly.py
+++ b/packages/shared-python/shared/services/retrieval/hydration/result_assembly.py
@@ -5,9 +5,12 @@
from sqlalchemy.ext.asyncio import AsyncSession
+from shared.services.retrieval.hydration.asset_inline import (
+ inline_assets_at_placeholders,
+ strip_path_placeholders,
+)
from shared.services.retrieval.hydration.connected import hydrate_connected_target_rows
from shared.services.retrieval.hydration.row_utils import (
- clean_content,
extract_page_nums,
filter_excluded_rows,
iter_connected_target_ids,
@@ -70,18 +73,12 @@ async def assemble_retrieval_results(
assembled_row['content'] = _compose_table_content(row, rows_by_chunk_id)
assembled_row['content_source'] = 'summary'
elif chunk_type == 'text':
- related_parts = _connected_media_parts(row, rows_by_chunk_id)
- if base_content and related_parts:
- assembled_row['content'] = '\n\n'.join([base_content, *related_parts])
- elif related_parts:
- assembled_row['content'] = '\n\n'.join(related_parts)
- else:
- assembled_row['content'] = base_content
+ assembled_row['content'] = _compose_text_content(row, rows_by_chunk_id)
assembled_row['content_source'] = 'content'
else:
assembled_row['content'] = base_content
assembled_row['content_source'] = 'content'
- assembled_row['content'] = clean_content(assembled_row['content'])
+ assembled_row['content'] = strip_path_placeholders(assembled_row['content'])
assembled.append(assembled_row)
return assembled
@@ -93,6 +90,26 @@ def _page_summary(row: dict[str, Any]) -> str:
return str(metadata.get('summary') or '').strip()
+def _compose_text_content(
+ row: dict[str, Any],
+ rows_by_chunk_id: dict[str, dict[str, Any]],
+) -> str:
+ base_content = str(row.get('content') or '')
+ display_by_target = _connected_display_by_target(row, rows_by_chunk_id)
+ if not display_by_target:
+ return base_content
+ metadata = row.get('chunk_metadata') or row.get('metadata') or {}
+ connections = (
+ metadata.get('connect_to') if isinstance(metadata, dict) else None
+ ) or []
+ content, _embedded = inline_assets_at_placeholders(
+ base_content,
+ connections=connections if isinstance(connections, list) else [],
+ display_by_target=display_by_target,
+ )
+ return content
+
+
def _compose_table_content(
row: dict[str, Any],
rows_by_chunk_id: dict[str, dict[str, Any]],
@@ -102,11 +119,11 @@ def _compose_table_content(
return '\n\n'.join(part for part in parts if part)
-def _connected_media_parts(
+def _connected_display_by_target(
row: dict[str, Any],
rows_by_chunk_id: dict[str, dict[str, Any]],
-) -> list[str]:
- connected_targets: list[tuple[int, str]] = []
+) -> dict[str, str]:
+ display: dict[str, str] = {}
for target_id in iter_connected_target_ids(row):
target_row = rows_by_chunk_id.get(target_id)
if not target_row:
@@ -119,10 +136,8 @@ def _connected_media_parts(
else:
continue
if target_content:
- sort_key = int(target_row.get('sort_order', 0) or 0)
- connected_targets.append((sort_key, target_content))
- connected_targets.sort(key=lambda item: item[0])
- return [content for _, content in connected_targets]
+ display[target_id] = target_content
+ return display
def _connected_image_parts(
diff --git a/packages/shared-python/shared/services/retrieval/hydration/row_utils.py b/packages/shared-python/shared/services/retrieval/hydration/row_utils.py
index 32bcb2ba2..74fa382e9 100644
--- a/packages/shared-python/shared/services/retrieval/hydration/row_utils.py
+++ b/packages/shared-python/shared/services/retrieval/hydration/row_utils.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import re
from typing import Any
from shared.services.retrieval.search.section_filters import is_excluded_section
@@ -22,15 +21,6 @@
ReferenceLookupKey = tuple[str, str, str, str]
-_PATH_REF_RE = re.compile(r'\[(?:images|tables)/[^\]\n]+\]')
-_SAME_AS_RE = re.compile(r'\[SAME-AS [^\]]+\]')
-
-
-def clean_content(content: str) -> str:
- text = _PATH_REF_RE.sub('', content)
- text = _SAME_AS_RE.sub('', text)
- return text.strip()
-
def normalize_chunk_type(raw: object) -> str:
return str(raw or '').strip().split('\n', 1)[0].lower()
diff --git a/packages/shared-python/shared/services/retrieval/map_unit_index.py b/packages/shared-python/shared/services/retrieval/map_unit_index.py
index 4fff8206a..ab7a05231 100644
--- a/packages/shared-python/shared/services/retrieval/map_unit_index.py
+++ b/packages/shared-python/shared/services/retrieval/map_unit_index.py
@@ -23,11 +23,11 @@
UnitRow,
)
from shared.services.retrieval.nav.nav_map_scores import build_score_units
+from shared.services.retrieval.nav.knowhere_hybrid import MAP_UNIT_INDEX_FORMAT_VERSION
from shared.services.retrieval.nav.persisted_score_load import average_idf_from_unit_dfs
from shared.services.retrieval.publication_models import DocumentPublicationScope
-
-MAP_UNIT_INDEX_FORMAT_VERSION = 1
+__all__ = ["MAP_UNIT_INDEX_FORMAT_VERSION", "replace_document_map_units"]
def replace_document_map_units(
@@ -88,6 +88,10 @@ def replace_document_map_units(
token_count = 0
path_unit_df: Counter[str] = Counter()
content_unit_df: Counter[str] = Counter()
+ path_document_count: int = 0
+ path_total_length: int = 0
+ content_document_count: int = 0
+ content_total_length: int = 0
for sort_order, unit in enumerate(score_units):
unit_id = str(unit.get("chunk_id") or "").strip()
section_id = str(unit.get("section_id") or "").strip()
@@ -96,6 +100,12 @@ def replace_document_map_units(
map_unit_id = f"dmu_{uuid4().hex}"
path_tokens = str(unit.get("path_search_text") or "").split()
content_tokens = str(unit.get("content_search_text") or "").split()
+ if path_tokens:
+ path_document_count += 1
+ path_total_length += len(path_tokens)
+ if content_tokens:
+ content_document_count += 1
+ content_total_length += len(content_tokens)
path_unit_df.update(set(path_tokens))
content_unit_df.update(set(content_tokens))
# ``provider.self_units`` already reflects root-asset remount (assets
@@ -152,6 +162,10 @@ def replace_document_map_units(
unit_count=persisted_count,
token_document_frequency=content_unit_df,
),
+ path_document_count=path_document_count,
+ path_total_length=path_total_length,
+ content_document_count=content_document_count,
+ content_total_length=content_total_length,
)
)
diff --git a/packages/shared-python/shared/services/retrieval/namespace_map_snapshot.py b/packages/shared-python/shared/services/retrieval/namespace_map_snapshot.py
index 41fa6c6b4..2c31b296f 100644
--- a/packages/shared-python/shared/services/retrieval/namespace_map_snapshot.py
+++ b/packages/shared-python/shared/services/retrieval/namespace_map_snapshot.py
@@ -19,8 +19,8 @@
)
from shared.services.retrieval.publication_models import DocumentPublicationScope
from shared.services.retrieval.serving_manifest import (
- decode_serving_manifest,
- encode_serving_manifest,
+ decode_namespace_map_snapshot,
+ encode_namespace_map_snapshot,
)
@@ -107,7 +107,7 @@ def _decode_documents(
if row is None:
return {}
try:
- payload = decode_serving_manifest(
+ payload = decode_namespace_map_snapshot(
row.payload_zlib,
checksum=row.checksum,
format_version=row.format_version,
@@ -127,7 +127,7 @@ def _write_snapshot(
documents: dict[str, dict[str, Any]],
target_generation: int,
) -> None:
- encoded, checksum, format_version = encode_serving_manifest(
+ encoded, checksum, format_version = encode_namespace_map_snapshot(
{"documents": documents}
)
if row is None:
diff --git a/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_cache.py b/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_cache.py
deleted file mode 100644
index 6cc5cab29..000000000
--- a/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_cache.py
+++ /dev/null
@@ -1,43 +0,0 @@
-"""Process-local cache for decoded namespace MAP snapshot documents.
-
-Keyed by ``(user_id, namespace, generation)`` so a publish/archive that bumps
-the namespace generation invalidates stale entries automatically -- no manual
-invalidation call is needed. Bounded LRU keeps memory use predictable across
-many namespaces sharing one worker process.
-"""
-
-from __future__ import annotations
-
-import threading
-from collections import OrderedDict
-from typing import Any
-
-_MAX_ENTRIES = 64
-_lock = threading.Lock()
-_cache: "OrderedDict[tuple[str, str, int], dict[str, dict[str, Any]]]" = OrderedDict()
-
-
-def get_cached_namespace_documents(
- *, user_id: str, namespace: str, generation: int
-) -> dict[str, dict[str, Any]] | None:
- key = (user_id, namespace, generation)
- with _lock:
- documents = _cache.get(key)
- if documents is not None:
- _cache.move_to_end(key)
- return documents
-
-
-def cache_namespace_documents(
- *,
- user_id: str,
- namespace: str,
- generation: int,
- documents: dict[str, dict[str, Any]],
-) -> None:
- key = (user_id, namespace, generation)
- with _lock:
- _cache[key] = documents
- _cache.move_to_end(key)
- while len(_cache) > _MAX_ENTRIES:
- _cache.popitem(last=False)
diff --git a/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_redis.py b/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_redis.py
new file mode 100644
index 000000000..162b85c25
--- /dev/null
+++ b/packages/shared-python/shared/services/retrieval/namespace_map_snapshot_redis.py
@@ -0,0 +1,42 @@
+"""Redis cache for compressed namespace MAP routing snapshots."""
+
+from __future__ import annotations
+
+from shared.models.schemas.retrieval_namespace import normalize_retrieval_namespace
+from shared.services.redis import RedisServiceFactory
+
+_CACHE_TTL_SECONDS = 3600
+_KEY_PREFIX = "retrieval:snapshot:v2"
+
+
+class NamespaceMapSnapshotRedisCache:
+ """Access generation-scoped compressed namespace snapshots in Redis."""
+
+ @staticmethod
+ def _build_key(*, user_id: str, namespace: str, generation: int) -> str:
+ normalized_namespace = normalize_retrieval_namespace(namespace)
+ return f"{_KEY_PREFIX}:{user_id}:{normalized_namespace}:g{int(generation)}"
+
+ @classmethod
+ async def get(
+ cls, *, user_id: str, namespace: str, generation: int
+ ) -> bytes | None:
+ service = RedisServiceFactory.get_service()
+ return await service.get_bytes(
+ cls._build_key(
+ user_id=user_id, namespace=namespace, generation=generation
+ )
+ )
+
+ @classmethod
+ async def set(
+ cls, *, user_id: str, namespace: str, generation: int, payload_zlib: bytes
+ ) -> bool:
+ service = RedisServiceFactory.get_service()
+ return await service.set_bytes(
+ cls._build_key(
+ user_id=user_id, namespace=namespace, generation=generation
+ ),
+ payload_zlib,
+ ex=_CACHE_TTL_SECONDS,
+ )
diff --git a/packages/shared-python/shared/services/retrieval/nav/knowhere_hybrid.py b/packages/shared-python/shared/services/retrieval/nav/knowhere_hybrid.py
index 2c4bbb48e..6e5a95f81 100644
--- a/packages/shared-python/shared/services/retrieval/nav/knowhere_hybrid.py
+++ b/packages/shared-python/shared/services/retrieval/nav/knowhere_hybrid.py
@@ -8,28 +8,34 @@
from __future__ import annotations
-import os
-import re
import math
+import os
from dataclasses import dataclass
from typing import Dict, List, Mapping, Optional, Sequence, Tuple
+from shared.utils.text_utils import tokenize_for_retrieval as _tokenize_word_level
+
RRF_K = 60
CHANNEL_WEIGHT_PATH = 1.0
CHANNEL_WEIGHT_CONTENT = 2.0
+# Persisted map-unit token format. Bump when tokenizer semantics change.
+# v1: character-level CJK regex. v2: word-level jieba (text_utils).
+MAP_UNIT_INDEX_FORMAT_VERSION = 2
def tokenize_for_retrieval(text: str, *, dedupe: bool = True) -> List[str]:
- tokens = re.findall(r"[a-z0-9_]+|[\u4e00-\u9fff]", str(text or "").lower())
- if not dedupe:
- return [t for t in tokens if t]
- seen: set[str] = set()
- out: List[str] = []
- for t in tokens:
- if t and t not in seen:
- seen.add(t)
- out.append(t)
- return out
+ """Word-level retrieval tokens (jieba + English), aligned with chunk publication.
+
+ Uses the same knobs as ``search.lexical_text``: no stopword filtering and
+ ``min_token_length=2`` so map-unit indexes match ``document_chunks`` search
+ text. Character-level regex tokenization was removed.
+ """
+ return _tokenize_word_level(
+ text,
+ stopwords=[],
+ dedupe=dedupe,
+ min_token_length=2,
+ )
def tokenize_query_for_ranker(query: str) -> List[str]:
diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_compose.py b/packages/shared-python/shared/services/retrieval/nav/nav_compose.py
index 3951a70c4..e23a5f825 100644
--- a/packages/shared-python/shared/services/retrieval/nav/nav_compose.py
+++ b/packages/shared-python/shared/services/retrieval/nav/nav_compose.py
@@ -12,6 +12,8 @@
from dataclasses import dataclass
from typing import Any, Dict, List, Optional, Sequence, Set, Tuple
+from shared.services.retrieval.hydration.evidence_text import render_evidence_blocks
+
from ._compat import Chunk
from ._compat import line_node_id
from ._compat import ToolSpace
@@ -100,17 +102,11 @@ def direct_parent_id(ts: ToolSpace, section_id: str, doc_id: str) -> Optional[st
return line_node_id(resolved, b.lines[p].line_id)
-def _section_title(ts: ToolSpace, section_id: str, doc_id: str, *, max_chars: int = 40) -> str:
+def _section_title(ts: ToolSpace, section_id: str, doc_id: str) -> str:
sid = str(section_id or "").strip()
if not sid:
return ""
- def _clip(text: str) -> str:
- t = (text or "").strip()
- if len(t) > max_chars:
- return t[:max_chars].rstrip()
- return t
-
# ``path_titles`` is a lightweight title lookup. Prefer it over
# ``get_structure`` because the latter may calculate the complete subtree
# chunk count, which is unnecessary while rendering evidence headers.
@@ -121,7 +117,7 @@ def _clip(text: str) -> str:
except TypeError:
path = str(path_titles(sid) or "").strip()
if path:
- return _clip(path.rsplit(" / ", 1)[-1])
+ return path
# Prefer structure title (Knowhere / ProviderToolSpace); never parse ids.
try:
@@ -131,7 +127,7 @@ def _clip(text: str) -> str:
if isinstance(st, dict):
raw = st.get("preview") or st.get("title") or ""
if isinstance(raw, str) and raw.strip():
- return _clip(raw.strip())
+ return raw.strip()
resolved = _section_doc_id(ts, sid, doc_id)
idx = getattr(ts, "_idx", None)
@@ -145,13 +141,13 @@ def _clip(text: str) -> str:
bb = getattr(idx, "_bundles", {}).get(resolved)
if bb and bb.lines:
title = (bb.lines[0].content or "").strip()
- return _clip(title) if title else sid
+ return title if title else sid
return sid
_, j = loc
if j < 0 or j >= len(b.lines):
return sid
title = (b.lines[j].content or "").strip()
- return _clip(title) if title else sid
+ return title if title else sid
def _chunk_body(chunk: Chunk) -> str:
@@ -302,7 +298,7 @@ def _build_groups(
if parent_id is None:
parent_id = owner
if parent_id not in groups:
- title = _section_title(ts, parent_id, owner_doc, max_chars=40)
+ title = _section_title(ts, parent_id, owner_doc)
groups[parent_id] = _ParentGroup(
parent_id=parent_id,
parent_title=title,
@@ -325,24 +321,13 @@ def _render_group(
selected: Sequence[_ChildItem],
*,
evidence_index: int,
- indent: bool,
) -> str:
- """Render one evidence block (full text only)."""
- parts: List[str] = [f"[E{evidence_index}]"]
- if group.parent_title:
- parts.append(f"[§ {group.parent_title}]")
- for child in selected:
- body = _chunk_body(child.chunk)
- if not body:
- continue
- if indent:
- indented = "\n".join(
- (" " + ln if ln.strip() else ln) for ln in body.splitlines()
- )
- parts.append(indented)
- else:
- parts.append(body)
- return "\n".join(parts).strip()
+ """Render one evidence block via the shared evidence renderer."""
+ bodies = [_chunk_body(child.chunk) for child in selected]
+ return render_evidence_blocks(
+ [(group.parent_title or "", bodies)],
+ start_index=evidence_index,
+ )
def _scored_flat(groups: Sequence[_ParentGroup]) -> List[Tuple[Chunk, float]]:
@@ -399,9 +384,8 @@ def _render_kept(
]
if not entries:
continue
- indent = len(entries) >= 2
block = _render_group(
- g, entries, evidence_index=len(parts) + 1, indent=indent
+ g, entries, evidence_index=len(parts) + 1
)
add = len(block) + (len(sep) if parts else 0)
if used + add <= budget_chars:
diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_hierarchy.py b/packages/shared-python/shared/services/retrieval/nav/nav_hierarchy.py
index 80b9cde01..ead3dfc6e 100644
--- a/packages/shared-python/shared/services/retrieval/nav/nav_hierarchy.py
+++ b/packages/shared-python/shared/services/retrieval/nav/nav_hierarchy.py
@@ -36,6 +36,10 @@
runtime_checkable,
)
+from shared.services.retrieval.hydration.asset_inline import (
+ inline_assets_at_placeholders,
+)
+
if TYPE_CHECKING:
from .knowhere_hybrid import PersistedScoreCorpus
@@ -179,17 +183,72 @@ def parent_id(self, section_id: str) -> Optional[str]:
return str(parent) if parent else None
def _node_unit_span(self, section_id: str) -> Tuple[str, int, int]:
- """(joined text, first sort_order, unit count) for one node's own units."""
+ """(joined text, first sort_order, unit count) for one node's own units.
+
+ Evidence display only: text units insert connected assets at placeholders.
+ Scoring still uses ``materialize_self_only_chunks`` / raw ``unit_text``.
+ """
self_units = getattr(self._provider, "self_units", None)
unit_text = getattr(self._provider, "unit_text", None)
if not callable(self_units) or not callable(unit_text):
return "", 0, 0
units = list(self_units(section_id) or ())
- texts = [t for t in (str(unit_text(u) or "").strip() for u in units) if t]
- if not texts:
- return "", 0, len(units)
+ if not units:
+ return "", 0, 0
first_order = int(getattr(units[0], "sort_order", 0) or 0)
- return "\n".join(texts), first_order, len(units)
+
+ asset_types = {"image", "table"}
+ text_units: List[Any] = []
+ asset_by_id: Dict[str, str] = {}
+ for unit in units:
+ chunk_type = str(getattr(unit, "chunk_type", "") or "").strip().lower()
+ chunk_id = str(getattr(unit, "chunk_id", "") or "").strip()
+ body = str(unit_text(unit) or "").strip()
+ if chunk_type in asset_types:
+ if chunk_id and body:
+ asset_by_id[chunk_id] = body
+ continue
+ text_units.append(unit)
+
+ if not text_units:
+ texts = [body for body in asset_by_id.values() if body]
+ return "\n".join(texts), first_order, len(units)
+
+ parts: List[str] = []
+ used_assets: Set[str] = set()
+ for unit in text_units:
+ content = str(unit_text(unit) or "").strip()
+ meta = getattr(unit, "metadata", None) or {}
+ connections = (
+ meta.get("connect_to") if isinstance(meta, dict) else None
+ ) or []
+ if not isinstance(connections, list):
+ connections = []
+ wanted = {
+ str(item.get("target") or "").strip()
+ for item in connections
+ if isinstance(item, dict)
+ }
+ display = {
+ target_id: asset_by_id[target_id]
+ for target_id in wanted
+ if target_id in asset_by_id
+ }
+ content, embedded = inline_assets_at_placeholders(
+ content,
+ connections=connections,
+ display_by_target=display,
+ )
+ used_assets.update(embedded)
+ if content:
+ parts.append(content)
+
+ for target_id, body in asset_by_id.items():
+ if target_id in used_assets or not body:
+ continue
+ parts.append(body)
+
+ return "\n".join(parts), first_order, len(units)
def _make_chunk(
self, node_id: str, doc_id: str, text: str, order: int, section_id: str
diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py b/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py
index bd4e24f76..cd5696312 100644
--- a/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py
+++ b/packages/shared-python/shared/services/retrieval/nav/nav_knowhere.py
@@ -42,6 +42,7 @@
from .nav_address import NavLevel
from .nav_hierarchy import NodeMeta
from .knowhere_hybrid import (
+ MAP_UNIT_INDEX_FORMAT_VERSION,
PersistedScoreCorpus,
PersistedScoreUnit,
tokenize_query_for_ranker,
@@ -51,7 +52,6 @@
# Knowhere sentinel path for the virtual document container (not a collectable leaf).
ROOT_SECTION_PATH = "Root"
_DEFAULT_DSN = "postgresql://root:root123@127.0.0.1:5433/Knowhere"
-_MAP_UNIT_INDEX_FORMAT_VERSION = 1
_MAP_SCORE_CHANNELS: Tuple[str, str] = ("path", "content")
_logger = logging.getLogger(__name__)
@@ -335,7 +335,9 @@ def load_persisted_score_corpus(
cur.execute(
"SELECT indexes.document_id, indexes.job_result_id, "
"indexes.format_version, indexes.unit_count, "
- "indexes.average_idf_path, indexes.average_idf_content "
+ "indexes.average_idf_path, indexes.average_idf_content, "
+ "indexes.path_document_count, indexes.path_total_length, "
+ "indexes.content_document_count, indexes.content_total_length "
"FROM document_map_unit_indexes AS indexes "
f"JOIN (VALUES {values_sql}) AS revisions(document_id, job_result_id) "
"ON indexes.document_id = revisions.document_id "
@@ -353,7 +355,8 @@ def load_persisted_score_corpus(
len(index_rows),
)
if len(index_rows) != len(revisions) or any(
- len(row) < 6 or int(row[2]) != _MAP_UNIT_INDEX_FORMAT_VERSION
+ len(row) < 10 or int(row[2]) != MAP_UNIT_INDEX_FORMAT_VERSION
+ or any(value is None for value in row[6:10])
for row in index_rows
):
return None
@@ -386,8 +389,67 @@ def load_persisted_score_corpus(
for document_id, section_ids in allowed_by_document.items()
for section_id in section_ids
}
+ cur.execute(
+ "SELECT sections.document_id, sections.job_result_id, count(*) "
+ "FROM document_sections AS sections "
+ f"JOIN (VALUES {values_sql}) AS revisions(document_id, job_result_id) "
+ "ON sections.document_id = revisions.document_id "
+ "AND sections.job_result_id = revisions.job_result_id "
+ "GROUP BY sections.document_id, sections.job_result_id",
+ revision_params,
+ )
+ section_counts = {
+ (str(document_id), str(job_result_id)): int(count)
+ for document_id, job_result_id, count in cur.fetchall()
+ }
+ has_complete_section_scope = all(
+ len(allowed_by_document.get(document_id, set()))
+ == section_counts.get((document_id, job_result_id), 0)
+ for document_id, job_result_id in revisions
+ )
all_unit_rows = self._score_unit_rows_cache.get(revision_key)
- if all_unit_rows is None:
+ if has_complete_section_scope and query_token_hashes:
+ stage_started = time.perf_counter()
+ cur.execute(
+ "WITH matching_tokens AS MATERIALIZED ("
+ "SELECT DISTINCT map_unit_id FROM document_map_unit_tokens "
+ "WHERE channel = ANY(%s) AND token_hash = ANY(%s)"
+ "), scoped_units AS MATERIALIZED ("
+ f"SELECT units.id, units.document_id, units.unit_id, units.section_id, "
+ "units.path_token_count, units.content_token_count "
+ "FROM document_map_units AS units "
+ f"JOIN (VALUES {values_sql}) AS revisions(document_id, job_result_id) "
+ "ON units.document_id = revisions.document_id "
+ "AND units.job_result_id = revisions.job_result_id"
+ ") SELECT scoped_units.id, scoped_units.document_id, "
+ "scoped_units.unit_id, scoped_units.section_id, "
+ "scoped_units.path_token_count, scoped_units.content_token_count "
+ "FROM matching_tokens JOIN scoped_units "
+ "ON scoped_units.id = matching_tokens.map_unit_id",
+ [
+ list(_MAP_SCORE_CHANNELS),
+ list(query_token_hashes),
+ *revision_params,
+ ],
+ )
+ unit_rows = [
+ {
+ "map_unit_id": str(row[0]),
+ "document_id": str(row[1]),
+ "unit_id": str(row[2]),
+ "section_id": str(row[3] or ""),
+ "path_token_count": int(row[4] or 0),
+ "content_token_count": int(row[5] or 0),
+ }
+ for row in cur.fetchall()
+ ]
+ _logger.info(
+ "retrieval map-index load stage=units-selective seconds=%.3f rows=%d",
+ time.perf_counter() - stage_started,
+ len(unit_rows),
+ )
+ all_unit_rows = None
+ elif all_unit_rows is None:
stage_started = time.perf_counter()
cur.execute(
"SELECT units.id, units.document_id, units.unit_id, "
@@ -427,26 +489,37 @@ def load_persisted_score_corpus(
True,
)
- unit_rows = [
- row
- for row in all_unit_rows
- if (str(row["document_id"]), str(row["section_id"])) in allowed_pairs_set
- ]
+ if not has_complete_section_scope or not query_token_hashes:
+ unit_rows = [
+ row
+ for row in all_unit_rows or []
+ if (str(row["document_id"]), str(row["section_id"]))
+ in allowed_pairs_set
+ ]
frequencies: Dict[Tuple[str, str], Dict[str, int]] = {}
if unit_rows and query_tokens:
stage_started = time.perf_counter()
- # Restrict the token scan to this episode's map units instead of
- # matching token_hash across the whole table then filtering.
+ # Keep the episode scope explicit without materializing every
+ # token row matching a common query term. PostgreSQL can choose
+ # either the scoped-unit side or the channel/token_hash-leading
+ # index, while the scope still limits results to the pinned
+ # revision and allowed sections.
allowed_map_unit_ids = [str(row["map_unit_id"]) for row in unit_rows]
cur.execute(
- "SELECT map_unit_id, channel, token, frequency "
- "FROM document_map_unit_tokens "
- "WHERE map_unit_id = ANY(%s) "
- "AND token_hash = ANY(%s) AND channel = ANY(%s)",
+ "WITH scoped_units AS MATERIALIZED ("
+ "SELECT unnest(%s::text[]) AS map_unit_id"
+ ") "
+ "SELECT tokens.map_unit_id, tokens.channel, tokens.token, "
+ "tokens.frequency "
+ "FROM document_map_unit_tokens AS tokens "
+ "JOIN scoped_units "
+ "ON scoped_units.map_unit_id = tokens.map_unit_id "
+ "WHERE tokens.channel = ANY(%s) "
+ "AND tokens.token_hash = ANY(%s)",
[
allowed_map_unit_ids,
- list(query_token_hashes),
list(_MAP_SCORE_CHANNELS),
+ list(query_token_hashes),
],
)
for map_unit_id, channel, token, frequency in cur.fetchall():
@@ -473,6 +546,16 @@ def load_persisted_score_corpus(
query_tokens=query_tokens,
frequencies=frequencies,
average_idf=average_idf_path,
+ document_count_override=(
+ sum(int(row[6] or 0) for row in index_rows)
+ if has_complete_section_scope and query_token_hashes
+ else None
+ ),
+ total_length_override=(
+ sum(int(row[7] or 0) for row in index_rows)
+ if has_complete_section_scope and query_token_hashes
+ else None
+ ),
)
content_stats = build_channel_bm25_stats(
unit_rows=unit_rows,
@@ -482,6 +565,16 @@ def load_persisted_score_corpus(
query_tokens=query_tokens,
frequencies=frequencies,
average_idf=average_idf_content,
+ document_count_override=(
+ sum(int(row[8] or 0) for row in index_rows)
+ if has_complete_section_scope and query_token_hashes
+ else None
+ ),
+ total_length_override=(
+ sum(int(row[9] or 0) for row in index_rows)
+ if has_complete_section_scope and query_token_hashes
+ else None
+ ),
)
return PersistedScoreCorpus(
units=[
diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py b/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py
index 28c046001..eb207ce3a 100644
--- a/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py
+++ b/packages/shared-python/shared/services/retrieval/nav/nav_map_scores.py
@@ -20,6 +20,22 @@
_logger = logging.getLogger(__name__)
+def _count_tree_shape(
+ tree_by_doc: Dict[
+ str,
+ Tuple[Dict[str, List[str]], Set[str], Dict[str, str]],
+ ],
+) -> Tuple[int, int, int]:
+ """Return reachable section nodes, parent-child edges, and leaves."""
+ section_nodes: int = sum(len(value[0]) for value in tree_by_doc.values())
+ section_edges: int = sum(
+ sum(len(children) for children in value[0].values())
+ for value in tree_by_doc.values()
+ )
+ leaf_sections: int = sum(len(value[1]) for value in tree_by_doc.values())
+ return section_nodes, section_edges, leaf_sections
+
+
def _build_legacy_score_corpus(ts: Any, doc_ids: Sequence[str]) -> PersistedScoreCorpus:
"""Build the retired in-memory scorer input when persisted indexes are absent."""
raw_units: List[dict] = []
@@ -227,31 +243,31 @@ def _pool_unit_scores_to_tree(
unit_scores: Dict[str, float],
) -> Dict[str, float]:
"""MAX-pool globally comparable unit scores onto one document tree."""
- map_scores = {
- leaf_id: float(unit_scores.get(leaf_id, 0.0) or 0.0) for leaf_id in leaves
- }
-
- def score_node(section_id: str) -> float:
- if section_id in map_scores:
- return map_scores[section_id]
+ # ``_walk_tree`` inserts every parent before its children. Reversing that
+ # order is therefore a postorder traversal without allocating descendant
+ # lists or revisiting nodes.
+ map_scores: Dict[str, float] = {}
+ descendant_leaf_max: Dict[str, float] = {}
+ for section_id in reversed(children_map):
kids = children_map.get(section_id) or []
if not kids:
- score = float(unit_scores.get(section_id, 0.0) or 0.0)
- map_scores[section_id] = score
- return score
- descendant_leaves = _collect_descendant_leaves(section_id, children_map, leaves)
- parts = [
- float(unit_scores.get(leaf_id, 0.0) or 0.0) for leaf_id in descendant_leaves
- ]
- self_key = f"{section_id}__self"
- if self_key in unit_scores:
- parts.append(float(unit_scores[self_key]))
- score = float(max(parts)) if parts else 0.0
- map_scores[section_id] = score
- return score
-
- for section_id in children_map:
- score_node(section_id)
+ leaf_score = float(unit_scores.get(section_id, 0.0) or 0.0)
+ descendant_leaf_max[section_id] = leaf_score
+ score = leaf_score
+ else:
+ child_leaf_scores = [
+ descendant_leaf_max.get(kid, float(unit_scores.get(kid, 0.0) or 0.0))
+ for kid in kids
+ ]
+ leaf_score = max(child_leaf_scores, default=0.0)
+ descendant_leaf_max[section_id] = leaf_score
+ self_score = float(unit_scores.get(f"{section_id}__self", 0.0) or 0.0)
+ score = max(leaf_score, self_score)
+ map_scores[section_id] = float(score)
+ # Preserve the legacy behavior for leaf ids that are not present in the
+ # children map (defensive support for sparse providers).
+ for leaf_id in leaves:
+ map_scores.setdefault(leaf_id, float(unit_scores.get(leaf_id, 0.0) or 0.0))
return map_scores
@@ -414,11 +430,15 @@ def compute_corpus_map_and_unit_scores_many(
cached = _walk_tree(ts, doc_id, root_ids)
tree_cache[doc_id] = cached
tree_by_doc[doc_id] = cached
+ section_nodes, section_edges, leaf_sections = _count_tree_shape(tree_by_doc)
_logger.info(
- "retrieval mapnav phase=tree_build seconds=%.3f documents=%d sections=%d",
+ "retrieval mapnav phase=tree_build seconds=%.3f documents=%d "
+ "section_nodes=%d section_edges=%d leaf_sections=%d",
time.perf_counter() - tree_started,
len(valid_doc_ids),
- sum(len(value[0]) for value in tree_by_doc.values()),
+ section_nodes,
+ section_edges,
+ leaf_sections,
)
persisted_loader = getattr(ts, "load_persisted_score_corpus", None)
@@ -471,10 +491,13 @@ def compute_corpus_map_and_unit_scores_many(
map_scores[doc_id] = doc_max
results[query] = (map_scores, unit_scores)
_logger.info(
- "retrieval mapnav phase=map_pooling seconds=%.3f documents=%d sections=%d",
+ "retrieval mapnav phase=map_pooling seconds=%.3f documents=%d "
+ "section_nodes=%d section_edges=%d leaf_sections=%d",
time.perf_counter() - pooling_started,
len(valid_doc_ids),
- sum(len(value[0]) for value in tree_by_doc.values()),
+ section_nodes,
+ section_edges,
+ leaf_sections,
)
return results
diff --git a/packages/shared-python/shared/services/retrieval/nav/persisted_score_load.py b/packages/shared-python/shared/services/retrieval/nav/persisted_score_load.py
index 9ce10c2e1..712f23b5e 100644
--- a/packages/shared-python/shared/services/retrieval/nav/persisted_score_load.py
+++ b/packages/shared-python/shared/services/retrieval/nav/persisted_score_load.py
@@ -47,12 +47,19 @@ def build_channel_bm25_stats(
query_tokens: Sequence[str],
frequencies: Mapping[tuple[str, str], Mapping[str, int]],
average_idf: float,
+ document_count_override: int | None = None,
+ total_length_override: int | None = None,
) -> PersistedBm25Stats:
"""Build channel stats from already-fetched unit rows and query-token freqs."""
lengths = [
int(row[length_field]) for row in unit_rows if int(row[length_field]) > 0
]
- document_count = len(lengths)
+ document_count = (
+ len(lengths) if document_count_override is None else document_count_override
+ )
+ total_length = (
+ sum(lengths) if total_length_override is None else total_length_override
+ )
document_frequency = {
token: sum(
1
@@ -66,7 +73,7 @@ def build_channel_bm25_stats(
}
return PersistedBm25Stats(
document_count=document_count,
- total_length=sum(lengths),
+ total_length=total_length,
document_frequency=document_frequency,
average_idf=float(average_idf),
)
diff --git a/packages/shared-python/shared/services/retrieval/nav_config.py b/packages/shared-python/shared/services/retrieval/nav_config.py
index be1d3dd83..df5e9b5e5 100644
--- a/packages/shared-python/shared/services/retrieval/nav_config.py
+++ b/packages/shared-python/shared/services/retrieval/nav_config.py
@@ -61,7 +61,7 @@
"max_waves": 0,
"max_harvest_depth": 3,
"plan_control_digest_chars": 600,
- "enable_node_filter": False,
+ "enable_node_filter": True,
"filter_max_rounds": 3,
"filter_min_hits": 1,
"filter_max_hits": 40,
diff --git a/packages/shared-python/shared/services/retrieval/nav_snapshot.py b/packages/shared-python/shared/services/retrieval/nav_snapshot.py
index ea502c54b..9f92617c4 100644
--- a/packages/shared-python/shared/services/retrieval/nav_snapshot.py
+++ b/packages/shared-python/shared/services/retrieval/nav_snapshot.py
@@ -9,7 +9,9 @@
import json
import logging
+import os
import time
+import zlib
from dataclasses import dataclass
from collections.abc import Callable, Iterator, Mapping
from typing import Any, Protocol
@@ -33,6 +35,7 @@
DocumentChunk,
DocumentSection,
RetrievalNamespaceMapSnapshot,
+ RetrievalNamespaceGeneration,
RetrievalServingRevisionManifest,
)
from shared.models.database.job_result import JobResult
@@ -46,12 +49,15 @@
knowhere_database_url,
)
from shared.services.retrieval.search.section_filters import is_excluded_section
-from shared.services.retrieval.serving_manifest import decode_serving_manifest
+from shared.services.retrieval.serving_manifest import (
+ decode_namespace_map_snapshot,
+ decode_serving_manifest,
+)
from shared.services.retrieval.manifest_cache import get_cached_manifest_payloads
-from shared.services.retrieval.namespace_map_snapshot_cache import (
- cache_namespace_documents,
- get_cached_namespace_documents,
+from shared.services.retrieval.namespace_map_snapshot_redis import (
+ NamespaceMapSnapshotRedisCache,
)
+from shared.core.exceptions.redis_exceptions import RedisOperationError
# Keep each payload query bounded under the API's 30-second statement timeout.
@@ -70,6 +76,36 @@
_logger = logging.getLogger(__name__)
+def _is_snapshot_timing_enabled() -> bool:
+ """Enable detailed snapshot timings for local performance diagnosis."""
+ return os.environ.get("RETRIEVAL_SNAPSHOT_TIMING", "0").strip().lower() in {
+ "1",
+ "true",
+ "yes",
+ "on",
+ }
+
+
+def _count_section_shape(
+ sections_by_doc: Mapping[str, list[SectionRow]],
+) -> tuple[int, int, int, int]:
+ """Return section rows, distinct paths, roots, and leaves for diagnostics."""
+ rows: list[SectionRow] = [
+ row for document_rows in sections_by_doc.values() for row in document_rows
+ ]
+ parent_ids: set[str] = {
+ str(row.parent_section_id).strip()
+ for row in rows
+ if str(row.parent_section_id or "").strip()
+ }
+ return (
+ len(rows),
+ len({str(row.section_path) for row in rows}),
+ sum(1 for row in rows if not str(row.parent_section_id or "").strip()),
+ sum(1 for row in rows if str(row.section_id) not in parent_ids),
+ )
+
+
class SnapshotSession(Protocol):
"""Minimal database interface required by the snapshot loader."""
@@ -149,12 +185,14 @@ async def load_nav_snapshot(
exclude_sections: list[dict[str, str]] | None = None,
lazy: bool = False,
revision_pins: Mapping[str, str] | None = None,
+ generation: int | None = None,
) -> NavSnapshot:
"""Preload namespace current revision into a sync map-nav snapshot."""
excluded_docs = [
str(x).strip() for x in (exclude_document_ids or ()) if str(x).strip()
]
excluded_secs = list(exclude_sections or ())
+ snapshot_started = time.perf_counter()
if revision_pins is None:
doc_stmt = (
@@ -182,7 +220,9 @@ async def load_nav_snapshot(
)
if excluded_docs:
doc_stmt = doc_stmt.where(Document.document_id.notin_(excluded_docs))
+ doc_query_started = time.perf_counter()
doc_rows = list((await db.execute(doc_stmt)).all())
+ doc_query_seconds = time.perf_counter() - doc_query_started
if not doc_rows:
raise ValueError(
f"no active documents with current revision for "
@@ -207,11 +247,13 @@ async def load_nav_snapshot(
current_job_result_ids.add(job_result_id)
document_revisions.append((did, job_result_id))
+ job_query_started = time.perf_counter()
job_result_rows = await db.execute(
select(JobResult.id, JobResult.job_id).where(
JobResult.id.in_(list(current_job_result_ids))
)
)
+ job_query_seconds = time.perf_counter() - job_query_started
job_id_by_result_id = {
str(job_result_id): str(job_id)
for job_result_id, job_id in job_result_rows.all()
@@ -223,13 +265,16 @@ async def load_nav_snapshot(
user_id=user_id,
namespace=namespace,
document_revisions=document_revisions,
+ expected_generation=generation,
)
if snapshot_entries is not None:
+ parse_started = time.perf_counter()
manifest_sections = _parse_manifest_entries(
snapshot_entries,
exclude_sections=excluded_secs,
job_id_by_result_id=job_id_by_result_id,
)
+ parse_seconds = time.perf_counter() - parse_started
else:
_logger.warning(
"retrieval snapshot fallback=manifest_merge user_id=%s namespace=%s documents=%d",
@@ -245,6 +290,7 @@ async def load_nav_snapshot(
exclude_sections=excluded_secs,
job_id_by_result_id=job_id_by_result_id,
)
+ parse_seconds = 0.0
if manifest_sections is None:
_logger.warning(
"retrieval snapshot fallback=table_scan user_id=%s namespace=%s documents=%d",
@@ -324,21 +370,47 @@ async def load_nav_snapshot(
for chunk_id in chunk_ids
},
)
+ ref_index_started = time.perf_counter()
+ lazy_ref_index = LazyChunkRefIndex(
+ chunk_ref_index,
+ resolver=store.load_chunk_reference_metadata,
+ )
+ ref_index_seconds = time.perf_counter() - ref_index_started
except Exception:
store.close()
raise
- return NavSnapshot(
+ snapshot = NavSnapshot(
provider=provider,
- chunk_ref_index=LazyChunkRefIndex(
- chunk_ref_index,
- resolver=store.load_chunk_reference_metadata,
- ),
+ chunk_ref_index=lazy_ref_index,
document_ids=list(provider.document_ids()),
document_titles={
did: kept_titles.get(did, did) for did in provider.document_ids()
},
document_revisions=dict(revisions),
)
+ if _is_snapshot_timing_enabled():
+ section_rows, section_paths, root_sections, leaf_sections = (
+ _count_section_shape(sections_by_doc)
+ )
+ _logger.info(
+ "retrieval snapshot timing total_seconds=%.3f parse_seconds=%.3f "
+ "ref_index_copy_seconds=%.3f doc_query_seconds=%.3f "
+ "job_query_seconds=%.3f documents=%d section_rows=%d "
+ "section_paths=%d root_sections=%d leaf_sections=%d refs=%d "
+ "mode=lazy",
+ time.perf_counter() - snapshot_started,
+ parse_seconds,
+ ref_index_seconds,
+ doc_query_seconds,
+ job_query_seconds,
+ len(snapshot.document_ids),
+ section_rows,
+ section_paths,
+ root_sections,
+ leaf_sections,
+ len(snapshot.chunk_ref_index),
+ )
+ return snapshot
units_by_doc, chunk_ref_index = await _load_chunks(
db,
@@ -358,7 +430,7 @@ async def load_nav_snapshot(
f"user_id={user_id!r} namespace={namespace!r}"
)
- return build_nav_snapshot(
+ snapshot = build_nav_snapshot(
document_titles=kept_titles,
sections_by_doc={did: sections_by_doc.get(did, []) for did in kept_titles},
units_by_doc={did: units_by_doc.get(did, []) for did in kept_titles},
@@ -369,6 +441,27 @@ async def load_nav_snapshot(
if document_id in kept_titles
},
)
+ if _is_snapshot_timing_enabled():
+ section_rows, section_paths, root_sections, leaf_sections = (
+ _count_section_shape(sections_by_doc)
+ )
+ _logger.info(
+ "retrieval snapshot timing total_seconds=%.3f parse_seconds=%.3f "
+ "doc_query_seconds=%.3f job_query_seconds=%.3f documents=%d "
+ "section_rows=%d section_paths=%d root_sections=%d "
+ "leaf_sections=%d refs=%d mode=eager",
+ time.perf_counter() - snapshot_started,
+ parse_seconds,
+ doc_query_seconds,
+ job_query_seconds,
+ len(snapshot.document_ids),
+ section_rows,
+ section_paths,
+ root_sections,
+ leaf_sections,
+ len(snapshot.chunk_ref_index),
+ )
+ return snapshot
async def _resolve_namespace_snapshot_entries(
@@ -377,21 +470,42 @@ async def _resolve_namespace_snapshot_entries(
user_id: str,
namespace: str,
document_revisions: list[tuple[str, str]],
+ expected_generation: int | None = None,
) -> list[tuple[object, ...]] | None:
"""Return manifest-shaped entries from the persisted namespace snapshot.
Returns ``None`` (triggering the exact per-revision fallback) when the
snapshot row is missing, corrupt, or stale for any requested revision.
"""
+ generation_statement = select(RetrievalNamespaceGeneration.generation).where(
+ RetrievalNamespaceGeneration.user_id == user_id,
+ RetrievalNamespaceGeneration.namespace == namespace,
+ )
+ try:
+ generation_result = await db.execute(generation_statement)
+ _current_generation = generation_result.scalar_one_or_none()
+ except SQLAlchemyError as exc:
+ await db.rollback()
+ _logger.warning(
+ "retrieval snapshot generation lookup failed; using fallback error=%s",
+ exc,
+ )
+ return None
+ if _is_snapshot_timing_enabled():
+ _logger.info(
+ "retrieval snapshot generation generation=%s",
+ _current_generation,
+ )
+ generation_value = int(expected_generation) if expected_generation is not None else None
statement = select(
RetrievalNamespaceMapSnapshot.generation,
- RetrievalNamespaceMapSnapshot.payload_zlib,
RetrievalNamespaceMapSnapshot.checksum,
RetrievalNamespaceMapSnapshot.format_version,
).where(
RetrievalNamespaceMapSnapshot.user_id == user_id,
RetrievalNamespaceMapSnapshot.namespace == namespace,
)
+ lookup_started = time.perf_counter()
try:
row = (await db.execute(statement)).first()
except SQLAlchemyError as exc:
@@ -403,30 +517,136 @@ async def _resolve_namespace_snapshot_entries(
exc,
)
return None
+ lookup_seconds = time.perf_counter() - lookup_started
+ if _is_snapshot_timing_enabled():
+ _logger.info(
+ "retrieval snapshot lookup seconds=%.3f found=%s",
+ lookup_seconds,
+ row is not None,
+ )
if row is None:
return None
- generation, payload_zlib, checksum, format_version = row
- documents = get_cached_namespace_documents(
- user_id=user_id, namespace=namespace, generation=int(generation)
- )
- if documents is None:
- try:
- payload = decode_serving_manifest(
- bytes(payload_zlib),
- checksum=str(checksum),
- format_version=int(format_version),
+ generation, checksum, format_version = row
+ row_generation = int(generation)
+ if generation_value is not None and row_generation != generation_value:
+ _logger.info(
+ "retrieval snapshot generation mismatch row=%d expected=%d",
+ row_generation,
+ generation_value,
+ )
+ return None
+ if (
+ generation_value is None
+ and _current_generation is not None
+ and int(_current_generation) > 0
+ and row_generation != int(_current_generation)
+ ):
+ _logger.info(
+ "retrieval snapshot generation mismatch row=%d current=%d",
+ row_generation,
+ int(_current_generation),
+ )
+ return None
+ cached_blob: bytes | None = None
+ try:
+ cached_blob = await NamespaceMapSnapshotRedisCache.get(
+ user_id=user_id, namespace=namespace, generation=row_generation
+ )
+ except RedisOperationError as exc:
+ _logger.warning("retrieval snapshot redis get failed error=%s", exc)
+ database_blob: bytes | None = None
+ if cached_blob is None:
+ payload_result = await db.execute(
+ select(RetrievalNamespaceMapSnapshot.payload_zlib).where(
+ RetrievalNamespaceMapSnapshot.user_id == user_id,
+ RetrievalNamespaceMapSnapshot.namespace == namespace,
)
- except (ValueError, TypeError):
+ )
+ payload_row = payload_result.first()
+ if payload_row is None or payload_row[0] is None:
return None
- decoded_documents = payload.get("documents")
- if not isinstance(decoded_documents, dict):
+ database_blob = bytes(payload_row[0])
+ blob: bytes = database_blob
+ else:
+ blob = cached_blob
+ is_timing_enabled = _is_snapshot_timing_enabled()
+ decode_timings: dict[str, float] | None = {} if is_timing_enabled else None
+ decode_started = time.perf_counter()
+ try:
+ payload = decode_namespace_map_snapshot(
+ blob,
+ checksum=str(checksum),
+ format_version=int(format_version),
+ timings=decode_timings,
+ )
+ except (ValueError, TypeError, zlib.error):
+ if cached_blob is not None:
+ # A stale/corrupt cache entry must never shadow the PostgreSQL source.
+ try:
+ if database_blob is None:
+ fallback_result = await db.execute(
+ select(RetrievalNamespaceMapSnapshot.payload_zlib).where(
+ RetrievalNamespaceMapSnapshot.user_id == user_id,
+ RetrievalNamespaceMapSnapshot.namespace == namespace,
+ )
+ )
+ fallback_row = fallback_result.first()
+ if fallback_row is None or fallback_row[0] is None:
+ return None
+ database_blob = bytes(fallback_row[0])
+ assert database_blob is not None
+ payload = decode_namespace_map_snapshot(
+ database_blob,
+ checksum=str(checksum),
+ format_version=int(format_version),
+ timings=decode_timings,
+ )
+ blob = database_blob
+ cached_blob = None
+ except (ValueError, TypeError, zlib.error):
+ return None
+ else:
return None
- documents = decoded_documents
- cache_namespace_documents(
- user_id=user_id,
- namespace=namespace,
- generation=int(generation),
- documents=documents,
+ decoded_documents = payload.get("documents")
+ if not isinstance(decoded_documents, dict):
+ return None
+ documents = decoded_documents
+ if cached_blob is None:
+ try:
+ await NamespaceMapSnapshotRedisCache.set(
+ user_id=user_id,
+ namespace=namespace,
+ generation=row_generation,
+ payload_zlib=blob,
+ )
+ except RedisOperationError as exc:
+ _logger.warning("retrieval snapshot redis set failed error=%s", exc)
+ if is_timing_enabled:
+ _logger.info(
+ "retrieval snapshot decode cache_hit=false seconds=%.3f "
+ "compressed_bytes=%d decompressed_bytes=%d "
+ "decompress_seconds=%.3f checksum_seconds=%.3f "
+ "json_decode_seconds=%.3f documents=%d",
+ time.perf_counter() - decode_started,
+ int((decode_timings or {}).get("compressed_bytes", 0.0)),
+ int((decode_timings or {}).get("decompressed_bytes", 0.0)),
+ (decode_timings or {}).get("decompress_seconds", 0.0),
+ (decode_timings or {}).get("checksum_seconds", 0.0),
+ (decode_timings or {}).get("json_decode_seconds", 0.0),
+ len(documents),
+ )
+ elif is_timing_enabled:
+ _logger.info(
+ "retrieval snapshot decode cache_hit=true seconds=%.3f "
+ "compressed_bytes=%d decompressed_bytes=%d decompress_seconds=%.3f "
+ "checksum_seconds=%.3f json_decode_seconds=%.3f documents=%d",
+ time.perf_counter() - decode_started,
+ int((decode_timings or {}).get("compressed_bytes", 0.0)),
+ int((decode_timings or {}).get("decompressed_bytes", 0.0)),
+ (decode_timings or {}).get("decompress_seconds", 0.0),
+ (decode_timings or {}).get("checksum_seconds", 0.0),
+ (decode_timings or {}).get("json_decode_seconds", 0.0),
+ len(documents),
)
entries: list[tuple[object, ...]] = []
for document_id, job_result_id in document_revisions:
@@ -532,6 +752,11 @@ def _parse_manifest_entries(
ref_index: dict[str, dict[str, Any]] = {}
root_assets_by_doc: dict[str, set[str]] = {}
text_connections_by_doc: dict[str, list[tuple[str, str]]] = {}
+ section_seconds = 0.0
+ chunk_seconds = 0.0
+ section_count = 0
+ chunk_count = 0
+ connection_count = 0
try:
for document_id, _job_result_id, payload_zlib, checksum, format_version in manifest_entries:
if isinstance(payload_zlib, dict):
@@ -546,6 +771,7 @@ def _parse_manifest_entries(
checksum=str(checksum),
format_version=int(str(format_version)),
)
+ section_started = time.perf_counter()
raw_sections = payload.get("sections")
if not isinstance(raw_sections, list):
return None
@@ -577,6 +803,9 @@ def _parse_manifest_entries(
)
by_doc.setdefault(str(document_id), []).append(section)
path_by_id[section_id] = section_path
+ section_count += 1
+ section_seconds += time.perf_counter() - section_started
+ chunk_started = time.perf_counter()
raw_chunks = payload.get("chunks")
if not isinstance(raw_chunks, list):
return None
@@ -628,15 +857,32 @@ def _parse_manifest_entries(
text_connections_by_doc.setdefault(document_key, []).append(
(section_id or "", target)
)
+ connection_count += 1
+ chunk_count += 1
+ chunk_seconds += time.perf_counter() - chunk_started
except (TypeError, ValueError, KeyError):
return None
remounted: dict[str, dict[str, Any]] = {}
+ remount_started = time.perf_counter()
for document_id, asset_ids in root_assets_by_doc.items():
owners: dict[str, list[str]] = {}
for section_id, target in text_connections_by_doc.get(document_id, ()):
if target in asset_ids:
owners.setdefault(section_id, []).append(target)
remounted[document_id] = {"root": sorted(asset_ids), "owners": owners}
+ if _is_snapshot_timing_enabled():
+ _logger.info(
+ "retrieval snapshot parse sections=%d chunks=%d connections=%d "
+ "section_seconds=%.3f chunk_seconds=%.3f remount_seconds=%.3f "
+ "ref_index_keys=%d",
+ section_count,
+ chunk_count,
+ connection_count,
+ section_seconds,
+ chunk_seconds,
+ time.perf_counter() - remount_started,
+ len(ref_index),
+ )
return by_doc, path_by_id, (ids_by_doc, ref_index, remounted)
diff --git a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py
index 95436458b..68e274022 100644
--- a/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py
+++ b/packages/shared-python/shared/services/retrieval/search/map_unit_discovery.py
@@ -20,7 +20,7 @@
from collections.abc import Mapping
from dataclasses import dataclass, field
from hashlib import sha256
-from typing import Any
+from typing import Any, cast
from loguru import logger
from sqlalchemy import text
@@ -32,6 +32,7 @@
normalize_chunk_type,
)
from shared.services.retrieval.nav.knowhere_hybrid import (
+ MAP_UNIT_INDEX_FORMAT_VERSION,
PersistedScoreCorpus,
PersistedScoreUnit,
score_persisted_corpus_many,
@@ -192,6 +193,8 @@ async def map_unit_discovery(
query_tokens = tokenize_query_for_ranker(query)
if not query_tokens:
return DiscoveryResult(status="discovery_done", payload={"fused_rows": []})
+ if revision_pins is not None and not revision_pins:
+ return DiscoveryResult(status="discovery_done", payload={"fused_rows": []})
query_token_hashes = [
sha256(token.encode("utf-8")).hexdigest() for token in query_tokens
]
@@ -210,6 +213,15 @@ async def map_unit_discovery(
params.update(type_params)
params.update(signal_params)
+ is_unfiltered_scope: bool = not any(
+ (
+ chunk_types,
+ signal_paths,
+ exclude_sections,
+ exclude_document_ids,
+ )
+ )
+
cte = _SCOPED_UNITS_CTE.format(
revision_join=revision_join,
revision_clause=revision_clause,
@@ -217,7 +229,25 @@ async def map_unit_discovery(
type_clause=type_clause,
signal_clause=signal_clause,
)
- unit_result = await db.execute(text(cte + "SELECT * FROM scoped_units"), params)
+ unit_statement = cte + "SELECT * FROM scoped_units"
+ if is_unfiltered_scope:
+ unit_statement = (
+ "WITH matching_tokens AS MATERIALIZED ("
+ "SELECT DISTINCT map_unit_id FROM document_map_unit_tokens "
+ "WHERE channel = ANY(:channels) AND token_hash = ANY(:token_hashes)"
+ "), "
+ + cte.lstrip().removeprefix("WITH ")
+ + " SELECT DISTINCT scoped_units.* "
+ "FROM matching_tokens JOIN scoped_units "
+ "ON scoped_units.map_unit_id = matching_tokens.map_unit_id"
+ )
+ params = {
+ **params,
+ "channels": list(_MAP_SCORE_CHANNELS),
+ "token_hashes": query_token_hashes,
+ }
+ stage_started = time.monotonic()
+ unit_result = await db.execute(text(unit_statement), params)
unit_rows = [dict(row._mapping) for row in unit_result.all()]
unit_rows = [
row
@@ -228,9 +258,11 @@ async def map_unit_discovery(
exclude_sections=exclude_sections,
)
]
-
- if not unit_rows:
- return DiscoveryResult(status="discovery_done", payload={"fused_rows": []})
+ logger.info(
+ "retrieval map-unit stage=units seconds={:.3f} rows={}",
+ time.monotonic() - stage_started,
+ len(unit_rows),
+ )
frequency_scope_cte = cte if signal_paths else _SCOPED_UNIT_IDS_CTE.format(
revision_join=revision_join,
@@ -254,6 +286,7 @@ async def map_unit_discovery(
ON scoped_units.map_unit_id = matching_tokens.map_unit_id
"""
)
+ stage_started = time.monotonic()
frequency_result = await db.execute(
frequency_query,
{
@@ -267,9 +300,37 @@ async def map_unit_discovery(
frequencies.setdefault((str(map_unit_id), str(channel)), {})[str(token)] = (
int(frequency)
)
+ logger.info(
+ "retrieval map-unit stage=frequencies seconds={:.3f} rows={} units={}",
+ time.monotonic() - stage_started,
+ sum(len(values) for values in frequencies.values()),
+ len(frequencies),
+ )
- index_result = await db.execute(
- text(
+ if is_unfiltered_scope and revision_pins is not None:
+ pinned_pairs = [
+ (str(document_id).strip(), str(job_result_id).strip())
+ for document_id, job_result_id in revision_pins.items()
+ if str(document_id).strip() and str(job_result_id).strip()
+ ]
+ pinned_values_sql = ", ".join(
+ f"(:_pin_document_{index}, :_pin_revision_{index})"
+ for index, _pair in enumerate(pinned_pairs)
+ )
+ index_statement = f"""
+ SELECT indexes.average_idf_path, indexes.average_idf_content,
+ indexes.unit_count, indexes.format_version,
+ indexes.path_document_count, indexes.path_total_length,
+ indexes.content_document_count, indexes.content_total_length,
+ indexes.token_count
+ FROM document_map_unit_indexes AS indexes
+ JOIN (VALUES {pinned_values_sql})
+ AS scoped_revisions(document_id, job_result_id)
+ ON indexes.document_id = scoped_revisions.document_id
+ AND indexes.job_result_id = scoped_revisions.job_result_id
+ """
+ else:
+ index_statement = (
(
cte
if signal_paths
@@ -282,7 +343,10 @@ async def map_unit_discovery(
)
+ """
SELECT indexes.average_idf_path, indexes.average_idf_content,
- indexes.unit_count
+ indexes.unit_count, indexes.format_version,
+ indexes.path_document_count, indexes.path_total_length,
+ indexes.content_document_count, indexes.content_total_length,
+ indexes.token_count
FROM document_map_unit_indexes AS indexes
JOIN (
SELECT DISTINCT document_id, job_result_id FROM scoped_units
@@ -290,28 +354,150 @@ async def map_unit_discovery(
ON indexes.document_id = scoped_revisions.document_id
AND indexes.job_result_id = scoped_revisions.job_result_id
"""
- ),
- params,
- )
+ )
+ stage_started = time.monotonic()
+ index_result = await db.execute(text(index_statement), params)
index_parts = [
- (float(path_idf or 0.0), float(content_idf or 0.0), int(unit_count or 0))
- for path_idf, content_idf, unit_count in index_result.all()
- ]
- expected_revisions = {
- (str(row["document_id"]), str(row["job_result_id"])) for row in unit_rows
- }
- unfiltered_scope = not any(
(
- chunk_types,
- signal_paths,
- exclude_sections,
- exclude_document_ids,
+ float(path_idf or 0.0),
+ float(content_idf or 0.0),
+ int(unit_count or 0),
+ int(format_version or 0),
+ path_document_count,
+ path_total_length,
+ content_document_count,
+ content_total_length,
+ int(token_count or 0),
)
+ for (
+ path_idf,
+ content_idf,
+ unit_count,
+ format_version,
+ path_document_count,
+ path_total_length,
+ content_document_count,
+ content_total_length,
+ token_count,
+ ) in index_result.all()
+ ]
+ logger.info(
+ "retrieval map-unit stage=indexes seconds={:.3f} rows={}",
+ time.monotonic() - stage_started,
+ len(index_parts),
+ )
+ if is_unfiltered_scope:
+ if revision_pins is not None:
+ # The route captures the active revision set before discovery. Reuse
+ # that immutable set instead of scanning scoped map units a second
+ # time just to derive the same revision keys. Missing index rows
+ # still fail the existing completeness check below.
+ expected_revisions = {
+ (str(document_id), str(job_result_id))
+ for document_id, job_result_id in revision_pins.items()
+ }
+ revision_check_seconds = 0.0
+ else:
+ stage_started = time.monotonic()
+ revision_result = await db.execute(
+ text(cte + "SELECT DISTINCT document_id, job_result_id FROM scoped_units"),
+ params,
+ )
+ expected_revisions = {
+ (str(row[0]), str(row[1])) for row in revision_result.all()
+ }
+ revision_check_seconds = time.monotonic() - stage_started
+ logger.info(
+ "retrieval map-unit stage=revision-check seconds={:.3f} rows={}",
+ revision_check_seconds,
+ len(expected_revisions),
+ )
+ else:
+ expected_revisions = {
+ (str(row["document_id"]), str(row["job_result_id"]))
+ for row in unit_rows
+ }
+ indexed_unit_count = sum(
+ unit_count for _path_idf, _content_idf, unit_count, *_rest in index_parts
+ )
+ has_index_storage_mismatch: bool = False
+ if is_unfiltered_scope and not unit_rows and expected_revisions:
+ storage_counts: tuple[int | None, int | None] = cast(
+ tuple[int | None, int | None],
+ (
+ await db.execute(
+ text(
+ _SCOPED_UNIT_IDS_CTE.format(
+ revision_join=revision_join,
+ revision_clause=revision_clause,
+ exclude_clause=exclude_clause,
+ type_clause=type_clause,
+ )
+ + """
+ SELECT
+ (SELECT COUNT(*) FROM scoped_units) AS unit_count,
+ (
+ SELECT COUNT(*)
+ FROM document_map_unit_tokens AS tokens
+ JOIN scoped_units
+ ON scoped_units.map_unit_id = tokens.map_unit_id
+ ) AS token_count
+ """
+ ),
+ params,
+ )
+ ).one(),
+ )
+ actual_unit_count: int | None = storage_counts[0]
+ actual_token_count: int | None = storage_counts[1]
+ indexed_token_count: int = sum(int(part[8]) for part in index_parts)
+ has_index_storage_mismatch = indexed_unit_count != int(
+ actual_unit_count or 0
+ ) or indexed_token_count != int(actual_token_count or 0)
+ has_index_unit_count_mismatch = (
+ indexed_unit_count < len(unit_rows)
+ if is_unfiltered_scope
+ else indexed_unit_count != len(unit_rows)
+ )
+ is_index_format_incompatible = any(
+ format_version != MAP_UNIT_INDEX_FORMAT_VERSION
+ for (
+ _path_idf,
+ _content_idf,
+ _unit_count,
+ format_version,
+ _path_document_count,
+ _path_total_length,
+ _content_document_count,
+ _content_total_length,
+ _token_count,
+ ) in index_parts
+ )
+ has_incomplete_index_statistics = is_unfiltered_scope and any(
+ format_version != MAP_UNIT_INDEX_FORMAT_VERSION
+ or path_document_count is None
+ or path_total_length is None
+ or content_document_count is None
+ or content_total_length is None
+ for (
+ _path_idf,
+ _content_idf,
+ _unit_count,
+ format_version,
+ path_document_count,
+ path_total_length,
+ content_document_count,
+ content_total_length,
+ _token_count,
+ ) in index_parts
+ )
+ has_unusable_index = (
+ len(index_parts) != len(expected_revisions)
+ or has_index_unit_count_mismatch
+ or has_index_storage_mismatch
+ or is_index_format_incompatible
)
- index_unit_count_mismatch = unfiltered_scope and sum(
- unit_count for _path_idf, _content_idf, unit_count in index_parts
- ) != len(unit_rows)
- if len(index_parts) != len(expected_revisions) or index_unit_count_mismatch:
+ if has_unusable_index:
try:
await record_retrieval_index_readiness(
user_id=user_id,
@@ -343,11 +529,46 @@ async def map_unit_discovery(
filter_mode=filter_mode,
revision_pins=revision_pins,
)
+ if has_incomplete_index_statistics:
+ logger.warning(
+ "retrieval map index statistics incomplete user_id=%s namespace=%s "
+ "using row-derived BM25 denominators",
+ user_id,
+ namespace,
+ )
+ # Token-selective projection is only sufficient when persisted channel
+ # denominators are available. Before the statistics backfill, restore
+ # the old full-scope unit projection so row-derived BM25 statistics use
+ # the same corpus as the legacy map-unit reader.
+ stage_started = time.monotonic()
+ full_unit_params = {
+ key: value
+ for key, value in params.items()
+ if key not in {"channels", "token_hashes"}
+ }
+ full_unit_result = await db.execute(
+ text(cte + "SELECT * FROM scoped_units"), full_unit_params
+ )
+ unit_rows = [dict(row._mapping) for row in full_unit_result.all()]
+ unit_rows = [
+ row
+ for row in unit_rows
+ if not is_excluded_section(
+ document_id=row.get("document_id"),
+ section_path=row.get("section_path"),
+ exclude_sections=exclude_sections,
+ )
+ ]
+ logger.info(
+ "retrieval map-unit stage=full-units-for-stats seconds={:.3f} rows={}",
+ time.monotonic() - stage_started,
+ len(unit_rows),
+ )
try:
await record_retrieval_index_readiness(
user_id=user_id,
namespace=namespace,
- ready=True,
+ ready=not has_incomplete_index_statistics,
expected_revisions=len(expected_revisions),
indexed_revisions=len(index_parts),
)
@@ -356,16 +577,17 @@ async def map_unit_discovery(
average_idf_path = combine_average_idf(
[
(path_idf, unit_count)
- for path_idf, _content_idf, unit_count in index_parts
+ for path_idf, _content_idf, unit_count, *_rest in index_parts
]
)
average_idf_content = combine_average_idf(
[
(content_idf, unit_count)
- for _path_idf, content_idf, unit_count in index_parts
+ for _path_idf, content_idf, unit_count, *_rest in index_parts
]
)
+ stage_started = time.monotonic()
path_stats = build_channel_bm25_stats(
unit_rows=unit_rows,
map_unit_id_field="map_unit_id",
@@ -374,6 +596,16 @@ async def map_unit_discovery(
query_tokens=query_tokens,
frequencies=frequencies,
average_idf=average_idf_path,
+ document_count_override=(
+ sum(int(part[4] or 0) for part in index_parts)
+ if is_unfiltered_scope and not has_incomplete_index_statistics
+ else None
+ ),
+ total_length_override=(
+ sum(int(part[5] or 0) for part in index_parts)
+ if is_unfiltered_scope and not has_incomplete_index_statistics
+ else None
+ ),
)
content_stats = build_channel_bm25_stats(
unit_rows=unit_rows,
@@ -383,6 +615,21 @@ async def map_unit_discovery(
query_tokens=query_tokens,
frequencies=frequencies,
average_idf=average_idf_content,
+ document_count_override=(
+ sum(int(part[6] or 0) for part in index_parts)
+ if is_unfiltered_scope and not has_incomplete_index_statistics
+ else None
+ ),
+ total_length_override=(
+ sum(int(part[7] or 0) for part in index_parts)
+ if is_unfiltered_scope and not has_incomplete_index_statistics
+ else None
+ ),
+ )
+ logger.info(
+ "retrieval map-unit stage=stats seconds={:.3f} units={}",
+ time.monotonic() - stage_started,
+ len(unit_rows),
)
corpus = PersistedScoreCorpus(
@@ -401,7 +648,13 @@ async def map_unit_discovery(
path_stats=path_stats,
content_stats=content_stats,
)
+ stage_started = time.monotonic()
scores_by_unit = score_persisted_corpus_many(corpus, [query]).get(query, {})
+ logger.info(
+ "retrieval map-unit stage=scoring seconds={:.3f} units={}",
+ time.monotonic() - stage_started,
+ len(scores_by_unit),
+ )
rows_by_unit_id = {row["map_unit_id"]: row for row in unit_rows}
ranked_unit_ids = sorted(
@@ -410,6 +663,7 @@ async def map_unit_discovery(
reverse=True,
)[:top_k]
+ stage_started = time.monotonic()
fused_rows = await _hydrate_winning_units(
db,
ranked_unit_ids=ranked_unit_ids,
@@ -420,6 +674,11 @@ async def map_unit_discovery(
exclude_sections=exclude_sections,
revision_pins=revision_pins,
)
+ logger.info(
+ "retrieval map-unit stage=hydration seconds={:.3f} rows={}",
+ time.monotonic() - stage_started,
+ len(fused_rows),
+ )
if fused_rows:
normalize_row_scores(
fused_rows,
diff --git a/packages/shared-python/shared/services/retrieval/serving_manifest.py b/packages/shared-python/shared/services/retrieval/serving_manifest.py
index c66ae5798..77a33e1cd 100644
--- a/packages/shared-python/shared/services/retrieval/serving_manifest.py
+++ b/packages/shared-python/shared/services/retrieval/serving_manifest.py
@@ -4,8 +4,9 @@
import hashlib
import json
+import time
import zlib
-from typing import Any
+from typing import Any, MutableMapping
from sqlalchemy import delete, select
from sqlalchemy.orm import Session
@@ -20,6 +21,7 @@
from shared.services.retrieval.publication_models import DocumentPublicationScope
SERVING_MANIFEST_FORMAT_VERSION = 1
+NAMESPACE_MAP_SNAPSHOT_FORMAT_VERSION = 2
def build_revision_serving_payload(
@@ -181,24 +183,143 @@ def decode_serving_manifest(
*,
checksum: str,
format_version: int,
+ timings: MutableMapping[str, float] | None = None,
) -> dict[str, Any]:
"""Validate and decode one persisted serving manifest."""
if format_version != SERVING_MANIFEST_FORMAT_VERSION:
raise ValueError(f"unsupported serving manifest version: {format_version}")
+ return _decode_compressed_json(
+ payload_zlib,
+ checksum=checksum,
+ timings=timings,
+ compression_error="invalid serving manifest compression",
+ checksum_error="serving manifest checksum mismatch",
+ json_error="invalid serving manifest JSON",
+ object_error="serving manifest payload must be an object",
+ )
+
+
+def _decode_compressed_json(
+ payload_zlib: bytes,
+ *,
+ checksum: str,
+ timings: MutableMapping[str, float] | None,
+ compression_error: str,
+ checksum_error: str,
+ json_error: str,
+ object_error: str,
+) -> dict[str, Any]:
+ """Decompress, validate, and decode a canonical JSON payload."""
+
+ started = time.perf_counter()
try:
canonical_payload = zlib.decompress(payload_zlib)
except zlib.error as exc:
- raise ValueError("invalid serving manifest compression") from exc
+ raise ValueError(compression_error) from exc
+ if timings is not None:
+ timings["decompress_seconds"] = time.perf_counter() - started
+ checksum_started = time.perf_counter()
actual_checksum = hashlib.sha256(canonical_payload).hexdigest()
if actual_checksum != checksum:
- raise ValueError("serving manifest checksum mismatch")
+ raise ValueError(checksum_error)
+ if timings is not None:
+ timings["checksum_seconds"] = time.perf_counter() - checksum_started
+ json_started = time.perf_counter()
try:
decoded = json.loads(canonical_payload.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
- raise ValueError("invalid serving manifest JSON") from exc
+ raise ValueError(json_error) from exc
+ if timings is not None:
+ timings["json_decode_seconds"] = time.perf_counter() - json_started
+ timings["compressed_bytes"] = float(len(payload_zlib))
+ timings["decompressed_bytes"] = float(len(canonical_payload))
+ timings["decode_seconds"] = time.perf_counter() - started
if not isinstance(decoded, dict):
- raise ValueError("serving manifest payload must be an object")
+ raise ValueError(object_error)
return decoded
+
+
+def encode_namespace_map_snapshot(
+ payload: dict[str, Any],
+) -> tuple[bytes, str, int]:
+ """Encode the routing-only namespace snapshot using its own format version."""
+ documents = payload.get("documents")
+ if not isinstance(documents, dict):
+ raise ValueError("namespace snapshot documents must be an object")
+ routing_documents: dict[str, dict[str, object]] = {}
+ for document_id, raw_document in documents.items():
+ if not isinstance(raw_document, dict):
+ raise ValueError(f"namespace snapshot document is not an object: {document_id}")
+ raw_sections = raw_document.get("sections")
+ raw_chunks = raw_document.get("chunks")
+ if not isinstance(raw_sections, list) or not isinstance(raw_chunks, list):
+ raise ValueError(f"namespace snapshot records are invalid: {document_id}")
+ sections: list[dict[str, object]] = []
+ for section in raw_sections:
+ if not isinstance(section, dict) or not str(section.get("section_id") or ""):
+ raise ValueError(f"namespace snapshot section is invalid: {document_id}")
+ sections.append(
+ {
+ key: section[key]
+ for key in (
+ "section_id", "parent_section_id", "section_path",
+ "section_title", "section_level", "summary", "sort_order",
+ )
+ if key in section
+ }
+ )
+ chunks: list[dict[str, object]] = []
+ for chunk in raw_chunks:
+ if not isinstance(chunk, dict) or not str(chunk.get("chunk_id") or ""):
+ raise ValueError(f"namespace snapshot chunk is invalid: {document_id}")
+ chunks.append(
+ {
+ key: chunk[key]
+ for key in ("chunk_id", "section_id", "chunk_type", "sort_order", "connect_to")
+ if key in chunk
+ }
+ )
+ routing_documents[str(document_id)] = {
+ "job_result_id": raw_document.get("job_result_id"),
+ "job_id": raw_document.get("job_id"),
+ "sections": sections,
+ "chunks": chunks,
+ "root_asset_ids": raw_document.get("root_asset_ids") or [],
+ "remounted_assets_by_section": raw_document.get(
+ "remounted_assets_by_section"
+ )
+ or {},
+ }
+ compressed, checksum, _ = encode_serving_manifest({"documents": routing_documents})
+ return compressed, checksum, NAMESPACE_MAP_SNAPSHOT_FORMAT_VERSION
+
+
+def decode_namespace_map_snapshot(
+ payload_zlib: bytes,
+ *,
+ checksum: str,
+ format_version: int,
+ timings: MutableMapping[str, float] | None = None,
+) -> dict[str, Any]:
+ """Decode namespace snapshots, retaining compatibility with legacy v1 rows."""
+ if format_version == SERVING_MANIFEST_FORMAT_VERSION:
+ return decode_serving_manifest(
+ payload_zlib,
+ checksum=checksum,
+ format_version=SERVING_MANIFEST_FORMAT_VERSION,
+ timings=timings,
+ )
+ if format_version != NAMESPACE_MAP_SNAPSHOT_FORMAT_VERSION:
+ raise ValueError(f"unsupported namespace snapshot version: {format_version}")
+ return _decode_compressed_json(
+ payload_zlib,
+ checksum=checksum,
+ timings=timings,
+ compression_error="invalid namespace snapshot compression",
+ checksum_error="namespace snapshot checksum mismatch",
+ json_error="invalid namespace snapshot JSON",
+ object_error="namespace snapshot payload must be an object",
+ )
diff --git a/packages/shared-python/shared/testing/contract_runtime.py b/packages/shared-python/shared/testing/contract_runtime.py
index 27a3c7de1..f5fbbd5f4 100644
--- a/packages/shared-python/shared/testing/contract_runtime.py
+++ b/packages/shared-python/shared/testing/contract_runtime.py
@@ -350,6 +350,13 @@ def configure_contract_environment(
"QSTASH_CURRENT_SIGNING_KEY": "qstash-current-test-key",
"QSTASH_NEXT_SIGNING_KEY": "qstash-next-test-key",
"QSTASH_CALLBACK_BASE_URL": "http://localhost:5005/api/v1",
+ # Pin product defaults so a developer's local apps/api/.env cannot
+ # change contract expectations (credits seed, upload allow-list).
+ "FREE_PLAN_INITIAL_CREDITS": "5",
+ "SUPPORTED_EXTENSIONS": (
+ ".doc,.docx,.pdf,.txt,.xls,.xlsx,.csv,.pptx,"
+ ".jpg,.jpeg,.png,.md,.html,.htm"
+ ),
}
if "BILLING_ENABLED" not in os.environ:
diff --git a/packages/shared-python/shared/tests/test_asset_inline.py b/packages/shared-python/shared/tests/test_asset_inline.py
new file mode 100644
index 000000000..a59652f09
--- /dev/null
+++ b/packages/shared-python/shared/tests/test_asset_inline.py
@@ -0,0 +1,154 @@
+"""Unit tests for placeholder-based asset inlining."""
+
+from __future__ import annotations
+
+import pytest
+
+from shared.services.retrieval.hydration.asset_inline import (
+ inline_assets_at_placeholders,
+)
+from shared.services.retrieval.hydration.result_assembly import (
+ assemble_retrieval_results,
+)
+from shared.services.retrieval.nav.nav_hierarchy import ProviderToolSpace
+from shared.services.retrieval.nav.nav_knowhere import (
+ KnowhereProvider,
+ SectionRow,
+ UnitRow,
+)
+
+
+def test_inline_replaces_placeholder_with_newlines() -> None:
+ body, embedded = inline_assets_at_placeholders(
+ "see [images/a.png] here",
+ connections=[
+ {
+ "target": "img-1",
+ "relation": "embeds",
+ "ref": "[images/a.png]",
+ }
+ ],
+ display_by_target={"img-1": "[Image: images/a.png]\nsummary"},
+ )
+ assert body == "see \n[Image: images/a.png]\nsummary\n here"
+ assert embedded == {"img-1"}
+ assert "[images/" not in body
+
+
+def test_inline_appends_when_placeholder_missing() -> None:
+ body, embedded = inline_assets_at_placeholders(
+ "plain text",
+ connections=[{"target": "img-1", "ref": "[images/a.png]"}],
+ display_by_target={"img-1": "[Image: images/a.png]"},
+ )
+ assert body == "plain text\n\n[Image: images/a.png]"
+ assert embedded == {"img-1"}
+
+
+def test_inline_does_not_duplicate_target() -> None:
+ body, embedded = inline_assets_at_placeholders(
+ "x [images/a.png] y",
+ connections=[
+ {"target": "img-1", "ref": "[images/a.png]"},
+ {"target": "img-1", "ref": "[images/a.png]"},
+ ],
+ display_by_target={"img-1": "[Image: images/a.png]"},
+ )
+ assert body.count("[Image: images/a.png]") == 1
+ assert embedded == {"img-1"}
+
+
+@pytest.mark.asyncio
+async def test_assemble_inserts_table_at_placeholder() -> None:
+ rows = [
+ {
+ "chunk_id": "text-1",
+ "chunk_type": "text",
+ "content": "见表 [tables/table-1.html] 结束",
+ "chunk_metadata": {
+ "connect_to": [
+ {
+ "target": "table-1",
+ "relation": "embeds",
+ "ref": "[tables/table-1.html]",
+ }
+ ]
+ },
+ },
+ {
+ "chunk_id": "table-1",
+ "chunk_type": "table",
+ "content": "",
+ "file_path": "tables/table-1.html",
+ "asset_url": "https://assets.example.com/job-1/tables/table-1.html",
+ "chunk_metadata": {
+ "summary": "企业入驻信息登记模板",
+ "keywords": ["企业名称"],
+ },
+ },
+ ]
+ assembled = await assemble_retrieval_results(
+ rows=rows,
+ exclude_document_ids=[],
+ exclude_sections=[],
+ )
+ assert len(assembled) == 1
+ content = assembled[0]["content"]
+ assert "[tables/" not in content
+ assert content.index("见表") < content.index("[Table:")
+ assert content.index("[Table:") < content.index("结束")
+ assert "企业入驻信息登记模板" in content
+ assert "SHOULD NOT LEAK" not in content
+
+
+def test_node_unit_span_inlines_section_assets() -> None:
+ provider = KnowhereProvider(
+ doc_id="doc-1",
+ sections=[
+ SectionRow(
+ section_id="sec-1",
+ parent_section_id=None,
+ section_path="One",
+ section_title="One",
+ section_level=1,
+ summary="",
+ sort_order=0,
+ )
+ ],
+ units=[
+ UnitRow(
+ chunk_id="text-1",
+ section_id="sec-1",
+ chunk_type="text",
+ content="see [images/a.png] end",
+ sort_order=0,
+ metadata={
+ "connect_to": [
+ {
+ "target": "img-1",
+ "relation": "embeds",
+ "ref": "[images/a.png]",
+ }
+ ]
+ },
+ ),
+ UnitRow(
+ chunk_id="img-1",
+ section_id="sec-1",
+ chunk_type="image",
+ content="images/a.png",
+ sort_order=1,
+ file_path="images/a.png",
+ metadata={"summary": "chart summary"},
+ ),
+ ],
+ )
+ ts = ProviderToolSpace(provider)
+ text, _order, count = ts._node_unit_span("sec-1")
+ assert count == 2
+ assert "[images/" not in text
+ assert text.index("see") < text.index("[Image:")
+ assert text.index("[Image:") < text.index("end")
+ assert "chart summary" in text
+ # Asset must not also appear as a trailing standalone copy.
+ assert text.count("[Image:") == 1
diff --git a/packages/shared-python/shared/tests/test_knowhere_hybrid_tokenize.py b/packages/shared-python/shared/tests/test_knowhere_hybrid_tokenize.py
new file mode 100644
index 000000000..2d4b96e1d
--- /dev/null
+++ b/packages/shared-python/shared/tests/test_knowhere_hybrid_tokenize.py
@@ -0,0 +1,47 @@
+"""Word-level map-unit tokenization (replaces character-level CJK regex)."""
+
+from __future__ import annotations
+
+from shared.services.retrieval.nav.knowhere_hybrid import (
+ MAP_UNIT_INDEX_FORMAT_VERSION,
+ build_content_search_text,
+ build_path_search_text,
+ tokenize_for_retrieval,
+ tokenize_query_for_ranker,
+)
+from shared.utils.text_utils import tokenize_for_retrieval as tokenize_publication
+
+
+def test_map_unit_index_format_is_word_level_v2() -> None:
+ assert MAP_UNIT_INDEX_FORMAT_VERSION == 2
+
+
+def test_query_uses_chinese_word_tokens_not_characters() -> None:
+ tokens = tokenize_query_for_ranker("冠心病的诊断标准")
+ assert "冠心病" in tokens
+ assert "诊断" in tokens
+ assert "标准" in tokens
+ assert "冠" not in tokens
+ assert "诊" not in tokens
+
+
+def test_index_text_matches_publication_tokenizer() -> None:
+ text = "冠心病患者的诊断标准与心肌炎鉴别"
+ map_tokens = tokenize_for_retrieval(text, dedupe=False)
+ pub_tokens = tokenize_publication(
+ text, stopwords=[], dedupe=False, min_token_length=2
+ )
+ assert map_tokens == pub_tokens
+ assert "冠心病" in map_tokens
+ assert "心肌炎" in map_tokens
+
+
+def test_build_search_text_joins_word_tokens() -> None:
+ content = build_content_search_text("冠心病诊断标准")
+ path = build_path_search_text(section_path="指南 / 冠心病 / 诊断")
+ assert "冠心病" in content.split()
+ assert "诊断" in content.split()
+ assert "冠心病" in path.split()
+ # Single CJK characters must not appear as standalone tokens.
+ assert "冠" not in content.split()
+ assert "冠" not in path.split()
diff --git a/packages/shared-python/shared/tests/test_nav_bridge_config.py b/packages/shared-python/shared/tests/test_nav_bridge_config.py
index bfd42c33e..076b66d3c 100644
--- a/packages/shared-python/shared/tests/test_nav_bridge_config.py
+++ b/packages/shared-python/shared/tests/test_nav_bridge_config.py
@@ -41,6 +41,7 @@ def test_build_nav_config_is_checklist_map_trim_stack() -> None:
assert cfg.planner_thinking == "enabled"
assert cfg.planner_think_max_tokens == 16_384
assert cfg.token_limit == 100_000
+ assert cfg.enable_node_filter is True
assert not hasattr(cfg, "llm_model_env")