perf(retrieval): add content trigram index

apps/api/alembic/versions/0a1b2c3d4e5f_add_content_trigram_index.py

@@ -0,0 +1,46 @@
+"""Add trigram acceleration for regex searches over published chunk content."""
+
+from __future__ import annotations
+
+from typing import Sequence
+
+from alembic import op
+
+
+revision: str = "0a1b2c3d4e5f"
+down_revision: str | None = "9f0a1b2c3d4e"
+branch_labels: Sequence[str] | None = None
+depends_on: Sequence[str] | None = None
+
+_INDEX_NAME = "idx_document_chunks_content_trgm"
+
+
+def upgrade() -> None:
+    op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
+    external_transaction = bool(
+        op.get_context().opts.get("knowhere_external_transaction", False)
+    )
+    if external_transaction:
+        op.execute(
+            f"CREATE INDEX IF NOT EXISTS {_INDEX_NAME} "
+            "ON document_chunks USING gin (content gin_trgm_ops) "
+            "WHERE content IS NOT NULL"
+        )
+        return
+    with op.get_context().autocommit_block():
+        op.execute(
+            f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {_INDEX_NAME} "
+            "ON document_chunks USING gin (content gin_trgm_ops) "
+            "WHERE content IS NOT NULL"
+        )
+
+
+def downgrade() -> None:
+    external_transaction = bool(
+        op.get_context().opts.get("knowhere_external_transaction", False)
+    )
+    if external_transaction:
+        op.execute(f"DROP INDEX IF EXISTS {_INDEX_NAME}")
+        return
+    with op.get_context().autocommit_block():
+        op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {_INDEX_NAME}")

apps/api/alembic/versions/d3e4f5a6b7c8_merge_retrieval_index_heads.py

@@ -0,0 +1,21 @@
+"""Merge the retrieval index migration branches."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+revision: str = "d3e4f5a6b7c8"
+down_revision: tuple[str, str] = (
+    "0a1b2c3d4e5f",
+    "c2d3e4f5a6b7",
+)
+branch_labels: Sequence[str] | None = None
+depends_on: Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Merge migration heads without applying additional schema changes."""
+
+
+def downgrade() -> None:
+    """Split the migration graph back into its two parent heads."""

apps/api/tests/migrations/test_schema_contract.py

@@ -260,6 +260,28 @@ def test_should_index_document_chunks_in_lazy_section_order(
     )
 
 
+def test_should_create_content_trigram_index_for_regex_search(
+    migrated_head_engine: Engine,
+) -> None:
+    with migrated_head_engine.begin() as connection:
+        index_definition = connection.execute(
+            text(
+                """
+                SELECT pg_get_indexdef(indexes.indexrelid)
+                FROM pg_index AS indexes
+                JOIN pg_class AS classes ON classes.oid = indexes.indexrelid
+                JOIN pg_namespace AS namespaces ON namespaces.oid = classes.relnamespace
+                WHERE namespaces.nspname = current_schema()
+                  AND classes.relname = 'idx_document_chunks_content_trgm'
+                """
+            )
+        ).scalar_one()
+
+    definition = str(index_definition)
+    assert "USING gin (content gin_trgm_ops)" in definition
+    assert "WHERE (content IS NOT NULL)" in definition
+
+
 def test_should_create_token_leading_map_unit_covering_index(
     migrated_head_engine: Engine,
 ) -> None: