diff --git a/apps/api/alembic/versions/0a1b2c3d4e5f_add_content_trigram_index.py b/apps/api/alembic/versions/0a1b2c3d4e5f_add_content_trigram_index.py new file mode 100644 index 00000000..9fd30d7d --- /dev/null +++ b/apps/api/alembic/versions/0a1b2c3d4e5f_add_content_trigram_index.py @@ -0,0 +1,46 @@ +"""Add trigram acceleration for regex searches over published chunk content.""" + +from __future__ import annotations + +from typing import Sequence + +from alembic import op + + +revision: str = "0a1b2c3d4e5f" +down_revision: str | None = "9f0a1b2c3d4e" +branch_labels: Sequence[str] | None = None +depends_on: Sequence[str] | None = None + +_INDEX_NAME = "idx_document_chunks_content_trgm" + + +def upgrade() -> None: + op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm") + external_transaction = bool( + op.get_context().opts.get("knowhere_external_transaction", False) + ) + if external_transaction: + op.execute( + f"CREATE INDEX IF NOT EXISTS {_INDEX_NAME} " + "ON document_chunks USING gin (content gin_trgm_ops) " + "WHERE content IS NOT NULL" + ) + return + with op.get_context().autocommit_block(): + op.execute( + f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {_INDEX_NAME} " + "ON document_chunks USING gin (content gin_trgm_ops) " + "WHERE content IS NOT NULL" + ) + + +def downgrade() -> None: + external_transaction = bool( + op.get_context().opts.get("knowhere_external_transaction", False) + ) + if external_transaction: + op.execute(f"DROP INDEX IF EXISTS {_INDEX_NAME}") + return + with op.get_context().autocommit_block(): + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {_INDEX_NAME}") diff --git a/apps/api/alembic/versions/d3e4f5a6b7c8_merge_retrieval_index_heads.py b/apps/api/alembic/versions/d3e4f5a6b7c8_merge_retrieval_index_heads.py new file mode 100644 index 00000000..347a1dfb --- /dev/null +++ b/apps/api/alembic/versions/d3e4f5a6b7c8_merge_retrieval_index_heads.py @@ -0,0 +1,21 @@ +"""Merge the retrieval index migration branches.""" + +from __future__ import annotations + +from collections.abc import Sequence + +revision: str = "d3e4f5a6b7c8" +down_revision: tuple[str, str] = ( + "0a1b2c3d4e5f", + "c2d3e4f5a6b7", +) +branch_labels: Sequence[str] | None = None +depends_on: Sequence[str] | None = None + + +def upgrade() -> None: + """Merge migration heads without applying additional schema changes.""" + + +def downgrade() -> None: + """Split the migration graph back into its two parent heads.""" diff --git a/apps/api/tests/migrations/test_schema_contract.py b/apps/api/tests/migrations/test_schema_contract.py index 8b4dc85d..735ed05c 100644 --- a/apps/api/tests/migrations/test_schema_contract.py +++ b/apps/api/tests/migrations/test_schema_contract.py @@ -260,6 +260,28 @@ def test_should_index_document_chunks_in_lazy_section_order( ) +def test_should_create_content_trigram_index_for_regex_search( + migrated_head_engine: Engine, +) -> None: + with migrated_head_engine.begin() as connection: + index_definition = connection.execute( + text( + """ + SELECT pg_get_indexdef(indexes.indexrelid) + FROM pg_index AS indexes + JOIN pg_class AS classes ON classes.oid = indexes.indexrelid + JOIN pg_namespace AS namespaces ON namespaces.oid = classes.relnamespace + WHERE namespaces.nspname = current_schema() + AND classes.relname = 'idx_document_chunks_content_trgm' + """ + ) + ).scalar_one() + + definition = str(index_definition) + assert "USING gin (content gin_trgm_ops)" in definition + assert "WHERE (content IS NOT NULL)" in definition + + def test_should_create_token_leading_map_unit_covering_index( migrated_head_engine: Engine, ) -> None: