diff --git a/packages/hyperdb-demo/CHANGELOG.md b/packages/hyperdb-demo/CHANGELOG.md index 8c36b96..1f1b911 100644 --- a/packages/hyperdb-demo/CHANGELOG.md +++ b/packages/hyperdb-demo/CHANGELOG.md @@ -1,5 +1,14 @@ # @will-be-done/hyperdb-demo-app +## 0.0.11 + +### Patch Changes + +- Updated dependencies [53a6916] +- Updated dependencies [0678e07] + - @will-be-done/hyperdb@0.4.3 + - @will-be-done/hyperdb-devtool@0.4.3 + ## 0.0.10 ### Patch Changes diff --git a/packages/hyperdb-devtool/CHANGELOG.md b/packages/hyperdb-devtool/CHANGELOG.md index 98f40e0..53708aa 100644 --- a/packages/hyperdb-devtool/CHANGELOG.md +++ b/packages/hyperdb-devtool/CHANGELOG.md @@ -1,5 +1,13 @@ # @will-be-done/hyperdb-devtool +## 0.4.3 + +### Patch Changes + +- Updated dependencies [53a6916] +- Updated dependencies [0678e07] + - @will-be-done/hyperdb@0.4.3 + ## 0.4.2 ### Patch Changes diff --git a/packages/hyperdb/CHANGELOG.md b/packages/hyperdb/CHANGELOG.md index f7a06a7..0a9eac0 100644 --- a/packages/hyperdb/CHANGELOG.md +++ b/packages/hyperdb/CHANGELOG.md @@ -1,5 +1,12 @@ # @will-be-done/hyperdb +## 0.4.3 + +### Patch Changes + +- 53a6916: catch firefox idb tx finish +- 0678e07: add long redonly tx + ## 0.4.2 ### Patch Changes diff --git a/packages/hyperdb/src/hyperdb/drivers/idb/idb-driver.browser.test.ts b/packages/hyperdb/src/hyperdb/drivers/idb/idb-driver.browser.test.ts index 6a8de77..31e6129 100644 --- a/packages/hyperdb/src/hyperdb/drivers/idb/idb-driver.browser.test.ts +++ b/packages/hyperdb/src/hyperdb/drivers/idb/idb-driver.browser.test.ts @@ -11,6 +11,7 @@ import type { HyperDB } from "../../core/contracts"; import { DB } from "../../runtime/db"; import { HybridDB } from "../../runtime/hybrid-db"; import { BptreeInmemDriver } from "../inmemory/bptree-inmem-driver"; +import { getSqliteIndexSortKeyValue } from "../sqlite/sqlite-common"; import { defineTable } from "../../schema/table"; import { v } from "../../schema/values"; import { @@ -70,7 +71,7 @@ type GetAllRecordsHost = { type RawStoredRecord = { row: Record; - indexes: Record; + indexes: Record; }; function spyOnGetAllRecords(prototype: T) { @@ -159,7 +160,14 @@ describe("IdbDriver", () => { $hyperdbType: "bigint", value: "42", }); - expect(stored.indexes.byCount).toEqual(expect.any(String)); + expect(stored.indexes.byCount).toBeInstanceOf(ArrayBuffer); + expect(new Uint8Array(stored.indexes.byCount)).toEqual( + getSqliteIndexSortKeyValue(rawRowsTable, "byCount", { + id: "row-1", + count: 42n, + bytes, + }), + ); } finally { rawDb.close(); await deleteDatabase(dbName); @@ -815,6 +823,42 @@ describe("IdbDriver", () => { } }); + it("validates and deduplicates primary-key equality scans", async () => { + const db = await createDB(); + await execAsync(db.loadTables([tasksTable])); + const task = { + id: "task-1", + title: "First", + projectId: "project-1", + rank: 1, + }; + await execAsync(db.insert(tasksTable, [task])); + + await expect( + execAsync( + db.intervalScan(tasksTable, "byId", [ + { eq: [{ col: "id", val: "task-1" }] }, + { eq: [{ col: "id", val: "task-1" }] }, + ]), + ), + ).resolves.toEqual([task]); + await expect( + execAsync( + db.driver.intervalScan( + tasksTable.tableName, + "byId", + [{ eq: [{ col: "title", val: "First" }] }], + {}, + ), + ), + ).rejects.toThrow("Primary-key index byId requires string IDs"); + await expect( + execAsync( + db.intervalScan(tasksTable, "byId", [{ eq: [{ col: "id", val: 1 }] }]), + ), + ).rejects.toThrow("Primary-key index byId requires string IDs"); + }); + it("rolls back duplicate insert batches without stale index entries", async () => { const db = await createDB(); await execAsync(db.loadTables([tasksTable])); diff --git a/packages/hyperdb/src/hyperdb/drivers/idb/idb-driver.ts b/packages/hyperdb/src/hyperdb/drivers/idb/idb-driver.ts index a7bee44..a0dd612 100644 --- a/packages/hyperdb/src/hyperdb/drivers/idb/idb-driver.ts +++ b/packages/hyperdb/src/hyperdb/drivers/idb/idb-driver.ts @@ -19,18 +19,21 @@ import { import { convertWhereToBound } from "../../core/query/bounds"; import type { TableDefinition } from "../../schema/table"; import { decodeValueFromStorage } from "../../storage/codec"; +import { + getPersistentIndexPlan, + getPersistentIndexSortKeyMode, + isPrimaryKeyBackedIndex, +} from "../persistent-index-plan"; import { assertSafeTableDefinition, getSqliteIndexSortKeyValue, - sqliteIndexSortColumns, - sqliteIndexSortKeyMode, } from "../sqlite/sqlite-common"; import { encodeSqliteSortKeyTuple } from "../sqlite/sqlite-sort-key"; type NativeStoredRecord = { id: string; row: unknown; - indexes: Record; + indexes: Record; }; type StoredTableMetadata = { @@ -130,7 +133,7 @@ const OLD_ROWS_STORE = "rows"; const OLD_INDEX_ENTRIES_STORE = "indexEntries"; const TABLE_METADATA_STORE = "tableMetadata"; const TABLE_STORE_PREFIX = "hyperdb:"; -const TABLE_INDEX_SIGNATURE_VERSION = 4; +const TABLE_INDEX_SIGNATURE_VERSION = 5; const IDB_READ_BATCH_SIZE = 1000; const STALE_CONNECTION_MESSAGE = "IndexedDB connection is stale; reopen the driver"; @@ -314,6 +317,10 @@ function validateHashBounds( } } +function toIdbSortKey(sortKey: Uint8Array): ArrayBuffer { + return Uint8Array.from(sortKey).buffer; +} + function createSortKeyRanges( factory: IDBFactory, tableDef: TableDefinition, @@ -324,8 +331,10 @@ function createSortKeyRanges( if (!indexDef) throw new Error(`Index ${indexName} not found`); const filterColumns = indexDef.cols.map(String); - const sortColumns = sqliteIndexSortColumns(tableDef, indexName); - const mode = sqliteIndexSortKeyMode(tableDef, indexName); + const sortColumns = + getPersistentIndexPlan(tableDef).byLogicalName.get(indexName) + ?.sortColumns ?? filterColumns; + const mode = getPersistentIndexSortKeyMode(tableDef, indexName); const rawBounds = convertWhereToBound(filterColumns, clauses); if (indexDef.type === "hash" || indexDef.type === "uniqhash") { @@ -343,14 +352,14 @@ function createSortKeyRanges( }; const lowerSortKey = bound.gte - ? encodeSqliteSortKeyTuple(bound.gte, mode) + ? toIdbSortKey(encodeSqliteSortKeyTuple(bound.gte, mode)) : bound.gt - ? encodeSqliteSortKeyTuple(bound.gt, mode) + ? toIdbSortKey(encodeSqliteSortKeyTuple(bound.gt, mode)) : undefined; const upperSortKey = bound.lte - ? encodeSqliteSortKeyTuple(bound.lte, mode) + ? toIdbSortKey(encodeSqliteSortKeyTuple(bound.lte, mode)) : bound.lt - ? encodeSqliteSortKeyTuple(bound.lt, mode) + ? toIdbSortKey(encodeSqliteSortKeyTuple(bound.lt, mode)) : undefined; if (lowerSortKey !== undefined && upperSortKey !== undefined) { @@ -400,17 +409,6 @@ function isIdOnlyIndex(tableDef: TableDefinition, indexName: string): boolean { return indexDef?.cols.length === 1 && String(indexDef.cols[0]) === "id"; } -function isUnfilteredClauses(clauses: WhereClause[]): boolean { - return clauses.every( - (clause) => - (!clause.eq || clause.eq.length === 0) && - (!clause.gte || clause.gte.length === 0) && - (!clause.gt || clause.gt.length === 0) && - (!clause.lte || clause.lte.length === 0) && - (!clause.lt || clause.lt.length === 0), - ); -} - function exactIdFromClauses(clauses: WhereClause[]): string | undefined { if (clauses.length !== 1) return undefined; const [clause] = clauses; @@ -428,6 +426,7 @@ function exactIdFromClauses(clauses: WhereClause[]): string | undefined { } function sortAndLimitRecords( + factory: IDBFactory, records: NativeStoredRecord[], indexName: string, selectOptions: SelectOptions, @@ -435,8 +434,12 @@ function sortAndLimitRecords( records.sort((left, right) => { const leftSortKey = left.indexes[indexName]; const rightSortKey = right.indexes[indexName]; - if (leftSortKey < rightSortKey) return -1; - if (leftSortKey > rightSortKey) return 1; + const sortKeyComparison = compareIdbKeys( + factory, + leftSortKey, + rightSortKey, + ); + if (sortKeyComparison !== 0) return sortKeyComparison; if (left.id < right.id) return -1; if (left.id > right.id) return 1; return 0; @@ -468,19 +471,27 @@ function indexKeyPath(indexName: string): string { } function indexIsUnique(tableDef: TableDefinition, indexName: string): boolean { - return tableDef.indexes[indexName]?.type === "uniqhash"; + return ( + getPersistentIndexPlan(tableDef).byLogicalName.get(indexName)?.unique ?? + false + ); } function createNativeRecordFromRow( tableDef: TableDefinition, row: Row, ): NativeStoredRecord { - const indexes: Record = {}; - - for (const indexName of Object.keys(tableDef.indexes)) { - const sortKey = getSqliteIndexSortKeyValue(tableDef, indexName, row); + const indexes: Record = {}; + + for (const physicalIndex of getPersistentIndexPlan(tableDef) + .physicalIndexes) { + const sortKey = getSqliteIndexSortKeyValue( + tableDef, + physicalIndex.name, + row, + ); if (sortKey !== null) { - indexes[indexName] = sortKey; + indexes[physicalIndex.name] = toIdbSortKey(sortKey); } } @@ -881,29 +892,45 @@ async function performScan( }); return result; } + } - if (isUnfilteredClauses(clauses) && selectOptions.limit === undefined) { - const records = await getAllRecords( - factory, - store, - undefined, - { - direction, - }, - ); - const result = records.map(decodeStoredRecord); - emitIdbDebug(debug, "scan", startedAt, { - txId, - traceContext: options.traceContext, - tableName, - indexName, - rowCount: result.length, - }); - return result; + if (isPrimaryKeyBackedIndex(tableDef, indexName)) { + const ids = new Set(); + for (const clause of clauses) { + for (const condition of clause.eq ?? []) { + if (condition.col !== "id" || typeof condition.val !== "string") { + throw new Error( + `Primary-key index ${indexName} requires string IDs`, + ); + } + ids.add(condition.val); + } } + const records = ( + await Promise.all( + [...ids].map((id) => + requestToPromise(store.get(id)), + ), + ) + ).filter((record): record is NativeStoredRecord => record !== undefined); + const result = records + .slice(0, selectOptions.limit) + .map(decodeStoredRecord); + emitIdbDebug(debug, "scan", startedAt, { + txId, + traceContext: options.traceContext, + tableName, + indexName, + rowCount: result.length, + }); + return result; } - const index = store.index(indexName); + const physicalIndex = + getPersistentIndexPlan(tableDef).byLogicalName.get(indexName); + if (!physicalIndex) + throw new Error(`Physical index ${indexName} not found`); + const index = store.index(physicalIndex.name); const ranges = createSortKeyRanges(factory, tableDef, indexName, clauses); const canPushLimit = ranges.length === 1; const records: NativeStoredRecord[] = []; @@ -925,7 +952,12 @@ async function performScan( ); } - const sorted = sortAndLimitRecords(records, indexName, selectOptions); + const sorted = sortAndLimitRecords( + factory, + records, + physicalIndex.name, + selectOptions, + ); const result = sorted.map(decodeStoredRecord); emitIdbDebug(debug, "scan", startedAt, { @@ -1635,7 +1667,11 @@ export class IdbDriver implements DBDriver { const tx = this.db.transaction(storeName, "readonly"); const store = tx.objectStore(storeName); - const expectedIndexes = new Set(Object.keys(tableDef.indexes)); + const expectedIndexes = new Set( + getPersistentIndexPlan(tableDef).physicalIndexes.map( + (physicalIndex) => physicalIndex.name, + ), + ); const actualIndexes = Array.from(store.indexNames); let storeNeedsUpgrade = actualIndexes.length !== expectedIndexes.size; @@ -1875,7 +1911,11 @@ function applySchemaUpgrade( const store = db.objectStoreNames.contains(storeName) ? tx.objectStore(storeName) : db.createObjectStore(storeName, { keyPath: "id" }); - const expectedIndexes = new Set(Object.keys(tableDef.indexes)); + const expectedIndexes = new Set( + getPersistentIndexPlan(tableDef).physicalIndexes.map( + (physicalIndex) => physicalIndex.name, + ), + ); for (const indexName of Array.from(store.indexNames)) { if ( diff --git a/packages/hyperdb/src/hyperdb/drivers/persistent-index-plan.test.ts b/packages/hyperdb/src/hyperdb/drivers/persistent-index-plan.test.ts new file mode 100644 index 0000000..468692a --- /dev/null +++ b/packages/hyperdb/src/hyperdb/drivers/persistent-index-plan.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; +import { defineTable, type TableDefinition } from "../schema/table"; +import { v } from "../schema/values"; +import { getPersistentIndexPlan } from "./persistent-index-plan"; + +describe("getPersistentIndexPlan", () => { + it("maps logical indexes to the physical indexes used by persistent drivers", () => { + const tableDef = defineTable("persistentIndexPlan", { + id: v.string(), + email: v.string(), + projectId: v.string(), + }) + .index("byEmailOrdered", ["email"]) + .index("byEmailUnique", ["email"], { type: "uniqhash" }) + .index("byProject", ["projectId"], { type: "hash" }); + + const plan = getPersistentIndexPlan(tableDef); + + expect(plan.physicalIndexes).toEqual([ + { + name: "byEmailOrdered", + sortColumns: ["email"], + unique: true, + mode: "scan", + }, + { + name: "byProject", + sortColumns: ["projectId", "id"], + unique: false, + mode: "scan", + }, + ]); + + const sharedEmailIndex = plan.physicalIndexes[0]; + expect(plan.byLogicalName.get("byEmailOrdered")).toBe(sharedEmailIndex); + expect(plan.byLogicalName.get("byEmailUnique")).toBe(sharedEmailIndex); + expect(plan.byLogicalName.get("byProject")).toBe(plan.physicalIndexes[1]); + expect(plan.byLogicalName.has("byId")).toBe(false); + }); + + it("caches the complete plan by table definition", () => { + const tableDef = defineTable("cachedPersistentIndexPlan", { + id: v.string(), + title: v.string(), + }).index("byTitle", ["title"]); + + expect(getPersistentIndexPlan(tableDef)).toBe( + getPersistentIndexPlan(tableDef), + ); + }); + + it("keeps schemaless indexes separate when their encodings differ", () => { + const tableDef = { + tableName: "schemalessPersistentIndexPlan", + schema: {}, + indexes: { + byId: { type: "uniqhash", cols: ["id"] }, + byValueOrdered: { type: "btree", cols: ["value"] }, + byValueUnique: { type: "uniqhash", cols: ["value"] }, + }, + idIndexName: "byId", + } as unknown as TableDefinition; + + const plan = getPersistentIndexPlan(tableDef); + + expect(plan.physicalIndexes).toHaveLength(2); + expect(plan.byLogicalName.get("byValueOrdered")?.mode).toBe("stored"); + expect(plan.byLogicalName.get("byValueUnique")?.mode).toBe("scan"); + expect(plan.byLogicalName.get("byValueOrdered")).not.toBe( + plan.byLogicalName.get("byValueUnique"), + ); + }); +}); diff --git a/packages/hyperdb/src/hyperdb/drivers/persistent-index-plan.ts b/packages/hyperdb/src/hyperdb/drivers/persistent-index-plan.ts new file mode 100644 index 0000000..f3ec875 --- /dev/null +++ b/packages/hyperdb/src/hyperdb/drivers/persistent-index-plan.ts @@ -0,0 +1,115 @@ +import type { TableDefinition } from "../schema/table"; + +export type PersistentSortKeyMode = "scan" | "stored"; + +export type PersistentPhysicalIndex = { + readonly name: string; + readonly sortColumns: readonly string[]; + readonly unique: boolean; + readonly mode: PersistentSortKeyMode; +}; + +export type PersistentIndexPlan = { + readonly physicalIndexes: readonly PersistentPhysicalIndex[]; + readonly byLogicalName: ReadonlyMap; +}; + +const persistentIndexPlanCache = new WeakMap< + TableDefinition, + PersistentIndexPlan +>(); + +export function isPrimaryKeyBackedIndex( + tableDef: TableDefinition, + indexName: string, +): boolean { + const indexDef = tableDef.indexes[indexName]; + return ( + indexDef !== undefined && + (indexDef.type === "hash" || indexDef.type === "uniqhash") && + indexDef.cols.length === 1 && + String(indexDef.cols[0]) === "id" + ); +} + +export function getPersistentIndexSortKeyMode( + tableDef: TableDefinition, + indexName: string, +): PersistentSortKeyMode { + const indexDef = tableDef.indexes[indexName]; + return indexDef?.type === "btree" && !tableDef.schemaValidator + ? "stored" + : "scan"; +} + +export function getPersistentIndexPlan( + tableDef: TableDefinition, +): PersistentIndexPlan { + const cached = persistentIndexPlanCache.get(tableDef); + if (cached) return cached; + + // Persistent drivers can use the native primary key directly, and a unique + // ordered index can serve compatible uniqhash and B-tree declarations. + const logicalNames = Object.keys(tableDef.indexes) + .filter((indexName) => !isPrimaryKeyBackedIndex(tableDef, indexName)) + .sort(); + const consumed = new Set(); + const physicalIndexes: PersistentPhysicalIndex[] = []; + const byLogicalName = new Map(); + + for (const logicalName of logicalNames) { + if (consumed.has(logicalName)) continue; + const indexDef = tableDef.indexes[logicalName]!; + const columns = indexDef.cols.map(String); + const mode = getPersistentIndexSortKeyMode(tableDef, logicalName); + const aliases = [logicalName]; + + if (indexDef.type === "btree" || indexDef.type === "uniqhash") { + for (const candidateName of logicalNames) { + if (candidateName === logicalName || consumed.has(candidateName)) { + continue; + } + const candidate = tableDef.indexes[candidateName]!; + const candidateColumns = candidate.cols.map(String); + const isUniqueOrderedPair = + new Set([indexDef.type, candidate.type]).size === 2 && + (indexDef.type === "uniqhash" || candidate.type === "uniqhash") && + (indexDef.type === "btree" || candidate.type === "btree"); + const sameColumns = + columns.length === candidateColumns.length && + columns.every((column, index) => column === candidateColumns[index]); + + if ( + isUniqueOrderedPair && + sameColumns && + mode === getPersistentIndexSortKeyMode(tableDef, candidateName) + ) { + aliases.push(candidateName); + } + } + } + + aliases.sort(); + for (const alias of aliases) consumed.add(alias); + const unique = aliases.some( + (alias) => tableDef.indexes[alias]?.type === "uniqhash", + ); + const sortColumns = [...columns]; + if (!unique && sortColumns[sortColumns.length - 1] !== "id") { + sortColumns.push("id"); + } + + const physicalIndex: PersistentPhysicalIndex = { + name: aliases[0]!, + sortColumns, + unique, + mode, + }; + physicalIndexes.push(physicalIndex); + for (const alias of aliases) byLogicalName.set(alias, physicalIndex); + } + + const plan = { physicalIndexes, byLogicalName }; + persistentIndexPlanCache.set(tableDef, plan); + return plan; +} diff --git a/packages/hyperdb/src/hyperdb/drivers/sqlite/async-sql-driver.test.ts b/packages/hyperdb/src/hyperdb/drivers/sqlite/async-sql-driver.test.ts index c951ff0..6243c0e 100644 --- a/packages/hyperdb/src/hyperdb/drivers/sqlite/async-sql-driver.test.ts +++ b/packages/hyperdb/src/hyperdb/drivers/sqlite/async-sql-driver.test.ts @@ -39,8 +39,44 @@ const uniqhashMigrationTableV2 = defineTable("asyncUniqhashMigration", { email: v.string(), }).index("byEmail", ["email"], { type: "uniqhash" }); +const suffixNamedIndexTable = defineTable("asyncSuffixNamedIndex", { + id: v.string(), + title: v.string(), +}) + .index("byTitle_sort_key", ["title"]) + .index("uniqueTitle", ["title"], { type: "uniqhash" }); + describe("db", async () => { for (const driver of [createSqlJsAsyncDriver]) { + it("preserves physical indexes whose logical names end in _sort_key", async () => { + const debug = vi.fn(); + const db = new DB(await driver({ debug })); + await execAsync(db.loadTables([suffixNamedIndexTable])); + await execAsync( + db.insert(suffixNamedIndexTable, [ + { id: "task-b", title: "B" }, + { id: "task-a", title: "A" }, + ]), + ); + debug.mockClear(); + + await execAsync(db.loadTables([suffixNamedIndexTable])); + + expect( + debug.mock.calls.some(([event]) => + event.normalizedSql.startsWith("DROP INDEX"), + ), + ).toBe(false); + await expect( + execAsync( + db.intervalScan(suffixNamedIndexTable, "byTitle_sort_key", [{}]), + ), + ).resolves.toEqual([ + { id: "task-a", title: "A" }, + { id: "task-b", title: "B" }, + ]); + }); + it("keeps SQL diagnostics silent by default", async () => { const logSpy = vi .spyOn(console, "log") diff --git a/packages/hyperdb/src/hyperdb/drivers/sqlite/async-sql-driver.ts b/packages/hyperdb/src/hyperdb/drivers/sqlite/async-sql-driver.ts index 68bd034..832847d 100644 --- a/packages/hyperdb/src/hyperdb/drivers/sqlite/async-sql-driver.ts +++ b/packages/hyperdb/src/hyperdb/drivers/sqlite/async-sql-driver.ts @@ -6,6 +6,7 @@ import type { DBCmd } from "../../commands/async"; import { unwrapCb } from "../../commands/async"; import { execAsync } from "../../core/executor"; import { cloneDeep } from "../../utils/toolkit"; +import { getPersistentIndexPlan } from "../persistent-index-plan"; import { buildSortKeyWhereClause, buildOrderClause, @@ -24,6 +25,8 @@ import { sqliteIndexSortKeyColumn, sqliteIndexIdentifier, isSqliteSortKeyColumn, + SQLITE_SORT_KEY_SUFFIX, + LEGACY_SQLITE_SORT_KEY_SUFFIX, assertSafeTableDefinition, buildRowInsertParams, parseSqliteStoredRow, @@ -683,16 +686,6 @@ export class AsyncSqlDriver implements DBDriver { tableDefinitions = cloneDeep(tableDefinitions); for (const tableDef of tableDefinitions) { - for (const [, indexDef] of Object.entries(tableDef.indexes)) { - if (indexDef.type !== "btree") continue; - const cols = [...indexDef.cols]; - - if (cols[cols.length - 1] !== "id") { - cols.push("id"); - } - (indexDef as unknown as { cols: typeof cols }).cols = cols; - } - await this.createTable(tableDef); const indexUniqueness = await this.getGeneratedIndexUniqueness( tableDef.tableName, @@ -801,16 +794,16 @@ export class AsyncSqlDriver implements DBDriver { tableDef: TableDefinition, ): Set { return new Set( - Object.keys(tableDef.indexes).map((indexName) => - sqliteIndexSortKeyColumn(indexName), + getPersistentIndexPlan(tableDef).physicalIndexes.map((physicalIndex) => + sqliteIndexSortKeyColumn(physicalIndex.name), ), ); } private getExpectedIndexNames(tableDef: TableDefinition): Set { return new Set( - Object.keys(tableDef.indexes).map((indexName) => - sqliteIndexIdentifier(tableDef.tableName, indexName), + getPersistentIndexPlan(tableDef).physicalIndexes.map((physicalIndex) => + sqliteIndexIdentifier(tableDef.tableName, physicalIndex.name), ), ); } @@ -818,7 +811,8 @@ export class AsyncSqlDriver implements DBDriver { private isGeneratedIndexName(tableName: string, indexName: string): boolean { return ( indexName.startsWith(`idx_${tableName}_`) && - indexName.endsWith("_sort_key") + (indexName.endsWith(SQLITE_SORT_KEY_SUFFIX) || + indexName.endsWith(LEGACY_SQLITE_SORT_KEY_SUFFIX)) ); } @@ -835,7 +829,8 @@ export class AsyncSqlDriver implements DBDriver { indexName, ); const expectedUnique = - tableDef.indexes[tableIndexName]?.type === "uniqhash"; + getPersistentIndexPlan(tableDef).byLogicalName.get(tableIndexName) + ?.unique ?? false; if (unique === expectedUnique) continue; } @@ -852,9 +847,11 @@ export class AsyncSqlDriver implements DBDriver { tableName: string, generatedIndexName: string, ): string { - return generatedIndexName - .slice(`idx_${tableName}_`.length) - .replace(/_sort_key$/, ""); + const indexName = generatedIndexName.slice(`idx_${tableName}_`.length); + const suffix = indexName.endsWith(SQLITE_SORT_KEY_SUFFIX) + ? SQLITE_SORT_KEY_SUFFIX + : LEGACY_SQLITE_SORT_KEY_SUFFIX; + return indexName.slice(0, -suffix.length); } // Sort-key columns whose encoding changed because the index flipped between @@ -872,9 +869,10 @@ export class AsyncSqlDriver implements DBDriver { tableDef.tableName, indexName, ); - const indexDef = tableDef.indexes[tableIndexName]; - if (!indexDef) continue; - const expectedUnique = indexDef.type === "uniqhash"; + const physicalIndex = + getPersistentIndexPlan(tableDef).byLogicalName.get(tableIndexName); + if (!physicalIndex) continue; + const expectedUnique = physicalIndex.unique; if (unique !== expectedUnique) { columns.push(sqliteIndexSortKeyColumn(tableIndexName)); } @@ -919,8 +917,9 @@ export class AsyncSqlDriver implements DBDriver { tableDef: TableDefinition, ): Promise { const existingColumns = await this.getTableColumns(tableDef.tableName); - for (const indexName of Object.keys(tableDef.indexes)) { - const sortKeyColumn = sqliteIndexSortKeyColumn(indexName); + for (const physicalIndex of getPersistentIndexPlan(tableDef) + .physicalIndexes) { + const sortKeyColumn = sqliteIndexSortKeyColumn(physicalIndex.name); if (existingColumns.has(sortKeyColumn)) continue; const sql = addSortKeyColumnSQL(tableDef.tableName, sortKeyColumn); @@ -933,8 +932,9 @@ export class AsyncSqlDriver implements DBDriver { private async backfillSortKeyColumns( tableDef: TableDefinition, ): Promise { - for (const indexName of Object.keys(tableDef.indexes)) { - const sortKeyColumn = sqliteIndexSortKeyColumn(indexName); + for (const physicalIndex of getPersistentIndexPlan(tableDef) + .physicalIndexes) { + const sortKeyColumn = sqliteIndexSortKeyColumn(physicalIndex.name); const sql = `SELECT data FROM ${tableDef.tableName} WHERE ${sortKeyColumn} IS NULL`; const startedAt = this.debug ? nowMs() : 0; const stmt = await this.db.prepare(sql); @@ -943,7 +943,7 @@ export class AsyncSqlDriver implements DBDriver { const rows = await stmt.values([]); emitAsyncSqlDebug(this.debug, "scan", sql, startedAt, () => ({ tableName: tableDef.tableName, - indexName, + indexName: physicalIndex.name, rowCount: rows.length, })); for (const chunk of chunkArray(rows, CHUNK_SIZE)) { @@ -964,7 +964,7 @@ export class AsyncSqlDriver implements DBDriver { startedAt, () => ({ tableName: tableDef.tableName, - indexName, + indexName: physicalIndex.name, }), error, ); @@ -976,8 +976,9 @@ export class AsyncSqlDriver implements DBDriver { } private async createIndexes(tableDef: TableDefinition): Promise { - for (const [indexName] of Object.entries(tableDef.indexes)) { - const indexSQL = createIndexSQL(tableDef, indexName); + for (const physicalIndex of getPersistentIndexPlan(tableDef) + .physicalIndexes) { + const indexSQL = createIndexSQL(tableDef, physicalIndex.name); await runAsyncSQL(this.db, indexSQL, undefined, this.debug); } } diff --git a/packages/hyperdb/src/hyperdb/drivers/sqlite/driver-edge-cases.test.ts b/packages/hyperdb/src/hyperdb/drivers/sqlite/driver-edge-cases.test.ts index 378b110..89e603c 100644 --- a/packages/hyperdb/src/hyperdb/drivers/sqlite/driver-edge-cases.test.ts +++ b/packages/hyperdb/src/hyperdb/drivers/sqlite/driver-edge-cases.test.ts @@ -12,13 +12,33 @@ import { type InspectableSqlDatabase, } from "../../test-utils/sql-js-driver"; import { v } from "../../schema/values"; -import type { SqlValue } from "./sqlite-common"; +import { buildSortKeyWhereClause, type SqlValue } from "./sqlite-common"; const noSideTablesTable = defineTable("driverEdgeNoSideTables", { id: v.string(), title: v.string(), }).index("byTitle", ["title"]); +const manyPrefixRangesTable = defineTable("driverEdgeManyPrefixRanges", { + id: v.string(), + entityId: v.string(), + tableName: v.string(), +}).index("byEntityAndTable", ["entityId", "tableName"]); + +const sharedUniqueOrderedTable = defineTable("driverEdgeSharedUniqueOrdered", { + id: v.string(), + email: v.string(), +}) + .index("byEmailOrdered", ["email"]) + .index("byEmailUnique", ["email"], { type: "uniqhash" }); + +const suffixNamedIndexTable = defineTable("driverEdgeSuffixNamedIndex", { + id: v.string(), + title: v.string(), +}) + .index("byTitle_sort_key", ["title"]) + .index("uniqueTitle", ["title"], { type: "uniqhash" }); + const sortKeyBackfillTableV1 = defineTable("driverEdgeSortKeyBackfill", { id: v.string(), title: v.string(), @@ -69,6 +89,37 @@ function sqliteRows(sqldb: InspectableSqlDatabase, sql: string): SqlValue[][] { } describe("SQLite driver edge case regressions", () => { + it("preserves physical indexes whose logical names end in _sort_key", async () => { + const { driver, execLog } = await createInspectableSqlDriver(); + const db = new SyncDB(new DB(driver)); + db.loadTables([suffixNamedIndexTable]); + db.insert(suffixNamedIndexTable, [ + { id: "task-b", title: "B" }, + { id: "task-a", title: "A" }, + ]); + execLog.length = 0; + + db.loadTables([suffixNamedIndexTable]); + + expect(execLog.some((sql) => sql.startsWith("DROP INDEX"))).toBe(false); + expect( + db + .intervalScan(suffixNamedIndexTable, "byTitle_sort_key", [{}]) + .map((row) => row.id), + ).toEqual(["task-a", "task-b"]); + }); + + it("rejects empty primary-key hash bounds", () => { + expect(() => + buildSortKeyWhereClause( + "byId", + noSideTablesTable.tableName, + [], + new Map([[noSideTablesTable.tableName, noSideTablesTable]]), + ), + ).toThrow(/Hash index should have equality conditions/); + }); + it("backfills sort keys for rows that predate a new index", async () => { const { driver, execLog } = await createInspectableSqlDriver(); const db = new SyncDB(new DB(driver)); @@ -91,6 +142,41 @@ describe("SQLite driver edge case regressions", () => { ).toEqual([{ id: "task-a", title: "A" }]); }); + it("replaces legacy textual sort-key columns with binary sort keys", async () => { + const { driver, sqldb } = await createInspectableSqlDriver(); + const db = new SyncDB(new DB(driver)); + db.loadTables([noSideTablesTable]); + db.insert(noSideTablesTable, [{ id: "task-a", title: "A" }]); + + sqldb.run("DROP INDEX idx_driverEdgeNoSideTables_byTitle_sort_key_v2"); + sqldb.run( + "ALTER TABLE driverEdgeNoSideTables RENAME COLUMN idx_byTitle_sort_key_v2 TO idx_byTitle_sort_key", + ); + sqldb.run( + "CREATE INDEX idx_driverEdgeNoSideTables_byTitle_sort_key ON driverEdgeNoSideTables(idx_byTitle_sort_key, id)", + ); + + db.loadTables([noSideTablesTable]); + + const columns = sqliteRows( + sqldb, + "PRAGMA table_info(driverEdgeNoSideTables)", + ).map((row) => String(row[1])); + expect(columns).not.toContain("idx_byTitle_sort_key"); + expect(columns).toContain("idx_byTitle_sort_key_v2"); + expect( + sqliteRows( + sqldb, + "SELECT typeof(idx_byTitle_sort_key_v2) FROM driverEdgeNoSideTables", + ), + ).toEqual([["blob"]]); + expect( + db.intervalScan(noSideTablesTable, "byTitle", [ + { eq: [{ col: "title", val: "A" }] }, + ]), + ).toEqual([{ id: "task-a", title: "A" }]); + }); + it("recomputes stored sort keys when a hash index is promoted to uniqhash", async () => { const { driver, sqldb } = await createInspectableSqlDriver(); const db = new SyncDB(new DB(driver)); @@ -109,7 +195,8 @@ describe("SQLite driver edge case regressions", () => { "PRAGMA index_list(driverEdgeUniqhashMigration)", ).find( (row) => - String(row[1]) === "idx_driverEdgeUniqhashMigration_byEmail_sort_key", + String(row[1]) === + "idx_driverEdgeUniqhashMigration_byEmail_sort_key_v2", ); expect(byEmail && Number(byEmail[2])).toBe(1); @@ -145,22 +232,17 @@ describe("SQLite driver edge case regressions", () => { sqldb, "PRAGMA table_info(driverEdgePruneSortKeys)", ).map((row) => String(row[1])); - expect(columns).toEqual([ - "id", - "data", - "idx_byId_sort_key", - "idx_byState_sort_key", - ]); + expect(columns).toEqual(["id", "data", "idx_byState_sort_key_v2"]); const indexNames = sqliteRows( sqldb, "SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'driverEdgePruneSortKeys'", ).map(([name]) => String(name)); expect(indexNames).toContain( - "idx_driverEdgePruneSortKeys_byState_sort_key", + "idx_driverEdgePruneSortKeys_byState_sort_key_v2", ); expect(indexNames).not.toContain( - "idx_driverEdgePruneSortKeys_byTitle_sort_key", + "idx_driverEdgePruneSortKeys_byTitle_sort_key_v2", ); expect( @@ -190,7 +272,7 @@ describe("SQLite driver edge case regressions", () => { ).toEqual([{ id: "task-a", projectId: "project-1", state: "open" }]); }); - it("uses IN for multiple exact sort-key lookups", async () => { + it("uses the primary key for multiple exact ID lookups", async () => { const { driver, execLog } = await createInspectableSqlDriver(); const db = new SyncDB(new DB(driver)); db.loadTables([noSideTablesTable]); @@ -213,10 +295,81 @@ describe("SQLite driver edge case regressions", () => { const selectSql = execLog.find((sql) => sql.includes("FROM driverEdgeNoSideTables"), ); - expect(selectSql).toContain("idx_byId_sort_key IN (?, ?)"); + expect(selectSql).toContain("WHERE id IN (?, ?)"); expect(selectSql).not.toContain(" OR "); }); + it("supports many exact-prefix ranges without exceeding SQLite expression depth", async () => { + const db = new SyncDB(new DB(await createSqlJsDriver())); + db.loadTables([manyPrefixRangesTable]); + + const rows = Array.from({ length: 400 }, (_, index) => ({ + id: `change-${index}`, + entityId: `entity-${index}`, + tableName: "tasks", + })); + db.insert(manyPrefixRangesTable, rows); + + const results = db.intervalScan( + manyPrefixRangesTable, + "byEntityAndTable", + rows.map((row) => ({ + eq: [ + { col: "entityId", val: row.entityId }, + { col: "tableName", val: row.tableName }, + ], + })), + ); + + expect(new Set(results.map((row) => row.id))).toEqual( + new Set(rows.map((row) => row.id)), + ); + }); + + it("shares one physical index between matching uniqhash and B-tree indexes", async () => { + const { driver, sqldb } = await createInspectableSqlDriver(); + const db = new SyncDB(new DB(driver)); + db.loadTables([sharedUniqueOrderedTable]); + db.insert(sharedUniqueOrderedTable, [ + { id: "user-b", email: "b@example.com" }, + { id: "user-a", email: "a@example.com" }, + ]); + + expect( + db.intervalScan(sharedUniqueOrderedTable, "byEmailUnique", [ + { eq: [{ col: "email", val: "a@example.com" }] }, + ]), + ).toEqual([{ id: "user-a", email: "a@example.com" }]); + expect( + db.intervalScan(sharedUniqueOrderedTable, "byEmailOrdered", [{}]), + ).toEqual([ + { id: "user-a", email: "a@example.com" }, + { id: "user-b", email: "b@example.com" }, + ]); + + const columns = sqliteRows( + sqldb, + "PRAGMA table_info(driverEdgeSharedUniqueOrdered)", + ).map((row) => String(row[1])); + expect(columns).toEqual(["id", "data", "idx_byEmailOrdered_sort_key_v2"]); + + const generatedIndexes = sqliteRows( + sqldb, + "PRAGMA index_list(driverEdgeSharedUniqueOrdered)", + ).filter((row) => String(row[1]).startsWith("idx_")); + expect(generatedIndexes).toHaveLength(1); + expect(String(generatedIndexes[0]?.[1])).toBe( + "idx_driverEdgeSharedUniqueOrdered_byEmailOrdered_sort_key_v2", + ); + expect(Number(generatedIndexes[0]?.[2])).toBe(1); + + expect(() => + db.insert(sharedUniqueOrderedTable, [ + { id: "user-c", email: "a@example.com" }, + ]), + ).toThrow(); + }); + it("chunks inserts by SQLite bind variable budget", async () => { const { driver, execLog } = await createInspectableSqlDriver(); const db = new SyncDB(new DB(driver)); @@ -225,7 +378,7 @@ describe("SQLite driver edge case regressions", () => { db.insert( noSideTablesTable, - Array.from({ length: 226 }, (_, index) => ({ + Array.from({ length: 301 }, (_, index) => ({ id: `task-${index}`, title: `Task ${index}`, })), @@ -236,7 +389,7 @@ describe("SQLite driver edge case regressions", () => { ); expect(inserts).toHaveLength(2); expect(inserts[0]!.match(/\?/g)).toHaveLength(900); - expect(inserts[1]!.match(/\?/g)).toHaveLength(4); + expect(inserts[1]!.match(/\?/g)).toHaveLength(3); }); it("stores index sort keys on the base table and scans without side-index tables", async () => { @@ -260,12 +413,7 @@ describe("SQLite driver edge case regressions", () => { sqldb, "PRAGMA table_info(driverEdgeNoSideTables)", ).map((row) => String(row[1])); - expect(columns).toEqual([ - "id", - "data", - "idx_byId_sort_key", - "idx_byTitle_sort_key", - ]); + expect(columns).toEqual(["id", "data", "idx_byTitle_sort_key_v2"]); const indexSql = sqliteRows( sqldb, @@ -273,12 +421,12 @@ describe("SQLite driver edge case regressions", () => { ).map(([sql]) => String(sql)); expect( indexSql.some((sql) => - sql.includes("ON driverEdgeNoSideTables(idx_byTitle_sort_key, id)"), + sql.includes("ON driverEdgeNoSideTables(idx_byTitle_sort_key_v2, id)"), ), ).toBe(true); expect( indexSql.some((sql) => - sql.includes("WHERE idx_byTitle_sort_key IS NOT NULL"), + sql.includes("WHERE idx_byTitle_sort_key_v2 IS NOT NULL"), ), ).toBe(true); diff --git a/packages/hyperdb/src/hyperdb/drivers/sqlite/sql-driver.ts b/packages/hyperdb/src/hyperdb/drivers/sqlite/sql-driver.ts index d96f3c0..3e94221 100644 --- a/packages/hyperdb/src/hyperdb/drivers/sqlite/sql-driver.ts +++ b/packages/hyperdb/src/hyperdb/drivers/sqlite/sql-driver.ts @@ -4,6 +4,7 @@ import type { DBDriver, DBDriverTX } from "../../core/driver"; import type { TableDefinition } from "../../schema/table"; import type { DBCmd } from "../../commands/async"; import { cloneDeep } from "../../utils/toolkit"; +import { getPersistentIndexPlan } from "../persistent-index-plan"; import { buildSortKeyWhereClause, buildOrderClause, @@ -22,6 +23,8 @@ import { sqliteIndexSortKeyColumn, sqliteIndexIdentifier, isSqliteSortKeyColumn, + SQLITE_SORT_KEY_SUFFIX, + LEGACY_SQLITE_SORT_KEY_SUFFIX, assertSafeTableDefinition, buildRowInsertParams, parseSqliteStoredRow, @@ -342,16 +345,6 @@ export class SqlDriver implements DBDriver { try { tableDefinitions = cloneDeep(tableDefinitions); for (const tableDef of tableDefinitions) { - for (const [, indexDef] of Object.entries(tableDef.indexes)) { - if (indexDef.type !== "btree") continue; - const cols = [...indexDef.cols]; - - if (cols[cols.length - 1] !== "id") { - cols.push("id"); - } - (indexDef as unknown as { cols: typeof cols }).cols = cols; - } - this.createTable(tableDef); const indexUniqueness = this.getGeneratedIndexUniqueness( tableDef.tableName, @@ -404,16 +397,16 @@ export class SqlDriver implements DBDriver { tableDef: TableDefinition, ): Set { return new Set( - Object.keys(tableDef.indexes).map((indexName) => - sqliteIndexSortKeyColumn(indexName), + getPersistentIndexPlan(tableDef).physicalIndexes.map((physicalIndex) => + sqliteIndexSortKeyColumn(physicalIndex.name), ), ); } private getExpectedIndexNames(tableDef: TableDefinition): Set { return new Set( - Object.keys(tableDef.indexes).map((indexName) => - sqliteIndexIdentifier(tableDef.tableName, indexName), + getPersistentIndexPlan(tableDef).physicalIndexes.map((physicalIndex) => + sqliteIndexIdentifier(tableDef.tableName, physicalIndex.name), ), ); } @@ -421,7 +414,8 @@ export class SqlDriver implements DBDriver { private isGeneratedIndexName(tableName: string, indexName: string): boolean { return ( indexName.startsWith(`idx_${tableName}_`) && - indexName.endsWith("_sort_key") + (indexName.endsWith(SQLITE_SORT_KEY_SUFFIX) || + indexName.endsWith(LEGACY_SQLITE_SORT_KEY_SUFFIX)) ); } @@ -438,7 +432,8 @@ export class SqlDriver implements DBDriver { indexName, ); const expectedUnique = - tableDef.indexes[tableIndexName]?.type === "uniqhash"; + getPersistentIndexPlan(tableDef).byLogicalName.get(tableIndexName) + ?.unique ?? false; if (unique === expectedUnique) continue; } @@ -450,9 +445,11 @@ export class SqlDriver implements DBDriver { tableName: string, generatedIndexName: string, ): string { - return generatedIndexName - .slice(`idx_${tableName}_`.length) - .replace(/_sort_key$/, ""); + const indexName = generatedIndexName.slice(`idx_${tableName}_`.length); + const suffix = indexName.endsWith(SQLITE_SORT_KEY_SUFFIX) + ? SQLITE_SORT_KEY_SUFFIX + : LEGACY_SQLITE_SORT_KEY_SUFFIX; + return indexName.slice(0, -suffix.length); } // Sort-key columns whose encoding changed because the index flipped between @@ -470,9 +467,10 @@ export class SqlDriver implements DBDriver { tableDef.tableName, indexName, ); - const indexDef = tableDef.indexes[tableIndexName]; - if (!indexDef) continue; - const expectedUnique = indexDef.type === "uniqhash"; + const physicalIndex = + getPersistentIndexPlan(tableDef).byLogicalName.get(tableIndexName); + if (!physicalIndex) continue; + const expectedUnique = physicalIndex.unique; if (unique !== expectedUnique) { columns.push(sqliteIndexSortKeyColumn(tableIndexName)); } @@ -503,8 +501,9 @@ export class SqlDriver implements DBDriver { private addMissingSortKeyColumns(tableDef: TableDefinition): void { const existingColumns = this.getTableColumns(tableDef.tableName); - for (const indexName of Object.keys(tableDef.indexes)) { - const sortKeyColumn = sqliteIndexSortKeyColumn(indexName); + for (const physicalIndex of getPersistentIndexPlan(tableDef) + .physicalIndexes) { + const sortKeyColumn = sqliteIndexSortKeyColumn(physicalIndex.name); if (existingColumns.has(sortKeyColumn)) continue; const sql = addSortKeyColumnSQL(tableDef.tableName, sortKeyColumn); @@ -515,8 +514,9 @@ export class SqlDriver implements DBDriver { // NOTE: backwards compatibility. Remove after v1. private backfillSortKeyColumns(tableDef: TableDefinition): void { - for (const indexName of Object.keys(tableDef.indexes)) { - const sortKeyColumn = sqliteIndexSortKeyColumn(indexName); + for (const physicalIndex of getPersistentIndexPlan(tableDef) + .physicalIndexes) { + const sortKeyColumn = sqliteIndexSortKeyColumn(physicalIndex.name); const q = this.db.prepare( `SELECT data FROM ${tableDef.tableName} WHERE ${sortKeyColumn} IS NULL`, ); @@ -536,8 +536,9 @@ export class SqlDriver implements DBDriver { } private createIndexes(tableDef: TableDefinition): void { - for (const indexName of Object.keys(tableDef.indexes)) { - const indexSQL = createIndexSQL(tableDef, indexName); + for (const physicalIndex of getPersistentIndexPlan(tableDef) + .physicalIndexes) { + const indexSQL = createIndexSQL(tableDef, physicalIndex.name); this.db.exec(indexSQL); } } diff --git a/packages/hyperdb/src/hyperdb/drivers/sqlite/sqlite-common.ts b/packages/hyperdb/src/hyperdb/drivers/sqlite/sqlite-common.ts index 2a54df4..b6f564d 100644 --- a/packages/hyperdb/src/hyperdb/drivers/sqlite/sqlite-common.ts +++ b/packages/hyperdb/src/hyperdb/drivers/sqlite/sqlite-common.ts @@ -12,10 +12,13 @@ import { decodeValueFromStorage, encodeValueForStorage, } from "../../storage/codec"; +import { + getPersistentIndexPlan, + isPrimaryKeyBackedIndex, +} from "../persistent-index-plan"; import { encodeSqliteSortKeyTuple, getSqliteSortKeyTuple, - type SqliteSortKeyMode, } from "./sqlite-sort-key"; export type SqlValue = number | string | Uint8Array | null; @@ -25,6 +28,8 @@ export const CHUNK_SIZE = 12000; export const SQL_BIND_PARAM_LIMIT = 900; const SAFE_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/; +export const SQLITE_SORT_KEY_SUFFIX = "_sort_key_v2"; +export const LEGACY_SQLITE_SORT_KEY_SUFFIX = "_sort_key"; export function chunkArray(array: T[], size: number): T[][] { const chunks: T[][] = []; @@ -35,7 +40,8 @@ export function chunkArray(array: T[], size: number): T[][] { } export function getSqliteInsertChunkSize(tableDef: TableDefinition): number { - const columnCount = 2 + Object.keys(tableDef.indexes).length; + const columnCount = + 2 + getPersistentIndexPlan(tableDef).physicalIndexes.length; return Math.max(1, Math.floor(SQL_BIND_PARAM_LIMIT / columnCount)); } @@ -63,7 +69,7 @@ export function assertSafeTableDefinition(tableDef: TableDefinition): void { export function sqliteIndexSortKeyColumn(indexName: string): string { assertSafeIdentifier("Index name", indexName); - const columnName = `idx_${indexName}_sort_key`; + const columnName = `idx_${indexName}${SQLITE_SORT_KEY_SUFFIX}`; assertSafeIdentifier("Sort-key column name", columnName); return columnName; } @@ -74,51 +80,36 @@ export function sqliteIndexIdentifier( ): string { assertSafeIdentifier("Table name", tableName); assertSafeIdentifier("Index name", indexName); - const indexIdentifier = `idx_${tableName}_${indexName}_sort_key`; + const indexIdentifier = `idx_${tableName}_${indexName}${SQLITE_SORT_KEY_SUFFIX}`; assertSafeIdentifier("SQLite index name", indexIdentifier); return indexIdentifier; } export function isSqliteSortKeyColumn(columnName: string): boolean { - return columnName.startsWith("idx_") && columnName.endsWith("_sort_key"); -} - -export function sqliteIndexSortColumns( - tableDef: TableDefinition, - indexName: string, -): string[] { - const indexDef = tableDef.indexes[indexName]; - if (!indexDef) throw new Error(`Index ${indexName} not found`); - - const cols = indexDef.cols.map(String); - if (indexDef.type !== "uniqhash" && cols[cols.length - 1] !== "id") { - cols.push("id"); - } - return cols; -} - -export function sqliteIndexSortKeyMode( - tableDef: TableDefinition, - indexName: string, -): SqliteSortKeyMode { - const indexDef = tableDef.indexes[indexName]; - return indexDef?.type === "btree" && isSchemalessTable(tableDef) - ? "stored" - : "scan"; + return ( + columnName.startsWith("idx_") && + (columnName.endsWith(SQLITE_SORT_KEY_SUFFIX) || + columnName.endsWith(LEGACY_SQLITE_SORT_KEY_SUFFIX)) + ); } export function getSqliteIndexSortKeyValue( tableDef: TableDefinition, indexName: string, row: Row, -): string | null { +): Uint8Array | null { const indexDef = tableDef.indexes[indexName]; if (!indexDef) throw new Error(`Index ${indexName} not found`); - const sortColumns = sqliteIndexSortColumns(tableDef, indexName); + const physicalIndex = + getPersistentIndexPlan(tableDef).byLogicalName.get(indexName); + if (!physicalIndex) { + throw new Error(`Index ${indexName} uses the primary-key access path`); + } + const sortColumns = physicalIndex.sortColumns; const includeMissing = indexDef.type === "btree" && isSchemalessTable(tableDef); - const mode = sqliteIndexSortKeyMode(tableDef, indexName); + const mode = physicalIndex.mode; const tuple = getSqliteSortKeyTuple(row, sortColumns, includeMissing); return tuple ? encodeSqliteSortKeyTuple(tuple, mode) : null; @@ -133,8 +124,8 @@ export function buildRowInsertParams( return [ storageRow.id, JSON.stringify(storageRow), - ...Object.keys(tableDef.indexes).map((indexName) => - getSqliteIndexSortKeyValue(tableDef, indexName, storageRow), + ...getPersistentIndexPlan(tableDef).physicalIndexes.map((physicalIndex) => + getSqliteIndexSortKeyValue(tableDef, physicalIndex.name, storageRow), ), ]; } @@ -159,6 +150,12 @@ function validateHashBounds( ): void { const indexColumn = indexColumns.join(", "); + if (bounds.length === 0 || bounds.some((bound) => !bound.gte)) { + throw new Error( + `Hash index should have equality conditions for columns '${indexColumn}' and index name '${indexName}': ${JSON.stringify(bounds)}`, + ); + } + for (const bound of bounds) { if ( (bound.gt !== undefined && bound.gt.length > 0) || @@ -171,7 +168,6 @@ function validateHashBounds( if ( !bound.lte || - !bound.gte || bound.lte.length !== indexColumns.length || bound.gte.length !== indexColumns.length ) { @@ -213,6 +209,18 @@ function isExactSortKeyBound(bound: { ); } +function joinBalancedOr(expressions: readonly string[]): string { + if (expressions.length === 0) { + throw new Error("Cannot join an empty list of SQL expressions"); + } + if (expressions.length === 1) return expressions[0]!; + + const middle = Math.ceil(expressions.length / 2); + return `(${joinBalancedOr(expressions.slice(0, middle))} OR ${joinBalancedOr( + expressions.slice(middle), + )})`; +} + export function buildSortKeyWhereClause( indexName: string, tableName: string, @@ -227,18 +235,34 @@ export function buildSortKeyWhereClause( const indexDef = tableDef.indexes[indexName]; if (!indexDef) throw new Error(`Index ${indexName} not found`); const filterColumns = indexDef.cols.map(String); - const sortColumns = sqliteIndexSortColumns(tableDef, indexName); - const mode = sqliteIndexSortKeyMode(tableDef, indexName); const rawBounds = convertWhereToBound(filterColumns, clauses); if (indexDef.type === "hash" || indexDef.type === "uniqhash") { validateHashBounds(indexName, filterColumns, rawBounds); } - const sortKeyColumn = sqliteIndexSortKeyColumn(indexName); + if (isPrimaryKeyBackedIndex(tableDef, indexName)) { + const ids = rawBounds.map((bound) => bound.gte?.[0]); + if (ids.some((id) => typeof id !== "string")) { + throw new Error(`Primary-key index ${indexName} requires string IDs`); + } + const placeholders = ids.map(() => "?").join(", "); + return { + where: `WHERE id IN (${placeholders})`, + params: ids, + }; + } + + const physicalIndex = + getPersistentIndexPlan(tableDef).byLogicalName.get(indexName); + if (!physicalIndex) throw new Error(`Physical index ${indexName} not found`); + const sortColumns = physicalIndex.sortColumns; + const mode = physicalIndex.mode; + + const sortKeyColumn = sqliteIndexSortKeyColumn(physicalIndex.name); const params: any[] = []; const rangeConditions: string[] = []; - const exactSortKeys: string[] = []; + const exactSortKeys: Uint8Array[] = []; let hasUnboundedRange = false; for (const rawBound of rawBounds) { @@ -255,7 +279,7 @@ export function buildSortKeyWhereClause( } const current: string[] = []; - const currentParams: string[] = []; + const currentParams: Uint8Array[] = []; if (bound.gte) { current.push(`${sortKeyColumn} >= ?`); @@ -291,12 +315,12 @@ export function buildSortKeyWhereClause( if (exactSortKeys.length > 0) { const placeholders = exactSortKeys.map(() => "?").join(", "); conditions.push( - `(${sortKeyColumn} IN (${placeholders}) OR ${rangeConditions.join( - " OR ", + `(${sortKeyColumn} IN (${placeholders}) OR ${joinBalancedOr( + rangeConditions, )})`, ); } else { - conditions.push(`(${rangeConditions.join(" OR ")})`); + conditions.push(joinBalancedOr(rangeConditions)); } } else if (exactSortKeys.length > 0) { const placeholders = exactSortKeys.map(() => "?").join(", "); @@ -326,7 +350,14 @@ export function buildOrderClause( return ""; } - return `ORDER BY ${sqliteIndexSortKeyColumn(indexName)} ${ + if (isPrimaryKeyBackedIndex(tableDef, indexName)) { + return ""; + } + const physicalIndex = + getPersistentIndexPlan(tableDef).byLogicalName.get(indexName); + if (!physicalIndex) return ""; + + return `ORDER BY ${sqliteIndexSortKeyColumn(physicalIndex.name)} ${ reverse ? "DESC" : "ASC" }`; } @@ -335,8 +366,8 @@ export function buildInsertSQL( tableDef: TableDefinition, valueCount: number, ): string { - const indexColumns = Object.keys(tableDef.indexes).map((indexName) => - sqliteIndexSortKeyColumn(indexName), + const indexColumns = getPersistentIndexPlan(tableDef).physicalIndexes.map( + (physicalIndex) => sqliteIndexSortKeyColumn(physicalIndex.name), ); const columns = ["id", "data", ...indexColumns]; const rowPlaceholders = `(${columns.map(() => "?").join(", ")})`; @@ -382,8 +413,8 @@ export function buildSelectSQL( } export function createTableSQL(tableDef: TableDefinition): string { - const sortKeyColumns = Object.keys(tableDef.indexes).map( - (indexName) => `${sqliteIndexSortKeyColumn(indexName)} TEXT`, + const sortKeyColumns = getPersistentIndexPlan(tableDef).physicalIndexes.map( + (physicalIndex) => `${sqliteIndexSortKeyColumn(physicalIndex.name)} BLOB`, ); const sql = ` CREATE TABLE IF NOT EXISTS ${tableDef.tableName} ( @@ -403,14 +434,16 @@ export function createIndexSQL( indexName: string, ): string { const tableName = tableDef.tableName; - const indexDef = tableDef.indexes[indexName]; - if (!indexDef) throw new Error(`Index ${indexName} not found`); - - const sortKeyColumn = sqliteIndexSortKeyColumn(indexName); - const indexIdentifier = sqliteIndexIdentifier(tableName, indexName); - const unique = indexDef.type === "uniqhash" ? "UNIQUE " : ""; - const indexColumns = - indexDef.type === "uniqhash" ? sortKeyColumn : `${sortKeyColumn}, id`; + const physicalIndex = + getPersistentIndexPlan(tableDef).byLogicalName.get(indexName); + if (!physicalIndex) throw new Error(`Physical index ${indexName} not found`); + + const sortKeyColumn = sqliteIndexSortKeyColumn(physicalIndex.name); + const indexIdentifier = sqliteIndexIdentifier(tableName, physicalIndex.name); + const unique = physicalIndex.unique ? "UNIQUE " : ""; + const indexColumns = physicalIndex.unique + ? sortKeyColumn + : `${sortKeyColumn}, id`; const sql = ` CREATE ${unique}INDEX IF NOT EXISTS ${indexIdentifier} ON ${tableName}(${indexColumns}) @@ -432,7 +465,7 @@ export function addSortKeyColumnSQL( sortKeyColumn: string, ): string { assertSafeIdentifier("Sort-key column name", sortKeyColumn); - const sql = `ALTER TABLE ${tableName} ADD COLUMN ${sortKeyColumn} TEXT` + const sql = `ALTER TABLE ${tableName} ADD COLUMN ${sortKeyColumn} BLOB` .trim() .replace(/\n+/g, " "); diff --git a/packages/hyperdb/src/hyperdb/drivers/sqlite/sqlite-sort-key.test.ts b/packages/hyperdb/src/hyperdb/drivers/sqlite/sqlite-sort-key.test.ts index cf92198..4b4c90f 100644 --- a/packages/hyperdb/src/hyperdb/drivers/sqlite/sqlite-sort-key.test.ts +++ b/packages/hyperdb/src/hyperdb/drivers/sqlite/sqlite-sort-key.test.ts @@ -22,8 +22,13 @@ function compareEncodedTuples( const encodedLeft = encodeSqliteSortKeyTuple(left, mode); const encodedRight = encodeSqliteSortKeyTuple(right, mode); - if (encodedLeft < encodedRight) return -1; - if (encodedLeft > encodedRight) return 1; + const length = Math.min(encodedLeft.length, encodedRight.length); + for (let index = 0; index < length; index++) { + if (encodedLeft[index]! < encodedRight[index]!) return -1; + if (encodedLeft[index]! > encodedRight[index]!) return 1; + } + if (encodedLeft.length < encodedRight.length) return -1; + if (encodedLeft.length > encodedRight.length) return 1; return 0; } @@ -52,6 +57,16 @@ describe("SqliteSortKey", () => { "a", "aa", "b", + "\u0000", + "\u007e", + "\u007f", + "\u07ff", + "\u0800", + "\ud7ff", + "\ud800", + "\ufffe", + "\uffff", + "𐀀", new Uint8Array([]), new Uint8Array([0]), new Uint8Array([0, 1]), @@ -69,7 +84,7 @@ describe("SqliteSortKey", () => { }); it("folds undefined into the null scan key", () => { - expect(encodeSqliteSortKeyTuple([undefined], "scan")).toBe( + expect(encodeSqliteSortKeyTuple([undefined], "scan")).toEqual( encodeSqliteSortKeyTuple([null], "scan"), ); }); @@ -87,7 +102,7 @@ describe("SqliteSortKey", () => { expect( encodeSqliteSortKeyTuple([new Uint8Array(buffer, 1, 2)], "scan"), - ).toBe( + ).toEqual( encodeSqliteSortKeyTuple( [{ $hyperdbType: "bytes", value: [1, 2] }], "scan", @@ -127,6 +142,16 @@ describe("SqliteSortKey", () => { "a", "aa", "b", + "\u0000", + "\u007e", + "\u007f", + "\u07ff", + "\u0800", + "\ud7ff", + "\ud800", + "\ufffe", + "\uffff", + "𐀀", new Uint8Array([]), new Uint8Array([0]), new Uint8Array([0, 1]), @@ -162,7 +187,7 @@ describe("SqliteSortKey", () => { it("keeps missing and null distinct in stored keys", () => { expect(compareEncodedTuples([undefined], [null], "stored")).toBe(-1); - expect(encodeSqliteSortKeyTuple([undefined], "stored")).not.toBe( + expect(encodeSqliteSortKeyTuple([undefined], "stored")).not.toEqual( encodeSqliteSortKeyTuple([null], "stored"), ); }); @@ -170,7 +195,7 @@ describe("SqliteSortKey", () => { it("encodes equivalent storage wrappers to equivalent sort keys", () => { const buffer = new Uint8Array([9, 1, 2, 8]).buffer; - expect(encodeSqliteSortKeyTuple([1n], "stored")).toBe( + expect(encodeSqliteSortKeyTuple([1n], "stored")).toEqual( encodeSqliteSortKeyTuple( [{ $hyperdbType: "bigint", value: "1" }], "stored", @@ -178,7 +203,7 @@ describe("SqliteSortKey", () => { ); expect( encodeSqliteSortKeyTuple([new Uint8Array(buffer, 1, 2)], "stored"), - ).toBe( + ).toEqual( encodeSqliteSortKeyTuple( [{ $hyperdbType: "bytes", value: [1, 2] }], "stored", @@ -198,7 +223,7 @@ describe("SqliteSortKey", () => { { $hyperdbType: "bytes", value: "AQI=" }, { $hyperdbType: "arrayBuffer", value: [Number.NaN] }, ]) { - expect(encodeSqliteSortKeyTuple([value], "stored")).not.toBe( + expect(encodeSqliteSortKeyTuple([value], "stored")).not.toEqual( validBytesKey, ); } @@ -251,6 +276,14 @@ describe("SqliteSortKey", () => { } }); + it("uses a compact binary representation for ordinary strings", () => { + const value = "checklist_items:00000000-0000-0000-0000-000000000000"; + const encoded = encodeSqliteSortKeyTuple([value], "scan"); + + expect(encoded).toBeInstanceOf(Uint8Array); + expect(encoded.length).toBeLessThan(value.length * 2); + }); + describe("getSqliteSortKeyTuple", () => { it("returns index values in column order", () => { expect( diff --git a/packages/hyperdb/src/hyperdb/drivers/sqlite/sqlite-sort-key.ts b/packages/hyperdb/src/hyperdb/drivers/sqlite/sqlite-sort-key.ts index 8cc3eb4..b76a73a 100644 --- a/packages/hyperdb/src/hyperdb/drivers/sqlite/sqlite-sort-key.ts +++ b/packages/hyperdb/src/hyperdb/drivers/sqlite/sqlite-sort-key.ts @@ -4,6 +4,20 @@ import { UnreachableError } from "../../utils"; export type SqliteSortKeyMode = "scan" | "stored"; const MAX_DECIMAL_LENGTH = 999999999999999; +const TERMINATOR = 0x00; +const TAG = { + min: 0x10, + missing: 0x20, + null: 0x30, + bigint: 0x40, + number: 0x50, + boolean: 0x60, + string: 0x70, + bytes: 0x80, + array: 0x90, + object: 0xa0, + max: 0xff, +} as const; function isEncodedObject( value: unknown, @@ -17,22 +31,6 @@ function isEncodedObject( ); } -function toHex(value: number, width: number): string { - return value.toString(16).padStart(width, "0"); -} - -function encodeCodeUnitString(value: string): string { - let encoded = ""; - for (let i = 0; i < value.length; i++) { - encoded += toHex(value.charCodeAt(i), 4); - } - return encoded + "!"; -} - -function encodeByteArray(bytes: readonly number[]): string { - return bytes.map((byte) => toHex(byte, 2)).join("") + "!"; -} - function isByteArray(value: unknown): value is number[] { return ( Array.isArray(value) && @@ -86,7 +84,11 @@ function bigintOf(value: unknown): bigint { throw new UnreachableError(value as never, "Expected bigint value"); } -function encodeBigint(value: unknown): string { +function asciiBytes(value: string): number[] { + return Array.from(value, (character) => character.charCodeAt(0)); +} + +function encodeBigintPayload(value: unknown): number[] { const bigint = bigintOf(value); const negative = bigint < 0n; const digits = (negative ? -bigint : bigint).toString(); @@ -95,7 +97,9 @@ function encodeBigint(value: unknown): string { } if (!negative) { - return `1${digits.length.toString().padStart(15, "0")}${digits}`; + return asciiBytes( + `1${digits.length.toString().padStart(15, "0")}${digits}`, + ); } const invertedLength = MAX_DECIMAL_LENGTH - digits.length; @@ -104,71 +108,120 @@ function encodeBigint(value: unknown): string { .map((digit) => String(9 - Number(digit))) .join(""); - return `0${invertedLength.toString().padStart(15, "0")}${invertedDigits}`; + return asciiBytes( + `0${invertedLength.toString().padStart(15, "0")}${invertedDigits}`, + ); } -function encodeNumber(value: number): string { +function encodeNumberPayload(value: number): number[] { const normalized = Object.is(value, -0) ? 0 : value; const buffer = new ArrayBuffer(8); const view = new DataView(buffer); view.setFloat64(0, normalized, false); const bytes = Array.from(new Uint8Array(buffer)); - if ((bytes[0] & 0x80) !== 0) { + if ((bytes[0]! & 0x80) !== 0) { for (let i = 0; i < bytes.length; i++) { - bytes[i] = ~bytes[i] & 0xff; + bytes[i] = ~bytes[i]! & 0xff; } } else { - bytes[0] = bytes[0] ^ 0x80; + bytes[0] = bytes[0]! ^ 0x80; } - return bytes.map((byte) => toHex(byte, 2)).join(""); + return bytes; } -function encodeArrayPayload(values: readonly unknown[]): string { - return values.map((item) => encodeStoredSortValue(item)).join("") + "!"; +// Encodes positive integers from 1 through 0x3fffff inclusive with the same +// bytewise order as their numeric value. Zero is reserved as a terminator; +// callers must keep values within the supported range. +function encodePositiveInteger(value: number): number[] { + if (value <= 0x7f) return [value]; + if (value <= 0x7ff) { + return [0xc0 | (value >> 6), 0x80 | (value & 0x3f)]; + } + if (value <= 0xffff) { + return [ + 0xe0 | (value >> 12), + 0x80 | ((value >> 6) & 0x3f), + 0x80 | (value & 0x3f), + ]; + } + return [ + 0xf0 | (value >> 18), + 0x80 | ((value >> 12) & 0x3f), + 0x80 | ((value >> 6) & 0x3f), + 0x80 | (value & 0x3f), + ]; } -function encodeObjectPayload(value: Record): string { +function encodeStringPayload(value: string): number[] { + const result: number[] = []; + for (let index = 0; index < value.length; index++) { + result.push(...encodePositiveInteger(value.charCodeAt(index) + 1)); + } + result.push(TERMINATOR); + return result; +} + +function encodeByteArrayPayload(bytes: readonly number[]): number[] { + const result: number[] = []; + for (const byte of bytes) { + result.push(...encodePositiveInteger(byte + 1)); + } + result.push(TERMINATOR); + return result; +} + +function encodeArrayPayload(values: readonly unknown[]): number[] { + return [...values.flatMap((item) => encodeStoredSortValue(item)), TERMINATOR]; +} + +function encodeObjectPayload(value: Record): number[] { const keys = Object.keys(value).sort(); - return ( - encodeArrayPayload(keys) + - keys.map((key) => encodeStoredSortValue(value[key])).join("") + - "!" - ); + return [ + ...encodeArrayPayload(keys), + ...keys.flatMap((key) => encodeStoredSortValue(value[key])), + TERMINATOR, + ]; } -function encodeScanSortValue(value: unknown): string { - if (value === MIN) return "00"; - if (value === MAX) return "zz"; - if (value === null || value === undefined) return "20"; +function encodeScanSortValue(value: unknown): number[] { + if (value === MIN) return [TAG.min]; + if (value === MAX) return [TAG.max]; + if (value === null || value === undefined) return [TAG.null]; if ( typeof value === "bigint" || (isEncodedObject(value) && value.$hyperdbType === "bigint" && typeof value.value === "string") ) { - return `30${encodeBigint(value)}`; + return [TAG.bigint, ...encodeBigintPayload(value)]; + } + if (typeof value === "number") { + return [TAG.number, ...encodeNumberPayload(value)]; + } + if (typeof value === "boolean") { + return [TAG.number, ...encodeNumberPayload(Number(value))]; + } + if (typeof value === "string") { + return [TAG.string, ...encodeStringPayload(value)]; } - if (typeof value === "number") return `40${encodeNumber(value)}`; - if (typeof value === "boolean") return `40${encodeNumber(Number(value))}`; - if (typeof value === "string") return `60${encodeCodeUnitString(value)}`; if ( value instanceof ArrayBuffer || ArrayBuffer.isView(value) || isEncodedBytesObject(value) ) { - return `70${encodeByteArray(bytesOf(value))}`; + return [TAG.bytes, ...encodeByteArrayPayload(bytesOf(value))]; } throw new UnreachableError(value as never, "Unknown scan sort-key value"); } -function encodeStoredSortValue(value: unknown): string { - if (value === MIN) return "00"; - if (value === MAX) return "zz"; - if (value === undefined) return "10"; - if (value === null) return "20"; +function encodeStoredSortValue(value: unknown): number[] { + if (value === MIN) return [TAG.min]; + if (value === MAX) return [TAG.max]; + if (value === undefined) return [TAG.missing]; + if (value === null) return [TAG.null]; if ( typeof value === "bigint" || @@ -176,23 +229,30 @@ function encodeStoredSortValue(value: unknown): string { value.$hyperdbType === "bigint" && typeof value.value === "string") ) { - return `30${encodeBigint(value)}`; + return [TAG.bigint, ...encodeBigintPayload(value)]; + } + if (typeof value === "number") { + return [TAG.number, ...encodeNumberPayload(value)]; + } + if (typeof value === "boolean") { + return [TAG.boolean, value ? 1 : 0]; + } + if (typeof value === "string") { + return [TAG.string, ...encodeStringPayload(value)]; } - - if (typeof value === "number") return `40${encodeNumber(value)}`; - if (typeof value === "boolean") return `50${value ? "1" : "0"}`; - if (typeof value === "string") return `60${encodeCodeUnitString(value)}`; - if ( value instanceof ArrayBuffer || ArrayBuffer.isView(value) || isEncodedBytesObject(value) ) { - return `70${encodeByteArray(bytesOf(value))}`; + return [TAG.bytes, ...encodeByteArrayPayload(bytesOf(value))]; + } + if (Array.isArray(value)) { + return [TAG.array, ...encodeArrayPayload(value)]; + } + if (isEncodedObject(value)) { + return [TAG.object, ...encodeObjectPayload(value)]; } - - if (Array.isArray(value)) return `80${encodeArrayPayload(value)}`; - if (isEncodedObject(value)) return `90${encodeObjectPayload(value)}`; throw new UnreachableError(value as never, "Unknown stored sort-key value"); } @@ -200,11 +260,10 @@ function encodeStoredSortValue(value: unknown): string { export function encodeSqliteSortKeyTuple( tuple: readonly unknown[], mode: SqliteSortKeyMode, -): string { +): Uint8Array { const encodeValue = mode === "stored" ? encodeStoredSortValue : encodeScanSortValue; - - return tuple.map((value) => encodeValue(value)).join(""); + return Uint8Array.from(tuple.flatMap((value) => encodeValue(value))); } export function getSqliteSortKeyTuple( diff --git a/packages/hyperdb/src/hyperdb/runtime/db.test.ts b/packages/hyperdb/src/hyperdb/runtime/db.test.ts index fd2fc52..846b6a3 100644 --- a/packages/hyperdb/src/hyperdb/runtime/db.test.ts +++ b/packages/hyperdb/src/hyperdb/runtime/db.test.ts @@ -78,6 +78,13 @@ const bigintHashErrorTable = defineTable("bigintHashError", { value: v.bigint(), }).index("byValueHash", ["value"], { type: "hash" }); +const sharedUniqueOrderedTable = defineTable("sharedUniqueOrdered", { + id: v.string(), + email: v.string(), +}) + .index("byEmailOrdered", ["email"]) + .index("byEmailUnique", ["email"], { type: "uniqhash" }); + describe("db", async () => { for (const [driverName, createDriver] of createDriverFactories()) { it("preloadTables is a no-op for plain DB - " + driverName, async () => { @@ -89,6 +96,38 @@ describe("db", async () => { ).resolves.toBeUndefined(); }); + it( + "queries matching uniqhash and B-tree logical indexes - " + driverName, + async () => { + const db = new AsyncDB(new DB(await createDriver())); + await db.loadTables([sharedUniqueOrderedTable]); + await db.insert(sharedUniqueOrderedTable, [ + { id: "user-b", email: "b@example.com" }, + { id: "user-a", email: "a@example.com" }, + ]); + + expect( + await db.intervalScan(sharedUniqueOrderedTable, "byEmailUnique", [ + { eq: [{ col: "email", val: "a@example.com" }] }, + ]), + ).toEqual([{ id: "user-a", email: "a@example.com" }]); + expect( + await db.intervalScan(sharedUniqueOrderedTable, "byEmailOrdered", [ + {}, + ]), + ).toEqual([ + { id: "user-a", email: "a@example.com" }, + { id: "user-b", email: "b@example.com" }, + ]); + + await expect( + db.insert(sharedUniqueOrderedTable, [ + { id: "user-c", email: "a@example.com" }, + ]), + ).rejects.toThrow(); + }, + ); + it("insert, delete, upsert - " + driverName, async () => { const db = new AsyncDB(new DB(await createDriver())); await db.loadTables([tasksTable, taskTemplatesTable]); @@ -747,7 +786,7 @@ describe("Database Operations Edge Cases", async () => { ) .index("byPostTitle", ["title"]) .index("byPostTitleHash", ["title"], { type: "hash" }) - .index("byPostTitleSlug", ["title", "slug"]); + .index("byPostSlugTitle", ["slug", "title"]); const db = new AsyncDB( new DB(await createDriver(), { runtimeRowsValidation: true }), @@ -826,24 +865,33 @@ describe("Database Operations Edge Cases", async () => { }), ).toEqual([nullTitlePost, firstPost, secondPost]); expect( - await db.intervalScan(documentsTable, "byPostTitleSlug", [ + await db.intervalScan(documentsTable, "byPostSlugTitle", [ { - eq: [{ col: "title", val: null }], + eq: [ + { col: "slug", val: "untitled" }, + { col: "title", val: null }, + ], }, ]), ).toEqual([nullTitlePost]); expect( - await db.intervalScan(documentsTable, "byPostTitleSlug", [ + await db.intervalScan(documentsTable, "byPostSlugTitle", [ { - eq: [{ col: "title", val: "Hello" }], + eq: [ + { col: "slug", val: "hello" }, + { col: "title", val: "Hello" }, + ], }, ]), ).toEqual([firstPost]); expect( - await db.intervalScan(documentsTable, "byPostTitleSlug", [ + await db.intervalScan(documentsTable, "byPostSlugTitle", [ { - eq: [{ col: "title", val: "Preview" }], + eq: [ + { col: "slug", val: "preview" }, + { col: "title", val: "Preview" }, + ], }, ]), ).toEqual([]); @@ -879,9 +927,12 @@ describe("Database Operations Edge Cases", async () => { await db.upsert(documentsTable, [promotedPreview]); expect( - await db.intervalScan(documentsTable, "byPostTitleSlug", [ + await db.intervalScan(documentsTable, "byPostSlugTitle", [ { - eq: [{ col: "title", val: "Preview" }], + eq: [ + { col: "slug", val: "preview" }, + { col: "title", val: "Preview" }, + ], }, ]), ).toEqual([promotedPreview]); @@ -903,9 +954,12 @@ describe("Database Operations Edge Cases", async () => { ]), ).toEqual([]); expect( - await db.intervalScan(documentsTable, "byPostTitleSlug", [ + await db.intervalScan(documentsTable, "byPostSlugTitle", [ { - eq: [{ col: "title", val: null }], + eq: [ + { col: "slug", val: "untitled" }, + { col: "title", val: null }, + ], }, ]), ).toEqual([]); diff --git a/packages/hyperdb/src/hyperdb/schema/table.test.ts b/packages/hyperdb/src/hyperdb/schema/table.test.ts index 5b3f03c..9e0ce0d 100644 --- a/packages/hyperdb/src/hyperdb/schema/table.test.ts +++ b/packages/hyperdb/src/hyperdb/schema/table.test.ts @@ -181,6 +181,48 @@ describe("defineTable", () => { } }); + it("rejects duplicate index definitions and overlapping B-tree prefixes", () => { + const schema = { + id: v.string(), + name: v.string(), + createdAt: v.string(), + }; + + expect(() => + defineTable("duplicateIndexName", schema).index("byId", ["id"]), + ).toThrow(/Index name byId is already defined/); + + expect(() => + defineTable("duplicateBtree", schema) + .index("byName", ["name"]) + .index("byNameAgain", ["name"]), + ).toThrow(/duplicate the same btree definition/); + + expect(() => + defineTable("duplicateUniqhash", schema) + .index("byName", ["name"], { type: "uniqhash" }) + .index("byNameAgain", ["name"], { type: "uniqhash" }), + ).toThrow(/duplicate the same uniqhash definition/); + + expect(() => + defineTable("overlappingBtree", schema) + .index("byName", ["name"]) + .index("byNameCreatedAt", ["name", "createdAt"]), + ).toThrow(/overlap by column prefix/); + + expect(() => + defineTable("overlappingBtreeReverse", schema) + .index("byNameCreatedAt", ["name", "createdAt"]) + .index("byName", ["name"]), + ).toThrow(/overlap by column prefix/); + + expect(() => + defineTable("sharedUniqueOrdered", schema) + .index("byNameUnique", ["name"], { type: "uniqhash" }) + .index("byNameOrdered", ["name"]), + ).not.toThrow(); + }); + it("uses defineTable for schema-backed table definitions", () => { const tasksTable = defineTable("tasks", { id: v.string(), diff --git a/packages/hyperdb/src/hyperdb/schema/table.ts b/packages/hyperdb/src/hyperdb/schema/table.ts index 7b50f07..d3d3649 100644 --- a/packages/hyperdb/src/hyperdb/schema/table.ts +++ b/packages/hyperdb/src/hyperdb/schema/table.ts @@ -210,6 +210,50 @@ export function validateIndexes( } } } + + const indexEntries = Object.entries(indexes); + for (let leftIndex = 0; leftIndex < indexEntries.length; leftIndex++) { + const [leftName, leftConfig] = indexEntries[leftIndex]!; + const leftColumns = leftConfig.cols.map(String); + + for ( + let rightIndex = leftIndex + 1; + rightIndex < indexEntries.length; + rightIndex++ + ) { + const [rightName, rightConfig] = indexEntries[rightIndex]!; + const rightColumns = rightConfig.cols.map(String); + const sameColumns = + leftColumns.length === rightColumns.length && + leftColumns.every((column, index) => column === rightColumns[index]); + + if (leftConfig.type === rightConfig.type && sameColumns) { + throw new Error( + `Indexes ${leftName} and ${rightName} duplicate the same ${leftConfig.type} definition on table: ${tableName}`, + ); + } + + if (leftConfig.type !== "btree" || rightConfig.type !== "btree") { + continue; + } + + const shorterColumns = + leftColumns.length < rightColumns.length ? leftColumns : rightColumns; + const longerColumns = + leftColumns.length < rightColumns.length ? rightColumns : leftColumns; + const isStrictPrefix = + shorterColumns.length < longerColumns.length && + shorterColumns.every( + (column, index) => column === longerColumns[index], + ); + + if (isStrictPrefix) { + throw new Error( + `B-tree indexes ${leftName} and ${rightName} overlap by column prefix on table: ${tableName}`, + ); + } + } + } } function addIndexMethod( @@ -231,6 +275,12 @@ function addIndexMethod( columns: readonly IndexableColumn[], options?: IndexOptions, ) { + if (Object.prototype.hasOwnProperty.call(tableDef.indexes, name)) { + throw new Error( + `Index name ${name} is already defined on table: ${tableDef.tableName}`, + ); + } + const type = options?.type ?? "btree"; const nextIndexes = { ...tableDef.indexes,