feat: improve indexes

packages/hyperdb-demo/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @will-be-done/hyperdb-demo-app
 
+## 0.0.11
+
+### Patch Changes
+
+- Updated dependencies [53a6916]
+- Updated dependencies [0678e07]
+  - @will-be-done/hyperdb@0.4.3
+  - @will-be-done/hyperdb-devtool@0.4.3
+
 ## 0.0.10
 
 ### Patch Changes

packages/hyperdb-devtool/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @will-be-done/hyperdb-devtool
 
+## 0.4.3
+
+### Patch Changes
+
+- Updated dependencies [53a6916]
+- Updated dependencies [0678e07]
+  - @will-be-done/hyperdb@0.4.3
+
 ## 0.4.2
 
 ### Patch Changes

packages/hyperdb/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @will-be-done/hyperdb
 
+## 0.4.3
+
+### Patch Changes
+
+- 53a6916: catch firefox idb tx finish
+- 0678e07: add long redonly tx
+
 ## 0.4.2
 
 ### Patch Changes

packages/hyperdb/src/hyperdb/drivers/idb/idb-driver.browser.test.ts

@@ -11,6 +11,7 @@ import type { HyperDB } from "../../core/contracts";
 import { DB } from "../../runtime/db";
 import { HybridDB } from "../../runtime/hybrid-db";
 import { BptreeInmemDriver } from "../inmemory/bptree-inmem-driver";
+import { getSqliteIndexSortKeyValue } from "../sqlite/sqlite-common";
 import { defineTable } from "../../schema/table";
 import { v } from "../../schema/values";
 import {
@@ -70,7 +71,7 @@ type GetAllRecordsHost = {
 
 type RawStoredRecord = {
   row: Record<string, unknown>;
-  indexes: Record<string, string>;
+  indexes: Record<string, ArrayBuffer>;
 };
 
 function spyOnGetAllRecords<T extends object>(prototype: T) {
@@ -159,7 +160,14 @@ describe("IdbDriver", () => {
         $hyperdbType: "bigint",
         value: "42",
       });
-      expect(stored.indexes.byCount).toEqual(expect.any(String));
+      expect(stored.indexes.byCount).toBeInstanceOf(ArrayBuffer);
+      expect(new Uint8Array(stored.indexes.byCount)).toEqual(
+        getSqliteIndexSortKeyValue(rawRowsTable, "byCount", {
+          id: "row-1",
+          count: 42n,
+          bytes,
+        }),
+      );
     } finally {
       rawDb.close();
       await deleteDatabase(dbName);
@@ -815,6 +823,42 @@ describe("IdbDriver", () => {
     }
   });
 
+  it("validates and deduplicates primary-key equality scans", async () => {
+    const db = await createDB();
+    await execAsync(db.loadTables([tasksTable]));
+    const task = {
+      id: "task-1",
+      title: "First",
+      projectId: "project-1",
+      rank: 1,
+    };
+    await execAsync(db.insert(tasksTable, [task]));
+
+    await expect(
+      execAsync(
+        db.intervalScan(tasksTable, "byId", [
+          { eq: [{ col: "id", val: "task-1" }] },
+          { eq: [{ col: "id", val: "task-1" }] },
+        ]),
+      ),
+    ).resolves.toEqual([task]);
+    await expect(
+      execAsync(
+        db.driver.intervalScan(
+          tasksTable.tableName,
+          "byId",
+          [{ eq: [{ col: "title", val: "First" }] }],
+          {},
+        ),
+      ),
+    ).rejects.toThrow("Primary-key index byId requires string IDs");
+    await expect(
+      execAsync(
+        db.intervalScan(tasksTable, "byId", [{ eq: [{ col: "id", val: 1 }] }]),
+      ),
+    ).rejects.toThrow("Primary-key index byId requires string IDs");
+  });
+
   it("rolls back duplicate insert batches without stale index entries", async () => {
     const db = await createDB();
     await execAsync(db.loadTables([tasksTable]));

packages/hyperdb/src/hyperdb/drivers/idb/idb-driver.ts

@@ -19,18 +19,21 @@ import {
 import { convertWhereToBound } from "../../core/query/bounds";
 import type { TableDefinition } from "../../schema/table";
 import { decodeValueFromStorage } from "../../storage/codec";
+import {
+  getPersistentIndexPlan,
+  getPersistentIndexSortKeyMode,
+  isPrimaryKeyBackedIndex,
+} from "../persistent-index-plan";
 import {
   assertSafeTableDefinition,
   getSqliteIndexSortKeyValue,
-  sqliteIndexSortColumns,
-  sqliteIndexSortKeyMode,
 } from "../sqlite/sqlite-common";
 import { encodeSqliteSortKeyTuple } from "../sqlite/sqlite-sort-key";
 
 type NativeStoredRecord = {
   id: string;
   row: unknown;
-  indexes: Record<string, string>;
+  indexes: Record<string, ArrayBuffer>;
 };
 
 type StoredTableMetadata = {
@@ -130,7 +133,7 @@ const OLD_ROWS_STORE = "rows";
 const OLD_INDEX_ENTRIES_STORE = "indexEntries";
 const TABLE_METADATA_STORE = "tableMetadata";
 const TABLE_STORE_PREFIX = "hyperdb:";
-const TABLE_INDEX_SIGNATURE_VERSION = 4;
+const TABLE_INDEX_SIGNATURE_VERSION = 5;
 const IDB_READ_BATCH_SIZE = 1000;
 const STALE_CONNECTION_MESSAGE =
   "IndexedDB connection is stale; reopen the driver";
@@ -314,6 +317,10 @@ function validateHashBounds(
   }
 }
 
+function toIdbSortKey(sortKey: Uint8Array): ArrayBuffer {
+  return Uint8Array.from(sortKey).buffer;
+}
+
 function createSortKeyRanges(
   factory: IDBFactory,
   tableDef: TableDefinition,
@@ -324,8 +331,10 @@ function createSortKeyRanges(
   if (!indexDef) throw new Error(`Index ${indexName} not found`);
 
   const filterColumns = indexDef.cols.map(String);
-  const sortColumns = sqliteIndexSortColumns(tableDef, indexName);
-  const mode = sqliteIndexSortKeyMode(tableDef, indexName);
+  const sortColumns =
+    getPersistentIndexPlan(tableDef).byLogicalName.get(indexName)
+      ?.sortColumns ?? filterColumns;
+  const mode = getPersistentIndexSortKeyMode(tableDef, indexName);
   const rawBounds = convertWhereToBound(filterColumns, clauses);
 
   if (indexDef.type === "hash" || indexDef.type === "uniqhash") {
@@ -343,14 +352,14 @@ function createSortKeyRanges(
     };
 
     const lowerSortKey = bound.gte
-      ? encodeSqliteSortKeyTuple(bound.gte, mode)
+      ? toIdbSortKey(encodeSqliteSortKeyTuple(bound.gte, mode))
       : bound.gt
-        ? encodeSqliteSortKeyTuple(bound.gt, mode)
+        ? toIdbSortKey(encodeSqliteSortKeyTuple(bound.gt, mode))
         : undefined;
     const upperSortKey = bound.lte
-      ? encodeSqliteSortKeyTuple(bound.lte, mode)
+      ? toIdbSortKey(encodeSqliteSortKeyTuple(bound.lte, mode))
       : bound.lt
-        ? encodeSqliteSortKeyTuple(bound.lt, mode)
+        ? toIdbSortKey(encodeSqliteSortKeyTuple(bound.lt, mode))
         : undefined;
 
     if (lowerSortKey !== undefined && upperSortKey !== undefined) {
@@ -400,17 +409,6 @@ function isIdOnlyIndex(tableDef: TableDefinition, indexName: string): boolean {
   return indexDef?.cols.length === 1 && String(indexDef.cols[0]) === "id";
 }
 
-function isUnfilteredClauses(clauses: WhereClause[]): boolean {
-  return clauses.every(
-    (clause) =>
-      (!clause.eq || clause.eq.length === 0) &&
-      (!clause.gte || clause.gte.length === 0) &&
-      (!clause.gt || clause.gt.length === 0) &&
-      (!clause.lte || clause.lte.length === 0) &&
-      (!clause.lt || clause.lt.length === 0),
-  );
-}
-
 function exactIdFromClauses(clauses: WhereClause[]): string | undefined {
   if (clauses.length !== 1) return undefined;
   const [clause] = clauses;
@@ -428,15 +426,20 @@ function exactIdFromClauses(clauses: WhereClause[]): string | undefined {
 }
 
 function sortAndLimitRecords(
+  factory: IDBFactory,
   records: NativeStoredRecord[],
   indexName: string,
   selectOptions: SelectOptions,
 ): NativeStoredRecord[] {
   records.sort((left, right) => {
     const leftSortKey = left.indexes[indexName];
     const rightSortKey = right.indexes[indexName];
-    if (leftSortKey < rightSortKey) return -1;
-    if (leftSortKey > rightSortKey) return 1;
+    const sortKeyComparison = compareIdbKeys(
+      factory,
+      leftSortKey,
+      rightSortKey,
+    );
+    if (sortKeyComparison !== 0) return sortKeyComparison;
     if (left.id < right.id) return -1;
     if (left.id > right.id) return 1;
     return 0;
@@ -468,19 +471,27 @@ function indexKeyPath(indexName: string): string {
 }
 
 function indexIsUnique(tableDef: TableDefinition, indexName: string): boolean {
-  return tableDef.indexes[indexName]?.type === "uniqhash";
+  return (
+    getPersistentIndexPlan(tableDef).byLogicalName.get(indexName)?.unique ??
+    false
+  );
 }
 
 function createNativeRecordFromRow(
   tableDef: TableDefinition,
   row: Row,
 ): NativeStoredRecord {
-  const indexes: Record<string, string> = {};
-
-  for (const indexName of Object.keys(tableDef.indexes)) {
-    const sortKey = getSqliteIndexSortKeyValue(tableDef, indexName, row);
+  const indexes: Record<string, ArrayBuffer> = {};
+
+  for (const physicalIndex of getPersistentIndexPlan(tableDef)
+    .physicalIndexes) {
+    const sortKey = getSqliteIndexSortKeyValue(
+      tableDef,
+      physicalIndex.name,
+      row,
+    );
     if (sortKey !== null) {
-      indexes[indexName] = sortKey;
+      indexes[physicalIndex.name] = toIdbSortKey(sortKey);
     }
   }
 
@@ -881,29 +892,45 @@ async function performScan(
         });
         return result;
       }
+    }
 
-      if (isUnfilteredClauses(clauses) && selectOptions.limit === undefined) {
-        const records = await getAllRecords<NativeStoredRecord>(
-          factory,
-          store,
-          undefined,
-          {
-            direction,
-          },
-        );
-        const result = records.map(decodeStoredRecord);
-        emitIdbDebug(debug, "scan", startedAt, {
-          txId,
-          traceContext: options.traceContext,
-          tableName,
-          indexName,
-          rowCount: result.length,
-        });
-        return result;
+    if (isPrimaryKeyBackedIndex(tableDef, indexName)) {
+      const ids = new Set<string>();
+      for (const clause of clauses) {
+        for (const condition of clause.eq ?? []) {
+          if (condition.col !== "id" || typeof condition.val !== "string") {
+            throw new Error(
+              `Primary-key index ${indexName} requires string IDs`,
+            );
+          }
+          ids.add(condition.val);
+        }
       }
+      const records = (
+        await Promise.all(
+          [...ids].map((id) =>
+            requestToPromise<NativeStoredRecord | undefined>(store.get(id)),
+          ),
+        )
+      ).filter((record): record is NativeStoredRecord => record !== undefined);
+      const result = records
+        .slice(0, selectOptions.limit)
+        .map(decodeStoredRecord);
+      emitIdbDebug(debug, "scan", startedAt, {
+        txId,
+        traceContext: options.traceContext,
+        tableName,
+        indexName,
+        rowCount: result.length,
+      });
+      return result;
     }
 
-    const index = store.index(indexName);
+    const physicalIndex =
+      getPersistentIndexPlan(tableDef).byLogicalName.get(indexName);
+    if (!physicalIndex)
+      throw new Error(`Physical index ${indexName} not found`);
+    const index = store.index(physicalIndex.name);
     const ranges = createSortKeyRanges(factory, tableDef, indexName, clauses);
     const canPushLimit = ranges.length === 1;
     const records: NativeStoredRecord[] = [];
@@ -925,7 +952,12 @@ async function performScan(
       );
     }
 
-    const sorted = sortAndLimitRecords(records, indexName, selectOptions);
+    const sorted = sortAndLimitRecords(
+      factory,
+      records,
+      physicalIndex.name,
+      selectOptions,
+    );
     const result = sorted.map(decodeStoredRecord);
 
     emitIdbDebug(debug, "scan", startedAt, {
@@ -1635,7 +1667,11 @@ export class IdbDriver implements DBDriver {
 
       const tx = this.db.transaction(storeName, "readonly");
       const store = tx.objectStore(storeName);
-      const expectedIndexes = new Set(Object.keys(tableDef.indexes));
+      const expectedIndexes = new Set(
+        getPersistentIndexPlan(tableDef).physicalIndexes.map(
+          (physicalIndex) => physicalIndex.name,
+        ),
+      );
       const actualIndexes = Array.from(store.indexNames);
       let storeNeedsUpgrade = actualIndexes.length !== expectedIndexes.size;
 
@@ -1875,7 +1911,11 @@ function applySchemaUpgrade(
     const store = db.objectStoreNames.contains(storeName)
       ? tx.objectStore(storeName)
       : db.createObjectStore(storeName, { keyPath: "id" });
-    const expectedIndexes = new Set(Object.keys(tableDef.indexes));
+    const expectedIndexes = new Set(
+      getPersistentIndexPlan(tableDef).physicalIndexes.map(
+        (physicalIndex) => physicalIndex.name,
+      ),
+    );
 
     for (const indexName of Array.from(store.indexNames)) {
       if (