diff --git a/.changeset/join-key-implicit-index.md b/.changeset/join-key-implicit-index.md new file mode 100644 index 0000000000..7e0a27547b --- /dev/null +++ b/.changeset/join-key-implicit-index.md @@ -0,0 +1,5 @@ +--- +"@tanstack/db": patch +--- + +Joins on a collection's primary key no longer require an explicit index: query optimization now falls back to a synthetic key index derived from `getKey` (when it reads a single property), so lazy joins on the key load only the matching rows instead of falling back to a full collection scan. diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index a23dec7648..415540d6f6 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -4,6 +4,7 @@ import { CollectionRequiresConfigError, CollectionRequiresSyncConfigError, } from '../errors' +import { createKeyIndexFromGetKey } from '../indexes/key-index.js' import { currentStateAsChanges } from './change-events' import { CollectionStateManager } from './state' @@ -13,6 +14,7 @@ import { CollectionSyncManager } from './sync' import { CollectionIndexesManager } from './indexes' import { CollectionMutationsManager } from './mutations' import { CollectionEventsManager } from './events.js' +import type { KeyIndex } from '../indexes/key-index.js' import type { CollectionSubscription } from './subscription' import type { AllCollectionEvents, @@ -311,6 +313,10 @@ export class CollectionImpl< private comparisonOpts: StringCollationConfig + // Lazily derived by the `keyIndex` getter; `null` records a failed + // derivation so introspection of `getKey` only ever runs once. + private _keyIndex: KeyIndex | null | undefined + /** * Creates a new Collection instance * @@ -678,6 +684,29 @@ export class CollectionImpl< return this._indexes.indexes } + /** + * Synthetic index over the collection's primary key, derived from + * `config.getKey` when it reads a single property (e.g. `(row) => row.id`). + * Query optimization consults it as a fallback when no user-created index + * matches the key field, so joins and lookups on the key don't require an + * explicit index. `undefined` when the key cannot be introspected (e.g. + * composite or computed keys). Note that `findIndexForField` conservatively + * skips this index for collections with a non-default `defaultStringCollation` + * (its compare options are the defaults), preserving the full-scan fallback + * there. + */ + get keyIndex(): KeyIndex | undefined { + if (this._keyIndex === undefined) { + this._keyIndex = + createKeyIndexFromGetKey( + this.config.getKey, + (key) => this.has(key), + () => this.size, + ) ?? null + } + return this._keyIndex ?? undefined + } + /** * Validates the data against the schema */ diff --git a/packages/db/src/indexes/key-index.ts b/packages/db/src/indexes/key-index.ts new file mode 100644 index 0000000000..ec43adff67 --- /dev/null +++ b/packages/db/src/indexes/key-index.ts @@ -0,0 +1,175 @@ +import { normalizeValue } from '../utils/comparison.js' +import { + createSingleRowRefProxy, + toExpression, +} from '../query/builder/ref-proxy.js' +import { BaseIndex } from './base-index.js' +import type { IndexOperation } from './base-index.js' +import type { BasicExpression } from '../query/ir.js' + +/** + * Synthetic read-only index over a collection's primary key. + * + * The collection's keyed state already provides O(1) key lookups, so this + * index stores nothing itself — `eq`/`in` lookups delegate straight to the + * collection. It exists so query optimization can serve equality lookups on + * the key field (most importantly lazy joins on the primary key) without the + * user having to create an explicit index. It is only consulted as a fallback + * when no user-created index matches the field. + */ +export class KeyIndex< + TKey extends string | number = string | number, +> extends BaseIndex { + public readonly supportedOperations = new Set([`eq`, `in`]) + + private hasKey: (key: TKey) => boolean + private getKeyCount: () => number + + constructor( + expression: BasicExpression, + hasKey: (key: TKey) => boolean, + getKeyCount: () => number, + ) { + // Never registered in collection.indexes — the negative id keeps it + // distinct from user-created index ids. + super(-1, expression, `key`) + this.hasKey = hasKey + this.getKeyCount = getKeyCount + } + + protected initialize(): void {} + + // The collection state is the backing store, so there is nothing to maintain. + add(): void {} + remove(): void {} + update(): void {} + build(): void {} + clear(): void {} + + lookup(operation: IndexOperation, value: any): Set { + const startTime = performance.now() + + let result: Set + switch (operation) { + case `eq`: + result = this.equalityLookup(value) + break + case `in`: + result = this.inArrayLookup(value) + break + default: + throw new Error(`Operation ${operation} not supported by KeyIndex`) + } + + this.trackLookup(startTime) + return result + } + + equalityLookup(value: any): Set { + // Normalize like BasicIndex does, so a lookup value behaves the same + // against the key field as it would against a user-created index. + const normalizedValue = normalizeValue(value) + return this.hasKey(normalizedValue) + ? new Set([normalizedValue as TKey]) + : new Set() + } + + inArrayLookup(values: Array): Set { + const result = new Set() + for (const value of values) { + const normalizedValue = normalizeValue(value) + if (this.hasKey(normalizedValue)) { + result.add(normalizedValue as TKey) + } + } + return result + } + + get keyCount(): number { + return this.getKeyCount() + } + + get supportsRangeOptimization(): boolean { + return false + } + + // The remaining IndexInterface members are mandated by BaseIndex's abstract + // contract but unreachable in practice: `supports()` reports only eq/in, so + // the optimizer and order-by never route range or ordered access here. + // Throwing (rather than returning empty results) keeps any future call path + // that does reach them loudly wrong instead of silently dropping rows. + private unsupported(feature: string): never { + throw new Error(`KeyIndex does not support ${feature}`) + } + + rangeQuery(): Set { + return this.unsupported(`range queries`) + } + + rangeQueryReversed(): Set { + return this.unsupported(`range queries`) + } + + take(): Array { + return this.unsupported(`ordered access`) + } + + takeFromStart(): Array { + return this.unsupported(`ordered access`) + } + + takeReversed(): Array { + return this.unsupported(`ordered access`) + } + + takeReversedFromEnd(): Array { + return this.unsupported(`ordered access`) + } + + get orderedEntriesArray(): Array<[any, Set]> { + return this.unsupported(`ordered access`) + } + + get orderedEntriesArrayReversed(): Array<[any, Set]> { + return this.unsupported(`ordered access`) + } + + get indexedKeysSet(): Set { + return this.unsupported(`key enumeration`) + } + + get valueMapData(): Map> { + return this.unsupported(`value enumeration`) + } +} + +/** + * Derives a {@link KeyIndex} from a collection's `getKey` function. + * + * `getKey` is called once with a ref proxy: when it reads a single property + * (e.g. `(row) => row.id`), that access is captured as the key field path — + * the same introspection `createIndex` uses for its index callback. Anything + * else — composite keys, computed keys, or a `getKey` that throws on the + * proxy — returns `undefined` and the collection simply has no implicit key + * index. + */ +export function createKeyIndexFromGetKey< + T extends object, + TKey extends string | number, +>( + getKey: (item: T) => TKey, + hasKey: (key: TKey) => boolean, + getKeyCount: () => number, +): KeyIndex | undefined { + let expression: BasicExpression + try { + const row = createSingleRowRefProxy() + expression = toExpression(getKey(row as unknown as T)) + } catch { + return undefined + } + if (expression.type !== `ref` || expression.path.length === 0) { + return undefined + } + return new KeyIndex(expression, hasKey, getKeyCount) +} diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index bae05a943f..e30b7027d8 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -17,7 +17,7 @@ export interface CollectionLike< TKey extends string | number = string | number, > extends Pick< Collection, - `get` | `has` | `entries` | `indexes` | `id` | `compareOptions` + `get` | `has` | `entries` | `indexes` | `keyIndex` | `id` | `compareOptions` > {} /** diff --git a/packages/db/src/utils/index-optimization.ts b/packages/db/src/utils/index-optimization.ts index 5a52a5ec54..d72576fce0 100644 --- a/packages/db/src/utils/index-optimization.ts +++ b/packages/db/src/utils/index-optimization.ts @@ -20,7 +20,7 @@ import { ReverseIndex } from '../indexes/reverse-index.js' import { hasVirtualPropPath } from '../virtual-props.js' import { makeComparator } from './comparison.js' import type { CompareOptions } from '../query/builder/types.js' -import type { IndexInterface, IndexOperation } from '../indexes/base-index.js' +import type { IndexInterface } from '../indexes/base-index.js' import type { BasicExpression } from '../query/ir.js' import type { CollectionLike } from '../types.js' @@ -40,7 +40,12 @@ export interface OptimizationResult { } /** - * Finds an index that matches a given field path + * Finds an index that matches a given field path. + * + * The returned index may be capability-limited (e.g. the implicit primary-key + * index only serves `eq`/`in`), so callers must check `supports(operation)` + * before range or ordered access — as all callers in this module and in + * order-by/change-events do. */ export function findIndexForField( collection: CollectionLike, @@ -66,6 +71,22 @@ export function findIndexForField( return index } } + + // Fall back to the collection's implicit primary-key index, so equality + // lookups on the key field (e.g. joins on the primary key) work without a + // user-created index. Checked last so an explicit index always wins. + const keyIndex = collection.keyIndex + if ( + keyIndex && + keyIndex.matchesField(fieldPath) && + keyIndex.matchesCompareOptions(compareOpts) + ) { + if (!keyIndex.matchesDirection(compareOpts.direction)) { + return new ReverseIndex(keyIndex) + } + return keyIndex + } + return undefined } @@ -471,37 +492,39 @@ function optimizeCompoundRangeQuery< } /** - * Optimizes simple comparison expressions (eq, gt, gte, lt, lte) + * A binary comparison normalized to `field op value` form. For `value op field` + * expressions the range operation is flipped (`gt`↔`lt`, `gte`↔`lte`; `eq` is + * symmetric), so `operation` is always the operation an index lookup serves. */ -function optimizeSimpleComparison< - T extends object, - TKey extends string | number, ->( +interface NormalizedComparison { + fieldPath: Array + queryValue: unknown + operation: `eq` | `gt` | `gte` | `lt` | `lte` +} + +/** + * Normalizes a comparison expression to `field op value` form, or returns + * `undefined` when it is not a binary comparison between a field reference + * and a literal value. Both the optimizer and the `canOptimize*` predicates + * classify through this helper so their decisions cannot drift apart. + */ +function normalizeSimpleComparison( expression: BasicExpression, - collection: CollectionLike, -): OptimizationResult { +): NormalizedComparison | undefined { if (expression.type !== `func` || expression.args.length !== 2) { - return { canOptimize: false, matchingKeys: new Set(), isExact: false } + return undefined } const leftArg = expression.args[0]! const rightArg = expression.args[1]! - - // Check both directions: field op value AND value op field - let fieldArg: BasicExpression | null = null - let valueArg: BasicExpression | null = null - let operation = expression.name as `eq` | `gt` | `gte` | `lt` | `lte` + let operation = expression.name as NormalizedComparison[`operation`] if (leftArg.type === `ref` && rightArg.type === `val`) { - // field op value - fieldArg = leftArg - valueArg = rightArg - } else if (leftArg.type === `val` && rightArg.type === `ref`) { - // value op field - need to flip the operation - fieldArg = rightArg - valueArg = leftArg - - // Flip the operation for reverse comparison + return { fieldPath: leftArg.path, queryValue: rightArg.value, operation } + } + + if (leftArg.type === `val` && rightArg.type === `ref`) { + // value op field - flip the range operation for the reverse comparison switch (operation) { case `gt`: operation = `lt` @@ -517,57 +540,65 @@ function optimizeSimpleComparison< break // eq stays the same } + return { fieldPath: rightArg.path, queryValue: leftArg.value, operation } } - if (fieldArg && valueArg) { - const fieldPath = (fieldArg as any).path - const index = findIndexForField(collection, fieldPath) - - if (index) { - const queryValue = (valueArg as any).value - - // Map operation to IndexOperation enum - const indexOperation = operation as IndexOperation + return undefined +} - // Check if the index supports this operation - if (!index.supports(indexOperation)) { - return { canOptimize: false, matchingKeys: new Set(), isExact: false } - } +/** + * Optimizes simple comparison expressions (eq, gt, gte, lt, lte) + */ +function optimizeSimpleComparison< + T extends object, + TKey extends string | number, +>( + expression: BasicExpression, + collection: CollectionLike, +): OptimizationResult { + const normalized = normalizeSimpleComparison(expression) + if (!normalized) { + return { canOptimize: false, matchingKeys: new Set(), isExact: false } + } + const { fieldPath, queryValue, operation } = normalized - // A range op can only use the index when the operand's domain orders the - // same way the index does and the index supports trustworthy traversal. - // Otherwise the index may omit matching rows, which re-filtering cannot - // recover, so fall back to a full scan. - if ( - (operation === `gt` || - operation === `gte` || - operation === `lt` || - operation === `lte`) && - !canRangeOptimize(queryValue, index, collection) - ) { - return { canOptimize: false, matchingKeys: new Set(), isExact: false } - } + const index = findIndexForField(collection, fieldPath) + if (index) { + // Check if the index supports this operation + if (!index.supports(operation)) { + return { canOptimize: false, matchingKeys: new Set(), isExact: false } + } - const matchingKeys = index.lookup(indexOperation, queryValue) - - // A comparison against a nullish value is never true, but BTree indexes - // store and return rows with nullish keys (they sort to the nulls end). - // Determine whether the index result is exact or a superset that the - // caller must re-filter: - // - eq/gt/gte: a nullish query value matches nothing while the index - // still returns nullish-keyed rows -> inexact. A non-nullish lower - // bound (gt/gte) excludes those bottom-sorted rows, so they stay exact. - // - lt/lte: the open lower bound always includes nullish-keyed rows, - // so the result is conservatively inexact. - // NaN/invalid Dates are exact here: under PostgreSQL float semantics the - // evaluator and the index agree on them (equal to self, greatest). - const isExact = - operation === `lt` || operation === `lte` - ? false - : isExactComparisonValue(queryValue) - - return { canOptimize: true, matchingKeys, isExact } + // A range op can only use the index when the operand's domain orders the + // same way the index does and the index supports trustworthy traversal. + // Otherwise the index may omit matching rows, which re-filtering cannot + // recover, so fall back to a full scan. + if ( + operation !== `eq` && + !canRangeOptimize(queryValue, index, collection) + ) { + return { canOptimize: false, matchingKeys: new Set(), isExact: false } } + + const matchingKeys = index.lookup(operation, queryValue) + + // A comparison against a nullish value is never true, but BTree indexes + // store and return rows with nullish keys (they sort to the nulls end). + // Determine whether the index result is exact or a superset that the + // caller must re-filter: + // - eq/gt/gte: a nullish query value matches nothing while the index + // still returns nullish-keyed rows -> inexact. A non-nullish lower + // bound (gt/gte) excludes those bottom-sorted rows, so they stay exact. + // - lt/lte: the open lower bound always includes nullish-keyed rows, + // so the result is conservatively inexact. + // NaN/invalid Dates are exact here: under PostgreSQL float semantics the + // evaluator and the index agree on them (equal to self, greatest). + const isExact = + operation === `lt` || operation === `lte` + ? false + : isExactComparisonValue(queryValue) + + return { canOptimize: true, matchingKeys, isExact } } return { canOptimize: false, matchingKeys: new Set(), isExact: false } @@ -580,28 +611,29 @@ function canOptimizeSimpleComparison< T extends object, TKey extends string | number, >(expression: BasicExpression, collection: CollectionLike): boolean { - if (expression.type !== `func` || expression.args.length !== 2) { + const normalized = normalizeSimpleComparison(expression) + if (!normalized) { return false } + const { fieldPath, queryValue, operation } = normalized - const leftArg = expression.args[0]! - const rightArg = expression.args[1]! - - // Check both directions: field op value AND value op field - let fieldPath: Array | null = null - - if (leftArg.type === `ref` && rightArg.type === `val`) { - fieldPath = (leftArg as any).path - } else if (leftArg.type === `val` && rightArg.type === `ref`) { - fieldPath = (rightArg as any).path + const index = findIndexForField(collection, fieldPath) + if (index === undefined) { + return false } - if (fieldPath) { - const index = findIndexForField(collection, fieldPath) - return index !== undefined + // Mirror optimizeSimpleComparison's gates: the index must support the + // normalized operation (e.g. the implicit key index only serves eq/in), + // and a range op additionally requires trustworthy index traversal for + // the operand's domain. + if (!index.supports(operation)) { + return false + } + if (operation !== `eq` && !canRangeOptimize(queryValue, index, collection)) { + return false } - return false + return true } /** @@ -805,7 +837,9 @@ function canOptimizeInArrayExpression< ) { const fieldPath = (fieldArg as any).path const index = findIndexForField(collection, fieldPath) - return index !== undefined + // Mirror optimizeInArrayExpression's gate: IN is served either natively + // or via per-value equality lookups. + return index !== undefined && (index.supports(`in`) || index.supports(`eq`)) } return false diff --git a/packages/db/tests/key-index.test.ts b/packages/db/tests/key-index.test.ts new file mode 100644 index 0000000000..a1e6140f51 --- /dev/null +++ b/packages/db/tests/key-index.test.ts @@ -0,0 +1,313 @@ +import { describe, expect, it } from 'vitest' +import { createCollection } from '../src/collection/index.js' +import { KeyIndex, createKeyIndexFromGetKey } from '../src/indexes/key-index.js' +import { BasicIndex } from '../src/indexes/basic-index.js' +import { + canOptimizeExpression, + findIndexForField, + optimizeExpressionWithIndexes, +} from '../src/utils/index-optimization.js' +import { Func, PropRef, Value } from '../src/query/ir.js' +import { mockSyncCollectionOptions } from './utils.js' +import type { IndexOperation } from '../src/indexes/base-index.js' + +type Item = { + id: string + category: string +} + +const sampleItems: Array = [ + { id: `a`, category: `one` }, + { id: `b`, category: `one` }, + { id: `c`, category: `two` }, +] + +function makeCollection(getKey: (item: Item) => string) { + return createCollection( + mockSyncCollectionOptions({ + id: `key-index-test`, + getKey, + initialData: sampleItems, + }), + ) +} + +describe(`createKeyIndexFromGetKey`, () => { + it(`derives an index from a single property access`, () => { + const keys = new Set([`a`, `b`]) + const index = createKeyIndexFromGetKey( + (item) => item.id, + (key) => keys.has(key), + () => keys.size, + ) + + expect(index).toBeInstanceOf(KeyIndex) + expect(index!.matchesField([`id`])).toBe(true) + expect(index!.matchesField([`category`])).toBe(false) + expect(index!.keyCount).toBe(2) + }) + + it(`returns undefined for a composite key`, () => { + const index = createKeyIndexFromGetKey( + (item) => `${item.id}:${item.category}`, + () => true, + () => 0, + ) + + expect(index).toBeUndefined() + }) + + it(`returns undefined when getKey does not read a property`, () => { + const index = createKeyIndexFromGetKey( + (item) => item as unknown as string, + () => true, + () => 0, + ) + + expect(index).toBeUndefined() + }) + + it(`returns undefined when getKey throws on the introspection proxy`, () => { + const index = createKeyIndexFromGetKey( + () => { + throw new Error(`boom`) + }, + () => true, + () => 0, + ) + + expect(index).toBeUndefined() + }) +}) + +describe(`KeyIndex lookups`, () => { + const keys = new Set([`a`, `b`]) + const index = createKeyIndexFromGetKey( + (item) => item.id, + (key) => keys.has(key), + () => keys.size, + )! + + it(`supports only eq and in`, () => { + expect(index.supports(`eq`)).toBe(true) + expect(index.supports(`in`)).toBe(true) + expect(index.supports(`gt`)).toBe(false) + expect(index.supports(`lte`)).toBe(false) + expect(index.supportsRangeOptimization).toBe(false) + }) + + it(`resolves eq lookups through the key set`, () => { + expect(index.lookup(`eq`, `a`)).toEqual(new Set([`a`])) + expect(index.lookup(`eq`, `missing`)).toEqual(new Set()) + expect(index.lookup(`eq`, null)).toEqual(new Set()) + }) + + it(`resolves in lookups by filtering to present keys`, () => { + expect(index.lookup(`in`, [`a`, `b`, `missing`])).toEqual( + new Set([`a`, `b`]), + ) + expect(index.lookup(`in`, [])).toEqual(new Set()) + }) + + it(`rejects unsupported operations`, () => { + expect(() => index.lookup(`gt`, `a`)).toThrow( + `Operation gt not supported by KeyIndex`, + ) + }) +}) + +describe(`collection.keyIndex`, () => { + it(`is derived for a plain property getKey and reflects live state`, () => { + const collection = makeCollection((item) => item.id) + const keyIndex = collection.keyIndex + + expect(keyIndex).toBeInstanceOf(KeyIndex) + expect(keyIndex!.matchesField([`id`])).toBe(true) + expect(keyIndex!.lookup(`eq`, `a`)).toEqual(new Set([`a`])) + expect(keyIndex!.lookup(`in`, [`a`, `c`, `zzz`])).toEqual( + new Set([`a`, `c`]), + ) + + collection.utils.begin() + collection.utils.write({ + type: `insert`, + value: { id: `d`, category: `two` }, + }) + collection.utils.commit() + + expect(keyIndex!.lookup(`eq`, `d`)).toEqual(new Set([`d`])) + }) + + it(`is undefined for a composite getKey`, () => { + const collection = makeCollection((item) => `${item.id}:${item.category}`) + + expect(collection.keyIndex).toBeUndefined() + }) +}) + +describe(`findIndexForField key-index fallback`, () => { + it(`serves the key field when no explicit index exists`, () => { + const collection = makeCollection((item) => item.id) + + const index = findIndexForField(collection, [`id`]) + + expect(index).toBe(collection.keyIndex) + expect(index!.lookup(`eq`, `b`)).toEqual(new Set([`b`])) + }) + + it(`prefers an explicit index on the key field`, () => { + const collection = makeCollection((item) => item.id) + collection.createIndex((row) => row.id, { indexType: BasicIndex }) + + const index = findIndexForField(collection, [`id`]) + + expect(index).toBeInstanceOf(BasicIndex) + }) + + it(`does not serve non-key fields`, () => { + const collection = makeCollection((item) => item.id) + + expect(findIndexForField(collection, [`category`])).toBeUndefined() + }) + + it(`does not serve collections with a composite key`, () => { + const collection = makeCollection((item) => `${item.id}:${item.category}`) + + expect(findIndexForField(collection, [`id`])).toBeUndefined() + }) + + it(`is conservatively skipped for collections with a custom string collation`, () => { + const collection = createCollection({ + getKey: (item) => item.id, + defaultStringCollation: { stringSort: `lexical` }, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + for (const item of sampleItems) { + write({ type: `insert`, value: item }) + } + commit() + markReady() + }, + }, + }) + + // The key index itself is derivable, but its compare options are the + // defaults, so lookups under a custom collation fall back to a full scan. + expect(collection.keyIndex).toBeInstanceOf(KeyIndex) + expect(findIndexForField(collection, [`id`])).toBeUndefined() + }) +}) + +describe(`canOptimizeExpression with only a key index`, () => { + const collection = makeCollection((item) => item.id) + + it(`reports eq and in on the key field as optimizable`, () => { + expect( + canOptimizeExpression( + new Func(`eq`, [new PropRef([`id`]), new Value(`a`)]), + collection, + ), + ).toBe(true) + expect( + canOptimizeExpression( + new Func(`in`, [new PropRef([`id`]), new Value([`a`, `b`])]), + collection, + ), + ).toBe(true) + }) + + it(`reports range operations on the key field as not optimizable`, () => { + // The key index exists but only supports eq/in, so the predicate must + // agree with what optimization would actually do. + expect( + canOptimizeExpression( + new Func(`gt`, [new PropRef([`id`]), new Value(`a`)]), + collection, + ), + ).toBe(false) + }) +}) + +describe(`canOptimizeExpression agrees with optimizeExpressionWithIndexes`, () => { + it(`rejects locale-ordered string ranges on a range-capable index`, () => { + const collection = makeCollection((item) => item.id) + collection.createIndex((row) => row.category, { indexType: BasicIndex }) + + // Default collections sort strings by locale, which diverges from the + // WHERE evaluator's code-point comparison, so a string range cannot be + // served by the index — and the predicate must say so too. + const rangeExpr = new Func(`gt`, [ + new PropRef([`category`]), + new Value(`m`), + ]) + expect( + optimizeExpressionWithIndexes(rangeExpr, collection).canOptimize, + ).toBe(false) + expect(canOptimizeExpression(rangeExpr, collection)).toBe(false) + + // The flipped operand form must classify the same way. + const flippedExpr = new Func(`gt`, [ + new Value(`m`), + new PropRef([`category`]), + ]) + expect( + optimizeExpressionWithIndexes(flippedExpr, collection).canOptimize, + ).toBe(false) + expect(canOptimizeExpression(flippedExpr, collection)).toBe(false) + + // Equality is unaffected by collation and stays optimizable. + const eqExpr = new Func(`eq`, [new PropRef([`category`]), new Value(`one`)]) + expect(optimizeExpressionWithIndexes(eqExpr, collection).canOptimize).toBe( + true, + ) + expect(canOptimizeExpression(eqExpr, collection)).toBe(true) + }) + + it(`normalizes flipped operands before consulting index capabilities`, () => { + // An index that serves upper-bound ranges only: `5 > score` normalizes to + // `score < 5`, so both the optimizer and the predicate must consult + // supports(`lt`), not the expression's literal `gt` — otherwise the two + // disagree on either flipped form. + class UpperBoundOnlyIndex< + TKey extends string | number = string | number, + > extends BasicIndex { + public readonly supportedOperations = new Set([ + `eq`, + `lt`, + `lte`, + ]) + } + + const collection = createCollection( + mockSyncCollectionOptions<{ id: string; score: number }>({ + id: `key-index-flip-test`, + getKey: (item) => item.id, + initialData: [ + { id: `a`, score: 1 }, + { id: `b`, score: 7 }, + ], + }), + ) + collection.createIndex((row) => row.score, { + indexType: UpperBoundOnlyIndex, + }) + + // `5 > score` means `score < 5`: served by the index, and the predicate + // must agree. + const flippedLt = new Func(`gt`, [new Value(5), new PropRef([`score`])]) + const optimizedLt = optimizeExpressionWithIndexes(flippedLt, collection) + expect(optimizedLt.canOptimize).toBe(true) + expect(optimizedLt.matchingKeys).toEqual(new Set([`a`])) + expect(canOptimizeExpression(flippedLt, collection)).toBe(true) + + // `5 < score` means `score > 5`: not served (no gt support), and the + // predicate must agree. + const flippedGt = new Func(`lt`, [new Value(5), new PropRef([`score`])]) + expect( + optimizeExpressionWithIndexes(flippedGt, collection).canOptimize, + ).toBe(false) + expect(canOptimizeExpression(flippedGt, collection)).toBe(false) + }) +}) diff --git a/packages/db/tests/query/indexes.test.ts b/packages/db/tests/query/indexes.test.ts index 6abc065f63..f033519d05 100644 --- a/packages/db/tests/query/indexes.test.ts +++ b/packages/db/tests/query/indexes.test.ts @@ -89,6 +89,24 @@ function createIndexUsageTracker(collection: any): { configurable: true, }) + // The implicit primary-key index lives outside `collection.indexes`, so + // patch its lookup separately to observe key-index-served queries. + const keyIndex = collection.keyIndex + const originalKeyIndexLookup = keyIndex?.lookup + if (keyIndex) { + keyIndex.lookup = function (operation: string, value: any) { + stats.rangeQueryCalls++ + stats.indexesUsed.push(`keyIndex`) + stats.queriesExecuted.push({ + type: `index`, + operation, + field: keyIndex.expression?.path?.join(`.`), + value, + }) + return originalKeyIndexLookup.call(this, operation, value) + } + } + // Track full scan calls (entries() iteration) const originalEntries = collection.entries collection.entries = function* () { @@ -125,6 +143,10 @@ function createIndexUsageTracker(collection: any): { } } + if (keyIndex && originalKeyIndexLookup) { + keyIndex.lookup = originalKeyIndexLookup + } + collection.entries = originalEntries } @@ -684,12 +706,14 @@ describe(`Query Index Optimization`, () => { // The WHERE clause on the non-nullable (left) side uses its index. // The WHERE clause on the nullable (right) side of the LEFT JOIN is NOT - // pushed down to avoid changing join semantics, so the right side does a full scan. + // pushed down to avoid changing join semantics, but the join key on the + // right side is its primary key, so the lazy join loads through the + // implicit key index instead of a full scan. expectIndexUsage(combinedStats, { shouldUseIndex: true, - shouldUseFullScan: true, - indexCallCount: 1, // Only item.status='active' uses index (non-nullable side) - fullScanCallCount: 1, // other collection does full scan (nullable side) + shouldUseFullScan: false, + indexCallCount: 2, // item.status='active' + join keys via the key index + fullScanCallCount: 0, }) } finally { tracker1.restore() @@ -804,7 +828,7 @@ describe(`Query Index Optimization`, () => { } }) - it(`should not optimize inner join if biggest collection has no index on the join key`, async () => { + it(`should optimize inner join via the implicit key index when the biggest collection has no explicit index on the join key`, async () => { // Create a second collection for the join with its own index const secondCollection = createCollection({ getKey: (item) => item.id2, @@ -878,7 +902,9 @@ describe(`Query Index Optimization`, () => { }, ]) - // We should have done an index lookup on the 1st collection to find active items + // We should have done an index lookup on the 1st collection to find + // active items, and the join keys are served by the implicit + // primary-key index instead of a full scan. expect(tracker1.stats.queriesExecuted).toEqual([ { type: `index`, @@ -886,6 +912,12 @@ describe(`Query Index Optimization`, () => { field: `status`, value: `active`, }, + { + type: `index`, + operation: `in`, + field: `id`, + value: [`1`], + }, ]) } finally { tracker1.restore() @@ -1018,9 +1050,11 @@ describe(`Query Index Optimization`, () => { }) it(`should not optimize left join if right collection has no index on the join key`, async () => { - // Create a second collection for the join with its own index + // Create a second collection for the join with its own index. + // The key is computed so the join key `id2` is not served by the + // implicit primary-key index and the collection truly has no index on it. const secondCollection = createCollection({ - getKey: (item) => item.id2, + getKey: (item) => `computed-${item.id2}`, autoIndex: `off`, startSync: true, sync: { @@ -1208,7 +1242,45 @@ describe(`Query Index Optimization`, () => { }) it(`should not optimize right join if left collection has no index on the join key`, async () => { - // Create a second collection for the join with its own index + // Create a local left collection with a computed key, so the join key + // `id` is not served by the implicit primary-key index and the + // collection truly has no index on it. + const firstCollection = createCollection({ + getKey: (item) => `computed-${item.id}`, + autoIndex: `off`, + startSync: true, + sync: { + sync: ({ begin, write, commit }) => { + begin() + write({ + type: `insert`, + value: { + id: `1`, + name: `Alice`, + age: 25, + status: `active`, + score: 95, + createdAt: new Date(`2023-01-01`), + }, + }) + write({ + type: `insert`, + value: { + id: `2`, + name: `Bob`, + age: 30, + status: `inactive`, + score: 80, + createdAt: new Date(`2023-01-02`), + }, + }) + commit() + }, + }, + }) + + await firstCollection.stateWhenReady() + const secondCollection = createCollection({ getKey: (item) => item.id2, autoIndex: `off`, @@ -1244,14 +1316,14 @@ describe(`Query Index Optimization`, () => { await secondCollection.stateWhenReady() // Track both collections - const tracker1 = createIndexUsageTracker(collection) + const tracker1 = createIndexUsageTracker(firstCollection) const tracker2 = createIndexUsageTracker(secondCollection) try { const liveQuery = createLiveQueryCollection({ query: (q: any) => q - .from({ item: collection }) + .from({ item: firstCollection }) .join( { other: secondCollection }, ({ item, other }: any) => eq(item.id, other.id2), diff --git a/packages/db/tests/query/join-key-index.test.ts b/packages/db/tests/query/join-key-index.test.ts new file mode 100644 index 0000000000..5897c276f4 --- /dev/null +++ b/packages/db/tests/query/join-key-index.test.ts @@ -0,0 +1,180 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest' +import { createLiveQueryCollection, eq } from '../../src/query/index.js' +import { createCollection } from '../../src/collection/index.js' +import { mockSyncCollectionOptions, stripVirtualProps } from '../utils.js' + +/** + * Lazy joins on a collection's primary key should load through the implicit + * key index instead of falling back to a full collection scan, without the + * user having to create an explicit index on the key field. + */ + +type Team = { id: string; name: string } +type Member = { id: string; teamId: string } + +const sampleTeams: Array = [ + { id: `t1`, name: `Team One` }, + { id: `t2`, name: `Team Two` }, + { id: `t3`, name: `Team Three` }, +] + +const sampleMembers: Array = [ + { id: `m1`, teamId: `t1` }, + { id: `m2`, teamId: `t1` }, + { id: `m3`, teamId: `t2` }, +] + +describe(`lazy join on the primary key without an explicit index`, () => { + let warnSpy: ReturnType + + beforeEach(() => { + warnSpy = vi.spyOn(console, `warn`).mockImplementation(() => {}) + }) + + afterEach(() => { + warnSpy.mockRestore() + }) + + const indexWarnings = () => + warnSpy.mock.calls + .map((c) => String(c[0])) + .filter((m) => m.includes(`Join requires an index`)) + + const makeTeamsCollection = () => + createCollection( + mockSyncCollectionOptions({ + id: `key-join-teams`, + getKey: (r) => r.id, + autoIndex: `off`, + initialData: sampleTeams, + }), + ) + + const makeMembersCollection = () => + createCollection( + mockSyncCollectionOptions({ + id: `key-join-members`, + getKey: (r) => r.id, + autoIndex: `off`, + initialData: sampleMembers, + }), + ) + + test(`loads through the key index and does not warn`, () => { + const teams = makeTeamsCollection() + const members = makeMembersCollection() + const keyLookup = vi.spyOn(teams.keyIndex!, `lookup`) + + const joined = createLiveQueryCollection({ + startSync: true, + query: (q) => + q + .from({ member: members }) + .join({ team: teams }, ({ member, team }) => + eq(team.id, member.teamId), + ) + .select(({ member, team }) => ({ + memberId: member.id, + teamName: team.name, + })), + }) + + expect( + joined.toArray + .map(stripVirtualProps) + .sort((a, b) => a.memberId.localeCompare(b.memberId)), + ).toEqual([ + { memberId: `m1`, teamName: `Team One` }, + { memberId: `m2`, teamName: `Team One` }, + { memberId: `m3`, teamName: `Team Two` }, + ]) + + expect(indexWarnings()).toEqual([]) + + // The load must actually have gone through the key index, not a scan + // that merely stopped warning. + expect(keyLookup).toHaveBeenCalledWith( + `in`, + expect.arrayContaining([`t1`, `t2`]), + ) + }) + + test(`serves join keys that appear after the initial load`, () => { + const teams = makeTeamsCollection() + const members = makeMembersCollection() + const keyLookup = vi.spyOn(teams.keyIndex!, `lookup`) + + const joined = createLiveQueryCollection({ + startSync: true, + query: (q) => + q + .from({ member: members }) + .join({ team: teams }, ({ member, team }) => + eq(team.id, member.teamId), + ) + .select(({ member, team }) => ({ + memberId: member.id, + teamName: team.name, + })), + }) + + // `t3` is not referenced by the initial members, so it was not part of + // the initial lazy snapshot. A new member pointing at it must trigger an + // index-served load for that key. + members.utils.begin() + members.utils.write({ + type: `insert`, + value: { id: `m4`, teamId: `t3` }, + }) + members.utils.commit() + + expect( + joined.toArray.map(stripVirtualProps).find((r) => r.memberId === `m4`), + ).toEqual({ + memberId: `m4`, + teamName: `Team Three`, + }) + + expect(indexWarnings()).toEqual([]) + + // The late-arriving key must have been loaded through the key index. + const t3Lookups = keyLookup.mock.calls.filter( + ([operation, values]) => + operation === `in` && Array.isArray(values) && values.includes(`t3`), + ) + expect(t3Lookups.length).toBeGreaterThan(0) + }) + + test(`still warns when the joined collection has a computed key`, () => { + const teams = createCollection( + mockSyncCollectionOptions({ + id: `key-join-teams-computed`, + // Computed key: cannot be introspected into a key index. + getKey: (r) => `team:${r.id}`, + autoIndex: `off`, + initialData: sampleTeams, + }), + ) + const members = makeMembersCollection() + + const joined = createLiveQueryCollection({ + startSync: true, + query: (q) => + q + .from({ member: members }) + .join({ team: teams }, ({ member, team }) => + eq(team.id, member.teamId), + ) + .select(({ member, team }) => ({ + memberId: member.id, + teamName: team.name, + })), + }) + + // Data still flows via the full-load fallback. + expect(joined.toArray).toHaveLength(3) + expect( + indexWarnings().filter((m) => m.includes(`key-join-teams-computed`)), + ).not.toEqual([]) + }) +})