From 6f5a70d275a8409366fc15a7c4e6220a7383457b Mon Sep 17 00:00:00 2001 From: balanced Date: Tue, 18 Aug 2026 16:39:04 +0300 Subject: [PATCH 1/5] fix: serve joins on the primary key via an implicit key index --- .changeset/join-key-implicit-index.md | 5 + packages/db/src/collection/index.ts | 29 +++ packages/db/src/indexes/key-index.ts | 168 +++++++++++++++ packages/db/src/types.ts | 2 +- packages/db/src/utils/index-optimization.ts | 16 ++ packages/db/tests/key-index.test.ts | 195 ++++++++++++++++++ packages/db/tests/query/indexes.test.ts | 94 ++++++++- .../db/tests/query/join-key-index.test.ts | 164 +++++++++++++++ 8 files changed, 661 insertions(+), 12 deletions(-) create mode 100644 .changeset/join-key-implicit-index.md create mode 100644 packages/db/src/indexes/key-index.ts create mode 100644 packages/db/tests/key-index.test.ts create mode 100644 packages/db/tests/query/join-key-index.test.ts diff --git a/.changeset/join-key-implicit-index.md b/.changeset/join-key-implicit-index.md new file mode 100644 index 0000000000..7e0a27547b --- /dev/null +++ b/.changeset/join-key-implicit-index.md @@ -0,0 +1,5 @@ +--- +"@tanstack/db": patch +--- + +Joins on a collection's primary key no longer require an explicit index: query optimization now falls back to a synthetic key index derived from `getKey` (when it reads a single property), so lazy joins on the key load only the matching rows instead of falling back to a full collection scan. diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index a23dec7648..415540d6f6 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -4,6 +4,7 @@ import { CollectionRequiresConfigError, CollectionRequiresSyncConfigError, } from '../errors' +import { createKeyIndexFromGetKey } from '../indexes/key-index.js' import { currentStateAsChanges } from './change-events' import { CollectionStateManager } from './state' @@ -13,6 +14,7 @@ import { CollectionSyncManager } from './sync' import { CollectionIndexesManager } from './indexes' import { CollectionMutationsManager } from './mutations' import { CollectionEventsManager } from './events.js' +import type { KeyIndex } from '../indexes/key-index.js' import type { CollectionSubscription } from './subscription' import type { AllCollectionEvents, @@ -311,6 +313,10 @@ export class CollectionImpl< private comparisonOpts: StringCollationConfig + // Lazily derived by the `keyIndex` getter; `null` records a failed + // derivation so introspection of `getKey` only ever runs once. + private _keyIndex: KeyIndex | null | undefined + /** * Creates a new Collection instance * @@ -678,6 +684,29 @@ export class CollectionImpl< return this._indexes.indexes } + /** + * Synthetic index over the collection's primary key, derived from + * `config.getKey` when it reads a single property (e.g. `(row) => row.id`). + * Query optimization consults it as a fallback when no user-created index + * matches the key field, so joins and lookups on the key don't require an + * explicit index. `undefined` when the key cannot be introspected (e.g. + * composite or computed keys). Note that `findIndexForField` conservatively + * skips this index for collections with a non-default `defaultStringCollation` + * (its compare options are the defaults), preserving the full-scan fallback + * there. + */ + get keyIndex(): KeyIndex | undefined { + if (this._keyIndex === undefined) { + this._keyIndex = + createKeyIndexFromGetKey( + this.config.getKey, + (key) => this.has(key), + () => this.size, + ) ?? null + } + return this._keyIndex ?? undefined + } + /** * Validates the data against the schema */ diff --git a/packages/db/src/indexes/key-index.ts b/packages/db/src/indexes/key-index.ts new file mode 100644 index 0000000000..c31f14790b --- /dev/null +++ b/packages/db/src/indexes/key-index.ts @@ -0,0 +1,168 @@ +import { normalizeValue } from '../utils/comparison.js' +import { + createSingleRowRefProxy, + toExpression, +} from '../query/builder/ref-proxy.js' +import { BaseIndex } from './base-index.js' +import type { IndexOperation } from './base-index.js' +import type { BasicExpression } from '../query/ir.js' + +/** + * Synthetic read-only index over a collection's primary key. + * + * The collection's keyed state already provides O(1) key lookups, so this + * index stores nothing itself — `eq`/`in` lookups delegate straight to the + * collection. It exists so query optimization can serve equality lookups on + * the key field (most importantly lazy joins on the primary key) without the + * user having to create an explicit index. It is only consulted as a fallback + * when no user-created index matches the field. + */ +export class KeyIndex< + TKey extends string | number = string | number, +> extends BaseIndex { + public readonly supportedOperations = new Set([`eq`, `in`]) + + private hasKey: (key: TKey) => boolean + private getKeyCount: () => number + + constructor( + expression: BasicExpression, + hasKey: (key: TKey) => boolean, + getKeyCount: () => number, + ) { + // Never registered in collection.indexes — the negative id keeps it + // distinct from user-created index ids. + super(-1, expression, `key`) + this.hasKey = hasKey + this.getKeyCount = getKeyCount + } + + protected initialize(): void {} + + // The collection state is the backing store, so there is nothing to maintain. + add(): void {} + remove(): void {} + update(): void {} + build(): void {} + clear(): void {} + + lookup(operation: IndexOperation, value: any): Set { + const startTime = performance.now() + + let result: Set + switch (operation) { + case `eq`: + result = this.equalityLookup(value) + break + case `in`: + result = this.inArrayLookup(value) + break + default: + throw new Error(`Operation ${operation} not supported by KeyIndex`) + } + + this.trackLookup(startTime) + return result + } + + equalityLookup(value: any): Set { + // Normalize like BasicIndex does, so a lookup value behaves the same + // against the key field as it would against a user-created index. + const normalizedValue = normalizeValue(value) + return this.hasKey(normalizedValue) + ? new Set([normalizedValue as TKey]) + : new Set() + } + + inArrayLookup(values: Array): Set { + const result = new Set() + for (const value of values) { + const normalizedValue = normalizeValue(value) + if (this.hasKey(normalizedValue)) { + result.add(normalizedValue as TKey) + } + } + return result + } + + get keyCount(): number { + return this.getKeyCount() + } + + get supportsRangeOptimization(): boolean { + return false + } + + // Range and ordered access are not supported: `supports()` reports only + // eq/in, so the optimizer and order-by never route these calls here. + rangeQuery(): Set { + throw new Error(`Range queries are not supported by KeyIndex`) + } + + rangeQueryReversed(): Set { + throw new Error(`Range queries are not supported by KeyIndex`) + } + + take(): Array { + throw new Error(`Ordered access is not supported by KeyIndex`) + } + + takeFromStart(): Array { + throw new Error(`Ordered access is not supported by KeyIndex`) + } + + takeReversed(): Array { + throw new Error(`Ordered access is not supported by KeyIndex`) + } + + takeReversedFromEnd(): Array { + throw new Error(`Ordered access is not supported by KeyIndex`) + } + + get orderedEntriesArray(): Array<[any, Set]> { + throw new Error(`Ordered access is not supported by KeyIndex`) + } + + get orderedEntriesArrayReversed(): Array<[any, Set]> { + throw new Error(`Ordered access is not supported by KeyIndex`) + } + + get indexedKeysSet(): Set { + throw new Error(`Key enumeration is not supported by KeyIndex`) + } + + get valueMapData(): Map> { + throw new Error(`Value enumeration is not supported by KeyIndex`) + } +} + +/** + * Derives a {@link KeyIndex} from a collection's `getKey` function. + * + * `getKey` is called once with a ref proxy: when it reads a single property + * (e.g. `(row) => row.id`), that access is captured as the key field path — + * the same introspection `createIndex` uses for its index callback. Anything + * else — composite keys, computed keys, or a `getKey` that throws on the + * proxy — returns `undefined` and the collection simply has no implicit key + * index. + */ +export function createKeyIndexFromGetKey< + T extends object, + TKey extends string | number, +>( + getKey: (item: T) => TKey, + hasKey: (key: TKey) => boolean, + getKeyCount: () => number, +): KeyIndex | undefined { + let expression: BasicExpression + try { + const row = createSingleRowRefProxy() + expression = toExpression(getKey(row as unknown as T)) + } catch { + return undefined + } + if (expression.type !== `ref` || expression.path.length === 0) { + return undefined + } + return new KeyIndex(expression, hasKey, getKeyCount) +} diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index bae05a943f..e30b7027d8 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -17,7 +17,7 @@ export interface CollectionLike< TKey extends string | number = string | number, > extends Pick< Collection, - `get` | `has` | `entries` | `indexes` | `id` | `compareOptions` + `get` | `has` | `entries` | `indexes` | `keyIndex` | `id` | `compareOptions` > {} /** diff --git a/packages/db/src/utils/index-optimization.ts b/packages/db/src/utils/index-optimization.ts index 5a52a5ec54..acfcad79c0 100644 --- a/packages/db/src/utils/index-optimization.ts +++ b/packages/db/src/utils/index-optimization.ts @@ -66,6 +66,22 @@ export function findIndexForField( return index } } + + // Fall back to the collection's implicit primary-key index, so equality + // lookups on the key field (e.g. joins on the primary key) work without a + // user-created index. Checked last so an explicit index always wins. + const keyIndex = collection.keyIndex + if ( + keyIndex && + keyIndex.matchesField(fieldPath) && + keyIndex.matchesCompareOptions(compareOpts) + ) { + if (!keyIndex.matchesDirection(compareOpts.direction)) { + return new ReverseIndex(keyIndex) + } + return keyIndex + } + return undefined } diff --git a/packages/db/tests/key-index.test.ts b/packages/db/tests/key-index.test.ts new file mode 100644 index 0000000000..31efdf2f7b --- /dev/null +++ b/packages/db/tests/key-index.test.ts @@ -0,0 +1,195 @@ +import { describe, expect, it } from 'vitest' +import { createCollection } from '../src/collection/index.js' +import { KeyIndex, createKeyIndexFromGetKey } from '../src/indexes/key-index.js' +import { BasicIndex } from '../src/indexes/basic-index.js' +import { findIndexForField } from '../src/utils/index-optimization.js' +import { mockSyncCollectionOptions } from './utils.js' + +type Item = { + id: string + category: string +} + +const sampleItems: Array = [ + { id: `a`, category: `one` }, + { id: `b`, category: `one` }, + { id: `c`, category: `two` }, +] + +function makeCollection(getKey: (item: Item) => string) { + return createCollection( + mockSyncCollectionOptions({ + id: `key-index-test`, + getKey, + initialData: sampleItems, + }), + ) +} + +describe(`createKeyIndexFromGetKey`, () => { + it(`derives an index from a single property access`, () => { + const keys = new Set([`a`, `b`]) + const index = createKeyIndexFromGetKey( + (item) => item.id, + (key) => keys.has(key), + () => keys.size, + ) + + expect(index).toBeInstanceOf(KeyIndex) + expect(index!.matchesField([`id`])).toBe(true) + expect(index!.matchesField([`category`])).toBe(false) + expect(index!.keyCount).toBe(2) + }) + + it(`returns undefined for a composite key`, () => { + const index = createKeyIndexFromGetKey( + (item) => `${item.id}:${item.category}`, + () => true, + () => 0, + ) + + expect(index).toBeUndefined() + }) + + it(`returns undefined when getKey does not read a property`, () => { + const index = createKeyIndexFromGetKey( + (item) => item as unknown as string, + () => true, + () => 0, + ) + + expect(index).toBeUndefined() + }) + + it(`returns undefined when getKey throws on the introspection proxy`, () => { + const index = createKeyIndexFromGetKey( + () => { + throw new Error(`boom`) + }, + () => true, + () => 0, + ) + + expect(index).toBeUndefined() + }) +}) + +describe(`KeyIndex lookups`, () => { + const keys = new Set([`a`, `b`]) + const index = createKeyIndexFromGetKey( + (item) => item.id, + (key) => keys.has(key), + () => keys.size, + )! + + it(`supports only eq and in`, () => { + expect(index.supports(`eq`)).toBe(true) + expect(index.supports(`in`)).toBe(true) + expect(index.supports(`gt`)).toBe(false) + expect(index.supports(`lte`)).toBe(false) + expect(index.supportsRangeOptimization).toBe(false) + }) + + it(`resolves eq lookups through the key set`, () => { + expect(index.lookup(`eq`, `a`)).toEqual(new Set([`a`])) + expect(index.lookup(`eq`, `missing`)).toEqual(new Set()) + expect(index.lookup(`eq`, null)).toEqual(new Set()) + }) + + it(`resolves in lookups by filtering to present keys`, () => { + expect(index.lookup(`in`, [`a`, `b`, `missing`])).toEqual( + new Set([`a`, `b`]), + ) + expect(index.lookup(`in`, [])).toEqual(new Set()) + }) + + it(`rejects unsupported operations`, () => { + expect(() => index.lookup(`gt`, `a`)).toThrow( + `Operation gt not supported by KeyIndex`, + ) + }) +}) + +describe(`collection.keyIndex`, () => { + it(`is derived for a plain property getKey and reflects live state`, () => { + const collection = makeCollection((item) => item.id) + const keyIndex = collection.keyIndex + + expect(keyIndex).toBeInstanceOf(KeyIndex) + expect(keyIndex!.matchesField([`id`])).toBe(true) + expect(keyIndex!.lookup(`eq`, `a`)).toEqual(new Set([`a`])) + expect(keyIndex!.lookup(`in`, [`a`, `c`, `zzz`])).toEqual( + new Set([`a`, `c`]), + ) + + collection.utils.begin() + collection.utils.write({ + type: `insert`, + value: { id: `d`, category: `two` }, + }) + collection.utils.commit() + + expect(keyIndex!.lookup(`eq`, `d`)).toEqual(new Set([`d`])) + }) + + it(`is undefined for a composite getKey`, () => { + const collection = makeCollection((item) => `${item.id}:${item.category}`) + + expect(collection.keyIndex).toBeUndefined() + }) +}) + +describe(`findIndexForField key-index fallback`, () => { + it(`serves the key field when no explicit index exists`, () => { + const collection = makeCollection((item) => item.id) + + const index = findIndexForField(collection, [`id`]) + + expect(index).toBe(collection.keyIndex) + expect(index!.lookup(`eq`, `b`)).toEqual(new Set([`b`])) + }) + + it(`prefers an explicit index on the key field`, () => { + const collection = makeCollection((item) => item.id) + collection.createIndex((row) => row.id, { indexType: BasicIndex }) + + const index = findIndexForField(collection, [`id`]) + + expect(index).toBeInstanceOf(BasicIndex) + }) + + it(`does not serve non-key fields`, () => { + const collection = makeCollection((item) => item.id) + + expect(findIndexForField(collection, [`category`])).toBeUndefined() + }) + + it(`does not serve collections with a composite key`, () => { + const collection = makeCollection((item) => `${item.id}:${item.category}`) + + expect(findIndexForField(collection, [`id`])).toBeUndefined() + }) + + it(`is conservatively skipped for collections with a custom string collation`, () => { + const collection = createCollection({ + getKey: (item) => item.id, + defaultStringCollation: { stringSort: `lexical` }, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + for (const item of sampleItems) { + write({ type: `insert`, value: item }) + } + commit() + markReady() + }, + }, + }) + + // The key index itself is derivable, but its compare options are the + // defaults, so lookups under a custom collation fall back to a full scan. + expect(collection.keyIndex).toBeInstanceOf(KeyIndex) + expect(findIndexForField(collection, [`id`])).toBeUndefined() + }) +}) diff --git a/packages/db/tests/query/indexes.test.ts b/packages/db/tests/query/indexes.test.ts index 6abc065f63..f033519d05 100644 --- a/packages/db/tests/query/indexes.test.ts +++ b/packages/db/tests/query/indexes.test.ts @@ -89,6 +89,24 @@ function createIndexUsageTracker(collection: any): { configurable: true, }) + // The implicit primary-key index lives outside `collection.indexes`, so + // patch its lookup separately to observe key-index-served queries. + const keyIndex = collection.keyIndex + const originalKeyIndexLookup = keyIndex?.lookup + if (keyIndex) { + keyIndex.lookup = function (operation: string, value: any) { + stats.rangeQueryCalls++ + stats.indexesUsed.push(`keyIndex`) + stats.queriesExecuted.push({ + type: `index`, + operation, + field: keyIndex.expression?.path?.join(`.`), + value, + }) + return originalKeyIndexLookup.call(this, operation, value) + } + } + // Track full scan calls (entries() iteration) const originalEntries = collection.entries collection.entries = function* () { @@ -125,6 +143,10 @@ function createIndexUsageTracker(collection: any): { } } + if (keyIndex && originalKeyIndexLookup) { + keyIndex.lookup = originalKeyIndexLookup + } + collection.entries = originalEntries } @@ -684,12 +706,14 @@ describe(`Query Index Optimization`, () => { // The WHERE clause on the non-nullable (left) side uses its index. // The WHERE clause on the nullable (right) side of the LEFT JOIN is NOT - // pushed down to avoid changing join semantics, so the right side does a full scan. + // pushed down to avoid changing join semantics, but the join key on the + // right side is its primary key, so the lazy join loads through the + // implicit key index instead of a full scan. expectIndexUsage(combinedStats, { shouldUseIndex: true, - shouldUseFullScan: true, - indexCallCount: 1, // Only item.status='active' uses index (non-nullable side) - fullScanCallCount: 1, // other collection does full scan (nullable side) + shouldUseFullScan: false, + indexCallCount: 2, // item.status='active' + join keys via the key index + fullScanCallCount: 0, }) } finally { tracker1.restore() @@ -804,7 +828,7 @@ describe(`Query Index Optimization`, () => { } }) - it(`should not optimize inner join if biggest collection has no index on the join key`, async () => { + it(`should optimize inner join via the implicit key index when the biggest collection has no explicit index on the join key`, async () => { // Create a second collection for the join with its own index const secondCollection = createCollection({ getKey: (item) => item.id2, @@ -878,7 +902,9 @@ describe(`Query Index Optimization`, () => { }, ]) - // We should have done an index lookup on the 1st collection to find active items + // We should have done an index lookup on the 1st collection to find + // active items, and the join keys are served by the implicit + // primary-key index instead of a full scan. expect(tracker1.stats.queriesExecuted).toEqual([ { type: `index`, @@ -886,6 +912,12 @@ describe(`Query Index Optimization`, () => { field: `status`, value: `active`, }, + { + type: `index`, + operation: `in`, + field: `id`, + value: [`1`], + }, ]) } finally { tracker1.restore() @@ -1018,9 +1050,11 @@ describe(`Query Index Optimization`, () => { }) it(`should not optimize left join if right collection has no index on the join key`, async () => { - // Create a second collection for the join with its own index + // Create a second collection for the join with its own index. + // The key is computed so the join key `id2` is not served by the + // implicit primary-key index and the collection truly has no index on it. const secondCollection = createCollection({ - getKey: (item) => item.id2, + getKey: (item) => `computed-${item.id2}`, autoIndex: `off`, startSync: true, sync: { @@ -1208,7 +1242,45 @@ describe(`Query Index Optimization`, () => { }) it(`should not optimize right join if left collection has no index on the join key`, async () => { - // Create a second collection for the join with its own index + // Create a local left collection with a computed key, so the join key + // `id` is not served by the implicit primary-key index and the + // collection truly has no index on it. + const firstCollection = createCollection({ + getKey: (item) => `computed-${item.id}`, + autoIndex: `off`, + startSync: true, + sync: { + sync: ({ begin, write, commit }) => { + begin() + write({ + type: `insert`, + value: { + id: `1`, + name: `Alice`, + age: 25, + status: `active`, + score: 95, + createdAt: new Date(`2023-01-01`), + }, + }) + write({ + type: `insert`, + value: { + id: `2`, + name: `Bob`, + age: 30, + status: `inactive`, + score: 80, + createdAt: new Date(`2023-01-02`), + }, + }) + commit() + }, + }, + }) + + await firstCollection.stateWhenReady() + const secondCollection = createCollection({ getKey: (item) => item.id2, autoIndex: `off`, @@ -1244,14 +1316,14 @@ describe(`Query Index Optimization`, () => { await secondCollection.stateWhenReady() // Track both collections - const tracker1 = createIndexUsageTracker(collection) + const tracker1 = createIndexUsageTracker(firstCollection) const tracker2 = createIndexUsageTracker(secondCollection) try { const liveQuery = createLiveQueryCollection({ query: (q: any) => q - .from({ item: collection }) + .from({ item: firstCollection }) .join( { other: secondCollection }, ({ item, other }: any) => eq(item.id, other.id2), diff --git a/packages/db/tests/query/join-key-index.test.ts b/packages/db/tests/query/join-key-index.test.ts new file mode 100644 index 0000000000..73a34687de --- /dev/null +++ b/packages/db/tests/query/join-key-index.test.ts @@ -0,0 +1,164 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest' +import { createLiveQueryCollection, eq } from '../../src/query/index.js' +import { createCollection } from '../../src/collection/index.js' +import { mockSyncCollectionOptions, stripVirtualProps } from '../utils.js' + +/** + * Lazy joins on a collection's primary key should load through the implicit + * key index instead of falling back to a full collection scan, without the + * user having to create an explicit index on the key field. + */ + +type Team = { id: string; name: string } +type Member = { id: string; teamId: string } + +const sampleTeams: Array = [ + { id: `t1`, name: `Team One` }, + { id: `t2`, name: `Team Two` }, + { id: `t3`, name: `Team Three` }, +] + +const sampleMembers: Array = [ + { id: `m1`, teamId: `t1` }, + { id: `m2`, teamId: `t1` }, + { id: `m3`, teamId: `t2` }, +] + +describe(`lazy join on the primary key without an explicit index`, () => { + let warnSpy: ReturnType + + beforeEach(() => { + warnSpy = vi.spyOn(console, `warn`).mockImplementation(() => {}) + }) + + afterEach(() => { + warnSpy.mockRestore() + }) + + const indexWarnings = () => + warnSpy.mock.calls + .map((c) => String(c[0])) + .filter((m) => m.includes(`Join requires an index`)) + + const makeTeamsCollection = () => + createCollection( + mockSyncCollectionOptions({ + id: `key-join-teams`, + getKey: (r) => r.id, + autoIndex: `off`, + initialData: sampleTeams, + }), + ) + + const makeMembersCollection = () => + createCollection( + mockSyncCollectionOptions({ + id: `key-join-members`, + getKey: (r) => r.id, + autoIndex: `off`, + initialData: sampleMembers, + }), + ) + + test(`loads through the key index and does not warn`, () => { + const teams = makeTeamsCollection() + const members = makeMembersCollection() + + const joined = createLiveQueryCollection({ + startSync: true, + query: (q) => + q + .from({ member: members }) + .join({ team: teams }, ({ member, team }) => + eq(team.id, member.teamId), + ) + .select(({ member, team }) => ({ + memberId: member.id, + teamName: team.name, + })), + }) + + expect( + joined.toArray + .map(stripVirtualProps) + .sort((a, b) => a.memberId.localeCompare(b.memberId)), + ).toEqual([ + { memberId: `m1`, teamName: `Team One` }, + { memberId: `m2`, teamName: `Team One` }, + { memberId: `m3`, teamName: `Team Two` }, + ]) + + expect(indexWarnings()).toEqual([]) + }) + + test(`serves join keys that appear after the initial load`, () => { + const teams = makeTeamsCollection() + const members = makeMembersCollection() + + const joined = createLiveQueryCollection({ + startSync: true, + query: (q) => + q + .from({ member: members }) + .join({ team: teams }, ({ member, team }) => + eq(team.id, member.teamId), + ) + .select(({ member, team }) => ({ + memberId: member.id, + teamName: team.name, + })), + }) + + // `t3` is not referenced by the initial members, so it was not part of + // the initial lazy snapshot. A new member pointing at it must trigger an + // index-served load for that key. + members.utils.begin() + members.utils.write({ + type: `insert`, + value: { id: `m4`, teamId: `t3` }, + }) + members.utils.commit() + + expect( + joined.toArray.map(stripVirtualProps).find((r) => r.memberId === `m4`), + ).toEqual({ + memberId: `m4`, + teamName: `Team Three`, + }) + + expect(indexWarnings()).toEqual([]) + }) + + test(`still warns when the joined collection has a computed key`, () => { + const teams = createCollection( + mockSyncCollectionOptions({ + id: `key-join-teams-computed`, + // Computed key: cannot be introspected into a key index. + getKey: (r) => `team:${r.id}`, + autoIndex: `off`, + initialData: sampleTeams, + }), + ) + const members = makeMembersCollection() + + const joined = createLiveQueryCollection({ + startSync: true, + query: (q) => + q + .from({ member: members }) + .join({ team: teams }, ({ member, team }) => + eq(team.id, member.teamId), + ) + .select(({ member, team }) => ({ + memberId: member.id, + teamName: team.name, + })), + }) + + // Data still flows via the full-load fallback. + expect(joined.toArray).toHaveLength(3) + expect( + indexWarnings().filter((m) => m.includes(`key-join-teams-computed`)), + ).not.toEqual([]) + }) +}) From 0941cf273a79f91054010243e55186426df47008 Mon Sep 17 00:00:00 2001 From: balanced Date: Tue, 18 Aug 2026 16:44:24 +0300 Subject: [PATCH 2/5] refactor: collapse KeyIndex unsupported-member stubs into one helper --- packages/db/src/indexes/key-index.ts | 31 +++++++++++++++++----------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/packages/db/src/indexes/key-index.ts b/packages/db/src/indexes/key-index.ts index c31f14790b..ec43adff67 100644 --- a/packages/db/src/indexes/key-index.ts +++ b/packages/db/src/indexes/key-index.ts @@ -93,46 +93,53 @@ export class KeyIndex< return false } - // Range and ordered access are not supported: `supports()` reports only - // eq/in, so the optimizer and order-by never route these calls here. + // The remaining IndexInterface members are mandated by BaseIndex's abstract + // contract but unreachable in practice: `supports()` reports only eq/in, so + // the optimizer and order-by never route range or ordered access here. + // Throwing (rather than returning empty results) keeps any future call path + // that does reach them loudly wrong instead of silently dropping rows. + private unsupported(feature: string): never { + throw new Error(`KeyIndex does not support ${feature}`) + } + rangeQuery(): Set { - throw new Error(`Range queries are not supported by KeyIndex`) + return this.unsupported(`range queries`) } rangeQueryReversed(): Set { - throw new Error(`Range queries are not supported by KeyIndex`) + return this.unsupported(`range queries`) } take(): Array { - throw new Error(`Ordered access is not supported by KeyIndex`) + return this.unsupported(`ordered access`) } takeFromStart(): Array { - throw new Error(`Ordered access is not supported by KeyIndex`) + return this.unsupported(`ordered access`) } takeReversed(): Array { - throw new Error(`Ordered access is not supported by KeyIndex`) + return this.unsupported(`ordered access`) } takeReversedFromEnd(): Array { - throw new Error(`Ordered access is not supported by KeyIndex`) + return this.unsupported(`ordered access`) } get orderedEntriesArray(): Array<[any, Set]> { - throw new Error(`Ordered access is not supported by KeyIndex`) + return this.unsupported(`ordered access`) } get orderedEntriesArrayReversed(): Array<[any, Set]> { - throw new Error(`Ordered access is not supported by KeyIndex`) + return this.unsupported(`ordered access`) } get indexedKeysSet(): Set { - throw new Error(`Key enumeration is not supported by KeyIndex`) + return this.unsupported(`key enumeration`) } get valueMapData(): Map> { - throw new Error(`Value enumeration is not supported by KeyIndex`) + return this.unsupported(`value enumeration`) } } From d9f7b29509f01444873f771ecdd2a25a8369c196 Mon Sep 17 00:00:00 2001 From: balanced Date: Tue, 18 Aug 2026 16:55:55 +0300 Subject: [PATCH 3/5] fix: align canOptimize predicates with index capability checks --- packages/db/src/utils/index-optimization.ts | 18 ++++++++-- packages/db/tests/key-index.test.ts | 36 ++++++++++++++++++- .../db/tests/query/join-key-index.test.ts | 16 +++++++++ 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/packages/db/src/utils/index-optimization.ts b/packages/db/src/utils/index-optimization.ts index acfcad79c0..f1497acdec 100644 --- a/packages/db/src/utils/index-optimization.ts +++ b/packages/db/src/utils/index-optimization.ts @@ -40,7 +40,12 @@ export interface OptimizationResult { } /** - * Finds an index that matches a given field path + * Finds an index that matches a given field path. + * + * The returned index may be capability-limited (e.g. the implicit primary-key + * index only serves `eq`/`in`), so callers must check `supports(operation)` + * before range or ordered access — as all callers in this module and in + * order-by/change-events do. */ export function findIndexForField( collection: CollectionLike, @@ -614,7 +619,12 @@ function canOptimizeSimpleComparison< if (fieldPath) { const index = findIndexForField(collection, fieldPath) - return index !== undefined + // Mirror optimizeSimpleComparison's gate: an index that exists but does + // not support the operation (e.g. the implicit key index only serves + // eq/in) cannot optimize this comparison. + return ( + index !== undefined && index.supports(expression.name as IndexOperation) + ) } return false @@ -821,7 +831,9 @@ function canOptimizeInArrayExpression< ) { const fieldPath = (fieldArg as any).path const index = findIndexForField(collection, fieldPath) - return index !== undefined + // Mirror optimizeInArrayExpression's gate: IN is served either natively + // or via per-value equality lookups. + return index !== undefined && (index.supports(`in`) || index.supports(`eq`)) } return false diff --git a/packages/db/tests/key-index.test.ts b/packages/db/tests/key-index.test.ts index 31efdf2f7b..87e95ff00d 100644 --- a/packages/db/tests/key-index.test.ts +++ b/packages/db/tests/key-index.test.ts @@ -2,7 +2,11 @@ import { describe, expect, it } from 'vitest' import { createCollection } from '../src/collection/index.js' import { KeyIndex, createKeyIndexFromGetKey } from '../src/indexes/key-index.js' import { BasicIndex } from '../src/indexes/basic-index.js' -import { findIndexForField } from '../src/utils/index-optimization.js' +import { + canOptimizeExpression, + findIndexForField, +} from '../src/utils/index-optimization.js' +import { Func, PropRef, Value } from '../src/query/ir.js' import { mockSyncCollectionOptions } from './utils.js' type Item = { @@ -193,3 +197,33 @@ describe(`findIndexForField key-index fallback`, () => { expect(findIndexForField(collection, [`id`])).toBeUndefined() }) }) + +describe(`canOptimizeExpression with only a key index`, () => { + const collection = makeCollection((item) => item.id) + + it(`reports eq and in on the key field as optimizable`, () => { + expect( + canOptimizeExpression( + new Func(`eq`, [new PropRef([`id`]), new Value(`a`)]), + collection, + ), + ).toBe(true) + expect( + canOptimizeExpression( + new Func(`in`, [new PropRef([`id`]), new Value([`a`, `b`])]), + collection, + ), + ).toBe(true) + }) + + it(`reports range operations on the key field as not optimizable`, () => { + // The key index exists but only supports eq/in, so the predicate must + // agree with what optimization would actually do. + expect( + canOptimizeExpression( + new Func(`gt`, [new PropRef([`id`]), new Value(`a`)]), + collection, + ), + ).toBe(false) + }) +}) diff --git a/packages/db/tests/query/join-key-index.test.ts b/packages/db/tests/query/join-key-index.test.ts index 73a34687de..5897c276f4 100644 --- a/packages/db/tests/query/join-key-index.test.ts +++ b/packages/db/tests/query/join-key-index.test.ts @@ -63,6 +63,7 @@ describe(`lazy join on the primary key without an explicit index`, () => { test(`loads through the key index and does not warn`, () => { const teams = makeTeamsCollection() const members = makeMembersCollection() + const keyLookup = vi.spyOn(teams.keyIndex!, `lookup`) const joined = createLiveQueryCollection({ startSync: true, @@ -89,11 +90,19 @@ describe(`lazy join on the primary key without an explicit index`, () => { ]) expect(indexWarnings()).toEqual([]) + + // The load must actually have gone through the key index, not a scan + // that merely stopped warning. + expect(keyLookup).toHaveBeenCalledWith( + `in`, + expect.arrayContaining([`t1`, `t2`]), + ) }) test(`serves join keys that appear after the initial load`, () => { const teams = makeTeamsCollection() const members = makeMembersCollection() + const keyLookup = vi.spyOn(teams.keyIndex!, `lookup`) const joined = createLiveQueryCollection({ startSync: true, @@ -127,6 +136,13 @@ describe(`lazy join on the primary key without an explicit index`, () => { }) expect(indexWarnings()).toEqual([]) + + // The late-arriving key must have been loaded through the key index. + const t3Lookups = keyLookup.mock.calls.filter( + ([operation, values]) => + operation === `in` && Array.isArray(values) && values.includes(`t3`), + ) + expect(t3Lookups.length).toBeGreaterThan(0) }) test(`still warns when the joined collection has a computed key`, () => { From e46e7156de26c0e8d371c97a14755c4fc1b3025a Mon Sep 17 00:00:00 2001 From: balanced Date: Tue, 18 Aug 2026 17:08:30 +0300 Subject: [PATCH 4/5] fix: mirror the range-trust gate in canOptimizeSimpleComparison --- packages/db/src/utils/index-optimization.ts | 30 +++++++++++++---- packages/db/tests/key-index.test.ts | 37 +++++++++++++++++++++ 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/packages/db/src/utils/index-optimization.ts b/packages/db/src/utils/index-optimization.ts index f1497acdec..bab1d45d30 100644 --- a/packages/db/src/utils/index-optimization.ts +++ b/packages/db/src/utils/index-optimization.ts @@ -610,21 +610,39 @@ function canOptimizeSimpleComparison< // Check both directions: field op value AND value op field let fieldPath: Array | null = null + let queryValue: unknown if (leftArg.type === `ref` && rightArg.type === `val`) { fieldPath = (leftArg as any).path + queryValue = (rightArg as any).value } else if (leftArg.type === `val` && rightArg.type === `ref`) { fieldPath = (rightArg as any).path + queryValue = (leftArg as any).value } if (fieldPath) { const index = findIndexForField(collection, fieldPath) - // Mirror optimizeSimpleComparison's gate: an index that exists but does - // not support the operation (e.g. the implicit key index only serves - // eq/in) cannot optimize this comparison. - return ( - index !== undefined && index.supports(expression.name as IndexOperation) - ) + if (index === undefined) { + return false + } + + // Mirror optimizeSimpleComparison's gates: the index must support the + // operation (e.g. the implicit key index only serves eq/in), and a range + // op additionally requires trustworthy index traversal for the operand's + // domain. The operand flip for `value op field` maps range ops onto range + // ops, so the classification needs no flip here. + const operation = expression.name as IndexOperation + if (!index.supports(operation)) { + return false + } + if ( + operation !== `eq` && + !canRangeOptimize(queryValue, index, collection) + ) { + return false + } + + return true } return false diff --git a/packages/db/tests/key-index.test.ts b/packages/db/tests/key-index.test.ts index 87e95ff00d..d6b5854032 100644 --- a/packages/db/tests/key-index.test.ts +++ b/packages/db/tests/key-index.test.ts @@ -5,6 +5,7 @@ import { BasicIndex } from '../src/indexes/basic-index.js' import { canOptimizeExpression, findIndexForField, + optimizeExpressionWithIndexes, } from '../src/utils/index-optimization.js' import { Func, PropRef, Value } from '../src/query/ir.js' import { mockSyncCollectionOptions } from './utils.js' @@ -227,3 +228,39 @@ describe(`canOptimizeExpression with only a key index`, () => { ).toBe(false) }) }) + +describe(`canOptimizeExpression agrees with optimizeExpressionWithIndexes`, () => { + it(`rejects locale-ordered string ranges on a range-capable index`, () => { + const collection = makeCollection((item) => item.id) + collection.createIndex((row) => row.category, { indexType: BasicIndex }) + + // Default collections sort strings by locale, which diverges from the + // WHERE evaluator's code-point comparison, so a string range cannot be + // served by the index — and the predicate must say so too. + const rangeExpr = new Func(`gt`, [ + new PropRef([`category`]), + new Value(`m`), + ]) + expect( + optimizeExpressionWithIndexes(rangeExpr, collection).canOptimize, + ).toBe(false) + expect(canOptimizeExpression(rangeExpr, collection)).toBe(false) + + // The flipped operand form must classify the same way. + const flippedExpr = new Func(`gt`, [ + new Value(`m`), + new PropRef([`category`]), + ]) + expect( + optimizeExpressionWithIndexes(flippedExpr, collection).canOptimize, + ).toBe(false) + expect(canOptimizeExpression(flippedExpr, collection)).toBe(false) + + // Equality is unaffected by collation and stays optimizable. + const eqExpr = new Func(`eq`, [new PropRef([`category`]), new Value(`one`)]) + expect(optimizeExpressionWithIndexes(eqExpr, collection).canOptimize).toBe( + true, + ) + expect(canOptimizeExpression(eqExpr, collection)).toBe(true) + }) +}) From 1f2128653de51cc670bd7487b5c3f5abd4653c79 Mon Sep 17 00:00:00 2001 From: balanced Date: Tue, 18 Aug 2026 17:50:24 +0300 Subject: [PATCH 5/5] fix: normalize flipped comparison operands before consulting index capabilities --- packages/db/src/utils/index-optimization.ts | 198 +++++++++----------- packages/db/tests/key-index.test.ts | 47 +++++ 2 files changed, 140 insertions(+), 105 deletions(-) diff --git a/packages/db/src/utils/index-optimization.ts b/packages/db/src/utils/index-optimization.ts index bab1d45d30..d72576fce0 100644 --- a/packages/db/src/utils/index-optimization.ts +++ b/packages/db/src/utils/index-optimization.ts @@ -20,7 +20,7 @@ import { ReverseIndex } from '../indexes/reverse-index.js' import { hasVirtualPropPath } from '../virtual-props.js' import { makeComparator } from './comparison.js' import type { CompareOptions } from '../query/builder/types.js' -import type { IndexInterface, IndexOperation } from '../indexes/base-index.js' +import type { IndexInterface } from '../indexes/base-index.js' import type { BasicExpression } from '../query/ir.js' import type { CollectionLike } from '../types.js' @@ -492,37 +492,39 @@ function optimizeCompoundRangeQuery< } /** - * Optimizes simple comparison expressions (eq, gt, gte, lt, lte) + * A binary comparison normalized to `field op value` form. For `value op field` + * expressions the range operation is flipped (`gt`↔`lt`, `gte`↔`lte`; `eq` is + * symmetric), so `operation` is always the operation an index lookup serves. */ -function optimizeSimpleComparison< - T extends object, - TKey extends string | number, ->( +interface NormalizedComparison { + fieldPath: Array + queryValue: unknown + operation: `eq` | `gt` | `gte` | `lt` | `lte` +} + +/** + * Normalizes a comparison expression to `field op value` form, or returns + * `undefined` when it is not a binary comparison between a field reference + * and a literal value. Both the optimizer and the `canOptimize*` predicates + * classify through this helper so their decisions cannot drift apart. + */ +function normalizeSimpleComparison( expression: BasicExpression, - collection: CollectionLike, -): OptimizationResult { +): NormalizedComparison | undefined { if (expression.type !== `func` || expression.args.length !== 2) { - return { canOptimize: false, matchingKeys: new Set(), isExact: false } + return undefined } const leftArg = expression.args[0]! const rightArg = expression.args[1]! - - // Check both directions: field op value AND value op field - let fieldArg: BasicExpression | null = null - let valueArg: BasicExpression | null = null - let operation = expression.name as `eq` | `gt` | `gte` | `lt` | `lte` + let operation = expression.name as NormalizedComparison[`operation`] if (leftArg.type === `ref` && rightArg.type === `val`) { - // field op value - fieldArg = leftArg - valueArg = rightArg - } else if (leftArg.type === `val` && rightArg.type === `ref`) { - // value op field - need to flip the operation - fieldArg = rightArg - valueArg = leftArg - - // Flip the operation for reverse comparison + return { fieldPath: leftArg.path, queryValue: rightArg.value, operation } + } + + if (leftArg.type === `val` && rightArg.type === `ref`) { + // value op field - flip the range operation for the reverse comparison switch (operation) { case `gt`: operation = `lt` @@ -538,57 +540,65 @@ function optimizeSimpleComparison< break // eq stays the same } + return { fieldPath: rightArg.path, queryValue: leftArg.value, operation } } - if (fieldArg && valueArg) { - const fieldPath = (fieldArg as any).path - const index = findIndexForField(collection, fieldPath) + return undefined +} - if (index) { - const queryValue = (valueArg as any).value +/** + * Optimizes simple comparison expressions (eq, gt, gte, lt, lte) + */ +function optimizeSimpleComparison< + T extends object, + TKey extends string | number, +>( + expression: BasicExpression, + collection: CollectionLike, +): OptimizationResult { + const normalized = normalizeSimpleComparison(expression) + if (!normalized) { + return { canOptimize: false, matchingKeys: new Set(), isExact: false } + } + const { fieldPath, queryValue, operation } = normalized - // Map operation to IndexOperation enum - const indexOperation = operation as IndexOperation + const index = findIndexForField(collection, fieldPath) + if (index) { + // Check if the index supports this operation + if (!index.supports(operation)) { + return { canOptimize: false, matchingKeys: new Set(), isExact: false } + } - // Check if the index supports this operation - if (!index.supports(indexOperation)) { - return { canOptimize: false, matchingKeys: new Set(), isExact: false } - } + // A range op can only use the index when the operand's domain orders the + // same way the index does and the index supports trustworthy traversal. + // Otherwise the index may omit matching rows, which re-filtering cannot + // recover, so fall back to a full scan. + if ( + operation !== `eq` && + !canRangeOptimize(queryValue, index, collection) + ) { + return { canOptimize: false, matchingKeys: new Set(), isExact: false } + } - // A range op can only use the index when the operand's domain orders the - // same way the index does and the index supports trustworthy traversal. - // Otherwise the index may omit matching rows, which re-filtering cannot - // recover, so fall back to a full scan. - if ( - (operation === `gt` || - operation === `gte` || - operation === `lt` || - operation === `lte`) && - !canRangeOptimize(queryValue, index, collection) - ) { - return { canOptimize: false, matchingKeys: new Set(), isExact: false } - } + const matchingKeys = index.lookup(operation, queryValue) - const matchingKeys = index.lookup(indexOperation, queryValue) - - // A comparison against a nullish value is never true, but BTree indexes - // store and return rows with nullish keys (they sort to the nulls end). - // Determine whether the index result is exact or a superset that the - // caller must re-filter: - // - eq/gt/gte: a nullish query value matches nothing while the index - // still returns nullish-keyed rows -> inexact. A non-nullish lower - // bound (gt/gte) excludes those bottom-sorted rows, so they stay exact. - // - lt/lte: the open lower bound always includes nullish-keyed rows, - // so the result is conservatively inexact. - // NaN/invalid Dates are exact here: under PostgreSQL float semantics the - // evaluator and the index agree on them (equal to self, greatest). - const isExact = - operation === `lt` || operation === `lte` - ? false - : isExactComparisonValue(queryValue) - - return { canOptimize: true, matchingKeys, isExact } - } + // A comparison against a nullish value is never true, but BTree indexes + // store and return rows with nullish keys (they sort to the nulls end). + // Determine whether the index result is exact or a superset that the + // caller must re-filter: + // - eq/gt/gte: a nullish query value matches nothing while the index + // still returns nullish-keyed rows -> inexact. A non-nullish lower + // bound (gt/gte) excludes those bottom-sorted rows, so they stay exact. + // - lt/lte: the open lower bound always includes nullish-keyed rows, + // so the result is conservatively inexact. + // NaN/invalid Dates are exact here: under PostgreSQL float semantics the + // evaluator and the index agree on them (equal to self, greatest). + const isExact = + operation === `lt` || operation === `lte` + ? false + : isExactComparisonValue(queryValue) + + return { canOptimize: true, matchingKeys, isExact } } return { canOptimize: false, matchingKeys: new Set(), isExact: false } @@ -601,51 +611,29 @@ function canOptimizeSimpleComparison< T extends object, TKey extends string | number, >(expression: BasicExpression, collection: CollectionLike): boolean { - if (expression.type !== `func` || expression.args.length !== 2) { + const normalized = normalizeSimpleComparison(expression) + if (!normalized) { return false } + const { fieldPath, queryValue, operation } = normalized - const leftArg = expression.args[0]! - const rightArg = expression.args[1]! - - // Check both directions: field op value AND value op field - let fieldPath: Array | null = null - let queryValue: unknown - - if (leftArg.type === `ref` && rightArg.type === `val`) { - fieldPath = (leftArg as any).path - queryValue = (rightArg as any).value - } else if (leftArg.type === `val` && rightArg.type === `ref`) { - fieldPath = (rightArg as any).path - queryValue = (leftArg as any).value + const index = findIndexForField(collection, fieldPath) + if (index === undefined) { + return false } - if (fieldPath) { - const index = findIndexForField(collection, fieldPath) - if (index === undefined) { - return false - } - - // Mirror optimizeSimpleComparison's gates: the index must support the - // operation (e.g. the implicit key index only serves eq/in), and a range - // op additionally requires trustworthy index traversal for the operand's - // domain. The operand flip for `value op field` maps range ops onto range - // ops, so the classification needs no flip here. - const operation = expression.name as IndexOperation - if (!index.supports(operation)) { - return false - } - if ( - operation !== `eq` && - !canRangeOptimize(queryValue, index, collection) - ) { - return false - } - - return true + // Mirror optimizeSimpleComparison's gates: the index must support the + // normalized operation (e.g. the implicit key index only serves eq/in), + // and a range op additionally requires trustworthy index traversal for + // the operand's domain. + if (!index.supports(operation)) { + return false + } + if (operation !== `eq` && !canRangeOptimize(queryValue, index, collection)) { + return false } - return false + return true } /** diff --git a/packages/db/tests/key-index.test.ts b/packages/db/tests/key-index.test.ts index d6b5854032..a1e6140f51 100644 --- a/packages/db/tests/key-index.test.ts +++ b/packages/db/tests/key-index.test.ts @@ -9,6 +9,7 @@ import { } from '../src/utils/index-optimization.js' import { Func, PropRef, Value } from '../src/query/ir.js' import { mockSyncCollectionOptions } from './utils.js' +import type { IndexOperation } from '../src/indexes/base-index.js' type Item = { id: string @@ -263,4 +264,50 @@ describe(`canOptimizeExpression agrees with optimizeExpressionWithIndexes`, () = ) expect(canOptimizeExpression(eqExpr, collection)).toBe(true) }) + + it(`normalizes flipped operands before consulting index capabilities`, () => { + // An index that serves upper-bound ranges only: `5 > score` normalizes to + // `score < 5`, so both the optimizer and the predicate must consult + // supports(`lt`), not the expression's literal `gt` — otherwise the two + // disagree on either flipped form. + class UpperBoundOnlyIndex< + TKey extends string | number = string | number, + > extends BasicIndex { + public readonly supportedOperations = new Set([ + `eq`, + `lt`, + `lte`, + ]) + } + + const collection = createCollection( + mockSyncCollectionOptions<{ id: string; score: number }>({ + id: `key-index-flip-test`, + getKey: (item) => item.id, + initialData: [ + { id: `a`, score: 1 }, + { id: `b`, score: 7 }, + ], + }), + ) + collection.createIndex((row) => row.score, { + indexType: UpperBoundOnlyIndex, + }) + + // `5 > score` means `score < 5`: served by the index, and the predicate + // must agree. + const flippedLt = new Func(`gt`, [new Value(5), new PropRef([`score`])]) + const optimizedLt = optimizeExpressionWithIndexes(flippedLt, collection) + expect(optimizedLt.canOptimize).toBe(true) + expect(optimizedLt.matchingKeys).toEqual(new Set([`a`])) + expect(canOptimizeExpression(flippedLt, collection)).toBe(true) + + // `5 < score` means `score > 5`: not served (no gt support), and the + // predicate must agree. + const flippedGt = new Func(`lt`, [new Value(5), new PropRef([`score`])]) + expect( + optimizeExpressionWithIndexes(flippedGt, collection).canOptimize, + ).toBe(false) + expect(canOptimizeExpression(flippedGt, collection)).toBe(false) + }) })