From 9bfaa49b686d15ec72b775a55a2337a473bcfbfd Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 24 Aug 2026 17:44:55 -0600 Subject: [PATCH 1/7] Apply canonical demand identity review fixes --- packages/db/src/query/index.ts | 5 + packages/db/src/query/ir-stable-identity.ts | 573 +++++++++++- packages/db/src/query/predicate-utils.ts | 71 +- .../src/query/runtime-reference-identity.ts | 41 + packages/db/src/query/subset-dedupe.ts | 19 +- .../db/tests/query/ir-stable-identity.test.ts | 867 ++++++++++++++++-- .../query/load-subset-oracle.property.test.ts | 136 ++- .../db/tests/query/predicate-utils.test.ts | 17 + .../query-db-collection/e2e/query-filter.ts | 118 +-- packages/query-db-collection/src/query.ts | 15 +- .../query-db-collection/src/serialization.ts | 135 --- .../load-subset-lifecycle-oracle.test.ts | 90 +- 12 files changed, 1662 insertions(+), 425 deletions(-) create mode 100644 packages/db/src/query/runtime-reference-identity.ts delete mode 100644 packages/query-db-collection/src/serialization.ts diff --git a/packages/db/src/query/index.ts b/packages/db/src/query/index.ts index c330b56b95..2c587b589b 100644 --- a/packages/db/src/query/index.ts +++ b/packages/db/src/query/index.ts @@ -98,9 +98,13 @@ export { type LiveQueryCollectionUtils } from './live/collection-config-builder. export { UnhashableQueryIRError, canonicalizeQueryIR, + getLoadSubsetDemandKey, + getQueryIdentity, getStableQueryBuilderHash, getStableQueryIRHash, getStableValueHash, + type DemandKey, + type QueryIdentity, } from './ir-stable-identity.js' // Predicate utilities for predicate push-down @@ -112,6 +116,7 @@ export { isLimitSubset, isOffsetLimitSubset, isPredicateSubset, + isLoadSubsetCoveredBy, } from './predicate-utils.js' export { DeduplicatedLoadSubset } from './subset-dedupe.js' diff --git a/packages/db/src/query/ir-stable-identity.ts b/packages/db/src/query/ir-stable-identity.ts index a479536fb2..871321601f 100644 --- a/packages/db/src/query/ir-stable-identity.ts +++ b/packages/db/src/query/ir-stable-identity.ts @@ -1,5 +1,7 @@ +import { normalizeValue } from '../utils/comparison.js' import { isRefProxy, toExpression } from './builder/ref-proxy.js' import { getQueryIR } from './builder/index.js' +import { getRuntimeReferenceIdentity } from './runtime-reference-identity.js' import type { Aggregate, BasicExpression, @@ -14,6 +16,7 @@ import type { Where, } from './ir.js' import type { InitialQueryBuilder, QueryBuilder } from './builder/index.js' +import type { LoadSubsetOptions } from '../types.js' type StableIdentityValue = | null @@ -23,6 +26,27 @@ type StableIdentityValue = | Array | { [key: string]: StableIdentityValue } +type RuntimeValueIdentity = `structural` | `semantic` + +type AliasScope = { + bindings: ReadonlyMap + hasUnqualifiedOutput: boolean + parent: AliasScope | undefined +} + +declare const queryIdentityBrand: unique symbol +declare const demandKeyBrand: unique symbol + +/** Semantic identity for a query plan, independent of its runtime owners. */ +export type QueryIdentity = string & { + readonly [queryIdentityBrand]: true +} + +/** Exact identity for one loadSubset demand, including its requested window. */ +export type DemandKey = string & { + readonly [demandKeyBrand]: true +} + export class UnhashableQueryIRError extends Error { constructor( public readonly path: string, @@ -34,7 +58,7 @@ export class UnhashableQueryIRError extends Error { } export function getStableQueryIRHash(query: QueryIR): string { - return JSON.stringify(canonicalizeQueryIR(query)) + return getQueryIdentity(query) } export function getStableQueryBuilderHash( @@ -47,14 +71,168 @@ export function getStableValueHash(value: unknown, path = `value`): string { return JSON.stringify(canonicalizeRuntimeValue(value, path, new WeakSet())) } +/** + * Returns the semantic identity of a structured query. + * + * Logical conjunctions and disjunctions are associative, commutative, and + * idempotent. Equality operands are commutative, while reversed inequalities + * are normalized by inverting their operator. Order-sensitive clauses and + * function arguments retain their original order. + */ +export function getQueryIdentity(query: QueryIR): QueryIdentity { + return JSON.stringify(canonicalizeQueryIR(query)) as QueryIdentity +} + +/** Returns the semantic identity of one structured expression. */ +export function getStableExpressionHash(expression: BasicExpression): string { + return JSON.stringify( + canonicalizeExpression(expression, `expression`, new WeakSet(), `semantic`), + ) +} + +/** + * Returns the exact semantic identity of a loadSubset request. + * + * Abort signals and subscriptions are owners of a request, not part of the + * requested data, and therefore do not affect the key. A demand generation + * scopes one asynchronous attempt rather than the data it requests. Code that + * rejects stale work compares this key alongside its generation; query-db uses + * the key alone so equivalent data demands can reuse one cache entry across + * generations. + */ +export function getLoadSubsetDemandKey( + options: LoadSubsetOptions, +): DemandKey | undefined { + if ( + options.where === undefined && + !options.orderBy?.length && + options.limit === undefined && + (options.offset === undefined || options.offset === 0) && + options.cursor === undefined + ) { + // Query-db uses its base query key for the one unconstrained demand. An + // owner-only option must not create another cache entry for the same data. + return undefined + } + + const seen = new WeakSet() + const result: Record = { + type: `loadSubsetDemand`, + query: canonicalizeLoadSubsetQuery(options, `loadSubset`, seen), + } + + if (options.limit !== undefined) { + result.limit = canonicalizeRuntimeValue( + options.limit, + `loadSubset.limit`, + seen, + ) + } + + if (options.offset !== undefined && options.offset !== 0) { + result.offset = canonicalizeRuntimeValue( + options.offset, + `loadSubset.offset`, + seen, + ) + } + + if (options.cursor !== undefined) { + const cursor: Record = { + whereFrom: canonicalizeExpression( + options.cursor.whereFrom, + `loadSubset.cursor.whereFrom`, + seen, + `semantic`, + ), + whereCurrent: canonicalizeExpression( + options.cursor.whereCurrent, + `loadSubset.cursor.whereCurrent`, + seen, + `semantic`, + ), + } + if (options.cursor.lastKey !== undefined) { + cursor.lastKey = canonicalizeRuntimeValue( + options.cursor.lastKey, + `loadSubset.cursor.lastKey`, + seen, + ) + } + result.cursor = cursor + } + + return JSON.stringify(result) as DemandKey +} + export function canonicalizeQueryIR(query: QueryIR): StableIdentityValue { return canonicalizeQuery(query, `query`, new WeakSet()) } +function createAliasScope( + query: QueryIR, + parent: AliasScope | undefined, +): AliasScope { + const bindings = new Map() + + const bindSource = (source: From): void => { + if (source.type === `unionFrom`) { + source.sources.forEach(bindSource) + return + } + if (source.type === `unionAll`) return + if (!bindings.has(source.alias)) { + bindings.set(source.alias, bindings.size) + } + } + + bindSource(query.from) + query.join?.forEach(({ from }) => bindSource(from)) + return { + bindings, + hasUnqualifiedOutput: query.from.type === `unionAll`, + parent, + } +} + +function resolveAliasBinding( + scope: AliasScope | undefined, + alias: string, +): readonly [number, number] | undefined { + let current = scope + let parentDistance = 0 + while (current) { + const binding = current.bindings.get(alias) + if (binding !== undefined) return [parentDistance, binding] + // A result-level union has no source alias. Every downstream ref starts at + // an output field, including nested paths such as profile.id, so it must + // not fall through and bind that field name to an enclosing query alias. + if (current.hasUnqualifiedOutput) return undefined + current = current.parent + parentDistance++ + } + return undefined +} + function canonicalizeQuery( query: QueryIR, path: string, seen: WeakSet, + parentScope?: AliasScope, +): StableIdentityValue { + return canonicalizeQueryInScope( + query, + path, + seen, + createAliasScope(query, parentScope), + ) +} + +function canonicalizeQueryInScope( + query: QueryIR, + path: string, + seen: WeakSet, + scope: AliasScope, ): StableIdentityValue { if (query.fnSelect) { throw new UnhashableQueryIRError(`${path}.fnSelect`, `function select`) @@ -70,40 +248,63 @@ function canonicalizeQuery( const result: Record = { type: `query`, - from: canonicalizeSource(query.from, `${path}.from`, seen), + from: canonicalizeSource(query.from, `${path}.from`, seen, scope), } if (query.select) { - result.select = canonicalizeSelect(query.select, `${path}.select`, seen) + result.select = canonicalizeSelect( + query.select, + `${path}.select`, + seen, + scope, + ) } if (query.join) { result.join = query.join.map((join, index) => - canonicalizeJoin(join, `${path}.join[${index}]`, seen), + canonicalizeJoin(join, `${path}.join[${index}]`, seen, scope), ) } if (query.where) { - result.where = query.where.map((where, index) => - canonicalizeWhere(where, `${path}.where[${index}]`, seen), + result.where = canonicalizeImplicitConjunction( + query.where, + `${path}.where`, + seen, + scope, ) } if (query.groupBy) { result.groupBy = query.groupBy.map((expression, index) => - canonicalizeExpression(expression, `${path}.groupBy[${index}]`, seen), + canonicalizeExpression( + expression, + `${path}.groupBy[${index}]`, + seen, + `semantic`, + scope, + ), ) } if (query.having) { - result.having = query.having.map((having, index) => - canonicalizeWhere(having, `${path}.having[${index}]`, seen), + result.having = canonicalizeImplicitConjunction( + query.having, + `${path}.having`, + seen, + scope, ) } if (query.orderBy) { result.orderBy = query.orderBy.map((orderBy, index) => - canonicalizeOrderBy(orderBy, `${path}.orderBy[${index}]`, seen), + canonicalizeOrderBy( + orderBy, + `${path}.orderBy[${index}]`, + seen, + `semantic`, + scope, + ), ) } @@ -111,7 +312,7 @@ function canonicalizeQuery( result.limit = canonicalizeRuntimeValue(query.limit, `${path}.limit`, seen) } - if (query.offset !== undefined) { + if (query.offset !== undefined && query.offset !== 0) { result.offset = canonicalizeRuntimeValue( query.offset, `${path}.offset`, @@ -130,16 +331,79 @@ function canonicalizeQuery( return result } +function canonicalizeImplicitConjunction( + clauses: ReadonlyArray, + path: string, + seen: WeakSet, + scope: AliasScope, +): Array { + const canonical = clauses.map((clause, index) => + canonicalizeWhere(clause, `${path}[${index}]`, seen, scope), + ) + canonical.sort(compareStableIdentityValues) + + return canonical.filter( + (clause, index) => + index === 0 || + compareStableIdentityValues(clause, canonical[index - 1]!) !== 0, + ) +} + +function canonicalizeLoadSubsetQuery( + options: LoadSubsetOptions, + path: string, + seen: WeakSet, +): StableIdentityValue { + const result: Record = { + type: `loadSubsetQuery`, + } + + if (options.where !== undefined) { + result.where = canonicalizeExpression( + options.where, + `${path}.where`, + seen, + `semantic`, + ) + } + + if (options.orderBy?.length) { + result.orderBy = options.orderBy.map((orderBy, index) => + canonicalizeOrderBy( + orderBy, + `${path}.orderBy[${index}]`, + seen, + `semantic`, + ), + ) + } + + return result +} + function canonicalizeJoin( join: JoinClause, path: string, seen: WeakSet, + scope: AliasScope, ): StableIdentityValue { return { type: join.type, - from: canonicalizeSource(join.from, `${path}.from`, seen), - left: canonicalizeExpression(join.left, `${path}.left`, seen), - right: canonicalizeExpression(join.right, `${path}.right`, seen), + from: canonicalizeSource(join.from, `${path}.from`, seen, scope), + left: canonicalizeExpression( + join.left, + `${path}.left`, + seen, + `semantic`, + scope, + ), + right: canonicalizeExpression( + join.right, + `${path}.right`, + seen, + `semantic`, + scope, + ), } } @@ -147,11 +411,11 @@ function canonicalizeSource( source: From, path: string, seen: WeakSet, + scope: AliasScope, ): StableIdentityValue { if (source.type === `collectionRef`) { return { type: `collectionRef`, - alias: source.alias, collectionId: canonicalizeRuntimeValue( source.collection.id, `${path}.collection.id`, @@ -164,7 +428,12 @@ function canonicalizeSource( return { type: `unionFrom`, sources: source.sources.map((unionSource, index) => - canonicalizeSource(unionSource, `${path}.sources[${index}]`, seen), + canonicalizeSource( + unionSource, + `${path}.sources[${index}]`, + seen, + scope, + ), ), } } @@ -173,15 +442,21 @@ function canonicalizeSource( return { type: `unionAll`, queries: source.queries.map((query, index) => - canonicalizeQuery(query, `${path}.queries[${index}]`, seen), + // Branches are peers that may capture the union query's outer scope; + // they are not children of the union result row itself. + canonicalizeQuery( + query, + `${path}.queries[${index}]`, + seen, + scope.parent, + ), ), } } return { type: `queryRef`, - alias: source.alias, - query: canonicalizeQuery(source.query, `${path}.query`, seen), + query: canonicalizeQuery(source.query, `${path}.query`, seen, scope), } } @@ -189,6 +464,7 @@ function canonicalizeSelect( select: Select, path: string, seen: WeakSet, + scope?: AliasScope, ): StableIdentityValue { return { type: `select`, @@ -196,7 +472,7 @@ function canonicalizeSelect( .sort() .map((key) => [ key, - canonicalizeSelectValue(select[key]!, `${path}.${key}`, seen), + canonicalizeSelectValue(select[key]!, `${path}.${key}`, seen, scope), ]), } } @@ -205,26 +481,34 @@ function canonicalizeSelectValue( value: unknown, path: string, seen: WeakSet, + scope?: AliasScope, ): StableIdentityValue { if (isRefProxy(value)) { - return canonicalizeExpression(toExpression(value), path, seen) + return canonicalizeExpression( + toExpression(value), + path, + seen, + `semantic`, + scope, + ) } if (isExpression(value)) { - return canonicalizeExpression(value, path, seen) + return canonicalizeExpression(value, path, seen, `semantic`, scope) } if (isPlainObject(value)) { - return canonicalizeSelect(value as Select, path, seen) + return canonicalizeSelect(value as Select, path, seen, scope) } - return canonicalizeRuntimeValue(value, path, seen) + return canonicalizeSemanticRuntimeValue(value, path, seen, scope) } function canonicalizeWhere( where: Where | Having, path: string, seen: WeakSet, + scope?: AliasScope, ): StableIdentityValue { if (isWhereObject(where)) { const result: Record = { @@ -233,6 +517,8 @@ function canonicalizeWhere( where.expression, `${path}.expression`, seen, + `semantic`, + scope, ), } @@ -243,19 +529,23 @@ function canonicalizeWhere( return result } - return canonicalizeExpression(where, path, seen) + return canonicalizeExpression(where, path, seen, `semantic`, scope) } function canonicalizeOrderBy( orderBy: OrderByClause, path: string, seen: WeakSet, + runtimeValueIdentity: RuntimeValueIdentity = `structural`, + scope?: AliasScope, ): StableIdentityValue { return { expression: canonicalizeExpression( orderBy.expression, `${path}.expression`, seen, + runtimeValueIdentity, + scope, ), compareOptions: canonicalizeRuntimeValue( orderBy.compareOptions, @@ -273,31 +563,96 @@ function canonicalizeExpression( | ConditionalSelect, path: string, seen: WeakSet, + runtimeValueIdentity: RuntimeValueIdentity = `structural`, + scope?: AliasScope, ): StableIdentityValue { if (expression.type === `ref`) { + const binding = resolveAliasBinding(scope, expression.path[0] ?? ``) return { type: `ref`, - path: expression.path.map((segment, index) => - canonicalizeRuntimeValue(segment, `${path}.path[${index}]`, seen), - ), + path: + binding === undefined + ? expression.path.map((segment, index) => + canonicalizeRuntimeValue(segment, `${path}.path[${index}]`, seen), + ) + : [ + [`binding`, ...binding], + ...expression.path + .slice(1) + .map((segment, index) => + canonicalizeRuntimeValue( + segment, + `${path}.path[${index + 1}]`, + seen, + ), + ), + ], } } if (expression.type === `val`) { return { type: `val`, - value: canonicalizeRuntimeValue(expression.value, `${path}.value`, seen), + value: + runtimeValueIdentity === `semantic` + ? canonicalizeSemanticRuntimeValue( + expression.value, + `${path}.value`, + seen, + scope, + ) + : canonicalizeRuntimeValue(expression.value, `${path}.value`, seen), } } if (expression.type === `func`) { - return { - type: `func`, - name: expression.name, - args: expression.args.map((arg, index) => - canonicalizeExpression(arg, `${path}.args[${index}]`, seen), - ), + if ( + expression.name === `in` && + expression.args.length === 2 && + expression.args[1]?.type === `val` && + Array.isArray(expression.args[1].value) + ) { + const candidates = expression.args[1].value.map((value, index) => + runtimeValueIdentity === `semantic` + ? canonicalizeSemanticRuntimeValue( + value, + `${path}.args[1].value[${index}]`, + seen, + scope, + ) + : canonicalizeRuntimeValue( + value, + `${path}.args[1].value[${index}]`, + seen, + ), + ) + return canonicalizeFunction(expression.name, [ + canonicalizeExpression( + expression.args[0]!, + `${path}.args[0]`, + seen, + runtimeValueIdentity, + scope, + ), + { + type: `val`, + // IN tests membership. Candidate order and duplicates do not change + // its result, but each candidate keeps its own equality semantics. + value: [`set`, sortUniqueStableIdentityValues(candidates)], + }, + ]) } + + const args = expression.args.map((arg, index) => + canonicalizeExpression( + arg, + `${path}.args[${index}]`, + seen, + runtimeValueIdentity, + scope, + ), + ) + return canonicalizeFunction(expression.name, args) } if (expression.type === `agg`) { @@ -305,7 +660,13 @@ function canonicalizeExpression( type: `agg`, name: expression.name, args: expression.args.map((arg, index) => - canonicalizeExpression(arg, `${path}.args[${index}]`, seen), + canonicalizeExpression( + arg, + `${path}.args[${index}]`, + seen, + runtimeValueIdentity, + scope, + ), ), } } @@ -318,11 +679,14 @@ function canonicalizeExpression( branch.condition, `${path}.branches[${index}].condition`, seen, + `semantic`, + scope, ), value: canonicalizeSelectValue( branch.value, `${path}.branches[${index}].value`, seen, + scope, ), })), } @@ -332,24 +696,35 @@ function canonicalizeExpression( expression.defaultValue, `${path}.defaultValue`, seen, + scope, ) } return result } + const childScope = createAliasScope(expression.query, scope) const result: Record = { type: `includesSubquery`, - query: canonicalizeQuery(expression.query, `${path}.query`, seen), + query: canonicalizeQueryInScope( + expression.query, + `${path}.query`, + seen, + childScope, + ), correlationField: canonicalizeExpression( expression.correlationField, `${path}.correlationField`, seen, + `semantic`, + scope, ), childCorrelationField: canonicalizeExpression( expression.childCorrelationField, `${path}.childCorrelationField`, seen, + `semantic`, + childScope, ), fieldName: expression.fieldName, materialization: expression.materialization, @@ -357,7 +732,7 @@ function canonicalizeExpression( if (expression.parentFilters) { result.parentFilters = expression.parentFilters.map((where, index) => - canonicalizeWhere(where, `${path}.parentFilters[${index}]`, seen), + canonicalizeWhere(where, `${path}.parentFilters[${index}]`, seen, scope), ) } @@ -368,6 +743,8 @@ function canonicalizeExpression( projection, `${path}.parentProjection[${index}]`, seen, + `semantic`, + scope, ), ) } @@ -379,6 +756,85 @@ function canonicalizeExpression( return result } +function canonicalizeFunction( + name: string, + args: Array, +): StableIdentityValue { + if ((name === `and` || name === `or`) && args.length > 0) { + const flattened = args.flatMap((arg) => + isCanonicalFunction(arg, name) ? arg.args : [arg], + ) + const unique = sortUniqueStableIdentityValues(flattened) + + return unique.length === 1 + ? unique[0]! + : { type: `func`, name, args: unique } + } + + if (name === `eq` && args.length === 2) { + args.sort(compareStableIdentityValues) + return { type: `func`, name, args } + } + + if ( + (name === `gt` || name === `gte` || name === `lt` || name === `lte`) && + args.length === 2 && + compareStableIdentityValues(args[0]!, args[1]!) > 0 + ) { + return { + type: `func`, + name: invertComparison(name), + args: [args[1]!, args[0]!], + } + } + + return { type: `func`, name, args } +} + +function sortUniqueStableIdentityValues( + values: Array, +): Array { + values.sort(compareStableIdentityValues) + return values.filter( + (value, index) => + index === 0 || + compareStableIdentityValues(value, values[index - 1]!) !== 0, + ) +} + +function isCanonicalFunction( + value: StableIdentityValue, + name: string, +): value is { + type: string + name: string + args: Array +} { + return ( + value !== null && + typeof value === `object` && + !Array.isArray(value) && + value.type === `func` && + value.name === name && + Array.isArray(value.args) + ) +} + +function invertComparison( + name: `gt` | `gte` | `lt` | `lte`, +): `gt` | `gte` | `lt` | `lte` { + switch (name) { + case `gt`: + return `lt` + case `gte`: + return `lte` + case `lt`: + return `gt` + case `lte`: + return `gte` + } +} + function canonicalizeRuntimeValue( value: unknown, path: string, @@ -497,6 +953,47 @@ function canonicalizeRuntimeValue( throw new UnhashableQueryIRError(path, `non-plain object value`) } +function canonicalizeSemanticRuntimeValue( + value: unknown, + path: string, + seen: WeakSet, + scope?: AliasScope, +): StableIdentityValue { + if (isRefProxy(value)) { + return canonicalizeExpression( + toExpression(value), + path, + seen, + `semantic`, + scope, + ) + } + + if (typeof value === `number` && Object.is(value, -0)) { + return canonicalizeRuntimeValue(0, path, seen) + } + + // Equality compares Uint8Array and Buffer values by content, independent of + // their concrete constructor and size. + const isUint8Array = + (typeof Buffer !== `undefined` && value instanceof Buffer) || + value instanceof Uint8Array + if (isUint8Array) { + return [`binary`, `Uint8Array`, Array.from(value as Uint8Array)] + } + + const normalized = normalizeValue(value) + if (normalized !== value) { + return canonicalizeRuntimeValue(normalized, path, seen) + } + + if (typeof value === `object` && value !== null) { + return getRuntimeReferenceIdentity(value) + } + + return canonicalizeRuntimeValue(value, path, seen) +} + function compareStableIdentityValues( left: StableIdentityValue, right: StableIdentityValue, diff --git a/packages/db/src/query/predicate-utils.ts b/packages/db/src/query/predicate-utils.ts index 4483d44ae2..2808a19937 100644 --- a/packages/db/src/query/predicate-utils.ts +++ b/packages/db/src/query/predicate-utils.ts @@ -1,4 +1,8 @@ import { Func, Value } from './ir.js' +import { + UnhashableQueryIRError, + getStableExpressionHash, +} from './ir-stable-identity.js' import type { BasicExpression, OrderBy, PropRef } from './ir.js' import type { LoadSubsetOptions } from '../types.js' @@ -872,6 +876,16 @@ export function isPredicateSubset( // Example: superset = {where: status='active', limit: 10, offset: 0, orderBy: desc} // subset = {where: status='active', limit: 5, offset: 0, orderBy: desc} // The top 5 active items ARE contained in the top 10 active items. + if (superset.limit !== undefined || superset.cursor !== undefined) { + // A cursor page only covers another request for the same page, whether or + // not the adapter also uses a numeric limit. + // Adapters may use the cursor expressions instead of offset, so matching + // offsets alone do not prove that two requests load the same rows. + if (!areCursorExpressionsEqual(subset.cursor, superset.cursor)) { + return false + } + } + if (superset.limit !== undefined) { // For limited supersets, where clauses must be equal if (!areWhereClausesEqual(subset.where, superset.where)) { @@ -893,6 +907,31 @@ export function isPredicateSubset( ) } +/** + * Returns whether established coverage satisfies a requested demand. + * + * Coverage is a directional relation. It must not be replaced with DemandKey + * equality, which answers whether two exact requests are the same. + */ +export function isLoadSubsetCoveredBy( + demand: LoadSubsetOptions, + coverage: LoadSubsetOptions, +): boolean { + return isPredicateSubset(demand, coverage) +} + +function areCursorExpressionsEqual( + a: LoadSubsetOptions[`cursor`], + b: LoadSubsetOptions[`cursor`], +): boolean { + if (a === undefined || b === undefined) return a === b + return ( + Object.is(a.lastKey, b.lastKey) && + areExpressionsEqual(a.whereFrom, b.whereFrom) && + areExpressionsEqual(a.whereCurrent, b.whereCurrent) + ) +} + /** * Check if two where clauses are structurally equal. * Used for limited query subset checks where subset relationship isn't sufficient. @@ -1047,32 +1086,34 @@ function findPredicateWithOperator( } function areExpressionsEqual(a: BasicExpression, b: BasicExpression): boolean { - if (a.type !== b.type) { - return false + try { + return getStableExpressionHash(a) === getStableExpressionHash(b) + } catch (error) { + if (!(error instanceof UnhashableQueryIRError)) throw error + return areExpressionsStructurallyEqual(a, b) } +} +function areExpressionsStructurallyEqual( + a: BasicExpression, + b: BasicExpression, +): boolean { + if (a.type !== b.type) return false if (a.type === `val` && b.type === `val`) { return areValuesEqual(a.value, b.value) } - if (a.type === `ref` && b.type === `ref`) { return areRefsEqual(a, b) } - if (a.type === `func` && b.type === `func`) { - const aFunc = a - const bFunc = b - if (aFunc.name !== bFunc.name) { - return false - } - if (aFunc.args.length !== bFunc.args.length) { - return false - } - return aFunc.args.every((arg, i) => - areExpressionsEqual(arg, bFunc.args[i]!), + return ( + a.name === b.name && + a.args.length === b.args.length && + a.args.every((arg, index) => + areExpressionsStructurallyEqual(arg, b.args[index]!), + ) ) } - return false } diff --git a/packages/db/src/query/runtime-reference-identity.ts b/packages/db/src/query/runtime-reference-identity.ts new file mode 100644 index 0000000000..d903060522 --- /dev/null +++ b/packages/db/src/query/runtime-reference-identity.ts @@ -0,0 +1,41 @@ +export type RuntimeReferenceIdentity = [ + `runtimeReference`, + namespace: string, + sequence: number, +] + +export function createRuntimeReferenceIdentityFactory(): ( + value: object, +) => RuntimeReferenceIdentity { + const namespace = createRuntimeReferenceNamespace() + const referenceIds = new WeakMap() + let sequence = 0 + + return (value) => { + let referenceId = referenceIds.get(value) + if (referenceId === undefined) { + referenceId = ++sequence + referenceIds.set(value, referenceId) + } + return [`runtimeReference`, namespace, referenceId] + } +} + +export const getRuntimeReferenceIdentity = + createRuntimeReferenceIdentityFactory() + +function createRuntimeReferenceNamespace(): string { + const randomValues = new Uint32Array(4) + const runtimeCrypto = Reflect.get(globalThis, `crypto`) as + | { getRandomValues: (values: Uint32Array) => Uint32Array } + | undefined + if (runtimeCrypto !== undefined) { + runtimeCrypto.getRandomValues(randomValues) + return Array.from(randomValues, (value) => value.toString(36)).join(`-`) + } + + // Reference equality cannot survive a runtime boundary. A per-runtime nonce + // prevents a persisted key from matching an unrelated reference after a + // reload, even on platforms without Web Crypto. + return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}-${Math.random().toString(36).slice(2)}` +} diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index c0381f78d5..f8e49d0f08 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -1,5 +1,5 @@ import { - isPredicateSubset, + isLoadSubsetCoveredBy, isWhereSubset, minusWherePredicates, unionWherePredicates, @@ -61,7 +61,7 @@ export class DeduplicatedLoadSubset { // Flag to track if we've loaded all data (unlimited call with no where clause) private hasLoadedAllData = false - // List of all limited calls (with limit, possibly with orderBy) + // List of calls with a finite or cursor-relative result window. // We clone options before storing to prevent mutation of stored predicates private limitedCalls: Array = [] @@ -109,9 +109,9 @@ export class DeduplicatedLoadSubset { } // Check against limited calls - if (options.limit !== undefined) { + if (options.limit !== undefined || options.cursor !== undefined) { const alreadyLoaded = this.limitedCalls.some((loaded) => - isPredicateSubset(options, loaded), + isLoadSubsetCoveredBy(options, loaded), ) if (alreadyLoaded) { @@ -124,7 +124,8 @@ export class DeduplicatedLoadSubset { // This prevents duplicate requests when concurrent calls have subset relationships const matchingInflight = this.inflightCalls.find( (inflight) => - !inflight.lease.aborted && isPredicateSubset(options, inflight.options), + !inflight.lease.aborted && + isLoadSubsetCoveredBy(options, inflight.options), ) if (matchingInflight !== undefined) { @@ -148,7 +149,11 @@ export class DeduplicatedLoadSubset { const lease = createSharedAbortLease(options.signal) const trackingOptions = cloneOptions({ ...options, signal: lease.signal }) const loadOptions = cloneOptions({ ...options, signal: lease.signal }) - if (this.unlimitedWhere !== undefined && options.limit === undefined) { + if ( + this.unlimitedWhere !== undefined && + options.limit === undefined && + options.cursor === undefined + ) { // Compute difference to get only the missing data // We can only do this for unlimited queries // and we can only remove data that was loaded from unlimited queries @@ -230,7 +235,7 @@ export class DeduplicatedLoadSubset { private updateTracking(options: LoadSubsetOptions): void { // Update tracking based on whether this was a limited or unlimited call - if (options.limit === undefined) { + if (options.limit === undefined && options.cursor === undefined) { // Unlimited call - update combined where predicate // We ignore orderBy for unlimited calls as mentioned in requirements if (options.where === undefined) { diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index 145b3232af..3d9bd86053 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -1,4 +1,6 @@ import { describe, expect, it } from 'vitest' +import { fc, test as fcTest } from '@fast-check/vitest' +import { Temporal } from 'temporal-polyfill' import { CollectionImpl } from '../../src/collection/index.js' import { Query, getQueryIR } from '../../src/query/builder/index.js' import { @@ -18,18 +20,36 @@ import { length, like, lower, + lt, max, not, or, + subtract, sum, upper, } from '../../src/query/builder/functions.js' import { UnhashableQueryIRError, + getLoadSubsetDemandKey, + getQueryIdentity, + getStableExpressionHash, getStableQueryIRHash, getStableValueHash, } from '../../src/query/ir-stable-identity.js' -import type { QueryIR } from '../../src/query/ir.js' +import { + CollectionRef, + Func, + IncludesSubquery, + PropRef, + QueryRef, + UnionAll, + Value, +} from '../../src/query/ir.js' +import { compileExpression } from '../../src/query/compiler/evaluators.js' +import { isLoadSubsetCoveredBy } from '../../src/query/predicate-utils.js' +import { createRuntimeReferenceIdentityFactory } from '../../src/query/runtime-reference-identity.js' +import type { BasicExpression, QueryIR } from '../../src/query/ir.js' +import type { LoadSubsetOptions } from '../../src/types.js' interface User { id: number @@ -52,6 +72,27 @@ interface User { largeViewCount?: bigint } +const referenceSemanticPairArbitrary = fc.oneof( + fc + .array(fc.integer()) + .map((values): [unknown, unknown] => [[...values], [...values]]), + fc + .dictionary(fc.string(), fc.integer()) + .map((value): [unknown, unknown] => [{ ...value }, { ...value }]), + fc + .array(fc.tuple(fc.string(), fc.integer())) + .map((entries): [unknown, unknown] => [new Map(entries), new Map(entries)]), + fc + .array(fc.integer()) + .map((values): [unknown, unknown] => [new Set(values), new Set(values)]), + fc + .int16Array() + .map((value): [unknown, unknown] => [ + new Int16Array(value), + new Int16Array(value), + ]), +) + interface Post { id: number userId: number @@ -81,12 +122,557 @@ describe(`stable runtime value hashing`, () => { }) }) +describe(`semantic expression identity`, () => { + const age = new PropRef([`user`, `age`]) + const active = new PropRef([`user`, `active`]) + type EquivalentExpressionPair = { + original: BasicExpression + equivalent: BasicExpression + } + + const comparisonPairArbitrary: fc.Arbitrary = fc + .record({ + operator: fc.constantFrom<`gt` | `gte` | `lt` | `lte`>( + `gt`, + `gte`, + `lt`, + `lte`, + ), + threshold: fc.integer(), + }) + .map(({ operator, threshold }) => { + const inverse: Record<`gt` | `gte` | `lt` | `lte`, string> = { + gt: `lt`, + gte: `lte`, + lt: `gt`, + lte: `gte`, + } + return { + original: new Func(operator, [age, new Value(threshold)]), + equivalent: new Func(inverse[operator], [ + new Value(threshold), + age, + ]), + } + }) + const equalityPairArbitrary: fc.Arbitrary = fc + .boolean() + .map((value) => ({ + original: new Func(`eq`, [active, new Value(value)]), + equivalent: new Func(`eq`, [new Value(value), active]), + })) + const membershipPairArbitrary: fc.Arbitrary = fc + .uniqueArray(fc.integer(), { minLength: 1, maxLength: 8 }) + .map((values) => ({ + original: new Func(`in`, [age, new Value(values)]), + equivalent: new Func(`in`, [ + age, + new Value([...values].reverse().concat(values[0]!)), + ]), + })) + const atomicExpressionPairArbitrary = fc.oneof( + comparisonPairArbitrary, + equalityPairArbitrary, + membershipPairArbitrary, + ) + const equivalentExpressionPairArbitrary = fc.oneof( + { weight: 3, arbitrary: atomicExpressionPairArbitrary }, + { + weight: 2, + arbitrary: fc + .tuple( + fc.constantFrom(`and`, `or`), + atomicExpressionPairArbitrary, + atomicExpressionPairArbitrary, + ) + .map(([operator, left, right]) => ({ + original: new Func(operator, [ + left.original, + new Func(operator, [right.original, left.original]), + ]), + equivalent: new Func(operator, [ + right.equivalent, + left.equivalent, + ]), + })), + }, + ) + + it(`normalizes associative, commutative, and idempotent boolean forms`, () => { + const adult = new Func(`gte`, [age, new Value(18)]) + const enabled = new Func(`eq`, [active, new Value(true)]) + const nested = new Func(`and`, [ + enabled, + new Func(`and`, [adult, enabled]), + ]) + const flat = new Func(`and`, [adult, enabled]) + + expect(getStableExpressionHash(nested)).toBe(getStableExpressionHash(flat)) + expect(getStableExpressionHash(new Func(`or`, [adult, adult]))).toBe( + getStableExpressionHash(adult), + ) + }) + + it(`normalizes equality and reversed inequalities`, () => { + expect(getStableExpressionHash(new Func(`eq`, [age, new Value(18)]))).toBe( + getStableExpressionHash(new Func(`eq`, [new Value(18), age])), + ) + expect(getStableExpressionHash(new Func(`gt`, [age, new Value(18)]))).toBe( + getStableExpressionHash(new Func(`lt`, [new Value(18), age])), + ) + }) + + it(`preserves order-sensitive function arguments`, () => { + expect( + getStableExpressionHash(new Func(`subtract`, [age, new Value(1)])), + ).not.toBe( + getStableExpressionHash(new Func(`subtract`, [new Value(1), age])), + ) + }) + + fcTest.prop([ + equivalentExpressionPairArbitrary, + fc.record({ age: fc.integer(), active: fc.boolean() }), + ])(`canonical expression grammar preserves semantics`, (pair, sample) => { + const row = { user: sample } + + expect(compileExpression(pair.original)(row)).toBe( + compileExpression(pair.equivalent)(row), + ) + expect(getStableExpressionHash(pair.original)).toBe( + getStableExpressionHash(pair.equivalent), + ) + }) + + fcTest.prop([referenceSemanticPairArbitrary])( + `keeps reference-semantic values distinct across identity and coverage`, + ([first, second]) => { + const value = new PropRef([`row`, `value`]) + const firstPredicate = new Func(`eq`, [value, new Value(first)]) + const secondPredicate = new Func(`eq`, [ + value, + new Value(second), + ]) + const row = { row: { value: first } } + + expect(compileExpression(firstPredicate)(row)).toBe(true) + expect(compileExpression(secondPredicate)(row)).toBe(false) + expect(getStableExpressionHash(firstPredicate)).not.toBe( + getStableExpressionHash(secondPredicate), + ) + expect( + getLoadSubsetDemandKey({ where: firstPredicate, limit: 1 }), + ).not.toBe(getLoadSubsetDemandKey({ where: secondPredicate, limit: 1 })) + expect( + isLoadSubsetCoveredBy( + { where: firstPredicate, limit: 1 }, + { where: secondPredicate, limit: 1 }, + ), + ).toBe(false) + }, + ) + + it(`does not reuse reference identities across runtimes`, () => { + const firstRuntime = createRuntimeReferenceIdentityFactory() + const secondRuntime = createRuntimeReferenceIdentityFactory() + + expect(firstRuntime({ a: 1 })).not.toEqual(secondRuntime({ b: 2 })) + }) + + fcTest.prop([ + fc.uniqueArray(fc.oneof(fc.integer(), fc.string(), fc.boolean()), { + minLength: 1, + maxLength: 8, + }), + ])(`treats IN candidates as a set`, (candidates) => { + const value = new PropRef([`row`, `value`]) + const ordered = new Func(`in`, [value, new Value(candidates)]) + const reordered = new Func(`in`, [ + value, + new Value([...candidates].reverse().concat(candidates[0]!)), + ]) + + for (const candidate of candidates) { + const row = { row: { value: candidate } } + expect(compileExpression(ordered)(row)).toBe( + compileExpression(reordered)(row), + ) + } + expect(getStableExpressionHash(ordered)).toBe( + getStableExpressionHash(reordered), + ) + expect(getLoadSubsetDemandKey({ where: ordered })).toBe( + getLoadSubsetDemandKey({ where: reordered }), + ) + }) +}) + +describe(`loadSubset demand identity`, () => { + const id = new PropRef([`id`]) + const group = new PropRef([`group`]) + const first = new Func(`eq`, [id, new Value(`a`)]) + const second = new Func(`eq`, [group, new Value(`x`)]) + const orderBy: NonNullable = [ + { + expression: id, + compareOptions: { direction: `asc`, nulls: `first` }, + }, + { + expression: group, + compareOptions: { direction: `desc`, nulls: `last` }, + }, + ] + + it(`includes the exact requested window`, () => { + const narrow = { where: first, orderBy, limit: 10, offset: 5 } + const wide = { where: first, orderBy, limit: 20, offset: 0 } + + expect(getLoadSubsetDemandKey(narrow)).not.toBe( + getLoadSubsetDemandKey(wide), + ) + }) + + it(`normalizes predicates but preserves orderBy sequence`, () => { + const left = new Func(`and`, [first, second]) + const right = new Func(`and`, [second, first]) + + expect(getLoadSubsetDemandKey({ where: left, orderBy })).toBe( + getLoadSubsetDemandKey({ where: right, orderBy }), + ) + expect(getLoadSubsetDemandKey({ where: left, orderBy })).not.toBe( + getLoadSubsetDemandKey({ where: right, orderBy: [...orderBy].reverse() }), + ) + }) + + it(`includes cursor shape and excludes runtime owners`, () => { + const cursor = { + whereFrom: new Func(`gt`, [id, new Value(`a`)]), + whereCurrent: first, + lastKey: `a`, + } + const firstOwner = new AbortController() + const secondOwner = new AbortController() + const subscription = {} as NonNullable + + expect( + getLoadSubsetDemandKey({ + where: first, + cursor, + signal: firstOwner.signal, + subscription, + }), + ).toBe( + getLoadSubsetDemandKey({ + where: first, + cursor, + signal: secondOwner.signal, + }), + ) + expect(getLoadSubsetDemandKey({ where: first, cursor })).not.toBe( + getLoadSubsetDemandKey({ + where: first, + cursor: { ...cursor, lastKey: `b` }, + }), + ) + }) + + it(`uses the base query key for an unconstrained owner-only demand`, () => { + expect(getLoadSubsetDemandKey({})).toBeUndefined() + expect(getLoadSubsetDemandKey({ offset: 0 })).toBeUndefined() + expect( + getLoadSubsetDemandKey({ signal: new AbortController().signal }), + ).toBeUndefined() + expect(getLoadSubsetDemandKey({ where: first, offset: 0 })).toBe( + getLoadSubsetDemandKey({ where: first }), + ) + }) + + it.each([ + [`signed zero`, -0, 0], + [`invalid Date`, new Date(Number.NaN), new Date(Number.NaN)], + [ + `Temporal.PlainDate`, + Temporal.PlainDate.from(`2024-01-15`), + Temporal.PlainDate.from(`2024-01-15`), + ], + [ + `Temporal.Duration`, + Temporal.Duration.from(`PT1H`), + Temporal.Duration.from(`PT1H`), + ], + [ + `large cross-constructor binary`, + new Uint8Array(129).fill(7), + Buffer.from(new Uint8Array(129).fill(7)), + ], + ])( + `uses comparison semantics for equivalent %s values`, + (_label, firstValue, secondValue) => { + const value = new PropRef([`row`, `value`]) + const firstPredicate = new Func(`eq`, [ + value, + new Value(firstValue), + ]) + const secondPredicate = new Func(`eq`, [ + value, + new Value(secondValue), + ]) + + expect( + compileExpression(firstPredicate)({ row: { value: secondValue } }), + ).toBe(true) + expect(getStableExpressionHash(firstPredicate)).toBe( + getStableExpressionHash(secondPredicate), + ) + expect(getLoadSubsetDemandKey({ where: firstPredicate })).toBe( + getLoadSubsetDemandKey({ where: secondPredicate }), + ) + expect(getQueryIdentity(createProfileValueQuery(firstValue))).toBe( + getQueryIdentity(createProfileValueQuery(secondValue)), + ) + }, + ) +}) + const postsCollection = new CollectionImpl({ id: `posts`, getKey: (item) => item.id, sync: { sync: () => {} }, }) +function createProfileValueQuery(value: unknown): QueryIR { + return { + ...getQueryIR(new Query().from({ user: usersCollection })), + where: [ + new Func(`eq`, [ + new PropRef([`user`, `profile`]), + new Value(value), + ]), + ], + } +} + +function createAlphaRenamedJoinQuery( + userAlias: string, + postAlias: string, +): QueryIR { + return { + from: new CollectionRef( + usersCollection as unknown as ConstructorParameters< + typeof CollectionRef + >[0], + userAlias, + ), + join: [ + { + type: `inner`, + from: new CollectionRef( + postsCollection as unknown as ConstructorParameters< + typeof CollectionRef + >[0], + postAlias, + ), + left: new PropRef([userAlias, `id`]), + right: new PropRef([postAlias, `userId`]), + }, + ], + where: [ + new Func(`eq`, [new PropRef([postAlias, `published`]), new Value(true)]), + ], + select: { + userId: new PropRef([userAlias, `id`]), + postTitle: new PropRef([postAlias, `title`]), + }, + orderBy: [ + { + expression: new PropRef([postAlias, `createdAt`]), + compareOptions: { direction: `desc`, nulls: `last` }, + }, + ], + } +} + +function createAlphaRenamedNestedQuery( + innerAlias: string, + outerAlias: string, +): QueryIR { + const inner: QueryIR = { + from: new CollectionRef( + usersCollection as unknown as ConstructorParameters< + typeof CollectionRef + >[0], + innerAlias, + ), + select: { + id: new PropRef([innerAlias, `id`]), + status: new PropRef([innerAlias, `status`]), + }, + } + return { + from: new QueryRef(inner, outerAlias), + where: [ + new Func(`eq`, [ + new PropRef([outerAlias, `status`]), + new Value(`active`), + ]), + ], + select: { id: new PropRef([outerAlias, `id`]) }, + } +} + +function createUnionDerivedNestedQuery(outerAlias: string): QueryIR { + const users = new Query() + .from({ user: usersCollection }) + .select(({ user }) => ({ id: user.id, kind: user.status })) + const posts = new Query() + .from({ post: postsCollection }) + .select(({ post }) => ({ id: post.id, kind: post.title })) + const union = new Query() + .unionAll(users, posts) + .where(({ kind }) => eq(kind, `active`)) + + return getQueryIR( + new Query().from({ [outerAlias]: union } as Record), + ) +} + +function createUnionDerivedNestedOutputQuery(outerAlias: string): QueryIR { + const users = new Query() + .from({ user: usersCollection }) + .select(({ user }) => ({ profile: { id: user.id } })) + const posts = new Query() + .from({ post: postsCollection }) + .select(({ post }) => ({ profile: { id: post.id } })) + const union = new Query() + .unionAll(users, posts) + .where(({ profile }) => eq(profile.id, 1)) + + return getQueryIR( + new Query().from({ [outerAlias]: union } as Record), + ) +} + +function createUnionDerivedIncludesQuery(parentAlias: string): QueryIR { + const firstPosts = new Query() + .from({ post: postsCollection }) + .select(({ post }) => ({ id: post.id, userId: post.userId })) + const secondPosts = new Query() + .from({ otherPost: postsCollection }) + .select(({ otherPost }) => ({ + id: otherPost.id, + userId: otherPost.userId, + })) + const childQuery = getQueryIR(new Query().unionAll(firstPosts, secondPosts)) + const posts = new IncludesSubquery( + childQuery, + new PropRef([parentAlias, `id`]), + new PropRef([`userId`]), + `posts`, + undefined, + undefined, + `array`, + ) + + return { + from: new CollectionRef( + usersCollection as unknown as ConstructorParameters< + typeof CollectionRef + >[0], + parentAlias, + ), + select: { + id: new PropRef([parentAlias, `id`]), + posts, + }, + } +} + +function createCorrelatedUnionIncludesQuery(parentAlias: string): QueryIR { + const createBranch = (childAlias: string): QueryIR => ({ + from: new CollectionRef( + postsCollection as unknown as ConstructorParameters< + typeof CollectionRef + >[0], + childAlias, + ), + select: { + profile: { id: new PropRef([childAlias, `id`]) }, + userId: new PropRef([childAlias, `userId`]), + parentAge: new PropRef([parentAlias, `age`]), + }, + }) + const childQuery: QueryIR = { + from: new UnionAll([createBranch(`firstPost`), createBranch(`secondPost`)]), + where: [new Func(`eq`, [new PropRef([`profile`, `id`]), new Value(1)])], + } + const posts = new IncludesSubquery( + childQuery, + new PropRef([parentAlias, `id`]), + new PropRef([`userId`]), + `posts`, + undefined, + [new PropRef([parentAlias, `age`])], + `array`, + ) + + return { + from: new CollectionRef( + usersCollection as unknown as ConstructorParameters< + typeof CollectionRef + >[0], + parentAlias, + ), + select: { + id: new PropRef([parentAlias, `id`]), + posts, + }, + } +} + +function createAlphaRenamedIncludesQuery( + parentAlias: string, + childAlias: string, +): QueryIR { + const childQuery: QueryIR = { + from: new CollectionRef( + postsCollection as unknown as ConstructorParameters< + typeof CollectionRef + >[0], + childAlias, + ), + select: { + id: new PropRef([childAlias, `id`]), + title: new PropRef([childAlias, `title`]), + }, + } + const posts = new IncludesSubquery( + childQuery, + new PropRef([parentAlias, `id`]), + new PropRef([childAlias, `userId`]), + `posts`, + [ + new Func(`eq`, [ + new PropRef([parentAlias, `status`]), + new Value(`active`), + ]), + ], + [new PropRef([parentAlias, `id`])], + `array`, + ) + return { + from: new CollectionRef( + usersCollection as unknown as ConstructorParameters< + typeof CollectionRef + >[0], + parentAlias, + ), + select: { + id: new PropRef([parentAlias, `id`]), + posts, + }, + } +} + const structuredQueries: Array<[string, () => QueryIR]> = [ [ `basic collection source`, @@ -359,6 +945,175 @@ describe(`stable QueryIR identity smoke test`, () => { ) }) + fcTest.prop([ + fc.uniqueArray(fc.stringMatching(/^[a-z][a-z0-9]{0,8}$/), { + minLength: 4, + maxLength: 4, + }), + ])(`does not depend on lexical source aliases`, (aliases) => { + const [firstUser, firstPost, secondUser, secondPost] = aliases + + expect( + getQueryIdentity(createAlphaRenamedJoinQuery(firstUser!, firstPost!)), + ).toBe( + getQueryIdentity(createAlphaRenamedJoinQuery(secondUser!, secondPost!)), + ) + expect( + getQueryIdentity(createAlphaRenamedNestedQuery(firstUser!, firstPost!)), + ).toBe( + getQueryIdentity(createAlphaRenamedNestedQuery(secondUser!, secondPost!)), + ) + expect( + getQueryIdentity(createAlphaRenamedIncludesQuery(firstUser!, firstPost!)), + ).toBe( + getQueryIdentity( + createAlphaRenamedIncludesQuery(secondUser!, secondPost!), + ), + ) + }) + + fcTest.prop([ + fc + .stringMatching(/^[a-z][a-z0-9]{0,8}$/) + .filter( + (alias) => + alias !== `kind` && alias !== `profile` && alias !== `userId`, + ), + ])(`does not bind union-derived output fields to outer aliases`, (alias) => { + expect(getQueryIdentity(createUnionDerivedNestedQuery(`kind`))).toBe( + getQueryIdentity(createUnionDerivedNestedQuery(alias)), + ) + expect( + getQueryIdentity(createUnionDerivedNestedOutputQuery(`profile`)), + ).toBe(getQueryIdentity(createUnionDerivedNestedOutputQuery(alias))) + expect(getQueryIdentity(createUnionDerivedIncludesQuery(`userId`))).toBe( + getQueryIdentity(createUnionDerivedIncludesQuery(alias)), + ) + expect( + getQueryIdentity(createCorrelatedUnionIncludesQuery(`profile`)), + ).toBe(getQueryIdentity(createCorrelatedUnionIncludesQuery(alias))) + }) + + it(`shares identity across equivalent predicate formulations`, () => { + const left = getQueryIR( + new Query() + .from({ user: usersCollection }) + .where(({ user }) => and(eq(user.status, `active`), gt(user.age, 18))), + ) + const right = getQueryIR( + new Query() + .from({ user: usersCollection }) + .where(({ user }) => and(lt(18, user.age), eq(`active`, user.status))), + ) + + expect(getQueryIdentity(left)).toBe(getQueryIdentity(right)) + }) + + it(`normalizes the implicit conjunction order of repeated where clauses`, () => { + const left = getQueryIR( + new Query() + .from({ user: usersCollection }) + .where(({ user }) => eq(user.status, `active`)) + .where(({ user }) => gt(user.age, 18)), + ) + const right = getQueryIR( + new Query() + .from({ user: usersCollection }) + .where(({ user }) => gt(user.age, 18)) + .where(({ user }) => eq(user.status, `active`)), + ) + const duplicate = getQueryIR( + new Query() + .from({ user: usersCollection }) + .where(({ user }) => eq(user.status, `active`)) + .where(({ user }) => gt(user.age, 18)) + .where(({ user }) => eq(user.status, `active`)), + ) + + expect(getQueryIdentity(left)).toBe(getQueryIdentity(right)) + expect(getQueryIdentity(left)).toBe(getQueryIdentity(duplicate)) + }) + + it(`normalizes the implicit conjunction order of repeated having clauses`, () => { + const left = getQueryIR( + new Query() + .from({ user: usersCollection }) + .groupBy(({ user }) => user.teamId) + .select(({ user }) => ({ + teamId: user.teamId, + userCount: count(user.id), + averageAge: avg(user.age), + })) + .having(({ $selected }) => gt($selected.userCount, 1)) + .having(({ $selected }) => gt($selected.averageAge, 18)), + ) + const right = getQueryIR( + new Query() + .from({ user: usersCollection }) + .groupBy(({ user }) => user.teamId) + .select(({ user }) => ({ + teamId: user.teamId, + userCount: count(user.id), + averageAge: avg(user.age), + })) + .having(({ $selected }) => gt($selected.averageAge, 18)) + .having(({ $selected }) => gt($selected.userCount, 1)), + ) + + expect(getQueryIdentity(left)).toBe(getQueryIdentity(right)) + }) + + it(`includes a query plan's result window`, () => { + const createQuery = (limit: number) => + getQueryIR( + new Query() + .from({ user: usersCollection }) + .orderBy(({ user }) => user.age) + .limit(limit), + ) + + expect(getQueryIdentity(createQuery(10))).not.toBe( + getQueryIdentity(createQuery(20)), + ) + }) + + it(`elides the default query offset`, () => { + const base = getQueryIR(new Query().from({ user: usersCollection })) + const offsetZero = getQueryIR( + new Query().from({ user: usersCollection }).offset(0), + ) + + expect(getQueryIdentity(base)).toBe(getQueryIdentity(offsetZero)) + }) + + it(`preserves function-argument and orderBy-clause order`, () => { + const subtractAge = getQueryIR( + new Query() + .from({ user: usersCollection }) + .orderBy(({ user }) => subtract(user.age, 1)) + .orderBy(({ user }) => user.name), + ) + const subtractFromOne = getQueryIR( + new Query() + .from({ user: usersCollection }) + .orderBy(({ user }) => subtract(1, user.age)) + .orderBy(({ user }) => user.name), + ) + const reversedClauses = getQueryIR( + new Query() + .from({ user: usersCollection }) + .orderBy(({ user }) => user.name) + .orderBy(({ user }) => subtract(user.age, 1)), + ) + + expect(getQueryIdentity(subtractAge)).not.toBe( + getQueryIdentity(subtractFromOne), + ) + expect(getQueryIdentity(subtractAge)).not.toBe( + getQueryIdentity(reversedClauses), + ) + }) + it(`preserves semantically significant union source ordering`, () => { const usersThenPosts = getQueryIR( new Query().unionAll({ user: usersCollection, post: postsCollection }), @@ -385,26 +1140,50 @@ describe(`stable QueryIR identity smoke test`, () => { ) }) - it(`normalizes object property ordering inside values`, () => { - const left = getQueryIR( - new Query().from({ user: usersCollection }).where(({ user }) => - eq(user.profile, { - skills: [`ts`, `db`], - experience: { years: 5 }, - }), - ), - ) + fcTest.prop([referenceSemanticPairArbitrary])( + `keeps queries distinct when captured values compare by reference`, + ([first, second]) => { + const firstQuery = createProfileValueQuery(first) + const secondQuery = createProfileValueQuery(second) + const firstPredicate = firstQuery.where![0] as BasicExpression + const secondPredicate = secondQuery.where![0] as BasicExpression + const row = { user: { profile: first } } - const right = getQueryIR( - new Query().from({ user: usersCollection }).where(({ user }) => - eq(user.profile, { - experience: { years: 5 }, - skills: [`ts`, `db`], - }), + expect(compileExpression(firstPredicate)(row)).toBe(true) + expect(compileExpression(secondPredicate)(row)).toBe(false) + expect(getQueryIdentity(firstQuery)).not.toBe( + getQueryIdentity(secondQuery), + ) + }, + ) + + it(`uses evaluator semantics for invalid Date and Temporal values`, () => { + expect(getQueryIdentity(createProfileValueQuery(new Date(`invalid`)))).toBe( + getQueryIdentity(createProfileValueQuery(new Date(`also invalid`))), + ) + expect( + getQueryIdentity( + createProfileValueQuery(Temporal.PlainDate.from(`2026-08-24`)), + ), + ).toBe( + getQueryIdentity( + createProfileValueQuery(Temporal.PlainDate.from(`2026-08-24`)), ), ) + }) - expect(getStableQueryIRHash(left)).toBe(getStableQueryIRHash(right)) + it(`normalizes object property ordering in structural value hashes`, () => { + expect( + getStableValueHash({ + skills: [`ts`, `db`], + experience: { years: 5 }, + }), + ).toBe( + getStableValueHash({ + experience: { years: 5 }, + skills: [`ts`, `db`], + }), + ) }) it(`keeps runtime values disjoint from internal identity tags`, () => { @@ -472,14 +1251,7 @@ describe(`stable QueryIR identity smoke test`, () => { } }) - it(`rejects opaque runtime values inside otherwise structured expressions`, () => { - const circularValue: Record = {} - circularValue.self = circularValue - - class OpaqueValue { - value = `Tanner` - } - + it(`rejects function and symbol values inside structured expressions`, () => { const queries = [ [ `function value`, @@ -499,35 +1271,6 @@ describe(`stable QueryIR identity smoke test`, () => { ), /symbol value/, ], - [ - `circular value`, - getQueryIR( - new Query() - .from({ user: usersCollection }) - .where(({ user }) => eq(user.profile, circularValue as never)), - ), - /circular value/, - ], - [ - `invalid date`, - getQueryIR( - new Query() - .from({ user: usersCollection }) - .where(({ user }) => - eq(user.createdAt, new Date(`invalid`) as never), - ), - ), - /invalid Date/, - ], - [ - `class instance`, - getQueryIR( - new Query() - .from({ user: usersCollection }) - .where(({ user }) => eq(user.name, new OpaqueValue() as never)), - ), - /non-plain object value/, - ], ] as const for (const [name, query, message] of queries) { @@ -537,4 +1280,20 @@ describe(`stable QueryIR identity smoke test`, () => { expect(() => getStableQueryIRHash(query), name).toThrow(message) } }) + + it(`accepts opaque object values by reference`, () => { + const circularValue: Record = {} + circularValue.self = circularValue + + class OpaqueValue { + value = `Tanner` + } + + expect(() => + getQueryIdentity(createProfileValueQuery(circularValue)), + ).not.toThrow() + expect(() => + getQueryIdentity(createProfileValueQuery(new OpaqueValue())), + ).not.toThrow() + }) }) diff --git a/packages/db/tests/query/load-subset-oracle.property.test.ts b/packages/db/tests/query/load-subset-oracle.property.test.ts index cf2645607c..fd6296e0b0 100644 --- a/packages/db/tests/query/load-subset-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-oracle.property.test.ts @@ -56,6 +56,7 @@ type WindowRequest = { direction: `asc` | `desc` nulls?: `first` | `last` stringSort?: `lexical` | `locale` + cursorBoundary?: number offset: number limit?: number } @@ -226,10 +227,23 @@ const rejectedWaiterScenarioArbitrary: fc.Arbitrary = const windowRequestArbitrary: fc.Arbitrary> = fc.record({ - orderField: fc.constantFrom(`none`, `rank`, `score`), - direction: fc.constantFrom(`asc`, `desc`), - nulls: fc.constantFrom(`first`, `last`), - stringSort: fc.constantFrom(`lexical`, `locale`), + orderField: fc.constantFrom>( + `none`, + `rank`, + `score`, + ), + direction: fc.constantFrom(`asc`, `desc`), + nulls: fc.constantFrom>( + `first`, + `last`, + ), + stringSort: fc.constantFrom>( + `lexical`, + `locale`, + ), + cursorBoundary: fc.option(fc.integer({ min: -3, max: 3 }), { + nil: undefined, + }), offset: fc.integer({ min: 0, max: 6 }), limit: fc.option(fc.integer({ min: 0, max: 6 }), { nil: undefined }), }) @@ -240,6 +254,9 @@ const finiteWindowRequestArbitrary: fc.Arbitrary> = direction: fc.constantFrom(`asc`, `desc`), nulls: fc.constantFrom(`first`, `last`), stringSort: fc.constantFrom(`lexical`, `locale`), + cursorBoundary: fc.option(fc.integer({ min: -3, max: 3 }), { + nil: undefined, + }), offset: fc.integer({ min: 0, max: 6 }), limit: fc.integer({ min: 0, max: 6 }), }) @@ -512,10 +529,25 @@ function readDedupeTrackingState(dedupe: DeduplicatedLoadSubset): { function toWindowOptions(request: WindowRequest): LoadSubsetOptions { const orderField = request.orderField ?? `rank` + const cursorRef = orderField === `score` ? scoreRef : rankRef return { where: request.where ? toWhere(request.where) : undefined, offset: request.offset, limit: request.limit, + cursor: + request.cursorBoundary === undefined + ? undefined + : { + whereFrom: new Func(request.direction === `asc` ? `gt` : `lt`, [ + cursorRef, + new Value(request.cursorBoundary), + ]), + whereCurrent: new Func(`eq`, [ + cursorRef, + new Value(request.cursorBoundary), + ]), + lastKey: request.cursorBoundary, + }, orderBy: orderField === `none` ? undefined @@ -552,6 +584,7 @@ type WindowCoverageDescriptor = { request: WindowRequest whereFingerprint: string orderFingerprint: string | undefined + cursorFingerprint: string | undefined matching: Set } @@ -565,6 +598,9 @@ function describeWindowCoverage( orderFingerprint: options.orderBy ? JSON.stringify(options.orderBy) : undefined, + cursorFingerprint: options.cursor + ? JSON.stringify(options.cursor) + : undefined, matching: matchingValues(options.where), } } @@ -576,10 +612,14 @@ function describedWindowCovers( if ( loaded.request.limit === undefined && loaded.request.offset === 0 && + loaded.cursorFingerprint === undefined && isSubset(requested.matching, loaded.matching) ) { return true } + if (requested.cursorFingerprint !== loaded.cursorFingerprint) { + return false + } if (requested.whereFingerprint !== loaded.whereFingerprint) return false if (requested.orderFingerprint === undefined) return true return requested.orderFingerprint === loaded.orderFingerprint @@ -1488,23 +1528,48 @@ describe(`loadSubset coverage oracle`, () => { ), ) - it( - `discovered trace: an identical filtered window reuses its load`, - expectAssertionFailure( - () => - Promise.resolve().then(() => { - const request: WindowRequest = { - where: { kind: `in`, values: [0] }, - orderField: `none`, - direction: `asc`, - offset: 0, - limit: 1, - } - expect(countWindowLoads([request, request])).toBe(1) - }), - { message: /expected 2 to be/ }, - ), - ) + it(`discovered trace: an identical filtered window reuses its load`, () => { + const request: WindowRequest = { + where: { kind: `in`, values: [0] }, + orderField: `none`, + direction: `asc`, + offset: 0, + limit: 1, + } + expect(countWindowLoads([request, request])).toBe(1) + }) + + it(`discovered trace: distinct cursor pages start distinct loads`, () => { + const request: WindowRequest = { + orderField: `rank`, + direction: `asc`, + offset: 0, + limit: 2, + cursorBoundary: 1, + } + + runWindowCoverageTrace([ + request, + { ...request, cursorBoundary: 2 }, + request, + ]) + }) + + it(`discovered trace: a cursor without a limit is not full coverage`, () => { + const request: WindowRequest = { + orderField: `rank`, + direction: `asc`, + offset: 0, + limit: undefined, + cursorBoundary: 1, + } + + runWindowCoverageTrace([ + request, + { ...request, cursorBoundary: 2 }, + { ...request, cursorBoundary: undefined }, + ]) + }) it(`rejects repeated transport work for one covered predicate`, () => { expect(() => @@ -2003,25 +2068,16 @@ describe(`loadSubset coverage oracle`, () => { ), ) - it( - `discovered trace: widening a window forgets an earlier covered window`, - expectAssertionFailure( - () => - Promise.resolve().then(() => { - const first: WindowRequest = { - orderField: `none`, - direction: `asc`, - offset: 0, - limit: 1, - where: { kind: `in`, values: [0] }, - } - expect(countWindowLoads([first, { ...first, limit: 2 }, first])).toBe( - 2, - ) - }), - { message: /expected 3 to be 2/ }, - ), - ) + it(`discovered trace: widening a window remembers an earlier covered window`, () => { + const first: WindowRequest = { + orderField: `none`, + direction: `asc`, + offset: 0, + limit: 1, + where: { kind: `in`, values: [0] }, + } + expect(countWindowLoads([first, { ...first, limit: 2 }, first])).toBe(2) + }) it( `discovered trace: complementary ranges redundantly reload an all-data request`, diff --git a/packages/db/tests/query/predicate-utils.test.ts b/packages/db/tests/query/predicate-utils.test.ts index 1f47eef23b..721c07f5ae 100644 --- a/packages/db/tests/query/predicate-utils.test.ts +++ b/packages/db/tests/query/predicate-utils.test.ts @@ -832,6 +832,23 @@ describe(`isPredicateSubset`, () => { expect(isPredicateSubset(subset, superset)).toBe(true) }) + it(`treats semantic predicate forms as equal coverage`, () => { + const age = ref(`age`) + const status = ref(`status`) + const ageCheck = gt(age, val(18)) + const statusCheck = eq(status, val(`active`)) + const subset: LoadSubsetOptions = { + where: func(`and`, ageCheck, statusCheck), + limit: 10, + } + const superset: LoadSubsetOptions = { + where: func(`and`, eq(val(`active`), status), func(`lt`, val(18), age)), + limit: 20, + } + + expect(isPredicateSubset(subset, superset)).toBe(true) + }) + it(`should return false for limited superset with different where clause`, () => { // Even if subset's where is more restrictive, it can't be a subset // of a limited superset with a different where clause. diff --git a/packages/query-db-collection/e2e/query-filter.ts b/packages/query-db-collection/e2e/query-filter.ts index aa3de76b15..cf9ac16508 100644 --- a/packages/query-db-collection/e2e/query-filter.ts +++ b/packages/query-db-collection/e2e/query-filter.ts @@ -3,7 +3,7 @@ * Uses expression helpers to implement proper predicate push-down */ -import { parseLoadSubsetOptions } from '@tanstack/db' +import { getLoadSubsetDemandKey, parseLoadSubsetOptions } from '@tanstack/db' import type { IR, LoadSubsetOptions, @@ -41,117 +41,11 @@ export function buildQueryKey( namespace: string, options: LoadSubsetOptions | undefined, ) { - return [`e2e`, namespace, serializeLoadSubsetOptions(options)] -} - -export function serializeLoadSubsetOptions( - options: LoadSubsetOptions | undefined, -): unknown { - if (!options) { - return null - } - - const result: Record = {} - - if (options.where) { - result.where = serializeExpression(options.where) - } - - if (options.orderBy?.length) { - result.orderBy = options.orderBy.map((clause) => ({ - expression: serializeExpression(clause.expression), - direction: clause.compareOptions.direction, - nulls: clause.compareOptions.nulls, - })) - } - - if (options.limit !== undefined) { - result.limit = options.limit - } - - // Include offset for pagination support - different offsets need different query keys - if (options.offset !== undefined) { - result.offset = options.offset - } - - return JSON.stringify(Object.keys(result).length === 0 ? null : result) -} - -function serializeExpression(expr: IR.BasicExpression | undefined): unknown { - if (!expr) { - return null - } - - switch (expr.type) { - case `val`: - return { - type: `val`, - value: serializeValue(expr.value), - } - case `ref`: - return { - type: `ref`, - path: [...expr.path], - } - case `func`: - return { - type: `func`, - name: expr.name, - args: expr.args.map((arg) => serializeExpression(arg)), - } - default: - return null - } -} - -function serializeValue(value: unknown): unknown { - if (value === undefined) { - return { __type: `undefined` } - } - - if (typeof value === `number`) { - if (Number.isNaN(value)) { - return { __type: `nan` } - } - if (value === Number.POSITIVE_INFINITY) { - return { __type: `infinity`, sign: 1 } - } - if (value === Number.NEGATIVE_INFINITY) { - return { __type: `infinity`, sign: -1 } - } - } - - if (typeof value === `bigint`) { - return { __type: `bigint`, value: value.toString() } - } - - if ( - value === null || - typeof value === `string` || - typeof value === `number` || - typeof value === `boolean` - ) { - return value - } - - if (value instanceof Date) { - return { __type: `date`, value: value.toJSON() } - } - - if (Array.isArray(value)) { - return value.map((item) => serializeValue(item)) - } - - if (typeof value === `object`) { - return Object.fromEntries( - Object.entries(value as Record).map(([key, val]) => [ - key, - serializeValue(val), - ]), - ) - } - - return value + return [ + `e2e`, + namespace, + options === undefined ? undefined : getLoadSubsetDemandKey(options), + ] } type Predicate = (item: T) => boolean diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index 5396bd49d6..7d611349fb 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -1,5 +1,9 @@ import { QueryObserver, hashKey } from '@tanstack/query-core' -import { deepEquals, withCollectionConfigFactory } from '@tanstack/db' +import { + deepEquals, + getLoadSubsetDemandKey, + withCollectionConfigFactory, +} from '@tanstack/db' import { GetKeyRequiredError, InitialDataInOnDemandModeError, @@ -8,7 +12,6 @@ import { QueryKeyRequiredError, } from './errors' import { createWriteUtils } from './manual-sync' -import { serializeLoadSubsetOptions } from './serialization' import type { BaseCollectionConfig, ChangeMessage, @@ -1215,10 +1218,10 @@ export function queryCollectionOptions( // Function-based queryKey: use it to build the key from opts return queryKey(opts) } else if (syncMode === `on-demand`) { - // Static queryKey in on-demand mode: automatically append serialized predicates - // to create separate cache entries for different predicate combinations - const serialized = serializeLoadSubsetOptions(opts) - return serialized !== undefined ? [...queryKey, serialized] : queryKey + // A static on-demand key is extended by exact semantic demand so + // equivalent predicates share one entry while distinct windows do not. + const demandKey = getLoadSubsetDemandKey(opts) + return demandKey !== undefined ? [...queryKey, demandKey] : queryKey } else { // Static queryKey in eager mode: use as-is return queryKey diff --git a/packages/query-db-collection/src/serialization.ts b/packages/query-db-collection/src/serialization.ts deleted file mode 100644 index 9849c4bd33..0000000000 --- a/packages/query-db-collection/src/serialization.ts +++ /dev/null @@ -1,135 +0,0 @@ -import type { IR, LoadSubsetOptions } from '@tanstack/db' - -/** - * Serializes LoadSubsetOptions into a stable, hashable format for query keys. - * Includes where, orderBy, limit, and offset for pagination support. - * Note: cursor expressions are not serialized as they are backend-specific. - * @internal - */ -export function serializeLoadSubsetOptions( - options: LoadSubsetOptions | undefined, -): string | undefined { - if (!options) { - return undefined - } - - const result: Record = {} - - if (options.where) { - result.where = serializeExpression(options.where) - } - - if (options.orderBy?.length) { - result.orderBy = options.orderBy.map((clause) => { - const baseOrderBy = { - expression: serializeExpression(clause.expression), - direction: clause.compareOptions.direction, - nulls: clause.compareOptions.nulls, - stringSort: clause.compareOptions.stringSort, - } - - // Handle locale-specific options when stringSort is 'locale' - if (clause.compareOptions.stringSort === `locale`) { - return { - ...baseOrderBy, - locale: clause.compareOptions.locale, - localeOptions: clause.compareOptions.localeOptions, - } - } - - return baseOrderBy - }) - } - - if (options.limit !== undefined) { - result.limit = options.limit - } - - // Include offset for pagination support - if (options.offset !== undefined) { - result.offset = options.offset - } - - return Object.keys(result).length === 0 ? undefined : JSON.stringify(result) -} - -/** - * Recursively serializes an IR expression for stable hashing - * @internal - */ -function serializeExpression(expr: IR.BasicExpression | undefined): unknown { - if (!expr) { - return null - } - - switch (expr.type) { - case `val`: - return { - type: `val`, - value: serializeValue(expr.value), - } - case `ref`: - return { - type: `ref`, - path: [...expr.path], - } - case `func`: - return { - type: `func`, - name: expr.name, - args: expr.args.map((arg) => serializeExpression(arg)), - } - default: - return null - } -} - -/** - * Serializes special JavaScript values (undefined, NaN, Infinity, Date) - * @internal - */ -function serializeValue(value: unknown): unknown { - if (value === undefined) { - return { __type: `undefined` } - } - - if (typeof value === `number`) { - if (Number.isNaN(value)) { - return { __type: `nan` } - } - if (value === Number.POSITIVE_INFINITY) { - return { __type: `infinity`, sign: 1 } - } - if (value === Number.NEGATIVE_INFINITY) { - return { __type: `infinity`, sign: -1 } - } - } - - if ( - value === null || - typeof value === `string` || - typeof value === `number` || - typeof value === `boolean` - ) { - return value - } - - if (value instanceof Date) { - return { __type: `date`, value: value.toJSON() } - } - - if (Array.isArray(value)) { - return value.map((item) => serializeValue(item)) - } - - if (typeof value === `object`) { - return Object.fromEntries( - Object.entries(value as Record).map(([key, val]) => [ - key, - serializeValue(val), - ]), - ) - } - - return value -} diff --git a/packages/query-db-collection/tests/load-subset-lifecycle-oracle.test.ts b/packages/query-db-collection/tests/load-subset-lifecycle-oracle.test.ts index 3e13faf653..38dc3e9a19 100644 --- a/packages/query-db-collection/tests/load-subset-lifecycle-oracle.test.ts +++ b/packages/query-db-collection/tests/load-subset-lifecycle-oracle.test.ts @@ -7,7 +7,6 @@ import { eq, } from '@tanstack/db' import { describe, expect, it, vi } from 'vitest' -import { expectAssertionFailure } from '../../db/tests/expected-failure.js' import { TraceAssertionError } from '../../db/tests/trace-runner.js' import { queryCollectionOptions } from '../src/query.js' import type { QueryFunctionContext } from '@tanstack/query-core' @@ -348,7 +347,7 @@ async function expectDeferredStartupReadyDoesNotOverrideError(): Promise { } async function expectEquivalentPredicatesShareOneLoad( - form: `commutative-and` | `reversed-equality`, + form: `commutative-and` | `commutative-or` | `reversed-equality`, ): Promise { const queryClient = createQueryClient() const id = `load-subset-canonical-predicate-${collectionSequence++}` @@ -373,14 +372,22 @@ async function expectEquivalentPredicatesShareOneLoad( new IR.PropRef([`group`]), new IR.Value(`x`), ]) - const first = - form === `commutative-and` - ? new IR.Func(`and`, [firstComparison, secondComparison]) - : firstComparison - const second = - form === `commutative-and` - ? new IR.Func(`and`, [secondComparison, firstComparison]) - : new IR.Func(`eq`, [new IR.Value(`a`), new IR.PropRef([`id`])]) + let first: IR.BasicExpression + let second: IR.BasicExpression + switch (form) { + case `commutative-and`: + first = new IR.Func(`and`, [firstComparison, secondComparison]) + second = new IR.Func(`and`, [secondComparison, firstComparison]) + break + case `commutative-or`: + first = new IR.Func(`or`, [firstComparison, secondComparison]) + second = new IR.Func(`or`, [secondComparison, firstComparison]) + break + case `reversed-equality`: + first = firstComparison + second = new IR.Func(`eq`, [new IR.Value(`a`), new IR.PropRef([`id`])]) + break + } try { await collection._sync.loadSubset({ where: first }) @@ -396,6 +403,41 @@ async function expectEquivalentPredicatesShareOneLoad( } } +async function expectEquivalentComparisonValuesShareOneLoad( + firstValue: unknown, + secondValue: unknown, +): Promise { + const queryClient = createQueryClient() + const id = `load-subset-comparison-value-${collectionSequence++}` + const queryFn = vi.fn().mockResolvedValue([{ id: `a` }]) + const collection = createCollection( + queryCollectionOptions({ + id, + queryClient, + queryKey: [id], + queryFn, + getKey: (row) => row.id, + startSync: true, + syncMode: `on-demand`, + retry: false, + }), + ) + const value = new IR.PropRef([`value`]) + + try { + await collection._sync.loadSubset({ + where: new IR.Func(`eq`, [value, new IR.Value(firstValue)]), + }) + await collection._sync.loadSubset({ + where: new IR.Func(`eq`, [value, new IR.Value(secondValue)]), + }) + expect(queryFn).toHaveBeenCalledOnce() + } finally { + await collection.cleanup() + queryClient.clear() + } +} + async function expectFinalOwnerCleanupAbortsQuery(): Promise { const queryClient = createQueryClient() const id = `load-subset-cancel-final-owner-${collectionSequence++}` @@ -526,19 +568,31 @@ describe(`loadSubset lifecycle oracle`, () => { }) it(`commutative predicate forms share one query-db transport load`, async () => { - await expectAssertionFailure(expectEquivalentPredicatesShareOneLoad, { - checkpoint: 0, - classify: ({ actual, expected }) => actual === 2 && expected === 1, - })(`commutative-and`) + await expectEquivalentPredicatesShareOneLoad(`commutative-and`) + await expectEquivalentPredicatesShareOneLoad(`commutative-or`) }) it(`reversed equality operands share one query-db transport load`, async () => { - await expectAssertionFailure(expectEquivalentPredicatesShareOneLoad, { - checkpoint: 0, - classify: ({ actual, expected }) => actual === 2 && expected === 1, - })(`reversed-equality`) + await expectEquivalentPredicatesShareOneLoad(`reversed-equality`) }) + it.each([ + [ + `valid Date`, + new Date(`2024-01-15T00:00:00Z`), + new Date(`2024-01-15T00:00:00Z`), + ], + [`invalid Date`, new Date(Number.NaN), new Date(Number.NaN)], + ])( + `shares one query-db transport load for equivalent %s values`, + async (_label, firstValue, secondValue) => { + await expectEquivalentComparisonValuesShareOneLoad( + firstValue, + secondValue, + ) + }, + ) + it(`aborts an in-flight query when its final live-query owner cleans up`, async () => { await expectFinalOwnerCleanupAbortsQuery() }) From 1d003a41688bf89a4dc752df8dfdc99c94cadb46 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 24 Aug 2026 17:45:52 -0600 Subject: [PATCH 2/7] docs: add canonical demand identity changeset --- .changeset/canonical-demand-identity.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/canonical-demand-identity.md diff --git a/.changeset/canonical-demand-identity.md b/.changeset/canonical-demand-identity.md new file mode 100644 index 0000000000..dc4d0602d4 --- /dev/null +++ b/.changeset/canonical-demand-identity.md @@ -0,0 +1,6 @@ +--- +'@tanstack/db': patch +'@tanstack/query-db-collection': patch +--- + +Canonicalize equivalent loadSubset queries to one demand identity while preserving distinct runtime values and window requests. Query DB now reuses the same canonical identity for its on-demand cache keys. From 53c270c3af52871b0e0157ff66478a5814d8747d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 24 Aug 2026 18:23:24 -0600 Subject: [PATCH 3/7] fix(db): fall back when runtime crypto is incomplete --- .../db/src/query/runtime-reference-identity.ts | 4 ++-- .../db/tests/query/ir-stable-identity.test.ts | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/db/src/query/runtime-reference-identity.ts b/packages/db/src/query/runtime-reference-identity.ts index d903060522..15d7b82b6d 100644 --- a/packages/db/src/query/runtime-reference-identity.ts +++ b/packages/db/src/query/runtime-reference-identity.ts @@ -27,9 +27,9 @@ export const getRuntimeReferenceIdentity = function createRuntimeReferenceNamespace(): string { const randomValues = new Uint32Array(4) const runtimeCrypto = Reflect.get(globalThis, `crypto`) as - | { getRandomValues: (values: Uint32Array) => Uint32Array } + | { getRandomValues?: (values: Uint32Array) => Uint32Array } | undefined - if (runtimeCrypto !== undefined) { + if (typeof runtimeCrypto?.getRandomValues === `function`) { runtimeCrypto.getRandomValues(randomValues) return Array.from(randomValues, (value) => value.toString(36)).join(`-`) } diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index 3d9bd86053..8a5447557f 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { fc, test as fcTest } from '@fast-check/vitest' import { Temporal } from 'temporal-polyfill' import { CollectionImpl } from '../../src/collection/index.js' @@ -279,6 +279,21 @@ describe(`semantic expression identity`, () => { expect(firstRuntime({ a: 1 })).not.toEqual(secondRuntime({ b: 2 })) }) + it(`falls back when the runtime crypto object lacks getRandomValues`, () => { + vi.stubGlobal(`crypto`, {}) + try { + const runtime = createRuntimeReferenceIdentityFactory() + + expect(runtime({ a: 1 })).toEqual([ + `runtimeReference`, + expect.any(String), + 1, + ]) + } finally { + vi.unstubAllGlobals() + } + }) + fcTest.prop([ fc.uniqueArray(fc.oneof(fc.integer(), fc.string(), fc.boolean()), { minLength: 1, From bdf9d7a332d75aa3bf9276663dc0257baa006ecb Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 24 Aug 2026 20:12:54 -0600 Subject: [PATCH 4/7] fix(db): preserve observable demand identity --- .changeset/canonical-demand-identity.md | 2 +- packages/db/src/query/index.ts | 2 +- packages/db/src/query/ir-stable-identity.ts | 161 +++++++++++++----- packages/db/src/query/live/ARCHITECTURE.md | 16 +- packages/db/src/query/predicate-utils.ts | 44 +++-- packages/db/src/query/subset-dedupe.ts | 6 +- .../db/tests/query/compiler/basic.test.ts | 57 +++++++ .../db/tests/query/ir-stable-identity.test.ts | 125 +++++++++++++- .../query/load-subset-oracle.property.test.ts | 27 +-- .../db/tests/query/predicate-utils.test.ts | 66 +++++++ 10 files changed, 420 insertions(+), 86 deletions(-) diff --git a/.changeset/canonical-demand-identity.md b/.changeset/canonical-demand-identity.md index dc4d0602d4..5a61f5db8d 100644 --- a/.changeset/canonical-demand-identity.md +++ b/.changeset/canonical-demand-identity.md @@ -3,4 +3,4 @@ '@tanstack/query-db-collection': patch --- -Canonicalize equivalent loadSubset queries to one demand identity while preserving distinct runtime values and window requests. Query DB now reuses the same canonical identity for its on-demand cache keys. +Canonicalize equivalent loadSubset queries to one demand identity while preserving observable output aliases, exact projected values, and distinct ordered windows. Query DB now reuses the same canonical identity for its on-demand cache keys. diff --git a/packages/db/src/query/index.ts b/packages/db/src/query/index.ts index 2c587b589b..75c020758e 100644 --- a/packages/db/src/query/index.ts +++ b/packages/db/src/query/index.ts @@ -116,7 +116,7 @@ export { isLimitSubset, isOffsetLimitSubset, isPredicateSubset, - isLoadSubsetCoveredBy, + isLoadSubsetRequestSubsumedBy, } from './predicate-utils.js' export { DeduplicatedLoadSubset } from './subset-dedupe.js' diff --git a/packages/db/src/query/ir-stable-identity.ts b/packages/db/src/query/ir-stable-identity.ts index 871321601f..3c0ead5ccf 100644 --- a/packages/db/src/query/ir-stable-identity.ts +++ b/packages/db/src/query/ir-stable-identity.ts @@ -26,7 +26,10 @@ type StableIdentityValue = | Array | { [key: string]: StableIdentityValue } -type RuntimeValueIdentity = `structural` | `semantic` +type ValueIdentityContext = + | `exact-output` + | `equality-operand` + | `ordering-operand` type AliasScope = { bindings: ReadonlyMap @@ -86,7 +89,12 @@ export function getQueryIdentity(query: QueryIR): QueryIdentity { /** Returns the semantic identity of one structured expression. */ export function getStableExpressionHash(expression: BasicExpression): string { return JSON.stringify( - canonicalizeExpression(expression, `expression`, new WeakSet(), `semantic`), + canonicalizeExpression( + expression, + `expression`, + new WeakSet(), + `exact-output`, + ), ) } @@ -143,13 +151,13 @@ export function getLoadSubsetDemandKey( options.cursor.whereFrom, `loadSubset.cursor.whereFrom`, seen, - `semantic`, + `exact-output`, ), whereCurrent: canonicalizeExpression( options.cursor.whereCurrent, `loadSubset.cursor.whereCurrent`, seen, - `semantic`, + `exact-output`, ), } if (options.cursor.lastKey !== undefined) { @@ -260,6 +268,18 @@ function canonicalizeQueryInScope( ) } + if ( + !query.select && + (Boolean(query.join?.length) || Boolean(query.groupBy?.length)) + ) { + // Without an explicit projection, these query shapes return a namespaced + // row. Its alias keys are public output and therefore part of identity. + result.implicitOutput = { + type: `namespaced`, + aliases: Array.from(scope.bindings.keys()), + } + } + if (query.join) { result.join = query.join.map((join, index) => canonicalizeJoin(join, `${path}.join[${index}]`, seen, scope), @@ -281,7 +301,7 @@ function canonicalizeQueryInScope( expression, `${path}.groupBy[${index}]`, seen, - `semantic`, + `exact-output`, scope, ), ) @@ -302,7 +322,7 @@ function canonicalizeQueryInScope( orderBy, `${path}.orderBy[${index}]`, seen, - `semantic`, + `ordering-operand`, scope, ), ) @@ -363,7 +383,7 @@ function canonicalizeLoadSubsetQuery( options.where, `${path}.where`, seen, - `semantic`, + `exact-output`, ) } @@ -373,7 +393,7 @@ function canonicalizeLoadSubsetQuery( orderBy, `${path}.orderBy[${index}]`, seen, - `semantic`, + `ordering-operand`, ), ) } @@ -394,14 +414,14 @@ function canonicalizeJoin( join.left, `${path}.left`, seen, - `semantic`, + `equality-operand`, scope, ), right: canonicalizeExpression( join.right, `${path}.right`, seen, - `semantic`, + `equality-operand`, scope, ), } @@ -488,20 +508,20 @@ function canonicalizeSelectValue( toExpression(value), path, seen, - `semantic`, + `exact-output`, scope, ) } if (isExpression(value)) { - return canonicalizeExpression(value, path, seen, `semantic`, scope) + return canonicalizeExpression(value, path, seen, `exact-output`, scope) } if (isPlainObject(value)) { return canonicalizeSelect(value as Select, path, seen, scope) } - return canonicalizeSemanticRuntimeValue(value, path, seen, scope) + return canonicalizeExactOutputRuntimeValue(value, path, seen) } function canonicalizeWhere( @@ -517,7 +537,7 @@ function canonicalizeWhere( where.expression, `${path}.expression`, seen, - `semantic`, + `exact-output`, scope, ), } @@ -529,14 +549,14 @@ function canonicalizeWhere( return result } - return canonicalizeExpression(where, path, seen, `semantic`, scope) + return canonicalizeExpression(where, path, seen, `exact-output`, scope) } function canonicalizeOrderBy( orderBy: OrderByClause, path: string, seen: WeakSet, - runtimeValueIdentity: RuntimeValueIdentity = `structural`, + valueContext: ValueIdentityContext = `exact-output`, scope?: AliasScope, ): StableIdentityValue { return { @@ -544,7 +564,7 @@ function canonicalizeOrderBy( orderBy.expression, `${path}.expression`, seen, - runtimeValueIdentity, + valueContext, scope, ), compareOptions: canonicalizeRuntimeValue( @@ -563,7 +583,7 @@ function canonicalizeExpression( | ConditionalSelect, path: string, seen: WeakSet, - runtimeValueIdentity: RuntimeValueIdentity = `structural`, + valueContext: ValueIdentityContext = `exact-output`, scope?: AliasScope, ): StableIdentityValue { if (expression.type === `ref`) { @@ -594,14 +614,24 @@ function canonicalizeExpression( return { type: `val`, value: - runtimeValueIdentity === `semantic` - ? canonicalizeSemanticRuntimeValue( + valueContext === `equality-operand` + ? canonicalizeEqualityRuntimeValue( expression.value, `${path}.value`, seen, scope, ) - : canonicalizeRuntimeValue(expression.value, `${path}.value`, seen), + : valueContext === `ordering-operand` + ? canonicalizeOrderingRuntimeValue( + expression.value, + `${path}.value`, + seen, + ) + : canonicalizeExactOutputRuntimeValue( + expression.value, + `${path}.value`, + seen, + ), } } @@ -613,25 +643,19 @@ function canonicalizeExpression( Array.isArray(expression.args[1].value) ) { const candidates = expression.args[1].value.map((value, index) => - runtimeValueIdentity === `semantic` - ? canonicalizeSemanticRuntimeValue( - value, - `${path}.args[1].value[${index}]`, - seen, - scope, - ) - : canonicalizeRuntimeValue( - value, - `${path}.args[1].value[${index}]`, - seen, - ), + canonicalizeEqualityRuntimeValue( + value, + `${path}.args[1].value[${index}]`, + seen, + scope, + ), ) return canonicalizeFunction(expression.name, [ canonicalizeExpression( expression.args[0]!, `${path}.args[0]`, seen, - runtimeValueIdentity, + `equality-operand`, scope, ), { @@ -643,12 +667,21 @@ function canonicalizeExpression( ]) } + const operandContext: ValueIdentityContext = + expression.name === `eq` + ? `equality-operand` + : expression.name === `gt` || + expression.name === `gte` || + expression.name === `lt` || + expression.name === `lte` + ? `ordering-operand` + : `exact-output` const args = expression.args.map((arg, index) => canonicalizeExpression( arg, `${path}.args[${index}]`, seen, - runtimeValueIdentity, + operandContext, scope, ), ) @@ -664,7 +697,7 @@ function canonicalizeExpression( arg, `${path}.args[${index}]`, seen, - runtimeValueIdentity, + `exact-output`, scope, ), ), @@ -679,7 +712,7 @@ function canonicalizeExpression( branch.condition, `${path}.branches[${index}].condition`, seen, - `semantic`, + `exact-output`, scope, ), value: canonicalizeSelectValue( @@ -716,14 +749,14 @@ function canonicalizeExpression( expression.correlationField, `${path}.correlationField`, seen, - `semantic`, + `equality-operand`, scope, ), childCorrelationField: canonicalizeExpression( expression.childCorrelationField, `${path}.childCorrelationField`, seen, - `semantic`, + `equality-operand`, childScope, ), fieldName: expression.fieldName, @@ -743,7 +776,7 @@ function canonicalizeExpression( projection, `${path}.parentProjection[${index}]`, seen, - `semantic`, + `exact-output`, scope, ), ) @@ -953,7 +986,19 @@ function canonicalizeRuntimeValue( throw new UnhashableQueryIRError(path, `non-plain object value`) } -function canonicalizeSemanticRuntimeValue( +function canonicalizeExactOutputRuntimeValue( + value: unknown, + path: string, + seen: WeakSet, +): StableIdentityValue { + if (typeof value === `object` && value !== null) { + return getRuntimeReferenceIdentity(value) + } + + return canonicalizeRuntimeValue(value, path, seen) +} + +function canonicalizeEqualityRuntimeValue( value: unknown, path: string, seen: WeakSet, @@ -964,7 +1009,7 @@ function canonicalizeSemanticRuntimeValue( toExpression(value), path, seen, - `semantic`, + `equality-operand`, scope, ) } @@ -994,6 +1039,38 @@ function canonicalizeSemanticRuntimeValue( return canonicalizeRuntimeValue(value, path, seen) } +function canonicalizeOrderingRuntimeValue( + value: unknown, + path: string, + seen: WeakSet, +): StableIdentityValue { + if (typeof value === `number` && Object.is(value, -0)) { + return canonicalizeRuntimeValue(0, path, seen) + } + + if (value instanceof Date && Number.isNaN(value.getTime())) { + return canonicalizeRuntimeValue(Number.NaN, path, seen) + } + + const normalized = normalizeValue(value) + if (normalized !== value && !(value instanceof Uint8Array)) { + return canonicalizeRuntimeValue(normalized, path, seen) + } + + try { + return canonicalizeRuntimeValue(value, path, seen) + } catch (error) { + if ( + error instanceof UnhashableQueryIRError && + typeof value === `object` && + value !== null + ) { + return getRuntimeReferenceIdentity(value) + } + throw error + } +} + function compareStableIdentityValues( left: StableIdentityValue, right: StableIdentityValue, diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index ad44e21ae5..4953a96f5f 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -98,8 +98,8 @@ reduction that enforces public-key congruence and multiplicity. ## Identity -Aliases are lexical query-language names. They are not runtime identities. The -query builder requires collection aliases to be unique within each lexical +Aliases are lexical query-language names rather than source runtime identities. +The query builder requires collection aliases to be unique within each lexical scope and rejects nested queries that shadow an ancestor alias. Sibling include scopes may reuse an alias because neither alias is visible to the other. Compilation then assigns opaque IDs to the accepted plan: @@ -110,8 +110,11 @@ type RelationNodeId = Brand type MaterializationEdgeId = Brand ``` -Alias text may remain as debug metadata. Renaming an accepted alias to another -unused name cannot change the compiled graph or its result. +An explicit projection can alpha-normalize aliases because its field names +define the public shape. Without a projection, joined and grouped queries return +a namespaced row whose keys are the lexical aliases. Those observable keys are +part of query identity. Alias text may otherwise remain as debug metadata +without becoming source identity. A `CanonicalCorrelationKey` is the canonical tuple of every evaluated parent-dependent value that can affect the child plan. This includes values @@ -522,8 +525,9 @@ create recursive Collection machinery. ## Normative laws 1. **Alpha-renaming:** changing any accepted alias to another unused name cannot - change results; aliases must be unique within one lexical scope and cannot - shadow an ancestor alias. Sibling scopes may reuse aliases. + change an explicitly projected result. An implicit namespaced result keeps + its aliases as public field names. Aliases must be unique within one lexical + scope and cannot shadow an ancestor alias. Sibling scopes may reuse aliases. 2. **Contribution conservation:** a public row exists exactly when its reduced supporting weight and collision policy produce one. 3. **Batch partition:** equivalent valid split and atomic deliveries converge. diff --git a/packages/db/src/query/predicate-utils.ts b/packages/db/src/query/predicate-utils.ts index 2808a19937..deb63d9c88 100644 --- a/packages/db/src/query/predicate-utils.ts +++ b/packages/db/src/query/predicate-utils.ts @@ -2,9 +2,11 @@ import { Func, Value } from './ir.js' import { UnhashableQueryIRError, getStableExpressionHash, + getStableValueHash, } from './ir-stable-identity.js' import type { BasicExpression, OrderBy, PropRef } from './ir.js' import type { LoadSubsetOptions } from '../types.js' +import type { CompareOptions } from './builder/types.js' /** * Check if one where clause is a logical subset of another. @@ -908,16 +910,17 @@ export function isPredicateSubset( } /** - * Returns whether established coverage satisfies a requested demand. + * Returns whether one acquisition request subsumes another demand. * - * Coverage is a directional relation. It must not be replaced with DemandKey + * This is a directional relationship between request shapes, not proof of + * applied or authoritative coverage. It must not be replaced with DemandKey * equality, which answers whether two exact requests are the same. */ -export function isLoadSubsetCoveredBy( +export function isLoadSubsetRequestSubsumedBy( demand: LoadSubsetOptions, - coverage: LoadSubsetOptions, + acquisitionRequest: LoadSubsetOptions, ): boolean { - return isPredicateSubset(demand, coverage) + return isPredicateSubset(demand, acquisitionRequest) } function areCursorExpressionsEqual( @@ -1220,12 +1223,31 @@ function minValue(a: any, b: any): any { return Math.min(a, b) } -function areCompareOptionsEqual( - a: { direction?: `asc` | `desc`; [key: string]: any }, - b: { direction?: `asc` | `desc`; [key: string]: any }, -): boolean { - // For now, just compare direction - could be enhanced for other options - return a.direction === b.direction +function areCompareOptionsEqual(a: CompareOptions, b: CompareOptions): boolean { + if ( + a.direction !== b.direction || + a.nulls !== b.nulls || + a.stringSort !== b.stringSort + ) { + return false + } + + if (a.stringSort !== `locale` || b.stringSort !== `locale`) { + return true + } + + if (a.locale !== b.locale) return false + if (Object.is(a.localeOptions, b.localeOptions)) return true + + try { + return ( + getStableValueHash(a.localeOptions) === + getStableValueHash(b.localeOptions) + ) + } catch (error) { + if (!(error instanceof UnhashableQueryIRError)) throw error + return false + } } interface ComparisonField { diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index f8e49d0f08..37ce49e0bf 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -1,5 +1,5 @@ import { - isLoadSubsetCoveredBy, + isLoadSubsetRequestSubsumedBy, isWhereSubset, minusWherePredicates, unionWherePredicates, @@ -111,7 +111,7 @@ export class DeduplicatedLoadSubset { // Check against limited calls if (options.limit !== undefined || options.cursor !== undefined) { const alreadyLoaded = this.limitedCalls.some((loaded) => - isLoadSubsetCoveredBy(options, loaded), + isLoadSubsetRequestSubsumedBy(options, loaded), ) if (alreadyLoaded) { @@ -125,7 +125,7 @@ export class DeduplicatedLoadSubset { const matchingInflight = this.inflightCalls.find( (inflight) => !inflight.lease.aborted && - isLoadSubsetCoveredBy(options, inflight.options), + isLoadSubsetRequestSubsumedBy(options, inflight.options), ) if (matchingInflight !== undefined) { diff --git a/packages/db/tests/query/compiler/basic.test.ts b/packages/db/tests/query/compiler/basic.test.ts index 2ff6b56adb..52f84b4df9 100644 --- a/packages/db/tests/query/compiler/basic.test.ts +++ b/packages/db/tests/query/compiler/basic.test.ts @@ -188,6 +188,63 @@ describe(`Query2 Compiler`, () => { }) }) + test(`implicit joined results expose their lexical aliases`, () => { + type Post = { id: number; userId: number; title: string } + const usersCollection = { + id: `users`, + config: { autoIndex: `off` }, + } as CollectionImpl + const postsCollection = { + id: `posts`, + config: { autoIndex: `off` }, + } as CollectionImpl + + const resultKeys = (userAlias: string, postAlias: string) => { + const graph = new D2() + const usersInput = graph.newInput<[number, User]>() + const postsInput = graph.newInput<[number, Post]>() + const query: QueryIR = { + from: new CollectionRef(usersCollection, userAlias), + join: [ + { + type: `inner`, + from: new CollectionRef(postsCollection, postAlias), + left: new PropRef([userAlias, `id`]), + right: new PropRef([postAlias, `userId`]), + }, + ], + } + const { pipeline } = compileQuery( + query, + { [userAlias]: usersInput, [postAlias]: postsInput }, + { users: usersCollection, posts: postsCollection }, + {}, + {}, + new Set(), + {}, + () => {}, + ) + const messages: Array> = [] + pipeline.pipe(output((message) => messages.push(message))) + graph.finalize() + + usersInput.sendData(new MultiSet([[[1, sampleUsers[0]!], 1]])) + postsInput.sendData( + new MultiSet([[[10, { id: 10, userId: 1, title: `Hello` }], 1]]), + ) + graph.run() + + const result = messages + .flatMap((message) => message.getInner()) + .map(([data]) => data[1][0]) + .find((row) => row !== undefined) + return Object.keys(result).sort() + } + + expect(resultKeys(`user`, `post`)).toEqual([`post`, `user`]) + expect(resultKeys(`account`, `article`)).toEqual([`account`, `article`]) + }) + test(`compiles a query with WHERE clause`, () => { const usersCollection = { id: `users`, diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index 8a5447557f..5a85d9ec48 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -46,7 +46,7 @@ import { Value, } from '../../src/query/ir.js' import { compileExpression } from '../../src/query/compiler/evaluators.js' -import { isLoadSubsetCoveredBy } from '../../src/query/predicate-utils.js' +import { isLoadSubsetRequestSubsumedBy } from '../../src/query/predicate-utils.js' import { createRuntimeReferenceIdentityFactory } from '../../src/query/runtime-reference-identity.js' import type { BasicExpression, QueryIR } from '../../src/query/ir.js' import type { LoadSubsetOptions } from '../../src/types.js' @@ -93,6 +93,28 @@ const referenceSemanticPairArbitrary = fc.oneof( ]), ) +const outputExpressionPairArbitrary: fc.Arbitrary<{ + first: BasicExpression + second: BasicExpression +}> = fc.oneof( + fc.integer().map((value) => ({ + first: new Value(value), + second: new Value(value), + })), + fc.string().map((value) => ({ + first: new Func(`concat`, [new Value(value)]), + second: new Func(`concat`, [new Value(value)]), + })), + fc.uint8Array({ minLength: 1, maxLength: 8 }).map((value) => ({ + first: new Func(`concat`, [new Value(Buffer.from(value))]), + second: new Func(`concat`, [new Value(new Uint8Array(value))]), + })), + fc.constant({ + first: new Value(-0), + second: new Value(0), + }), +) + interface Post { id: number userId: number @@ -264,7 +286,7 @@ describe(`semantic expression identity`, () => { getLoadSubsetDemandKey({ where: firstPredicate, limit: 1 }), ).not.toBe(getLoadSubsetDemandKey({ where: secondPredicate, limit: 1 })) expect( - isLoadSubsetCoveredBy( + isLoadSubsetRequestSubsumedBy( { where: firstPredicate, limit: 1 }, { where: secondPredicate, limit: 1 }, ), @@ -507,6 +529,45 @@ function createAlphaRenamedJoinQuery( } } +function createAlphaRenamedImplicitJoinQuery( + userAlias: string, + postAlias: string, +): QueryIR { + return { + from: new CollectionRef( + usersCollection as unknown as ConstructorParameters< + typeof CollectionRef + >[0], + userAlias, + ), + join: [ + { + type: `inner`, + from: new CollectionRef( + postsCollection as unknown as ConstructorParameters< + typeof CollectionRef + >[0], + postAlias, + ), + left: new PropRef([userAlias, `id`]), + right: new PropRef([postAlias, `userId`]), + }, + ], + } +} + +function createProjectedExpressionQuery(expression: BasicExpression): QueryIR { + return { + from: new CollectionRef( + usersCollection as unknown as ConstructorParameters< + typeof CollectionRef + >[0], + `user`, + ), + select: { value: expression }, + } +} + function createAlphaRenamedNestedQuery( innerAlias: string, outerAlias: string, @@ -987,6 +1048,66 @@ describe(`stable QueryIR identity smoke test`, () => { ) }) + it(`keeps aliases that define an implicit joined result shape`, () => { + expect( + getQueryIdentity(createAlphaRenamedImplicitJoinQuery(`user`, `post`)), + ).not.toBe( + getQueryIdentity( + createAlphaRenamedImplicitJoinQuery(`account`, `article`), + ), + ) + }) + + it(`keeps output-producing runtime values exact`, () => { + const bufferExpression = new Func(`concat`, [new Value(Buffer.from([65]))]) + const uint8Expression = new Func(`concat`, [ + new Value(new Uint8Array([65])), + ]) + + expect(compileExpression(bufferExpression)({})).toBe(`A`) + expect(compileExpression(uint8Expression)({})).toBe(`65`) + expect( + getQueryIdentity(createProjectedExpressionQuery(bufferExpression)), + ).not.toBe( + getQueryIdentity(createProjectedExpressionQuery(uint8Expression)), + ) + + expect( + getQueryIdentity(createProjectedExpressionQuery(new Value(-0))), + ).not.toBe(getQueryIdentity(createProjectedExpressionQuery(new Value(0)))) + + const firstObject = { value: 1 } + const secondObject = { value: 1 } + expect( + getQueryIdentity(createProjectedExpressionQuery(new Value(firstObject))), + ).not.toBe( + getQueryIdentity(createProjectedExpressionQuery(new Value(secondObject))), + ) + expect(compileExpression(new Value(firstObject))({})).toBe(firstObject) + expect(compileExpression(new Value(secondObject))({})).toBe(secondObject) + }) + + fcTest.prop([outputExpressionPairArbitrary])( + `equal query identities imply equal projected expression results`, + ({ first, second }) => { + const firstIdentity = getQueryIdentity( + createProjectedExpressionQuery(first), + ) + const secondIdentity = getQueryIdentity( + createProjectedExpressionQuery(second), + ) + + if (firstIdentity === secondIdentity) { + expect( + Object.is( + compileExpression(first)({}), + compileExpression(second)({}), + ), + ).toBe(true) + } + }, + ) + fcTest.prop([ fc .stringMatching(/^[a-z][a-z0-9]{0,8}$/) diff --git a/packages/db/tests/query/load-subset-oracle.property.test.ts b/packages/db/tests/query/load-subset-oracle.property.test.ts index fd6296e0b0..6c3bdead0e 100644 --- a/packages/db/tests/query/load-subset-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-oracle.property.test.ts @@ -1865,7 +1865,7 @@ describe(`loadSubset coverage oracle`, () => { ], ] as const)( `discovered trace: a different %s starts a distinct window load`, - async (_name, firstOptions, secondOptions) => { + (_name, firstOptions, secondOptions) => { const createRequest = ( compareOptions: typeof firstOptions | typeof secondOptions, ): WindowRequest => ({ @@ -1875,25 +1875,12 @@ describe(`loadSubset coverage oracle`, () => { limit: 1, ...compareOptions, }) - await expectAssertionFailure( - () => - Promise.resolve().then(() => { - try { - expect( - countWindowLoads([ - createRequest(firstOptions), - createRequest(secondOptions), - ]), - ).toBe(2) - } catch (error) { - throw new TraceAssertionError(0, error) - } - }), - { - checkpoint: 0, - classify: ({ actual, expected }) => actual === 1 && expected === 2, - }, - )() + expect( + countWindowLoads([ + createRequest(firstOptions), + createRequest(secondOptions), + ]), + ).toBe(2) }, ) diff --git a/packages/db/tests/query/predicate-utils.test.ts b/packages/db/tests/query/predicate-utils.test.ts index 721c07f5ae..9d832f39f7 100644 --- a/packages/db/tests/query/predicate-utils.test.ts +++ b/packages/db/tests/query/predicate-utils.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { isLimitSubset, + isLoadSubsetRequestSubsumedBy, isOffsetLimitSubset, isOrderBySubset, isPredicateSubset, @@ -676,6 +677,71 @@ describe(`isOrderBySubset`, () => { expect(isOrderBySubset(subset, superset)).toBe(false) }) + it.each([ + [ + `null placement`, + { direction: `asc`, nulls: `first`, stringSort: `lexical` } as const, + { direction: `asc`, nulls: `last`, stringSort: `lexical` } as const, + ], + [ + `string sort mode`, + { direction: `asc`, nulls: `last`, stringSort: `lexical` } as const, + { direction: `asc`, nulls: `last`, stringSort: `locale` } as const, + ], + [ + `locale`, + { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + locale: `en-US`, + } as const, + { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + locale: `de-DE`, + } as const, + ], + [ + `locale options`, + { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + locale: `en-US`, + localeOptions: { numeric: true, sensitivity: `base` }, + } as const, + { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + locale: `en-US`, + localeOptions: { numeric: false, sensitivity: `base` }, + } as const, + ], + ])(`should return false when %s differs`, (_label, first, second) => { + const expression = ref(`name`) + expect( + isOrderBySubset( + [{ expression, compareOptions: first }], + [{ expression, compareOptions: second }], + ), + ).toBe(false) + expect( + isLoadSubsetRequestSubsumedBy( + { + orderBy: [{ expression, compareOptions: first }], + limit: 10, + }, + { + orderBy: [{ expression, compareOptions: second }], + limit: 20, + }, + ), + ).toBe(false) + }) + it(`should return false when subset is longer than superset`, () => { const subset: OrderBy = [ orderByClause(ref(`age`), `asc`), From 1384a47c959138efd160917f1f04f38f99075d6c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 24 Aug 2026 20:39:14 -0600 Subject: [PATCH 5/7] fix(db): address demand identity review findings --- packages/db/src/query/ir-stable-identity.ts | 17 +++-- packages/db/src/query/predicate-utils.ts | 73 ++++++++++++++++--- packages/db/src/query/subset-dedupe.ts | 27 ++++++- .../db/tests/query/predicate-utils.test.ts | 44 +++++++++++ packages/db/tests/query/subset-dedupe.test.ts | 25 +++++++ .../load-subset-lifecycle-oracle.test.ts | 63 ++++++++-------- 6 files changed, 200 insertions(+), 49 deletions(-) diff --git a/packages/db/src/query/ir-stable-identity.ts b/packages/db/src/query/ir-stable-identity.ts index 3c0ead5ccf..d7fe07e306 100644 --- a/packages/db/src/query/ir-stable-identity.ts +++ b/packages/db/src/query/ir-stable-identity.ts @@ -827,12 +827,17 @@ function canonicalizeFunction( function sortUniqueStableIdentityValues( values: Array, ): Array { - values.sort(compareStableIdentityValues) - return values.filter( - (value, index) => - index === 0 || - compareStableIdentityValues(value, values[index - 1]!) !== 0, - ) + const keyedValues = values.map((value) => ({ + key: JSON.stringify(value), + value, + })) + keyedValues.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)) + return keyedValues + .filter( + (entry, index) => + index === 0 || entry.key !== keyedValues[index - 1]!.key, + ) + .map((entry) => entry.value) } function isCanonicalFunction( diff --git a/packages/db/src/query/predicate-utils.ts b/packages/db/src/query/predicate-utils.ts index deb63d9c88..27a43f0f16 100644 --- a/packages/db/src/query/predicate-utils.ts +++ b/packages/db/src/query/predicate-utils.ts @@ -46,7 +46,7 @@ export function isWhereSubset( return true } - return isWhereSubsetInternal(subset!, superset!) + return isWhereSubsetInternal(subset!, superset!, new WeakMap()) } function makeDisjunction( @@ -71,6 +71,7 @@ function convertInToOr(inField: InField) { function isWhereSubsetInternal( subset: BasicExpression, superset: BasicExpression, + expressionHashes: ExpressionHashCache, ): boolean { // If subset is false it is requesting no data, // thus the result set is empty @@ -80,7 +81,7 @@ function isWhereSubsetInternal( } // If expressions are structurally equal, subset relationship holds - if (areExpressionsEqual(subset, superset)) { + if (areExpressionsEqual(subset, superset, expressionHashes)) { return true } @@ -89,7 +90,11 @@ function isWhereSubsetInternal( // Example: (age > 20) ⊆ (age > 10 AND status = 'active') is false (doesn't imply status condition) if (superset.type === `func` && superset.name === `and`) { return superset.args.every((arg) => - isWhereSubsetInternal(subset, arg as BasicExpression), + isWhereSubsetInternal( + subset, + arg as BasicExpression, + expressionHashes, + ), ) } @@ -98,7 +103,11 @@ function isWhereSubsetInternal( // decomposes the subset first: A ⊆ or(C, D) AND B ⊆ or(C, D). if (subset.type === `func` && subset.name === `or`) { return subset.args.every((arg) => - isWhereSubsetInternal(arg as BasicExpression, superset), + isWhereSubsetInternal( + arg as BasicExpression, + superset, + expressionHashes, + ), ) } @@ -107,7 +116,11 @@ function isWhereSubsetInternal( // match a structurally equal disjunct via areExpressionsEqual. if (superset.type === `func` && superset.name === `or`) { return superset.args.some((arg) => - isWhereSubsetInternal(subset, arg as BasicExpression), + isWhereSubsetInternal( + subset, + arg as BasicExpression, + expressionHashes, + ), ) } @@ -115,7 +128,11 @@ function isWhereSubsetInternal( if (subset.type === `func` && subset.name === `and`) { // For (A AND B) ⊆ C, since (A AND B) implies A, we check if any conjunct implies C return subset.args.some((arg) => - isWhereSubsetInternal(arg as BasicExpression, superset), + isWhereSubsetInternal( + arg as BasicExpression, + superset, + expressionHashes, + ), ) } @@ -124,14 +141,22 @@ function isWhereSubsetInternal( if (subset.type === `func` && subset.name === `in`) { const inField = extractInField(subset) if (inField) { - return isWhereSubsetInternal(convertInToOr(inField), superset) + return isWhereSubsetInternal( + convertInToOr(inField), + superset, + expressionHashes, + ) } } if (superset.type === `func` && superset.name === `in`) { const inField = extractInField(superset) if (inField) { - return isWhereSubsetInternal(subset, convertInToOr(inField)) + return isWhereSubsetInternal( + subset, + convertInToOr(inField), + expressionHashes, + ) } } @@ -1088,13 +1113,41 @@ function findPredicateWithOperator( }) } -function areExpressionsEqual(a: BasicExpression, b: BasicExpression): boolean { +const unhashableExpression = Symbol(`unhashableExpression`) +type ExpressionHashCache = WeakMap< + BasicExpression, + string | typeof unhashableExpression +> + +function getCachedExpressionHash( + expression: BasicExpression, + expressionHashes: ExpressionHashCache, +): string | typeof unhashableExpression { + const cachedHash = expressionHashes.get(expression) + if (cachedHash !== undefined) return cachedHash + try { - return getStableExpressionHash(a) === getStableExpressionHash(b) + const hash = getStableExpressionHash(expression) + expressionHashes.set(expression, hash) + return hash } catch (error) { if (!(error instanceof UnhashableQueryIRError)) throw error + expressionHashes.set(expression, unhashableExpression) + return unhashableExpression + } +} + +function areExpressionsEqual( + a: BasicExpression, + b: BasicExpression, + expressionHashes: ExpressionHashCache = new WeakMap(), +): boolean { + const aHash = getCachedExpressionHash(a, expressionHashes) + const bHash = getCachedExpressionHash(b, expressionHashes) + if (aHash === unhashableExpression || bHash === unhashableExpression) { return areExpressionsStructurallyEqual(a, b) } + return aHash === bHash } function areExpressionsStructurallyEqual( diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index 37ce49e0bf..4e0a978ec4 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -4,6 +4,7 @@ import { minusWherePredicates, unionWherePredicates, } from './predicate-utils.js' +import { Func, PropRef, Value } from './ir.js' import type { BasicExpression } from './ir.js' import type { LoadSubsetOptions } from '../types.js' @@ -324,10 +325,34 @@ function createSharedAbortLease( export function cloneOptions(options: LoadSubsetOptions): LoadSubsetOptions { return { ...options, + where: options.where ? cloneBasicExpression(options.where) : undefined, orderBy: options.orderBy?.map((clause) => ({ ...clause, + expression: cloneBasicExpression(clause.expression), compareOptions: { ...clause.compareOptions }, })), - cursor: options.cursor ? { ...options.cursor } : undefined, + cursor: options.cursor + ? { + ...options.cursor, + whereFrom: cloneBasicExpression(options.cursor.whereFrom), + whereCurrent: cloneBasicExpression(options.cursor.whereCurrent), + } + : undefined, + } +} + +function cloneBasicExpression( + expression: BasicExpression, +): BasicExpression { + switch (expression.type) { + case `ref`: + return new PropRef([...expression.path]) + case `val`: + return new Value(expression.value) + case `func`: + return new Func( + expression.name, + expression.args.map((arg) => cloneBasicExpression(arg)), + ) } } diff --git a/packages/db/tests/query/predicate-utils.test.ts b/packages/db/tests/query/predicate-utils.test.ts index 9d832f39f7..26e5938790 100644 --- a/packages/db/tests/query/predicate-utils.test.ts +++ b/packages/db/tests/query/predicate-utils.test.ts @@ -915,6 +915,50 @@ describe(`isPredicateSubset`, () => { expect(isPredicateSubset(subset, superset)).toBe(true) }) + it(`does not normalize distinct comparison operators at a limited boundary`, () => { + const subset: LoadSubsetOptions = { + where: gt(ref(`age`), val(18)), + limit: 10, + } + const superset: LoadSubsetOptions = { + where: gte(ref(`age`), val(18)), + limit: 20, + } + + expect(isPredicateSubset(subset, superset)).toBe(false) + }) + + it(`does not retain expression hashes across comparison operations`, () => { + const subset = gt(ref(`age`), val(18)) + const superset = gt(ref(`age`), val(18)) + + expect(isWhereSubset(subset, superset)).toBe(true) + superset.name = `lt` + expect(isWhereSubset(subset, superset)).toBe(false) + }) + + it(`hashes a repeated expression once per subset comparison`, () => { + let valueReads = 0 + const countedValue = val(1) + Object.defineProperty(countedValue, `value`, { + configurable: true, + get: () => { + valueReads++ + return 1 + }, + }) + const subset = func(`custom-subset`, countedValue) + const superset = func( + `or`, + ...Array.from({ length: 4 }, (_, index) => + func(`custom-superset-${index}`, val(index)), + ), + ) + + expect(isWhereSubset(subset, superset)).toBe(false) + expect(valueReads).toBe(1) + }) + it(`should return false for limited superset with different where clause`, () => { // Even if subset's where is more restrictive, it can't be a subset // of a limited superset with a different where clause. diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 4c5d8d4f96..4ad426b26b 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -1387,5 +1387,30 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(result).toBe(true) expect(callCount).toBe(1) }) + + it(`does not let caller mutations change a stored cursor boundary`, async () => { + const loadSubset = vi.fn().mockResolvedValue(undefined) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const mutableBoundary = val(1) + const firstCursor = { + whereFrom: gt(ref(`id`), mutableBoundary), + whereCurrent: eq(ref(`id`), mutableBoundary), + lastKey: 1, + } + + await deduplicated.loadSubset({ cursor: firstCursor, limit: 10 }) + mutableBoundary.value = 2 + + await deduplicated.loadSubset({ + cursor: { + whereFrom: gt(ref(`id`), val(2)), + whereCurrent: eq(ref(`id`), val(2)), + lastKey: 1, + }, + limit: 10, + }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + }) }) }) diff --git a/packages/query-db-collection/tests/load-subset-lifecycle-oracle.test.ts b/packages/query-db-collection/tests/load-subset-lifecycle-oracle.test.ts index 38dc3e9a19..b7d4b395ad 100644 --- a/packages/query-db-collection/tests/load-subset-lifecycle-oracle.test.ts +++ b/packages/query-db-collection/tests/load-subset-lifecycle-oracle.test.ts @@ -349,20 +349,9 @@ async function expectDeferredStartupReadyDoesNotOverrideError(): Promise { async function expectEquivalentPredicatesShareOneLoad( form: `commutative-and` | `commutative-or` | `reversed-equality`, ): Promise { - const queryClient = createQueryClient() - const id = `load-subset-canonical-predicate-${collectionSequence++}` - const queryFn = vi.fn().mockResolvedValue([{ id: `a`, group: `x` }]) - const collection = createCollection( - queryCollectionOptions({ - id, - queryClient, - queryKey: [id], - queryFn, - getKey: (row) => row.id, - startSync: true, - syncMode: `on-demand`, - retry: false, - }), + const { queryClient, collection, queryFn } = createOnDemandCollection( + `load-subset-canonical-predicate`, + [{ id: `a`, group: `x` }], ) const firstComparison = new IR.Func(`eq`, [ new IR.PropRef([`id`]), @@ -407,20 +396,9 @@ async function expectEquivalentComparisonValuesShareOneLoad( firstValue: unknown, secondValue: unknown, ): Promise { - const queryClient = createQueryClient() - const id = `load-subset-comparison-value-${collectionSequence++}` - const queryFn = vi.fn().mockResolvedValue([{ id: `a` }]) - const collection = createCollection( - queryCollectionOptions({ - id, - queryClient, - queryKey: [id], - queryFn, - getKey: (row) => row.id, - startSync: true, - syncMode: `on-demand`, - retry: false, - }), + const { queryClient, collection, queryFn } = createOnDemandCollection( + `load-subset-comparison-value`, + [{ id: `a` }], ) const value = new IR.PropRef([`value`]) @@ -438,6 +416,25 @@ async function expectEquivalentComparisonValuesShareOneLoad( } } +function createOnDemandCollection(idPrefix: string, rows: Array) { + const queryClient = createQueryClient() + const id = `${idPrefix}-${collectionSequence++}` + const queryFn = vi.fn().mockResolvedValue(rows) + const collection = createCollection( + queryCollectionOptions({ + id, + queryClient, + queryKey: [id], + queryFn, + getKey: (row) => row.id, + startSync: true, + syncMode: `on-demand`, + retry: false, + }), + ) + return { queryClient, collection, queryFn } +} + async function expectFinalOwnerCleanupAbortsQuery(): Promise { const queryClient = createQueryClient() const id = `load-subset-cancel-final-owner-${collectionSequence++}` @@ -567,10 +564,12 @@ describe(`loadSubset lifecycle oracle`, () => { await expectDeferredStartupReadyDoesNotOverrideError() }) - it(`commutative predicate forms share one query-db transport load`, async () => { - await expectEquivalentPredicatesShareOneLoad(`commutative-and`) - await expectEquivalentPredicatesShareOneLoad(`commutative-or`) - }) + it.each([`commutative-and`, `commutative-or`] as const)( + `%s predicate forms share one query-db transport load`, + async (form) => { + await expectEquivalentPredicatesShareOneLoad(form) + }, + ) it(`reversed equality operands share one query-db transport load`, async () => { await expectEquivalentPredicatesShareOneLoad(`reversed-equality`) From a6b5ffaac35c2ab69898add9187681d02bb82d09 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 25 Aug 2026 07:32:52 -0600 Subject: [PATCH 6/7] fix(db): keep demand history immutable --- packages/db/src/query/predicate-utils.ts | 9 +++ packages/db/src/query/subset-dedupe.ts | 70 +++++++++++++++++-- .../db/tests/query/predicate-utils.test.ts | 18 +++++ packages/db/tests/query/subset-dedupe.test.ts | 33 +++++++++ 4 files changed, 125 insertions(+), 5 deletions(-) diff --git a/packages/db/src/query/predicate-utils.ts b/packages/db/src/query/predicate-utils.ts index 27a43f0f16..3241f9e55d 100644 --- a/packages/db/src/query/predicate-utils.ts +++ b/packages/db/src/query/predicate-utils.ts @@ -913,6 +913,15 @@ export function isPredicateSubset( } } + // A cursor-relative request is also a finite window. Even when it has no + // numeric limit, a different predicate can select rows outside that window. + if ( + superset.cursor !== undefined && + !areWhereClausesEqual(subset.where, superset.where) + ) { + return false + } + if (superset.limit !== undefined) { // For limited supersets, where clauses must be equal if (!areWhereClausesEqual(subset.where, superset.where)) { diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index 4e0a978ec4..f5b98b2531 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -325,7 +325,9 @@ function createSharedAbortLease( export function cloneOptions(options: LoadSubsetOptions): LoadSubsetOptions { return { ...options, - where: options.where ? cloneBasicExpression(options.where) : undefined, + where: options.where + ? cloneBasicExpression(options.where, `predicate`) + : undefined, orderBy: options.orderBy?.map((clause) => ({ ...clause, expression: cloneBasicExpression(clause.expression), @@ -334,25 +336,83 @@ export function cloneOptions(options: LoadSubsetOptions): LoadSubsetOptions { cursor: options.cursor ? { ...options.cursor, - whereFrom: cloneBasicExpression(options.cursor.whereFrom), - whereCurrent: cloneBasicExpression(options.cursor.whereCurrent), + whereFrom: cloneBasicExpression( + options.cursor.whereFrom, + `predicate`, + ), + whereCurrent: cloneBasicExpression( + options.cursor.whereCurrent, + `predicate`, + ), } : undefined, } } +type ExpressionCloneContext = `exact` | `predicate` | `comparison` + function cloneBasicExpression( expression: BasicExpression, + context: ExpressionCloneContext = `exact`, ): BasicExpression { switch (expression.type) { case `ref`: return new PropRef([...expression.path]) case `val`: - return new Value(expression.value) + return new Value( + context === `comparison` + ? snapshotComparisonValue(expression.value) + : expression.value, + ) case `func`: return new Func( expression.name, - expression.args.map((arg) => cloneBasicExpression(arg)), + expression.args.map((arg, index) => { + if ( + context === `predicate` && + expression.name === `in` && + index === 1 && + arg.type === `val` && + Array.isArray(arg.value) + ) { + return new Value( + arg.value.map((value) => snapshotComparisonValue(value)), + ) + } + + const argumentContext = + context === `predicate` && isComparisonFunction(expression.name) + ? `comparison` + : context + return cloneBasicExpression(arg, argumentContext) + }), ) } } + +function isComparisonFunction(name: string): boolean { + return ( + name === `eq` || + name === `gt` || + name === `gte` || + name === `lt` || + name === `lte` + ) +} + +function snapshotComparisonValue(value: T): T { + if (value instanceof Date) { + return new Date(value.getTime()) as T + } + + if (typeof Buffer !== `undefined` && value instanceof Buffer) { + return Buffer.from(value) as T + } + + if (value instanceof Uint8Array) { + return value.slice() as T + } + + // Other objects use reference equality in predicate identity and comparison. + return value +} diff --git a/packages/db/tests/query/predicate-utils.test.ts b/packages/db/tests/query/predicate-utils.test.ts index 26e5938790..6471950dee 100644 --- a/packages/db/tests/query/predicate-utils.test.ts +++ b/packages/db/tests/query/predicate-utils.test.ts @@ -928,6 +928,24 @@ describe(`isPredicateSubset`, () => { expect(isPredicateSubset(subset, superset)).toBe(false) }) + it(`requires equal predicates for a cursor-relative superset`, () => { + const cursor = { + whereFrom: gt(ref(`id`), val(10)), + whereCurrent: eq(ref(`id`), val(10)), + lastKey: 10, + } + const subset: LoadSubsetOptions = { + where: gt(ref(`age`), val(18)), + cursor, + } + const superset: LoadSubsetOptions = { + where: gte(ref(`age`), val(18)), + cursor, + } + + expect(isPredicateSubset(subset, superset)).toBe(false) + }) + it(`does not retain expression hashes across comparison operations`, () => { const subset = gt(ref(`age`), val(18)) const superset = gt(ref(`age`), val(18)) diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 4ad426b26b..b76ba2963c 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -1412,5 +1412,38 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(loadSubset).toHaveBeenCalledTimes(2) }) + + it(`does not let Date mutation change a stored cursor boundary`, async () => { + const loadSubset = vi.fn().mockResolvedValue(undefined) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const mutableBoundary = new Date(`2025-01-01T00:00:00.000Z`) + + await deduplicated.loadSubset({ + cursor: { + whereFrom: gt(ref(`createdAt`), val(mutableBoundary)), + whereCurrent: eq(ref(`createdAt`), val(mutableBoundary)), + lastKey: 1, + }, + limit: 10, + }) + mutableBoundary.setUTCFullYear(2026) + + await deduplicated.loadSubset({ + cursor: { + whereFrom: gt( + ref(`createdAt`), + val(new Date(`2026-01-01T00:00:00.000Z`)), + ), + whereCurrent: eq( + ref(`createdAt`), + val(new Date(`2026-01-01T00:00:00.000Z`)), + ), + lastKey: 1, + }, + limit: 10, + }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + }) }) }) From 16ad1c3ccf8c164ab18d33d8f5f8885259954841 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 25 Aug 2026 09:08:27 -0600 Subject: [PATCH 7/7] fix(db): preserve observable query identity --- packages/db/src/query/ir-stable-identity.ts | 11 ++-- .../db/tests/query/ir-stable-identity.test.ts | 55 ++++++++++++++++++- 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/packages/db/src/query/ir-stable-identity.ts b/packages/db/src/query/ir-stable-identity.ts index d7fe07e306..1c07d0b748 100644 --- a/packages/db/src/query/ir-stable-identity.ts +++ b/packages/db/src/query/ir-stable-identity.ts @@ -270,7 +270,9 @@ function canonicalizeQueryInScope( if ( !query.select && - (Boolean(query.join?.length) || Boolean(query.groupBy?.length)) + (query.from.type === `unionFrom` || + query.join !== undefined || + query.groupBy !== undefined) ) { // Without an explicit projection, these query shapes return a namespaced // row. Its alias keys are public output and therefore part of identity. @@ -799,9 +801,10 @@ function canonicalizeFunction( ) const unique = sortUniqueStableIdentityValues(flattened) - return unique.length === 1 - ? unique[0]! - : { type: `func`, name, args: unique } + // The evaluator gives `and` and `or` boolean results even when their sole + // operand returns another truthy or falsy value. Keep that coercion in the + // identity because expression result types are erased at runtime. + return { type: `func`, name, args: unique } } if (name === `eq` && args.length === 2) { diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index 5a85d9ec48..ca745d9de5 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -45,7 +45,10 @@ import { UnionAll, Value, } from '../../src/query/ir.js' -import { compileExpression } from '../../src/query/compiler/evaluators.js' +import { + compileExpression, + toBooleanPredicate, +} from '../../src/query/compiler/evaluators.js' import { isLoadSubsetRequestSubsumedBy } from '../../src/query/predicate-utils.js' import { createRuntimeReferenceIdentityFactory } from '../../src/query/runtime-reference-identity.js' import type { BasicExpression, QueryIR } from '../../src/query/ir.js' @@ -231,7 +234,19 @@ describe(`semantic expression identity`, () => { expect(getStableExpressionHash(nested)).toBe(getStableExpressionHash(flat)) expect(getStableExpressionHash(new Func(`or`, [adult, adult]))).toBe( - getStableExpressionHash(adult), + getStableExpressionHash(new Func(`or`, [adult])), + ) + }) + + it(`keeps a boolean wrapper when duplicate operands coerce their result`, () => { + const bareAge = new PropRef([`user`, `age`]) + const duplicateAnd = new Func(`and`, [bareAge, bareAge]) + const row = { user: { age: 18 } } + + expect(toBooleanPredicate(compileExpression(bareAge)(row))).toBe(false) + expect(toBooleanPredicate(compileExpression(duplicateAnd)(row))).toBe(true) + expect(getStableExpressionHash(duplicateAnd)).not.toBe( + getStableExpressionHash(bareAge), ) }) @@ -1058,6 +1073,42 @@ describe(`stable QueryIR identity smoke test`, () => { ) }) + it(`keeps aliases that define an implicit union-source result shape`, () => { + const usersAndPosts = getQueryIR( + new Query().unionAll({ + user: usersCollection, + post: postsCollection, + }), + ) + const accountsAndArticles = getQueryIR( + new Query().unionAll({ + account: usersCollection, + article: postsCollection, + }), + ) + + expect(usersAndPosts.from.type).toBe(`unionFrom`) + expect(getQueryIdentity(usersAndPosts)).not.toBe( + getQueryIdentity(accountsAndArticles), + ) + }) + + it(`keeps aliases when an empty groupBy still selects a namespaced row`, () => { + const createQuery = (alias: string) => + getQueryIR( + new Query() + .from({ [alias]: usersCollection } as Record< + string, + typeof usersCollection + >) + .groupBy(() => []), + ) + + expect(getQueryIdentity(createQuery(`user`))).not.toBe( + getQueryIdentity(createQuery(`account`)), + ) + }) + it(`keeps output-producing runtime values exact`, () => { const bufferExpression = new Func(`concat`, [new Value(Buffer.from([65]))]) const uint8Expression = new Func(`concat`, [