From 40b7dad88b72ee77cf1f206b5fe3135af763a0eb Mon Sep 17 00:00:00 2001 From: Ivan Vasilov Date: Mon, 21 Sep 2026 00:08:26 +0200 Subject: [PATCH 1/2] fix: page collection reads past the PostgREST db-max-rows cap Collection reads issued a single PostgREST request per page, which PostgREST silently truncates at its db-max-rows cap (1000 by default). A matching set larger than the cap loaded incomplete data with no error. Wrap reads in an offset-paging loop (fetchAllPages) that fetches the complete matching set, or the caller's limit. The first request asks for count=exact so the loop knows the total up front and stops without a trailing empty probe; later pages advance the offset by the rows actually received (self-correcting under concurrent writes) and are capped by the observed page size and the rows still owed. - postgrest-request: return { data, count }; add count and signal options. The signal threads straight into the builder config and on to fetch. - functions: supabaseQueryFn routes both the main read and the boundary-tie read through the loop and threads ctx.signal. The tie read is paged but unbounded by the caller's limit so every tied row returns. - query-once: non-aggregate queryOnce is paginated for free (it loads its source collections through supabaseQueryFn); executeQuery (aggregate / groupBy / having) stays a single request and adopts the new return shape. The query key keeps limit/offset (the loop overrides them per page on a clone), so distinct windows of a subset still get distinct keys. Co-Authored-By: Dhruv Vaishnav --- README.md | 6 ++ src/functions.ts | 97 ++++++++++++++++++++--- src/postgrest-request.ts | 37 ++++++++- src/query-once.ts | 5 +- supabase/config.toml | 6 +- tests/e2e/reads.test.ts | 36 +++++++++ tests/pagination.test.ts | 157 +++++++++++++++++++++++++++++++++++++- tests/pagination.utils.ts | 42 +++++++--- tests/query-once.test.ts | 53 +++++++++++++ tests/test.utils.ts | 23 +++++- 10 files changed, 429 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 24aaaaf..3020983 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,12 @@ Most query operations are translated to PostgREST filters and run server-side. A The client-side filter fallback above applies to live queries and ordinary `queryOnce` queries. The server aggregate path in `queryOnce` requires fully pushable `WHERE` expressions and throws for unsupported filters, rather than returning an aggregate over unfiltered rows. +### Row cap and pagination + +PostgREST caps every response at its `db-max-rows` setting (1000 by default on Supabase). Collection reads and non-aggregate `queryOnce` queries page past this cap automatically: the adapter loops with `offset`, using a one-time `count=exact` on the first request to fetch the complete matching set (or the caller's `limit`). + +The **aggregate / `GROUP BY` / `HAVING`** path of `queryOnce` is the one exception — it issues a single request and is **not** paginated. Aggregate results are a few rows, so the cap is moot, but a `GROUP BY` / `HAVING` query whose grouped output exceeds `db-max-rows` is silently truncated at the cap. Add an explicit `limit`, or narrow the query, if you expect more grouped rows than the cap. + **Evaluated Client-Side** These operations fetch the required rows and process them in memory: diff --git a/src/functions.ts b/src/functions.ts index 50f8c5d..d2be1f4 100644 --- a/src/functions.ts +++ b/src/functions.ts @@ -52,6 +52,71 @@ export const subsetOptionsToQueryKey = ( return key ? [tableName, key] : [tableName] } +type PageOptions = { + /** The caller's row limit; `undefined` means "the full matching set". */ + limit?: number + /** The starting offset (always 0 for cursor reads). */ + offset: number + signal?: AbortSignal +} + +/** + * Read a full matching set (or the caller's `limit` rows) with offset paging, + * looping past PostgREST's `db-max-rows` cap instead of silently truncating at + * it. The first request asks for `count=exact` so the loop knows the total up + * front and stops without a trailing empty probe; each later page advances the + * offset by the rows actually received (self-correcting under concurrent + * writes) and is capped by the observed page size and the rows still owed. + */ +async function fetchAllPages( + supabase: SupabaseClient, + tableName: string, + baseSearch: URLSearchParams, + { limit, offset: startOffset, signal }: PageOptions +): Promise { + // A zero limit is an empty window: return it without touching the network. + if (limit === 0) { + return [] + } + + // The base search already carries the caller's limit/offset (from the search + // builders), so the first request is sent verbatim — its URL stays identical + // to the single-request behaviour, and the query key that mirrors it. + const first = await postgrestRequest(supabase, tableName, { + method: "GET", + search: baseSearch, + signal, + count: "exact", + }) + const rows: any[] = first.data ? [...first.data] : [] + + // The rows the first page returned size the server's cap; a later page that + // comes back shorter than this means the server has no more rows. + const pageSize = rows.length + // How many rows we ultimately want: the caller's limit, bounded by the total + // the server reported. When the count header is missing we fall back to + // paging until a short page appears. + const total = first.count ?? Number.POSITIVE_INFINITY + const target = limit === undefined ? total : Math.min(limit, total) + + let lastPageSize = pageSize + while (lastPageSize === pageSize && pageSize > 0 && rows.length < target) { + const pageSearch = new URLSearchParams(baseSearch) + pageSearch.set("offset", `${startOffset + rows.length}`) + pageSearch.set("limit", `${Math.min(pageSize, target - rows.length)}`) + const page = await postgrestRequest(supabase, tableName, { + method: "GET", + search: pageSearch, + signal, + }) + const pageRows: any[] = page.data ?? [] + rows.push(...pageRows) + lastPageSize = pageRows.length + } + + return rows +} + export const supabaseQueryFn = async ( supabase: SupabaseClient, tableName: string, @@ -66,6 +131,9 @@ export const supabaseQueryFn = async ( ) => { const options = ctx.meta?.loadSubsetOptions ?? {} const search = loadSubsetOptionsToSearch(options) + // Cursor reads pin their own window start, so they never carry an offset (see + // loadSubsetOptionsToSearch); only a cursor-less read advances from `offset`. + const startOffset = options.offset && !options.cursor ? options.offset : 0 // A keyset request whose boundary column has ties (e.g. `orderBy(created_at)` // with repeated values) needs a second, unlimited request for the rows equal @@ -80,23 +148,30 @@ export const supabaseQueryFn = async ( // Honouring `whereCurrent` here is what makes the cursor correct on its own. const tiesSearch = cursorCurrentToSearch(options) if (!tiesSearch) { - const data = await postgrestRequest(supabase, tableName, { - method: "GET", - search, + return await fetchAllPages(supabase, tableName, search, { + limit: options.limit, + offset: startOffset, + signal: ctx.signal, }) - return data || [] } const [ties, rows] = await Promise.all([ - postgrestRequest(supabase, tableName, { - method: "GET", - search: tiesSearch, + // The tie read is deliberately unbounded by the caller's limit — every row + // tied at the boundary must come back — but it is still paged so a tie class + // larger than the cap is not truncated. + fetchAllPages(supabase, tableName, tiesSearch, { + offset: 0, + signal: ctx.signal, + }), + fetchAllPages(supabase, tableName, search, { + limit: options.limit, + offset: startOffset, + signal: ctx.signal, }), - postgrestRequest(supabase, tableName, { method: "GET", search }), ]) // `whereCurrent` (== boundary) and `whereFrom` (> / < boundary) are disjoint, // so concatenation never duplicates; the collection re-sorts locally. - return [...(ties || []), ...(rows || [])] + return [...ties, ...rows] } export const supabaseOnInsert = async ( @@ -106,7 +181,7 @@ export const supabaseOnInsert = async ( ) => { await Promise.all( transaction.mutations.map(async (mutation) => { - const data = await postgrestRequest(supabase, tableName, { + const { data } = await postgrestRequest(supabase, tableName, { method: "POST", search: new URLSearchParams({ select: "*" }), body: { ...mutation.modified }, @@ -136,7 +211,7 @@ export const supabaseOnUpdate = async ( const { original, changes } = mutation const search = keyColumnsToSearch(keys, original) search.set("select", "*") - const data = await postgrestRequest(supabase, tableName, { + const { data } = await postgrestRequest(supabase, tableName, { method: "PATCH", search, body: { ...original, ...changes }, diff --git a/src/postgrest-request.ts b/src/postgrest-request.ts index 72af421..6a2702a 100644 --- a/src/postgrest-request.ts +++ b/src/postgrest-request.ts @@ -8,15 +8,30 @@ const SINGLE_ROW_ACCEPT = "application/vnd.pgrst.object+json" interface PostgrestRequestOptions { /** JSON body for insert/update. */ body?: unknown + /** + * Ask PostgREST for the total matching-row count (`Prefer: count=…`). The + * count is read back off the `Content-Range` header; the paging loop uses it + * to fetch a set larger than the server's `db-max-rows` cap without probing. + */ + count?: "exact" | "planned" | "estimated" method: "GET" | "POST" | "PATCH" | "DELETE" /** Ask PostgREST to return the affected rows (`Prefer: return=representation`). */ returnRows?: boolean /** The full query string, built by the params helpers. */ search: URLSearchParams + /** Abort signal, threaded from the query's `ctx.signal`. */ + signal?: AbortSignal /** Expect exactly one row (`Accept: application/vnd.pgrst.object+json`). */ single?: boolean } +/** A PostgREST read result: the rows plus the total count when one was asked for. */ +export interface PostgrestResult { + /** Total matching rows when `count` was requested, else `null`. */ + count: number | null + data: any +} + /** * Issue a PostgREST request with a hand-built query string while leaving * supabase-js in charge of auth, schema, tracing, timeout, and transport. @@ -33,8 +48,16 @@ interface PostgrestRequestOptions { export async function postgrestRequest( supabase: SupabaseClient, table: string, - { method, search, body, returnRows, single }: PostgrestRequestOptions -): Promise { + { + method, + search, + body, + returnRows, + single, + count, + signal, + }: PostgrestRequestOptions +): Promise { const queryBuilder = supabase.from(table) const url = new URL(queryBuilder.url.toString()) @@ -45,6 +68,9 @@ export async function postgrestRequest( if (returnRows) { headers.append("Prefer", "return=representation") } + if (count) { + headers.append("Prefer", `count=${count}`) + } if (single) { headers.set("Accept", SINGLE_ROW_ACCEPT) } @@ -56,11 +82,14 @@ export async function postgrestRequest( schema: queryBuilder.schema, body, fetch: queryBuilder.fetch, + // The builder threads its own `signal` straight into the underlying fetch, + // so aborting the query cancels the in-flight request. + signal, }) - const { data, error } = await builder + const { data, error, count: total } = await builder if (error) { throw error } - return data + return { data, count: total ?? null } } diff --git a/src/query-once.ts b/src/query-once.ts index 3b34beb..077b5cb 100644 --- a/src/query-once.ts +++ b/src/query-once.ts @@ -30,7 +30,10 @@ export async function executeQuery( ir: SerializedQueryIR ): Promise { const { tableName, search } = queryIrToSearch(ir) - const data = await postgrestRequest(supabase, tableName, { + // The aggregate / groupBy / having path stays a single request: aggregate + // output is a handful of rows, so the db-max-rows cap does not apply and no + // paging loop is needed here. + const { data } = await postgrestRequest(supabase, tableName, { method: "GET", search, }) diff --git a/supabase/config.toml b/supabase/config.toml index b75c2d1..3815dbf 100644 --- a/supabase/config.toml +++ b/supabase/config.toml @@ -14,8 +14,10 @@ schemas = ["public", "graphql_public"] # Extra schemas to add to the search_path of every request. extra_search_path = ["public", "extensions"] # The maximum number of rows returns from a view, table, or stored procedure. Limits payload size -# for accidental or malicious requests. -max_rows = 1000 +# for accidental or malicious requests. Deliberately tiny for the e2e suite: it forces the read +# pagination loop (src/functions.ts `fetchAllPages`) to page through sets larger than the cap, so +# the multi-page path is exercised without seeding thousands of rows. +max_rows = 2 # Controls whether new tables, views, sequences and functions created in the `public` schema by # `postgres` are reachable through the Data API roles (`anon`, `authenticated`, `service_role`) # without explicit GRANTs. When unset, new entities are NOT auto-exposed, matching the new cloud diff --git a/tests/e2e/reads.test.ts b/tests/e2e/reads.test.ts index e2e774c..635f199 100644 --- a/tests/e2e/reads.test.ts +++ b/tests/e2e/reads.test.ts @@ -40,3 +40,39 @@ test("queryOnce runs a one-shot filtered query", async ({ users }) => { expect(rows).toHaveLength(1) expect(rows[0]?.name).toBe("Alice") }) + +test("reads a set larger than the server row cap across pages", async ({ + users, + other, +}) => { + // config.toml caps responses at 2 rows; reset_e2e seeds Alice + Bob, so three + // more rows push the matching set past the cap and force the paging loop. + const { error } = await other.from("users").insert([ + { name: "Carol", email: "carol@test.com", active: true }, + { name: "Dave", email: "dave@test.com", active: true }, + { name: "Erin", email: "erin@test.com", active: true }, + ] as unknown as never) + expect(error).toBeNull() + + const live = createLiveQueryCollection((q) => + q + .from({ row: users.collection }) + .select(({ row }) => ({ id: row.id, name: row.name })) + ) + + try { + await live.preload() + await vi.waitFor(() => expect(live.size).toBe(5), WAIT) + // The full set comes back despite the cap, with each row exactly once. + expect(live.toArray.map((row) => row.name).sort()).toEqual([ + "Alice", + "Bob", + "Carol", + "Dave", + "Erin", + ]) + expect(new Set(live.toArray.map((row) => row.id)).size).toBe(5) + } finally { + await live.cleanup() + } +}) diff --git a/tests/pagination.test.ts b/tests/pagination.test.ts index 307eb91..5a60728 100644 --- a/tests/pagination.test.ts +++ b/tests/pagination.test.ts @@ -3,9 +3,12 @@ import { createCollection, createLiveQueryCollection, createLiveQueryWindowController, + IR, } from "@tanstack/db" -import { afterEach, beforeEach, describe, expect, test } from "vitest" +import { QueryClient } from "@tanstack/query-core" +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest" import { z } from "zod" +import { supabaseQueryFn } from "../src/functions" import { supabaseCollectionOptions } from "../src/index" import { getSearches, makePaginatingFetch } from "./pagination.utils" import { SUPABASE_KEY, SUPABASE_URL } from "./test.utils" @@ -189,3 +192,155 @@ describe("live infinite query (windowed) pagination", () => { ]) }) }) + +// A collection load with no windowing (a plain subscribe, or a `queryOnce`) +// issues a single `supabaseQueryFn` call that must return the FULL matching set, +// looping past PostgREST's `db-max-rows` cap instead of truncating at it. +describe("db-max-rows paging loop (supabaseQueryFn)", () => { + // Ordered fixture the capped mock pages through. + const ROWS = [{ id: 1 }, { id: 2 }, { id: 3 }, { id: 4 }, { id: 5 }] + + const ORDER_BY = [ + { + expression: new IR.PropRef(["id"]), + compareOptions: { direction: "asc" as const, nulls: "last" as const }, + }, + ] + + const prefer = (call: unknown[]): string => + new Headers((call[1] as RequestInit)?.headers).get("prefer") ?? "" + + const run = ( + mockFetch: ReturnType, + loadSubsetOptions: Record, + signal: AbortSignal = new AbortController().signal + ) => { + const supabase = createClient(SUPABASE_URL, SUPABASE_KEY, { + global: { fetch: mockFetch }, + }) + return supabaseQueryFn(supabase, "items", { + client: new QueryClient(), + queryKey: ["items"], + signal, + meta: { loadSubsetOptions } as never, + }) + } + + test("pages through the whole set when it exceeds the server cap", async () => { + const mockFetch = makePaginatingFetch(ROWS, { cap: 2 }) + const rows = await run(mockFetch, { orderBy: ORDER_BY }) + + // Every row returned exactly once, in order. + expect((rows as Array<{ id: number }>).map((r) => r.id)).toEqual([ + 1, 2, 3, 4, 5, + ]) + // The first request has no artificial page-size limit — the cap sizes it — + // and each later page advances the offset by the rows received, capped by + // the rows still owed (the last page asks for just 1). + expect(getSearches(mockFetch)).toEqual([ + "select=*&order=id.asc", + "select=*&order=id.asc&offset=2&limit=2", + "select=*&order=id.asc&offset=4&limit=1", + ]) + }) + + test("requests count=exact on the first page only", async () => { + const mockFetch = makePaginatingFetch(ROWS, { cap: 2 }) + await run(mockFetch, { orderBy: ORDER_BY }) + + const prefers = mockFetch.mock.calls.map(prefer) + expect(prefers[0]).toContain("count=exact") + for (const p of prefers.slice(1)) { + expect(p).not.toContain("count=") + } + }) + + test("an exact multiple of the cap needs no trailing empty request", async () => { + const mockFetch = makePaginatingFetch(ROWS.slice(0, 4), { cap: 2 }) + const rows = await run(mockFetch, { orderBy: ORDER_BY }) + + expect((rows as Array<{ id: number }>).map((r) => r.id)).toEqual([ + 1, 2, 3, 4, + ]) + // Two requests, not three: the count tells the loop it is done at offset 4. + expect(getSearches(mockFetch)).toEqual([ + "select=*&order=id.asc", + "select=*&order=id.asc&offset=2&limit=2", + ]) + }) + + test("a set within the cap is a single request", async () => { + const mockFetch = makePaginatingFetch(ROWS.slice(0, 2), { cap: 2 }) + const rows = await run(mockFetch, { orderBy: ORDER_BY }) + + expect((rows as Array<{ id: number }>).map((r) => r.id)).toEqual([1, 2]) + expect(getSearches(mockFetch)).toEqual(["select=*&order=id.asc"]) + }) + + test("the loop owns limit: a caller limit above the cap still pages", async () => { + const mockFetch = makePaginatingFetch(ROWS, { cap: 2 }) + const rows = await run(mockFetch, { orderBy: ORDER_BY, limit: 3 }) + + // Exactly the caller's 3 rows, fetched across cap-sized pages. + expect((rows as Array<{ id: number }>).map((r) => r.id)).toEqual([1, 2, 3]) + expect(getSearches(mockFetch)).toEqual([ + "select=*&order=id.asc&limit=3", + "select=*&order=id.asc&limit=1&offset=2", + ]) + }) + + test("limit: 0 returns an empty set with zero requests", async () => { + const mockFetch = makePaginatingFetch(ROWS, { cap: 2 }) + const rows = await run(mockFetch, { orderBy: ORDER_BY, limit: 0 }) + + expect(rows).toEqual([]) + expect(mockFetch).not.toHaveBeenCalled() + }) + + test("threads the abort signal to every request", async () => { + const mockFetch = makePaginatingFetch(ROWS, { cap: 2 }) + const signal = new AbortController().signal + await run(mockFetch, { orderBy: ORDER_BY }, signal) + + expect(mockFetch.mock.calls.length).toBeGreaterThan(1) + for (const call of mockFetch.mock.calls) { + expect((call[1] as RequestInit).signal).toBe(signal) + } + }) + + test("one failed page fails the whole load (no partial data)", async () => { + let calls = 0 + const failingFetch = vi.fn().mockImplementation(() => { + calls += 1 + if (calls === 1) { + return Promise.resolve( + new Response(JSON.stringify([{ id: 1 }, { id: 2 }]), { + status: 200, + headers: { + "content-type": "application/json", + "content-range": "0-1/5", + }, + }) + ) + } + return Promise.resolve( + new Response(JSON.stringify({ message: "boom" }), { + status: 500, + headers: { "content-type": "application/json" }, + }) + ) + }) + const supabase = createClient(SUPABASE_URL, SUPABASE_KEY, { + global: { fetch: failingFetch }, + }) + + await expect( + supabaseQueryFn(supabase, "items", { + client: new QueryClient(), + queryKey: ["items"], + signal: new AbortController().signal, + meta: { loadSubsetOptions: { orderBy: ORDER_BY } } as never, + }) + ).rejects.toBeDefined() + }) +}) diff --git a/tests/pagination.utils.ts b/tests/pagination.utils.ts index af96a0f..40c84ca 100644 --- a/tests/pagination.utils.ts +++ b/tests/pagination.utils.ts @@ -15,8 +15,16 @@ const COMPARATORS: Record boolean> = { * the pagination adapter emits: single-column `order`, `limit`, `offset`, and * `col=op.value` scalar filters (`gt`/`gte`/`lt`/`lte`/`eq`). Non-GET requests * echo their body back so inserts/updates parse. + * + * `cap` simulates PostgREST's `db-max-rows`: no response ever returns more than + * `cap` rows, even when a larger `limit` is requested, while the Content-Range + * count still reports the true total — exactly the condition the paging loop + * exists to handle. */ -export function makePaginatingFetch(fixture: Row[]) { +export function makePaginatingFetch( + fixture: Row[], + { cap }: { cap?: number } = {} +) { const columns = Object.keys(fixture[0] ?? {}) return vi.fn().mockImplementation((input, init) => { const method = init?.method ?? "GET" @@ -46,21 +54,33 @@ export function makePaginatingFetch(fixture: Row[]) { rows.sort((a, b) => (Number(a[column]) - Number(b[column])) * sign) } + // The total matching set before paging — what PostgREST reports in the + // Content-Range header when `count=exact` is requested. + const total = rows.length const offset = params.get("offset") ? Number(params.get("offset")) : 0 const limit = params.get("limit") - rows = rows.slice( - offset, - limit === null ? undefined : offset + Number(limit) - ) - return Promise.resolve(json(rows)) + const requested = limit === null ? undefined : offset + Number(limit) + const upperBound = + cap === undefined + ? requested + : Math.min(requested ?? offset + cap, offset + cap) + rows = rows.slice(offset, upperBound) + const end = offset + rows.length - 1 + const contentRange = rows.length + ? `${offset}-${end}/${total}` + : `*/${total}` + return Promise.resolve(json(rows, contentRange)) }) } -function json(body: unknown): Response { - return new Response(JSON.stringify(body), { - status: 200, - headers: { "content-type": "application/json" }, - }) +function json(body: unknown, contentRange?: string): Response { + const headers: Record = { + "content-type": "application/json", + } + if (contentRange !== undefined) { + headers["content-range"] = contentRange + } + return new Response(JSON.stringify(body), { status: 200, headers }) } /** diff --git a/tests/query-once.test.ts b/tests/query-once.test.ts index e03e3d3..cd7ddbe 100644 --- a/tests/query-once.test.ts +++ b/tests/query-once.test.ts @@ -27,6 +27,7 @@ import { import { afterEach, beforeEach, describe, expect, test } from "vitest" import { queryOnce } from "../src/index" import { VERSION } from "../src/version" +import { makePaginatingFetch } from "./pagination.utils" import { createMockedTodosCollection, createMockedUsersCollection, @@ -1097,3 +1098,55 @@ describe("queryOnce PostgREST query generation", () => { }) }) }) + +describe("queryOnce pagination", () => { + // Five users behind a mock whose server cap is 2 rows per request. + const USERS = [ + { id: 1, name: "A", email: "a@t", active: true }, + { id: 2, name: "B", email: "b@t", active: true }, + { id: 3, name: "C", email: "c@t", active: true }, + { id: 4, name: "D", email: "d@t", active: true }, + { id: 5, name: "E", email: "e@t", active: true }, + ] + + test("a non-aggregate queryOnce returns the full multi-page set", async () => { + // Non-aggregate queryOnce loads its source collection through the same + // supabaseQueryFn as live queries, so it inherits the db-max-rows paging + // loop for free — the full set comes back despite the cap. + const mockFetch = makePaginatingFetch(USERS, { cap: 2 }) + const users = createMockedUsersCollection(mockFetch) + const supabase = createClient(SUPABASE_URL, SUPABASE_KEY, { + global: { fetch: mockFetch }, + }) + + const rows = await queryOnce((q) => q.from({ user: users }), supabase) + + expect( + (rows as Array<{ id: number }>).map((r) => r.id).sort((a, b) => a - b) + ).toEqual([1, 2, 3, 4, 5]) + + await users.cleanup() + }) + + test("an aggregate queryOnce stays a single executeQuery request", async () => { + // The aggregate / groupBy / having path bypasses supabaseQueryFn, so it does + // NOT paginate: exactly one request is issued. + const mockFetch = makePaginatingFetch(USERS, { cap: 2 }) + const users = createMockedUsersCollection(mockFetch) + const supabase = createClient(SUPABASE_URL, SUPABASE_KEY, { + global: { fetch: mockFetch }, + }) + + await queryOnce( + (q) => q.from({ user: users }).groupBy(({ user }) => user.active), + supabase + ) + + const gets = mockFetch.mock.calls.filter( + (call) => (call[1]?.method ?? "GET") === "GET" + ) + expect(gets).toHaveLength(1) + + await users.cleanup() + }) +}) diff --git a/tests/test.utils.ts b/tests/test.utils.ts index 9f5c295..11f1c8c 100644 --- a/tests/test.utils.ts +++ b/tests/test.utils.ts @@ -65,11 +65,28 @@ export function createMockFetch() { ) } - const response = mockResponses[table] ?? [] + const all = mockResponses[table] ?? [] + // Honour offset/limit so the pagination loop's follow-up pages terminate + // (a page past the data returns empty) instead of re-serving the same rows. + const params = url.searchParams + const offset = params.get("offset") ? Number(params.get("offset")) : 0 + const limit = params.get("limit") + const rows = all.slice( + offset, + limit === null ? undefined : offset + Number(limit) + ) + const end = offset + rows.length - 1 return Promise.resolve( - new Response(JSON.stringify(response), { + new Response(JSON.stringify(rows), { status: 200, - headers: { "content-type": "application/json" }, + headers: { + "content-type": "application/json", + // The loop reads the total off Content-Range to size its paging; the + // format mirrors PostgREST's `-/`. + "content-range": rows.length + ? `${offset}-${end}/${all.length}` + : `*/${all.length}`, + }, }) ) }) From 5f36d2fbd7ab68537264cd0bb39d47ee8cb05c04 Mon Sep 17 00:00:00 2001 From: Ivan Vasilov Date: Mon, 21 Sep 2026 00:37:26 +0200 Subject: [PATCH 2/2] fix: count a caller offset against the total when paging; document limitations `Content-Range` reports the whole matching set, so a read that starts at a caller `offset` has only `total - offset` rows left to fetch. Using the raw total as the target made an exact-multiple read issue one trailing empty request. Compute the remaining rows past the starting offset instead, and add a unit test for offset 2 over six rows at a cap of 2 (two requests, not three). Document the paging loop's known limitations in the README: offset paging needs a total order, it is not stable under concurrent writes, and each read now costs one `count=exact`. Co-Authored-By: Claude Fable 5.1 --- README.md | 6 ++++++ src/functions.ts | 9 +++++---- tests/pagination.test.ts | 16 ++++++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3020983..0392ee3 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,12 @@ PostgREST caps every response at its `db-max-rows` setting (1000 by default on S The **aggregate / `GROUP BY` / `HAVING`** path of `queryOnce` is the one exception — it issues a single request and is **not** paginated. Aggregate results are a few rows, so the cap is moot, but a `GROUP BY` / `HAVING` query whose grouped output exceeds `db-max-rows` is silently truncated at the cap. Add an explicit `limit`, or narrow the query, if you expect more grouped rows than the cap. +Known limitations of the paging loop: + +- **Ordering.** Pages are stitched together with `offset`, and PostgreSQL does not guarantee a consistent row order across separate `LIMIT`/`OFFSET` queries unless the sort is total. A read with no `orderBy`, or one ordered on a non-unique column, can skip or repeat rows across page boundaries. Order by a unique column (or include the primary key as a final sort key) when a collection is expected to exceed the cap. +- **Concurrent writes.** Offset paging is not keyset-stable: a row inserted or deleted by another client while a multi-page read is in flight shifts the remaining rows by one offset, so a row can be missed or duplicated until the next refetch. +- **Count cost.** Every collection read issues one `Prefer: count=exact` on its first request so the loop knows when to stop. On very large tables, or under heavy RLS, that is a full `count(*)` per read. + **Evaluated Client-Side** These operations fetch the required rows and process them in memory: diff --git a/src/functions.ts b/src/functions.ts index d2be1f4..2ebec94 100644 --- a/src/functions.ts +++ b/src/functions.ts @@ -93,11 +93,12 @@ async function fetchAllPages( // The rows the first page returned size the server's cap; a later page that // comes back shorter than this means the server has no more rows. const pageSize = rows.length - // How many rows we ultimately want: the caller's limit, bounded by the total - // the server reported. When the count header is missing we fall back to - // paging until a short page appears. + // `Content-Range` reports the whole matching set, so the rows reachable from + // the starting offset are what remains past it. When the count header is + // missing we fall back to paging until a short page appears. const total = first.count ?? Number.POSITIVE_INFINITY - const target = limit === undefined ? total : Math.min(limit, total) + const remaining = Math.max(0, total - startOffset) + const target = limit === undefined ? remaining : Math.min(limit, remaining) let lastPageSize = pageSize while (lastPageSize === pageSize && pageSize > 0 && rows.length < target) { diff --git a/tests/pagination.test.ts b/tests/pagination.test.ts index 5a60728..54a7c0d 100644 --- a/tests/pagination.test.ts +++ b/tests/pagination.test.ts @@ -269,6 +269,22 @@ describe("db-max-rows paging loop (supabaseQueryFn)", () => { ]) }) + test("a caller offset counts against the total: no trailing empty request", async () => { + // Six rows, offset 2, cap 2: rows 3-4 then 5-6 — the count (6) minus the + // offset (2) tells the loop it is done after two requests, not three. + const SIX = [...ROWS, { id: 6 }] + const mockFetch = makePaginatingFetch(SIX, { cap: 2 }) + const rows = await run(mockFetch, { orderBy: ORDER_BY, offset: 2 }) + + expect((rows as Array<{ id: number }>).map((r) => r.id)).toEqual([ + 3, 4, 5, 6, + ]) + expect(getSearches(mockFetch)).toEqual([ + "select=*&order=id.asc&offset=2", + "select=*&order=id.asc&offset=4&limit=2", + ]) + }) + test("a set within the cap is a single request", async () => { const mockFetch = makePaginatingFetch(ROWS.slice(0, 2), { cap: 2 }) const rows = await run(mockFetch, { orderBy: ORDER_BY })