feat: add automatic collection pagination

README.md

@@ -188,6 +188,18 @@ Most query operations are translated to PostgREST filters and run server-side. A
 
 The client-side filter fallback above applies to live queries and ordinary `queryOnce` queries. The server aggregate path in `queryOnce` requires fully pushable `WHERE` expressions and throws for unsupported filters, rather than returning an aggregate over unfiltered rows.
 
+### Row cap and pagination
+
+PostgREST caps every response at its `db-max-rows` setting (1000 by default on Supabase). Collection reads and non-aggregate `queryOnce` queries page past this cap automatically: the adapter loops with `offset`, using a one-time `count=exact` on the first request to fetch the complete matching set (or the caller's `limit`).
+
+The **aggregate / `GROUP BY` / `HAVING`** path of `queryOnce` is the one exception — it issues a single request and is **not** paginated. Aggregate results are a few rows, so the cap is moot, but a `GROUP BY` / `HAVING` query whose grouped output exceeds `db-max-rows` is silently truncated at the cap. Add an explicit `limit`, or narrow the query, if you expect more grouped rows than the cap.
+
+Known limitations of the paging loop:
+
+- **Ordering.** Pages are stitched together with `offset`, and PostgreSQL does not guarantee a consistent row order across separate `LIMIT`/`OFFSET` queries unless the sort is total. A read with no `orderBy`, or one ordered on a non-unique column, can skip or repeat rows across page boundaries. Order by a unique column (or include the primary key as a final sort key) when a collection is expected to exceed the cap.
+- **Concurrent writes.** Offset paging is not keyset-stable: a row inserted or deleted by another client while a multi-page read is in flight shifts the remaining rows by one offset, so a row can be missed or duplicated until the next refetch.
+- **Count cost.** Every collection read issues one `Prefer: count=exact` on its first request so the loop knows when to stop. On very large tables, or under heavy RLS, that is a full `count(*)` per read.
+
 **Evaluated Client-Side**
 
 These operations fetch the required rows and process them in memory:

src/functions.ts

@@ -52,6 +52,72 @@ export const subsetOptionsToQueryKey = (
   return key ? [tableName, key] : [tableName]
 }
 
+type PageOptions = {
+  /** The caller's row limit; `undefined` means "the full matching set". */
+  limit?: number
+  /** The starting offset (always 0 for cursor reads). */
+  offset: number
+  signal?: AbortSignal
+}
+
+/**
+ * Read a full matching set (or the caller's `limit` rows) with offset paging,
+ * looping past PostgREST's `db-max-rows` cap instead of silently truncating at
+ * it. The first request asks for `count=exact` so the loop knows the total up
+ * front and stops without a trailing empty probe; each later page advances the
+ * offset by the rows actually received (self-correcting under concurrent
+ * writes) and is capped by the observed page size and the rows still owed.
+ */
+async function fetchAllPages(
+  supabase: SupabaseClient,
+  tableName: string,
+  baseSearch: URLSearchParams,
+  { limit, offset: startOffset, signal }: PageOptions
+): Promise<any[]> {
+  // A zero limit is an empty window: return it without touching the network.
+  if (limit === 0) {
+    return []
+  }
+
+  // The base search already carries the caller's limit/offset (from the search
+  // builders), so the first request is sent verbatim — its URL stays identical
+  // to the single-request behaviour, and the query key that mirrors it.
+  const first = await postgrestRequest(supabase, tableName, {
+    method: "GET",
+    search: baseSearch,
+    signal,
+    count: "exact",
+  })
+  const rows: any[] = first.data ? [...first.data] : []
+
+  // The rows the first page returned size the server's cap; a later page that
+  // comes back shorter than this means the server has no more rows.
+  const pageSize = rows.length
+  // `Content-Range` reports the whole matching set, so the rows reachable from
+  // the starting offset are what remains past it. When the count header is
+  // missing we fall back to paging until a short page appears.
+  const total = first.count ?? Number.POSITIVE_INFINITY
+  const remaining = Math.max(0, total - startOffset)
+  const target = limit === undefined ? remaining : Math.min(limit, remaining)
+
+  let lastPageSize = pageSize
+  while (lastPageSize === pageSize && pageSize > 0 && rows.length < target) {
+    const pageSearch = new URLSearchParams(baseSearch)
+    pageSearch.set("offset", `${startOffset + rows.length}`)
+    pageSearch.set("limit", `${Math.min(pageSize, target - rows.length)}`)
+    const page = await postgrestRequest(supabase, tableName, {
+      method: "GET",
+      search: pageSearch,
+      signal,
+    })
+    const pageRows: any[] = page.data ?? []
+    rows.push(...pageRows)
+    lastPageSize = pageRows.length
+  }
+
+  return rows
+}
+
 export const supabaseQueryFn = async (
   supabase: SupabaseClient,
   tableName: string,
@@ -66,6 +132,9 @@ export const supabaseQueryFn = async (
 ) => {
   const options = ctx.meta?.loadSubsetOptions ?? {}
   const search = loadSubsetOptionsToSearch(options)
+  // Cursor reads pin their own window start, so they never carry an offset (see
+  // loadSubsetOptionsToSearch); only a cursor-less read advances from `offset`.
+  const startOffset = options.offset && !options.cursor ? options.offset : 0
 
   // A keyset request whose boundary column has ties (e.g. `orderBy(created_at)`
   // with repeated values) needs a second, unlimited request for the rows equal
@@ -80,23 +149,30 @@ export const supabaseQueryFn = async (
   // Honouring `whereCurrent` here is what makes the cursor correct on its own.
   const tiesSearch = cursorCurrentToSearch(options)
   if (!tiesSearch) {
-    const data = await postgrestRequest(supabase, tableName, {
-      method: "GET",
-      search,
+    return await fetchAllPages(supabase, tableName, search, {
+      limit: options.limit,
+      offset: startOffset,
+      signal: ctx.signal,
     })
-    return data || []
   }
 
   const [ties, rows] = await Promise.all([
-    postgrestRequest(supabase, tableName, {
-      method: "GET",
-      search: tiesSearch,
+    // The tie read is deliberately unbounded by the caller's limit — every row
+    // tied at the boundary must come back — but it is still paged so a tie class
+    // larger than the cap is not truncated.
+    fetchAllPages(supabase, tableName, tiesSearch, {
+      offset: 0,
+      signal: ctx.signal,
+    }),
+    fetchAllPages(supabase, tableName, search, {
+      limit: options.limit,
+      offset: startOffset,
+      signal: ctx.signal,
     }),
-    postgrestRequest(supabase, tableName, { method: "GET", search }),
   ])
   // `whereCurrent` (== boundary) and `whereFrom` (> / < boundary) are disjoint,
   // so concatenation never duplicates; the collection re-sorts locally.
-  return [...(ties || []), ...(rows || [])]
+  return [...ties, ...rows]
 }
 
 export const supabaseOnInsert = async (
@@ -106,7 +182,7 @@ export const supabaseOnInsert = async (
 ) => {
   await Promise.all(
     transaction.mutations.map(async (mutation) => {
-      const data = await postgrestRequest(supabase, tableName, {
+      const { data } = await postgrestRequest(supabase, tableName, {
         method: "POST",
         search: new URLSearchParams({ select: "*" }),
         body: { ...mutation.modified },
@@ -136,7 +212,7 @@ export const supabaseOnUpdate = async (
       const { original, changes } = mutation
       const search = keyColumnsToSearch(keys, original)
       search.set("select", "*")
-      const data = await postgrestRequest(supabase, tableName, {
+      const { data } = await postgrestRequest(supabase, tableName, {
         method: "PATCH",
         search,
         body: { ...original, ...changes },

src/postgrest-request.ts

@@ -8,15 +8,30 @@ const SINGLE_ROW_ACCEPT = "application/vnd.pgrst.object+json"
 interface PostgrestRequestOptions {
   /** JSON body for insert/update. */
   body?: unknown
+  /**
+   * Ask PostgREST for the total matching-row count (`Prefer: count=…`). The
+   * count is read back off the `Content-Range` header; the paging loop uses it
+   * to fetch a set larger than the server's `db-max-rows` cap without probing.
+   */
+  count?: "exact" | "planned" | "estimated"
   method: "GET" | "POST" | "PATCH" | "DELETE"
   /** Ask PostgREST to return the affected rows (`Prefer: return=representation`). */
   returnRows?: boolean
   /** The full query string, built by the params helpers. */
   search: URLSearchParams
+  /** Abort signal, threaded from the query's `ctx.signal`. */
+  signal?: AbortSignal
   /** Expect exactly one row (`Accept: application/vnd.pgrst.object+json`). */
   single?: boolean
 }
 
+/** A PostgREST read result: the rows plus the total count when one was asked for. */
+export interface PostgrestResult {
+  /** Total matching rows when `count` was requested, else `null`. */
+  count: number | null
+  data: any
+}
+
 /**
  * Issue a PostgREST request with a hand-built query string while leaving
  * supabase-js in charge of auth, schema, tracing, timeout, and transport.
@@ -33,8 +48,16 @@ interface PostgrestRequestOptions {
 export async function postgrestRequest(
   supabase: SupabaseClient,
   table: string,
-  { method, search, body, returnRows, single }: PostgrestRequestOptions
-): Promise<any> {
+  {
+    method,
+    search,
+    body,
+    returnRows,
+    single,
+    count,
+    signal,
+  }: PostgrestRequestOptions
+): Promise<PostgrestResult> {
   const queryBuilder = supabase.from(table)
 
   const url = new URL(queryBuilder.url.toString())
@@ -45,6 +68,9 @@ export async function postgrestRequest(
   if (returnRows) {
     headers.append("Prefer", "return=representation")
   }
+  if (count) {
+    headers.append("Prefer", `count=${count}`)
+  }
   if (single) {
     headers.set("Accept", SINGLE_ROW_ACCEPT)
   }
@@ -56,11 +82,14 @@ export async function postgrestRequest(
     schema: queryBuilder.schema,
     body,
     fetch: queryBuilder.fetch,
+    // The builder threads its own `signal` straight into the underlying fetch,
+    // so aborting the query cancels the in-flight request.
+    signal,
   })
 
-  const { data, error } = await builder
+  const { data, error, count: total } = await builder
   if (error) {
     throw error
   }
-  return data
+  return { data, count: total ?? null }
 }

src/query-once.ts

@@ -30,7 +30,10 @@ export async function executeQuery(
   ir: SerializedQueryIR
 ): Promise<unknown[]> {
   const { tableName, search } = queryIrToSearch(ir)
-  const data = await postgrestRequest(supabase, tableName, {
+  // The aggregate / groupBy / having path stays a single request: aggregate
+  // output is a handful of rows, so the db-max-rows cap does not apply and no
+  // paging loop is needed here.
+  const { data } = await postgrestRequest(supabase, tableName, {
     method: "GET",
     search,
   })

supabase/config.toml

@@ -14,8 +14,10 @@ schemas = ["public", "graphql_public"]
 # Extra schemas to add to the search_path of every request.
 extra_search_path = ["public", "extensions"]
 # The maximum number of rows returns from a view, table, or stored procedure. Limits payload size
-# for accidental or malicious requests.
-max_rows = 1000
+# for accidental or malicious requests. Deliberately tiny for the e2e suite: it forces the read
+# pagination loop (src/functions.ts `fetchAllPages`) to page through sets larger than the cap, so
+# the multi-page path is exercised without seeding thousands of rows.
+max_rows = 2
 # Controls whether new tables, views, sequences and functions created in the `public` schema by
 # `postgres` are reachable through the Data API roles (`anon`, `authenticated`, `service_role`)
 # without explicit GRANTs. When unset, new entities are NOT auto-exposed, matching the new cloud

tests/e2e/reads.test.ts

@@ -40,3 +40,39 @@ test("queryOnce runs a one-shot filtered query", async ({ users }) => {
   expect(rows).toHaveLength(1)
   expect(rows[0]?.name).toBe("Alice")
 })
+
+test("reads a set larger than the server row cap across pages", async ({
+  users,
+  other,
+}) => {
+  // config.toml caps responses at 2 rows; reset_e2e seeds Alice + Bob, so three
+  // more rows push the matching set past the cap and force the paging loop.
+  const { error } = await other.from("users").insert([
+    { name: "Carol", email: "carol@test.com", active: true },
+    { name: "Dave", email: "dave@test.com", active: true },
+    { name: "Erin", email: "erin@test.com", active: true },
+  ] as unknown as never)
+  expect(error).toBeNull()
+
+  const live = createLiveQueryCollection((q) =>
+    q
+      .from({ row: users.collection })
+      .select(({ row }) => ({ id: row.id, name: row.name }))
+  )
+
+  try {
+    await live.preload()
+    await vi.waitFor(() => expect(live.size).toBe(5), WAIT)
+    // The full set comes back despite the cap, with each row exactly once.
+    expect(live.toArray.map((row) => row.name).sort()).toEqual([
+      "Alice",
+      "Bob",
+      "Carol",
+      "Dave",
+      "Erin",
+    ])
+    expect(new Set(live.toArray.map((row) => row.id)).size).toBe(5)
+  } finally {
+    await live.cleanup()
+  }
+})