diff --git a/docs/content/docs/guides/sync-and-async.mdx b/docs/content/docs/guides/sync-and-async.mdx index 46e96b6e..c58a627d 100644 --- a/docs/content/docs/guides/sync-and-async.mdx +++ b/docs/content/docs/guides/sync-and-async.mdx @@ -49,6 +49,8 @@ The [`NitroSQLite` export](/api/react-native-nitro-sqlite/variables/NitroSQLite) `NitroSQLite.native` exposes the underlying [`NitroSQLiteNative` hybrid object](/api/react-native-nitro-sqlite/hybrid-objects/NitroSQLiteNative). Its methods take a database name and return raw results without the connection helper's `rows` container. They also throw native errors without converting them to `NitroSQLiteError`. Its `executeAsync()` accepts an optional `onRows` callback that receives leading rows in batches, in order; the resolved `results` then holds only the rows after the last batch. +The native promise can resolve before those callbacks run. Its `rowBatchCount` reports how many batches to expect. Wait for all of them before appending `results` to the collected rows. Without `onRows`, the count is zero and `results` contains every row. The managed `executeAsync()` API handles this waiting and assembly for you. + ```ts import { NitroSQLite } from 'react-native-nitro-sqlite' diff --git a/example/tests/unit/specs/operations/execute.spec.ts b/example/tests/unit/specs/operations/execute.spec.ts index 74ff233f..d5dcedf0 100644 --- a/example/tests/unit/specs/operations/execute.spec.ts +++ b/example/tests/unit/specs/operations/execute.spec.ts @@ -168,6 +168,7 @@ export default function registerExecuteUnitTests() { ) expectQueryResultRows(result, size) + expect(result.rowBatchCount).toBe(0) } } finally { db.close() @@ -186,6 +187,7 @@ export default function registerExecuteUnitTests() { ) expectQueryResultRows(result, size) + expect(result.rowBatchCount).toBe(Math.floor(size / 256)) expect(result.results).toBe(result.rows._array) expect( result.rows._array.every((row, index) => row.id === index + 1), @@ -198,6 +200,7 @@ export default function registerExecuteUnitTests() { [1_000], ) expectQueryResultRows(result, 1_000) + expect(result.rowBatchCount).toBe(3) expect( result.rows._array.every((row, index) => row.id === index + 1), ).toBe(true) diff --git a/packages/react-native-nitro-sqlite/cpp/NitroSQLiteOperations.cpp b/packages/react-native-nitro-sqlite/cpp/NitroSQLiteOperations.cpp index 2c8c49fd..6a547e53 100644 --- a/packages/react-native-nitro-sqlite/cpp/NitroSQLiteOperations.cpp +++ b/packages/react-native-nitro-sqlite/cpp/NitroSQLiteOperations.cpp @@ -221,6 +221,7 @@ namespace { int columnCount = 0; std::vector columnNames; std::vector rows; + size_t rowBatchCount = 0; bool columnsCaptured = false; const auto captureColumns = [&] { @@ -283,6 +284,7 @@ namespace { rows.push_back(std::move(row)); if (onRows && rows.size() == kRowBatchSize) { onRows(SQLiteQueryResults(columnNames, std::move(rows))); + rowBatchCount++; rows = {}; } }); @@ -309,6 +311,7 @@ namespace { result.insertId = static_cast(sqlite3_last_insert_rowid(db)); result.results = SQLiteQueryResults(std::move(columnNames), std::move(rows)); result.metadata = std::move(metadata); + result.rowBatchCount = static_cast(rowBatchCount); return result; } diff --git a/packages/react-native-nitro-sqlite/cpp/NitroSQLiteOperations.hpp b/packages/react-native-nitro-sqlite/cpp/NitroSQLiteOperations.hpp index 1bbd578a..20b150c2 100644 --- a/packages/react-native-nitro-sqlite/cpp/NitroSQLiteOperations.hpp +++ b/packages/react-native-nitro-sqlite/cpp/NitroSQLiteOperations.hpp @@ -54,7 +54,8 @@ void sqliteAttachDb(const SQLiteConnectionPtr& connection, const std::string& do void sqliteDetachDb(const SQLiteConnectionPtr& connection, const std::string& alias); /** Execute one statement. With @p onRows, full batches of rows go to it as they are read and the - * returned result holds only the rows after the last batch. + * returned result holds only the rows after the last batch. Its rowBatchCount reports how many + * batches were sent; JavaScript delivery may still be pending when this function returns. */ NitroSQLiteQueryResult sqliteExecute(const SQLiteConnectionPtr& connection, const std::string& query, const std::optional& params, const SQLiteRowBatchHandler& onRows = nullptr); diff --git a/packages/react-native-nitro-sqlite/nitrogen/generated/shared/c++/NitroSQLiteQueryResult.hpp b/packages/react-native-nitro-sqlite/nitrogen/generated/shared/c++/NitroSQLiteQueryResult.hpp index 693a4c1a..ae50d9cc 100644 --- a/packages/react-native-nitro-sqlite/nitrogen/generated/shared/c++/NitroSQLiteQueryResult.hpp +++ b/packages/react-native-nitro-sqlite/nitrogen/generated/shared/c++/NitroSQLiteQueryResult.hpp @@ -48,10 +48,11 @@ namespace margelo::nitro::rnnitrosqlite { std::optional insertId SWIFT_PRIVATE; margelo::nitro::rnnitrosqlite::SQLiteQueryResults results SWIFT_PRIVATE; std::optional> metadata SWIFT_PRIVATE; + std::optional rowBatchCount SWIFT_PRIVATE; public: NitroSQLiteQueryResult() = default; - explicit NitroSQLiteQueryResult(double rowsAffected, std::optional insertId, margelo::nitro::rnnitrosqlite::SQLiteQueryResults results, std::optional> metadata): rowsAffected(rowsAffected), insertId(insertId), results(results), metadata(metadata) {} + explicit NitroSQLiteQueryResult(double rowsAffected, std::optional insertId, margelo::nitro::rnnitrosqlite::SQLiteQueryResults results, std::optional> metadata, std::optional rowBatchCount): rowsAffected(rowsAffected), insertId(insertId), results(results), metadata(metadata), rowBatchCount(rowBatchCount) {} public: // NitroSQLiteQueryResult is not equatable because these properties are not equatable: results @@ -70,7 +71,8 @@ namespace margelo::nitro { JSIConverter::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "rowsAffected"))), JSIConverter>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "insertId"))), JSIConverter::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "results"))), - JSIConverter>>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "metadata"))) + JSIConverter>>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "metadata"))), + JSIConverter>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "rowBatchCount"))) ); } static inline jsi::Value toJSI(jsi::Runtime& runtime, const margelo::nitro::rnnitrosqlite::NitroSQLiteQueryResult& arg) { @@ -79,6 +81,7 @@ namespace margelo::nitro { obj.setProperty(runtime, PropNameIDCache::get(runtime, "insertId"), JSIConverter>::toJSI(runtime, arg.insertId)); obj.setProperty(runtime, PropNameIDCache::get(runtime, "results"), JSIConverter::toJSI(runtime, arg.results)); obj.setProperty(runtime, PropNameIDCache::get(runtime, "metadata"), JSIConverter>>::toJSI(runtime, arg.metadata)); + obj.setProperty(runtime, PropNameIDCache::get(runtime, "rowBatchCount"), JSIConverter>::toJSI(runtime, arg.rowBatchCount)); return obj; } static inline bool canConvert(jsi::Runtime& runtime, const jsi::Value& value) { @@ -93,6 +96,7 @@ namespace margelo::nitro { if (!JSIConverter>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "insertId")))) return false; if (!JSIConverter::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "results")))) return false; if (!JSIConverter>>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "metadata")))) return false; + if (!JSIConverter>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "rowBatchCount")))) return false; return true; } }; diff --git a/packages/react-native-nitro-sqlite/src/__tests__/execute.test.ts b/packages/react-native-nitro-sqlite/src/__tests__/execute.test.ts index e56dd39b..2347118b 100644 --- a/packages/react-native-nitro-sqlite/src/__tests__/execute.test.ts +++ b/packages/react-native-nitro-sqlite/src/__tests__/execute.test.ts @@ -10,7 +10,7 @@ import { executeAsyncManaged, executeManaged, } from '../operations/execute' -import { nativeResult } from './testUtils' +import { deferred, nativeResult } from './testUtils' const dbName = 'execute-test' const query = 'SELECT ? AS value' @@ -127,6 +127,65 @@ describe('execute', () => { expect(result.rows.item(3)).toEqual({ value: 4 }) }) + it.each([ + { size: 256, earlyBatches: 0, pendingNative: false }, + { size: 257, earlyBatches: 0, pendingNative: true }, + { size: 512, earlyBatches: 0, pendingNative: false }, + { size: 1_000, earlyBatches: 0, pendingNative: false }, + { size: 1_000, earlyBatches: 1, pendingNative: false }, + { size: 1_000, earlyBatches: 3, pendingNative: false }, + ])( + 'waits for all batches with $size rows, $earlyBatches early batches, and pendingNative=$pendingNative', + async ({ size, earlyBatches, pendingNative }) => { + const rows = Array.from({ length: size }, (_, value) => ({ value })) + const rowBatchCount = Math.floor(size / 256) + const native = { + ...nativeResult(rows.slice(rowBatchCount * 256)), + rowBatchCount, + } + const completion = deferred() + const deliveries: (() => void)[] = [] + jest + .mocked(HybridNitroSQLite.executeAsync) + .mockImplementationOnce((_dbName, _query, _params, onRows) => { + for (let batch = 0; batch < rowBatchCount; batch++) { + deliveries.push(() => + onRows?.(rows.slice(batch * 256, (batch + 1) * 256)), + ) + } + deliveries.splice(0, earlyBatches).forEach((deliver) => deliver()) + return pendingNative ? completion.promise : Promise.resolve(native) + }) + + const completed = jest.fn() + const pending = executeAsync<{ value: number }>( + dbName, + query, + params, + ).then((result) => { + completed() + return result + }) + completion.resolve(native) + await new Promise((resolve) => setImmediate(resolve)) + expect(completed).toHaveBeenCalledTimes(deliveries.length === 0 ? 1 : 0) + + while (deliveries.length > 0) { + deliveries.shift()?.() + if (deliveries.length > 0) { + await new Promise((resolve) => setImmediate(resolve)) + expect(completed).not.toHaveBeenCalled() + } + } + + const result = await pending + expect(result.results).toEqual(rows) + expect(result.rows._array).toBe(result.results) + expect(result.rows.length).toBe(size) + expect(result.rows.item(size - 1)).toEqual(rows[size - 1]) + }, + ) + it('keeps the native rows when no batch was delivered', async () => { const native = nativeResult([{ value: 7 }]) jest.mocked(HybridNitroSQLite.executeAsync).mockResolvedValueOnce(native) @@ -136,6 +195,26 @@ describe('execute', () => { expect(result.rows._array).toBe(native.results) }) + it('keeps a managed connection busy until row delivery completes', async () => { + openDatabaseQueue(dbName) + const delivery = deferred<() => void>() + jest + .mocked(HybridNitroSQLite.executeAsync) + .mockImplementationOnce(async (_dbName, _query, _params, onRows) => { + delivery.resolve(() => onRows?.([{ value: 1 }])) + return { ...nativeResult([{ value: 2 }]), rowBatchCount: 1 } + }) + + const pending = executeAsync(dbName, query) + const deliver = await delivery.promise + await new Promise((resolve) => setImmediate(resolve)) + expect(() => execute(dbName, query)).toThrow('busy') + + deliver() + await expect(pending).resolves.toMatchObject({ rows: { length: 2 } }) + expect(() => closeDatabaseQueue(dbName)).not.toThrow() + }) + it('converts native asynchronous errors and releases a managed queue', async () => { openDatabaseQueue(dbName) jest diff --git a/packages/react-native-nitro-sqlite/src/operations/execute.ts b/packages/react-native-nitro-sqlite/src/operations/execute.ts index 47e0f86c..5ed5511c 100644 --- a/packages/react-native-nitro-sqlite/src/operations/execute.ts +++ b/packages/react-native-nitro-sqlite/src/operations/execute.ts @@ -57,7 +57,7 @@ export function executeNative( * @param dbName Name of an open native database. * @param query SQL statement with optional positional placeholders. * @param params Values bound to the placeholders. - * @returns A promise of the query result with typed rows. + * @returns A promise of the complete query result after all row batches arrive. */ export async function executeAsync( dbName: string, @@ -90,6 +90,8 @@ export async function executeAsyncNative( try { // Large results arrive in batches, so rows are converted while SQLite reads the rest. let leadingRows: Row[] | undefined + let receivedBatchCount = 0 + let onBatchReceived: (() => void) | undefined const nativeResult = await HybridNitroSQLite.executeAsync( dbName, query, @@ -100,8 +102,19 @@ export async function executeAsyncNative( } else { Array.prototype.push.apply(leadingRows, rows as Row[]) } + receivedBatchCount++ + onBatchReceived?.() }, ) + // Native computation can finish before its queued JavaScript callbacks run. + const expectedBatchCount = nativeResult.rowBatchCount ?? 0 + if (receivedBatchCount < expectedBatchCount) { + await new Promise((resolve) => { + onBatchReceived = () => { + if (receivedBatchCount >= expectedBatchCount) resolve() + } + }) + } return buildJSQueryResult(nativeResult, leadingRows) } catch (error) { throw NitroSQLiteError.fromError(error) diff --git a/packages/react-native-nitro-sqlite/src/specs/NitroSQLite.nitro.ts b/packages/react-native-nitro-sqlite/src/specs/NitroSQLite.nitro.ts index 38f4986d..dcd40766 100644 --- a/packages/react-native-nitro-sqlite/src/specs/NitroSQLite.nitro.ts +++ b/packages/react-native-nitro-sqlite/src/specs/NitroSQLite.nitro.ts @@ -100,7 +100,8 @@ export interface NitroSQLite * @param onRows Receives leading rows in batches, in order, while SQLite still reads the rest, so * JavaScript can convert a batch while the next one is read. Batched rows are not repeated in the * result's `results`, which then holds only the rows after the last batch. - * @returns A promise of the native query result. + * @returns A promise of the native query result. Row callbacks may still be queued when it + * resolves. Wait for `rowBatchCount` callbacks before combining their rows with `results`. */ executeAsync( dbName: string, diff --git a/packages/react-native-nitro-sqlite/src/specs/NitroSQLiteQueryResult.nitro.ts b/packages/react-native-nitro-sqlite/src/specs/NitroSQLiteQueryResult.nitro.ts index efab64c1..455d3727 100644 --- a/packages/react-native-nitro-sqlite/src/specs/NitroSQLiteQueryResult.nitro.ts +++ b/packages/react-native-nitro-sqlite/src/specs/NitroSQLiteQueryResult.nitro.ts @@ -22,6 +22,13 @@ export interface NitroSQLiteQueryResult { /** Column metadata keyed by result column name, when available. */ readonly metadata?: Record + + /** Number of row batches sent through the native `onRows` callback, or zero when none were sent. + * Callbacks may arrive after the native promise resolves. Wait for this many batches before + * combining them with `results`. The managed async API waits for them automatically. + * An omitted count is treated as zero. Batch tracking requires an updated native build. + */ + readonly rowBatchCount?: number } // TODO: Investigate why this cannot be represented in Nitro