fix(blob): serialize commands as binary

Daniele Briggi · Daniele Briggi · commit 2f873ff6baef · 2025-05-07T15:17:35.000Z
Blobs have to be serialized as binary and thus
all the parameters have to be converted to binary
as well.
diff --git a/.gitignore b/.gitignore
@@ -118,7 +118,7 @@ dist
 # Compiled code
 coverage/
 docs/
-lib/
+#lib/
 
 # Mac files
 .DS_Store
diff --git a/lib/drivers/connection-tls.js b/lib/drivers/connection-tls.js
@@ -156,12 +156,12 @@ class SQLiteCloudTlsConnection extends connection_1.SQLiteCloudConnection {
                 (_a = this.socket) === null || _a === void 0 ? void 0 : _a.destroy();
                 this.socket = undefined;
             }, timeoutMs);
-            (_d = this.socket) === null || _d === void 0 ? void 0 : _d.write(formattedCommands, 'utf-8', () => {
+            (_d = this.socket) === null || _d === void 0 ? void 0 : _d.write(formattedCommands, () => {
                 clearTimeout(timeout); // Clear the timeout on successful write
             });
         }
         else {
-            (_e = this.socket) === null || _e === void 0 ? void 0 : _e.write(formattedCommands, 'utf-8');
+            (_e = this.socket) === null || _e === void 0 ? void 0 : _e.write(formattedCommands);
         }
         return this;
     }
diff --git a/lib/drivers/connection.d.ts b/lib/drivers/connection.d.ts
@@ -1,7 +1,7 @@
 /**
  * connection.ts - base abstract class for sqlitecloud server connections
  */
-import { SQLiteCloudConfig, ErrorCallback, ResultsCallback, SQLiteCloudCommand } from './types';
+import { SQLiteCloudConfig, ErrorCallback, ResultsCallback, SQLiteCloudCommand, SQLiteCloudDataTypes } from './types';
 import { OperationsQueue } from './queue';
 /**
  * Base class for SQLiteCloudConnection handles basics and defines methods.
@@ -39,7 +39,7 @@ export declare abstract class SQLiteCloudConnection {
      * @returns An array of rows in case of selections or an object with
      * metadata in case of insert, update, delete.
      */
-    sql(sql: TemplateStringsArray | string | SQLiteCloudCommand, ...values: any[]): Promise<any>;
+    sql(sql: TemplateStringsArray | string | SQLiteCloudCommand, ...values: SQLiteCloudDataTypes[]): Promise<any>;
     /** Disconnect from server, release transport. */
     abstract close(): this;
 }
diff --git a/lib/drivers/database.d.ts b/lib/drivers/database.d.ts
@@ -1,7 +1,7 @@
 import EventEmitter from 'eventemitter3';
 import { PubSub } from './pubsub';
 import { Statement } from './statement';
-import { ErrorCallback as ConnectionCallback, ResultsCallback, RowCallback, RowCountCallback, RowsCallback, SQLiteCloudCommand, SQLiteCloudConfig } from './types';
+import { ErrorCallback as ConnectionCallback, ResultsCallback, RowCallback, RowCountCallback, RowsCallback, SQLiteCloudCommand, SQLiteCloudConfig, SQLiteCloudDataTypes } from './types';
 /**
  * Creating a Database object automatically opens a connection to the SQLite database.
  * When the connection is established the Database object emits an open event and calls
@@ -153,7 +153,7 @@ export declare class Database extends EventEmitter {
      * @returns An array of rows in case of selections or an object with
      * metadata in case of insert, update, delete.
      */
-    sql(sql: TemplateStringsArray | string | SQLiteCloudCommand, ...values: any[]): Promise<any>;
+    sql(sql: TemplateStringsArray | string | SQLiteCloudCommand, ...values: SQLiteCloudDataTypes[]): Promise<any>;
     /**
      * Returns true if the database connection is open.
      */
diff --git a/lib/drivers/protocol.d.ts b/lib/drivers/protocol.d.ts
@@ -50,4 +50,4 @@ export declare function popData(buffer: Buffer): {
     fwdBuffer: Buffer;
 };
 /** Format a command to be sent via SCSP protocol */
-export declare function formatCommand(command: SQLiteCloudCommand): string;
+export declare function formatCommand(command: SQLiteCloudCommand): Buffer;
diff --git a/lib/drivers/protocol.js b/lib/drivers/protocol.js
@@ -314,15 +314,15 @@ function formatCommand(command) {
 }
 function serializeCommand(data, zeroString = false) {
     const n = data.length;
-    let serializedData = `${n} `;
+    let serializedData = buffer_1.Buffer.from(`${n} `);
     for (let i = 0; i < n; i++) {
         // the first string is the sql and it must be zero-terminated
         const zs = i == 0 || zeroString;
-        serializedData += serializeData(data[i], zs);
+        serializedData = buffer_1.Buffer.concat([serializedData, serializeData(data[i], zs)]);
     }
-    const bytesTotal = buffer_1.Buffer.byteLength(serializedData, 'utf-8');
-    const header = `${exports.CMD_ARRAY}${bytesTotal} `;
-    return header + serializedData;
+    const bytesTotal = serializedData.byteLength;
+    const header = buffer_1.Buffer.from(`${exports.CMD_ARRAY}${bytesTotal} `);
+    return buffer_1.Buffer.concat([header, serializedData]);
 }
 function serializeData(data, zeroString = false) {
     if (typeof data === 'string') {
@@ -332,22 +332,22 @@ function serializeData(data, zeroString = false) {
             data += '\0';
         }
         const header = `${cmd}${buffer_1.Buffer.byteLength(data, 'utf-8')} `;
-        return header + data;
+        return buffer_1.Buffer.from(header + data);
     }
     if (typeof data === 'number') {
         if (Number.isInteger(data)) {
-            return `${exports.CMD_INT}${data} `;
+            return buffer_1.Buffer.from(`${exports.CMD_INT}${data} `);
         }
         else {
-            return `${exports.CMD_FLOAT}${data} `;
+            return buffer_1.Buffer.from(`${exports.CMD_FLOAT}${data} `);
         }
     }
     if (buffer_1.Buffer.isBuffer(data)) {
-        const header = `${exports.CMD_BLOB}${data.length} `;
-        return header + data.toString('utf-8');
+        const header = `${exports.CMD_BLOB}${data.byteLength} `;
+        return buffer_1.Buffer.concat([buffer_1.Buffer.from(header), data]);
     }
     if (data === null || data === undefined) {
-        return `${exports.CMD_NULL} `;
+        return buffer_1.Buffer.from(`${exports.CMD_NULL} `);
     }
     if (Array.isArray(data)) {
         return serializeCommand(data, zeroString);
diff --git a/lib/sqlitecloud.drivers.dev.js b/lib/sqlitecloud.drivers.dev.js
diff --git a/lib/sqlitecloud.drivers.js b/lib/sqlitecloud.drivers.js
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@sqlitecloud/drivers",
-  "version": "1.0.491",
+  "version": "1.0.492",
   "description": "SQLiteCloud drivers for Typescript/Javascript in edge, web and node clients",
   "main": "./lib/index.js",
   "types": "./lib/index.d.ts",
diff --git a/src/drivers/connection-tls.ts b/src/drivers/connection-tls.ts
@@ -141,11 +141,11 @@ export class SQLiteCloudTlsConnection extends SQLiteCloudConnection {
         this.socket = undefined
       }, timeoutMs)
 
-      this.socket?.write(formattedCommands, 'utf-8', () => {
+      this.socket?.write(formattedCommands, () => {
         clearTimeout(timeout) // Clear the timeout on successful write
       })
     } else {
-      this.socket?.write(formattedCommands, 'utf-8')
+      this.socket?.write(formattedCommands)
     }
 
     return this
diff --git a/src/drivers/connection.ts b/src/drivers/connection.ts
@@ -2,7 +2,7 @@
  * connection.ts - base abstract class for sqlitecloud server connections
  */
 
-import { SQLiteCloudConfig, SQLiteCloudError, ErrorCallback, ResultsCallback, SQLiteCloudCommand } from './types'
+import { SQLiteCloudConfig, SQLiteCloudError, ErrorCallback, ResultsCallback, SQLiteCloudCommand, SQLiteCloudDataTypes } from './types'
 import { validateConfiguration } from './utilities'
 import { OperationsQueue } from './queue'
 import { anonimizeCommand, getUpdateResults } from './utilities'
@@ -112,7 +112,7 @@ export abstract class SQLiteCloudConnection {
    * @returns An array of rows in case of selections or an object with
    * metadata in case of insert, update, delete.
    */
-  public async sql(sql: TemplateStringsArray | string | SQLiteCloudCommand, ...values: any[]): Promise<any> {
+  public async sql(sql: TemplateStringsArray | string | SQLiteCloudCommand, ...values: SQLiteCloudDataTypes[]): Promise<any> {
     let commands = { query: '' } as SQLiteCloudCommand
 
     // sql is a TemplateStringsArray, the 'raw' property is specific to TemplateStringsArray
diff --git a/src/drivers/database.ts b/src/drivers/database.ts
@@ -22,6 +22,7 @@ import {
   SQLiteCloudArrayType,
   SQLiteCloudCommand,
   SQLiteCloudConfig,
+  SQLiteCloudDataTypes,
   SQLiteCloudError
 } from './types'
 import { isBrowser, popCallback } from './utilities'
@@ -436,7 +437,7 @@ export class Database extends EventEmitter {
    * @returns An array of rows in case of selections or an object with
    * metadata in case of insert, update, delete.
    */
-  public async sql(sql: TemplateStringsArray | string | SQLiteCloudCommand, ...values: any[]): Promise<any> {
+  public async sql(sql: TemplateStringsArray | string | SQLiteCloudCommand, ...values: SQLiteCloudDataTypes[]): Promise<any> {
     let commands = { query: '' } as SQLiteCloudCommand
 
     // sql is a TemplateStringsArray, the 'raw' property is specific to TemplateStringsArray
diff --git a/src/drivers/protocol.ts b/src/drivers/protocol.ts
@@ -334,7 +334,7 @@ export function popData(buffer: Buffer): { data: SQLiteCloudDataTypes | SQLiteCl
 }
 
 /** Format a command to be sent via SCSP protocol */
-export function formatCommand(command: SQLiteCloudCommand): string {
+export function formatCommand(command: SQLiteCloudCommand): Buffer {
   // core returns null if there's a space after the semi column
   // we want to maintain a compatibility with the standard sqlite3 driver
   command.query = command.query.trim()
@@ -346,22 +346,23 @@ export function formatCommand(command: SQLiteCloudCommand): string {
   return serializeData(command.query, false)
 }
 
-function serializeCommand(data: SQLiteCloudDataTypes[], zeroString: boolean = false): string {
+function serializeCommand(data: SQLiteCloudDataTypes[], zeroString: boolean = false): Buffer {
   const n = data.length
-  let serializedData = `${n} `
+  let serializedData = Buffer.from(`${n} `)
 
   for (let i = 0; i < n; i++) {
     // the first string is the sql and it must be zero-terminated
     const zs = i == 0 || zeroString
-    serializedData += serializeData(data[i], zs)
+    serializedData = Buffer.concat([serializedData, serializeData(data[i], zs)])
   }
 
-  const bytesTotal = Buffer.byteLength(serializedData, 'utf-8')
-  const header = `${CMD_ARRAY}${bytesTotal} `
-  return header + serializedData
+  const bytesTotal = serializedData.byteLength
+  const header = Buffer.from(`${CMD_ARRAY}${bytesTotal} `)
+
+  return Buffer.concat([header, serializedData])
 }
 
-function serializeData(data: SQLiteCloudDataTypes, zeroString: boolean = false): string {
+function serializeData(data: SQLiteCloudDataTypes, zeroString: boolean = false): Buffer {
   if (typeof data === 'string') {
     let cmd = CMD_STRING
     if (zeroString) {
@@ -370,24 +371,24 @@ function serializeData(data: SQLiteCloudDataTypes, zeroString: boolean = false):
     }
 
     const header = `${cmd}${Buffer.byteLength(data, 'utf-8')} `
-    return header + data
+    return Buffer.from(header + data)
   }
 
   if (typeof data === 'number') {
     if (Number.isInteger(data)) {
-      return `${CMD_INT}${data} `
+      return Buffer.from(`${CMD_INT}${data} `)
     } else {
-      return `${CMD_FLOAT}${data} `
+      return Buffer.from(`${CMD_FLOAT}${data} `)
     }
   }
 
   if (Buffer.isBuffer(data)) {
-    const header = `${CMD_BLOB}${data.length} `
-    return header + data.toString('utf-8')
+    const header = `${CMD_BLOB}${data.byteLength} `
+    return Buffer.concat([Buffer.from(header), data])
   }
 
   if (data === null || data === undefined) {
-    return `${CMD_NULL} `
+    return Buffer.from(`${CMD_NULL} `)
   }
 
   if (Array.isArray(data)) {
diff --git a/src/drivers/utilities.ts b/src/drivers/utilities.ts
@@ -82,7 +82,7 @@ export function getInitializationCommands(config: SQLiteCloudConfig): string {
     }
     commands += `USE DATABASE ${config.database};`
   }
-
+  
   return commands
 }
 
diff --git a/test/database.test.ts b/test/database.test.ts
@@ -6,6 +6,7 @@ import { describe, expect, it } from '@jest/globals'
 import { RowCountCallback } from '../src/drivers/types'
 import { Database, SQLiteCloudError, SQLiteCloudRow, SQLiteCloudRowset, sanitizeSQLiteIdentifier } from '../src/index'
 import { LONG_TIMEOUT, getChinookDatabase, getTestingDatabase, getTestingDatabaseAsync, removeDatabase, removeDatabaseAsync } from './shared'
+const crypto = require('crypto')
 
 //
 // utility methods to setup and destroy temporary test databases
@@ -604,6 +605,26 @@ describe('Database.sql (async)', () => {
       await removeDatabaseAsync(database)
     }
   })
+
+  it('binding should work with blob', async () => {
+    let database
+    try {
+      database = await getTestingDatabaseAsync()
+      const hash = crypto.createHash('sha256').update('my blob data').digest()
+
+      await database.sql('CREATE TABLE IF NOT EXISTS blobs (id INTEGER PRIMARY KEY, hash BLOB NOT NULL, myindex INTEGER NOT NULL);')
+      let results = await database.sql('INSERT INTO blobs (hash, myindex) VALUES (?, ?);', hash, 1)
+      expect(results.changes).toEqual(1)
+
+      results = await database.sql('SELECT * FROM blobs WHERE id = ? and hash = ?;', results.lastID, hash)
+
+      expect(results).toHaveLength(1)
+      expect(results[0].hash).toEqual(hash)
+      expect(results[0].myindex).toEqual(1)
+    } finally {
+      await removeDatabaseAsync(database)
+    }
+  })
 })
 
 it('should be connected', async () => {
diff --git a/test/protocol.test.ts b/test/protocol.test.ts
@@ -50,7 +50,7 @@ describe('Format command', () => {
     it(`should serialize ${JSON.stringify([query, ...parameters])}`, () => {
       const command: SQLiteCloudCommand = { query, parameters }
       const serialized = formatCommand(command)
-      expect(serialized).toEqual(expected)
+      expect(serialized).toEqual(Buffer.from(expected))
     })
   })
 })

Original file line number	Diff line number	Diff line change
`@@ -156,12 +156,12 @@ class SQLiteCloudTlsConnection extends connection_1.SQLiteCloudConnection {`
`156`	`156`	`(_a = this.socket) === null \|\| _a === void 0 ? void 0 : _a.destroy();`
`157`	`157`	`this.socket = undefined;`
`158`	`158`	`}, timeoutMs);`
`159`		`- (_d = this.socket) === null \|\| _d === void 0 ? void 0 : _d.write(formattedCommands, 'utf-8', () => {`
	`159`	`+ (_d = this.socket) === null \|\| _d === void 0 ? void 0 : _d.write(formattedCommands, () => {`
`160`	`160`	`clearTimeout(timeout); // Clear the timeout on successful write`
`161`	`161`	`});`
`162`	`162`	`}`
`163`	`163`	`else {`
`164`		`- (_e = this.socket) === null \|\| _e === void 0 ? void 0 : _e.write(formattedCommands, 'utf-8');`
	`164`	`+ (_e = this.socket) === null \|\| _e === void 0 ? void 0 : _e.write(formattedCommands);`
`165`	`165`	`}`
`166`	`166`	`return this;`
`167`	`167`	`}`
Original file line number	Diff line number	Diff line change
`@@ -314,15 +314,15 @@ function formatCommand(command) {`
`314`	`314`	`}`
`315`	`315`	`function serializeCommand(data, zeroString = false) {`
`316`	`316`	`const n = data.length;`
`317`		- let serializedData = `${n} `;
	`317`	+ let serializedData = buffer_1.Buffer.from(`${n} `);
`318`	`318`	`for (let i = 0; i < n; i++) {`
`319`	`319`	`// the first string is the sql and it must be zero-terminated`
`320`	`320`	`const zs = i == 0 \|\| zeroString;`
`321`		`- serializedData += serializeData(data[i], zs);`
	`321`	`+ serializedData = buffer_1.Buffer.concat([serializedData, serializeData(data[i], zs)]);`
`322`	`322`	`}`
`323`		`- const bytesTotal = buffer_1.Buffer.byteLength(serializedData, 'utf-8');`
`324`		- const header = `${exports.CMD_ARRAY}${bytesTotal} `;
`325`		`- return header + serializedData;`
	`323`	`+ const bytesTotal = serializedData.byteLength;`
	`324`	+ const header = buffer_1.Buffer.from(`${exports.CMD_ARRAY}${bytesTotal} `);
	`325`	`+ return buffer_1.Buffer.concat([header, serializedData]);`
`326`	`326`	`}`
`327`	`327`	`function serializeData(data, zeroString = false) {`
`328`	`328`	`if (typeof data === 'string') {`
`@@ -332,22 +332,22 @@ function serializeData(data, zeroString = false) {`
`332`	`332`	`data += '\0';`
`333`	`333`	`}`
`334`	`334`	const header = `${cmd}${buffer_1.Buffer.byteLength(data, 'utf-8')} `;
`335`		`- return header + data;`
	`335`	`+ return buffer_1.Buffer.from(header + data);`
`336`	`336`	`}`
`337`	`337`	`if (typeof data === 'number') {`
`338`	`338`	`if (Number.isInteger(data)) {`
`339`		- return `${exports.CMD_INT}${data} `;
	`339`	+ return buffer_1.Buffer.from(`${exports.CMD_INT}${data} `);
`340`	`340`	`}`
`341`	`341`	`else {`
`342`		- return `${exports.CMD_FLOAT}${data} `;
	`342`	+ return buffer_1.Buffer.from(`${exports.CMD_FLOAT}${data} `);
`343`	`343`	`}`
`344`	`344`	`}`
`345`	`345`	`if (buffer_1.Buffer.isBuffer(data)) {`
`346`		- const header = `${exports.CMD_BLOB}${data.length} `;
`347`		`- return header + data.toString('utf-8');`
	`346`	+ const header = `${exports.CMD_BLOB}${data.byteLength} `;
	`347`	`+ return buffer_1.Buffer.concat([buffer_1.Buffer.from(header), data]);`
`348`	`348`	`}`
`349`	`349`	`if (data === null \|\| data === undefined) {`
`350`		- return `${exports.CMD_NULL} `;
	`350`	+ return buffer_1.Buffer.from(`${exports.CMD_NULL} `);
`351`	`351`	`}`
`352`	`352`	`if (Array.isArray(data)) {`
`353`	`353`	`return serializeCommand(data, zeroString);`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@sqlitecloud/drivers",`
`3`		`- "version": "1.0.491",`
	`3`	`+ "version": "1.0.492",`
`4`	`4`	`"description": "SQLiteCloud drivers for Typescript/Javascript in edge, web and node clients",`
`5`	`5`	`"main": "./lib/index.js",`
`6`	`6`	`"types": "./lib/index.d.ts",`