diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ebf0cb4..56444a3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Release History +## Unreleased + +- Add the default-enabled `enableGeoSpatialSupport` option for kernel-backed GEOMETRY / GEOGRAPHY results. Enabled mode returns the canonical `{ srid, wkb }` Arrow value; disabled mode returns WKT / EWKT. +- Pin transitive `brace-expansion` 1.x / 2.x and `ip-address` dependencies to patched releases for GHSA-6j4f-fj2g-mc7p, GHSA-qhr7-859c-m2p7, and GHSA-2p57-rm9w-gvfp. + ## 2.2.0 - Upgrade the kernel backend native packages to 1.1.0; the kernel dependency is now stable and no longer experimental. diff --git a/CONNECTION_PARAMETERS.md b/CONNECTION_PARAMETERS.md index af1c11c2..50a97a96 100644 --- a/CONNECTION_PARAMETERS.md +++ b/CONNECTION_PARAMETERS.md @@ -98,11 +98,12 @@ disabling verification. ## Results and type rendering -| Option | Type | Thrift | Kernel | Default Value | Note | -| ----------------------------- | --------- | :----: | :----: | ------------- | -------------------------------------------------------------------------------------------------------------------------------------- | -| `preserveBigNumericPrecision` | `boolean` | ✅ | ✅ | `false` | Returns DECIMAL as an exact string and BIGINT as `bigint` on both. | -| `disableRowMaterialization` | `boolean` | ✅ | ✅ | `false` | Fetches and parses Arrow batches but returns `null` row placeholders instead of converting cells. Intended for fetch-throughput tests. | -| `enableMetricViewMetadata` | `boolean` | ✅ | ⚠️ | `false` | Injected into session configuration on both paths. Kernel may drop its non-allowlisted configuration key. | +| Option | Type | Thrift | Kernel | Default Value | Note | +| ----------------------------- | --------- | :----: | :----: | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `preserveBigNumericPrecision` | `boolean` | ✅ | ✅ | `false` | Returns DECIMAL as an exact string and BIGINT as `bigint` on both. | +| `disableRowMaterialization` | `boolean` | ✅ | ✅ | `false` | Fetches and parses Arrow batches but returns `null` row placeholders instead of converting cells. Intended for fetch-throughput tests. | +| `enableGeoSpatialSupport` | `boolean` | ❌ | ✅ | `true` | Returns GEOMETRY / GEOGRAPHY as canonical `{ srid: number, wkb: Buffer }` values when `true`, or WKT / EWKT strings when `false`. The choice is local and is not forwarded to SEA. | +| `enableMetricViewMetadata` | `boolean` | ✅ | ⚠️ | `false` | Injected into session configuration on both paths. Kernel may drop its non-allowlisted configuration key. | ## Session defaults (`openSession(request)`) diff --git a/KERNEL_REV b/KERNEL_REV index f22f2168..a07683fb 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -80f2aee7d884994d7b0af9a9ea6078872859a9cd +b7e9310b27be16a6c42490e58c4b5c4a525320bc diff --git a/lib/contracts/IDBSQLClient.ts b/lib/contracts/IDBSQLClient.ts index 8a5bdef1..b3f5ee33 100644 --- a/lib/contracts/IDBSQLClient.ts +++ b/lib/contracts/IDBSQLClient.ts @@ -157,6 +157,17 @@ export type ConnectionOptions = { */ disableRowMaterialization?: boolean; + /** + * Enable native kernel-backed `GEOMETRY` and `GEOGRAPHY` results. `true` + * (the default) returns the canonical Arrow value as + * `{ srid: number, wkb: Buffer }`; `false` returns WKT / EWKT strings. + * This is a client-side conversion choice and is never forwarded to the SQL + * Execution API. + * + * Only the kernel backend uses this option. + */ + enableGeoSpatialSupport?: boolean; + /** * Extra HTTP headers attached to driver-owned out-of-band requests * (telemetry POSTs and feature-flag GETs). Not applied to the primary diff --git a/lib/kernel/KernelAuth.ts b/lib/kernel/KernelAuth.ts index b2f1443e..c0dc4d84 100644 --- a/lib/kernel/KernelAuth.ts +++ b/lib/kernel/KernelAuth.ts @@ -115,6 +115,12 @@ export interface KernelSessionDefaults { * remove one. */ complexTypesAsJson?: boolean; + /** + * Enable native GEOMETRY / GEOGRAPHY values. `true` (the default) returns + * canonical `struct` values; `false` returns WKT / + * EWKT strings. Applied locally by the kernel and never sent to SEA. + */ + enableGeoSpatialSupport?: boolean; /** * Per-session kernel connection-pool size * (kernel `ConnectionOptions.max_connections`). Validated as a positive @@ -819,6 +825,7 @@ export function buildKernelConnectionOptions(options: ConnectionOptions): Kernel hostName: string; httpPath: string; intervalsAsString: boolean; + enableGeoSpatialSupport: boolean; maxConnections?: number; } & KernelTlsOptions & KernelHttpOptions & @@ -833,6 +840,9 @@ export function buildKernelConnectionOptions(options: ConnectionOptions): Kernel // (native Arrow) — they already decode identically to Thrift via the // shared Arrow converter, so `complexTypesAsJson` is not forced on. intervalsAsString: true, + // Match the public driver contract and the kernel default: geospatial + // support is enabled unless the caller explicitly requests WKT / EWKT. + enableGeoSpatialSupport: true, // TLS knobs (server-cert verification toggle + custom CA + mTLS client // identity). Validated and normalised (string PEM → Buffer) here so the // napi shape only sees a Buffer. @@ -843,6 +853,15 @@ export function buildKernelConnectionOptions(options: ConnectionOptions): Kernel ...buildKernelProxyOptions(options), }; + if (options.enableGeoSpatialSupport !== undefined) { + if (typeof options.enableGeoSpatialSupport !== 'boolean') { + throw new HiveDriverError( + `kernel backend: \`enableGeoSpatialSupport\` must be a boolean; got ${typeof options.enableGeoSpatialSupport}.`, + ); + } + base.enableGeoSpatialSupport = options.enableGeoSpatialSupport; + } + // kernel-only pool sizing; read via cast to match how this function reads the // other kernel-specific options (TLS) — they live on the internal options // surface, not the published public `ConnectionOptions` `.d.ts`. diff --git a/native/kernel/index.d.ts b/native/kernel/index.d.ts index 51644c72..b91c52a6 100644 --- a/native/kernel/index.d.ts +++ b/native/kernel/index.d.ts @@ -405,11 +405,12 @@ export declare class Connection { */ getPrimaryKeys(catalog: string, schema: string, table: string): Promise /** - * Foreign-key relationships. The foreign side must be fully - * specified (catalog + schema + table); the parent side is - * optional. All identifiers are exact — no LIKE patterns. + * Foreign-key relationships. The parent side is optional. When the + * foreign table is omitted, returns an empty result without issuing a + * server statement. When it is provided, its catalog and schema are + * required. All identifiers are exact — no LIKE patterns. */ - getCrossReference(parentCatalog: string | undefined | null, parentSchema: string | undefined | null, parentTable: string | undefined | null, foreignCatalog: string, foreignSchema: string, foreignTable: string): Promise + getCrossReference(parentCatalog?: string | undefined | null, parentSchema?: string | undefined | null, parentTable?: string | undefined | null, foreignCatalog?: string | undefined | null, foreignSchema?: string | undefined | null, foreignTable?: string | undefined | null): Promise } /** @@ -761,6 +762,14 @@ export interface ConnectionOptions { * `session_confs`. Unknown keys are rejected server-side. */ sessionConf?: Record + /** + * Select whether `GEOMETRY` / `GEOGRAPHY` results use their native Arrow + * representation. `true` (the default) requests Arrow + * `struct` values; `false` requests WKT / EWKT + * in Arrow UTF-8 values. Binary mode requires the native Reyden Arrow + * path. This choice is applied locally and never sent to SEA. + */ + enableGeoSpatialSupport?: boolean /** * Driver name reported in telemetry system configuration. Omitted ⇒ * kernel default. diff --git a/package-lock.json b/package-lock.json index 5d7804ab..519bcf28 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1860,9 +1860,9 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.18", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", - "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", + "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", "dev": true, "license": "MIT", "dependencies": { @@ -3993,9 +3993,9 @@ } }, "node_modules/ip-address": { - "version": "10.3.1", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.3.1.tgz", - "integrity": "sha512-1e9d3kb97NHJTIJDZW9rKqW2h6+dFa50Dy0fpPSMQp2ADje5gvKsXmdiK6dwY5t76TaTt5+P5N1Y/LoToIxP6g==", + "version": "10.7.1", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.1.tgz", + "integrity": "sha512-4OUAqU9Z1i3vCnS05hzGiFnEMDpQ+62pAD/MVQOp83fYyNC8GleCqaS0QikQBmcWCrKFiUs/B8ztRRiYOAXuCA==", "license": "MIT", "engines": { "node": ">= 12" @@ -4786,9 +4786,9 @@ } }, "node_modules/minimatch/node_modules/brace-expansion": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", - "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.7.tgz", + "integrity": "sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==", "dev": true, "license": "MIT", "dependencies": { @@ -7357,7 +7357,7 @@ "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "requires": { - "brace-expansion": "^1.1.7" + "brace-expansion": "1.1.21" } } } @@ -7385,7 +7385,7 @@ "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "requires": { - "brace-expansion": "^1.1.7" + "brace-expansion": "1.1.21" } } } @@ -8109,9 +8109,9 @@ "dev": true }, "brace-expansion": { - "version": "1.1.18", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", - "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", + "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", "dev": true, "requires": { "balanced-match": "^1.0.0", @@ -8758,7 +8758,7 @@ "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "requires": { - "brace-expansion": "^1.1.7" + "brace-expansion": "1.1.21" } } } @@ -8896,7 +8896,7 @@ "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "requires": { - "brace-expansion": "^1.1.7" + "brace-expansion": "1.1.21" } }, "ms": { @@ -8934,7 +8934,7 @@ "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "requires": { - "brace-expansion": "^1.1.7" + "brace-expansion": "1.1.21" } }, "semver": { @@ -8982,7 +8982,7 @@ "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "requires": { - "brace-expansion": "^1.1.7" + "brace-expansion": "1.1.21" } }, "resolve": { @@ -9412,7 +9412,7 @@ "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "requires": { - "brace-expansion": "^1.1.7" + "brace-expansion": "1.1.21" } } } @@ -9633,9 +9633,9 @@ } }, "ip-address": { - "version": "10.3.1", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.3.1.tgz", - "integrity": "sha512-1e9d3kb97NHJTIJDZW9rKqW2h6+dFa50Dy0fpPSMQp2ADje5gvKsXmdiK6dwY5t76TaTt5+P5N1Y/LoToIxP6g==" + "version": "10.7.1", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.1.tgz", + "integrity": "sha512-4OUAqU9Z1i3vCnS05hzGiFnEMDpQ+62pAD/MVQOp83fYyNC8GleCqaS0QikQBmcWCrKFiUs/B8ztRRiYOAXuCA==" }, "is-arrayish": { "version": "0.3.2", @@ -10200,13 +10200,13 @@ "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", "dev": true, "requires": { - "brace-expansion": "^2.0.1" + "brace-expansion": "2.1.7" }, "dependencies": { "brace-expansion": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", - "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.7.tgz", + "integrity": "sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==", "dev": true, "requires": { "balanced-match": "^1.0.0" @@ -11109,7 +11109,7 @@ "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.3.tgz", "integrity": "sha512-vfuYK48HXCTFD03G/1/zkIls3Ebr2YNa4qU9gHDZdblHLiqhJrJGkY3+0Nx0JpN9qBhJbVObc1CNciT1bIZJxw==", "requires": { - "ip-address": "^10.1.1", + "ip-address": "10.7.1", "smart-buffer": "^4.2.0" } }, @@ -11309,7 +11309,7 @@ "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "requires": { - "brace-expansion": "^1.1.7" + "brace-expansion": "1.1.21" } } } diff --git a/package.json b/package.json index 438b0220..4f4516ea 100644 --- a/package.json +++ b/package.json @@ -104,9 +104,11 @@ "basic-ftp": "^5.3.1", "@75lb/deep-merge": "^1.1.2", "ws": "^8.18.0", - "ip-address": "^10.1.1", + "ip-address": "10.7.1", "form-data": "^4.0.4", "uuid": "^11.1.1", - "serialize-javascript": "^7.0.5" + "serialize-javascript": "^7.0.5", + "brace-expansion@1": "1.1.21", + "brace-expansion@2": "2.1.7" } } diff --git a/tests/unit/kernel/KernelOperationBackend.test.ts b/tests/unit/kernel/KernelOperationBackend.test.ts index 2980ecce..98d71391 100644 --- a/tests/unit/kernel/KernelOperationBackend.test.ts +++ b/tests/unit/kernel/KernelOperationBackend.test.ts @@ -257,6 +257,47 @@ describe('KernelOperationBackend — M0 datatype round-trip via napi → ArrowRe expect(row.s).to.deep.equal({ a: 1, b: 'hi' }); }); + it('surfaces geospatial string and binary values in idiomatic JS shapes', async () => { + const wkb = new Uint8Array([ + 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x40, + ]); + const geoType = new Struct([ + new Field('srid', new Int32(), false), + new Field( + 'wkb', + new Binary(), + false, + new Map([ + ['geometry', 'true'], + ['srid', '-1'], + ]), + ), + ]); + const schema = new Schema([ + withTypeName(new Field('geom_text', new Utf8(), true), 'GEOMETRY'), + withTypeName(new Field('geom_binary', geoType, true), 'GEOMETRY'), + ]); + const stub = new StatementStub(ipcSchemaOnly(schema), [ + ipcFromColumns(schema, { + geom_text: ['SRID=4326;POINT(1 2)', null], + geom_binary: [{ srid: 4326, wkb }, null], + }), + ]); + const backend = new KernelOperationBackend({ + statement: stub, + context: new ClientContextStub(), + }); + + const rows = (await backend.fetchChunk({ limit: 100 })) as Array>; + expect(rows[0].geom_text).to.equal('SRID=4326;POINT(1 2)'); + expect(rows[0].geom_binary).to.deep.equal({ + srid: 4326, + wkb: Buffer.from(wkb), + }); + expect(rows[1]).to.deep.equal({ geom_text: null, geom_binary: null }); + }); + it('streams multiple batches and reports hasMore correctly', async () => { const schema = new Schema([withTypeName(new Field('x', new Int32(), true), 'INT')]); const schemaIpc = ipcSchemaOnly(schema); diff --git a/tests/unit/kernel/_helpers/nativeOptions.ts b/tests/unit/kernel/_helpers/nativeOptions.ts index 6c42e87c..886bc341 100644 --- a/tests/unit/kernel/_helpers/nativeOptions.ts +++ b/tests/unit/kernel/_helpers/nativeOptions.ts @@ -81,7 +81,7 @@ export default function expectNativeConnectionOptions(actual: unknown, expectedR ]) { delete rest[key]; } - expect(rest).to.deep.equal(expectedRest); + expect(rest).to.deep.equal({ enableGeoSpatialSupport: true, ...expectedRest }); expect(customHeaders, 'customHeaders').to.be.an('array').with.lengthOf(1); expect(customHeaders?.[0].name).to.equal('User-Agent'); expect(customHeaders?.[0].value).to.match(/NodejsDatabricksSqlConnector\//); diff --git a/tests/unit/kernel/connectionOptions.test.ts b/tests/unit/kernel/connectionOptions.test.ts index b5b13d41..fe1182f9 100644 --- a/tests/unit/kernel/connectionOptions.test.ts +++ b/tests/unit/kernel/connectionOptions.test.ts @@ -40,6 +40,29 @@ describe('KernelAuth connection options — intervalsAsString default', () => { }); }); +describe('KernelAuth connection options — geospatial result representation', () => { + it('enables native geospatial support by default', () => { + const native = buildKernelConnectionOptions(opts({})) as { enableGeoSpatialSupport?: boolean }; + expect(native.enableGeoSpatialSupport).to.equal(true); + }); + + for (const value of [true, false]) { + it(`forwards enableGeoSpatialSupport=${value}`, () => { + const native = buildKernelConnectionOptions(opts({ enableGeoSpatialSupport: value })) as { + enableGeoSpatialSupport?: boolean; + }; + expect(native.enableGeoSpatialSupport).to.equal(value); + }); + } + + it('rejects non-boolean values at runtime', () => { + expect(() => buildKernelConnectionOptions(opts({ enableGeoSpatialSupport: 'false' }))).to.throw( + HiveDriverError, + /must be a boolean/, + ); + }); +}); + describe('KernelAuth connection options — maxConnections', () => { it('forwards a valid positive integer', () => { const native = buildKernelConnectionOptions(opts({ maxConnections: 10 })) as { maxConnections?: number }; diff --git a/tests/unit/kernel/execution.test.ts b/tests/unit/kernel/execution.test.ts index af7d53c3..313b39fd 100644 --- a/tests/unit/kernel/execution.test.ts +++ b/tests/unit/kernel/execution.test.ts @@ -533,6 +533,7 @@ describe('KernelBackend', () => { host: 'workspace.example', path: '/sql/1.0/warehouses/xyz', token: 'dapi-token', + enableGeoSpatialSupport: false, } as ConnectionOptions); await backend.openSession({}); @@ -549,6 +550,7 @@ describe('KernelBackend', () => { authMode: 'Pat', token: 'dapi-token', intervalsAsString: true, + enableGeoSpatialSupport: false, }); });