From c5b7c7afbf37d7381cedb3672f19c5c5646605b1 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Tue, 17 Nov 2026 10:54:13 +0530 Subject: [PATCH 1/7] FEAT: Add native FLOAT32 and FLOAT16 vector support Add immutable base_type-aware vectors, hinted list binding, TVPs and buffered procedure outputs. Include numeric validation, documentation and unit/live SQL coverage. Vector bulk copy and Arrow remain explicit follow-up work. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 11 + README.md | 152 ++++ mssql_python/__init__.py | 8 + mssql_python/_ddbc_types.pyi | 17 + mssql_python/_native_fetch.py | 2 + mssql_python/connection_string_parser.py | 6 + mssql_python/constants.py | 6 + mssql_python/cursor.py | 207 ++++- mssql_python/ddbc_bindings.py | 2 + mssql_python/mssql_python.pyi | 8 + mssql_python/pybind/ddbc_bindings.cpp | 325 ++++++- mssql_python/pybind/ddbc_bindings.h | 5 + mssql_python/pybind/param_detect.hpp | 40 + mssql_python/pybind/py_type_cache.hpp | 6 + mssql_python/pybind/vector_support.hpp | 76 ++ mssql_python/table_parameter.py | 70 ++ mssql_python/vector.py | 116 +++ tests/test_004_cursor.py | 10 +- tests/test_025_vector.py | 1058 ++++++++++++++++++++++ tests/test_typing.py | 29 + 20 files changed, 2134 insertions(+), 20 deletions(-) create mode 100644 mssql_python/pybind/vector_support.hpp create mode 100644 mssql_python/table_parameter.py create mode 100644 mssql_python/vector.py create mode 100644 tests/test_025_vector.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e3f9ce6e..73bdaa5cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] ### Added +- Opt-in synchronous native vector support with immutable `Vector` values, + FLOAT32/FLOAT16 subtype metadata, scalar/array parameter binding, and row + fetches. `base_type` selects SQL storage; values are normalized to float32 at + construction, with early finite/float32 and conservative FLOAT16 range checks. + Explicit vector hints and TVP schemas accept numeric lists with the same + validation; unhinted lists keep their existing meanings. Added + `TableValuedParameter` and buffered `callproc()` with vector + `OutputParameter` / `InputOutputParameter` wrappers. FLOAT16 requires Microsoft + ODBC Driver 18.7.1.1+ (not supplied by the current bundled driver); ODBC owns + half-precision conversion. Vector bulk copy remains pending pycore's contract, + and Arrow vector fetches are not supported. - New feature: Support for macOS and Linux. - Documentation: Added API documentation in the Wiki. - New `token_provider=` parameter on `connect()` / `Connection` for Microsoft diff --git a/README.md b/README.md index 620cc63e9..8e5ab3ec3 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,158 @@ example, requesting ASCII with `SQL_CHAR` does not cause non-ASCII parameters to raise encoding errors. Use `setencoding()` with no arguments to restore the supported defaults. `setdecoding()` independently controls how result data is read. +### Native vector values + +The synchronous ODBC API accepts and returns immutable `Vector` values. SQL NULL +is represented by `None`. FLOAT32 supports 1–1998 dimensions; FLOAT16 supports +1–3996. A vector's dimensions and subtype must match its SQL column or parameter. + +Enable native vector negotiation **when connecting**: `VectorTypeSupport=v1` +for FLOAT32, or `VectorTypeSupport=v2` for both subtypes. The default remains +`off` (the ODBC driver's existing JSON/text behavior). FLOAT32 requires Microsoft +ODBC Driver 18.6.1.1+; FLOAT16 requires **18.7.1.1+** and a server configured to +support FLOAT16 vectors. The currently published `mssql-python-odbc` 18.6.2.2 +package does not meet the FLOAT16 requirement. These Python changes do not +upgrade the bundled driver or add support to the alternate Rust ODBC provider. + +| Connection setting | Native FLOAT32 | Native FLOAT16 | +|---|---|---| +| `VectorTypeSupport=off` (default) | No; reads return JSON text | No; reads return JSON text | +| `VectorTypeSupport=v1` | Yes | No; native FLOAT16 inputs are rejected by ODBC | +| `VectorTypeSupport=v2` | Yes | Yes | + +`Vector(values, base_type="float16")` selects the value's SQL base type; it does not upgrade a +connection from v1 to v2. The connection setting is passed through to ODBC for +negotiation rather than emulated in Python. + +```python +from mssql_python import connect, Vector, SQL_SS_VECTOR + +connection = connect(connection_string, VectorTypeSupport="v1") +cursor = connection.cursor() +cursor.execute("CREATE TABLE #embeddings (embedding VECTOR(3) NULL)") +vector = Vector([1.25, -2.5, 0]) # FLOAT16 values require a v2 connection +cursor.execute("INSERT INTO #embeddings VALUES (?)", vector) +cursor.executemany("INSERT INTO #embeddings VALUES (?)", [(None,), (vector,)]) +cursor.setinputsizes([(SQL_SS_VECTOR, 3, 0)]) # dimensions, subtype: 0=FLOAT32, 1=FLOAT16 +cursor.execute("INSERT INTO #embeddings VALUES (?)", None) +cursor.execute("SELECT embedding FROM #embeddings") +rows = cursor.fetchall() # fetchone(), fetchmany(), and iteration work too +values = list(vector) # Independent list; the Vector remains immutable +``` + +`Vector(values, base_type="float32")` snapshots a numeric iterable and normalizes +each element to float32 immediately. Its `.base_type` and `.dimensions` +properties describe the SQL value; iteration and indexing return Python floats. +Ordinary float32 rounding is allowed, including subnormal values and underflow +to zero. Exact float32 representability is not required. + +Non-numeric elements (including bool and None) raise `TypeError`. NaN, infinity, +and magnitudes above the finite float32 maximum (approximately 3.402823466e38) +raise `ValueError` in the constructor, before binding or network I/O. FLOAT16 +values additionally reject magnitudes above 65504 conservatively, before +float32 rounding. Errors identify the element index without including its value. +Values are never clamped, promoted to float64, or silently changed to JSON. + +An explicit vector input hint also accepts a numeric **list**, applying the +same checks during binding. Without a hint, ordinary lists retain their existing +parameter meanings and are not automatically vectors. Hints are consumed by a +successful execution; reapply them for subsequent calls. + +```python +cursor.setinputsizes([(SQL_SS_VECTOR, 3, 0)]) +cursor.execute("INSERT INTO #embeddings VALUES (?)", [[0.1, 0.2, 0.3]]) +cursor.setinputsizes([(SQL_SS_VECTOR, 3, 0)]) +cursor.executemany( + "INSERT INTO #embeddings VALUES (?)", + [([0.1, 0.2, 0.3],), (None,), (vector,)], +) +``` + +Non-NULL values must match the hinted dimensions and base type. A conflicting +`Vector` is rejected, not reinterpreted. Batch vector columns must agree in +dimensions/base type; hinted columns may mix lists, matching Vectors, and None. + +Both subtypes are marshaled to ODBC as `SQL_C_SS_VECTOR` structures containing +C float32 arrays and bound with SQL type `SQL_SS_VECTOR`. ODBC owns conversion +to/from FLOAT16, including half-precision rounding and backend diagnostics; +Python performs only the early finite/range checks above. Python does not +pack TDS headers, encode half floats, or serialize vectors as binary/JSON. + +**Table-valued parameters:** use an existing server-side table type and an +explicit schema. Each column specification is `(SQL type, size, scale)`; for +vectors, size means dimensions and scale is the subtype (0 or 1). Empty tables +and NULL vector cells are supported. Vector columns also accept lists because +the explicit schema provides dimensions and base type; cells are snapshotted +as Vectors with the same validation. UDT/sql_variant TVP columns and +data-at-execution TVP cells are not supported. + +```python +from mssql_python import TableValuedParameter, SQL_INTEGER + +# dbo.EmbeddingRows: (id INT, embedding VECTOR(3)) +table = TableValuedParameter( + "dbo.EmbeddingRows", + [(1, vector), (2, None)], + [(SQL_INTEGER, 10, 0), (SQL_SS_VECTOR, 3, 0)], +) +cursor.execute("EXEC dbo.InsertEmbeddings ?", table) +``` + +**Stored procedures:** `callproc()` accepts ordinary input parameters and +vector `OutputParameter(dimensions, base_type)` / `InputOutputParameter(vector)` +wrappers. It returns a modified parameter sequence and also fills each wrapper's +`.value`. Result sets are buffered in memory so output values are final when the +call returns; use normal row fetches and `nextset()` to read them. +Input parameters may be `None`; parameter-description metadata is not exposed as +a procedure result set, including when reusing the cursor. + +```python +from mssql_python import OutputParameter, InputOutputParameter + +# dbo.EchoEmbedding: @input VECTOR(3), @output VECTOR(3) OUTPUT, +# @inout VECTOR(3) OUTPUT +output = OutputParameter(3) +inout = InputOutputParameter(vector) +parameters = cursor.callproc("dbo.EchoEmbedding", [vector, output, inout]) +print(output.value) +``` + +Output wrappers currently support vectors only. Procedure text/binary inputs +must fit the inline binding limits. Arrow vector fetches and Arrow access to +buffered procedure results are not supported; use row fetch methods instead. +**Vector bulk copy is pending the pycore vector input contract.** `bulkcopy()` +explicitly rejects `Vector` cells rather than guessing a serialization format. +Once that backend contract is available, pycore will own its FLOAT16 conversion. + +Vector unit tests run without SQL Server. End-to-end tests in +`tests/test_025_vector.py` exercise v1/FLOAT32, v2/FLOAT32, and v2/FLOAT16: +scalar and batch binding (including explicitly hinted lists, pyformat, +typed/all-NULL arrays and streamed text), numeric boundaries and error recovery, +every row fetch method including `fetchval()`, metadata/row access, output converters, +commit/rollback, TVPs, stored procedure outputs/result sets, and error recovery. +Negotiation tests assert that v1 rejects native FLOAT16, `off` rejects native +vector inputs, and `off` retains JSON/NULL reads. Arrow and bulk-copy cases assert +the current explicit unsupported boundaries; they are not positive vector +bulk-copy tests. + +Set `DB_CONNECTION_STRING` outside source control to enable the live tests. +The database principal needs permission to create tables, procedures, and table +types in `dbo`; named test objects use unique names and are dropped in cleanup. +The v2 cases additionally require `MSSQL_TEST_VECTOR_FLOAT16=1`, ODBC 18.7.1.1+, +and a FLOAT16-capable server. For example, from PowerShell: + +```powershell +python -m pytest tests\test_025_vector.py -k live -v +# Enable the full v2 matrix only after provisioning a compatible driver/server: +$env:MSSQL_TEST_VECTOR_FLOAT16 = "1" +python -m pytest tests\test_025_vector.py -v +``` + +Missing database/capability configuration causes explicit skips. Once enabled, +connection failures and unsupported backend behavior fail the tests rather than +being converted to skips. + ## Getting Started Examples Connect to SQL Server and execute a simple query: diff --git a/mssql_python/__init__.py b/mssql_python/__init__.py index 6e79ca239..265bc9c8f 100644 --- a/mssql_python/__init__.py +++ b/mssql_python/__init__.py @@ -66,6 +66,8 @@ # Row Objects from .row import Row, RowMapping +from .vector import Vector, OutputParameter, InputOutputParameter +from .table_parameter import TableValuedParameter # Logging Configuration (Simplified single-level DEBUG system) from .logging import logger, setup_logging, driver_logger @@ -183,6 +185,7 @@ def _cleanup_connections() -> None: SQL_SS_TIME2, SQL_SS_XML, SQL_SS_VARIANT, + SQL_SS_VECTOR, # Connection attribute constants SQL_ATTR_ACCESS_MODE, SQL_ATTR_CONNECTION_TIMEOUT, @@ -324,6 +327,11 @@ def _cleanup_connections() -> None: ) __all__ = [ + "Vector", + "OutputParameter", + "InputOutputParameter", + "TableValuedParameter", + "SQL_SS_VECTOR", # Exception classes "Warning", "Error", diff --git a/mssql_python/_ddbc_types.pyi b/mssql_python/_ddbc_types.pyi index 52a8473d0..f06dfc13e 100644 --- a/mssql_python/_ddbc_types.pyi +++ b/mssql_python/_ddbc_types.pyi @@ -11,6 +11,7 @@ __all__ = [ "ARCHITECTURE", "Connection", "DDBCSQLCheckError", + "DDBCCallProc", "DDBCSQLColumns", "DDBCSQLDescribeCol", "DDBCSQLExecDirect", @@ -66,6 +67,8 @@ class ColumnMetadata(TypedDict): DecimalDigits: int Nullable: int +ProcedureResult = tuple[list[ColumnMetadata], list[list[Any]], int] + class InfoResult(TypedDict): data: bytes length: int @@ -197,6 +200,20 @@ def DDBCSetDecimalSeparator(separator: str) -> None: ... def DDBCSQLCheckError(handle_type: int, handle: SqlHandle | None, ret: int) -> ErrorInfo: ... def DDBCSQLGetAllDiagRecords(handle: SqlHandle | None) -> list[tuple[str, str]]: ... def DDBCSQLExecDirect(handle: SqlHandle | None, query: str) -> int: ... +def _vector_buffer_size(dimensions: int) -> int: ... +def _detect_param_types( + params: list[Any], sizes: list[tuple[int, int, int, int]] | None = None +) -> list[ParamInfo]: ... +def DDBCCallProc( + statement: SqlHandle | None, + query: str, + parameters: list[Any], + encodingSettings: EncodingSettings, + messages: list[tuple[str, str]], + charEncoding: str, + wcharEncoding: str, + charCtype: int, +) -> tuple[list[Any], list[ProcedureResult]]: ... def DDBCSQLExecute( statementHandle: SqlHandle | None, query: str, diff --git a/mssql_python/_native_fetch.py b/mssql_python/_native_fetch.py index 743d776a4..a1d0c74c5 100644 --- a/mssql_python/_native_fetch.py +++ b/mssql_python/_native_fetch.py @@ -119,6 +119,8 @@ def __init__( ) def __call__(self, cursor: Cursor, data: dict[str, Any]) -> tuple[object, ...] | None: + if data.get("_procedure_rows") is not None: + return None # Watch even an ineligible cursor so restoring its state can re-enable it. connection = data.get("_connection") if type(cursor) is not self.cursor_type or connection is None: diff --git a/mssql_python/connection_string_parser.py b/mssql_python/connection_string_parser.py index 42dc8ef41..3dd3221e0 100644 --- a/mssql_python/connection_string_parser.py +++ b/mssql_python/connection_string_parser.py @@ -106,6 +106,12 @@ def _normalize_params(params: Dict[str, str], warn_rejected: bool = True) -> Dic normalized_key = _ConnectionStringParser.normalize_key(key) if normalized_key: + if normalized_key == "VectorTypeSupport": + value = value.strip().lower() + if value not in ("off", "v1", "v2"): + raise ConnectionStringParseError( + ["VectorTypeSupport must be off, v1 or v2"] + ) # Driver selection is controlled by the driver. if normalized_key in _RESERVED_PARAMETERS: continue diff --git a/mssql_python/constants.py b/mssql_python/constants.py index c02f8b39a..76bd262c1 100644 --- a/mssql_python/constants.py +++ b/mssql_python/constants.py @@ -120,6 +120,8 @@ class ConstantsDDBC(Enum): SQL_SS_TIME2 = -154 SQL_SS_XML = -152 SQL_SS_VARIANT = -150 + SQL_SS_VECTOR = -156 + SQL_C_SS_VECTOR = 0x4002 SQL_C_SS_TIMESTAMPOFFSET = 0x4001 SQL_SCOPE_CURROW = 0 SQL_BEST_ROWID = 1 @@ -420,6 +422,7 @@ def get_valid_types(cls) -> set[int]: ConstantsDDBC.SQL_GUID.value, ConstantsDDBC.SQL_SS_UDT.value, ConstantsDDBC.SQL_SS_VARIANT.value, + ConstantsDDBC.SQL_SS_VECTOR.value, } # Could also add category methods for convenience @@ -572,6 +575,7 @@ def get_attribute_set_timing(attribute: int) -> AttributeSetTime: # (with spaces) ODBC only honors "PacketSize" without spaces # internally. "packetsize": "PacketSize", + "vectortypesupport": "VectorTypeSupport", } @@ -629,6 +633,7 @@ def get_info_constants() -> Dict[str, int]: "SQL_SS_TIME2", "SQL_SS_XML", "SQL_SS_VARIANT", + "SQL_SS_VECTOR", # Connection attribute constants (ODBC-standard, driver-independent only) "SQL_ATTR_ACCESS_MODE", "SQL_ATTR_CONNECTION_TIMEOUT", @@ -695,6 +700,7 @@ def get_info_constants() -> Dict[str, int]: SQL_SS_TIME2: int SQL_SS_XML: int SQL_SS_VARIANT: int + SQL_SS_VECTOR: int SQL_ATTR_ACCESS_MODE: int SQL_ATTR_CONNECTION_TIMEOUT: int SQL_ATTR_CURRENT_CATALOG: int diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 54010e5d7..a56c81389 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -20,6 +20,7 @@ import warnings import threading import weakref +from collections import deque from collections.abc import Callable from types import FrameType, GeneratorType from importlib import import_module @@ -37,6 +38,7 @@ DatabaseError, ) from mssql_python.row import Row, OutputConverter, _ColumnNames +from mssql_python.vector import Vector, _vector_from_hint from mssql_python.perf_timer import perf_phase from mssql_python import get_settings from mssql_python.parameter_helper import ( @@ -47,7 +49,7 @@ if TYPE_CHECKING: import pyarrow - from mssql_python._ddbc_types import ColumnMetadata, EncodingSettings + from mssql_python._ddbc_types import ColumnMetadata, EncodingSettings, ProcedureResult from mssql_python._pycore_types import ( BulkCopyResult, CoreContext, @@ -466,6 +468,8 @@ def __init__(self, connection: "Connection", timeout: int = 0) -> None: self._connection: "Connection" = connection # Store as private attribute self._timeout: int = timeout self._inputsizes: Optional[List[Tuple[int, int, int, int]]] = None + self._procedure_results: deque[ProcedureResult] | None = None + self._procedure_rows: deque[list[Any]] | None = None # self.connection.autocommit = False self._initialize_cursor() self.description: _Description | None = None @@ -776,6 +780,15 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg - A tuple containing the SQL type, C type, column size, and decimal digits. """ logger.debug("_map_sql_type: Mapping param index=%d, type=%s", i, type(param).__name__) + if isinstance(param, Vector): + info = ddbc_bindings._detect_param_types([param])[0] + return ( + info.paramSQLType, + info.paramCType, + info.columnSize, + info.decimalDigits, + False, + ) if param is None: logger.debug("_map_sql_type: NULL parameter - index=%d", i) # GH-610: Send SQL_UNKNOWN_TYPE to C++ where the describe-cache @@ -1127,6 +1140,8 @@ def _reset_cursor(self) -> None: """ Reset the DDBC statement handle. """ + self._procedure_results = None + self._procedure_rows = None if self.hstmt: self.hstmt.free() self.hstmt = None @@ -1147,6 +1162,8 @@ def _soft_reset_cursor(self) -> None: parameter definitions are unchanged (so the driver keeps its prepared handle and sends sp_execute) and resets them otherwise. """ + self._procedure_results = None + self._procedure_rows = None if self.hstmt: ret = ddbc_bindings.DDBCSQLResetStmt(self.hstmt) try: @@ -1179,6 +1196,8 @@ def close(self) -> None: # Clear messages per DBAPI self.messages = [] + self._procedure_results = None + self._procedure_rows = None # Remove this cursor from the connection's tracking if hasattr(self, "connection") and self.connection and hasattr(self.connection, "_cursors"): @@ -1256,6 +1275,11 @@ def setinputsizes(self, sizes: Sequence[int | tuple[int, ...]] | None) -> None: cursor.executemany(sql, params) Note: + SQL_SS_VECTOR hints use size as dimensions and decimal_digits as + base type (0=FLOAT32, 1=FLOAT16). They accept Vector, list, or None. + Lists use the same validation as Vector; a hint never changes an + existing Vector's base type. Successful execution consumes hints. + SQL_SS_UDT parameters require a statement whose parameter metadata SQL Server can describe. The driver discovers the UDT identity before binding; this may require a metadata round trip. Temporary tables and @@ -1388,6 +1412,7 @@ def _get_sql_to_c_type_map(cls) -> dict[int, int]: ddbc_sql_const.SQL_GUID.value: ddbc_sql_const.SQL_C_GUID.value, ddbc_sql_const.SQL_SS_XML.value: ddbc_sql_const.SQL_C_WCHAR.value, ddbc_sql_const.SQL_SS_VARIANT.value: ddbc_sql_const.SQL_C_BINARY.value, + ddbc_sql_const.SQL_SS_VECTOR.value: ddbc_sql_const.SQL_C_SS_VECTOR.value, } return cls._SQL_TO_C_TYPE @@ -1641,6 +1666,7 @@ def _map_data_type(self, sql_type: int) -> type: ddbc_sql_const.SQL_FLOAT.value: float, ddbc_sql_const.SQL_DOUBLE.value: float, ddbc_sql_const.SQL_REAL.value: float, + ddbc_sql_const.SQL_SS_VECTOR.value: Vector, # Exact numeric types ddbc_sql_const.SQL_DECIMAL.value: decimal.Decimal, ddbc_sql_const.SQL_NUMERIC.value: decimal.Decimal, @@ -1845,6 +1871,8 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state # the cursor associated with hstmt without destroying the # prepared statement, which is the standard ODBC pattern for # re-executing a prepared query. + self._procedure_results = None + self._procedure_rows = None if self.hstmt: logger.debug("execute: Closing cursor for re-execution (reset_cursor=False)") self.hstmt._close_cursor() @@ -2491,6 +2519,17 @@ def _compute_column_type( non_nulls = [v for v in column if v is not None] if not non_nulls: return None, None, None, 0 + vectors = [v for v in non_nulls if isinstance(v, Vector)] + if vectors: + sample = vectors[0] + if len(vectors) != len(non_nulls): + raise TypeError("Vector batch columns must contain only Vector or None") + if any( + v.dimensions != sample.dimensions or v.base_type != sample.base_type + for v in vectors + ): + raise ValueError("Vector batch dimensions and subtype must agree") + return sample, None, None, 0 int_values: list[int] = [] for v in non_nulls: @@ -2669,6 +2708,24 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s ) # Prepare parameter type information + if self._inputsizes: + vector_hints = [ + (index, size, scale) + for index, (sql_type, _, size, scale) in enumerate(self._inputsizes) + if sql_type == ddbc_sql_const.SQL_SS_VECTOR.value and index < param_count + ] + if vector_hints: + normalized_rows = [] + for row in seq_of_parameters: + normalized_row = list(row) + for index, size, scale in vector_hints: + normalized_row[index] = _vector_from_hint( + normalized_row[index], size, scale + ) + normalized_rows.append(normalized_row) + seq_of_parameters = normalized_rows + sample_row = seq_of_parameters[0] + with perf_phase("py::executemany::param_type_detection"): for col_index in range(param_count): column = ( @@ -2735,6 +2792,13 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s paraminfo.inputOutputType = ddbc_sql_const.SQL_PARAM_INPUT.value paraminfo.columnSize = column_size paraminfo.decimalDigits = decimal_digits + if sql_type == ddbc_sql_const.SQL_SS_VECTOR.value: + from mssql_python.vector import _check_shape + + if decimal_digits not in (0, 1): + raise ValueError("Vector subtype must be 0 (FLOAT32) or 1 (FLOAT16)") + _check_shape(column_size, "float16" if decimal_digits else "float32") + paraminfo.columnSize = ddbc_bindings._vector_buffer_size(column_size) paraminfo.isDAE = is_dae # Ensure we never have SQL_C_DEFAULT (0) for C-type @@ -2962,6 +3026,27 @@ def fetchone(self) -> Union[None, Row]: """ self._check_closed() # Check if the cursor is closed + procedure_rows: deque[list[Any]] | None = getattr(self, "_procedure_rows", None) + if procedure_rows is not None: + if not self.description: + raise ProgrammingError("The current procedure result has no rows", "") + if not procedure_rows: + self.rowcount = self._next_row_index + return None + buffered_row = procedure_rows.popleft() + self._increment_rownumber() + self.rowcount = self._next_row_index + column_map, converter_map, lower_map = self._get_column_and_converter_maps() + return Row( + buffered_row, + column_map, + cursor=self, + converter_map=converter_map, + uuid_str_indices=self._uuid_str_indices, + column_map_lower=lower_map, + column_names=self._cached_result_columns, + ) + if self._cached_decoding_generation != self._connection._decoding_generation: self._refresh_decoding_cache() char_enc = self._cached_char_encoding @@ -3095,6 +3180,15 @@ def fetchmany(self, size: Optional[int] = None) -> List[Row]: if size <= 0: return [] + if getattr(self, "_procedure_rows", None) is not None: + rows = [] + for _ in range(size): + row = self.fetchone() + if row is None: + break + rows.append(row) + return rows + if self._cached_decoding_generation != self._connection._decoding_generation: self._refresh_decoding_cache() char_enc = self._cached_char_encoding @@ -3200,6 +3294,9 @@ def fetchall(self) -> List[Row]: List of Row objects. """ self._check_closed() # Check if the cursor is closed + procedure_rows: deque[list[Any]] | None = getattr(self, "_procedure_rows", None) + if procedure_rows is not None: + return self.fetchmany(len(procedure_rows)) if self.description else self.fetchmany(1) if not self._has_result_set and self.description: self._reset_rownumber() @@ -3313,6 +3410,13 @@ def _fetch_arrow_batch( self, batch_size: int, reader_state: ddbc_bindings._ArrowReaderState | None = None ) -> Optional["pyarrow.RecordBatch"]: self._check_closed() # Check if the cursor is closed + if getattr(self, "_procedure_rows", None) is not None or any( + column[1] is Vector for column in (self.description or ()) + ): + raise NotSupportedError( + driver_error="Use row fetch methods for vectors and buffered callproc results", + ddbc_error="Arrow vector conversion is not implemented", + ) pa = self._ensure_pyarrow() if not self._has_result_set and self.description: @@ -3520,6 +3624,10 @@ def nextset(self) -> Optional[bool]: logger.debug("nextset: Moving to next result set") self._check_closed() # Check if the cursor is closed + if getattr(self, "_procedure_results", None) is not None: + self.messages = [] + return self._next_procedure_result() + # Clear messages per DBAPI self.messages = [] @@ -3903,6 +4011,18 @@ def bulkcopy( pycore_connection = None pycore_cursor = None + vector_validation_error: NotSupportedError | None = None + + def _validated_vector_rows( + rows: Iterable[tuple[Any, ...]], + ) -> Iterator[tuple[Any, ...]]: + nonlocal vector_validation_error + try: + yield from self._validate_bulkcopy_vectors(rows) + except NotSupportedError as error: + vector_validation_error = error + raise + try: # Only pass logger to Rust if logging is enabled (performance optimization) pycore_connection = mssql_py_core.PyCoreConnection( @@ -3928,11 +4048,13 @@ def _prepare_row_iterator( if first is None: return iter(()) if isinstance(first, tuple): - return chain((first,), cast("Iterator[tuple[Any, ...]]", it)) + rows = chain((first,), cast("Iterator[tuple[Any, ...]]", it)) + return _validated_vector_rows(rows) if isinstance(first, Row): - return ( + row_values = ( tuple(item._values) for item in chain((first,), cast("Iterator[Row]", it)) ) + return _validated_vector_rows(row_values) raise TypeError( f"bulkcopy data rows must be tuples or Row objects, " f"got {type(first).__name__}" @@ -3954,6 +4076,8 @@ def _prepare_row_iterator( use_internal_transaction=use_internal_transaction, python_logger=logger if is_logging_enabled else None, # Only pass logger if enabled ) + if vector_validation_error is not None: + raise vector_validation_error logger.info( "_bulkcopy: Bulk copy completed successfully - rows_copied=%s, batch_count=%s, elapsed_time=%s", @@ -3972,6 +4096,9 @@ def _prepare_row_iterator( type(e).__name__, str(e), ) + # Rust can replace Python iterator exceptions with RuntimeError. + if vector_validation_error is not None and vector_validation_error is not e: + raise vector_validation_error from e # Re-raise the original exception, preserving its type, args, and # traceback. The finally block scrubs credentials, and Python # tracebacks don't expose local values, so no reconstruction is needed. @@ -4455,6 +4582,40 @@ def tables( logger.error(f"Error executing tables query: {e}") raise + @staticmethod + def _validate_bulkcopy_vectors( + rows: Iterable[tuple[Any, ...]], + ) -> Iterator[tuple[Any, ...]]: + for row in rows: + if any(isinstance(value, Vector) for value in row): + raise NotSupportedError( + driver_error="Vector bulk copy requires the developing pycore vector contract", + ddbc_error="No vector serialization is performed by the Python wrapper", + ) + yield row + + def _next_procedure_result(self) -> bool: + self._cached_column_map = None + self._cached_column_map_lower = None + self._cached_converter_map = None + self._cached_converters_generation = -1 + self._uuid_str_indices = None + self._clear_rownumber() + if not self._procedure_results: + self.description = None + self._column_sql_types = None + self._procedure_rows = deque() + self.rowcount = -1 + return False + metadata, rows, rowcount = self._procedure_results.popleft() + self._initialize_description(metadata) + self._procedure_rows = deque(rows) + self.rowcount = rowcount + if self.description: + self._reset_rownumber() + self._uuid_str_indices = self._compute_uuid_str_indices() + return True + def callproc( self, procname: str, parameters: Optional[Sequence[Any]] = None ) -> Optional[Sequence[Any]]: @@ -4469,13 +4630,43 @@ def callproc( A sequence containing the result parameters (input parameters unchanged, output parameters with their new values) - Raises: - NotSupportedError: This method is not yet implemented + Vector outputs use OutputParameter(dimensions, base_type) or + InputOutputParameter(Vector(...)). Results are buffered so output values + are available on return; fetch methods and nextset() expose those results. """ - raise NotSupportedError( - driver_error="callproc() is not yet implemented", - ddbc_error="Stored procedure calls are not currently supported", + self._check_closed() + if not isinstance(procname, str) or not procname or "\0" in procname: + raise ValueError("procname must be a non-empty procedure name") + parts = procname.split(".") + if not 1 <= len(parts) <= 3 or any(not part for part in parts): + raise ValueError( + "procname must be a procedure, schema.procedure or database.schema.procedure" + ) + if isinstance(parameters, (str, bytes, bytearray)): + raise TypeError("parameters must be a sequence of parameter values") + values = list(parameters) if parameters is not None else [] + name = ".".join("[" + part.replace("]", "]]") + "]" for part in parts) + query = "{CALL " + name + "(" + ",".join("?" for _ in values) + ")}" + self._soft_reset_cursor() + self.messages = [] + if self._cached_decoding_generation != self._connection._decoding_generation: + self._refresh_decoding_cache() + outputs, results = ddbc_bindings.DDBCCallProc( + self.hstmt, + query, + values, + self._get_encoding_settings(), + self.messages, + self._cached_char_encoding, + self._cached_wchar_encoding, + self._cached_char_ctype, ) + self.last_executed_stmt = "" + self.is_stmt_prepared = [False] + self._reset_inputsizes() + self._procedure_results = deque(results) + self._next_procedure_result() + return outputs def setoutputsize(self, size: int, column: Optional[int] = None) -> None: """ diff --git a/mssql_python/ddbc_bindings.py b/mssql_python/ddbc_bindings.py index ea98b6416..d326572a2 100644 --- a/mssql_python/ddbc_bindings.py +++ b/mssql_python/ddbc_bindings.py @@ -23,6 +23,7 @@ _disable_native_fetch_install as _disable_native_fetch_install, _fetch_conversion_ineligible as _fetch_conversion_ineligible, _fetch_observer as _fetch_observer, + _detect_param_types as _detect_param_types, _get_odbc_driver_path as _get_odbc_driver_path, _invalidate_native_fetch as _invalidate_native_fetch, _make_row_mapping_base as _make_row_mapping_base, @@ -30,6 +31,7 @@ _native_fetch_methods as _native_fetch_methods, _set_odbc_provider as _set_odbc_provider, _watch_fetch_dependencies as _watch_fetch_dependencies, + _vector_buffer_size as _vector_buffer_size, ) diff --git a/mssql_python/mssql_python.pyi b/mssql_python/mssql_python.pyi index 9fe22dcf1..ff74b6c6b 100644 --- a/mssql_python/mssql_python.pyi +++ b/mssql_python/mssql_python.pyi @@ -24,6 +24,14 @@ from .logging import MSSQLLogger import datetime import logging import pyarrow +from .vector import ( + Vector as Vector, + OutputParameter as OutputParameter, + InputOutputParameter as InputOutputParameter, +) +from .table_parameter import TableValuedParameter as TableValuedParameter + +SQL_SS_VECTOR: int # GLOBALS - DB-API 2.0 Required Module Globals # https://www.python.org/dev/peps/pep-0249/#module-interface diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index cc07dd852..811675712 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -489,6 +489,16 @@ static SQLRETURN PreResolveUnknownNullTypes(SqlHandle& handle, SQLHANDLE hStmt, paramInfo.paramSQLType = resolved.sqlType; paramInfo.columnSize = resolved.columnSize; paramInfo.decimalDigits = resolved.decimalDigits; + if (resolved.sqlType == SQL_SS_VECTOR) { + if (resolved.columnSize < sizeof(SQL_SS_VECTOR_STRUCT) || + (resolved.columnSize - sizeof(SQL_SS_VECTOR_STRUCT)) % sizeof(float) != 0) + throw py::value_error("ODBC returned invalid vector parameter metadata"); + size_t dimensions = + (resolved.columnSize - sizeof(SQL_SS_VECTOR_STRUCT)) / sizeof(float); + paramInfo.vectorBinding = + std::make_shared(dimensions, resolved.decimalDigits); + paramInfo.paramCType = SQL_C_SS_VECTOR; + } } return SQL_SUCCESS; } @@ -541,6 +551,11 @@ static SQLRETURN PreResolveUdtTypes(SqlHandle& handle, SQLHANDLE hStmt, // rebind over those records instead of issuing SQL_RESET_PARAMS first. An // unchanged IPD lets msodbcsql and mssql-odbc keep their prepared handle and // send sp_execute; any difference resets first, as before. +SQLRETURN BindTableParameter(SqlHandle& handle, SQLHANDLE hStmt, SQLUSMALLINT ordinal, + py::handle parameter, + std::vector>& buffers, + const std::string& charEncoding); + SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& params, std::vector& paramInfos, std::vector>& paramBuffers, @@ -565,7 +580,10 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par signature.push_back({info.inputOutputType, info.paramCType, info.paramSQLType, info.columnSize, info.decimalDigits, info.isDAE}); } - if (handle.paramsBound && (signature.empty() || signature != handle.boundParamSignature)) { + const bool hasTable = std::any_of(paramInfos.begin(), paramInfos.end(), + [](const ParamInfo& info) { return info.paramSQLType == SQL_SS_TABLE; }); + if (handle.paramsBound && + (signature.empty() || signature != handle.boundParamSignature || hasTable)) { SQLRETURN resetRc = ResetParamBindings(handle, hStmt); if (!SQL_SUCCEEDED(resetRc)) return resetRc; @@ -587,8 +605,28 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par SQLLEN bufferLength = 0; SQLLEN* strLenOrIndPtr = nullptr; - // TODO: Add more data types like money, guid, interval, TVPs etc. + if (paramInfo.paramSQLType == SQL_SS_TABLE) { + SQLRETURN rc = BindTableParameter(handle, hStmt, + static_cast(paramIndex + 1), param, paramBuffers, charEncoding); + if (!SQL_SUCCEEDED(rc)) return rc; + continue; + } + + // TODO: Add more data types like money and intervals. switch (paramInfo.paramCType) { + case SQL_C_SS_VECTOR: { + if (!paramInfo.vectorBinding) { + if (param.is_none()) + throw py::value_error("NULL vector parameters require dimensions and subtype"); + paramInfo.vectorBinding = std::make_shared(param); + } + auto binding = paramInfo.vectorBinding; + paramBuffers.push_back(binding); + dataPtr = &binding->value; + bufferLength = static_cast(VectorBufferSize(binding->elements.size())); + strLenOrIndPtr = &binding->indicator; + break; + } case SQL_C_CHAR: { if (!py::isinstance(param) && !py::isinstance(param) && !py::isinstance(param)) { @@ -1745,11 +1783,13 @@ SQLRETURN SqlHandle::freeHandle() { describeCache.clear(); if (_implicitly_freed || (_cleanupState && _cleanupState->disconnected)) { _handle = nullptr; + boundParamBuffers.clear(); return SQL_SUCCESS; } SQLRETURN ret = SQLFreeHandle_ptr ? SQLFreeHandle_ptr(_type, _handle) : SQL_INVALID_HANDLE; if (SQL_SUCCEEDED(ret)) { _handle = nullptr; + boundParamBuffers.clear(); } else if (_type == SQL_HANDLE_STMT && _cleanupState) { _cleanupState->childFreeFailed = true; } @@ -2406,7 +2446,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& "param_count=%zu, param_set_size=%zu", columnwise_params.size(), paramSetSize); - std::vector> tempBuffers; + auto& tempBuffers = paramBuffers; try { SQLRETURN describeRc = PreResolveUdtTypes(handle, hStmt, paramInfos); @@ -2435,6 +2475,44 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& SQLLEN* strLenOrIndArray = nullptr; SQLLEN bufferLength = 0; switch (info.paramCType) { + case SQL_C_SS_VECTOR: { + size_t dimensions = 0; + int subtype = info.decimalDigits; + for (auto item : columnValues) { + if (!item.is_none()) { + dimensions = item.attr("dimensions").cast(); + subtype = item.attr("base_type").cast() == "float16" ? 1 : 0; + break; + } + } + if (dimensions == 0) { + if (info.columnSize < sizeof(SQL_SS_VECTOR_STRUCT)) + throw py::value_error("NULL vector arrays require vector metadata"); + dimensions = (info.columnSize - sizeof(SQL_SS_VECTOR_STRUCT)) / sizeof(float); + } + ValidateVectorShape(dimensions, subtype); + if (info.columnSize != VectorBufferSize(dimensions) || + info.decimalDigits != subtype) + throw py::value_error("Vector array must match its declared dimensions and subtype"); + auto structs = AllocateParamBufferArray( + tempBuffers, paramSetSize); + strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); + for (size_t i = 0; i < paramSetSize; ++i) { + auto binding = columnValues[i].is_none() + ? std::make_shared(dimensions, subtype) + : std::make_shared(columnValues[i]); + if (binding->value.dimension != dimensions || binding->value.type != subtype) + throw py::value_error("Vector array dimensions and subtype must agree"); + structs[i] = binding->value; + strLenOrIndArray[i] = binding->indicator; + tempBuffers.push_back(binding); + } + dataPtr = structs; + bufferLength = static_cast(VectorBufferSize(dimensions)); + info.columnSize = static_cast(bufferLength); + info.decimalDigits = static_cast(subtype); + break; + } case SQL_C_LONG: { LOG("BindParameterArray: Binding SQL_C_LONG array - " "param_index=%d, count=%zu", @@ -3033,13 +3111,100 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& "buffers"); throw; } - paramBuffers.insert(paramBuffers.end(), tempBuffers.begin(), tempBuffers.end()); LOG("BindParameterArray: Successfully bound all parameters - " "total_params=%zu, buffer_count=%zu", columnwise_params.size(), paramBuffers.size()); return SQL_SUCCESS; } +namespace { +SQLRETURN BindTableParameter(SqlHandle& handle, SQLHANDLE hStmt, SQLUSMALLINT ordinal, + py::handle parameter, + std::vector>& buffers, + const std::string& charEncoding) { + auto rows = parameter.attr("rows").cast(); + auto columns = parameter.attr("columns").cast(); + auto name = parameter.attr("type_name").cast(); + auto separator = name.find(u'.'); + std::u16string schema; + if (separator != std::u16string::npos) { + schema = name.substr(0, separator); + name = name.substr(separator + 1); + } + auto typeName = AllocateParamBuffer(buffers, name); + auto rowCount = AllocateParamBuffer(buffers, static_cast(rows.size())); + SQLRETURN rc = SQLBindParameter_ptr(hStmt, ordinal, SQL_PARAM_INPUT, SQL_C_DEFAULT, + SQL_SS_TABLE, std::max(1, rows.size()), 0, + reinterpretU16stringAsSqlWChar(*typeName), SQL_NTS, rowCount); + if (!SQL_SUCCEEDED(rc)) return rc; + if (!schema.empty()) { + SQLHDESC ipd = nullptr; + rc = SQLGetStmtAttr_ptr(hStmt, SQL_ATTR_IMP_PARAM_DESC, &ipd, 0, nullptr); + if (!SQL_SUCCEEDED(rc)) return rc; + auto schemaName = AllocateParamBuffer(buffers, schema); + rc = SQLSetDescField_ptr(ipd, ordinal, SQL_CA_SS_SCHEMA_NAME, + reinterpretU16stringAsSqlWChar(*schemaName), SQL_NTS); + if (!SQL_SUCCEEDED(rc)) return rc; + } + py::list samples, overrides, values; + auto cTypes = py::module_::import("mssql_python.cursor") + .attr("Cursor").attr("_get_sql_to_c_type_map")().cast(); + for (size_t col = 0; col < columns.size(); ++col) { + auto spec = columns[col].cast(); + int sqlType = spec[0].cast(); + py::list column; + py::object sample = py::none(); + for (auto row : rows) { + py::object cell = row.cast()[col]; + column.append(cell); + if (sample.is_none() && !cell.is_none()) sample = cell; + } + if (rows.empty()) column.append(py::none()); + samples.append(sample); + overrides.append(py::make_tuple(sqlType, + cTypes[py::int_(sqlType)], spec[1], spec[2])); + values.append(column); + } + // Apply the same explicit coercions to every cell, not just the sample row. + for (size_t rowIndex = 0; rowIndex < rows.size(); ++rowIndex) { + py::list normalized(rows[rowIndex]); + DetectParamTypes(normalized.ptr(), overrides.ptr()); + for (size_t col = 0; col < columns.size(); ++col) + values[col].cast()[rowIndex] = normalized[col]; + } + auto infos = DetectParamTypes(samples.ptr(), overrides.ptr()); + for (auto& info : infos) { + if (info.isDAE) + throw py::value_error("TVP column values must fit their declared inline column sizes"); + } + rc = SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, + reinterpret_cast(static_cast(ordinal)), SQL_IS_INTEGER); + if (!SQL_SUCCEEDED(rc)) return rc; + // Always restore focus, including when Python conversion or binding throws. + struct FocusGuard { + SQLHANDLE statement; + bool active = true; + ~FocusGuard() { + if (!active) return; + SQLRETURN result = SQLSetStmtAttr_ptr(statement, SQL_SOPT_SS_PARAM_FOCUS, nullptr, + SQL_IS_INTEGER); + if (!SQL_SUCCEEDED(result)) LOG_ERROR("Failed to restore TVP parameter focus"); + } + } focus{hStmt}; + rc = SQLSetStmtAttr_ptr(hStmt, SQL_ATTR_PARAMSET_SIZE, + reinterpret_cast(std::max(1, rows.size())), 0); + if (!SQL_SUCCEEDED(rc)) return rc; + rc = BindParameterArray(handle, hStmt, values, infos, std::max(1, rows.size()), + buffers, charEncoding); + if (!SQL_SUCCEEDED(rc)) + py::module_::import("mssql_python.helpers").attr("check_error")( + SQL_HANDLE_STMT, py::cast(&handle, py::return_value_policy::reference), rc); + SQLRETURN restore = SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, nullptr, SQL_IS_INTEGER); + focus.active = false; + return SQL_SUCCEEDED(rc) ? restore : rc; +} +} // namespace + SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::u16string& query, const py::list& columnwise_params, std::vector& paramInfos, size_t paramSetSize, @@ -3090,7 +3255,7 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::u16 LOG("SQLExecuteMany: Using array binding (non-DAE) - calling " "BindParameterArray with encoding '%s'", charEncoding.c_str()); - std::vector> paramBuffers; + auto& paramBuffers = statementHandle->boundParamBuffers; rc = BindParameterArray(*statementHandle, hStmt, columnwise_params, paramInfos, paramSetSize, paramBuffers, charEncoding); if (!SQL_SUCCEEDED(rc)) { @@ -3548,7 +3713,8 @@ SQLSMALLINT MapVariantCTypeToSQLType(SQLLEN variantCType) { // Helper function to check if a column requires SQLGetData streaming (LOB or sql_variant) static inline bool IsLobOrVariantColumn(SQLSMALLINT dataType, SQLULEN columnSize) { - return dataType == SQL_SS_VARIANT || + // Vector buffers contain pointers; use the existing per-row GetData path. + return dataType == SQL_SS_VECTOR || dataType == SQL_SS_VARIANT || ((dataType == SQL_WVARCHAR || dataType == SQL_WLONGVARCHAR || dataType == SQL_VARCHAR || dataType == SQL_LONGVARCHAR || dataType == SQL_VARBINARY || dataType == SQL_LONGVARBINARY || dataType == SQL_SS_XML || dataType == SQL_SS_UDT) && @@ -3641,6 +3807,27 @@ static SQLRETURN GetRowData(const SqlHandlePtr& StatementHandle, SQLUSMALLINT co } } + if (dataType == SQL_SS_VECTOR) { + if (columnSize < sizeof(SQL_SS_VECTOR_STRUCT) || + (columnSize - sizeof(SQL_SS_VECTOR_STRUCT)) % sizeof(float) != 0) + throw py::value_error("ODBC returned invalid vector column metadata"); + SQLLEN subtype = 0; + ret = SQLColAttribute_ptr(hStmt, i, SQL_DESC_SCALE, nullptr, 0, nullptr, &subtype); + CaptureFetchDiagnostics(hStmt, ret, messages); + if (!SQL_SUCCEEDED(ret)) return ret; + size_t dimensions = (columnSize - sizeof(SQL_SS_VECTOR_STRUCT)) / sizeof(float); + VectorBinding binding(dimensions, static_cast(subtype)); + { + py::gil_scoped_release release; + ret = SQLGetData_ptr(hStmt, i, SQL_C_SS_VECTOR, &binding.value, + static_cast(VectorBufferSize(dimensions)), &binding.indicator); + } + CaptureFetchDiagnostics(hStmt, ret, messages); + if (!SQL_SUCCEEDED(ret)) return ret; + row.append(binding.toPython()); + continue; + } + // Preprocess sql_variant: detect underlying type to route to correct conversion logic SQLSMALLINT effectiveDataType = dataType; if (dataType == SQL_SS_VARIANT) { @@ -6636,6 +6823,126 @@ SQLRETURN FetchOne_wrap(SqlHandlePtr StatementHandle, py::list& row, return ret; } +py::tuple CallProcedure_wrap(SqlHandlePtr statement, const std::u16string& query, + py::list parameters, const py::dict& encodingSettings, + py::list messages, const std::string& charEncoding, + const std::string& wcharEncoding, int charCtype) { + SQLHSTMT hStmt = statement->get(); + statement->resultMetadata.clear(); + py::list values = parameters.attr("copy")().cast(); + auto vectorModule = py::module_::import("mssql_python.vector"); + auto outputClass = vectorModule.attr("OutputParameter"); + auto inoutClass = vectorModule.attr("InputOutputParameter"); + std::vector directions(parameters.size(), SQL_PARAM_INPUT); + for (size_t i = 0; i < parameters.size(); ++i) { + auto item = parameters[i]; + if (py::isinstance(item, outputClass)) { + directions[i] = py::isinstance(item, inoutClass) + ? SQL_PARAM_INPUT_OUTPUT : SQL_PARAM_OUTPUT; + if (directions[i] == SQL_PARAM_INPUT_OUTPUT) { + if (item.attr("value").is_none()) + throw py::value_error("INPUT_OUTPUT parameters require a non-NULL Vector"); + values[i] = item.attr("value"); + } else { + size_t dimensions = item.attr("dimensions").cast(); + auto baseType = item.attr("base_type").cast(); + ValidateVectorShape(dimensions, baseType == "float16" ? 1 : 0); + py::tuple zeros(dimensions); + for (size_t j = 0; j < dimensions; ++j) zeros[j] = py::float_(0.0); + values[i] = vectorModule.attr("Vector")(zeros, baseType); + } + } + } + auto infos = DetectParamTypes(values.ptr(), Py_None); + for (size_t i = 0; i < infos.size(); ++i) { + infos[i].inputOutputType = directions[i]; + if (infos[i].isDAE) + throw py::value_error("callproc input strings/binary must fit inline parameter limits"); + } + SQLRETURN rc = ResetParamBindings(*statement, hStmt); + CheckFetchError(statement, rc); + { + py::gil_scoped_release release; + rc = SQLPrepare_ptr(hStmt, reinterpretU16stringAsSqlWChar(query), SQL_NTS); + } + CheckFetchError(statement, rc); + statement->clearDescribeCache(); + const bool hasUnknownNull = std::any_of(infos.begin(), infos.end(), [](const ParamInfo& info) { + return info.paramCType == SQL_C_DEFAULT && info.paramSQLType == SQL_UNKNOWN_TYPE; + }); + if (hasUnknownNull) { + rc = PreResolveUnknownNullTypes(*statement, hStmt, infos, &values); + CheckFetchError(statement, rc); + // SQLDescribeParam can leave its metadata as the no-result RPC's IRD. + // Reprepare after resolving types so only procedure results are exposed. + { + py::gil_scoped_release release; + rc = SQLPrepare_ptr(hStmt, reinterpretU16stringAsSqlWChar(query), SQL_NTS); + } + CheckFetchError(statement, rc); + } + auto& buffers = statement->boundParamBuffers; + // ODBC may retain input and output pointers until all results are consumed. + struct ParameterGuard { + SqlHandle& handle; + SQLHSTMT statement; + bool active = true; + ~ParameterGuard() { + if (!active) return; + SQLRETURN rc = ResetParamBindings(handle, statement); + if (!SQL_SUCCEEDED(rc)) LOG_ERROR("Failed to reset procedure parameter bindings"); + } + } reset{*statement, hStmt}; + std::string encoding = "utf-8"; + if (encodingSettings.contains("encoding") && encodingSettings.contains("ctype") && + encodingSettings["ctype"].cast() == SQL_C_CHAR) + encoding = encodingSettings["encoding"].cast(); + rc = BindParameters(*statement, hStmt, values, infos, buffers, encoding); + CheckFetchError(statement, rc); + { + py::gil_scoped_release release; + rc = SQLExecute_ptr(hStmt); + } + CheckFetchError(statement, rc); + CaptureFetchDiagnostics(hStmt, rc, messages); + py::list results; + while (rc != SQL_NO_DATA) { + py::list metadata, rows; + rc = SQLDescribeCol_wrap(statement, metadata, messages); + CheckFetchError(statement, rc); + SQLLEN rowCount = -1; + rc = SQLRowCount_ptr(hStmt, &rowCount); + CheckFetchError(statement, rc); + if (!metadata.empty()) { + rc = FetchAll_wrap(statement, rows, charEncoding, wcharEncoding, charCtype, messages); + CheckFetchError(statement, rc); + rowCount = -1; + } + results.append(py::make_tuple(metadata, rows, rowCount)); + statement->resultMetadata.clear(); + { + py::gil_scoped_release release; + rc = SQLMoreResults_ptr(hStmt); + } + CheckFetchError(statement, rc); + CaptureFetchDiagnostics(hStmt, rc, messages); + } + py::list output; + for (size_t i = 0; i < infos.size(); ++i) { + if (directions[i] == SQL_PARAM_INPUT) { + output.append(parameters[i]); + } else { + auto result = infos[i].vectorBinding->toPython(); + parameters[i].attr("value") = result; + output.append(result); + } + } + rc = ResetParamBindings(*statement, hStmt); + CheckFetchError(statement, rc); + reset.active = false; + return py::make_tuple(output, results); +} + // Wrap SQLMoreResults SQLRETURN SQLMoreResults_wrap(SqlHandlePtr StatementHandle) { PERF_TIMER("SQLMoreResults_wrap"); @@ -6734,8 +7041,14 @@ PYBIND11_MODULE(ddbc_bindings, m) { m.attr("ARCHITECTURE") = ARCHITECTURE; m.attr("SQL_NO_TOTAL") = static_cast(SQL_NO_TOTAL); + m.def("_vector_buffer_size", &VectorBufferSize); + m.def("DDBCCallProc", &CallProcedure_wrap); // Expose the C++ functions to Python + m.def("_detect_param_types", [](py::list params, py::object sizes) { + py::list copy = params.attr("copy")().cast(); + return DetectParamTypes(copy.ptr(), sizes.ptr()); + }, py::arg("params"), py::arg("sizes") = py::none()); m.def("ThrowStdException", &ThrowStdException); m.def("GetDriverPathCpp", &GetDriverPathCpp, "Get the path to the ODBC driver"); m.def("_get_odbc_driver_path", &GetDriverPathForProviderCpp, diff --git a/mssql_python/pybind/ddbc_bindings.h b/mssql_python/pybind/ddbc_bindings.h index 2f1aa4fd4..729f84251 100644 --- a/mssql_python/pybind/ddbc_bindings.h +++ b/mssql_python/pybind/ddbc_bindings.h @@ -56,6 +56,11 @@ using py::literals::operator""_a; #define SQL_SS_UDT (-151) #define SQL_SS_VARIANT (-150) #define SQL_CA_SS_VARIANT_TYPE (1215) +#define SQL_SS_VECTOR (-156) +#define SQL_C_SS_VECTOR (0x4002) +#define SQL_SS_TABLE (-153) +#define SQL_SOPT_SS_PARAM_FOCUS (1236) +#define SQL_CA_SS_SCHEMA_NAME (1226) // Include logger bridge for LOG macros #include "logger_bridge.hpp" diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index 172a98b51..f6e88c573 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -23,6 +23,7 @@ #include "ddbc_bindings.h" // ParamInfo consumers, SQL Server ODBC constants #include "py_ref.hpp" // steal / borrow #include "py_type_cache.hpp" // PyTypeCache::get_*_class +#include "vector_support.hpp" #include #include // CPython datetime API (PyDateTime_Check, PyDateTime_GET_*, etc.) @@ -59,6 +60,7 @@ struct ParamInfo { // move operations are all correct and this struct needs no rule-of-five. py::object dataPtr; Py_ssize_t utf16Len = 0; // UTF-16 code unit count for string params + std::shared_ptr vectorBinding; }; #ifdef __GNUC__ #pragma GCC diagnostic pop @@ -243,6 +245,25 @@ inline void ApplyInputSizeOverride(PyObject* params, PyObject* inputSize, Py_ssi } PyObject* obj = PyList_GET_ITEM(params, index); + if (info.paramSQLType == SQL_SS_VECTOR) { + auto dimensions = static_cast(info.columnSize); + py::object vector = py::module_::import("mssql_python.vector") + .attr("_vector_from_hint")(borrow(obj), dimensions, info.decimalDigits); + if (PyList_SetItem(params, index, vector.release().ptr()) != 0) { + throw py::error_already_set(); + } + obj = PyList_GET_ITEM(params, index); + info.paramCType = SQL_C_SS_VECTOR; + info.vectorBinding = obj == Py_None + ? std::make_shared(dimensions, info.decimalDigits) + : std::make_shared(borrow(obj)); + if (info.vectorBinding->value.dimension != dimensions || + info.vectorBinding->value.type != info.decimalDigits) + throw py::value_error("Vector dimensions and subtype must match setinputsizes"); + info.columnSize = VectorBufferSize(dimensions); + info.isDAE = false; + return; + } if (obj == Py_None) { info.paramCType = SQL_C_DEFAULT; return; @@ -318,6 +339,25 @@ inline std::vector DetectParamTypes(PyObject* params, PyObject* input PyObject* obj = PyList_GET_ITEM(params, i); + int isVector = PyObject_IsInstance(obj, PyTypeCache::vector_class); + if (isVector < 0) throw py::error_already_set(); + if (isVector) { + info.vectorBinding = std::make_shared(borrow(obj)); + info.paramSQLType = SQL_SS_VECTOR; + info.paramCType = SQL_C_SS_VECTOR; + info.columnSize = VectorBufferSize(info.vectorBinding->elements.size()); + info.decimalDigits = static_cast(info.vectorBinding->value.type); + continue; + } + int isTvp = PyObject_IsInstance(obj, PyTypeCache::tvp_class); + if (isTvp < 0) throw py::error_already_set(); + if (isTvp) { + info.paramSQLType = SQL_SS_TABLE; + info.paramCType = SQL_C_DEFAULT; + info.columnSize = static_cast(py::len(borrow(obj).attr("rows"))); + continue; + } + // --- None --- if (obj == Py_None) { info.paramSQLType = SQL_UNKNOWN_TYPE; diff --git a/mssql_python/pybind/py_type_cache.hpp b/mssql_python/pybind/py_type_cache.hpp index ef06ef1ce..50c01f4d0 100644 --- a/mssql_python/pybind/py_type_cache.hpp +++ b/mssql_python/pybind/py_type_cache.hpp @@ -24,6 +24,8 @@ inline PyObject* date_class = nullptr; inline PyObject* time_class = nullptr; inline PyObject* decimal_class = nullptr; inline PyObject* uuid_class = nullptr; +inline PyObject* vector_class = nullptr; +inline PyObject* tvp_class = nullptr; inline bool cache_initialized = false; // Import a module and extract an attribute. Returns a new reference. @@ -53,6 +55,8 @@ inline void initialize() { py::object dec_cls = steal(import_attr("decimal", "Decimal")); py::object uuid_cls = steal(import_attr("uuid", "UUID")); + py::object vec_cls = steal(import_attr("mssql_python.vector", "Vector")); + py::object table_cls = steal(import_attr("mssql_python.table_parameter", "TableValuedParameter")); // Commit to globals — all acquisitions succeeded. datetime_class = dt_cls.release().ptr(); @@ -60,6 +64,8 @@ inline void initialize() { time_class = time_cls.release().ptr(); decimal_class = dec_cls.release().ptr(); uuid_class = uuid_cls.release().ptr(); + vector_class = vec_cls.release().ptr(); + tvp_class = table_cls.release().ptr(); cache_initialized = true; } diff --git a/mssql_python/pybind/vector_support.hpp b/mssql_python/pybind/vector_support.hpp new file mode 100644 index 000000000..f03a9c3f9 --- /dev/null +++ b/mssql_python/pybind/vector_support.hpp @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +#pragma once + +#include "ddbc_bindings.h" +#include +#include + +enum SQLVECTORTYPE { SQL_VECTOR_TYPE_FLOAT32 = 0, SQL_VECTOR_TYPE_FLOAT16 = 1 }; + +#pragma pack(push, 8) +struct SQL_SS_VECTOR_STRUCT { + SQLSMALLINT dimension; + SQLVECTORTYPE type; + union { + float* f32; + float* f16; + } data; +}; +#pragma pack(pop) + +inline size_t VectorBufferSize(size_t dimensions) { + return sizeof(SQL_SS_VECTOR_STRUCT) + dimensions * sizeof(float); +} + +inline void ValidateVectorShape(size_t dimensions, int subtype) { + if (subtype != 0 && subtype != 1) + throw py::value_error("Invalid vector subtype; expected FLOAT32 (0) or FLOAT16 (1)"); + if (dimensions == 0 || dimensions > (subtype == 0 ? 1998u : 3996u)) + throw py::value_error("Invalid vector dimensions for the selected subtype"); +} + +struct VectorBinding { + std::vector elements; + SQL_SS_VECTOR_STRUCT value{}; + SQLLEN indicator = SQL_NULL_DATA; + + VectorBinding(size_t dimensions, int subtype) { + ValidateVectorShape(dimensions, subtype); + elements.resize(dimensions); + value.dimension = static_cast(dimensions); + value.type = static_cast(subtype); + value.data.f32 = elements.data(); + } + + explicit VectorBinding(py::handle source) + : VectorBinding(source.attr("dimensions").cast(), + source.attr("base_type").cast() == "float16" ? 1 : 0) { + auto values = source.attr("values").cast(); + if (values.size() != elements.size()) + throw py::value_error("Vector dimensions do not match its values"); + for (size_t i = 0; i < elements.size(); ++i) { + double number = values[i].cast(); + if (!std::isfinite(number) || std::abs(number) > std::numeric_limits::max()) + throw py::value_error("Vector elements must be finite float32 values"); + elements[i] = static_cast(number); + } + indicator = static_cast(VectorBufferSize(elements.size())); + } + + VectorBinding(const VectorBinding&) = delete; + VectorBinding& operator=(const VectorBinding&) = delete; + + py::object toPython() const { + if (indicator == SQL_NULL_DATA) return py::none(); + ValidateVectorShape(value.dimension, value.type); + if (static_cast(value.dimension) > elements.size() || + indicator != static_cast(VectorBufferSize(value.dimension))) + throw py::value_error("ODBC returned an invalid vector length"); + py::tuple values(value.dimension); + for (size_t i = 0; i < static_cast(value.dimension); ++i) + values[i] = py::float_(elements[i]); + return py::module_::import("mssql_python.vector") + .attr("Vector")(values, value.type == SQL_VECTOR_TYPE_FLOAT16 ? "float16" : "float32"); + } +}; diff --git a/mssql_python/table_parameter.py b/mssql_python/table_parameter.py new file mode 100644 index 000000000..ff640747a --- /dev/null +++ b/mssql_python/table_parameter.py @@ -0,0 +1,70 @@ +"""Explicit table-valued parameter schema and immutable row snapshots.""" + +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from typing import Any + +from .constants import SQLTypes, ConstantsDDBC +from .vector import _check_shape, _vector_from_hint + + +@dataclass(frozen=True, init=False) +class TableValuedParameter: + """A READONLY TVP. + + columns contains (sql_type, size, scale) hints, as for setinputsizes(). + For vector columns, size is the dimension count and scale is 0 (FLOAT32) + or 1 (FLOAT16). Vector cells accept Vector, list, or None; lists are + snapshotted as Vectors. Explicit schema also supports empty/all-NULL tables. + """ + + type_name: str + columns: tuple[tuple[int, int, int], ...] + rows: tuple[tuple[Any, ...], ...] + + def __init__( + self, + type_name: str, + rows: Iterable[Sequence[Any]], + columns: Sequence[tuple[int, int, int]], + ) -> None: + if not isinstance(type_name, str) or not type_name or "\0" in type_name: + raise ValueError("type_name must be a non-empty SQL table type name") + parts = type_name.split(".") + if len(parts) not in (1, 2) or any( + not part or not all(c.isalnum() or c in "_@" for c in part) for part in parts + ): + raise ValueError("type_name must be an unquoted type or schema.type name") + schema = tuple(tuple(column) for column in columns) + if not schema: + raise ValueError("columns must contain at least one column") + valid = SQLTypes.get_valid_types() + for column in schema: + if len(column) != 3 or any( + isinstance(item, bool) or not isinstance(item, int) for item in column + ): + raise ValueError("columns must contain (sql_type, size, scale) integer tuples") + sql_type, size, scale = column + if sql_type not in valid or size < 0 or scale < 0: + raise ValueError("invalid TVP column type, size or scale") + if sql_type in ( + ConstantsDDBC.SQL_SS_UDT.value, + ConstantsDDBC.SQL_SS_VARIANT.value, + ): + raise ValueError("UDT and sql_variant TVP columns are not supported") + if sql_type == ConstantsDDBC.SQL_SS_VECTOR.value: + if scale not in (0, 1): + raise ValueError("vector subtype must be 0 (FLOAT32) or 1 (FLOAT16)") + _check_shape(size, "float16" if scale else "float32") + snapshot = [] + for row in rows: + values = list(row) + if len(values) != len(schema): + raise ValueError("TVP rows must match the number of columns") + for index, (sql_type, size, scale) in enumerate(schema): + if sql_type == ConstantsDDBC.SQL_SS_VECTOR.value: + values[index] = _vector_from_hint(values[index], size, scale) + snapshot.append(tuple(values)) + object.__setattr__(self, "type_name", type_name) + object.__setattr__(self, "columns", schema) + object.__setattr__(self, "rows", tuple(snapshot)) diff --git a/mssql_python/vector.py b/mssql_python/vector.py new file mode 100644 index 000000000..56f0530fa --- /dev/null +++ b/mssql_python/vector.py @@ -0,0 +1,116 @@ +"""Native SQL Server vectors. Half-precision wire conversion belongs to the backend.""" + +from collections.abc import Iterable, Iterator, Sequence +from dataclasses import dataclass +import math +import struct +from numbers import Real +from typing import Literal, Optional, Union, overload + +VectorBaseType = Literal["float32", "float16"] +_FLOAT32_MAX = 3.4028234663852886e38 +_FLOAT16_MAX = 65504.0 +_FLOAT32 = struct.Struct("=f") + + +def _check_shape(dimensions: int, base_type: VectorBaseType) -> None: + if base_type not in ("float32", "float16"): + raise ValueError("base_type must be 'float32' or 'float16'") + limit = 1998 if base_type == "float32" else 3996 + if isinstance(dimensions, bool) or not isinstance(dimensions, int): + raise TypeError("dimensions must be an integer") + if not 1 <= dimensions <= limit: + raise ValueError(f"{base_type} vector dimensions must be between 1 and {limit}") + + +@dataclass(frozen=True, init=False) +class Vector(Sequence[float]): + """An immutable vector with explicit SQL storage subtype. + + Values are normalized to float32 and exposed as Python floats. base_type + selects SQL storage, not a Python scalar type. FLOAT16 range is checked + conservatively; the backend still performs half-precision conversion. + """ + + values: tuple[float, ...] + base_type: VectorBaseType + + def __init__(self, values: Iterable[float], base_type: VectorBaseType = "float32") -> None: + _check_shape(1, base_type) + if isinstance(values, (str, bytes, bytearray)): + raise TypeError("vector values must be an iterable of real numbers") + converted: list[float] = [] + for index, value in enumerate(values): + if isinstance(value, bool) or not isinstance(value, Real): + raise TypeError( + f"vector element at index {index} must be a real number, not bool or None" + ) + if abs(value) > _FLOAT32_MAX: + raise ValueError(f"vector element at index {index} must be a finite float32 value") + number = float(value) + if not math.isfinite(number) or abs(number) > _FLOAT32_MAX: + raise ValueError(f"vector element at index {index} must be a finite float32 value") + if base_type == "float16" and abs(number) > _FLOAT16_MAX: + raise ValueError( + f"vector element at index {index} exceeds the finite float16 range" + ) + converted.append(_FLOAT32.unpack(_FLOAT32.pack(number))[0]) + _check_shape(len(converted), base_type) + _check_shape(len(converted), base_type) + object.__setattr__(self, "values", tuple(converted)) + object.__setattr__(self, "base_type", base_type) + + @property + def dimensions(self) -> int: + return len(self.values) + + def __len__(self) -> int: + return len(self.values) + + def __iter__(self) -> Iterator[float]: + return iter(self.values) + + @overload + def __getitem__(self, index: int) -> float: ... + + @overload + def __getitem__(self, index: slice) -> tuple[float, ...]: ... + + def __getitem__(self, index: Union[int, slice]) -> Union[float, tuple[float, ...]]: + return self.values[index] + + +def _vector_from_hint(value: object, dimensions: int, subtype: int) -> Optional[Vector]: + if subtype not in (0, 1): + raise ValueError("Vector subtype must be 0 (FLOAT32) or 1 (FLOAT16)") + base_type: VectorBaseType = "float16" if subtype else "float32" + _check_shape(dimensions, base_type) + if value is None: + return None + if isinstance(value, list): + value = Vector(value, base_type) + if not isinstance(value, Vector): + raise TypeError("Vector parameters must be Vector, list, or None") + if value.dimensions != dimensions or value.base_type != base_type: + raise ValueError("Vector dimensions and base_type must match the input hint") + return value + + +class OutputParameter: + """A vector OUTPUT parameter for callproc(); value is filled before return.""" + + def __init__(self, dimensions: int, base_type: VectorBaseType = "float32") -> None: + _check_shape(dimensions, base_type) + self.dimensions = dimensions + self.base_type = base_type + self.value: Optional[Vector] = None + + +class InputOutputParameter(OutputParameter): + """A vector INPUT_OUTPUT parameter initialized from a Vector.""" + + def __init__(self, value: Vector) -> None: + if not isinstance(value, Vector): + raise TypeError("InputOutputParameter requires a Vector") + super().__init__(value.dimensions, value.base_type) + self.value = value diff --git a/tests/test_004_cursor.py b/tests/test_004_cursor.py index 8d2b8f9af..91ca774f9 100644 --- a/tests/test_004_cursor.py +++ b/tests/test_004_cursor.py @@ -14883,12 +14883,10 @@ def test_tables_error_handling(cursor): pass -def test_callproc_not_supported_error(cursor): - """Test callproc NotSupportedError (Lines 2413-2421).""" - - # This should always raise NotSupportedError (lines 2417-2420) - with pytest.raises(mssql_python.NotSupportedError, match="callproc.*is not yet implemented"): - cursor.callproc("test_proc") +def test_callproc_invalid_name(cursor): + """Reject invalid procedure names before sending a request.""" + with pytest.raises(ValueError, match="procedure"): + cursor.callproc("dbo..test_proc") def test_setoutputsize_no_op(cursor): diff --git a/tests/test_025_vector.py b/tests/test_025_vector.py new file mode 100644 index 000000000..8b7111754 --- /dev/null +++ b/tests/test_025_vector.py @@ -0,0 +1,1058 @@ +"""Vector unit contracts and live SQL Server end-to-end API coverage. + +Live tests require DB_CONNECTION_STRING. Enable the v2 matrix with +MSSQL_TEST_VECTOR_FLOAT16=1 on an ODBC 18.7.1.1+ / FLOAT16-capable server. +No live operation is mocked; backend/connection errors fail rather than skip. +""" + +import json +import math +import os +import struct +import uuid +from collections import deque +from itertools import repeat +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +import mssql_python +from mssql_python import InputOutputParameter, OutputParameter, TableValuedParameter, Vector +from mssql_python import ddbc_bindings +from mssql_python.constants import ConstantsDDBC +from mssql_python.cursor import Cursor + + +@pytest.mark.parametrize("dtype,limit", [("float32", 1998), ("float16", 3996)]) +def test_vector_dimensions_and_subtype(dtype, limit): + value = Vector(range(limit), base_type=dtype) + assert value.dimensions == limit + assert value.base_type == dtype + assert value[0] == 0.0 + assert Vector(value, base_type=dtype) == value + with pytest.raises(ValueError, match="dimensions"): + Vector(range(limit + 1), base_type=dtype) + + +@pytest.mark.parametrize("values", [[], [math.nan], [math.inf], [-math.inf], [1e40]]) +def test_invalid_vector_values(values): + with pytest.raises(ValueError): + Vector(values) + + +@pytest.mark.parametrize("values", ["123", b"123", [None], [True], ["1"]]) +def test_non_numeric_vector_values(values): + with pytest.raises(TypeError): + Vector(values) + + +def test_vector_is_immutable_and_preserves_input(): + source = [1, 2, 3] + value = Vector(source, base_type="float16") + source[0] = 9 + assert value.values == (1.0, 2.0, 3.0) + with pytest.raises(AttributeError): + value.base_type = "float32" + with pytest.raises(ValueError, match="base_type"): + Vector([1], base_type="float64") + + +def test_vector_procedure_parameters(): + output = OutputParameter(3, base_type="float16") + assert output.base_type == "float16" + assert output.dimensions == 3 + assert output.value is None + value = Vector([1, 2, 3], base_type="float16") + inout = InputOutputParameter(value) + assert inout.value == value + with pytest.raises(TypeError): + InputOutputParameter([1, 2]) + + +def test_tvp_schema_and_rows(): + columns = [(mssql_python.SQL_INTEGER, 0, 0), (mssql_python.SQL_SS_VECTOR, 3, 1)] + value = Vector([1, 2, 3], base_type="float16") + tvp = TableValuedParameter("dbo.Embeddings", [(1, value), (2, None)], columns) + assert tvp.rows == ((1, value), (2, None)) + assert TableValuedParameter("dbo.Embeddings", [], columns).rows == () + with pytest.raises(ValueError, match="columns"): + TableValuedParameter("dbo.Embeddings", [(1,)], columns) + with pytest.raises(ValueError, match="dimensions|subtype"): + TableValuedParameter("dbo.Embeddings", [(1, Vector([1, 2]))], columns) + + +@pytest.mark.parametrize("mode", ["off", "v1", "v2"]) +def test_vector_connection_keyword(mode): + from mssql_python.connection_string_parser import _ConnectionStringParser + + parser = _ConnectionStringParser(validate_keywords=True) + params = parser._parse(f"Server=localhost;VectorTypeSupport={mode.upper()}") + assert params["vectortypesupport"] == mode.upper() + assert parser._normalize_params(params)["VectorTypeSupport"] == mode + + +@pytest.mark.parametrize("dtype,dimensions,subtype", [("float32", 1998, 0), ("float16", 3996, 1)]) +def test_native_vector_detection(dtype, dimensions, subtype): + value = Vector(repeat(1.25, dimensions), dtype) + info = ddbc_bindings._detect_param_types([value])[0] + assert info.paramSQLType == mssql_python.SQL_SS_VECTOR + assert info.paramCType == ConstantsDDBC.SQL_C_SS_VECTOR.value + assert info.columnSize == ddbc_bindings._vector_buffer_size(dimensions) + assert info.decimalDigits == subtype + assert not info.isDAE + if dtype == "float16": + assert info.columnSize > 8000 + + +@pytest.mark.parametrize("value", [None, Vector([1, 2, 3], "float16"), [1, 2, 3]]) +def test_native_vector_input_sizes(value): + hints = [(mssql_python.SQL_SS_VECTOR, ConstantsDDBC.SQL_C_SS_VECTOR.value, 3, 1)] + info = ddbc_bindings._detect_param_types([value], hints)[0] + assert info.columnSize == ddbc_bindings._vector_buffer_size(3) + assert info.decimalDigits == 1 + assert info.paramCType == ConstantsDDBC.SQL_C_SS_VECTOR.value + assert not info.isDAE + + +@pytest.mark.parametrize( + "value,size,subtype,error", + [ + (Vector([1, 2]), 3, 0, ValueError), + (Vector([1, 2], "float16"), 2, 0, ValueError), + (None, 3997, 1, ValueError), + (None, 0, 0, ValueError), + (None, 2, 2, ValueError), + ((1, 2), 2, 0, TypeError), + ([1, 2], 3, 0, ValueError), + ([1e40, 2], 2, 0, ValueError), + ([70000, 2], 2, 1, ValueError), + ([[1], 2], 2, 0, TypeError), + ("x" * 8001, 2, 0, TypeError), + ], +) +def test_native_vector_invalid_input_sizes(value, size, subtype, error): + hints = [(mssql_python.SQL_SS_VECTOR, ConstantsDDBC.SQL_C_SS_VECTOR.value, size, subtype)] + with pytest.raises(error): + ddbc_bindings._detect_param_types([value], hints) + + +def test_vector_large_integer_and_unbounded_iterable(): + with pytest.raises(ValueError, match="finite float32"): + Vector([10**1000]) + with pytest.raises(ValueError, match="dimensions"): + Vector(repeat(1)) + + +def test_no_python_half_precision_conversion(): + number = 1.0001 + float32 = struct.unpack("=f", struct.pack("=f", number))[0] + float16 = struct.unpack("=e", struct.pack("=e", number))[0] + assert Vector([number], "float16")[0] == float32 + assert float32 != float16 + + +@pytest.mark.parametrize("base_type", ["float32", "float16"]) +def test_vector_normalizes_to_float32_and_supports_list(base_type): + source = [0.1, -0.1, -0.0, 1e-50, 1e-40] + expected = [struct.unpack("=f", struct.pack("=f", value))[0] for value in source] + value = Vector(iter(source), base_type=base_type) + assert list(value) == expected + assert value.values == tuple(expected) + assert math.copysign(1, value[2]) == -1 + assert value[3] == 0 + assert value[4] != 0 + assert value[:2] == tuple(expected[:2]) + assert hash(value) == hash(Vector(expected, base_type=base_type)) + copy = list(value) + copy[0] = 9 + assert value[0] == expected[0] + assert not hasattr(value, "dtype") + + +@pytest.mark.parametrize( + "base_type,limit", + [("float32", 3.4028234663852886e38), ("float16", 65504.0)], +) +def test_vector_accepts_finite_range_boundaries(base_type, limit): + assert list(Vector([limit, -limit], base_type=base_type)) == [limit, -limit] + for outside in (math.nextafter(limit, math.inf), math.nextafter(-limit, -math.inf)): + with pytest.raises(ValueError, match=f"index 1.*{base_type}"): + Vector([0, outside], base_type=base_type) + + +@pytest.mark.parametrize("base_type", ["float32", "float16"]) +@pytest.mark.parametrize("invalid", [math.nan, math.inf, -math.inf, 3.5e38, -3.5e38, 10**1000]) +def test_vector_overflow_is_rejected_at_construction(base_type, invalid): + with pytest.raises(ValueError, match="index 1.*finite float32"): + Vector([0, invalid], base_type=base_type) + + +@pytest.mark.parametrize("invalid", [65505, -65505, 70000, -70000]) +def test_vector_half_range_is_checked_without_encoding(invalid): + with pytest.raises(ValueError, match="index 1.*float16"): + Vector([0, invalid], base_type="float16") + + +@pytest.mark.parametrize("base_type,subtype", [("float32", 0), ("float16", 1)]) +def test_native_hinted_list_preserves_input(base_type, subtype): + source = [0.1, 2, 3] + parameters = [source] + hints = [(mssql_python.SQL_SS_VECTOR, ConstantsDDBC.SQL_C_SS_VECTOR.value, 3, subtype)] + info = ddbc_bindings._detect_param_types(parameters, hints)[0] + assert parameters[0] is source + assert source == [0.1, 2, 3] + assert info.columnSize == ddbc_bindings._vector_buffer_size(3) + assert info.decimalDigits == subtype + + +def test_unhinted_list_is_not_implicitly_a_vector(): + with pytest.raises(TypeError, match="Unsupported parameter type"): + ddbc_bindings._detect_param_types([[1, 2, 3]]) + + +@pytest.mark.parametrize("rows", [[], [[1], [2]]]) +def test_native_fetch_policy_excludes_buffered_procedures(rows): + from mssql_python._native_fetch import _Policy + + policy = object.__new__(_Policy) + assert policy(Cursor.__new__(Cursor), {"_procedure_rows": deque(rows)}) is None + + +def test_tvp_vector_lists_use_explicit_schema_and_snapshot(): + source = [0.1, 2, 3] + columns = [(mssql_python.SQL_SS_VECTOR, 3, 1)] + parameter = TableValuedParameter("dbo.v", [(source,), (None,)], columns) + expected = Vector(source, base_type="float16") + source[0] = 9 + assert parameter.rows == ((expected,), (None,)) + for invalid, error in (([1, 2], ValueError), ([1e40, 2, 3], ValueError), ("123", TypeError)): + with pytest.raises(error): + TableValuedParameter("dbo.v", [(invalid,)], columns) + + +@pytest.mark.parametrize("dimensions", [True, 1.5, "3"]) +def test_output_parameter_invalid_shape_type(dimensions): + with pytest.raises(TypeError, match="integer"): + OutputParameter(dimensions) + + +@pytest.mark.parametrize("mode", ["", "v3", "true"]) +def test_invalid_vector_negotiation(mode): + from mssql_python.connection_string_parser import _ConnectionStringParser + from mssql_python.exceptions import ConnectionStringParseError + + parser = _ConnectionStringParser(validate_keywords=True) + with pytest.raises(ConnectionStringParseError, match="(?i)vectortypesupport"): + parser._normalize_params(parser._parse(f"Server=localhost;VectorTypeSupport={mode}")) + + +def test_vector_negotiation_is_opt_in(): + from mssql_python.connection_string_parser import _ConnectionStringParser + + parser = _ConnectionStringParser(validate_keywords=True) + assert "VectorTypeSupport" not in parser._normalize_params(parser._parse("Server=localhost")) + assert parser._normalize_params({"vectortypesupport": "V1"})["VectorTypeSupport"] == "v1" + + +@pytest.mark.parametrize("name", ["", "dbo..v", "dbo.v;drop", "a.b.c", "dbo.\0v"]) +def test_invalid_tvp_name(name): + with pytest.raises(ValueError): + TableValuedParameter(name, [], [(mssql_python.SQL_SS_VECTOR, 3, 0)]) + + +def test_tvp_immutable_snapshot_and_native_detection(): + rows = [[Vector([1, 2, 3])]] + columns = [[mssql_python.SQL_SS_VECTOR, 3, 0]] + parameter = TableValuedParameter("dbo.v", rows, columns) + rows.clear() + columns.clear() + assert len(parameter.rows) == 1 + with pytest.raises(AttributeError): + parameter.rows = () + info = ddbc_bindings._detect_param_types([parameter])[0] + assert info.paramSQLType == -153 + assert not info.isDAE + + +@pytest.fixture +def unit_cursor(monkeypatch): + monkeypatch.setattr(Cursor, "_initialize_cursor", lambda self: None) + connection = SimpleNamespace( + _converters_generation=0, + _decoding_generation=0, + _output_converters={}, + _native_uuid=None, + getdecoding=lambda sql_type: {"encoding": "utf-16le", "ctype": -8}, + getencoding=lambda: {"encoding": "utf-16le", "ctype": -8}, + ) + connection.get_output_converter = connection._output_converters.get + cursor = Cursor(connection) + yield cursor + cursor.close() + + +def vector_metadata(dtype="float32"): + return [ + { + "ColumnName": "embedding", + "DataType": mssql_python.SQL_SS_VECTOR, + "ColumnSize": ddbc_bindings._vector_buffer_size(3), + "DecimalDigits": int(dtype == "float16"), + "Nullable": 1, + } + ] + + +def test_callproc_buffered_results_and_cursor_reuse(unit_cursor, monkeypatch): + vectors = [Vector([i, 2, 3]) for i in range(4)] + output = OutputParameter(3) + + def callproc(handle, sql, params, settings, messages, *decoding): + assert sql == "{CALL [dbo].[a]]b](?,?)}" + assert params == [vectors[0], output] + assert decoding == ("utf-16le", "utf-16le", -8) + output.value = vectors[0] + return [vectors[0], vectors[0]], [ + (vector_metadata(), [[value] for value in vectors], -1), + ([], [], 4), + (vector_metadata(), [[None]], -1), + ] + + monkeypatch.setattr(ddbc_bindings, "DDBCCallProc", callproc) + assert unit_cursor.callproc("dbo.a]b", [vectors[0], output]) == [vectors[0], vectors[0]] + assert output.value == vectors[0] + assert unit_cursor.description[0][1] is Vector + assert unit_cursor.fetchone().embedding == vectors[0] + assert [row[0] for row in unit_cursor.fetchmany(2)] == vectors[1:3] + assert [row[0] for row in unit_cursor.fetchall()] == vectors[3:] + assert unit_cursor.fetchone() is None + assert unit_cursor.rowcount == 4 + assert unit_cursor.rownumber == 3 + assert unit_cursor.nextset() + assert unit_cursor.rowcount == 4 + with pytest.raises(mssql_python.ProgrammingError): + unit_cursor.fetchone() + assert unit_cursor.nextset() + assert unit_cursor.fetchone()[0] is None + assert not unit_cursor.nextset() + assert unit_cursor.description is None + unit_cursor._soft_reset_cursor() + assert unit_cursor._procedure_rows is None + assert unit_cursor._procedure_results is None + + +@pytest.mark.parametrize("name", ["", "a..b", "a.b.c.d", "a\0b"]) +def test_callproc_invalid_name_without_database(unit_cursor, name): + with pytest.raises(ValueError): + unit_cursor.callproc(name) + + +def test_buffered_procedure_rejects_arrow_and_releases_rows(unit_cursor, monkeypatch): + monkeypatch.setattr( + ddbc_bindings, + "DDBCCallProc", + lambda *args: ([], [(vector_metadata(), [[Vector([1, 2, 3])]], -1)]), + ) + unit_cursor.callproc("dbo.v") + with pytest.raises(mssql_python.NotSupportedError, match="row fetch"): + unit_cursor.arrow_batch() + unit_cursor.close() + assert unit_cursor._procedure_rows is None + assert unit_cursor._procedure_results is None + + +def test_callproc_propagates_backend_errors(unit_cursor, monkeypatch): + execute = Mock(side_effect=mssql_python.ProgrammingError("procedure failed", "42000")) + monkeypatch.setattr(ddbc_bindings, "DDBCCallProc", execute) + with pytest.raises(mssql_python.ProgrammingError, match="procedure failed"): + unit_cursor.callproc("dbo.failure", [Vector([1, 2, 3])]) + assert unit_cursor._procedure_rows is None + + +def test_procedure_result_converters_are_refreshed(unit_cursor, monkeypatch): + value = Vector([1, 2, 3]) + unit_cursor.connection._output_converters[mssql_python.SQL_SS_VECTOR] = lambda v: v.dimensions + monkeypatch.setattr( + ddbc_bindings, + "DDBCCallProc", + lambda *args: ([], [(vector_metadata(), [[value]], -1)] * 2), + ) + unit_cursor.callproc("dbo.v") + assert unit_cursor.fetchone()[0] == 3 + assert unit_cursor.nextset() + assert unit_cursor.fetchone()[0] == 3 + + +@pytest.mark.parametrize("explicit", [False, True]) +@pytest.mark.parametrize("dtype", ["float32", "float16"]) +def test_executemany_vector_type_mapping(unit_cursor, monkeypatch, explicit, dtype): + execute = Mock(return_value=0) + monkeypatch.setattr(ddbc_bindings, "SQLExecuteMany", execute) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLRowCount", lambda handle: 3) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLDescribeCol", lambda *args: 0) + subtype = int(dtype == "float16") + if explicit: + unit_cursor.setinputsizes([(mssql_python.SQL_SS_VECTOR, 3, subtype)]) + value = Vector([1, 2, 3], dtype) + unit_cursor.executemany("INSERT INTO v VALUES (?)", [(None,), (value,), (value,)]) + arguments = execute.call_args.args + assert arguments[2] == [[None, value, value]] + info = arguments[3][0] + assert info.paramSQLType == mssql_python.SQL_SS_VECTOR + assert info.paramCType == ConstantsDDBC.SQL_C_SS_VECTOR.value + assert info.decimalDigits == subtype + assert info.columnSize == ddbc_bindings._vector_buffer_size(3) + assert not info.isDAE + + +@pytest.mark.parametrize("base_type,subtype", [("float32", 0), ("float16", 1)]) +@pytest.mark.parametrize("pyformat", [False, True]) +def test_executemany_hinted_lists_are_normalized( + unit_cursor, monkeypatch, base_type, subtype, pyformat +): + execute = Mock(return_value=0) + monkeypatch.setattr(ddbc_bindings, "SQLExecuteMany", execute) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLRowCount", lambda handle: 3) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLDescribeCol", lambda *args: 0) + source = [0.1, 2, 3] + expected = Vector(source, base_type=base_type) + rows = [(None,), (source,), (expected,)] + sql = "INSERT INTO v VALUES (?)" + if pyformat: + rows = [{"v": row[0]} for row in rows] + sql = "INSERT INTO v VALUES (%(v)s)" + unit_cursor.setinputsizes([(mssql_python.SQL_SS_VECTOR, 3, subtype)]) + unit_cursor.executemany(sql, rows) + source[0] = 9 + assert execute.call_args.args[2] == [[None, expected, expected]] + assert execute.call_args.args[3][0].decimalDigits == subtype + assert unit_cursor._inputsizes is None + + +@pytest.mark.parametrize( + "invalid,error", + [ + ([1, 2], ValueError), + ([1e40, 2, 3], ValueError), + ([70000, 2, 3], ValueError), + ([None, 2, 3], TypeError), + ("123", TypeError), + (Vector([1, 2, 3]), ValueError), + ], +) +def test_executemany_hinted_lists_validate_before_execution( + unit_cursor, monkeypatch, invalid, error +): + execute = Mock() + monkeypatch.setattr(ddbc_bindings, "SQLExecuteMany", execute) + unit_cursor.setinputsizes([(mssql_python.SQL_SS_VECTOR, 3, 1)]) + with pytest.raises(error): + unit_cursor.executemany("INSERT INTO v VALUES (?)", [([1, 2, 3],), (invalid,)]) + execute.assert_not_called() + + +@pytest.mark.parametrize( + "values,error", + [ + ([Vector([1]), Vector([1, 2])], ValueError), + ([Vector([1]), Vector([1], "float16")], ValueError), + ([Vector([1]), 1], TypeError), + ], +) +def test_executemany_rejects_mixed_vector_columns(unit_cursor, values, error): + with pytest.raises(error): + unit_cursor.executemany("INSERT INTO v VALUES (?)", [(value,) for value in values]) + + +def test_bulkcopy_does_not_invent_pycore_serialization(): + rows = [(1, "x"), (2, None)] + assert list(Cursor._validate_bulkcopy_vectors(rows)) == rows + with pytest.raises(mssql_python.NotSupportedError, match="pycore"): + list(Cursor._validate_bulkcopy_vectors([(1, Vector([1, 2, 3]))])) + + +@pytest.mark.parametrize("leading_row", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("wrap_error", [False, True]) +def test_bulkcopy_preserves_vector_rejection( + unit_cursor, monkeypatch, leading_row, streaming, wrap_error +): + def consume(table_name, rows, **kwargs): + try: + list(rows) + except mssql_python.NotSupportedError: + if wrap_error: + raise RuntimeError("Failed to read bulk row") from None + return {"rows_copied": 0} + + core_cursor = Mock() + core_cursor.bulkcopy.side_effect = consume + core_connection = Mock() + core_connection.cursor.return_value = core_cursor + core = SimpleNamespace(PyCoreConnection=Mock(return_value=core_connection)) + monkeypatch.setattr(Cursor, "_bulkcopy_core_and_validate", staticmethod(lambda *args: core)) + monkeypatch.setattr(Cursor, "_build_pycore_context", lambda self: {}) + rows = ([(1,)] if leading_row else []) + [(Vector([1, 2, 3]),)] + with pytest.raises(mssql_python.NotSupportedError, match="pycore"): + unit_cursor.bulkcopy("dbo.vectors", iter(rows) if streaming else rows) + core_cursor.close.assert_called_once() + core_connection.close.assert_called_once() + + +@pytest.fixture( + params=[ + pytest.param(("v1", "float32"), id="v1-float32"), + pytest.param(("v2", "float32"), id="v2-float32"), + pytest.param(("v2", "float16"), id="v2-float16"), + ] +) +def live_vector_connection(request, conn_str): + if not conn_str: + pytest.skip("DB_CONNECTION_STRING is not configured") + mode, dtype = request.param + if mode == "v2" and os.getenv("MSSQL_TEST_VECTOR_FLOAT16") != "1": + pytest.skip("Set MSSQL_TEST_VECTOR_FLOAT16=1 with ODBC 18.7.1.1+ and server support") + # Keyword overrides avoid duplicate keys or a pre-existing mode in conn_str. + connection = mssql_python.connect(conn_str, VectorTypeSupport=mode) + try: + yield connection, dtype + finally: + connection.close() + + +def _expected_stored_vector(values, base_type): + normalized = [struct.unpack("=f", struct.pack("=f", value))[0] for value in values] + if base_type == "float16": + normalized = [struct.unpack("=e", struct.pack("=e", value))[0] for value in normalized] + return Vector(normalized, base_type=base_type) + + +@pytest.mark.parametrize("style", ["tuple", "list", "pyformat"]) +def test_live_vector_hinted_lists_and_hint_lifetime(live_vector_connection, style): + connection, base_type = live_vector_connection + source = [0.1, 1.0001, -0.0] + expected = _expected_stored_vector(source, base_type) + with connection.cursor() as cursor: + cursor.setinputsizes([(mssql_python.SQL_SS_VECTOR, 3, int(base_type == "float16"))]) + if style == "pyformat": + cursor.execute("SELECT %(v)s AS embedding", {"v": source}) + else: + cursor.execute("SELECT ? AS embedding", (source,) if style == "tuple" else [source]) + assert cursor._inputsizes is None + assert source == [0.1, 1.0001, -0.0] + source[0] = 9 + retained = cursor.fetchval() + assert retained == expected + assert retained.base_type == base_type + assert cursor.fetchval() is None + with pytest.raises(TypeError, match="Unsupported parameter type"): + cursor.execute("SELECT ? AS embedding", (source,)) + cursor.execute("SELECT 1") + assert cursor.fetchval() == 1 + assert retained == expected + + +@pytest.mark.parametrize("pyformat", [False, True]) +@pytest.mark.parametrize("streamed", [False, True]) +def test_live_vector_hinted_list_batches(live_vector_connection, pyformat, streamed): + connection, base_type = live_vector_connection + source = [0.1, 1.0001, -2.5] + expected = _expected_stored_vector(source, base_type) + wrapped = Vector(source, base_type=base_type) + text = "x" * (5001 if streamed else 5) + rows = [(0, None, text), (1, source, text), (2, wrapped, text)] + sql = "INSERT INTO #hinted_vectors VALUES (?, ?, ?)" + if pyformat: + rows = [dict(id=row[0], v=row[1], label=row[2]) for row in rows] + sql = "INSERT INTO #hinted_vectors VALUES (%(id)s, %(v)s, %(label)s)" + with connection.cursor() as cursor: + cursor.execute( + f"CREATE TABLE #hinted_vectors (id INT, embedding VECTOR(3, {base_type}), label NVARCHAR(MAX))" + ) + cursor.setinputsizes( + [ + (mssql_python.SQL_INTEGER, 10, 0), + (mssql_python.SQL_SS_VECTOR, 3, int(base_type == "float16")), + ( + mssql_python.SQL_WLONGVARCHAR if streamed else mssql_python.SQL_WVARCHAR, + len(text), + 0, + ), + ] + ) + cursor.executemany(sql, rows) + assert source == [0.1, 1.0001, -2.5] + assert cursor._inputsizes is None + cursor.execute("SELECT embedding, label FROM #hinted_vectors ORDER BY id") + assert [tuple(row) for row in cursor.fetchall()] == [ + (None, text), + (expected, text), + (expected, text), + ] + + +@pytest.mark.parametrize("method", ["execute", "executemany"]) +@pytest.mark.parametrize("invalid", [[1, 2], [1e40, 2, 3], [None, 2, 3], [[1], 2, 3]]) +def test_live_vector_hinted_list_validation_and_recovery(live_vector_connection, method, invalid): + connection, base_type = live_vector_connection + with connection.cursor() as cursor: + cursor.execute(f"CREATE TABLE #hint_errors (embedding VECTOR(3, {base_type}))") + connection.commit() + cursor.setinputsizes([(mssql_python.SQL_SS_VECTOR, 3, int(base_type == "float16"))]) + with pytest.raises((TypeError, ValueError)): + if method == "execute": + cursor.execute("INSERT INTO #hint_errors VALUES (?)", (invalid,)) + else: + cursor.executemany( + "INSERT INTO #hint_errors VALUES (?)", [([1, 2, 3],), (invalid,)] + ) + cursor.setinputsizes([]) + cursor.execute("SELECT COUNT(*) FROM #hint_errors") + assert cursor.fetchval() == 0 + cursor.setinputsizes([(mssql_python.SQL_SS_VECTOR, 3, int(base_type == "float16"))]) + cursor.execute("INSERT INTO #hint_errors VALUES (?)", ([1, 2, 3],)) + cursor.execute("SELECT embedding FROM #hint_errors") + assert cursor.fetchval() == Vector([1, 2, 3], base_type=base_type) + + +@pytest.mark.parametrize("method", ["execute", "executemany"]) +def test_live_vector_numeric_boundaries(live_vector_connection, method): + connection, base_type = live_vector_connection + limit = 65504.0 if base_type == "float16" else 3.4028234663852886e38 + source = [limit, -limit, 0.1] + expected = _expected_stored_vector(source, base_type) + with connection.cursor() as cursor: + cursor.execute(f"CREATE TABLE #range_vectors (embedding VECTOR(3, {base_type}))") + connection.commit() + cursor.setinputsizes([(mssql_python.SQL_SS_VECTOR, 3, int(base_type == "float16"))]) + if method == "execute": + cursor.execute("INSERT INTO #range_vectors VALUES (?)", (source,)) + else: + cursor.executemany("INSERT INTO #range_vectors VALUES (?)", [(source,)]) + cursor.execute("SELECT embedding FROM #range_vectors") + assert cursor.fetchval() == expected + for invalid in (3.5e38, -3.5e38, math.inf, math.nan): + with pytest.raises(ValueError, match="finite float32"): + Vector([invalid], base_type=base_type) + if base_type == "float16": + with pytest.raises(ValueError, match="float16"): + Vector([70000], base_type=base_type) + cursor.execute("SELECT COUNT(*) FROM #range_vectors") + assert cursor.fetchval() == 1 + + +def test_live_vector_tiny_values(live_vector_connection): + connection, base_type = live_vector_connection + source = [1e-50, 1e-40, -0.0] + with connection.cursor() as cursor: + cursor.execute("SELECT ?", Vector(source, base_type=base_type)) + assert cursor.fetchval() == _expected_stored_vector(source, base_type) + + +def test_live_vector_rebinding_prepared_statement(live_vector_connection): + connection, base_type = live_vector_connection + with connection.cursor() as cursor: + for dimensions in (3, 4, 1, 3): + value = Vector(range(dimensions), base_type=base_type) + cursor.execute("SELECT ? AS embedding", value) + assert cursor.fetchval() == value + cursor.setinputsizes( + [(mssql_python.SQL_SS_VECTOR, dimensions, int(base_type == "float16"))] + ) + cursor.execute("SELECT ? AS embedding", None) + assert cursor.fetchval() is None + + +@pytest.mark.parametrize("method", ["fetchone", "fetchmany", "fetchval", "iteration"]) +def test_live_vector_buffered_procedure_fetch_reuse(live_vector_connection, method): + connection, base_type = live_vector_connection + value = Vector([1, 2, 3], base_type=base_type) + with connection.cursor() as cursor: + cursor.execute("SELECT n FROM (VALUES (1), (2), (3)) AS t(n) ORDER BY n") + assert cursor.fetchone()[0] == 1 + assert cursor.fetchone()[0] == 2 + cursor.execute( + f"CREATE PROCEDURE #buffered_vector @v VECTOR(3, {base_type}) AS " + "BEGIN SET NOCOUNT ON; " + "SELECT @v AS embedding FROM (VALUES (1), (2), (3), (4)) AS t(n) ORDER BY n; END" + ) + assert cursor.callproc("#buffered_vector", [value]) == [value] + if method == "fetchone": + actual = [cursor.fetchone()[0] for _ in range(4)] + assert cursor.fetchone() is None + elif method == "fetchmany": + actual = [row[0] for _ in range(4) for row in cursor.fetchmany(1)] + assert cursor.fetchmany(1) == [] + elif method == "fetchval": + actual = [cursor.fetchval() for _ in range(4)] + assert cursor.fetchval() is None + else: + actual = [row[0] for row in cursor] + assert actual == [value] * 4 + cursor.execute("SELECT 42") + assert cursor.fetchval() == 42 + + +def test_live_vector_scalar_array_and_nulls(live_vector_connection): + connection, dtype = live_vector_connection + value = Vector([1.25, -2.5, 0], dtype) + with connection.cursor() as cursor: + cursor.execute(f"CREATE TABLE #vectors (id INT, embedding VECTOR(3, {dtype}) NULL)") + cursor.execute("INSERT INTO #vectors VALUES (?, ?)", 1, value) + cursor.executemany("INSERT INTO #vectors VALUES (?, ?)", [(2, None), (3, value)]) + cursor.setinputsizes([(mssql_python.SQL_SS_VECTOR, 3, int(dtype == "float16"))]) + cursor.execute("INSERT INTO #vectors VALUES (4, ?)", None) + cursor.execute("SELECT embedding FROM #vectors ORDER BY id") + assert cursor.description[0][1] is Vector + assert cursor.fetchone()[0] == value + assert cursor.fetchmany(1)[0][0] is None + assert [row[0] for row in cursor.fetchall()] == [value, None] + assert cursor.fetchone() is None + cursor.execute("SELECT embedding FROM #vectors WHERE id=1") + assert cursor.fetchall()[0][0] == value + + +@pytest.mark.parametrize("with_results", [False, True], ids=["outputs-only", "multiple-results"]) +def test_live_vector_procedure_outputs(live_vector_connection, with_results): + connection, dtype = live_vector_connection + value = Vector([1.25, -2.5, 0], dtype) + selects = "SELECT @i AS embedding; SELECT @o AS embedding;" if with_results else "" + with connection.cursor() as cursor: + cursor.execute( + f"CREATE PROCEDURE #vector_echo @i VECTOR(3, {dtype}), " + f"@o VECTOR(3, {dtype}) OUTPUT, @io VECTOR(3, {dtype}) OUTPUT AS " + f"BEGIN SET NOCOUNT ON; SET @o=@i; SET @io=@i; {selects} END" + ) + output = OutputParameter(3, dtype) + inout = InputOutputParameter(Vector([0, 0, 0], dtype)) + assert cursor.callproc("#vector_echo", [value, output, inout]) == [value] * 3 + assert output.value == inout.value == value + if with_results: + assert cursor.fetchone()[0] == value + assert cursor.nextset() + assert cursor.fetchall()[0][0] == value + else: + assert cursor.description is None + assert not cursor.nextset() + assert cursor.callproc("#vector_echo", [None, output, inout]) == [None] * 3 + assert output.value is None and inout.value is None + if not with_results: + assert cursor.description is None + if with_results: + assert cursor.fetchone()[0] is None + assert cursor.nextset() + assert cursor.fetchone()[0] is None + cursor.execute("SELECT 1", reset_cursor=False) + assert cursor.fetchone()[0] == 1 + + +@pytest.mark.parametrize("method", ["execute", "executemany"]) +def test_live_vector_maximum_dimensions_and_rounding(live_vector_connection, method): + connection, dtype = live_vector_connection + dimensions = 3996 if dtype == "float16" else 1998 + value = Vector(repeat(0.1, dimensions), dtype) + format_code = "=4 ORDER BY id") + assert [row[0] for row in cursor.fetchall()] == [value, None] + + +def test_live_vector_executemany_with_streamed_text(live_vector_connection): + connection, dtype = live_vector_connection + first = Vector([1, 2, 3], dtype) + second = Vector([4, 5, 6], dtype) + text = "streamed " * 1000 + with connection.cursor() as cursor: + cursor.execute( + f"CREATE TABLE #stream_vectors (id INT, embedding VECTOR(3, {dtype}), label NVARCHAR(MAX))" + ) + cursor.executemany( + "INSERT INTO #stream_vectors VALUES (?, ?, ?)", + [(1, first, text), (2, second, text + "second"), (3, None, text)], + ) + cursor.execute("SELECT embedding, label FROM #stream_vectors ORDER BY id") + assert [tuple(row) for row in cursor.fetchall()] == [ + (first, text), + (second, text + "second"), + (None, text), + ] + + +def test_live_vector_commit_rollback_and_multiple_cursors(live_vector_connection): + connection, dtype = live_vector_connection + value = Vector([1, 2, 3], dtype) + with connection.cursor() as writer, connection.cursor() as reader: + writer.execute(f"CREATE TABLE #tx_vectors (embedding VECTOR(3, {dtype}))") + connection.commit() + writer.execute("INSERT INTO #tx_vectors VALUES (?)", value) + connection.commit() + writer.execute("INSERT INTO #tx_vectors VALUES (?)", Vector([4, 5, 6], dtype)) + connection.rollback() + reader.execute("SELECT embedding FROM #tx_vectors") + assert [row[0] for row in reader.fetchall()] == [value] + writer.close() + reader.execute("SELECT embedding FROM #tx_vectors") + assert reader.fetchone()[0] == value + + +def test_live_vector_input_hints_reject_mismatch_and_cursor_recovers(live_vector_connection): + connection, dtype = live_vector_connection + wrong_dtype = "float32" if dtype == "float16" else "float16" + with connection.cursor() as cursor: + for value in (Vector([1, 2], dtype), Vector([1, 2, 3], wrong_dtype)): + cursor.setinputsizes([(mssql_python.SQL_SS_VECTOR, 3, int(dtype == "float16"))]) + with pytest.raises(ValueError, match="dimensions|subtype"): + cursor.execute("SELECT ?", value) + cursor.execute("SELECT 1") + assert cursor.fetchone()[0] == 1 + cursor.execute(f"CREATE TABLE #mismatch_vectors (embedding VECTOR(3, {dtype}))") + connection.commit() + with pytest.raises(mssql_python.Error): + cursor.execute("INSERT INTO #mismatch_vectors VALUES (?)", Vector([1, 2], dtype)) + cursor.execute("SELECT COUNT(*) FROM #mismatch_vectors") + assert cursor.fetchone()[0] == 0 + + +def test_live_vector_output_converters(live_vector_connection): + connection, dtype = live_vector_connection + value = Vector([1, 2, 3], dtype) + with connection.cursor() as cursor: + for key in (mssql_python.SQL_SS_VECTOR, Vector): + connection.add_output_converter(key, lambda item: tuple(item)) + try: + cursor.execute("SELECT ? AS embedding", value) + assert cursor.fetchone()[0] == value.values + cursor.execute( + f"CREATE OR ALTER PROCEDURE #converted_vector @v VECTOR(3, {dtype}) AS " + "BEGIN SET NOCOUNT ON; SELECT @v AS embedding; SELECT @v AS embedding; END" + ) + assert cursor.callproc("#converted_vector", [value]) == [value] + assert cursor.fetchone()[0] == value.values + assert cursor.nextset() + assert cursor.fetchone()[0] == value.values + finally: + connection.remove_output_converter(key) + cursor.execute("SELECT ?", value) + assert cursor.fetchone()[0] == value + + +@pytest.mark.parametrize("method", ["arrow_batch", "arrow", "arrow_reader"]) +def test_live_vector_arrow_boundary(live_vector_connection, method): + pytest.importorskip("pyarrow") + connection, dtype = live_vector_connection + value = Vector([1, 2, 3], dtype) + with connection.cursor() as cursor: + cursor.execute("SELECT ? AS embedding", value) + with pytest.raises(mssql_python.NotSupportedError, match="row fetch"): + getattr(cursor, method)() + assert cursor.fetchone()[0] == value + + +def test_live_vector_bulkcopy_boundary(live_vector_connection): + connection, dtype = live_vector_connection + table_name = f"dbo.pytest_vector_bulk_{uuid.uuid4().hex}" + connection.autocommit = True + with connection.cursor() as cursor: + cursor.execute(f"CREATE TABLE {table_name} (embedding VECTOR(3, {dtype}))") + try: + with pytest.raises(mssql_python.NotSupportedError, match="pycore"): + cursor.bulkcopy(table_name, [(Vector([1, 2, 3], dtype),)]) + cursor.execute(f"SELECT COUNT(*) FROM {table_name}") + assert cursor.fetchone()[0] == 0 + finally: + cursor.execute(f"DROP TABLE {table_name}") + + +@pytest.mark.parametrize( + "mode,dtype", + [ + pytest.param("off", "float32", id="off-rejects-float32"), + pytest.param("off", "float16", id="off-rejects-float16"), + pytest.param("v1", "float16", id="v1-rejects-float16"), + ], +) +def test_live_vector_negotiation_rejects_unsupported_inputs(conn_str, mode, dtype): + if not conn_str: + pytest.skip("DB_CONNECTION_STRING is not configured") + if dtype == "float16" and os.getenv("MSSQL_TEST_VECTOR_FLOAT16") != "1": + pytest.skip("Set MSSQL_TEST_VECTOR_FLOAT16=1 with a FLOAT16-capable server") + connection = mssql_python.connect(conn_str, VectorTypeSupport=mode) + try: + with connection.cursor() as cursor: + with pytest.raises(mssql_python.Error): + cursor.execute("SELECT ?", Vector([1, 2, 3], dtype)) + cursor.execute("SELECT 1") + assert cursor.fetchone()[0] == 1 + finally: + connection.close() + + +@pytest.mark.parametrize("dtype", ["float32", "float16"]) +def test_live_vector_off_returns_json(conn_str, dtype): + if not conn_str: + pytest.skip("DB_CONNECTION_STRING is not configured") + if dtype == "float16" and os.getenv("MSSQL_TEST_VECTOR_FLOAT16") != "1": + pytest.skip("Set MSSQL_TEST_VECTOR_FLOAT16=1 with a FLOAT16-capable server") + connection = mssql_python.connect(conn_str, VectorTypeSupport="off") + try: + with connection.cursor() as cursor: + cursor.execute(f"SELECT CAST('[1.25,-2.5,0]' AS VECTOR(3, {dtype})) AS embedding") + assert cursor.description[0][1] is str + value = cursor.fetchone().embedding + assert isinstance(value, str) + assert json.loads(value) == [1.25, -2.5, 0] + cursor.execute(f"SELECT CAST(NULL AS VECTOR(3, {dtype}))") + assert cursor.fetchone()[0] is None + finally: + connection.close() diff --git a/tests/test_typing.py b/tests/test_typing.py index d492f73a6..1859692d2 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -94,3 +94,32 @@ def fetch_with_converters( encoding="utf-8", ) _check_typing(tmp_path, str(usage)) + + +@pytest.mark.parametrize("module", ["mssql_python", "mssql_python.mssql_python"]) +def test_vector_consumer_call_shapes(tmp_path: Path, module: str) -> None: + usage = tmp_path / "vector_consumer.py" + usage.write_text( + f"""\ +from {module} import ( + Cursor, Vector, OutputParameter, InputOutputParameter, + TableValuedParameter, SQL_SS_VECTOR, +) + +def supported_calls(cursor: Cursor) -> None: + vector = Vector([0.1, 0.2, 0.3], base_type="float16") + values: list[float] = list(vector) + dimensions: int = vector.dimensions + base_type: str = vector.base_type + output = OutputParameter(3, base_type="float16") + inout = InputOutputParameter(vector) + table = TableValuedParameter("dbo.v", [([1, 2, 3],)], [(SQL_SS_VECTOR, 3, 1)]) + cursor.execute("SELECT ?", [vector]) + cursor.execute("EXEC dbo.p ?", table) + cursor.setinputsizes([(SQL_SS_VECTOR, 3, 1)]) + cursor.executemany("INSERT INTO v VALUES (?)", [([1, 2, 3],), (None,), (vector,)]) + cursor.callproc("dbo.p", [vector, output, inout]) +""", + encoding="utf-8", + ) + _check_typing(tmp_path, str(usage)) From 38e00d2c4224f60739341a36a36f4b923bea0937 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Tue, 17 Nov 2026 11:11:36 +0530 Subject: [PATCH 2/7] CHORE: Gate vector integration tests on supported backends Use server type metadata and the qualified provider to skip unsupported CI legs without hiding connection failures. Cover both gate outcomes and cleanup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 5 +++ tests/test_025_vector.py | 71 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 69 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8e5ab3ec3..422b21c41 100644 --- a/README.md +++ b/README.md @@ -322,6 +322,11 @@ $env:MSSQL_TEST_VECTOR_FLOAT16 = "1" python -m pytest tests\test_025_vector.py -v ``` +Servers without the built-in vector type and the unqualified alternate provider +are explicitly skipped, so existing SQL Server 2022 validation legs remain +usable. The server check reads `sys.types`, not a version banner, and does not +suppress connection or query errors. + Missing database/capability configuration causes explicit skips. Once enabled, connection failures and unsupported backend behavior fail the tests rather than being converted to skips. diff --git a/tests/test_025_vector.py b/tests/test_025_vector.py index 8b7111754..0fcd5fa09 100644 --- a/tests/test_025_vector.py +++ b/tests/test_025_vector.py @@ -2,7 +2,8 @@ Live tests require DB_CONNECTION_STRING. Enable the v2 matrix with MSSQL_TEST_VECTOR_FLOAT16=1 on an ODBC 18.7.1.1+ / FLOAT16-capable server. -No live operation is mocked; backend/connection errors fail rather than skip. +Live tests require the Microsoft ODBC provider and a server advertising the +vector type. No live operation is mocked; connection/backend errors fail. """ import json @@ -13,7 +14,7 @@ from collections import deque from itertools import repeat from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import MagicMock, Mock import pytest @@ -500,6 +501,64 @@ def consume(table_name, rows, **kwargs): core_connection.close.assert_called_once() +def _require_live_vector_server(conn_str): + if not conn_str: + pytest.skip("DB_CONNECTION_STRING is not configured") + if mssql_python.get_native_provider_info()["id"] != "msodbcsql18": + pytest.skip("Vector integration currently qualifies the Microsoft ODBC provider only") + connection = mssql_python.connect(conn_str) + try: + with connection.cursor() as cursor: + cursor.execute( + "SELECT COUNT(*) FROM sys.types WHERE name=N'vector' AND is_user_defined=0" + ) + if not cursor.fetchval(): + pytest.skip("Server does not advertise the native vector type") + finally: + connection.close() + + +@pytest.mark.parametrize("supported", [False, True]) +def test_vector_server_gate_uses_type_metadata_and_closes_connection(monkeypatch, supported): + connection = MagicMock() + cursor = connection.cursor.return_value.__enter__.return_value + cursor.fetchval.return_value = int(supported) + monkeypatch.setattr(mssql_python, "connect", Mock(return_value=connection)) + monkeypatch.setattr(mssql_python, "get_native_provider_info", lambda: {"id": "msodbcsql18"}) + if supported: + _require_live_vector_server("Server=localhost") + else: + with pytest.raises(pytest.skip.Exception, match="native vector type"): + _require_live_vector_server("Server=localhost") + cursor.execute.assert_called_once_with( + "SELECT COUNT(*) FROM sys.types WHERE name=N'vector' AND is_user_defined=0" + ) + connection.close.assert_called_once() + + +def test_vector_server_gate_does_not_hide_connection_failures(monkeypatch): + monkeypatch.setattr(mssql_python, "get_native_provider_info", lambda: {"id": "msodbcsql18"}) + monkeypatch.setattr( + mssql_python, "connect", Mock(side_effect=mssql_python.OperationalError("failed", "08001")) + ) + with pytest.raises(mssql_python.OperationalError, match="failed"): + _require_live_vector_server("Server=localhost") + + +def test_vector_server_gate_does_not_connect_unqualified_provider(monkeypatch): + connect = Mock() + monkeypatch.setattr(mssql_python, "connect", connect) + monkeypatch.setattr(mssql_python, "get_native_provider_info", lambda: {"id": "mssql-odbc"}) + with pytest.raises(pytest.skip.Exception, match="provider"): + _require_live_vector_server("Server=localhost") + connect.assert_not_called() + + +@pytest.fixture(scope="session") +def vector_server(conn_str): + _require_live_vector_server(conn_str) + + @pytest.fixture( params=[ pytest.param(("v1", "float32"), id="v1-float32"), @@ -507,9 +566,7 @@ def consume(table_name, rows, **kwargs): pytest.param(("v2", "float16"), id="v2-float16"), ] ) -def live_vector_connection(request, conn_str): - if not conn_str: - pytest.skip("DB_CONNECTION_STRING is not configured") +def live_vector_connection(request, conn_str, vector_server): mode, dtype = request.param if mode == "v2" and os.getenv("MSSQL_TEST_VECTOR_FLOAT16") != "1": pytest.skip("Set MSSQL_TEST_VECTOR_FLOAT16=1 with ODBC 18.7.1.1+ and server support") @@ -1022,7 +1079,7 @@ def test_live_vector_bulkcopy_boundary(live_vector_connection): pytest.param("v1", "float16", id="v1-rejects-float16"), ], ) -def test_live_vector_negotiation_rejects_unsupported_inputs(conn_str, mode, dtype): +def test_live_vector_negotiation_rejects_unsupported_inputs(conn_str, vector_server, mode, dtype): if not conn_str: pytest.skip("DB_CONNECTION_STRING is not configured") if dtype == "float16" and os.getenv("MSSQL_TEST_VECTOR_FLOAT16") != "1": @@ -1039,7 +1096,7 @@ def test_live_vector_negotiation_rejects_unsupported_inputs(conn_str, mode, dtyp @pytest.mark.parametrize("dtype", ["float32", "float16"]) -def test_live_vector_off_returns_json(conn_str, dtype): +def test_live_vector_off_returns_json(conn_str, vector_server, dtype): if not conn_str: pytest.skip("DB_CONNECTION_STRING is not configured") if dtype == "float16" and os.getenv("MSSQL_TEST_VECTOR_FLOAT16") != "1": From 9c515b5582c8573ced97c6e812756b581b29321a Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Tue, 17 Nov 2026 13:29:55 +0530 Subject: [PATCH 3/7] FIX: Harden vector stored procedure failure handling Invalidate prepared-query caches before calling procedures and reject overwritten non-vector INPUT_OUTPUT values before native binding. Add unit and Azure regressions and remove a stale TODO. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 6 ++- mssql_python/cursor.py | 4 +- mssql_python/pybind/ddbc_bindings.cpp | 8 ++-- tests/test_025_vector.py | 66 +++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73bdaa5cb..56af0b246 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), Explicit vector hints and TVP schemas accept numeric lists with the same validation; unhinted lists keep their existing meanings. Added `TableValuedParameter` and buffered `callproc()` with vector - `OutputParameter` / `InputOutputParameter` wrappers. FLOAT16 requires Microsoft - ODBC Driver 18.7.1.1+ (not supplied by the current bundled driver); ODBC owns + `OutputParameter` / `InputOutputParameter` wrappers. Procedure calls invalidate + the previous prepared-query cache even when the procedure fails; overwritten + non-vector INPUT_OUTPUT values are rejected before binding. FLOAT16 requires + Microsoft ODBC Driver 18.7.1.1+ (not supplied by the current bundled driver); ODBC owns half-precision conversion. Vector bulk copy remains pending pycore's contract, and Arrow vector fetches are not supported. - New feature: Support for macOS and Linux. diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index a56c81389..056ffc4df 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -4651,6 +4651,8 @@ def callproc( self.messages = [] if self._cached_decoding_generation != self._connection._decoding_generation: self._refresh_decoding_cache() + self.last_executed_stmt = "" + self.is_stmt_prepared = [False] outputs, results = ddbc_bindings.DDBCCallProc( self.hstmt, query, @@ -4661,8 +4663,6 @@ def callproc( self._cached_wchar_encoding, self._cached_char_ctype, ) - self.last_executed_stmt = "" - self.is_stmt_prepared = [False] self._reset_inputsizes() self._procedure_results = deque(results) self._next_procedure_result() diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 811675712..cc1bc9b78 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -612,7 +612,6 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par continue; } - // TODO: Add more data types like money and intervals. switch (paramInfo.paramCType) { case SQL_C_SS_VECTOR: { if (!paramInfo.vectorBinding) { @@ -6840,9 +6839,12 @@ py::tuple CallProcedure_wrap(SqlHandlePtr statement, const std::u16string& query directions[i] = py::isinstance(item, inoutClass) ? SQL_PARAM_INPUT_OUTPUT : SQL_PARAM_OUTPUT; if (directions[i] == SQL_PARAM_INPUT_OUTPUT) { - if (item.attr("value").is_none()) + py::object input = item.attr("value"); + if (input.is_none()) throw py::value_error("INPUT_OUTPUT parameters require a non-NULL Vector"); - values[i] = item.attr("value"); + if (!py::isinstance(input, vectorModule.attr("Vector"))) + throw py::type_error("INPUT_OUTPUT parameters require a Vector"); + values[i] = input; } else { size_t dimensions = item.attr("dimensions").cast(); auto baseType = item.attr("base_type").cast(); diff --git a/tests/test_025_vector.py b/tests/test_025_vector.py index 0fcd5fa09..3a564d592 100644 --- a/tests/test_025_vector.py +++ b/tests/test_025_vector.py @@ -371,6 +371,34 @@ def test_callproc_propagates_backend_errors(unit_cursor, monkeypatch): assert unit_cursor._procedure_rows is None +def test_failing_callproc_invalidates_prepared_query(unit_cursor, monkeypatch): + unit_cursor.hstmt = Mock() + monkeypatch.setattr(ddbc_bindings, "DDBCSQLResetStmt", lambda *args: 0) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLRowCount", lambda *args: -1) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLDescribeCol", lambda *args: 0) + + def execute(handle, sql, params, hints, prepared, use_prepare, settings): + prepared[0] = True + return 0 + + execute = Mock(side_effect=execute) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLExecute", execute) + query = "SELECT CAST(? AS INT)" + unit_cursor.execute(query, 1) + assert unit_cursor.last_executed_stmt == query + assert unit_cursor.is_stmt_prepared == [True] + monkeypatch.setattr( + ddbc_bindings, + "DDBCCallProc", + Mock(side_effect=mssql_python.ProgrammingError("procedure failed", "42000")), + ) + with pytest.raises(mssql_python.ProgrammingError, match="procedure failed"): + unit_cursor.callproc("dbo.failure", [Vector([1, 2, 3])]) + unit_cursor.execute(query, 2) + assert execute.call_count == 2 + assert all(call.args[5] is True for call in execute.call_args_list) + + def test_procedure_result_converters_are_refreshed(unit_cursor, monkeypatch): value = Vector([1, 2, 3]) unit_cursor.connection._output_converters[mssql_python.SQL_SS_VECTOR] = lambda v: v.dimensions @@ -804,6 +832,44 @@ def test_live_vector_procedure_outputs(live_vector_connection, with_results): assert cursor.fetchone()[0] == 1 +@pytest.mark.parametrize("failure", ["missing-procedure", "procedure-error"]) +def test_live_failing_callproc_reprepares_previous_query(live_vector_connection, failure): + connection, base_type = live_vector_connection + value = Vector([1, 2, 3], base_type=base_type) + with connection.cursor() as cursor: + if failure == "procedure-error": + cursor.execute( + f"CREATE PROCEDURE #failing_vector @v VECTOR(3, {base_type}) AS " + "BEGIN SET NOCOUNT ON; THROW 50000, 'expected procedure failure', 1; END" + ) + query = f"SELECT CAST(? AS VECTOR(3, {base_type})) AS embedding" + cursor.execute(query, value) + assert cursor.fetchval() == value + assert cursor.is_stmt_prepared == [True] + with pytest.raises(mssql_python.DatabaseError): + cursor.callproc("#failing_vector", [value]) + assert cursor.last_executed_stmt == "" + assert cursor.is_stmt_prepared == [False] + replacement = Vector([4, 5, 6], base_type=base_type) + cursor.execute(query, replacement) + assert cursor.fetchval() == replacement + + +@pytest.mark.parametrize( + "invalid,error", + [(None, ValueError), (1, TypeError), (True, TypeError), ("abc", TypeError), ([1], TypeError)], +) +def test_live_vector_rejects_mutated_inout(live_vector_connection, invalid, error): + connection, base_type = live_vector_connection + parameter = InputOutputParameter(Vector([1, 2, 3], base_type=base_type)) + parameter.value = invalid + with connection.cursor() as cursor: + with pytest.raises(error, match="INPUT_OUTPUT.*Vector"): + cursor.callproc("#invalid_inout", [parameter]) + cursor.execute("SELECT 1") + assert cursor.fetchval() == 1 + + @pytest.mark.parametrize("method", ["execute", "executemany"]) def test_live_vector_maximum_dimensions_and_rounding(live_vector_connection, method): connection, dtype = live_vector_connection From 84358b834789d6a85b7668132c5e61d351e18920 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 18 Nov 2026 13:17:28 +0530 Subject: [PATCH 4/7] FIX: Preserve TVP decimal/time values and buffered fetchall rowcount Bind TVP TIME columns as text so fractional seconds survive, size numeric/time text buffers separately from SQL precision, normalize non-Decimal DECIMAL/NUMERIC TVP cells like executemany(), and finalize rowcount when fetchall() exhausts an empty buffered procedure result. Add unit and live regressions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 4 +- mssql_python/cursor.py | 3 +- mssql_python/pybind/ddbc_bindings.cpp | 41 +++++++-- mssql_python/table_parameter.py | 25 +++++- tests/test_025_vector.py | 125 ++++++++++++++++++++++++++ 5 files changed, 185 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 422b21c41..034dc8344 100644 --- a/README.md +++ b/README.md @@ -257,7 +257,9 @@ explicit schema. Each column specification is `(SQL type, size, scale)`; for vectors, size means dimensions and scale is the subtype (0 or 1). Empty tables and NULL vector cells are supported. Vector columns also accept lists because the explicit schema provides dimensions and base type; cells are snapshotted -as Vectors with the same validation. UDT/sql_variant TVP columns and +as Vectors with the same validation. DECIMAL/NUMERIC cells are snapshotted as +`Decimal` (other values convert through `str()`, as in `executemany()`), and +TIME columns preserve fractional seconds. UDT/sql_variant TVP columns and data-at-execution TVP cells are not supported. ```python diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 056ffc4df..c0f3b1762 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -3296,7 +3296,8 @@ def fetchall(self) -> List[Row]: self._check_closed() # Check if the cursor is closed procedure_rows: deque[list[Any]] | None = getattr(self, "_procedure_rows", None) if procedure_rows is not None: - return self.fetchmany(len(procedure_rows)) if self.description else self.fetchmany(1) + # Request one extra row so exhaustion finalizes rowcount, even when empty. + return self.fetchmany(len(procedure_rows) + 1) if not self._has_result_set and self.description: self._reset_rownumber() diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index cc1bc9b78..e24ab5488 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -2556,24 +2556,36 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& LOG("BindParameterArray: Binding SQL_C_WCHAR array - " "param_index=%d, count=%zu, column_size=%zu", paramIndex, paramSetSize, info.columnSize); + // Numeric/time text can exceed SQL precision (sign, separators, fractions); + // size the buffer separately and keep columnSize as the SQL metadata. + size_t capacity = info.columnSize; + if (info.paramSQLType == SQL_DECIMAL || info.paramSQLType == SQL_NUMERIC || + info.paramSQLType == SQL_TYPE_TIME || info.paramSQLType == SQL_TIME || + info.paramSQLType == SQL_SS_TIME2) { + for (auto item : columnValues) { + if (!item.is_none()) + capacity = std::max(capacity, + item.cast().length()); + } + } SQLWCHAR* wcharArray = AllocateParamBufferArray( - tempBuffers, paramSetSize * (info.columnSize + 1)); + tempBuffers, paramSetSize * (capacity + 1)); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; - std::memset(wcharArray + i * (info.columnSize + 1), 0, - (info.columnSize + 1) * sizeof(SQLWCHAR)); + std::memset(wcharArray + i * (capacity + 1), 0, + (capacity + 1) * sizeof(SQLWCHAR)); } else { std::u16string wstr = columnValues[i].cast(); // u16string is already UTF-16, so the // original check is sufficient - if (wstr.length() > info.columnSize) { + if (wstr.length() > capacity) { ThrowStdException("Input string exceeds allowed column size " "at parameter index " + std::to_string(paramIndex)); } - std::memcpy(wcharArray + i * (info.columnSize + 1), wstr.c_str(), + std::memcpy(wcharArray + i * (capacity + 1), wstr.c_str(), (wstr.length() + 1) * sizeof(SQLWCHAR)); strLenOrIndArray[i] = SQL_NTS; } @@ -2582,7 +2594,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& "param_index=%d", paramIndex); dataPtr = wcharArray; - bufferLength = (info.columnSize + 1) * sizeof(SQLWCHAR); + bufferLength = (capacity + 1) * sizeof(SQLWCHAR); break; } case SQL_C_TINYINT: @@ -3117,6 +3129,10 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& } namespace { +bool IsTvpTimeType(int sqlType) { + return sqlType == SQL_TYPE_TIME || sqlType == SQL_TIME || sqlType == SQL_SS_TIME2; +} + SQLRETURN BindTableParameter(SqlHandle& handle, SQLHANDLE hStmt, SQLUSMALLINT ordinal, py::handle parameter, std::vector>& buffers, @@ -3160,8 +3176,10 @@ SQLRETURN BindTableParameter(SqlHandle& handle, SQLHANDLE hStmt, SQLUSMALLINT or } if (rows.empty()) column.append(py::none()); samples.append(sample); - overrides.append(py::make_tuple(sqlType, - cTypes[py::int_(sqlType)], spec[1], spec[2])); + // SQL_TIME_STRUCT has no fractional seconds; send time text as execute() does. + py::object cType = IsTvpTimeType(sqlType) ? py::int_(PARAM_C_TYPE_TEXT) + : cTypes[py::int_(sqlType)]; + overrides.append(py::make_tuple(sqlType, cType, spec[1], spec[2])); values.append(column); } // Apply the same explicit coercions to every cell, not just the sample row. @@ -3172,9 +3190,14 @@ SQLRETURN BindTableParameter(SqlHandle& handle, SQLHANDLE hStmt, SQLUSMALLINT or values[col].cast()[rowIndex] = normalized[col]; } auto infos = DetectParamTypes(samples.ptr(), overrides.ptr()); - for (auto& info : infos) { + for (size_t col = 0; col < infos.size(); ++col) { + auto& info = infos[col]; if (info.isDAE) throw py::value_error("TVP column values must fit their declared inline column sizes"); + // Keep the declared TIME size; text capacity is sized per value when binding. + auto spec = columns[col].cast(); + if (IsTvpTimeType(spec[0].cast()) && info.paramCType == PARAM_C_TYPE_TEXT) + info.columnSize = spec[1].cast(); } rc = SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, reinterpret_cast(static_cast(ordinal)), SQL_IS_INTEGER); diff --git a/mssql_python/table_parameter.py b/mssql_python/table_parameter.py index ff640747a..7d5fbd86e 100644 --- a/mssql_python/table_parameter.py +++ b/mssql_python/table_parameter.py @@ -2,11 +2,28 @@ from collections.abc import Iterable, Sequence from dataclasses import dataclass +from decimal import Decimal, InvalidOperation from typing import Any from .constants import SQLTypes, ConstantsDDBC from .vector import _check_shape, _vector_from_hint +_NUMERIC_TYPES = (ConstantsDDBC.SQL_DECIMAL.value, ConstantsDDBC.SQL_NUMERIC.value) + + +def _decimal_cell(value: Any, row: int, column: int) -> Any: + if value is None or isinstance(value, Decimal): + return value + # Match executemany(): convert through str() and never echo the rejected value. + try: + return Decimal(str(value)) + except (InvalidOperation, ValueError): + pass + raise ValueError( + f"Failed to convert TVP value to Decimal at row {row}, column {column} " + f"(value type: {type(value).__name__})" + ) + @dataclass(frozen=True, init=False) class TableValuedParameter: @@ -15,7 +32,9 @@ class TableValuedParameter: columns contains (sql_type, size, scale) hints, as for setinputsizes(). For vector columns, size is the dimension count and scale is 0 (FLOAT32) or 1 (FLOAT16). Vector cells accept Vector, list, or None; lists are - snapshotted as Vectors. Explicit schema also supports empty/all-NULL tables. + snapshotted as Vectors. DECIMAL/NUMERIC cells are snapshotted as Decimal, + converting other values through str() as executemany() does. Explicit + schema also supports empty/all-NULL tables. """ type_name: str @@ -57,13 +76,15 @@ def __init__( raise ValueError("vector subtype must be 0 (FLOAT32) or 1 (FLOAT16)") _check_shape(size, "float16" if scale else "float32") snapshot = [] - for row in rows: + for row_index, row in enumerate(rows): values = list(row) if len(values) != len(schema): raise ValueError("TVP rows must match the number of columns") for index, (sql_type, size, scale) in enumerate(schema): if sql_type == ConstantsDDBC.SQL_SS_VECTOR.value: values[index] = _vector_from_hint(values[index], size, scale) + elif sql_type in _NUMERIC_TYPES: + values[index] = _decimal_cell(values[index], row_index, index) snapshot.append(tuple(values)) object.__setattr__(self, "type_name", type_name) object.__setattr__(self, "columns", schema) diff --git a/tests/test_025_vector.py b/tests/test_025_vector.py index 3a564d592..4363768e8 100644 --- a/tests/test_025_vector.py +++ b/tests/test_025_vector.py @@ -6,12 +6,14 @@ vector type. No live operation is mocked; connection/backend errors fail. """ +import datetime import json import math import os import struct import uuid from collections import deque +from decimal import Decimal from itertools import repeat from types import SimpleNamespace from unittest.mock import MagicMock, Mock @@ -399,6 +401,49 @@ def execute(handle, sql, params, hints, prepared, use_prepare, settings): assert all(call.args[5] is True for call in execute.call_args_list) +def test_buffered_fetchall_finalizes_rowcount_for_empty_results(unit_cursor, monkeypatch): + value = Vector([1, 2, 3]) + monkeypatch.setattr( + ddbc_bindings, + "DDBCCallProc", + lambda *args: ( + [], + [ + (vector_metadata(), [], -1), + (vector_metadata(), [[value], [value]], -1), + (vector_metadata(), [], -1), + ], + ), + ) + unit_cursor.callproc("dbo.v") + assert unit_cursor.fetchall() == [] + assert unit_cursor.rowcount == 0 + assert unit_cursor.nextset() + assert [row[0] for row in unit_cursor.fetchall()] == [value, value] + assert unit_cursor.rowcount == 2 + assert unit_cursor.nextset() + assert unit_cursor.fetchall() == [] + assert unit_cursor.rowcount == 0 + + +def test_tvp_numeric_cells_are_normalized_like_executemany(): + columns = [(mssql_python.SQL_DECIMAL, 3, 2)] + rows = [(None,), (1,), (1.25,), (Decimal("-9.99"),), ("0.5",)] + parameter = TableValuedParameter("dbo.t", rows, columns) + assert parameter.rows == ( + (None,), + (Decimal("1"),), + (Decimal("1.25"),), + (Decimal("-9.99"),), + (Decimal("0.5"),), + ) + assert all(type(row[0]) is Decimal for row in parameter.rows[1:]) + for invalid in (True, "secret-value", object()): + with pytest.raises(ValueError, match="row 1, column 0") as error: + TableValuedParameter("dbo.t", [(1,), (invalid,)], columns) + assert "secret-value" not in str(error.value) + + def test_procedure_result_converters_are_refreshed(unit_cursor, monkeypatch): value = Vector([1, 2, 3]) unit_cursor.connection._output_converters[mssql_python.SQL_SS_VECTOR] = lambda v: v.dimensions @@ -936,6 +981,86 @@ def test_live_vector_tvp(live_vector_connection, method): cursor.execute(f"DROP TYPE {type_name}") +@pytest.mark.parametrize( + "time_spec", + [(mssql_python.SQL_TYPE_TIME, 16, 6), (mssql_python.SQL_SS_TIME2, 15, 6)], + ids=["sql-type-time", "ss-time2"], +) +@pytest.mark.parametrize("method", ["execute", "callproc"]) +def test_live_tvp_decimal_and_time_values(live_vector_connection, time_spec, method): + connection, dtype = live_vector_connection + suffix = uuid.uuid4().hex + type_name = f"dbo.pytest_tvp_types_{suffix}" + proc_name = f"dbo.pytest_tvp_types_proc_{suffix}" + connection.autocommit = True + value = Vector([1, 2, 3], dtype) + precise = datetime.time(12, 34, 56, 123456) + columns = [ + (mssql_python.SQL_INTEGER, 10, 0), + (mssql_python.SQL_DECIMAL, 3, 2), + time_spec, + (mssql_python.SQL_SS_VECTOR, 3, int(dtype == "float16")), + ] + rows = [ + (1, None, None, None), + (2, Decimal("-9.99"), precise, value), + (3, Decimal("9.99"), datetime.time(0, 0, 0, 1), value), + (4, 1, datetime.time(23, 59, 59, 999999), None), + (5, 1.25, precise, value), + (6, Decimal("-0.01"), datetime.time(1, 2, 3), value), + ] + expected = [ + (1, None, None, None), + (2, Decimal("-9.99"), precise, value), + (3, Decimal("9.99"), datetime.time(0, 0, 0, 1), value), + (4, Decimal("1.00"), datetime.time(23, 59, 59, 999999), None), + (5, Decimal("1.25"), precise, value), + (6, Decimal("-0.01"), datetime.time(1, 2, 3), value), + ] + with connection.cursor() as cursor: + cursor.execute( + f"CREATE TYPE {type_name} AS TABLE " + f"(id INT, amount DECIMAL(3, 2), at TIME(6), embedding VECTOR(3, {dtype}))" + ) + try: + cursor.execute( + f"CREATE PROCEDURE {proc_name} @rows {type_name} READONLY AS " + "BEGIN SET NOCOUNT ON; SELECT id, amount, at, embedding FROM @rows ORDER BY id; END" + ) + try: + parameter = TableValuedParameter(type_name, rows, columns) + if method == "callproc": + cursor.callproc(proc_name, [parameter]) + else: + cursor.execute(f"EXEC {proc_name} ?", parameter) + assert [tuple(row) for row in cursor.fetchall()] == expected + finally: + cursor.execute(f"DROP PROCEDURE {proc_name}") + finally: + cursor.execute(f"DROP TYPE {type_name}") + + +def test_live_buffered_procedure_empty_results_rowcount(live_vector_connection): + connection, dtype = live_vector_connection + value = Vector([1, 2, 3], dtype) + with connection.cursor() as cursor: + cursor.execute( + f"CREATE PROCEDURE #empty_vector_results @v VECTOR(3, {dtype}) AS " + "BEGIN SET NOCOUNT ON; SELECT @v AS embedding WHERE 1 = 0; " + "SELECT @v AS embedding; SELECT @v AS embedding WHERE 1 = 0; END" + ) + assert cursor.callproc("#empty_vector_results", [value]) == [value] + assert cursor.fetchall() == [] + assert cursor.rowcount == 0 + assert cursor.nextset() + assert [row[0] for row in cursor.fetchall()] == [value] + assert cursor.rowcount == 1 + assert cursor.nextset() + assert cursor.fetchall() == [] + assert cursor.rowcount == 0 + assert not cursor.nextset() + + @pytest.mark.parametrize("method", ["fetchone", "fetchmany", "fetchall", "iteration"]) @pytest.mark.parametrize("with_lob", [False, True], ids=["vector", "vector-with-lob"]) def test_live_vector_fetch_apis(live_vector_connection, method, with_lob): From c2dd361ed88b987a02934f5e4429a9589c628b9c Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 18 Nov 2026 18:34:36 +0530 Subject: [PATCH 5/7] FIX: Address vector review feedback Drop the PR-local TableValuedParameter and TVP binder so the shared TVP definition (#789) is the single public API; vector TVP columns will build on it. Restore main's array-binding buffer ownership. OutputParameter now takes the setinputsizes (sql_type, size, scale) triple; only SQL_SS_VECTOR is supported and other types raise NotSupportedError. InputOutputParameter infers its type from the initial Vector. Avoid per-cell ABC isinstance scans in executemany() and bulkcopy() by gating on exact Vector type. Build fetched and output vectors through a cached trusted constructor instead of revalidating driver-produced values. Add a bulk fast path for list/tuple float inputs with identical values and errors. Accept bracket-quoted callproc name parts and reject malformed brackets. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 9 +- README.md | 43 +-- mssql_python/__init__.py | 2 - mssql_python/cursor.py | 67 ++++- mssql_python/mssql_python.pyi | 1 - mssql_python/pybind/ddbc_bindings.cpp | 145 +---------- mssql_python/pybind/ddbc_bindings.h | 3 - mssql_python/pybind/param_detect.hpp | 8 - mssql_python/pybind/py_type_cache.hpp | 7 +- mssql_python/pybind/vector_support.hpp | 8 +- mssql_python/vector.py | 71 ++++- tests/test_025_vector.py | 347 +++++++++++++------------ tests/test_typing.py | 9 +- 13 files changed, 341 insertions(+), 379 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56af0b246..db7616423 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,10 +11,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), FLOAT32/FLOAT16 subtype metadata, scalar/array parameter binding, and row fetches. `base_type` selects SQL storage; values are normalized to float32 at construction, with early finite/float32 and conservative FLOAT16 range checks. - Explicit vector hints and TVP schemas accept numeric lists with the same - validation; unhinted lists keep their existing meanings. Added - `TableValuedParameter` and buffered `callproc()` with vector - `OutputParameter` / `InputOutputParameter` wrappers. Procedure calls invalidate + Explicit vector hints accept numeric lists with the same + validation; unhinted lists keep their existing meanings. Added buffered + `callproc()` (1-3 part names, optionally bracket-quoted) with + `OutputParameter(sql_type, size, scale)` / `InputOutputParameter` wrappers; + output parameters currently support `SQL_SS_VECTOR`. Procedure calls invalidate the previous prepared-query cache even when the procedure fails; overwritten non-vector INPUT_OUTPUT values are rejected before binding. FLOAT16 requires Microsoft ODBC Driver 18.7.1.1+ (not supplied by the current bundled driver); ODBC owns diff --git a/README.md b/README.md index 034dc8344..e6f88f9d2 100644 --- a/README.md +++ b/README.md @@ -252,49 +252,34 @@ to/from FLOAT16, including half-precision rounding and backend diagnostics; Python performs only the early finite/range checks above. Python does not pack TDS headers, encode half floats, or serialize vectors as binary/JSON. -**Table-valued parameters:** use an existing server-side table type and an -explicit schema. Each column specification is `(SQL type, size, scale)`; for -vectors, size means dimensions and scale is the subtype (0 or 1). Empty tables -and NULL vector cells are supported. Vector columns also accept lists because -the explicit schema provides dimensions and base type; cells are snapshotted -as Vectors with the same validation. DECIMAL/NUMERIC cells are snapshotted as -`Decimal` (other values convert through `str()`, as in `executemany()`), and -TIME columns preserve fractional seconds. UDT/sql_variant TVP columns and -data-at-execution TVP cells are not supported. - -```python -from mssql_python import TableValuedParameter, SQL_INTEGER - -# dbo.EmbeddingRows: (id INT, embedding VECTOR(3)) -table = TableValuedParameter( - "dbo.EmbeddingRows", - [(1, vector), (2, None)], - [(SQL_INTEGER, 10, 0), (SQL_SS_VECTOR, 3, 0)], -) -cursor.execute("EXEC dbo.InsertEmbeddings ?", table) -``` - **Stored procedures:** `callproc()` accepts ordinary input parameters and -vector `OutputParameter(dimensions, base_type)` / `InputOutputParameter(vector)` -wrappers. It returns a modified parameter sequence and also fills each wrapper's +`OutputParameter(sql_type, size, scale)` / `InputOutputParameter(vector)` +wrappers. `OutputParameter` takes the same triple as `setinputsizes()`; today +only `SQL_SS_VECTOR` is supported (size is dimensions, scale is 0 for FLOAT32 or +1 for FLOAT16), and other SQL types raise `NotSupportedError`. +`InputOutputParameter` infers its type from the initial `Vector`. `callproc()` +returns a modified parameter sequence and also fills each wrapper's `.value`. Result sets are buffered in memory so output values are final when the call returns; use normal row fetches and `nextset()` to read them. Input parameters may be `None`; parameter-description metadata is not exposed as -a procedure result set, including when reusing the cursor. +a procedure result set, including when reusing the cursor. Procedure names have +1-3 parts; parts may already be bracket-quoted, e.g. `[dbo].[my.proc]`. ```python from mssql_python import OutputParameter, InputOutputParameter # dbo.EchoEmbedding: @input VECTOR(3), @output VECTOR(3) OUTPUT, # @inout VECTOR(3) OUTPUT -output = OutputParameter(3) +output = OutputParameter(SQL_SS_VECTOR, 3, 0) inout = InputOutputParameter(vector) parameters = cursor.callproc("dbo.EchoEmbedding", [vector, output, inout]) print(output.value) ``` -Output wrappers currently support vectors only. Procedure text/binary inputs -must fit the inline binding limits. Arrow vector fetches and Arrow access to +Procedure text/binary inputs +must fit the inline binding limits. Table-valued parameters are not part of this +vector support; vector TVP columns will build on the shared TVP definition. +Arrow vector fetches and Arrow access to buffered procedure results are not supported; use row fetch methods instead. **Vector bulk copy is pending the pycore vector input contract.** `bulkcopy()` explicitly rejects `Vector` cells rather than guessing a serialization format. @@ -305,7 +290,7 @@ Vector unit tests run without SQL Server. End-to-end tests in scalar and batch binding (including explicitly hinted lists, pyformat, typed/all-NULL arrays and streamed text), numeric boundaries and error recovery, every row fetch method including `fetchval()`, metadata/row access, output converters, -commit/rollback, TVPs, stored procedure outputs/result sets, and error recovery. +commit/rollback, stored procedure outputs/result sets, and error recovery. Negotiation tests assert that v1 rejects native FLOAT16, `off` rejects native vector inputs, and `off` retains JSON/NULL reads. Arrow and bulk-copy cases assert the current explicit unsupported boundaries; they are not positive vector diff --git a/mssql_python/__init__.py b/mssql_python/__init__.py index 8f4ee0222..a9d86d291 100644 --- a/mssql_python/__init__.py +++ b/mssql_python/__init__.py @@ -67,7 +67,6 @@ # Row Objects from .row import Row, RowMapping from .vector import Vector, OutputParameter, InputOutputParameter -from .table_parameter import TableValuedParameter # Logging Configuration (Simplified single-level DEBUG system) from .logging import logger, setup_logging, driver_logger @@ -330,7 +329,6 @@ def _cleanup_connections() -> None: "Vector", "OutputParameter", "InputOutputParameter", - "TableValuedParameter", "SQL_SS_VECTOR", # Exception classes "Warning", diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index c0f3b1762..3b81bf612 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -121,6 +121,51 @@ def _normalize_time_param(value: object, c_type: int) -> str | None: return None +def _quote_procedure_name(procname: str) -> str: + """Bracket-quote a 1-3 part procedure name, accepting already bracketed parts. + + Unquoted parts split on ".". A part starting with "[" runs to its closing "]" + ("]]" escapes "]"), so it may contain ".". Malformed brackets raise ValueError. + """ + shape = "procname must be a procedure, schema.procedure or database.schema.procedure" + parts: list[str] = [] + index = 0 + while True: + if procname.startswith("[", index): + chunks = [] + start = index + 1 + while True: + close = procname.find("]", start) + if close < 0: + raise ValueError("procname has an unterminated [bracketed] part") + if procname.startswith("]", close + 1): + chunks.append(procname[start : close + 1]) + start = close + 2 + continue + chunks.append(procname[start:close]) + index = close + 1 + break + part = "".join(chunks) + if index < len(procname) and procname[index] != ".": + raise ValueError("procname has characters after a closing ]") + else: + end = procname.find(".", index) + end = len(procname) if end < 0 else end + part = procname[index:end] + index = end + if not part: + raise ValueError(shape) + parts.append(part) + if index == len(procname): + break + index += 1 # skip "." + if index == len(procname): + raise ValueError(shape) + if len(parts) > 3: + raise ValueError(shape) + return ".".join("[" + part.replace("]", "]]") + "]" for part in parts) + + class _ArrowBatchIterator: """Serialize generator advancement with close, including delegated Arrow reads.""" @@ -2519,14 +2564,14 @@ def _compute_column_type( non_nulls = [v for v in column if v is not None] if not non_nulls: return None, None, None, 0 - vectors = [v for v in non_nulls if isinstance(v, Vector)] - if vectors: - sample = vectors[0] - if len(vectors) != len(non_nulls): + # Gate on the first value so non-vector columns skip a per-cell ABC isinstance scan. + if type(non_nulls[0]) is Vector: + sample = non_nulls[0] + if any(type(v) is not Vector for v in non_nulls): raise TypeError("Vector batch columns must contain only Vector or None") if any( v.dimensions != sample.dimensions or v.base_type != sample.base_type - for v in vectors + for v in non_nulls ): raise ValueError("Vector batch dimensions and subtype must agree") return sample, None, None, 0 @@ -4588,7 +4633,7 @@ def _validate_bulkcopy_vectors( rows: Iterable[tuple[Any, ...]], ) -> Iterator[tuple[Any, ...]]: for row in rows: - if any(isinstance(value, Vector) for value in row): + if any(type(value) is Vector for value in row): raise NotSupportedError( driver_error="Vector bulk copy requires the developing pycore vector contract", ddbc_error="No vector serialization is performed by the Python wrapper", @@ -4631,22 +4676,18 @@ def callproc( A sequence containing the result parameters (input parameters unchanged, output parameters with their new values) - Vector outputs use OutputParameter(dimensions, base_type) or + Vector outputs use OutputParameter(SQL_SS_VECTOR, dimensions, subtype) or InputOutputParameter(Vector(...)). Results are buffered so output values are available on return; fetch methods and nextset() expose those results. + procname may be 1-3 parts; parts may already be bracket-quoted. """ self._check_closed() if not isinstance(procname, str) or not procname or "\0" in procname: raise ValueError("procname must be a non-empty procedure name") - parts = procname.split(".") - if not 1 <= len(parts) <= 3 or any(not part for part in parts): - raise ValueError( - "procname must be a procedure, schema.procedure or database.schema.procedure" - ) + name = _quote_procedure_name(procname) if isinstance(parameters, (str, bytes, bytearray)): raise TypeError("parameters must be a sequence of parameter values") values = list(parameters) if parameters is not None else [] - name = ".".join("[" + part.replace("]", "]]") + "]" for part in parts) query = "{CALL " + name + "(" + ",".join("?" for _ in values) + ")}" self._soft_reset_cursor() self.messages = [] diff --git a/mssql_python/mssql_python.pyi b/mssql_python/mssql_python.pyi index ff74b6c6b..e465f8128 100644 --- a/mssql_python/mssql_python.pyi +++ b/mssql_python/mssql_python.pyi @@ -29,7 +29,6 @@ from .vector import ( OutputParameter as OutputParameter, InputOutputParameter as InputOutputParameter, ) -from .table_parameter import TableValuedParameter as TableValuedParameter SQL_SS_VECTOR: int diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index e24ab5488..7bb165334 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -551,11 +551,6 @@ static SQLRETURN PreResolveUdtTypes(SqlHandle& handle, SQLHANDLE hStmt, // rebind over those records instead of issuing SQL_RESET_PARAMS first. An // unchanged IPD lets msodbcsql and mssql-odbc keep their prepared handle and // send sp_execute; any difference resets first, as before. -SQLRETURN BindTableParameter(SqlHandle& handle, SQLHANDLE hStmt, SQLUSMALLINT ordinal, - py::handle parameter, - std::vector>& buffers, - const std::string& charEncoding); - SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& params, std::vector& paramInfos, std::vector>& paramBuffers, @@ -580,10 +575,7 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par signature.push_back({info.inputOutputType, info.paramCType, info.paramSQLType, info.columnSize, info.decimalDigits, info.isDAE}); } - const bool hasTable = std::any_of(paramInfos.begin(), paramInfos.end(), - [](const ParamInfo& info) { return info.paramSQLType == SQL_SS_TABLE; }); - if (handle.paramsBound && - (signature.empty() || signature != handle.boundParamSignature || hasTable)) { + if (handle.paramsBound && (signature.empty() || signature != handle.boundParamSignature)) { SQLRETURN resetRc = ResetParamBindings(handle, hStmt); if (!SQL_SUCCEEDED(resetRc)) return resetRc; @@ -605,13 +597,6 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par SQLLEN bufferLength = 0; SQLLEN* strLenOrIndPtr = nullptr; - if (paramInfo.paramSQLType == SQL_SS_TABLE) { - SQLRETURN rc = BindTableParameter(handle, hStmt, - static_cast(paramIndex + 1), param, paramBuffers, charEncoding); - if (!SQL_SUCCEEDED(rc)) return rc; - continue; - } - switch (paramInfo.paramCType) { case SQL_C_SS_VECTOR: { if (!paramInfo.vectorBinding) { @@ -2445,7 +2430,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& "param_count=%zu, param_set_size=%zu", columnwise_params.size(), paramSetSize); - auto& tempBuffers = paramBuffers; + std::vector> tempBuffers; try { SQLRETURN describeRc = PreResolveUdtTypes(handle, hStmt, paramInfos); @@ -2556,36 +2541,24 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& LOG("BindParameterArray: Binding SQL_C_WCHAR array - " "param_index=%d, count=%zu, column_size=%zu", paramIndex, paramSetSize, info.columnSize); - // Numeric/time text can exceed SQL precision (sign, separators, fractions); - // size the buffer separately and keep columnSize as the SQL metadata. - size_t capacity = info.columnSize; - if (info.paramSQLType == SQL_DECIMAL || info.paramSQLType == SQL_NUMERIC || - info.paramSQLType == SQL_TYPE_TIME || info.paramSQLType == SQL_TIME || - info.paramSQLType == SQL_SS_TIME2) { - for (auto item : columnValues) { - if (!item.is_none()) - capacity = std::max(capacity, - item.cast().length()); - } - } SQLWCHAR* wcharArray = AllocateParamBufferArray( - tempBuffers, paramSetSize * (capacity + 1)); + tempBuffers, paramSetSize * (info.columnSize + 1)); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; - std::memset(wcharArray + i * (capacity + 1), 0, - (capacity + 1) * sizeof(SQLWCHAR)); + std::memset(wcharArray + i * (info.columnSize + 1), 0, + (info.columnSize + 1) * sizeof(SQLWCHAR)); } else { std::u16string wstr = columnValues[i].cast(); // u16string is already UTF-16, so the // original check is sufficient - if (wstr.length() > capacity) { + if (wstr.length() > info.columnSize) { ThrowStdException("Input string exceeds allowed column size " "at parameter index " + std::to_string(paramIndex)); } - std::memcpy(wcharArray + i * (capacity + 1), wstr.c_str(), + std::memcpy(wcharArray + i * (info.columnSize + 1), wstr.c_str(), (wstr.length() + 1) * sizeof(SQLWCHAR)); strLenOrIndArray[i] = SQL_NTS; } @@ -2594,7 +2567,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& "param_index=%d", paramIndex); dataPtr = wcharArray; - bufferLength = (capacity + 1) * sizeof(SQLWCHAR); + bufferLength = (info.columnSize + 1) * sizeof(SQLWCHAR); break; } case SQL_C_TINYINT: @@ -3122,111 +3095,13 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& "buffers"); throw; } + paramBuffers.insert(paramBuffers.end(), tempBuffers.begin(), tempBuffers.end()); LOG("BindParameterArray: Successfully bound all parameters - " "total_params=%zu, buffer_count=%zu", columnwise_params.size(), paramBuffers.size()); return SQL_SUCCESS; } -namespace { -bool IsTvpTimeType(int sqlType) { - return sqlType == SQL_TYPE_TIME || sqlType == SQL_TIME || sqlType == SQL_SS_TIME2; -} - -SQLRETURN BindTableParameter(SqlHandle& handle, SQLHANDLE hStmt, SQLUSMALLINT ordinal, - py::handle parameter, - std::vector>& buffers, - const std::string& charEncoding) { - auto rows = parameter.attr("rows").cast(); - auto columns = parameter.attr("columns").cast(); - auto name = parameter.attr("type_name").cast(); - auto separator = name.find(u'.'); - std::u16string schema; - if (separator != std::u16string::npos) { - schema = name.substr(0, separator); - name = name.substr(separator + 1); - } - auto typeName = AllocateParamBuffer(buffers, name); - auto rowCount = AllocateParamBuffer(buffers, static_cast(rows.size())); - SQLRETURN rc = SQLBindParameter_ptr(hStmt, ordinal, SQL_PARAM_INPUT, SQL_C_DEFAULT, - SQL_SS_TABLE, std::max(1, rows.size()), 0, - reinterpretU16stringAsSqlWChar(*typeName), SQL_NTS, rowCount); - if (!SQL_SUCCEEDED(rc)) return rc; - if (!schema.empty()) { - SQLHDESC ipd = nullptr; - rc = SQLGetStmtAttr_ptr(hStmt, SQL_ATTR_IMP_PARAM_DESC, &ipd, 0, nullptr); - if (!SQL_SUCCEEDED(rc)) return rc; - auto schemaName = AllocateParamBuffer(buffers, schema); - rc = SQLSetDescField_ptr(ipd, ordinal, SQL_CA_SS_SCHEMA_NAME, - reinterpretU16stringAsSqlWChar(*schemaName), SQL_NTS); - if (!SQL_SUCCEEDED(rc)) return rc; - } - py::list samples, overrides, values; - auto cTypes = py::module_::import("mssql_python.cursor") - .attr("Cursor").attr("_get_sql_to_c_type_map")().cast(); - for (size_t col = 0; col < columns.size(); ++col) { - auto spec = columns[col].cast(); - int sqlType = spec[0].cast(); - py::list column; - py::object sample = py::none(); - for (auto row : rows) { - py::object cell = row.cast()[col]; - column.append(cell); - if (sample.is_none() && !cell.is_none()) sample = cell; - } - if (rows.empty()) column.append(py::none()); - samples.append(sample); - // SQL_TIME_STRUCT has no fractional seconds; send time text as execute() does. - py::object cType = IsTvpTimeType(sqlType) ? py::int_(PARAM_C_TYPE_TEXT) - : cTypes[py::int_(sqlType)]; - overrides.append(py::make_tuple(sqlType, cType, spec[1], spec[2])); - values.append(column); - } - // Apply the same explicit coercions to every cell, not just the sample row. - for (size_t rowIndex = 0; rowIndex < rows.size(); ++rowIndex) { - py::list normalized(rows[rowIndex]); - DetectParamTypes(normalized.ptr(), overrides.ptr()); - for (size_t col = 0; col < columns.size(); ++col) - values[col].cast()[rowIndex] = normalized[col]; - } - auto infos = DetectParamTypes(samples.ptr(), overrides.ptr()); - for (size_t col = 0; col < infos.size(); ++col) { - auto& info = infos[col]; - if (info.isDAE) - throw py::value_error("TVP column values must fit their declared inline column sizes"); - // Keep the declared TIME size; text capacity is sized per value when binding. - auto spec = columns[col].cast(); - if (IsTvpTimeType(spec[0].cast()) && info.paramCType == PARAM_C_TYPE_TEXT) - info.columnSize = spec[1].cast(); - } - rc = SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, - reinterpret_cast(static_cast(ordinal)), SQL_IS_INTEGER); - if (!SQL_SUCCEEDED(rc)) return rc; - // Always restore focus, including when Python conversion or binding throws. - struct FocusGuard { - SQLHANDLE statement; - bool active = true; - ~FocusGuard() { - if (!active) return; - SQLRETURN result = SQLSetStmtAttr_ptr(statement, SQL_SOPT_SS_PARAM_FOCUS, nullptr, - SQL_IS_INTEGER); - if (!SQL_SUCCEEDED(result)) LOG_ERROR("Failed to restore TVP parameter focus"); - } - } focus{hStmt}; - rc = SQLSetStmtAttr_ptr(hStmt, SQL_ATTR_PARAMSET_SIZE, - reinterpret_cast(std::max(1, rows.size())), 0); - if (!SQL_SUCCEEDED(rc)) return rc; - rc = BindParameterArray(handle, hStmt, values, infos, std::max(1, rows.size()), - buffers, charEncoding); - if (!SQL_SUCCEEDED(rc)) - py::module_::import("mssql_python.helpers").attr("check_error")( - SQL_HANDLE_STMT, py::cast(&handle, py::return_value_policy::reference), rc); - SQLRETURN restore = SQLSetStmtAttr_ptr(hStmt, SQL_SOPT_SS_PARAM_FOCUS, nullptr, SQL_IS_INTEGER); - focus.active = false; - return SQL_SUCCEEDED(rc) ? restore : rc; -} -} // namespace - SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::u16string& query, const py::list& columnwise_params, std::vector& paramInfos, size_t paramSetSize, @@ -3277,7 +3152,7 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::u16 LOG("SQLExecuteMany: Using array binding (non-DAE) - calling " "BindParameterArray with encoding '%s'", charEncoding.c_str()); - auto& paramBuffers = statementHandle->boundParamBuffers; + std::vector> paramBuffers; rc = BindParameterArray(*statementHandle, hStmt, columnwise_params, paramInfos, paramSetSize, paramBuffers, charEncoding); if (!SQL_SUCCEEDED(rc)) { diff --git a/mssql_python/pybind/ddbc_bindings.h b/mssql_python/pybind/ddbc_bindings.h index 729f84251..28fbaad3c 100644 --- a/mssql_python/pybind/ddbc_bindings.h +++ b/mssql_python/pybind/ddbc_bindings.h @@ -58,9 +58,6 @@ using py::literals::operator""_a; #define SQL_CA_SS_VARIANT_TYPE (1215) #define SQL_SS_VECTOR (-156) #define SQL_C_SS_VECTOR (0x4002) -#define SQL_SS_TABLE (-153) -#define SQL_SOPT_SS_PARAM_FOCUS (1236) -#define SQL_CA_SS_SCHEMA_NAME (1226) // Include logger bridge for LOG macros #include "logger_bridge.hpp" diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index f6e88c573..8e44e85b4 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -349,14 +349,6 @@ inline std::vector DetectParamTypes(PyObject* params, PyObject* input info.decimalDigits = static_cast(info.vectorBinding->value.type); continue; } - int isTvp = PyObject_IsInstance(obj, PyTypeCache::tvp_class); - if (isTvp < 0) throw py::error_already_set(); - if (isTvp) { - info.paramSQLType = SQL_SS_TABLE; - info.paramCType = SQL_C_DEFAULT; - info.columnSize = static_cast(py::len(borrow(obj).attr("rows"))); - continue; - } // --- None --- if (obj == Py_None) { diff --git a/mssql_python/pybind/py_type_cache.hpp b/mssql_python/pybind/py_type_cache.hpp index 50c01f4d0..81288cd13 100644 --- a/mssql_python/pybind/py_type_cache.hpp +++ b/mssql_python/pybind/py_type_cache.hpp @@ -25,7 +25,7 @@ inline PyObject* time_class = nullptr; inline PyObject* decimal_class = nullptr; inline PyObject* uuid_class = nullptr; inline PyObject* vector_class = nullptr; -inline PyObject* tvp_class = nullptr; +inline PyObject* vector_from_trusted = nullptr; inline bool cache_initialized = false; // Import a module and extract an attribute. Returns a new reference. @@ -56,7 +56,8 @@ inline void initialize() { py::object dec_cls = steal(import_attr("decimal", "Decimal")); py::object uuid_cls = steal(import_attr("uuid", "UUID")); py::object vec_cls = steal(import_attr("mssql_python.vector", "Vector")); - py::object table_cls = steal(import_attr("mssql_python.table_parameter", "TableValuedParameter")); + py::object vec_trusted = steal(PyObject_GetAttrString(vec_cls.ptr(), "_from_trusted")); + if (!vec_trusted) throw py::error_already_set(); // Commit to globals — all acquisitions succeeded. datetime_class = dt_cls.release().ptr(); @@ -65,7 +66,7 @@ inline void initialize() { decimal_class = dec_cls.release().ptr(); uuid_class = uuid_cls.release().ptr(); vector_class = vec_cls.release().ptr(); - tvp_class = table_cls.release().ptr(); + vector_from_trusted = vec_trusted.release().ptr(); cache_initialized = true; } diff --git a/mssql_python/pybind/vector_support.hpp b/mssql_python/pybind/vector_support.hpp index f03a9c3f9..9c6640b40 100644 --- a/mssql_python/pybind/vector_support.hpp +++ b/mssql_python/pybind/vector_support.hpp @@ -3,6 +3,7 @@ #pragma once #include "ddbc_bindings.h" +#include "py_type_cache.hpp" #include #include @@ -70,7 +71,10 @@ struct VectorBinding { py::tuple values(value.dimension); for (size_t i = 0; i < static_cast(value.dimension); ++i) values[i] = py::float_(elements[i]); - return py::module_::import("mssql_python.vector") - .attr("Vector")(values, value.type == SQL_VECTOR_TYPE_FLOAT16 ? "float16" : "float32"); + // The shape check above covers what ODBC can get wrong, and SQL Server cannot + // store non-finite elements, so skip the public constructor's per-element checks. + PyTypeCache::initialize(); + return py::reinterpret_borrow(PyTypeCache::vector_from_trusted)( + values, value.type == SQL_VECTOR_TYPE_FLOAT16 ? "float16" : "float32"); } }; diff --git a/mssql_python/vector.py b/mssql_python/vector.py index 56f0530fa..6ff078dea 100644 --- a/mssql_python/vector.py +++ b/mssql_python/vector.py @@ -7,6 +7,9 @@ from numbers import Real from typing import Literal, Optional, Union, overload +from .constants import ConstantsDDBC +from .exceptions import NotSupportedError + VectorBaseType = Literal["float32", "float16"] _FLOAT32_MAX = 3.4028234663852886e38 _FLOAT16_MAX = 65504.0 @@ -23,6 +26,24 @@ def _check_shape(dimensions: int, base_type: VectorBaseType) -> None: raise ValueError(f"{base_type} vector dimensions must be between 1 and {limit}") +def _try_fast_init(vector: object, values: object, base_type: VectorBaseType) -> bool: + """Validate a list/tuple of plain floats in bulk; return False to use the checked loop.""" + if not isinstance(values, (list, tuple)) or type(values) not in (list, tuple): + return False + count = len(values) + if not 1 <= count <= (1998 if base_type == "float32" else 3996): + return False + if not all(type(value) is float for value in values): + return False + peak = _FLOAT16_MAX if base_type == "float16" else _FLOAT32_MAX + if not all(map(math.isfinite, values)) or max(map(abs, values)) > peak: + return False + layout = f"={count}f" + object.__setattr__(vector, "values", struct.unpack(layout, struct.pack(layout, *values))) + object.__setattr__(vector, "base_type", base_type) + return True + + @dataclass(frozen=True, init=False) class Vector(Sequence[float]): """An immutable vector with explicit SQL storage subtype. @@ -39,6 +60,8 @@ def __init__(self, values: Iterable[float], base_type: VectorBaseType = "float32 _check_shape(1, base_type) if isinstance(values, (str, bytes, bytearray)): raise TypeError("vector values must be an iterable of real numbers") + if _try_fast_init(self, values, base_type): + return converted: list[float] = [] for index, value in enumerate(values): if isinstance(value, bool) or not isinstance(value, Real): @@ -60,6 +83,14 @@ def __init__(self, values: Iterable[float], base_type: VectorBaseType = "float32 object.__setattr__(self, "values", tuple(converted)) object.__setattr__(self, "base_type", base_type) + @classmethod + def _from_trusted(cls, values: tuple[float, ...], base_type: VectorBaseType) -> "Vector": + """Wrap float32 values produced by the driver without revalidating them.""" + vector = object.__new__(cls) + object.__setattr__(vector, "values", values) + object.__setattr__(vector, "base_type", base_type) + return vector + @property def dimensions(self) -> int: return len(self.values) @@ -97,20 +128,46 @@ def _vector_from_hint(value: object, dimensions: int, subtype: int) -> Optional[ class OutputParameter: - """A vector OUTPUT parameter for callproc(); value is filled before return.""" + """An OUTPUT parameter for callproc(); value is filled before return. + + Takes the same (sql_type, size, scale) triple as setinputsizes(). Only + SQL_SS_VECTOR is supported today: size is the dimension count and scale is + 0 (FLOAT32) or 1 (FLOAT16). + """ - def __init__(self, dimensions: int, base_type: VectorBaseType = "float32") -> None: - _check_shape(dimensions, base_type) - self.dimensions = dimensions - self.base_type = base_type + def __init__(self, sql_type: int, size: int, scale: int = 0) -> None: + if sql_type != ConstantsDDBC.SQL_SS_VECTOR.value: + raise NotSupportedError( + driver_error="callproc() output parameters currently support SQL_SS_VECTOR only", + ddbc_error="Unsupported output parameter SQL type", + ) + if isinstance(scale, bool) or scale not in (0, 1): + raise ValueError("Vector subtype must be 0 (FLOAT32) or 1 (FLOAT16)") + base_type: VectorBaseType = "float16" if scale else "float32" + _check_shape(size, base_type) + self.sql_type = sql_type + self.size = size + self.scale = scale self.value: Optional[Vector] = None + @property + def dimensions(self) -> int: + return self.size + + @property + def base_type(self) -> VectorBaseType: + return "float16" if self.scale else "float32" + class InputOutputParameter(OutputParameter): - """A vector INPUT_OUTPUT parameter initialized from a Vector.""" + """An INPUT_OUTPUT parameter whose SQL type is inferred from its initial Vector.""" def __init__(self, value: Vector) -> None: if not isinstance(value, Vector): raise TypeError("InputOutputParameter requires a Vector") - super().__init__(value.dimensions, value.base_type) + super().__init__( + ConstantsDDBC.SQL_SS_VECTOR.value, + value.dimensions, + int(value.base_type == "float16"), + ) self.value = value diff --git a/tests/test_025_vector.py b/tests/test_025_vector.py index 4363768e8..9a0527c68 100644 --- a/tests/test_025_vector.py +++ b/tests/test_025_vector.py @@ -6,14 +6,12 @@ vector type. No live operation is mocked; connection/backend errors fail. """ -import datetime import json import math import os import struct import uuid from collections import deque -from decimal import Decimal from itertools import repeat from types import SimpleNamespace from unittest.mock import MagicMock, Mock @@ -21,7 +19,7 @@ import pytest import mssql_python -from mssql_python import InputOutputParameter, OutputParameter, TableValuedParameter, Vector +from mssql_python import InputOutputParameter, OutputParameter, Vector from mssql_python import ddbc_bindings from mssql_python.constants import ConstantsDDBC from mssql_python.cursor import Cursor @@ -62,27 +60,86 @@ def test_vector_is_immutable_and_preserves_input(): def test_vector_procedure_parameters(): - output = OutputParameter(3, base_type="float16") + output = OutputParameter(mssql_python.SQL_SS_VECTOR, 3, 1) + assert (output.sql_type, output.size, output.scale) == (mssql_python.SQL_SS_VECTOR, 3, 1) assert output.base_type == "float16" assert output.dimensions == 3 assert output.value is None + assert OutputParameter(mssql_python.SQL_SS_VECTOR, 3).base_type == "float32" value = Vector([1, 2, 3], base_type="float16") inout = InputOutputParameter(value) assert inout.value == value + assert (inout.sql_type, inout.size, inout.scale) == (mssql_python.SQL_SS_VECTOR, 3, 1) with pytest.raises(TypeError): InputOutputParameter([1, 2]) -def test_tvp_schema_and_rows(): - columns = [(mssql_python.SQL_INTEGER, 0, 0), (mssql_python.SQL_SS_VECTOR, 3, 1)] - value = Vector([1, 2, 3], base_type="float16") - tvp = TableValuedParameter("dbo.Embeddings", [(1, value), (2, None)], columns) - assert tvp.rows == ((1, value), (2, None)) - assert TableValuedParameter("dbo.Embeddings", [], columns).rows == () - with pytest.raises(ValueError, match="columns"): - TableValuedParameter("dbo.Embeddings", [(1,)], columns) - with pytest.raises(ValueError, match="dimensions|subtype"): - TableValuedParameter("dbo.Embeddings", [(1, Vector([1, 2]))], columns) +@pytest.mark.parametrize("sql_type", [mssql_python.SQL_INTEGER, mssql_python.SQL_WVARCHAR]) +def test_output_parameter_other_sql_types_not_supported_yet(sql_type): + with pytest.raises(mssql_python.NotSupportedError, match="SQL_SS_VECTOR"): + OutputParameter(sql_type, 10, 0) + + +@pytest.mark.parametrize("scale", [2, -1, True]) +def test_output_parameter_invalid_vector_subtype(scale): + with pytest.raises(ValueError, match="subtype"): + OutputParameter(mssql_python.SQL_SS_VECTOR, 3, scale) + + +@pytest.mark.parametrize("base_type,limit", [("float32", 1998), ("float16", 3996)]) +@pytest.mark.parametrize("container", [list, tuple]) +def test_vector_float_fast_path_matches_checked_loop(monkeypatch, base_type, limit, container): + import mssql_python.vector as vector_module + + values = [(-1) ** i * (i * 0.1 + 1e-40) for i in range(limit)] + values[1] = 65504.0 if base_type == "float16" else 3.4028234663852886e38 + expected = tuple(struct.unpack("=f", struct.pack("=f", value))[0] for value in values) + # The per-element packer is only used by the checked loop. + monkeypatch.setattr(vector_module, "_FLOAT32", None) + vector = Vector(container(values), base_type=base_type) + assert vector.values == expected + assert vector.base_type == base_type + + +@pytest.mark.parametrize("base_type", ["float32", "float16"]) +@pytest.mark.parametrize( + "invalid,error,message", + [ + (math.nan, ValueError, "index 1.*finite float32"), + (math.inf, ValueError, "index 1.*finite float32"), + (-3.5e38, ValueError, "index 1.*finite float32"), + (True, TypeError, "index 1"), + (None, TypeError, "index 1"), + ], +) +def test_vector_all_float_errors_match_checked_loop(base_type, invalid, error, message): + with pytest.raises(error, match=message): + Vector([0.0, invalid, 1.0], base_type=base_type) + + +@pytest.mark.parametrize("values", [[0.0, 65505.0], (0.0, -70000.0)]) +def test_vector_all_float_half_range_errors_match_checked_loop(values): + with pytest.raises(ValueError, match="index 1.*float16"): + Vector(values, base_type="float16") + + +@pytest.mark.parametrize("values", [[], [0.5] * 1999]) +def test_vector_all_float_shape_errors_match_checked_loop(values): + with pytest.raises(ValueError, match="dimensions"): + Vector(values) + + +def test_trusted_vector_construction_skips_public_validation(monkeypatch): + def reject(*args, **kwargs): + raise AssertionError("trusted construction must not call Vector.__init__") + + monkeypatch.setattr(Vector, "__init__", reject) + vector = Vector._from_trusted((1.25, -2.5), "float16") + assert vector.values == (1.25, -2.5) + assert vector.base_type == "float16" + assert vector.dimensions == 2 + with pytest.raises(AttributeError): + vector.values = () @pytest.mark.parametrize("mode", ["off", "v1", "v2"]) @@ -222,22 +279,10 @@ def test_native_fetch_policy_excludes_buffered_procedures(rows): assert policy(Cursor.__new__(Cursor), {"_procedure_rows": deque(rows)}) is None -def test_tvp_vector_lists_use_explicit_schema_and_snapshot(): - source = [0.1, 2, 3] - columns = [(mssql_python.SQL_SS_VECTOR, 3, 1)] - parameter = TableValuedParameter("dbo.v", [(source,), (None,)], columns) - expected = Vector(source, base_type="float16") - source[0] = 9 - assert parameter.rows == ((expected,), (None,)) - for invalid, error in (([1, 2], ValueError), ([1e40, 2, 3], ValueError), ("123", TypeError)): - with pytest.raises(error): - TableValuedParameter("dbo.v", [(invalid,)], columns) - - @pytest.mark.parametrize("dimensions", [True, 1.5, "3"]) def test_output_parameter_invalid_shape_type(dimensions): with pytest.raises(TypeError, match="integer"): - OutputParameter(dimensions) + OutputParameter(mssql_python.SQL_SS_VECTOR, dimensions, 0) @pytest.mark.parametrize("mode", ["", "v3", "true"]) @@ -258,26 +303,6 @@ def test_vector_negotiation_is_opt_in(): assert parser._normalize_params({"vectortypesupport": "V1"})["VectorTypeSupport"] == "v1" -@pytest.mark.parametrize("name", ["", "dbo..v", "dbo.v;drop", "a.b.c", "dbo.\0v"]) -def test_invalid_tvp_name(name): - with pytest.raises(ValueError): - TableValuedParameter(name, [], [(mssql_python.SQL_SS_VECTOR, 3, 0)]) - - -def test_tvp_immutable_snapshot_and_native_detection(): - rows = [[Vector([1, 2, 3])]] - columns = [[mssql_python.SQL_SS_VECTOR, 3, 0]] - parameter = TableValuedParameter("dbo.v", rows, columns) - rows.clear() - columns.clear() - assert len(parameter.rows) == 1 - with pytest.raises(AttributeError): - parameter.rows = () - info = ddbc_bindings._detect_param_types([parameter])[0] - assert info.paramSQLType == -153 - assert not info.isDAE - - @pytest.fixture def unit_cursor(monkeypatch): monkeypatch.setattr(Cursor, "_initialize_cursor", lambda self: None) @@ -309,7 +334,7 @@ def vector_metadata(dtype="float32"): def test_callproc_buffered_results_and_cursor_reuse(unit_cursor, monkeypatch): vectors = [Vector([i, 2, 3]) for i in range(4)] - output = OutputParameter(3) + output = OutputParameter(mssql_python.SQL_SS_VECTOR, 3, 0) def callproc(handle, sql, params, settings, messages, *decoding): assert sql == "{CALL [dbo].[a]]b](?,?)}" @@ -351,6 +376,66 @@ def test_callproc_invalid_name_without_database(unit_cursor, name): unit_cursor.callproc(name) +@pytest.mark.parametrize( + "name,expected", + [ + ("dbo.vecproc", "[dbo].[vecproc]"), + ("[dbo].[vecproc]", "[dbo].[vecproc]"), + ("[dbo].[vec.proc]", "[dbo].[vec.proc]"), + ("db.[dbo].vecproc", "[db].[dbo].[vecproc]"), + ("[a]]b].c", "[a]]b].[c]"), + ("dbo.a]b", "[dbo].[a]]b]"), + ("[#temp.proc]", "[#temp.proc]"), + ], +) +def test_callproc_accepts_bracketed_names(unit_cursor, monkeypatch, name, expected): + calls = [] + monkeypatch.setattr( + ddbc_bindings, "DDBCCallProc", lambda handle, sql, *args: calls.append(sql) or ([], []) + ) + unit_cursor.callproc(name) + assert calls == ["{CALL " + expected + "()}"] + + +@pytest.mark.parametrize( + "name,message", + [ + ("[dbo", "unterminated"), + ("[dbo]x.p", "after a closing"), + ("[].p", "procedure"), + ("[a].[b].[c].[d]", "procedure"), + ("[dbo].", "procedure"), + ], +) +def test_callproc_rejects_malformed_bracketed_names(unit_cursor, monkeypatch, name, message): + native = Mock() + monkeypatch.setattr(ddbc_bindings, "DDBCCallProc", native) + with pytest.raises(ValueError, match=message): + unit_cursor.callproc(name) + native.assert_not_called() + + +def test_executemany_non_vector_columns_skip_vector_instance_checks(unit_cursor, monkeypatch): + from abc import ABCMeta + + import mssql_python.cursor as cursor_module + + checks = [] + + class CountingMeta(ABCMeta): + def __instancecheck__(cls, instance): + checks.append(instance) + return super().__instancecheck__(instance) + + class Probe(metaclass=CountingMeta): + pass + + monkeypatch.setattr(cursor_module, "Vector", Probe) + unit_cursor._compute_column_type([None, 1, 2, 3]) + unit_cursor._compute_column_type(["a", None, "b"]) + assert checks == [] + + def test_buffered_procedure_rejects_arrow_and_releases_rows(unit_cursor, monkeypatch): monkeypatch.setattr( ddbc_bindings, @@ -426,24 +511,6 @@ def test_buffered_fetchall_finalizes_rowcount_for_empty_results(unit_cursor, mon assert unit_cursor.rowcount == 0 -def test_tvp_numeric_cells_are_normalized_like_executemany(): - columns = [(mssql_python.SQL_DECIMAL, 3, 2)] - rows = [(None,), (1,), (1.25,), (Decimal("-9.99"),), ("0.5",)] - parameter = TableValuedParameter("dbo.t", rows, columns) - assert parameter.rows == ( - (None,), - (Decimal("1"),), - (Decimal("1.25"),), - (Decimal("-9.99"),), - (Decimal("0.5"),), - ) - assert all(type(row[0]) is Decimal for row in parameter.rows[1:]) - for invalid in (True, "secret-value", object()): - with pytest.raises(ValueError, match="row 1, column 0") as error: - TableValuedParameter("dbo.t", [(1,), (invalid,)], columns) - assert "secret-value" not in str(error.value) - - def test_procedure_result_converters_are_refreshed(unit_cursor, monkeypatch): value = Vector([1, 2, 3]) unit_cursor.connection._output_converters[mssql_python.SQL_SS_VECTOR] = lambda v: v.dimensions @@ -854,7 +921,7 @@ def test_live_vector_procedure_outputs(live_vector_connection, with_results): f"@o VECTOR(3, {dtype}) OUTPUT, @io VECTOR(3, {dtype}) OUTPUT AS " f"BEGIN SET NOCOUNT ON; SET @o=@i; SET @io=@i; {selects} END" ) - output = OutputParameter(3, dtype) + output = OutputParameter(mssql_python.SQL_SS_VECTOR, 3, int(dtype == "float16")) inout = InputOutputParameter(Vector([0, 0, 0], dtype)) assert cursor.callproc("#vector_echo", [value, output, inout]) == [value] * 3 assert output.value == inout.value == value @@ -941,105 +1008,6 @@ def test_live_vector_maximum_dimensions_and_rounding(live_vector_connection, met assert [row[0] for row in cursor.fetchall()] == [None, actual] -@pytest.mark.parametrize("method", ["execute", "callproc"]) -def test_live_vector_tvp(live_vector_connection, method): - connection, dtype = live_vector_connection - suffix = uuid.uuid4().hex - type_name = f"dbo.pytest_vector_type_{suffix}" - proc_name = f"dbo.pytest_vector_proc_{suffix}" - # Named types cannot be temporary; use unique names and scoped cleanup. - connection.autocommit = True - value = Vector([1.25, -2.5, 0], dtype) - with connection.cursor() as cursor: - cursor.execute(f"CREATE TYPE {type_name} AS TABLE (id INT, embedding VECTOR(3, {dtype}))") - try: - cursor.execute( - f"CREATE PROCEDURE {proc_name} @rows {type_name} READONLY AS " - "BEGIN SET NOCOUNT ON; SELECT id, embedding FROM @rows ORDER BY id; END" - ) - try: - columns = [ - (mssql_python.SQL_INTEGER, 10, 0), - (mssql_python.SQL_SS_VECTOR, 3, int(dtype == "float16")), - ] - for rows in ( - [(1, value), (2, None), (3, value)], - [(1, [1.25, -2.5, 0])], - [(1, None), (2, None)], - [], - ): - parameter = TableValuedParameter(type_name, rows, columns) - if method == "callproc": - assert cursor.callproc(proc_name, [parameter]) == [parameter] - else: - cursor.execute(f"EXEC {proc_name} ?", parameter) - assert cursor.description[1][1] is Vector - assert [tuple(row) for row in cursor.fetchall()] == list(parameter.rows) - finally: - cursor.execute(f"DROP PROCEDURE {proc_name}") - finally: - cursor.execute(f"DROP TYPE {type_name}") - - -@pytest.mark.parametrize( - "time_spec", - [(mssql_python.SQL_TYPE_TIME, 16, 6), (mssql_python.SQL_SS_TIME2, 15, 6)], - ids=["sql-type-time", "ss-time2"], -) -@pytest.mark.parametrize("method", ["execute", "callproc"]) -def test_live_tvp_decimal_and_time_values(live_vector_connection, time_spec, method): - connection, dtype = live_vector_connection - suffix = uuid.uuid4().hex - type_name = f"dbo.pytest_tvp_types_{suffix}" - proc_name = f"dbo.pytest_tvp_types_proc_{suffix}" - connection.autocommit = True - value = Vector([1, 2, 3], dtype) - precise = datetime.time(12, 34, 56, 123456) - columns = [ - (mssql_python.SQL_INTEGER, 10, 0), - (mssql_python.SQL_DECIMAL, 3, 2), - time_spec, - (mssql_python.SQL_SS_VECTOR, 3, int(dtype == "float16")), - ] - rows = [ - (1, None, None, None), - (2, Decimal("-9.99"), precise, value), - (3, Decimal("9.99"), datetime.time(0, 0, 0, 1), value), - (4, 1, datetime.time(23, 59, 59, 999999), None), - (5, 1.25, precise, value), - (6, Decimal("-0.01"), datetime.time(1, 2, 3), value), - ] - expected = [ - (1, None, None, None), - (2, Decimal("-9.99"), precise, value), - (3, Decimal("9.99"), datetime.time(0, 0, 0, 1), value), - (4, Decimal("1.00"), datetime.time(23, 59, 59, 999999), None), - (5, Decimal("1.25"), precise, value), - (6, Decimal("-0.01"), datetime.time(1, 2, 3), value), - ] - with connection.cursor() as cursor: - cursor.execute( - f"CREATE TYPE {type_name} AS TABLE " - f"(id INT, amount DECIMAL(3, 2), at TIME(6), embedding VECTOR(3, {dtype}))" - ) - try: - cursor.execute( - f"CREATE PROCEDURE {proc_name} @rows {type_name} READONLY AS " - "BEGIN SET NOCOUNT ON; SELECT id, amount, at, embedding FROM @rows ORDER BY id; END" - ) - try: - parameter = TableValuedParameter(type_name, rows, columns) - if method == "callproc": - cursor.callproc(proc_name, [parameter]) - else: - cursor.execute(f"EXEC {proc_name} ?", parameter) - assert [tuple(row) for row in cursor.fetchall()] == expected - finally: - cursor.execute(f"DROP PROCEDURE {proc_name}") - finally: - cursor.execute(f"DROP TYPE {type_name}") - - def test_live_buffered_procedure_empty_results_rowcount(live_vector_connection): connection, dtype = live_vector_connection value = Vector([1, 2, 3], dtype) @@ -1061,6 +1029,53 @@ def test_live_buffered_procedure_empty_results_rowcount(live_vector_connection): assert not cursor.nextset() +@pytest.mark.parametrize("name", ["[#quoted.vector]", "[#quoted]]vector]"]) +def test_live_callproc_bracketed_names(live_vector_connection, name): + connection, dtype = live_vector_connection + value = Vector([1, 2, 3], dtype) + with connection.cursor() as cursor: + cursor.execute( + f"CREATE PROCEDURE {name} @v VECTOR(3, {dtype}) AS " + "BEGIN SET NOCOUNT ON; SELECT @v AS embedding; END" + ) + assert cursor.callproc(name, [value]) == [value] + assert cursor.fetchone()[0] == value + + +def test_live_fetch_and_outputs_use_trusted_vector_construction( + live_vector_connection, monkeypatch +): + connection, dtype = live_vector_connection + value = Vector([1.25, -2.5, 0], dtype) + output = OutputParameter(mssql_python.SQL_SS_VECTOR, 3, int(dtype == "float16")) + with connection.cursor() as cursor: + cursor.execute( + f"CREATE PROCEDURE #trusted_vector @i VECTOR(3, {dtype}), " + f"@o VECTOR(3, {dtype}) OUTPUT AS BEGIN SET NOCOUNT ON; SET @o=@i; END" + ) + cursor.execute(f"CREATE TABLE #trusted (id INT, embedding VECTOR(3, {dtype}))") + cursor.executemany( + "INSERT INTO #trusted VALUES (?, ?)", [(i, value) for i in range(50)] + [(50, None)] + ) + # Values produced by the driver must not pass through the public constructor. + public_init = Vector.__init__ + calls = [] + + def counting_init(self, *args, **kwargs): + calls.append(args) + public_init(self, *args, **kwargs) + + monkeypatch.setattr(Vector, "__init__", counting_init) + cursor.execute("SELECT embedding FROM #trusted ORDER BY id") + rows = [row[0] for row in cursor.fetchall()] + assert rows == [value] * 50 + [None] + assert all(row.base_type == dtype for row in rows[:-1]) + assert calls == [] + monkeypatch.setattr(Vector, "__init__", public_init) + cursor.callproc("#trusted_vector", [value, output]) + assert output.value == value + + @pytest.mark.parametrize("method", ["fetchone", "fetchmany", "fetchall", "iteration"]) @pytest.mark.parametrize("with_lob", [False, True], ids=["vector", "vector-with-lob"]) def test_live_vector_fetch_apis(live_vector_connection, method, with_lob): diff --git a/tests/test_typing.py b/tests/test_typing.py index 1859692d2..07b26d953 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -102,8 +102,7 @@ def test_vector_consumer_call_shapes(tmp_path: Path, module: str) -> None: usage.write_text( f"""\ from {module} import ( - Cursor, Vector, OutputParameter, InputOutputParameter, - TableValuedParameter, SQL_SS_VECTOR, + Cursor, Vector, OutputParameter, InputOutputParameter, SQL_SS_VECTOR, ) def supported_calls(cursor: Cursor) -> None: @@ -111,14 +110,12 @@ def supported_calls(cursor: Cursor) -> None: values: list[float] = list(vector) dimensions: int = vector.dimensions base_type: str = vector.base_type - output = OutputParameter(3, base_type="float16") + output = OutputParameter(SQL_SS_VECTOR, 3, 1) inout = InputOutputParameter(vector) - table = TableValuedParameter("dbo.v", [([1, 2, 3],)], [(SQL_SS_VECTOR, 3, 1)]) cursor.execute("SELECT ?", [vector]) - cursor.execute("EXEC dbo.p ?", table) cursor.setinputsizes([(SQL_SS_VECTOR, 3, 1)]) cursor.executemany("INSERT INTO v VALUES (?)", [([1, 2, 3],), (None,), (vector,)]) - cursor.callproc("dbo.p", [vector, output, inout]) + cursor.callproc("[dbo].[p]", [vector, output, inout]) """, encoding="utf-8", ) From 754b910806185bad2d3db065c8f197e151a47f7f Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 18 Nov 2026 18:36:46 +0530 Subject: [PATCH 6/7] FIX: Remove unused TableValuedParameter module Complete dropping the PR-local TVP API in favor of the shared definition (#789); nothing imports this module any more. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mssql_python/table_parameter.py | 91 --------------------------------- 1 file changed, 91 deletions(-) delete mode 100644 mssql_python/table_parameter.py diff --git a/mssql_python/table_parameter.py b/mssql_python/table_parameter.py deleted file mode 100644 index 7d5fbd86e..000000000 --- a/mssql_python/table_parameter.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Explicit table-valued parameter schema and immutable row snapshots.""" - -from collections.abc import Iterable, Sequence -from dataclasses import dataclass -from decimal import Decimal, InvalidOperation -from typing import Any - -from .constants import SQLTypes, ConstantsDDBC -from .vector import _check_shape, _vector_from_hint - -_NUMERIC_TYPES = (ConstantsDDBC.SQL_DECIMAL.value, ConstantsDDBC.SQL_NUMERIC.value) - - -def _decimal_cell(value: Any, row: int, column: int) -> Any: - if value is None or isinstance(value, Decimal): - return value - # Match executemany(): convert through str() and never echo the rejected value. - try: - return Decimal(str(value)) - except (InvalidOperation, ValueError): - pass - raise ValueError( - f"Failed to convert TVP value to Decimal at row {row}, column {column} " - f"(value type: {type(value).__name__})" - ) - - -@dataclass(frozen=True, init=False) -class TableValuedParameter: - """A READONLY TVP. - - columns contains (sql_type, size, scale) hints, as for setinputsizes(). - For vector columns, size is the dimension count and scale is 0 (FLOAT32) - or 1 (FLOAT16). Vector cells accept Vector, list, or None; lists are - snapshotted as Vectors. DECIMAL/NUMERIC cells are snapshotted as Decimal, - converting other values through str() as executemany() does. Explicit - schema also supports empty/all-NULL tables. - """ - - type_name: str - columns: tuple[tuple[int, int, int], ...] - rows: tuple[tuple[Any, ...], ...] - - def __init__( - self, - type_name: str, - rows: Iterable[Sequence[Any]], - columns: Sequence[tuple[int, int, int]], - ) -> None: - if not isinstance(type_name, str) or not type_name or "\0" in type_name: - raise ValueError("type_name must be a non-empty SQL table type name") - parts = type_name.split(".") - if len(parts) not in (1, 2) or any( - not part or not all(c.isalnum() or c in "_@" for c in part) for part in parts - ): - raise ValueError("type_name must be an unquoted type or schema.type name") - schema = tuple(tuple(column) for column in columns) - if not schema: - raise ValueError("columns must contain at least one column") - valid = SQLTypes.get_valid_types() - for column in schema: - if len(column) != 3 or any( - isinstance(item, bool) or not isinstance(item, int) for item in column - ): - raise ValueError("columns must contain (sql_type, size, scale) integer tuples") - sql_type, size, scale = column - if sql_type not in valid or size < 0 or scale < 0: - raise ValueError("invalid TVP column type, size or scale") - if sql_type in ( - ConstantsDDBC.SQL_SS_UDT.value, - ConstantsDDBC.SQL_SS_VARIANT.value, - ): - raise ValueError("UDT and sql_variant TVP columns are not supported") - if sql_type == ConstantsDDBC.SQL_SS_VECTOR.value: - if scale not in (0, 1): - raise ValueError("vector subtype must be 0 (FLOAT32) or 1 (FLOAT16)") - _check_shape(size, "float16" if scale else "float32") - snapshot = [] - for row_index, row in enumerate(rows): - values = list(row) - if len(values) != len(schema): - raise ValueError("TVP rows must match the number of columns") - for index, (sql_type, size, scale) in enumerate(schema): - if sql_type == ConstantsDDBC.SQL_SS_VECTOR.value: - values[index] = _vector_from_hint(values[index], size, scale) - elif sql_type in _NUMERIC_TYPES: - values[index] = _decimal_cell(values[index], row_index, index) - snapshot.append(tuple(values)) - object.__setattr__(self, "type_name", type_name) - object.__setattr__(self, "columns", schema) - object.__setattr__(self, "rows", tuple(snapshot)) From b14c0bb2554256c85e9656a272f799372a42a0c4 Mon Sep 17 00:00:00 2001 From: gargsaumya Date: Wed, 18 Nov 2026 22:00:56 +0530 Subject: [PATCH 7/7] DOC: Document SQL Server 2025 FLOAT16 preview and server-side conversion FLOAT16 is preview-only on SQL Server 2025 (PREVIEW_FEATURES) and GA on Azure SQL DB, MI and Fabric. The driver never converts between vector base types; SQL Server converts native FLOAT32/FLOAT16 parameters to the column type, matching Microsoft.Data.SqlClient. Add a live test pinning conversion, rounding and the out-of-range error. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 14 +++++++++++--- tests/test_025_vector.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e6f88f9d2..4d34f940d 100644 --- a/README.md +++ b/README.md @@ -183,21 +183,29 @@ is represented by `None`. FLOAT32 supports 1–1998 dimensions; FLOAT16 supports Enable native vector negotiation **when connecting**: `VectorTypeSupport=v1` for FLOAT32, or `VectorTypeSupport=v2` for both subtypes. The default remains `off` (the ODBC driver's existing JSON/text behavior). FLOAT32 requires Microsoft -ODBC Driver 18.6.1.1+; FLOAT16 requires **18.7.1.1+** and a server configured to -support FLOAT16 vectors. The currently published `mssql-python-odbc` 18.6.2.2 +ODBC Driver 18.6.1.1+; FLOAT16 requires **18.7.1.1+** and a server that +supports FLOAT16 vectors. FLOAT16 is generally available in Azure SQL Database, +Azure SQL Managed Instance and SQL database in Fabric; in SQL Server 2025 it is a +preview feature that requires `ALTER DATABASE SCOPED CONFIGURATION SET +PREVIEW_FEATURES = ON`. FLOAT32 works on all of these. The currently published `mssql-python-odbc` 18.6.2.2 package does not meet the FLOAT16 requirement. These Python changes do not upgrade the bundled driver or add support to the alternate Rust ODBC provider. | Connection setting | Native FLOAT32 | Native FLOAT16 | |---|---|---| | `VectorTypeSupport=off` (default) | No; reads return JSON text | No; reads return JSON text | -| `VectorTypeSupport=v1` | Yes | No; native FLOAT16 inputs are rejected by ODBC | +| `VectorTypeSupport=v1` | Yes | No; FLOAT16 inputs are rejected by ODBC and FLOAT16 columns are read as JSON text | | `VectorTypeSupport=v2` | Yes | Yes | `Vector(values, base_type="float16")` selects the value's SQL base type; it does not upgrade a connection from v1 to v2. The connection setting is passed through to ODBC for negotiation rather than emulated in Python. +The driver never converts between FLOAT32 and FLOAT16. A `Vector` is sent with +its own base type and SQL Server converts it to the column's base type, +rounding to nearest; FLOAT32 values outside the FLOAT16 range fail with a server +error. This matches Microsoft.Data.SqlClient. + ```python from mssql_python import connect, Vector, SQL_SS_VECTOR diff --git a/tests/test_025_vector.py b/tests/test_025_vector.py index 9a0527c68..cd336ca82 100644 --- a/tests/test_025_vector.py +++ b/tests/test_025_vector.py @@ -1319,3 +1319,33 @@ def test_live_vector_off_returns_json(conn_str, vector_server, dtype): assert cursor.fetchone()[0] is None finally: connection.close() + + +def test_live_vector_server_converts_between_base_types(conn_str, vector_server): + if not conn_str: + pytest.skip("DB_CONNECTION_STRING is not configured") + if os.getenv("MSSQL_TEST_VECTOR_FLOAT16") != "1": + pytest.skip("Set MSSQL_TEST_VECTOR_FLOAT16=1 with a FLOAT16-capable server") + connection = mssql_python.connect(conn_str, VectorTypeSupport="v2", autocommit=True) + try: + with connection.cursor() as cursor: + cursor.execute("CREATE TABLE #cross (f16 VECTOR(3, float16) NULL, f32 VECTOR(3) NULL)") + # The driver sends each Vector with its own base type; SQL Server converts. + cursor.execute("INSERT INTO #cross (f16) VALUES (?)", Vector([0.1, 65519.0, 1e-8])) + cursor.execute( + "INSERT INTO #cross (f32) VALUES (?)", Vector([0.1, 2, 3], base_type="float16") + ) + cursor.execute("SELECT f16 FROM #cross WHERE f16 IS NOT NULL") + stored = cursor.fetchval() + assert stored.base_type == "float16" + assert stored == _expected_stored_vector([0.1, 65504.0, 0.0], "float16") + cursor.execute("SELECT f32 FROM #cross WHERE f32 IS NOT NULL") + widened = cursor.fetchval() + assert widened.base_type == "float32" + assert widened.values == _expected_stored_vector([0.1, 2, 3], "float16").values + with pytest.raises(mssql_python.Error, match="out-of-range"): + cursor.execute("INSERT INTO #cross (f16) VALUES (?)", Vector([70000.0, 0, 0])) + cursor.execute("SELECT COUNT(*) FROM #cross") + assert cursor.fetchval() == 2 + finally: + connection.close()