Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions docs/query.rst
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,33 @@ characters appear, in the order they appear.
Always use the parameter interpolation feature of the client library to
guard against malicious input, as demonstrated in the example above.

Named parameters
----------------

For queries with many parameters or repeated values, named parameters improve
readability. Pass a :class:`py:dict` as the second argument using
``%(name)s`` placeholders:

>>> cursor.execute(
... "INSERT INTO locations (name, date, kind, position) "
... "VALUES (%(name)s, %(date)s, %(kind)s, %(pos)s)",
... {"name": "Einstein Cross", "date": "2007-03-11", "kind": "Quasar", "pos": 7})

The same parameter name may appear multiple times in the query:

>>> cursor.execute(
... "SELECT * FROM locations WHERE name = %(q)s OR kind = %(q)s",
... {"q": "Quasar"})

The client converts the ``%(name)s`` placeholders to positional ``?`` markers
before sending the query to CrateDB, so no server-side changes are required.

.. NOTE::

Named parameters are not yet supported by ``executemany()``. Use
positional ``?`` placeholders with a :class:`py:list` of tuples for bulk
operations.

Bulk inserts
------------

Expand Down
2 changes: 1 addition & 1 deletion src/crate/client/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,4 +46,4 @@
# codeql[py/unused-global-variable]
apilevel = "2.0"
threadsafety = 1
paramstyle = "qmark"
paramstyle = "pyformat"
Comment thread Dismissed
4 changes: 4 additions & 0 deletions src/crate/client/cursor.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

from .converter import Converter, DataType
from .exceptions import ProgrammingError
from .params import convert_named_to_positional


class Cursor:
Expand Down Expand Up @@ -54,6 +55,9 @@ def execute(self, sql, parameters=None, bulk_parameters=None):
if self._closed:
raise ProgrammingError("Cursor closed")

if isinstance(parameters, dict):
sql, parameters = convert_named_to_positional(sql, parameters)

self._result = self.connection.client.sql(
sql, parameters, bulk_parameters
)
Expand Down
61 changes: 61 additions & 0 deletions src/crate/client/params.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
# -*- coding: utf-8; -*-
#
# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
# license agreements. See the NOTICE file distributed with this work for
# additional information regarding copyright ownership. Crate licenses
# this file to you under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. You may
# obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# However, if you have executed another commercial license agreement
# with Crate these terms will supersede the license and you may use the
# software solely pursuant to the terms of the relevant commercial agreement.
import re
import typing as t

from .exceptions import ProgrammingError

_NAMED_PARAM_RE = re.compile(r"%\((\w+)\)s")


def convert_named_to_positional(
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd tend to move the method into the cursor module and prefix it with _.

It's not supposed to be public API and a dedicated module doesn't seem warranted for this tiny and simple function.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good call. I was thinking to isolate the regex logic in its own module to keep cursor.py focused on DB-API concerns and make the function independently testable.

I'm moving to _convert_named_to_positional and _NAMED_PARAM_RE directly into cursor.py at module level (not inside the class, since it doesn't touch self or any class state). And will delete params.py. What do you think about this structure?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm moving to _convert_named_to_positional and _NAMED_PARAM_RE directly into cursor.py at module level (not inside the class, since it doesn't touch self or any class state). And will delete params.py. What do you think about this structure?

Sounds good

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for your review. I refactored the code based on that.

sql: str, params: t.Dict[str, t.Any]
) -> t.Tuple[str, t.List[t.Any]]:
"""Convert pyformat-style named parameters to positional qmark parameters.

Converts ``%(name)s`` placeholders to ``?`` and returns an ordered list
of corresponding values extracted from ``params``.

The same name may appear multiple times; each occurrence appends the
value to the positional list independently.

Raises ``ProgrammingError`` if a placeholder name is absent from ``params``.
Extra keys in ``params`` are silently ignored.

Example::

sql = "SELECT * FROM t WHERE a = %(a)s AND b = %(b)s"
params = {"a": 1, "b": 2}
# returns: ("SELECT * FROM t WHERE a = ? AND b = ?", [1, 2])
"""
positional: t.List[t.Any] = []

def _replace(match: "re.Match[str]") -> str:
name = match.group(1)
if name not in params:
raise ProgrammingError(
f"Named parameter '{name}' not found in the parameters dict"
)
positional.append(params[name])
return "?"

converted_sql = _NAMED_PARAM_RE.sub(_replace, sql)
return converted_sql, positional
38 changes: 38 additions & 0 deletions tests/client/test_cursor.py
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,44 @@ def test_execute_with_timezone(mocked_connection):
assert result[0][1].tzname() == "UTC"


def test_execute_with_named_params(mocked_connection):
"""
Verify that named %(name)s parameters are converted to positional ? markers
and the values are passed as an ordered list.
"""
cursor = mocked_connection.cursor()
cursor.execute(
"SELECT * FROM t WHERE a = %(a)s AND b = %(b)s",
{"a": 1, "b": 2},
)
mocked_connection.client.sql.assert_called_once_with(
"SELECT * FROM t WHERE a = ? AND b = ?", [1, 2], None
)


def test_execute_with_named_params_repeated(mocked_connection):
"""
Verify that a parameter name used multiple times in the SQL is resolved
correctly each time it appears.
"""
cursor = mocked_connection.cursor()
cursor.execute("SELECT %(x)s, %(x)s", {"x": 42})
mocked_connection.client.sql.assert_called_once_with(
"SELECT ?, ?", [42, 42], None
)
Comment on lines +516 to +519
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Btw. as a follow up we should change this behavior to have this return "SELECT $1, $1", [42]

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for suggestion. We can keep track of of already seen names and reuse them inside _convert_named_to_positional() How should we follow up? should I open a new issue or create a PR directly?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I already created a PR: #796

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok super!



def test_execute_with_named_params_missing(mocked_connection):
"""
Verify that a ProgrammingError is raised when a placeholder name is absent
from the parameters dict, and that the client is never called.
"""
cursor = mocked_connection.cursor()
with pytest.raises(ProgrammingError, match="Named parameter 'z' not found"):
cursor.execute("SELECT %(z)s", {"a": 1})
mocked_connection.client.sql.assert_not_called()


def test_cursor_close(mocked_connection):
"""
Verify that a cursor is not closed if not specifically closed.
Expand Down
91 changes: 91 additions & 0 deletions tests/client/test_params.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
# -*- coding: utf-8; -*-
#
# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
# license agreements. See the NOTICE file distributed with this work for
# additional information regarding copyright ownership. Crate licenses
# this file to you under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. You may
# obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# However, if you have executed another commercial license agreement
# with Crate these terms will supersede the license and you may use the
# software solely pursuant to the terms of the relevant commercial agreement.

import pytest

from crate.client.exceptions import ProgrammingError
from crate.client.params import convert_named_to_positional


def test_basic_conversion():
"""Named placeholders are replaced with ? and values are ordered."""
sql, params = convert_named_to_positional(
"SELECT * FROM t WHERE a = %(a)s AND b = %(b)s",
{"a": 1, "b": 2},
)
assert sql == "SELECT * FROM t WHERE a = ? AND b = ?"
assert params == [1, 2]


def test_repeated_param():
"""The same name appearing multiple times appends the value each time."""
sql, params = convert_named_to_positional(
"SELECT %(x)s, %(x)s",
{"x": 42},
)
assert sql == "SELECT ?, ?"
assert params == [42, 42]


def test_missing_param_raises():
"""A placeholder without a matching key raises ProgrammingError."""
with pytest.raises(ProgrammingError, match="Named parameter 'z' not found"):
convert_named_to_positional("SELECT %(z)s", {"a": 1})


def test_extra_params_ignored():
"""Extra keys in the params dict cause no error."""
sql, params = convert_named_to_positional(
"SELECT %(a)s",
{"a": 10, "b": 99, "c": "unused"},
)
assert sql == "SELECT ?"
assert params == [10]


def test_no_named_params():
"""SQL without %(...)s placeholders is returned unchanged."""
sql, params = convert_named_to_positional(
"SELECT * FROM t WHERE a = ?",
{},
)
assert sql == "SELECT * FROM t WHERE a = ?"
assert params == []


def test_various_value_types():
"""Different value types (str, int, float, None, bool) are handled."""
sql, params = convert_named_to_positional(
"INSERT INTO t VALUES (%(s)s, %(i)s, %(f)s, %(n)s, %(b)s)",
{"s": "hello", "i": 7, "f": 3.14, "n": None, "b": True},
)
assert sql == "INSERT INTO t VALUES (?, ?, ?, ?, ?)"
assert params == ["hello", 7, 3.14, None, True]


def test_preserves_surrounding_text():
"""Non-placeholder text in the SQL is not modified."""
sql, params = convert_named_to_positional(
"SELECT name FROM locations WHERE name = %(name)s ORDER BY name",
{"name": "Algol"},
)
assert sql == "SELECT name FROM locations WHERE name = ? ORDER BY name"
assert params == ["Algol"]
Loading