From 589e205dfd680e8275916b6ed6ab2e0364a52956 Mon Sep 17 00:00:00 2001 From: jasper blues Date: Sun, 1 Nov 2026 10:53:28 +1000 Subject: [PATCH 1/2] Group, de-duplicate and sort a map's null and a missing key as one null, and merge a CALL's UNION ALL marks from every part, as drivine4j does, at Cypher spec v0.20.0 On Postgres and Oracle a null read out of a map or a list is JSON's null and a missing key SQL's, so the two grouped, de-duplicated and sorted apart where Neo4j has one null (#314, port of liberation-data/drivine4j#500). A JSON value now carries `json_null` where it may be JSON's null: a property or an item of a map or a list not known free of nulls, a parameter's entry, an unwound item, head() and last() of such a list, a null written, a reduce(), and a CASE that may give one. A projection that groups or keeps its rows distinct compares such an item with the two nulls as one (`grouped_null`, on Oracle `NVL(x, JSON('null'))`), an order term that is the item sorts by the same column, a part of a UNION without ALL gives it as SQL's null (`compared_null`), and Oracle's order ties the two nulls so the next term decides (`json_null_sort_key`). A property read from the store is never flagged, so its SQL is unchanged. A CALL's UNION ALL merged its parts' marks from the first part only (#321, port of liberation-data/drivine4j#512): the merged column now may be JSON's null where any part's may be, and is a list, a list of lists or flat only where every part's is. The Cypher spec moves to v0.20.0, which adds two steps to "A null in a map and a missing key are one null"; the #314 holds are taken out. Fixes #314 Fixes #321 --- drivine/cypher/sql/oracle_cypher_compiler.py | 18 ++ drivine/cypher/sql/sql_cypher_compiler.py | 168 +++++++++++++++--- drivine/cypher/sql/value.py | 9 + tests/cypher/VERSION | 2 +- tests/cypher/cypher_spec.py | 9 - tests/cypher/queries.yaml | 8 + tests/integration/test_cypher_compiled.py | 141 +++++++++++++++ .../sync_integration/test_cypher_compiled.py | 141 +++++++++++++++ tests/unit/test_cypher_kind_order.py | 6 +- 9 files changed, 466 insertions(+), 36 deletions(-) diff --git a/drivine/cypher/sql/oracle_cypher_compiler.py b/drivine/cypher/sql/oracle_cypher_compiler.py index 3ee2c2a..45b5019 100644 --- a/drivine/cypher/sql/oracle_cypher_compiler.py +++ b/drivine/cypher/sql/oracle_cypher_compiler.py @@ -287,6 +287,24 @@ def open_sort_key(self, json: str) -> list[str]: rank = self.shared1(json, lambda x: f"DECODE({_kind(x)}, {ranks})") return [rank, json] + def json_null_sort_key(self, json: str) -> list[str]: + """JSON's null is ranked null, as SQL's is, but `JSON` orders it apart from SQL's: so the value is read as + `grouped_null` has it (#314).""" + return [self.open_sort_key(json)[0], self.grouped_null(json)] + + def compared_null(self, json: str) -> str: + """Oracle groups by no subquery, so the value is not bound in one, as `null_if_json_null` binds a long one: it + is read twice.""" + return f"CASE WHEN {_kind(json)} <> 'null' THEN {json} END" + + def grouped_null(self, json: str) -> str: + """ + SQL's null read as JSON's, which reads the value once, and reads no kind of it: `compared_null` reads both, and + kept `DISTINCT` over 300,000 maps' values 67% longer than without it, and an order by them 39% longer; this, 8% + and 3% (#314). + """ + return f"NVL({json}, {self.literal_json(None)})" + def number(self, json: str) -> str: return self.shared1( json, lambda x: f"CASE WHEN {self.type_of(x)} = 'number' THEN JSON_VALUE({x}, '$' RETURNING NUMBER) END" diff --git a/drivine/cypher/sql/sql_cypher_compiler.py b/drivine/cypher/sql/sql_cypher_compiler.py index 67b6ed8..dbd2c2f 100644 --- a/drivine/cypher/sql/sql_cypher_compiler.py +++ b/drivine/cypher/sql/sql_cypher_compiler.py @@ -650,10 +650,14 @@ def _union(self, query: Query, parameters: Mapping[str, Any]) -> CompiledCypher: ) # Every query's names, before any is compiled: a name made in one is no variable of a later one's. self._give(query) - rows = "\nUNION ALL\n".join( - f"SELECT {i + 1} AS part, {self.row_number} AS n, q.result " - f"FROM (\n{indent(self._query(Query(clauses)))}\n) q" - for i, clauses in enumerate(parts) + # Its rows are compared, as a UNION without ALL compares them: see `_comparing_parts`. + rows = self._comparing_parts( + not query.unions[0].all, + lambda: "\nUNION ALL\n".join( + f"SELECT {i + 1} AS part, {self.row_number} AS n, q.result " + f"FROM (\n{indent(self._query(Query(clauses)))}\n) q" + for i, clauses in enumerate(parts) + ), ) if query.unions[0].all: sql = f"SELECT u.result FROM (\n{indent(rows)}\n) u\nORDER BY u.part, u.n" @@ -1005,6 +1009,30 @@ def open_sort_key(self, json: str) -> list[str]: """ return [self.sort_key(json)] + def json_null_sort_key(self, json: str) -> list[str]: + """ + `open_sort_key` of a value that may be JSON's null (Json.json_null): JSON's null ties with SQL's, as Cypher has + one null, so a term after it decides their order (#314). Where a store's `open_sort_key` does not tie them + already, it says so. + """ + return self.open_sort_key(json) + + def compared_null(self, json: str) -> str: + """ + The JSON value `json`, JSON's null read as SQL's, where rows are given by it to a `UNION` that compares them, + and by default where they are grouped or kept distinct (`grouped_null`): as `null_if_json_null`, but where a + store binds a long value in a subquery there, which no store groups by, written out where it is read. + """ + return self.null_if_json_null(json) + + def grouped_null(self, json: str) -> str: + """ + The JSON value `json` as rows are grouped or kept distinct by it, JSON's null and SQL's one: as + `compared_null`, but where a store makes them one more cheaply in a value that each row reads alike, as SQL's + null read as JSON's, it says so (#314). + """ + return self.compared_null(json) + def list_sort_key(self, json: str) -> str: """What an order sorts the JSON by where the statement makes it a list (Json.list): as `sort_key`, where the store orders lists as Cypher does.""" @@ -1390,6 +1418,12 @@ def _begin(self, parameters: Parameters) -> None: The order of the rows the projection being compiled reads, which `collect` gathers them in, as Cypher gathers them: a group's rows are not read in the order of the query they come from. """ + self._distinct_parts = False + """ + Whether the query compiled is a part of a `UNION` without ALL, whose rows are compared as a whole: there a value + that may be JSON's null (Json.json_null) is given as SQL's, as Cypher has one null (#314). Set only for the work + `_comparing_parts` does, so an exception leaves it as it was. + """ self._stepping = False """Whether a quantified path pattern's conditions are being compiled: what they hold is in its walk's step.""" self._filtering = False @@ -1690,14 +1724,18 @@ def _correlated(self, query: Query, scope: Scope, what: str) -> str: if query.unions: # Each query's rows as one value each, as a statement's are; without ALL, each once. distinct = not query.unions[0].all - parts: list[str] = [] - for clauses in [query.clauses, *(it.clauses for it in query.unions)]: + + def part(clauses: Sequence[Clause]) -> str: select = _Select(scope) for clause in clauses[:-1]: select = self._read(select, clause, scope) sql = self._project(select, _returned(clauses).projection, final=True).sql result = self.comparable(Json("q.result", ordered=True)) if distinct else "q.result" - parts.append(f"SELECT {result} AS result FROM (\n{indent(sql)}\n) q") + return f"SELECT {result} AS result FROM (\n{indent(sql)}\n) q" + + parts = self._comparing_parts( + distinct, lambda: [part(it) for it in [query.clauses, *(u.clauses for u in query.unions)]] + ) return ("\nUNION\n" if distinct else "\nUNION ALL\n").join(parts) select = _Select(scope) for clause in query.clauses: @@ -1840,6 +1878,16 @@ def _imports_apart(self, scope: Scope, clauses: Sequence[Clause]) -> tuple[str, imports = {variable: _columns_of(value, name, column, self._identities) for variable, value, column in named} return f"({self.bound_select(', '.join(columns))}) {name}", imports + def _comparing_parts(self, distinct: bool, work: Callable[[], T]) -> T: + """The work, its parts of a `UNION` compared as a whole where `distinct` (`_distinct_parts`), as it was + after.""" + was = self._distinct_parts + self._distinct_parts = distinct + try: + return work() + finally: + self._distinct_parts = was + def _called_union(self, select: _Select, call: Call) -> _Projected: """A `UNION` in a `CALL { … }`: each query's rows, the same items of the same kinds, matched by name as Cypher matches them.""" @@ -1854,7 +1902,9 @@ def _called_union(self, select: _Select, call: Call) -> _Projected: if any({item.name for item in it.projection.items} != set(names) for it in returns): raise DrivineError("Each query of a UNION returns the same names, as in Cypher.") distinct = not query.unions[0].all - projected = [self._called_part(select, call, it, names, distinct) for it in parts] + projected = self._comparing_parts( + distinct, lambda: [self._called_part(select, call, it, names, distinct) for it in parts] + ) kinds = {tuple(type(value) for value in it.scope.values()) for it in projected} if len(kinds) > 1: unsupported("A UNION in CALL { … } whose queries return values of different kinds") @@ -1869,7 +1919,9 @@ def _called_union(self, select: _Select, call: Call) -> _Projected: was = self._identities self._identities = False try: - projected = [self._called_part(select, call, it, names, distinct) for it in parts] + projected = self._comparing_parts( + distinct, lambda: [self._called_part(select, call, it, names, distinct) for it in parts] + ) finally: self._identities = was sql = ("\nUNION\n" if distinct else "\nUNION ALL\n").join( @@ -1878,7 +1930,10 @@ def _called_union(self, select: _Select, call: Call) -> _Projected: # A value is of a temporal kind where each query makes it of that one, and a text, or a list of texts, where # each makes it one. It is at keys of a write's row (`row_keys`) only where each query takes it from those: a # property of it is read from the row there, which would read the first query's value for every query's rows. - # It is read from an unwound item (`from_list`) only where each query reads it from one. + # It is read from an unwound item (`from_list`) only where each query reads it from one. It may be JSON's null + # (`json_null`) where any query's may be, so that what reads it after compares that null and SQL's as one + # (#321). It is a list (`list`) only where each query makes it one, so that an order sorts the others' numbers + # and maps by their kinds, not as lists; and it holds no null (`flat`) only where each query's holds none. scope: dict[str, Value] = {} for variable, value in projected[0].scope.items(): if not isinstance(value, Json): @@ -1902,6 +1957,10 @@ def _called_union(self, select: _Select, call: Call) -> _Projected: texts=all(_holds_texts(it) for it in all_), # Its kind is known only where each query writes it as a value of that one. known=next(iter(known)) if len(known) == 1 else None, + json_null=any(_may_be_json_null(it) for it in all_), + list=all(isinstance(it, Json) and it.list for it in all_), + item_lists=all(isinstance(it, Json) and it.item_lists for it in all_), + flat=all(isinstance(it, Json) and it.flat for it in all_), ) return _Projected(sql, scope, []) @@ -1929,7 +1988,8 @@ def items_of(x: str) -> str: items = self.shared1(self.comparable(self.json(listed)), items_of) select.source(self._elements_of(listed, items, name)) - # An item of a list of texts, or of a text, is a text; an item of a list that holds no null holds none. + # An item of a list of texts, or of a text, is a text; an item of a list that holds no null holds none, nor is + # JSON's null. select.bind( unwind.variable, Json( @@ -1941,6 +2001,7 @@ def items_of(x: str) -> str: text=_holds_texts(listed) or _is_text(listed), flat=self._is_flat(listed), from_list=True, + json_null=not self._is_flat(listed), ), ) select.order.append((f"{name}.item_no", False)) @@ -3090,13 +3151,11 @@ def as_written(part: Expression) -> Expression | None: for item in items ] aggregating = any(contains(it.expression, is_aggregate) for it in items) - # Where rows are compared (DISTINCT, GROUP BY), JSON is compared as what compares. + # Where rows are compared (DISTINCT, GROUP BY), each item is compared as `_row_compared` has it. compared = projection.distinct or aggregating - columns: list[tuple[str, Value]] = [] - for i, (_, value) in enumerate(compiled): - if compared and isinstance(value, Json) and value.ordered: - value = dataclasses.replace(value, sql=self.comparable(value), ordered=False) - columns.append((f"c{i + 1}", value)) + columns: list[tuple[str, Value]] = [ + (f"c{i + 1}", self._row_compared(value) if compared else value) for i, (_, value) in enumerate(compiled) + ] select_list = [it for column, value in columns for it in _columns(value, column, self._identities)] # An order is written over the items' names, and where nothing is grouped or kept distinct, the variables too. name = self._alias("p") @@ -3134,7 +3193,15 @@ def carried(column: str, value: Callable[[], Value], aggregates: Expression) -> after[column] = self._sortable(self._expression(sort.expression, item_columns)) sorts_anew = True elif item >= 0: - carried(column, lambda item=item: compiled[item][1], sort.expression) + # One that may be JSON's null, where rows are compared, as its column is grouped by: see + # `_row_compared`. + carried( + column, + lambda item=item: ( + columns[item][1] if compared and _may_be_json_null(compiled[item][1]) else compiled[item][1] + ), + sort.expression, + ) elif compared and not contains(sort.expression, _aggregates): after[column] = self._sortable(self._expression(sort.expression, item_columns)) elif over_items is not None: @@ -3203,7 +3270,13 @@ def sorted_by(column: str) -> list[tuple[str, str]]: order = [(named, descending) for column, descending in sorts for named, _ in sorted_by(column)] if final: values = [self.json(_columns_of(value, name, column, self._identities)) for column, value in columns] - result = values[0].sql if len(values) == 1 else self.json_array([it.sql for it in values]) + # A value that may be JSON's null is given as SQL's where a UNION compares rows by it; in a row of several, + # a list's item, SQL's null is JSON's (#314). + if len(values) == 1: + one = values[0] + result = self.compared_null(one.sql) if self._distinct_parts and one.json_null else one.sql + else: + result = self.json_array([it.sql for it in values]) outer_columns = f"{self.returned(result)} AS result" else: outer = [ @@ -3233,6 +3306,24 @@ def sorted_by(column: str) -> list[tuple[str, str]]: } return _Projected("\n".join(outer_lines), scope_out, order) + def _row_compared(self, value: Value) -> Value: + """ + The value as rows are compared by it, where they are grouped or kept distinct: JSON as what compares, and JSON's + null, which a value read out of a map or a list may be (Json.json_null), and SQL's as one, as Cypher has one + null (#314). It may still be JSON's null, as Oracle's `grouped_null` gives it, so it is still read as one that + may be. In a part of a `UNION` that compares rows (`_distinct_parts`) it is SQL's, as another part's value that + is never JSON's null gives it. + """ + if not isinstance(value, Json): + return value + if value.json_null: + it = self.comparable(value) + sql = self.compared_null(it) if self._distinct_parts else self.grouped_null(it) + return dataclasses.replace(value, sql=sql, ordered=False) + if value.ordered: + return dataclasses.replace(value, sql=self.comparable(value), ordered=False) + return value + def _anew_first( self, select: _Select, projection: Projection, items: list[ProjectionItem] ) -> tuple[_Select, Projection] | None: @@ -3719,7 +3810,10 @@ def _held_first( if not contains(item.expression, is_aggregate) and ( contains(item.expression, lambda it: isinstance(it, Parameter)) - or any("SELECT" in key.upper() for key in _group_keys(self._expression(item.expression, select.scope))) + or any( + "SELECT" in key.upper() + for key in _group_keys(self._row_compared(self._expression(item.expression, select.scope))) + ) ) ] if not held: @@ -3808,7 +3902,12 @@ def _expression(self, expression: Expression, scope: Scope) -> Value: def _value_of(self, expression: Expression, scope: Scope) -> Value: if isinstance(expression, Literal): - return Json(self.literal_json(expression.value), text=isinstance(expression.value, str)) + # A null written is JSON's null, as a map's is: see Json.json_null. + return Json( + self.literal_json(expression.value), + text=isinstance(expression.value, str), + json_null=expression.value is None, + ) if isinstance(expression, Parameter): if self._given_parameters is not None and expression.name in self._given_parameters: return self._given_parameters[expression.name] @@ -3826,7 +3925,7 @@ def _value_of(self, expression: Expression, scope: Scope) -> Value: if isinstance(expression, Property): in_parameter = self._in_parameter(expression) if in_parameter is not None: - return Json(in_parameter) + return Json(in_parameter, json_null=True) return self._property(self._expression(expression.subject, scope), expression.key) if isinstance(expression, HasLabels): subject = self._expression(expression.subject, scope) @@ -4040,7 +4139,7 @@ def subject() -> str: if isinstance(known, str): in_parameter = self._in_parameter(expression) if in_parameter is not None: - return Json(in_parameter) + return Json(in_parameter, json_null=True) return self._property(self._expression(expression.subject, scope), known) # An item of a list of nodes or relationships is the node or the relationship, so it is compared by its id; of # one collect gathers in this query, its properties, for a store reads no aggregate from a subquery. A pattern @@ -4059,6 +4158,7 @@ def subject() -> str: list=_holds_lists(subject_value()), text=_holds_texts(subject_value()), temporals=_item_temporals_of(subject_value()), + json_null=not self._is_flat(subject_value()), ) def item(list_: str) -> str: @@ -4080,6 +4180,7 @@ def read(values: list[str]) -> str: list=_holds_lists(subject_value()), text=_holds_texts(subject_value()), temporals=_item_temporals_of(subject_value()), + json_null=not self._is_flat(subject_value()), ) def _entity_at(self, entities: Entities, at: str) -> Value: @@ -4258,6 +4359,8 @@ def context_step(accumulator: str, item: str, map_: str) -> str: self.reduce(self._alias("f"), initial, list_, step, context, context_step), # It may hold what its start, its list or a value of the row its step reads holds (#347). temporals=_has_temporals(start) or _has_temporals(listed) or any(_has_temporals(scope[it]) for it in read), + # Its step may give JSON's null, as a null written or read from a map does. + json_null=True, ) def _pattern_comprehension(self, expression: PatternComprehension, scope: Scope) -> Value: @@ -4467,7 +4570,11 @@ def _item(self, expression: Expression, select: _Select) -> Value: # One of a kind known other than a map is left to _property, which refuses it. and not _known_no_map(subject) ): - return Json(self.item_value(self.comparable(subject), expression.key), temporals=subject.temporals) + return Json( + self.item_value(self.comparable(subject), expression.key), + temporals=subject.temporals, + json_null=not subject.flat, + ) return self._expression(expression, select.scope) def _property(self, subject: Value, key: str) -> Value: @@ -4487,12 +4594,13 @@ def _property(self, subject: Value, key: str) -> Value: return field_ if subject.row_keys is not None: keys = (*subject.row_keys, key) - return Json(self.row_value(keys), row_keys=keys) + return Json(self.row_value(keys), row_keys=keys, json_null=not subject.flat) taken = self.property_of(self.comparable(subject), key) return Json( taken, identity=self._identity_taken(subject, taken, True, lambda it: self.property_of(it, key)), temporals=subject.temporals, + json_null=not subject.flat, ) if isinstance(subject, Bool): raise DrivineError("A truth value has no properties.") @@ -5607,6 +5715,7 @@ def written_as(value: Callable[[int, Value], str]) -> str: item_lists=all(_holds_lists(result) for _, result in results), text=all(_is_text(result) for _, result in results), texts=all(_holds_texts(result) for _, result in results), + json_null=any(_may_be_json_null(result) for _, result in results), ) def _function(self, function: Function, scope: Scope) -> Value: @@ -5993,6 +6102,7 @@ def applied(list_: str) -> str: list=_holds_lists(one()), text=_holds_texts(one()), temporals=_item_temporals_of(one()), + json_null=not self._is_flat(one()), ) return Json(result, text=name in TEXT_FUNCTIONS) @@ -6325,6 +6435,9 @@ def _sort_term(self, value: Value) -> tuple[str, Callable[[str], list[str]]]: return self.comparable(self.json(value)), lambda it: [self.list_sort_key(it)] if not isinstance(value, Json) or temporal is not None or _is_text(value): return self.comparable(self.json(value)), lambda it: [self.sort_key(it)] + # One that may be JSON's null ties it with SQL's, so the next term decides between them (#314). + if _may_be_json_null(value): + return self.comparable(self.json(value)), self.json_null_sort_key return self.comparable(self.json(value)), self.open_sort_key @@ -7205,6 +7318,11 @@ def _holds_texts(value: Value) -> bool: return isinstance(value, Json) and value.texts +def _may_be_json_null(value: Value) -> bool: + """Whether the value may be JSON's null: see Json.json_null.""" + return isinstance(value, Json) and value.json_null + + def utf8_bytes(given: Any) -> int | None: """ How many bytes the value the statement gives is in UTF-8, where it is a text; else None. A lone surrogate, which diff --git a/drivine/cypher/sql/value.py b/drivine/cypher/sql/value.py index c032996..98255ea 100644 --- a/drivine/cypher/sql/value.py +++ b/drivine/cypher/sql/value.py @@ -103,6 +103,15 @@ class Json: when it plans the statement: a number, a text or a truth value written, a list or a map written, or a variable a `WITH` binds to one. None where the statement leaves its kind open, as for a property or a parameter. """ + json_null: bool = False + """ + Whether it may be JSON's null: a value read out of a map or a list that may hold a null, as `x.a` of `{a: null}` + is, an entry of a parameter, a null written, a `reduce()`, or a `CASE` that may give one of them. A key that is + missing reads as SQL's null, and the two compare apart, where Cypher has one null: so a projection that groups its + rows or keeps them distinct, and a part of a `UNION`, compares it with the two nulls read as one, and an order ties + them (#314). A property read from the store is never JSON's null, as a graph engine stores none, so the SQL that + reads one is unchanged. + """ @dataclass(frozen=True) diff --git a/tests/cypher/VERSION b/tests/cypher/VERSION index 96fb87f..1847373 100644 --- a/tests/cypher/VERSION +++ b/tests/cypher/VERSION @@ -1 +1 @@ -v0.19.0 +v0.20.0 diff --git a/tests/cypher/cypher_spec.py b/tests/cypher/cypher_spec.py index 63eee7a..e47bdd5 100644 --- a/tests/cypher/cypher_spec.py +++ b/tests/cypher/cypher_spec.py @@ -187,19 +187,10 @@ def _steps(first: int, last: int) -> tuple[int, ...]: HELD: dict[str, dict[str, list[Hold]]] = { "queries.yaml": { - # v0.17.0, liberation-data/drivine-cypher-spec#42: JSON's null and a missing key group, de-duplicate and sort - # apart. - "A null in a map and a missing key are one null": [ - Hold(314, ("postgres",), (*_steps(1, 23), 25, 26)), - Hold(314, ("oracle",), (*_steps(1, 27), 29)), - ], # v0.18.0, liberation-data/drivine-cypher-spec#43: a key read in a body nested in a list comprehension's WHERE, # in an order term's body, and any(y IN … WHERE y = e). "Implicit grouping key": [Hold(316, ("sql",), (29, 30, 31, 33, 34, 35, 37, 38, 39, 40, 42, 43, 44))], }, - "writes.yaml": { - "A null in a written row and a missing key are one null": [Hold(314, ("sql",), (1, 2))], - }, } """ The steps of a spec tag past what drivine-python runs that a port here has yet to make pass, by file and case. Each is diff --git a/tests/cypher/queries.yaml b/tests/cypher/queries.yaml index 5a23783..726634b 100644 --- a/tests/cypher/queries.yaml +++ b/tests/cypher/queries.yaml @@ -1512,6 +1512,14 @@ cases: only: [neo4j, sql] parameters: {p: {a: null}, q: {}} rows: [null] + - note: "A UNION ALL in a CALL gives the one null where a later query gives it as a list's or a map's null and another as a missing key." + cypher: "CALL { RETURN {}.a AS v UNION ALL RETURN [null][0] AS v } RETURN DISTINCT v" + only: [neo4j, sql] + rows: [null] + - cypher: "CALL { RETURN 1 AS v UNION ALL RETURN [null][0] AS v UNION ALL RETURN {}.a AS v } RETURN v, count(*) AS c" + only: [neo4j, sql] + unordered: true + rows: [[1, 1], [null, 2]] - note: "A null written is the same null as a property a node has not." cypher: "MATCH (p:Person) RETURN CASE WHEN p.born > 1960 THEN p.nickname ELSE null END AS v, count(*) AS c" only: [neo4j, sql] diff --git a/tests/integration/test_cypher_compiled.py b/tests/integration/test_cypher_compiled.py index c0bd575..097acec 100644 --- a/tests/integration/test_cypher_compiled.py +++ b/tests/integration/test_cypher_compiled.py @@ -5246,3 +5246,144 @@ async def test_a_pattern_comprehension_whose_body_reads_none_of_its_patterns_var # In a write, which Postgres refused. await cypher_pm.query(spec("MATCH (p:Person) SET p.knows = size([(p)-->() | p.name])")) assert await cypher_pm.query(spec("MATCH (p:Person) RETURN p.name, p.knows ORDER BY p.name")) == sizes + + +def unordered(rows: list[Any]) -> list[str]: + """Rows as a multiset, a null row included, to compare those of a statement that gives no order.""" + return sorted(json.dumps(it) for it in rows) + + +async def test_a_null_in_a_map_or_a_list_and_a_missing_key_are_one_null_beyond_the_spec(cypher_pm: PersistenceManager): + # #314: drivine4j's CypherQueryContract cases of liberation-data/drivine4j#500 that the spec does not hold, and + # Drivine for TypeScript's own beside them (liberation-data/drivine#501). The rows are Neo4j's (2026.10). + await seeded(cypher_pm) + cases: list[tuple[str, dict[str, Any], list[Any]]] = [ + # A UNION in a CALL, and a part beside a CALL's UNION ALL, which leaves the part's rows compared. + ( + "CALL { UNWIND [{a: null}] AS x RETURN x.a AS v UNION UNWIND [{b: 1}] AS x RETURN x.a AS v } " + "RETURN v, count(*) AS c", + {}, + [[None, 1]], + ), + ("CALL { RETURN null AS v UNION RETURN {}.a AS v } RETURN v, count(*) AS c", {}, [[None, 1]]), + ( + "UNWIND [{a: null}] AS x CALL { RETURN 1 AS z UNION ALL RETURN 2 AS z } RETURN x.a AS v " + "UNION UNWIND [{b: 1}] AS x RETURN x.a AS v", + {}, + [None], + ), + # An order ties a value kept distinct, which on Oracle may be JSON's null still. + ( + "UNWIND [{a: null, i: 1}, {b: 1, i: 2}, {a: 0, i: 3}] AS x WITH DISTINCT x.a AS a, x.i AS i " + "RETURN i ORDER BY a, i DESC", + {}, + [3, 2, 1], + ), + # A value kept distinct, on Oracle JSON's null, is still the same null as a property a node has not. + ( + "UNWIND [{a: null}] AS x WITH DISTINCT x.a AS a RETURN a AS v " + "UNION MATCH (p:Person {name: 'Hugo'}) RETURN p.nickname AS v", + {}, + [None], + ), + ( + "UNWIND [{a: null}, {b: 1}] AS x WITH DISTINCT x.a AS a MATCH (p:Person) " + "RETURN CASE WHEN p.born IS NULL THEN p.nickname ELSE a END AS v, count(*) AS c", + {}, + [[None, 4]], + ), + # Grouped by a value of an unwound list that holds a null, and sorted by it. + ("UNWIND [null, 1, null] AS v RETURN v, count(*) AS c ORDER BY v", {}, [[1, 1], [None, 2]]), + # A grouped or kept null comes back as null, not as JSON's null or its text. + ( + "UNWIND [{a: null}, {b: 1}] AS x WITH x.a AS a, count(*) AS c RETURN {a: a, c: c} AS m", + {}, + [{"a": None, "c": 2}], + ), + ("UNWIND [{a: null}, {a: null}] AS x WITH DISTINCT x.a AS a RETURN [a, a IS NULL] AS l", {}, [[None, True]]), + # A CALL's UNION ALL whose later part's value may be JSON's null, in a part of an outer UNION. + ("CALL { RETURN {}.a AS v UNION ALL RETURN [null][0] AS v } RETURN v UNION RETURN {}.a AS v", {}, [None]), + # A subquery's UNION compares its queries' rows with the two nulls as one. + ("UNWIND [{a: null}] AS x RETURN COUNT { RETURN x.a AS v UNION RETURN {}.a AS v } AS n", {}, [1]), + # head(), last(), a parameter's entry by its key, and a parameter that is null. + ("UNWIND [[null], []] AS l RETURN head(l) AS h, last(l) AS t, count(*) AS c", {}, [[None, None, 2]]), + ("RETURN $p['a'] AS a UNION RETURN $q.a AS a", {"p": {"a": None}, "q": {}}, [None]), + ( + "UNWIND [{b: 1}, {b: 2}] AS x RETURN CASE WHEN x.b = 1 THEN $p ELSE x.a END AS v, count(*) AS c", + {"p": None}, + [[None, 2]], + ), + ] + for cypher, parameters, rows in cases: + assert (cypher, await cypher_pm.query(spec(cypher, parameters))) == (cypher, rows) + + +async def test_a_writes_rows_group_a_maps_null_and_a_missing_key_as_one_null_beyond_the_spec( + cypher_pm: PersistenceManager, +): + # #314: drivine4j's CypherWriteContract case of liberation-data/drivine4j#500 that the spec does not hold. + await clear(cypher_pm) + assert await cypher_pm.query( + spec("UNWIND [{a: 1}, {a: 1}] AS r CREATE (:CyWrite {b: 9}) RETURN r.a AS a, count(*) AS c") + ) == [[1, 2]] + + +async def test_a_calls_union_all_whose_later_part_gives_a_lists_null_keeps_it_one_null_with_a_missing_key( + cypher_pm: PersistenceManager, +): + # #321, drivine4j's of liberation-data/drivine4j#512: the column may be JSON's null where any part's value may be, + # not only the first's. The rows are Neo4j's (2026.10), in any order where the statement gives none. + assert await cypher_pm.query( + spec("CALL { RETURN {}.a AS v UNION ALL RETURN [null][0] AS v } RETURN DISTINCT v") + ) == [None] + assert unordered( + await cypher_pm.query( + spec( + "CALL { RETURN 1 AS v UNION ALL RETURN [null][0] AS v UNION ALL RETURN {}.a AS v } " + "RETURN v, count(*) AS c" + ) + ) + ) == unordered([[1, 1], [None, 2]]) + # In a part of an outer UNION, which compares rows by it. + assert unordered( + await cypher_pm.query( + spec( + "CALL { RETURN {}.a AS v UNION ALL RETURN 1 AS v UNION ALL RETURN [null][0] AS v } RETURN v " + "UNION RETURN 2 AS v" + ) + ) + ) == unordered([None, 1, 2]) + + +async def test_a_calls_union_all_whose_first_part_gives_a_list_and_a_later_one_a_number_or_a_map_sorts_each_by_its_kind( + cypher_pm: PersistenceManager, +): + # #321, drivine4j's of liberation-data/drivine4j#512: the column is a list only where every part's value is, so a + # number or a map a later part gives sorts by its kind, not as a list would. The rows are Neo4j's (2026.10). + # Postgres takes jsonb's order of kinds, a number before a list (liberation-data/drivine4j#307), and refuses a + # written list's json beside a number's jsonb in one UNION ALL. + if cypher_pm.type is DatabaseType.POSTGRES: + return + ordered: list[tuple[str, list[Any]]] = [ + ("CALL { RETURN [10] AS v UNION ALL RETURN 10 AS v } RETURN v ORDER BY v", [[10], 10]), + ("CALL { RETURN [1, 2] AS v UNION ALL RETURN 10 AS v } RETURN v ORDER BY v", [[1, 2], 10]), + ("CALL { RETURN [10] AS v UNION ALL RETURN {a: 1} AS v } RETURN v ORDER BY v", [{"a": 1}, [10]]), + ( + "CALL { RETURN [10] AS v UNION ALL RETURN 10 AS v UNION ALL RETURN [9] AS v UNION ALL RETURN [2] AS v } " + "RETURN v ORDER BY v", + [[2], [9], [10], 10], + ), + # A later part's list of numbers unwinds to numbers, which sort after the first part's list. + ("CALL { RETURN [[1]] AS v UNION ALL RETURN [1] AS v } UNWIND v AS x RETURN x AS r ORDER BY r", [[1], 1]), + ] + for cypher, rows in ordered: + assert (cypher, await cypher_pm.query(spec(cypher))) == (cypher, rows) + # A later part's number compares as a number, and a later part's null in a list makes `=` and IN null, not false. + # These give no order. + unordered_: list[tuple[str, list[Any]]] = [ + ("CALL { RETURN [1] AS v UNION ALL RETURN 1 AS v } RETURN v < 2 AS r", [None, True]), + ("CALL { RETURN [1] AS v UNION ALL RETURN [null] AS v } RETURN v = [1] AS r", [True, None]), + ("CALL { RETURN [1] AS v UNION ALL RETURN [null] AS v } RETURN v IN [[1], 1] AS r", [True, None]), + ] + for cypher, rows in unordered_: + assert (cypher, unordered(await cypher_pm.query(spec(cypher)))) == (cypher, unordered(rows)) diff --git a/tests/sync_integration/test_cypher_compiled.py b/tests/sync_integration/test_cypher_compiled.py index 4292b0e..f08476a 100644 --- a/tests/sync_integration/test_cypher_compiled.py +++ b/tests/sync_integration/test_cypher_compiled.py @@ -5218,3 +5218,144 @@ def test_a_pattern_comprehension_whose_body_reads_none_of_its_patterns_variables # In a write, which Postgres refused. cypher_pm.query(spec("MATCH (p:Person) SET p.knows = size([(p)-->() | p.name])")) assert cypher_pm.query(spec("MATCH (p:Person) RETURN p.name, p.knows ORDER BY p.name")) == sizes + + +def unordered(rows: list[Any]) -> list[str]: + """Rows as a multiset, a null row included, to compare those of a statement that gives no order.""" + return sorted(json.dumps(it) for it in rows) + + +def test_a_null_in_a_map_or_a_list_and_a_missing_key_are_one_null_beyond_the_spec(cypher_pm: PersistenceManager): + # #314: drivine4j's CypherQueryContract cases of liberation-data/drivine4j#500 that the spec does not hold, and + # Drivine for TypeScript's own beside them (liberation-data/drivine#501). The rows are Neo4j's (2026.10). + seeded(cypher_pm) + cases: list[tuple[str, dict[str, Any], list[Any]]] = [ + # A UNION in a CALL, and a part beside a CALL's UNION ALL, which leaves the part's rows compared. + ( + "CALL { UNWIND [{a: null}] AS x RETURN x.a AS v UNION UNWIND [{b: 1}] AS x RETURN x.a AS v } " + "RETURN v, count(*) AS c", + {}, + [[None, 1]], + ), + ("CALL { RETURN null AS v UNION RETURN {}.a AS v } RETURN v, count(*) AS c", {}, [[None, 1]]), + ( + "UNWIND [{a: null}] AS x CALL { RETURN 1 AS z UNION ALL RETURN 2 AS z } RETURN x.a AS v " + "UNION UNWIND [{b: 1}] AS x RETURN x.a AS v", + {}, + [None], + ), + # An order ties a value kept distinct, which on Oracle may be JSON's null still. + ( + "UNWIND [{a: null, i: 1}, {b: 1, i: 2}, {a: 0, i: 3}] AS x WITH DISTINCT x.a AS a, x.i AS i " + "RETURN i ORDER BY a, i DESC", + {}, + [3, 2, 1], + ), + # A value kept distinct, on Oracle JSON's null, is still the same null as a property a node has not. + ( + "UNWIND [{a: null}] AS x WITH DISTINCT x.a AS a RETURN a AS v " + "UNION MATCH (p:Person {name: 'Hugo'}) RETURN p.nickname AS v", + {}, + [None], + ), + ( + "UNWIND [{a: null}, {b: 1}] AS x WITH DISTINCT x.a AS a MATCH (p:Person) " + "RETURN CASE WHEN p.born IS NULL THEN p.nickname ELSE a END AS v, count(*) AS c", + {}, + [[None, 4]], + ), + # Grouped by a value of an unwound list that holds a null, and sorted by it. + ("UNWIND [null, 1, null] AS v RETURN v, count(*) AS c ORDER BY v", {}, [[1, 1], [None, 2]]), + # A grouped or kept null comes back as null, not as JSON's null or its text. + ( + "UNWIND [{a: null}, {b: 1}] AS x WITH x.a AS a, count(*) AS c RETURN {a: a, c: c} AS m", + {}, + [{"a": None, "c": 2}], + ), + ("UNWIND [{a: null}, {a: null}] AS x WITH DISTINCT x.a AS a RETURN [a, a IS NULL] AS l", {}, [[None, True]]), + # A CALL's UNION ALL whose later part's value may be JSON's null, in a part of an outer UNION. + ("CALL { RETURN {}.a AS v UNION ALL RETURN [null][0] AS v } RETURN v UNION RETURN {}.a AS v", {}, [None]), + # A subquery's UNION compares its queries' rows with the two nulls as one. + ("UNWIND [{a: null}] AS x RETURN COUNT { RETURN x.a AS v UNION RETURN {}.a AS v } AS n", {}, [1]), + # head(), last(), a parameter's entry by its key, and a parameter that is null. + ("UNWIND [[null], []] AS l RETURN head(l) AS h, last(l) AS t, count(*) AS c", {}, [[None, None, 2]]), + ("RETURN $p['a'] AS a UNION RETURN $q.a AS a", {"p": {"a": None}, "q": {}}, [None]), + ( + "UNWIND [{b: 1}, {b: 2}] AS x RETURN CASE WHEN x.b = 1 THEN $p ELSE x.a END AS v, count(*) AS c", + {"p": None}, + [[None, 2]], + ), + ] + for cypher, parameters, rows in cases: + assert (cypher, cypher_pm.query(spec(cypher, parameters))) == (cypher, rows) + + +def test_a_writes_rows_group_a_maps_null_and_a_missing_key_as_one_null_beyond_the_spec( + cypher_pm: PersistenceManager, +): + # #314: drivine4j's CypherWriteContract case of liberation-data/drivine4j#500 that the spec does not hold. + clear(cypher_pm) + assert cypher_pm.query( + spec("UNWIND [{a: 1}, {a: 1}] AS r CREATE (:CyWrite {b: 9}) RETURN r.a AS a, count(*) AS c") + ) == [[1, 2]] + + +def test_a_calls_union_all_whose_later_part_gives_a_lists_null_keeps_it_one_null_with_a_missing_key( + cypher_pm: PersistenceManager, +): + # #321, drivine4j's of liberation-data/drivine4j#512: the column may be JSON's null where any part's value may be, + # not only the first's. The rows are Neo4j's (2026.10), in any order where the statement gives none. + assert cypher_pm.query(spec("CALL { RETURN {}.a AS v UNION ALL RETURN [null][0] AS v } RETURN DISTINCT v")) == [ + None + ] + assert unordered( + cypher_pm.query( + spec( + "CALL { RETURN 1 AS v UNION ALL RETURN [null][0] AS v UNION ALL RETURN {}.a AS v } " + "RETURN v, count(*) AS c" + ) + ) + ) == unordered([[1, 1], [None, 2]]) + # In a part of an outer UNION, which compares rows by it. + assert unordered( + cypher_pm.query( + spec( + "CALL { RETURN {}.a AS v UNION ALL RETURN 1 AS v UNION ALL RETURN [null][0] AS v } RETURN v " + "UNION RETURN 2 AS v" + ) + ) + ) == unordered([None, 1, 2]) + + +def test_a_calls_union_all_whose_first_part_gives_a_list_and_a_later_one_a_number_or_a_map_sorts_each_by_its_kind( + cypher_pm: PersistenceManager, +): + # #321, drivine4j's of liberation-data/drivine4j#512: the column is a list only where every part's value is, so a + # number or a map a later part gives sorts by its kind, not as a list would. The rows are Neo4j's (2026.10). + # Postgres takes jsonb's order of kinds, a number before a list (liberation-data/drivine4j#307), and refuses a + # written list's json beside a number's jsonb in one UNION ALL. + if cypher_pm.type is DatabaseType.POSTGRES: + return + ordered: list[tuple[str, list[Any]]] = [ + ("CALL { RETURN [10] AS v UNION ALL RETURN 10 AS v } RETURN v ORDER BY v", [[10], 10]), + ("CALL { RETURN [1, 2] AS v UNION ALL RETURN 10 AS v } RETURN v ORDER BY v", [[1, 2], 10]), + ("CALL { RETURN [10] AS v UNION ALL RETURN {a: 1} AS v } RETURN v ORDER BY v", [{"a": 1}, [10]]), + ( + "CALL { RETURN [10] AS v UNION ALL RETURN 10 AS v UNION ALL RETURN [9] AS v UNION ALL RETURN [2] AS v } " + "RETURN v ORDER BY v", + [[2], [9], [10], 10], + ), + # A later part's list of numbers unwinds to numbers, which sort after the first part's list. + ("CALL { RETURN [[1]] AS v UNION ALL RETURN [1] AS v } UNWIND v AS x RETURN x AS r ORDER BY r", [[1], 1]), + ] + for cypher, rows in ordered: + assert (cypher, cypher_pm.query(spec(cypher))) == (cypher, rows) + # A later part's number compares as a number, and a later part's null in a list makes `=` and IN null, not false. + # These give no order. + unordered_: list[tuple[str, list[Any]]] = [ + ("CALL { RETURN [1] AS v UNION ALL RETURN 1 AS v } RETURN v < 2 AS r", [None, True]), + ("CALL { RETURN [1] AS v UNION ALL RETURN [null] AS v } RETURN v = [1] AS r", [True, None]), + ("CALL { RETURN [1] AS v UNION ALL RETURN [null] AS v } RETURN v IN [[1], 1] AS r", [True, None]), + ] + for cypher, rows in unordered_: + assert (cypher, unordered(cypher_pm.query(spec(cypher)))) == (cypher, unordered(rows)) diff --git a/tests/unit/test_cypher_kind_order.py b/tests/unit/test_cypher_kind_order.py index 2f9e885..85a70ce 100644 --- a/tests/unit/test_cypher_kind_order.py +++ b/tests/unit/test_cypher_kind_order.py @@ -15,8 +15,12 @@ def test_oracle_sorts_a_value_of_open_kind_by_where_its_kind_comes_then_by_the_value() -> None: - sql = OracleCypherCompiler("g").compile("UNWIND [2, 'b', null] AS t RETURN t ORDER BY t DESC").sql + sql = OracleCypherCompiler("g").compile("UNWIND [2, 'b'] AS t RETURN t ORDER BY t DESC").sql assert sql.endswith(f"ORDER BY {RANK} DESC, p2.s1 DESC"), sql + # An item of a list that holds a null may be JSON's null, which ties with SQL's: the value is read as + # `grouped_null` has it (#314). + sql = OracleCypherCompiler("g").compile("UNWIND [2, 'b', null] AS t RETURN t ORDER BY t DESC").sql + assert sql.endswith(f"ORDER BY {RANK} DESC, NVL(p2.s1, JSON('null')) DESC"), sql assert RANK in OracleCypherCompiler("g").compile("MATCH (n:Doc) RETURN n.a ORDER BY n.a").sql From f09563f36366feca99c9f956465a10aa977a4595 Mon Sep 17 00:00:00 2001 From: jasper blues Date: Sun, 1 Nov 2026 17:44:17 +1000 Subject: [PATCH 2/2] Merge a CALL union's list marks with the shared helpers, and let Oracle's null test reuse compared_null --- drivine/cypher/sql/oracle_cypher_compiler.py | 2 +- drivine/cypher/sql/sql_cypher_compiler.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivine/cypher/sql/oracle_cypher_compiler.py b/drivine/cypher/sql/oracle_cypher_compiler.py index 45b5019..114bb15 100644 --- a/drivine/cypher/sql/oracle_cypher_compiler.py +++ b/drivine/cypher/sql/oracle_cypher_compiler.py @@ -275,7 +275,7 @@ def is_json_null(self, json: str) -> str: return f"NVL({_kind(json)}, 'null') = 'null'" def null_if_json_null(self, json: str) -> str: - return self.shared1(json, lambda x: f"CASE WHEN {_kind(x)} <> 'null' THEN {x} END") + return self.shared1(json, self.compared_null) def open_sort_key(self, json: str) -> list[str]: """ diff --git a/drivine/cypher/sql/sql_cypher_compiler.py b/drivine/cypher/sql/sql_cypher_compiler.py index dbd2c2f..ff3c1a3 100644 --- a/drivine/cypher/sql/sql_cypher_compiler.py +++ b/drivine/cypher/sql/sql_cypher_compiler.py @@ -1958,8 +1958,8 @@ def _called_union(self, select: _Select, call: Call) -> _Projected: # Its kind is known only where each query writes it as a value of that one. known=next(iter(known)) if len(known) == 1 else None, json_null=any(_may_be_json_null(it) for it in all_), - list=all(isinstance(it, Json) and it.list for it in all_), - item_lists=all(isinstance(it, Json) and it.item_lists for it in all_), + list=all(_is_list(it) for it in all_), + item_lists=all(_holds_lists(it) for it in all_), flat=all(isinstance(it, Json) and it.flat for it in all_), ) return _Projected(sql, scope, [])