Pass on a CALL's imports through a leading WITH *, and work out a pattern comprehension that reads only the row per match

drivine/cypher/sql/sql_cypher_compiler.py

@@ -1433,7 +1433,10 @@ def _query(self, query: Query) -> str:
     def _read(self, select: _Select, clause: Clause, outer: Scope | None = None) -> _Select:
         """
         A reading clause over the rows the select reads, and the rows the clauses after it read: a `WITH` is a query of
-        its own. `outer` is what a subquery reads from the row it is in.
+        its own. `outer` is what a subquery reads from the row it is in. A `WITH *` alone where nothing is bound passes
+        on nothing, as Neo4j has it, and is left out: SQL has no query of no column (#304). Its `WHERE` is met on the
+        rows it reads, which hold what that reads; with `DISTINCT`, an order, `SKIP` or `LIMIT` it is refused, as it
+        was.
         """
         if isinstance(clause, Match):
             return self._match(select, clause)
@@ -1443,6 +1446,13 @@ def _read(self, select: _Select, clause: Clause, outer: Scope | None = None) ->
             self._unwind(rows, dataclasses.replace(clause, list=listed))
             return rows
         if isinstance(clause, With):
+            if (
+                clause.projection.star
+                and not clause.projection.items
+                and plain_with(dataclasses.replace(clause, where=None))
+                and all(it in self._made_anew for it in select.bound)
+            ):
+                return self._filtered_rows(select, clause.where) if clause.where is not None else select
             return self._with(select, clause, outer or {})
         if isinstance(clause, Call):
             return self._anew_call(select, clause)
@@ -1647,7 +1657,7 @@ def _called_part(
         them, where a `UNION` names them so, and kept distinct with `distinct`. It sees only the variables of the row it
         imports (`_called_scope`): any other name in it is a new variable, as in Cypher (#294). What its brackets
         import it sees throughout, after a `WITH` that does not pass it on too, as Neo4j has it; what a leading `WITH`
-        imports is gone after one.
+        imports is gone after one. A leading `WITH *` passes on what it imports (`_starred_imports`).
         """
         imported = self._called_scope(call, clauses, select.scope)
         inner = _Select(imported)
@@ -1658,7 +1668,8 @@ def _called_part(
             for variable, value in values.items():
                 inner.bind(variable, value)
         outer: Scope = imported if bracketed(call) else {}
-        for clause in clauses[:-1]:
+        reading = clauses[:-1] if bracketed(call) else self._starred_imports(clauses[:-1], list(imported))
+        for clause in reading:
             inner = self._read(inner, clause, outer)
         projection = _returned(clauses).projection
         if names is not None:
@@ -1669,6 +1680,30 @@ def _called_part(
             inner, dataclasses.replace(projection, items=items, distinct=distinct or projection.distinct), final=False
         )
 
+    def _starred_imports(self, clauses: Sequence[Clause], imports: list[str]) -> Sequence[Clause]:
+        """
+        The reading `clauses` of a `CALL { … }` without brackets, its leading `WITH *` naming each variable it `imports`
+        in place of its star (#304): those are no variables its query binds, so the star gave none of them, and was
+        refused where nothing else was bound. Not one the compiler made (see `_anew_call`), which the statement never
+        named. Where it imports none, the `WITH` is as it was, and passes on nothing (`_read`). With brackets, Neo4j's
+        star gives none of what they import, which the query sees throughout anyway, so their clauses are as they were.
+        """
+        leading = next(
+            (i for i, it in enumerate(clauses) if not (isinstance(it, Unwind) and it.variable in self._made_anew)),
+            None,
+        )
+        clause = clauses[leading] if leading is not None else None
+        named = sorted(it for it in imports if it not in self._made_anew)
+        if leading is None or not isinstance(clause, With) or not clause.projection.star or not named:
+            return clauses
+        items = tuple(
+            ProjectionItem(Variable(name), name)
+            for name in named
+            if not any(it.name == name for it in clause.projection.items)
+        )
+        projection = dataclasses.replace(clause.projection, star=False, items=items + clause.projection.items)
+        return [*clauses[:leading], dataclasses.replace(clause, projection=projection), *clauses[leading + 1 :]]
+
     def _called_scope(self, call: Call, clauses: Sequence[Clause], scope: Scope) -> dict[str, Value]:
         """
         The variables of the row, `scope`, that the query `clauses` of the call sees: those it imports (`imports`), and
@@ -4086,13 +4121,24 @@ def _pattern_comprehension(self, expression: PatternComprehension, scope: Scope)
         if expression.where is not None:
             inner.where.append(self._condition(expression.where, inner.scope))
         projected = self._expression(expression.projection, inner.scope)
+        # A body that reads no variable of the pattern's own, `[(p)-->() | p.name]`, is worked out for each of the
+        # pattern's rows in a derived table, and gathered from there: an aggregate that reads only the row's columns is
+        # an aggregate of the row's query, which Postgres refuses (#305). One that reads nothing of any row, a literal
+        # or a parameter, is the comprehension's aggregate as it is. Its path variable is its own even where the row
+        # holds that name, which it leaves to a new variable.
+        own = [it for it in variables_in_pattern(expression.pattern) if it not in scope]
+        own += _present(expression.pattern.variable)
+        read = variables_read_in(expression.projection)
+        apart = not any(it in own for it in read) and (len(read) > 0 or holds_subquery(expression.projection))
 
         def gathered(value: str) -> str:
-            return (
-                "("
-                + " ".join([f"SELECT {self.aggregate_list(value)}", *inner.from_(), *_present(inner.where_clause())])
-                + ")"
-            )
+            def rows(select: str) -> str:
+                return " ".join([select, *inner.from_(), *_present(inner.where_clause())])
+
+            if not apart:
+                return f"({rows(f'SELECT {self.aggregate_list(value)}')})"
+            each = self._alias("pc")
+            return f"(SELECT {self.aggregate_list(f'{each}.v')} FROM ({rows(f'SELECT {value} AS v')}) {each})"
 
         # A comprehension of nodes or relationships is a list of them, so a write to an item of it reaches them.
         if isinstance(projected, Node):
@@ -6297,6 +6343,49 @@ def variables_in(expression: Expression) -> list[str]:
     return _ordered_of(variables_in, children(expression))
 
 
+def variables_read_in(expression: Expression) -> list[str]:
+    """
+    The variables the expression reads of the scope it is in, each once, in the order it names them: those
+    `variables_in` gives, but for a name that a comprehension, a quantifier or a reduce binds of its own and reads only
+    inside, so `[m IN [p.name] | m]` reads `p` alone. A pattern's new variables can't be named so in Cypher, nor a
+    subquery's, so a pattern's are as `variables_in_pattern` gives them.
+    """
+
+    def without(names: list[str], *own: str) -> list[str]:
+        return [it for it in names if it not in own]
+
+    if isinstance(expression, Variable):
+        return [expression.name]
+    if isinstance(expression, MapProjection):
+        named = [it.name for it in expression.elements if isinstance(it, VariableElement)]
+        return _ordered([expression.variable], named, _ordered_of(variables_read_in, children(expression)))
+    if isinstance(expression, ListComprehension):
+        body = _ordered_of(variables_read_in, _present(expression.where, expression.projection))
+        return _ordered(variables_read_in(expression.list), without(body, expression.variable))
+    if isinstance(expression, Quantifier):
+        body = _ordered_of(variables_read_in, _present(expression.where))
+        return _ordered(variables_read_in(expression.list), without(body, expression.variable))
+    if isinstance(expression, Reduce):
+        body = variables_read_in(expression.expression)
+        return _ordered(
+            _ordered_of(variables_read_in, [expression.initial, expression.list]),
+            without(body, expression.accumulator, expression.variable),
+        )
+    if isinstance(expression, PatternComprehension):
+        return _ordered(
+            variables_in_pattern(expression.pattern),
+            _ordered_of(variables_read_in, _present(expression.where, expression.projection)),
+        )
+    if isinstance(expression, PatternPredicate):
+        return variables_in_pattern(expression.pattern)
+    if isinstance(expression, Exists):
+        return _ordered(
+            *(variables_in_pattern(it) for it in expression.patterns),
+            variables_read_in(expression.where) if expression.where is not None else [],
+        )
+    return _ordered_of(variables_read_in, children(expression))
+
+
 def parameters_in(expression: Expression) -> set[str]:
     """Every parameter the expression names, in any expression it holds."""
     if isinstance(expression, Parameter):

tests/cypher/VERSION

@@ -1 +1 @@
-v0.10.0
+v0.12.0

tests/cypher/queries.yaml

@@ -166,6 +166,82 @@ cases:
       - cypher: "MATCH (p:Person {name: 'Hugo'}) RETURN [(p)-[:ACTED_IN]->(m) | m.title]"
         rows: [[]]
 
+  - name: "Pattern comprehension whose body reads only the row"
+    description: "A pattern comprehension whose body reads none of its pattern's own variables gives one item for each match of its pattern, for each row."
+    steps:
+      - note: "Its body reads only a variable of the row: the value again for each match."
+        only: [neo4j, sql]
+        cypher: "MATCH (p:Person) RETURN p.name, [(p)-->(m) | p.name] AS names ORDER BY p.name"
+        rows: [["Carrie", ["Carrie", "Carrie"]], ["Hugo", []], ["Keanu", ["Keanu", "Keanu", "Keanu", "Keanu"]], ["Laurence", ["Laurence", "Laurence"]]]
+      - cypher: "MATCH (p:Person) RETURN p.name, [(p)-[:ACTED_IN]->(:Movie) | p.born] AS l ORDER BY p.name"
+        only: [neo4j, sql]
+        rows: [["Carrie", [1967, 1967]], ["Hugo", []], ["Keanu", [1964, 1964, 1964]], ["Laurence", [1961]]]
+      - cypher: "MATCH (p:Person) RETURN p.name, size([(p)-->() | p]) AS l ORDER BY p.name"
+        only: [neo4j, sql]
+        rows: [["Carrie", 2], ["Hugo", 0], ["Keanu", 4], ["Laurence", 2]]
+      - cypher: "MATCH (p:Person) WITH p, 7 AS k RETURN p.name, [(p)-->() | k] AS l ORDER BY p.name"
+        only: [neo4j, sql]
+        rows: [["Carrie", [7, 7]], ["Hugo", []], ["Keanu", [7, 7, 7, 7]], ["Laurence", [7, 7]]]
+      - cypher: "MATCH (p:Person) RETURN p.name, [(p)-->(m) | COUNT { (p)-->() }] AS l ORDER BY p.name"
+        only: [neo4j, sql]
+        rows: [["Carrie", [2, 2]], ["Hugo", []], ["Keanu", [4, 4, 4, 4]], ["Laurence", [2, 2]]]
+      - note: "Or a literal."
+        only: [neo4j, sql]
+        cypher: "MATCH (p:Person) RETURN p.name, [(p)-->(m) | 1] AS l ORDER BY p.name"
+        rows: [["Carrie", [1, 1]], ["Hugo", []], ["Keanu", [1, 1, 1, 1]], ["Laurence", [1, 1]]]
+      - note: "randomUUID() in it is made again for each match."
+        only: [neo4j, sql]
+        cypher: "MATCH (p:Person) UNWIND [(p)-->(m) | p.name + randomUUID()] AS u RETURN count(u) AS n, count(DISTINCT u) AS d"
+        rows: [[8, 8]]
+      - note: "Its WHERE reads only the row too."
+        only: [neo4j, sql]
+        cypher: "MATCH (p:Person) RETURN p.name, [(p)-->(m) WHERE p.born > 1962 | p.name] AS l ORDER BY p.name"
+        rows: [["Carrie", ["Carrie", "Carrie"]], ["Hugo", []], ["Keanu", ["Keanu", "Keanu", "Keanu", "Keanu"]], ["Laurence", []]]
+      - note: "Read by a WHERE, an ORDER BY, an UNWIND, an aggregate, or as a grouping key."
+        only: [neo4j, sql]
+        cypher: "MATCH (p:Person) WHERE size([(p)-->() | p.name]) > 1 RETURN p.name ORDER BY p.name"
+        rows: ["Carrie", "Keanu", "Laurence"]
+      - cypher: "MATCH (p:Person) RETURN p.name ORDER BY size([(p)-->() | p.name]), p.name"
+        only: [neo4j, sql]
+        rows: ["Hugo", "Carrie", "Laurence", "Keanu"]
+      - cypher: "MATCH (p:Person) UNWIND [(p)-->() | p.name] AS n RETURN n, count(*) AS c ORDER BY n"
+        only: [neo4j, sql]
+        rows: [["Carrie", 2], ["Keanu", 4], ["Laurence", 2]]
+      - cypher: "MATCH (p:Person) RETURN collect(size([(p)-->() | p.name])) AS l"
+        only: [neo4j, sql]
+        unorderedLists: true
+        rows: [[0, 2, 2, 4]]
+      - cypher: "MATCH (p:Person) RETURN [(p)-->() | p.name] AS names, count(*) AS c ORDER BY c, size(names), names[0]"
+        only: [neo4j, sql]
+        rows: [[[], 1], [["Carrie", "Carrie"], 1], [["Laurence", "Laurence"], 1], [["Keanu", "Keanu", "Keanu", "Keanu"], 1]]
+      - note: "In another pattern comprehension's body, reading the outer one's variable or the row's."
+        only: [neo4j, sql]
+        cypher: "MATCH (p:Person) UNWIND [(p)-->(m) | [(m)-->() | m.name]] AS l RETURN p.name, l ORDER BY p.name, size(l)"
+        rows: [["Carrie", []], ["Carrie", []], ["Keanu", []], ["Keanu", []], ["Keanu", []], ["Keanu", ["Carrie", "Carrie"]], ["Laurence", []], ["Laurence", ["Keanu", "Keanu", "Keanu", "Keanu"]]]
+      - cypher: "MATCH (p:Person) UNWIND [(p)-->(m) | [(m)-->() | p.name]] AS l RETURN p.name, l ORDER BY p.name, size(l)"
+        only: [neo4j, sql]
+        rows: [["Carrie", []], ["Carrie", []], ["Keanu", []], ["Keanu", []], ["Keanu", []], ["Keanu", ["Keanu", "Keanu"]], ["Laurence", []], ["Laurence", ["Laurence", "Laurence", "Laurence", "Laurence"]]]
+      - note: "Of a row an OPTIONAL MATCH found nothing for, an empty list."
+        only: [neo4j, sql]
+        cypher: "MATCH (m:Movie) OPTIONAL MATCH (m)<-[:ACTED_IN]-(p:Person {name: 'Carrie'}) RETURN m.title, [(p)-->(x) | p.name] AS l ORDER BY m.title"
+        rows: [["Johnny Mnemonic", []], ["The Matrix", ["Carrie", "Carrie"]], ["The Matrix Reloaded", ["Carrie", "Carrie"]]]
+      - cypher: "OPTIONAL MATCH (p:Person {name: 'Nobody'}) RETURN [(p)-->() | p.name] AS l"
+        only: [neo4j, sql]
+        rows: [[]]
+      - cypher: "OPTIONAL MATCH (p:Person) RETURN p.name, [(p)-->() | p.name] AS l ORDER BY p.name"
+        only: [neo4j, sql]
+        rows: [["Carrie", ["Carrie", "Carrie"]], ["Hugo", []], ["Keanu", ["Keanu", "Keanu", "Keanu", "Keanu"]], ["Laurence", ["Laurence", "Laurence"]]]
+      - note: "In a CALL, reading what it imports."
+        only: [neo4j, sql]
+        cypher: "MATCH (p:Person) CALL (p) { RETURN [(p)-->() | p.name] AS names } RETURN p.name, names ORDER BY p.name"
+        rows: [["Carrie", ["Carrie", "Carrie"]], ["Hugo", []], ["Keanu", ["Keanu", "Keanu", "Keanu", "Keanu"]], ["Laurence", ["Laurence", "Laurence"]]]
+      - cypher: "MATCH (p:Person) CALL { WITH p RETURN [(p)-->(m) | p.name] AS names } RETURN p.name, names ORDER BY p.name"
+        only: [neo4j, sql]
+        rows: [["Carrie", ["Carrie", "Carrie"]], ["Hugo", []], ["Keanu", ["Keanu", "Keanu", "Keanu", "Keanu"]], ["Laurence", ["Laurence", "Laurence"]]]
+      - cypher: "MATCH (p:Person) CALL (p) { RETURN [(p)-->(m) WHERE p.born > 1962 | p.name] AS l } RETURN p.name, l ORDER BY p.name"
+        only: [neo4j, sql]
+        rows: [["Carrie", ["Carrie", "Carrie"]], ["Hugo", []], ["Keanu", ["Keanu", "Keanu", "Keanu", "Keanu"]], ["Laurence", []]]
+
   - name: "Patterns and labels as conditions"
     description: "Patterns, EXISTS, label tests and list comprehensions work as conditions and as values."
     steps:
@@ -1008,6 +1084,71 @@ cases:
       - cypher: "MATCH (a:Person {name: 'Keanu'}) CALL { WITH * MATCH (a)-[:ACTED_IN]->(m) RETURN count(m) AS n } RETURN n"
         only: [neo4j, sql]
         rows: [3]
+      - note: "A leading WITH * passes on what the CALL imports, with brackets or without."
+        only: [neo4j, sql]
+        cypher: "UNWIND [1, 2] AS i CALL { WITH * RETURN i AS x } RETURN x"
+        rows: [1, 2]
+      - cypher: "UNWIND [1, 2] AS i CALL (*) { WITH * RETURN i AS x } RETURN x"
+        only: [neo4j, sql]
+        rows: [1, 2]
+      - cypher: "UNWIND [1, 2] AS i CALL (i) { WITH * RETURN i AS x } RETURN x"
+        only: [neo4j, sql]
+        rows: [1, 2]
+      - cypher: "UNWIND [1, 2] AS i CALL (i) { WITH * WITH * RETURN i * 10 AS x } RETURN x"
+        only: [neo4j, sql]
+        rows: [10, 20]
+      - cypher: "UNWIND [1] AS i WITH i, 2 AS j CALL { WITH * RETURN i + j AS x } RETURN x"
+        only: [neo4j, sql]
+        rows: [3]
+      - cypher: "UNWIND [1] AS i CALL { WITH * UNWIND [5] AS k WITH * RETURN i + k AS x } RETURN x"
+        only: [neo4j, sql]
+        rows: [6]
+      - cypher: "MATCH (p:Person) CALL { WITH * RETURN p.name AS n } RETURN n ORDER BY n"
+        only: [neo4j, sql]
+        rows: ["Carrie", "Hugo", "Keanu", "Laurence"]
+      - cypher: "UNWIND [1, 2] AS i CALL { WITH * RETURN i AS x UNION WITH * RETURN i * 10 AS x } RETURN x ORDER BY x"
+        only: [neo4j, sql]
+        rows: [1, 2, 10, 20]
+      - cypher: "UNWIND [1] AS i CALL { WITH * CALL { WITH * RETURN i AS y } RETURN y AS x } RETURN x"
+        only: [neo4j, sql]
+        rows: [1]
+      - cypher: "UNWIND [1, 2] AS i OPTIONAL CALL { WITH * WITH * WHERE i > 1 RETURN i AS x } RETURN i, x"
+        only: [neo4j, sql]
+        rows: [[1, null], [2, 2]]
+      - note: "A RETURN * after it returns what the CALL imports, which the row holds already."
+        only: [neo4j, sql]
+        cypher: "UNWIND [1, 2] AS i CALL { WITH * RETURN * } RETURN i"
+        error: true
+      - note: "A star gives none of what brackets import, which the CALL sees throughout anyway."
+        only: [neo4j, sql]
+        cypher: "UNWIND [1, 2] AS i CALL (i) { WITH 1 AS one WITH * RETURN * } RETURN *"
+        rows: [[1, 1], [2, 1]]
+      - cypher: "UNWIND [1, 2] AS i CALL (i) { WITH *, 2 AS y RETURN * } RETURN *"
+        only: [neo4j, sql]
+        rows: [[1, 2], [2, 2]]
+      - note: "Where the CALL imports nothing, a leading WITH * passes on nothing."
+        only: [neo4j, sql]
+        cypher: "CALL { WITH * RETURN 1 AS x } RETURN x"
+        rows: [1]
+      - cypher: "CALL (*) { WITH * RETURN 1 AS x } RETURN x"
+        only: [neo4j, sql]
+        rows: [1]
+      - cypher: "UNWIND [1] AS i CALL () { WITH * RETURN 1 AS x } RETURN i, x"
+        only: [neo4j, sql]
+        rows: [[1, 1]]
+      - cypher: "UNWIND [1] AS i CALL () { WITH * RETURN i AS x } RETURN x"
+        only: [neo4j, sql]
+        error: true
+      - cypher: "UNWIND [1, 2] AS i CALL () { WITH * WITH * RETURN 3 AS x } RETURN i, x"
+        only: [neo4j, sql]
+        rows: [[1, 3], [2, 3]]
+      - note: "Its WHERE is met still, and reads what the brackets import."
+        only: [neo4j, sql]
+        cypher: "UNWIND [1, 2] AS i CALL () { WITH * WHERE false RETURN 1 AS x } RETURN i, x"
+        rows: []
+      - cypher: "UNWIND [1, 2] AS i CALL (i) { WITH * WHERE i > 1 RETURN i AS x } RETURN x"
+        only: [neo4j, sql]
+        rows: [2]
       - note: "In a UNION, each query imports what its own leading WITH names."
         only: [neo4j, sql]
         cypher: "MATCH (a:Person {name: 'Keanu'}) CALL { WITH a MATCH (a)-[:ACTED_IN]->(m) RETURN count(m) AS n UNION ALL MATCH (a:Movie) RETURN count(a) AS n } RETURN a.name, n"
@@ -1340,6 +1481,16 @@ cases:
         rows: [[2, 1, 0]]
       - cypher: "WITH 1 AS b, 2 AS a WITH * RETURN *"
         rows: [[2, 1]]
+      - note: "A WITH * where nothing is bound passes on nothing."
+        only: [neo4j, sql]
+        cypher: "WITH * RETURN 1 AS x"
+        rows: [1]
+      - cypher: "WITH * WHERE false RETURN 1 AS x"
+        only: [neo4j, sql]
+        rows: []
+      - cypher: "RETURN 1 AS x UNION ALL WITH * WHERE false RETURN 2 AS x"
+        only: [neo4j, sql]
+        rows: [1]
       - cypher: "UNWIND [2, 1] AS b WITH b, 10 - b AS a RETURN * ORDER BY b"
         rows: [[9, 1], [8, 2]]