From ba5d72d343a048faf9fba1ee064fb225061167fb Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Sat, 17 Oct 2026 16:51:28 -0500 Subject: [PATCH 1/2] fix(declare): refuse a part reference that closes a master-part group cycle A part that references a table which already depends on the part's own master makes the master-part group depend on itself. topo_sort orders each master after every table its parts reference, so it could no longer order such tables, and every delete, drop, Diagram.trace and Schema.list_tables on the connection failed with networkx.NetworkXUnfeasible, including dropping the offending part. - Part._declare_check refuses such a reference before any DDL runs, with an error that names the part, the referenced table and the cycle. A new table has no children, so only a part's reference to a table outside its group can close a cycle, and this check refuses exactly those. - topo_sort raises a DataJointError that names an existing cycle and the part that closes it, and takes an optional set of tables to sort. Cascade and trace propagation, Diagram iteration and Schema.list_tables sort only their own tables, so an existing cycle affects only the operations that must order it, and the offending part can be dropped. - A part that does not inherit its master's primary key, unrenamed and non-nullable, through a reference to the master or a sibling part gets a warning at declaration. - declare() reports every foreign key through a new foreign_keys argument, which the _declare_check hook now receives. --- src/datajoint/autopopulate.py | 10 +- src/datajoint/declare.py | 34 +- src/datajoint/dependencies.py | 202 +++++++++++- src/datajoint/diagram.py | 22 +- src/datajoint/schemas.py | 14 +- src/datajoint/table.py | 20 +- src/datajoint/user_tables.py | 68 ++++ tests/integration/test_group_cycle.py | 427 ++++++++++++++++++++++++++ tests/unit/test_group_cycle.py | 232 ++++++++++++++ 9 files changed, 993 insertions(+), 36 deletions(-) create mode 100644 tests/integration/test_group_cycle.py create mode 100644 tests/unit/test_group_cycle.py diff --git a/src/datajoint/autopopulate.py b/src/datajoint/autopopulate.py index 61b36c5c9..c6ad04931 100644 --- a/src/datajoint/autopopulate.py +++ b/src/datajoint/autopopulate.py @@ -189,7 +189,12 @@ def __get__(self, obj, objtype=None): jobs: Job = _JobsDescriptor() - def _declare_check(self, primary_key: list[str], fk_attribute_map: dict[str, tuple[str, str]]) -> None: + def _declare_check( + self, + primary_key: list[str], + fk_attribute_map: dict[str, tuple[str, str]], + foreign_keys: list[tuple[str, dict[str, str], bool]], + ) -> None: """ Validate FK-only primary key constraint for auto-populated tables. @@ -203,6 +208,9 @@ def _declare_check(self, primary_key: list[str], fk_attribute_map: dict[str, tup List of primary key attribute names. fk_attribute_map : dict Mapping of child_attr -> (parent_table, parent_attr). + foreign_keys : list + One ``(parent_table, attr_map, nullable)`` entry per foreign key. + Not used here. Raises ------ diff --git a/src/datajoint/declare.py b/src/datajoint/declare.py index e207b8bca..524cb22ec 100644 --- a/src/datajoint/declare.py +++ b/src/datajoint/declare.py @@ -204,6 +204,7 @@ def compile_foreign_key( fk_attribute_map: dict[str, tuple[str, str]] | None = None, fk_index_candidates: list[list[str]] | None = None, column_comments: dict[str, str] | None = None, + foreign_keys: list[tuple[str, dict[str, str], bool]] | None = None, ) -> None: """ Parse a foreign key line and update declaration components. @@ -239,6 +240,11 @@ def compile_foreign_key( out of line (PostgreSQL's ``COMMENT ON COLUMN``). Each copied column's comment, with its core-type marker (e.g. ``:int32:``), is added so the child keeps the parent's declared type. Updated in place. + foreign_keys : list, optional + One ``(parent_table, attr_map, nullable)`` entry per foreign key, where + ``attr_map`` maps each child attribute to the parent attribute it + references. Unlike ``fk_attribute_map``, it keeps every reference when + several share an attribute. Updated in place. Raises ------ @@ -327,6 +333,9 @@ def compile_foreign_key( foreign_key_sql.append( f"FOREIGN KEY ({fk_cols}) REFERENCES {ref_table_name} ({pk_cols}) ON UPDATE CASCADE ON DELETE RESTRICT" ) + if foreign_keys is not None: + attr_map = {attr: ref.heading[attr].original_name for attr in ref.primary_key} + foreign_keys.append((ref_table_name, attr_map, is_nullable)) # Supporting index on the foreign-key columns. # @@ -382,7 +391,13 @@ def _tier_attributes(table_name: str | None, config) -> tuple[str, ...]: def prepare_declare( - definition: str, context: dict, adapter, *, table_name: str | None = None, config=None + definition: str, + context: dict, + adapter, + *, + table_name: str | None = None, + config=None, + foreign_keys: list[tuple[str, dict[str, str], bool]] | None = None, ) -> tuple[str, list[str], list[str], list[str], list[str], list[str], dict[str, tuple[str, str]], dict[str, str]]: """ Parse a table definition into its components. @@ -407,6 +422,9 @@ def prepare_declare( declares them. config : Config, optional Read for ``jobs.add_job_metadata`` and ``provenance.capture``. + foreign_keys : list, optional + Receives one ``(parent_table, attr_map, nullable)`` entry per foreign + key; see :func:`compile_foreign_key`. Updated in place. Returns ------- @@ -469,6 +487,7 @@ def add_attribute(line: str, in_key: bool) -> None: fk_attribute_map, fk_index_candidates, column_comments, + foreign_keys, ) elif re.match(r"^(unique\s+)?index\s*\(.*\)\s*(#.*)?$", line, re.I): # index compile_index(re.sub(r"\s*#.*$", "", line), index_sql, adapter) @@ -534,7 +553,13 @@ def _covered(candidate: list[str]) -> bool: def declare( - full_table_name: str, definition: str, context: dict, adapter, *, config=None + full_table_name: str, + definition: str, + context: dict, + adapter, + *, + config=None, + foreign_keys: list[tuple[str, dict[str, str], bool]] | None = None, ) -> tuple[str, list[str], list[str], dict[str, tuple[str, str]], list[str], list[str]]: r""" Parse a definition and generate SQL CREATE TABLE statement. @@ -551,6 +576,9 @@ def declare( Database adapter for backend-specific SQL generation. config : Config, optional Configuration object. If None, falls back to global config. + foreign_keys : list, optional + Receives one ``(parent_table, attr_map, nullable)`` entry per foreign + key; see :func:`compile_foreign_key`. Updated in place. Returns ------- @@ -593,7 +621,7 @@ def declare( external_stores, fk_attribute_map, column_comments, - ) = prepare_declare(definition, context, adapter, table_name=table_name, config=config) + ) = prepare_declare(definition, context, adapter, table_name=table_name, config=config, foreign_keys=foreign_keys) pre_ddl = [] # DDL to run BEFORE CREATE TABLE (e.g., CREATE TYPE for enums) post_ddl = [] # DDL to run AFTER CREATE TABLE (e.g., COMMENT ON) diff --git a/src/datajoint/dependencies.py b/src/datajoint/dependencies.py index cef7c6f0e..164834349 100644 --- a/src/datajoint/dependencies.py +++ b/src/datajoint/dependencies.py @@ -40,7 +40,155 @@ def extract_master(part_table: str) -> str | None: return None -def topo_sort(graph: nx.MultiDiGraph) -> list[str]: +#: How to read a chain of tables joined by `` -> `` in an error message. +CHAIN_LEGEND = "each table references the one before it or is the master of the part before it" + + +def group_graph(graph: nx.MultiDiGraph) -> nx.MultiDiGraph: + """ + Copy a dependency graph and add an edge from each part table to its master. + + A master's rows are inserted and deleted together with its parts' rows, so a + master depends on everything its parts depend on. With the added edges, + ``nx.ancestors`` gives a table's ancestors where reaching a master also + reaches all of its parts, and ``nx.descendants`` gives the converse, where + reaching a part also reaches its master. These are the same dependencies that + :func:`topo_sort` orders. + + The added edges form a cycle with each part that references its master, so + the result is for reachability only, not for sorting. + + Parameters + ---------- + graph : nx.MultiDiGraph + Dependency graph. + + Returns + ------- + nx.MultiDiGraph + A copy of ``graph`` with an edge from each part to its master. + """ + graph = nx.MultiDiGraph(graph) + for part in list(graph.nodes): + if (master := extract_master(part)) in graph: + graph.add_edge(part, master, key="master") + return graph + + +def group_ancestors(graph: nx.MultiDiGraph, table: str) -> set[str]: + """ + Return a table and its ancestors, where reaching a master reaches all its parts. + + The closure starts from ``table`` and adds the parents of every table in it, + and all the parts of every master in it, until nothing new is added. A part + reaches its own master through its reference to the master (``-> master``). + + Parameters + ---------- + graph : nx.MultiDiGraph + Dependency graph. + table : str + Full table name. + + Returns + ------- + set[str] + ``table`` and every table it depends on. + """ + return {table} | nx.ancestors(group_graph(graph), table) + + +def group_cycle(graph: nx.MultiDiGraph, part: str, parents: list[str]) -> list[str] | None: + """ + Find the cycle that a new part table's references would close. + + A part may reference its master, another part of the same master, or a table + outside its master-part group. A reference to a table outside the group closes + a cycle when that table already depends on the part's master (see + :func:`group_ancestors`): the group would then depend on itself, and + :func:`topo_sort` could no longer order its tables. + + A new table has no children yet, so only a new part's references to tables + outside its group can close such a cycle, and checking them when the part is + declared keeps every cycle out. + + Parameters + ---------- + graph : nx.MultiDiGraph + Dependency graph of the declared tables, which does not include ``part``. + part : str + Full name of the part table being declared. + parents : list[str] + Full names of the tables the part references. + + Returns + ------- + list[str] or None + The cycle as a chain of tables joined by `` -> `` that starts and + ends at the part's master, or None if the references close no cycle. + """ + master = extract_master(part) + if master not in graph: + return None + outside = [p for p in parents if p in graph and master not in (p, extract_master(p))] + if not outside: + return None + extended = group_graph(graph) + for parent in outside: + try: + path = nx.shortest_path(extended, master, parent) + except nx.NetworkXNoPath: + continue + return path + [part, master] + return None + + +def _reachable(graph: nx.MultiDiGraph, sources: set[str], upstream: bool) -> set[str]: + """Return the tables reachable from any of ``sources``, sources excluded unless reached.""" + step = graph.predecessors if upstream else graph.successors + reached = set() + frontier = list(sources) + while frontier: + for node in step(frontier.pop()): + if node not in reached: + reached.add(node) + frontier.append(node) + return reached + + +def _cycle_error(graph: nx.MultiDiGraph) -> DataJointError: + """ + Describe a cycle in the graph that :func:`topo_sort` failed to sort. + + ``graph`` carries the edges ``topo_sort`` adds from a part's parents to its + master, each marked with the part it stands for. + """ + edges = nx.find_cycle(graph) + # Start the chain at a master that the cycle reaches through one of its parts. + added = [i for i, (u, v, k) in enumerate(edges) if graph.edges[u, v, k].get("part")] + if added: + edges = edges[added[0] + 1 :] + edges[: added[0] + 1] + chain = [edges[0][0]] + parts = [] + for u, v, k in edges: + if part := graph.edges[u, v, k].get("part"): + chain.append(part) + parts.append(part) + chain.append(v) + message = f"Cannot order tables whose dependencies form a cycle: {' -> '.join(chain)} ({CHAIN_LEGEND})." + if parts: + names = " and ".join(parts) + which = f"part table {names} references" if len(parts) == 1 else f"part tables {names} each reference" + message += ( + f" The cycle runs through master-part groups: {which} a table that depends on its own master, which" + " can no longer be declared. To restore delete, drop and trace on these tables, drop " + f"{'that part' if len(parts) == 1 else 'one of these parts'} with `drop(part_integrity='ignore')` and" + " declare it without that reference." + ) + return DataJointError(message) + + +def topo_sort(graph: nx.MultiDiGraph, nodes=None) -> list[str]: """ Topological sort keeping part tables with their masters. @@ -48,14 +196,34 @@ def topo_sort(graph: nx.MultiDiGraph) -> list[str]: ---------- graph : nx.MultiDiGraph Dependency graph. + nodes : iterable of str, optional + Tables to sort. If given, only these tables and the tables on dependency + paths between them are sorted, so a cycle elsewhere in the graph does not + prevent ordering them, and only these tables are returned. Tables not in + the graph are ignored. If None (default), all tables are sorted. Returns ------- list[str] Table names in topological order with parts following masters. - """ - graph = nx.MultiDiGraph(graph) # make a mutable copy + Raises + ------ + DataJointError + If the dependencies of the tables to sort form a cycle, for example one + that runs through a master and its parts. + """ + if nodes is None: + graph = nx.MultiDiGraph(graph) # make a mutable copy + else: + # Sort the tables on paths between the requested ones too, so that their + # order respects every dependency between them. + nodes = set(nodes) & set(graph.nodes) + keep = nodes | (_reachable(graph, nodes, upstream=False) & _reachable(graph, nodes, upstream=True)) + subgraph = nx.MultiDiGraph() + subgraph.add_nodes_from(keep) + subgraph.add_edges_from((u, v) for u, v in graph.edges(keep) if v in keep) + graph = subgraph # Add parts' dependencies to their masters' dependencies # to ensure correct topological ordering of the masters. @@ -66,8 +234,11 @@ def topo_sort(graph: nx.MultiDiGraph) -> list[str]: parent = edge[0] if master not in (parent, extract_master(parent)): # if parent is neither master nor part of master - graph.add_edge(parent, master) - sorted_nodes = list(nx.topological_sort(graph)) + graph.add_edge(parent, master, part=part) + try: + sorted_nodes = list(nx.topological_sort(graph)) + except nx.NetworkXUnfeasible: + raise _cycle_error(graph) from None # bring parts up to their masters pos = len(sorted_nodes) - 1 @@ -84,6 +255,8 @@ def topo_sort(graph: nx.MultiDiGraph) -> list[str]: del sorted_nodes[pos] sorted_nodes.insert(insert_pos, part) + if nodes is not None: + return [node for node in sorted_nodes if node in nodes] return sorted_nodes @@ -259,7 +432,7 @@ def load_all_downstream(self) -> None: return self.load(force=True, schema_names=known_schemas) - def load_all_upstream(self) -> None: + def load_all_upstream(self, schema_names: set[str] | None = None) -> None: """ Load dependencies including all upstream schemas referenced via FK chains. @@ -271,9 +444,15 @@ def load_all_upstream(self) -> None: Called automatically by ``Diagram.trace()``. Symmetric to :meth:`load_all_downstream`. + + Parameters + ---------- + schema_names : set[str], optional + Schemas to start from in addition to the activated schemas, such + as the schemas of the tables a new table references. """ adapter = self._conn.adapter - known_schemas = set(self._conn.schemas) + known_schemas = set(self._conn.schemas) | set(schema_names or ()) if not known_schemas: self.load() return @@ -293,16 +472,21 @@ def load_all_upstream(self) -> None: return self.load(force=True, schema_names=known_schemas) - def topo_sort(self) -> list[str]: + def topo_sort(self, nodes=None) -> list[str]: """ Return table names in topological order. + Parameters + ---------- + nodes : iterable of str, optional + Tables to sort; all tables if None. See :func:`topo_sort`. + Returns ------- list[str] Table names sorted topologically. """ - return topo_sort(self) + return topo_sort(self, nodes) def parents(self, table_name: str, primary: bool | None = None) -> list[tuple[str, dict]]: r""" diff --git a/src/datajoint/diagram.py b/src/datajoint/diagram.py index 6ff60d061..db8a59a82 100644 --- a/src/datajoint/diagram.py +++ b/src/datajoint/diagram.py @@ -24,7 +24,7 @@ import networkx as nx from .condition import AndList -from .dependencies import extract_master, topo_sort +from .dependencies import extract_master, group_graph, topo_sort from .errors import DataJointError from .table import Table, lookup_class_name from .user_tables import Computed, Imported, Lookup, Manual, Part, _get_tier @@ -581,9 +581,9 @@ def _propagate_restrictions_upstream(self, start_node): passes until no new ancestor is restricted; termination is guaranteed because the dependency graph is a DAG. """ - sorted_nodes = topo_sort(self) # Only propagate through ancestors of start_node allowed_nodes = {start_node} | set(nx.ancestors(self, start_node)) + sorted_nodes = topo_sort(self, allowed_nodes) propagated_edges = set() restrictions = self._cascade_restrictions @@ -779,9 +779,15 @@ def _propagate_restrictions(self, start_node, mode, part_integrity="enforce"): restricted from every one of them. """ - sorted_nodes = topo_sort(self) # Only propagate through descendants of start_node allowed_nodes = {start_node} | set(nx.descendants(self, start_node)) + # Sort only the tables propagation can reach: part_integrity="cascade" also + # reaches the master of every part it reaches, and the master's descendants. + if part_integrity == "cascade" and mode == "cascade": + reachable = {start_node} | nx.descendants(group_graph(self), start_node) + else: + reachable = allowed_nodes + sorted_nodes = topo_sort(self, reachable) propagated_edges = set() # Dedup upward part→master walks by (part, master) PAIR, not by master # alone: distinct Parts of the same master can be restricted via @@ -1138,9 +1144,8 @@ def __iter__(self): Yields restricted ``FreeTable`` objects when cascade or restrict conditions have been applied, unrestricted ``FreeTable`` otherwise. """ - for node in topo_sort(self): - if node in self.nodes_to_show: - yield self._restricted_table(node) + for node in topo_sort(self, self.nodes_to_show): + yield self._restricted_table(node) def __reversed__(self): """ @@ -1149,9 +1154,8 @@ def __reversed__(self): Same as ``__iter__`` but reversed — useful for cascading deletes and drops. """ - for node in reversed(topo_sort(self)): - if node in self.nodes_to_show: - yield self._restricted_table(node) + for node in reversed(topo_sort(self, self.nodes_to_show)): + yield self._restricted_table(node) def prune(self): """ diff --git a/src/datajoint/schemas.py b/src/datajoint/schemas.py index 9f2cc1497..daf51ff5f 100644 --- a/src/datajoint/schemas.py +++ b/src/datajoint/schemas.py @@ -551,15 +551,11 @@ def list_tables(self) -> list[str]: list[str] Table names in topological order. """ - self.connection.dependencies.load() - return [ - t - for d, t in ( - self.connection.adapter.split_full_table_name(table_name) - for table_name in self.connection.dependencies.topo_sort() - ) - if d == self.database - ] + dependencies = self.connection.dependencies + dependencies.load() + split = self.connection.adapter.split_full_table_name + own = [table_name for table_name in dependencies.nodes if split(table_name)[0] == self.database] + return [split(table_name)[1] for table_name in dependencies.topo_sort(own)] def _find_table_name(self, name: str) -> str | None: """ diff --git a/src/datajoint/table.py b/src/datajoint/table.py index 9f6784fb3..bea6c0a8c 100644 --- a/src/datajoint/table.py +++ b/src/datajoint/table.py @@ -164,12 +164,18 @@ def declare(self, context=None): f"Table class name `{self.class_name}` is invalid. " "Class names must be in CamelCase, starting with a capital letter." ) + foreign_keys = [] sql, _external_stores, primary_key, fk_attribute_map, pre_ddl, post_ddl = declare( - self.full_table_name, self.definition, context, self.connection.adapter, config=self.connection._config + self.full_table_name, + self.definition, + context, + self.connection.adapter, + config=self.connection._config, + foreign_keys=foreign_keys, ) - # Call declaration hook for validation (subclasses like AutoPopulate can override) - self._declare_check(primary_key, fk_attribute_map) + # Call declaration hook for validation (subclasses like AutoPopulate and Part override it) + self._declare_check(primary_key, fk_attribute_map, foreign_keys) sql = _substitute_database(sql, self.database) try: @@ -198,12 +204,13 @@ def declare(self, context=None): # Populate lineage table for this table's attributes self._populate_lineage(primary_key, fk_attribute_map) - def _declare_check(self, primary_key, fk_attribute_map): + def _declare_check(self, primary_key, fk_attribute_map, foreign_keys): """ Hook for declaration-time validation. Subclasses can override. Called before the table is created in the database. Override this method - to add validation logic (e.g., AutoPopulate validates FK-only primary keys). + to add validation logic (e.g., AutoPopulate validates FK-only primary keys, + Part checks its references against its master). Parameters ---------- @@ -211,6 +218,9 @@ def _declare_check(self, primary_key, fk_attribute_map): List of primary key attribute names. fk_attribute_map : dict Dict mapping child_attr -> (parent_table, parent_attr). + foreign_keys : list + One ``(parent_table, attr_map, nullable)`` entry per foreign key, + where ``attr_map`` maps child_attr -> parent_attr. """ pass # Default: no validation diff --git a/src/datajoint/user_tables.py b/src/datajoint/user_tables.py index c497d69e5..8bc158a0a 100644 --- a/src/datajoint/user_tables.py +++ b/src/datajoint/user_tables.py @@ -3,8 +3,10 @@ """ import re +import warnings from .autopopulate import AutoPopulate +from .dependencies import CHAIN_LEGEND, extract_master, group_cycle from .errors import DataJointError from .table import Table from .utils import from_camel_case @@ -214,6 +216,72 @@ class Part(UserTable, metaclass=PartMeta): + ")" ) + def _declare_check(self, primary_key, fk_attribute_map, foreign_keys): + """ + Check the part's references before the part is created. + + A part may reference its master, another part of the same master, or a + table outside its master-part group that does not depend on its master. + A reference to a table that does depend on the master would make the group + depend on itself, and DataJoint could no longer order its tables for + delete, drop or trace, so the declaration is refused. + + A part that does not inherit its master's primary key, unrenamed and + non-nullable, through a reference to the master or to another part of it, + gets a warning. + + Parameters + ---------- + primary_key : list + List of primary key attribute names. + fk_attribute_map : dict + Dict mapping child_attr -> (parent_table, parent_attr). + foreign_keys : list + One ``(parent_table, attr_map, nullable)`` entry per foreign key. + + Raises + ------ + DataJointError + If a reference would close a cycle through the part's master-part group. + """ + super()._declare_check(primary_key, fk_attribute_map, foreign_keys) + part = self.full_table_name + master = extract_master(part) + parents = [parent for parent, _, _ in foreign_keys] + + if any(master not in (parent, extract_master(parent)) for parent in parents): + # Reload: the graph may predate tables declared or dropped since. A + # cycle runs through the ancestors of the referenced tables, so their + # schemas and everything upstream of them is enough. + dependencies = self.connection.dependencies + dependencies.clear() + split = self.connection.adapter.split_full_table_name + dependencies.load_all_upstream({self.database} | {split(parent)[0] for parent in parents}) + if cycle := group_cycle(dependencies, part, parents): + parent = cycle[-3] # the cycle ends with the referenced table, the part, and the master + raise DataJointError( + f"Part table {part} cannot reference {parent}, which depends on the part's master {master}. " + f"The reference would close the cycle {' -> '.join(cycle)} ({CHAIN_LEGEND}), and DataJoint " + "could no longer order these tables for delete, drop or trace. A part table may reference its " + "master, another part of the same master, or a table that does not depend on its master." + ) + + master_key = self._master.primary_key + if master_key and not any( + not nullable and all(attr_map.get(attr) == attr for attr in master_key) + for parent, attr_map, nullable in foreign_keys + if master in (parent, extract_master(parent)) + ): + warnings.warn( + f"Part table {part} does not inherit the primary key of its master {master} " + f"({', '.join(master_key)}), unrenamed, through a non-nullable reference to the master or to another " + "part of it. Without that key its rows cannot be matched to master rows by key, and operations that " + "keep a master and its parts together may treat it as a separate table. " + "Reference the master with `-> master` unless this is intended.", + UserWarning, + stacklevel=2, + ) + def delete(self, part_integrity: str = "enforce", **kwargs): """ Delete from a Part table. diff --git a/tests/integration/test_group_cycle.py b/tests/integration/test_group_cycle.py new file mode 100644 index 000000000..f08c17827 --- /dev/null +++ b/tests/integration/test_group_cycle.py @@ -0,0 +1,427 @@ +""" +Integration tests for the master-part group cycle rule. + +A part may reference its master, another part of the same master, or a table +that does not depend on its master. A reference to a table that depends on the +master would make the master-part group depend on itself, and DataJoint could +no longer order the tables for delete, drop or trace, so the declaration is +refused before any DDL runs. + +A cycle declared before the rule existed (here, with the check switched off) +makes the operations that must order it raise an error that names the cycle, +while operations on the rest of the schema keep working. +""" + +import itertools +import time +import warnings + +import pytest + +import datajoint as dj +from datajoint.errors import DataJointError + +_counter = itertools.count() + + +@pytest.fixture(scope="function") +def make_schema(connection_by_backend, db_creds_by_backend): + """Create fresh schemas for a test and drop them afterwards, last created first.""" + backend = db_creds_by_backend["backend"] + schemas = [] + + def make(): + name = f"djtest_groupcycle_{backend}_{str(int(time.time() * 1000))[-8:]}_{next(_counter)}"[:64] + schemas.append(dj.Schema(name, connection=connection_by_backend)) + return schemas[-1] + + yield make + + for schema in reversed(schemas): + try: + schema.drop(prompt=False) + except Exception: + pass + + +@pytest.fixture(scope="function") +def schema_by_backend(make_schema): + return make_schema() + + +def regrow(master, **parts): + """ + Return a new class for ``master``'s table that adds part tables to it, as + when parts are added to a pipeline that is already declared. + + Each keyword names a part and gives its definition. + """ + namespace = {"definition": master.definition} + for name, definition in parts.items(): + namespace[name] = type(name, (dj.Part,), {"definition": definition}) + return type(master.__name__, master.__bases__, namespace) + + +def master_key_warnings(record): + return [w for w in record if "does not inherit the primary key of its master" in str(w.message)] + + +# --------------------------------------------------------------------------- +# Refused declarations +# --------------------------------------------------------------------------- + + +def test_part_referencing_a_descendant_of_its_master_is_refused(schema_by_backend): + """Master -> X -> Master.Part: X references the master, the part references X.""" + schema = schema_by_backend + + @schema + class Master(dj.Manual): + definition = """ + master_id : int32 + """ + + @schema + class X(dj.Manual): + definition = """ + -> Master + x_id : int32 + """ + + Grown = regrow( + Master, + Part=""" + -> master + -> X + """, + ) + with pytest.raises(DataJointError) as error: + schema(Grown) + + part, master, x = Grown.Part.full_table_name, Master.full_table_name, X.full_table_name + message = str(error.value) + assert f"Part table {part} cannot reference {x}, which depends on the part's master {master}" in message + assert f"{master} -> {x} -> {part} -> {master}" in message + + # Refused before any DDL ran: the part does not exist, and the schema still sorts. + assert not Grown.Part().is_declared + assert set(schema.list_tables()) == {Master.table_name, X.table_name} + Master.insert1({"master_id": 1}) + X.insert1({"master_id": 1, "x_id": 1}) + Master.delete() + assert len(X()) == 0 + + +@pytest.mark.parametrize("reference", ["A", "A.Part"]) +def test_parts_referencing_each_others_masters_are_refused(schema_by_backend, reference): + """A.Part references B; a part of B that references A (or A.Part) closes the cycle.""" + schema = schema_by_backend + + @schema + class B(dj.Manual): + definition = """ + b_id : int32 + """ + + @schema + class A(dj.Manual): + definition = """ + a_id : int32 + """ + + class Part(dj.Part): + definition = """ + -> master + -> B + """ + + Grown = regrow( + B, + Part=f""" + -> master + -> {reference} + """, + ) + with pytest.raises(DataJointError) as error: + schema(Grown) + + part, b, a, a_part = Grown.Part.full_table_name, B.full_table_name, A.full_table_name, A.Part.full_table_name + message = str(error.value) + if reference == "A": + assert f"Part table {part} cannot reference {a}" in message + assert f"{b} -> {a_part} -> {a} -> {part} -> {b}" in message + else: + assert f"Part table {part} cannot reference {a_part}" in message + assert f"{b} -> {a_part} -> {part} -> {b}" in message + assert not Grown.Part().is_declared + + +def test_cycle_across_schemas_is_refused(make_schema): + """The table that depends on the master lives in another schema.""" + upstream, downstream = make_schema(), make_schema() + + @upstream + class Master(dj.Manual): + definition = """ + master_id : int32 + """ + + @downstream + class Analysis(dj.Manual): + definition = """ + -> Master + analysis_id : int32 + """ + + @downstream + class Report(dj.Manual): + definition = """ + -> Analysis + """ + + Grown = regrow( + Master, + Summary=""" + -> master + -> Report + """, + ) + with pytest.raises(DataJointError) as error: + upstream(Grown) + master, analysis, report = Master.full_table_name, Analysis.full_table_name, Report.full_table_name + assert f"{master} -> {analysis} -> {report} -> {Grown.Summary.full_table_name} -> {master}" in str(error.value) + assert not Grown.Summary().is_declared + + +# --------------------------------------------------------------------------- +# Allowed declarations +# --------------------------------------------------------------------------- + + +def test_allowed_part_references(schema_by_backend): + """A part may reference its master, a sibling, an upstream table, or a table downstream of another group.""" + schema = schema_by_backend + + @schema + class Probe(dj.Lookup): + definition = """ + probe_id : int32 + """ + contents = [(1,), (2,)] + + @schema + class Other(dj.Manual): + definition = """ + other_id : int32 + """ + + class Item(dj.Part): + definition = """ + -> master + item_id : int32 + """ + + @schema + class OtherSummary(dj.Manual): + definition = """ + -> Other + """ + + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + + @schema + class Session(dj.Manual): + definition = """ + session_id : int32 + """ + + class Channel(dj.Part): + definition = """ + -> master + channel_id : int32 + --- + -> Probe + """ + + class Spike(dj.Part): + definition = """ + -> master.Channel + spike_id : int32 + """ + + class Link(dj.Part): + definition = """ + -> master + -> OtherSummary + -> Other.Item + """ + + assert not master_key_warnings(record) + assert {Session.Channel.table_name, Session.Spike.table_name, Session.Link.table_name} <= set(schema.list_tables()) + + Other.insert1({"other_id": 1}) + Other.Item.insert1({"other_id": 1, "item_id": 1}) + OtherSummary.insert1({"other_id": 1}) + Session.insert1({"session_id": 1}) + Session.Channel.insert1({"session_id": 1, "channel_id": 1, "probe_id": 1}) + Session.Spike.insert1({"session_id": 1, "channel_id": 1, "spike_id": 1}) + Session.Link.insert1({"session_id": 1, "other_id": 1, "item_id": 1}) + + trace = dj.Diagram.trace(Session.Link & {"session_id": 1}) + assert len(trace[Other]) == 1 + assert dj.Diagram.cascade(Other & {"other_id": 1}).counts()[Session.Link.full_table_name] == 1 + (Session & {"session_id": 1}).delete() + assert len(Session.Spike()) == len(Session.Link()) == 0 + (Other & {"other_id": 1}).delete() + assert len(OtherSummary()) == 0 + + +# --------------------------------------------------------------------------- +# Cycles declared before the rule +# --------------------------------------------------------------------------- + + +def test_legacy_cycle_is_named_and_contained(schema_by_backend, monkeypatch): + schema = schema_by_backend + + @schema + class Master(dj.Manual): + definition = """ + master_id : int32 + """ + + @schema + class X(dj.Manual): + definition = """ + -> Master + x_id : int32 + """ + + @schema + class Unrelated(dj.Manual): + definition = """ + unrelated_id : int32 + """ + + @schema + class UnrelatedChild(dj.Manual): + definition = """ + -> Unrelated + child_id : int32 + """ + + # Declare the cycle the way a schema declared before the rule would hold it. + monkeypatch.setattr(dj.user_tables, "group_cycle", lambda graph, part, parents: None) + Cyclic = regrow( + Master, + Part=""" + -> master + -> X + """, + ) + schema(Cyclic) + monkeypatch.undo() + assert Cyclic.Part().is_declared + + Master.insert1({"master_id": 1}) + X.insert1({"master_id": 1, "x_id": 1}) + Cyclic.Part.insert1({"master_id": 1, "x_id": 1}) + Unrelated.insert1({"unrelated_id": 1}) + UnrelatedChild.insert1({"unrelated_id": 1, "child_id": 1}) + + master, x, part = Master.full_table_name, X.full_table_name, Cyclic.Part.full_table_name + cycle = f"{master} -> {x} -> {part} -> {master}" + + # Operations that must order the cycle name it, instead of a networkx error. + for operation in ( + lambda: Master.delete(), + lambda: dj.Diagram.cascade(Master & {"master_id": 1}), + lambda: dj.Diagram.trace(Cyclic.Part & {"master_id": 1}), + lambda: schema.list_tables(), + ): + with pytest.raises(DataJointError) as error: + operation() + assert cycle in str(error.value) + assert f"part table {part} references a table that depends on its own master" in str(error.value) + assert len(Master()) == len(X()) == len(Cyclic.Part()) == 1 + + # Operations elsewhere do not need to order the cycle and keep working. + assert len(dj.Diagram.trace(UnrelatedChild & {"child_id": 1})[Unrelated]) == 1 + (Unrelated & {"unrelated_id": 1}).delete() + assert len(Unrelated()) == len(UnrelatedChild()) == 0 + UnrelatedChild.drop() + + # Recovery, as the error suggests: drop the part and declare it without the reference. + Cyclic.Part.drop(part_integrity="ignore") + schema( + regrow( + Master, + Part=""" + -> master + x_id : int32 + """, + ) + ) + assert set(schema.list_tables()) >= {Master.table_name, X.table_name, Cyclic.Part.table_name} + Master.delete() + assert len(Master()) == len(X()) == 0 + + +# --------------------------------------------------------------------------- +# A part that does not inherit its master's primary key +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "definition", + [ + """ + part_id : int32 + """, + """ + -> master.proj(other_id="master_id") + part_id : int32 + """, + """ + part_id : int32 + --- + -> [nullable] master + """, + ], + ids=["no-reference", "renamed", "nullable"], +) +def test_part_without_master_key_warns(schema_by_backend, definition): + schema = schema_by_backend + + @schema + class Master(dj.Manual): + definition = """ + master_id : int32 + """ + + with pytest.warns(UserWarning, match="does not inherit the primary key of its master") as record: + schema(regrow(Master, Part=definition)) + assert Master.full_table_name in str(master_key_warnings(record)[0].message) + + +def test_part_with_master_key_through_secondary_reference_does_not_warn(schema_by_backend): + schema = schema_by_backend + + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + + @schema + class Master(dj.Manual): + definition = """ + master_id : int32 + """ + + class Part(dj.Part): + definition = """ + part_id : int32 + --- + -> master + """ + + assert not master_key_warnings(record) diff --git a/tests/unit/test_group_cycle.py b/tests/unit/test_group_cycle.py new file mode 100644 index 000000000..e8cc0f29d --- /dev/null +++ b/tests/unit/test_group_cycle.py @@ -0,0 +1,232 @@ +""" +Unit tests for the master-part group cycle rule, on hand-built dependency graphs. + +A master and its parts form one group. ``topo_sort`` orders a master after +every table its parts reference, so it fails when the groups depend on each +other in a cycle. ``group_cycle`` finds the cycle a new part's references would +close, and the declaration of such a part is refused. +""" + +import random + +import networkx as nx +import pytest + +from datajoint.dependencies import group_ancestors, group_cycle, topo_sort +from datajoint.errors import DataJointError + + +def t(name): + """Full table name in schema `s`.""" + return f"`s`.`{name}`" + + +def graph(*edges): + """Dependency graph from (parent, child) pairs of short table names.""" + g = nx.MultiDiGraph() + g.add_edges_from((t(parent), t(child)) for parent, child in edges) + return g + + +# --------------------------------------------------------------------------- +# group_ancestors +# --------------------------------------------------------------------------- + + +def test_group_ancestors_plain_chain(): + g = graph(("a", "b"), ("b", "c")) + assert group_ancestors(g, t("c")) == {t("a"), t("b"), t("c")} + assert group_ancestors(g, t("a")) == {t("a")} + + +def test_group_ancestors_master_brings_its_parts_and_their_parents(): + # x is referenced by a part of m, so everything downstream of m depends on x. + g = graph(("m", "m__p"), ("x", "m__p"), ("m", "y")) + assert group_ancestors(g, t("y")) == {t("y"), t("m"), t("m__p"), t("x")} + + +def test_group_ancestors_part_reaches_master_through_its_reference(): + g = graph(("m", "m__p"), ("x", "m__q"), ("m", "m__q")) + # m__p references m, and m brings its other part m__q and its parent x. + assert group_ancestors(g, t("m__p")) == {t("m__p"), t("m"), t("m__q"), t("x")} + + +def test_group_ancestors_part_without_master_reference_is_plain(): + # m__p does not reference m: reaching it does not reach its master. + g = graph(("x", "m__p"), ("y", "m")) + g.add_node(t("m")) + assert group_ancestors(g, t("m__p")) == {t("m__p"), t("x")} + + +def test_group_ancestors_across_two_groups(): + g = graph(("b", "a__p"), ("a", "a__p"), ("b", "b__p"), ("z", "b__p")) + assert group_ancestors(g, t("a")) == {t("a"), t("a__p"), t("b"), t("b__p"), t("z")} + + +# --------------------------------------------------------------------------- +# group_cycle +# --------------------------------------------------------------------------- + + +def test_group_cycle_master_x_part(): + g = graph(("m", "x")) + assert group_cycle(g, t("m__p"), [t("m"), t("x")]) == [t("m"), t("x"), t("m__p"), t("m")] + + +def test_group_cycle_two_groups(): + # a__p references b; a part of b that references a closes the cycle. + g = graph(("a", "a__p"), ("b", "a__p"), ("b", "b__p")) + assert group_cycle(g, t("b__q"), [t("b"), t("a")]) == [t("b"), t("a__p"), t("a"), t("b__q"), t("b")] + # so does a reference to the other group's part + assert group_cycle(g, t("b__q"), [t("b"), t("a__p")]) == [t("b"), t("a__p"), t("b__q"), t("b")] + + +def test_group_cycle_through_a_master_downstream(): + # m -> y -> n__q, a part of n; n -> z. A part of m referencing z closes a cycle. + g = graph(("m", "y"), ("y", "n__q"), ("n", "n__q"), ("n", "z")) + assert group_cycle(g, t("m__p"), [t("m"), t("z")]) == [t("m"), t("y"), t("n__q"), t("n"), t("z"), t("m__p"), t("m")] + + +@pytest.mark.parametrize( + "parents", + [ + ["m"], # its master + ["m", "m__s"], # a sibling part + ["m", "u"], # an unrelated upstream table + ["m", "y"], # a table downstream of another group + ["m__s"], # a sibling alone + ], +) +def test_group_cycle_allowed(parents): + g = graph(("m", "m__s"), ("u", "m"), ("n", "n__q"), ("n", "y"), ("u", "y")) + assert group_cycle(g, t("m__p"), [t(p) for p in parents]) is None + + +def test_group_cycle_ignores_parents_not_in_graph(): + g = graph(("m", "x")) + assert group_cycle(g, t("m__p"), [t("m"), t("elsewhere")]) is None + + +def test_group_cycle_master_not_in_graph(): + g = graph(("a", "b")) + assert group_cycle(g, t("m__p"), [t("b")]) is None + + +# --------------------------------------------------------------------------- +# topo_sort +# --------------------------------------------------------------------------- + + +def test_topo_sort_names_a_group_cycle(): + g = graph(("m", "x"), ("x", "m__p"), ("m", "m__p")) + with pytest.raises(DataJointError, match="cycle") as error: + topo_sort(g) + message = str(error.value) + assert f"{t('m')} -> {t('x')} -> {t('m__p')} -> {t('m')}" in message + assert f"part table {t('m__p')} references a table that depends on its own master" in message + + +def test_topo_sort_names_both_parts_of_a_two_group_cycle(): + g = graph(("a", "a__p"), ("b", "a__p"), ("b", "b__p"), ("a", "b__p")) + with pytest.raises(DataJointError, match="part tables .* each reference") as error: + topo_sort(g) + assert t("a__p") in str(error.value) and t("b__p") in str(error.value) + + +def test_topo_sort_names_a_plain_cycle(): + g = graph(("a", "b"), ("b", "a")) + with pytest.raises(DataJointError, match="form a cycle") as error: + topo_sort(g) + assert "master-part" not in str(error.value) + + +def test_topo_sort_nodes_skip_a_cycle_elsewhere(): + g = graph(("m", "x"), ("x", "m__p"), ("m", "m__p"), ("u", "v"), ("v", "w")) + assert topo_sort(g, [t("u"), t("v"), t("w")]) == [t("u"), t("v"), t("w")] + assert topo_sort(g, [t("m__p")]) == [t("m__p")] + with pytest.raises(DataJointError, match="cycle"): + topo_sort(g, [t("m"), t("m__p")]) + + +def test_topo_sort_nodes_keep_order_through_tables_left_out(): + # a -> z -> b: z is not requested, but a must still come before b. + g = graph(("z", "b"), ("a", "z")) + g.add_node(t("c")) + assert topo_sort(g, [t("b"), t("a")]) == [t("a"), t("b")] + assert topo_sort(g, [t("b"), t("a"), t("missing")]) == [t("a"), t("b")] + assert topo_sort(g, []) == [] + + +def test_topo_sort_nodes_keep_parts_after_masters(): + g = graph(("a", "m"), ("a", "z"), ("m", "m__part"), ("z", "m__part")) + assert topo_sort(g, [t("m"), t("m__part"), t("z")]) == [t("z"), t("m"), t("m__part")] + + +# --------------------------------------------------------------------------- +# Exactness: refusing what group_cycle finds keeps topo_sort working, and +# refuses nothing topo_sort could sort. +# --------------------------------------------------------------------------- + + +def sorts(g): + try: + topo_sort(g) + except DataJointError: + return False + return True + + +def test_group_cycle_is_exact(): + """ + Declare random tables in order, each referencing random earlier tables. + + A new part is refused exactly when adding it would make topo_sort fail. No + other declaration ever makes topo_sort fail: not a plain table, not a + master, and not a master declared after a part of it that does not + reference it (as a part left behind by a dropped master would be). + """ + counts = {"refused": 0, "accepted": 0, "late masters": 0} + for seed in range(300): + rng = random.Random(seed) + g = nx.MultiDiGraph() + masters, orphans = [], [] + for i in range(14): + existing = list(g) + kind = rng.choice(["plain", "master", "part", "part", "part"]) + if kind == "part" and masters and rng.random() < 0.9: + master = rng.choice(masters) + elif kind == "part": + master = t(f"m{i}") # its master is not declared yet + orphans.append(master) + if kind == "part": + table = t(f"{master[5:-1]}__p{i}") + elif kind == "master" and orphans and rng.random() < 0.5: + table = orphans.pop() + counts["late masters"] += 1 + else: + table = t(f"{'m' if kind == 'master' else 't'}{i}") + if table in g: + continue + parents = rng.sample(existing, k=min(len(existing), rng.randint(0, 3))) + if kind == "part" and master in g and rng.random() < 0.8: + parents.append(master) # `-> master`, usually + + candidate = nx.MultiDiGraph(g) + candidate.add_node(table) + candidate.add_edges_from((parent, table) for parent in parents) + + if kind == "part": + cycle = group_cycle(g, table, parents) + assert (cycle is None) == sorts(candidate), (seed, table, parents, cycle) + if cycle is not None: + counts["refused"] += 1 + continue + counts["accepted"] += 1 + else: + assert sorts(candidate), (seed, table, parents) + if kind == "master": + masters.append(table) + g = candidate + assert sorts(g) + # the random declarations exercise every case + assert all(counts.values()), counts From 1dee94d43e3091efb9bcb63042d1dde4cde14670 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Sat, 17 Oct 2026 17:25:35 -0500 Subject: [PATCH 2/2] fix(declare): group only owned parts with their masters A part is owned by its master when a foreign key in its primary key references the master, or another owned part of the same master; the reference may rename the attributes. Grouping parts by table name alone made topo_sort fail on schemas with no cycle (a part that does not reference its master but references a table downstream of it) and accept a cycle through a part that references its master only through a nullable or secondary foreign key. - owned_parts() reads ownership from the dependency graph through each edge's primary flag; is_owned() decides it for a part being declared. - topo_sort keeps only owned parts with their masters; other parts are sorted as separate tables. group_graph, group_ancestors and group_cycle use owned parts, so a reference to a part the master does not own counts as a reference outside the group. - Part._declare_check checks only owned parts for cycles and warns on a part that is not owned. The foreign-key records drop nullability. - The cascade's sort scope keeps following the name-based part-to-master lift of the current part_integrity engine. --- src/datajoint/autopopulate.py | 4 +- src/datajoint/declare.py | 14 +- src/datajoint/dependencies.py | 159 ++++++++++---- src/datajoint/diagram.py | 18 +- src/datajoint/table.py | 2 +- src/datajoint/user_tables.py | 59 ++--- tests/integration/test_group_cycle.py | 177 +++++++++++++-- tests/unit/test_group_cycle.py | 299 ++++++++++++++++++++------ 8 files changed, 569 insertions(+), 163 deletions(-) diff --git a/src/datajoint/autopopulate.py b/src/datajoint/autopopulate.py index c6ad04931..d59b8ba85 100644 --- a/src/datajoint/autopopulate.py +++ b/src/datajoint/autopopulate.py @@ -193,7 +193,7 @@ def _declare_check( self, primary_key: list[str], fk_attribute_map: dict[str, tuple[str, str]], - foreign_keys: list[tuple[str, dict[str, str], bool]], + foreign_keys: list[tuple[str, dict[str, str]]], ) -> None: """ Validate FK-only primary key constraint for auto-populated tables. @@ -209,7 +209,7 @@ def _declare_check( fk_attribute_map : dict Mapping of child_attr -> (parent_table, parent_attr). foreign_keys : list - One ``(parent_table, attr_map, nullable)`` entry per foreign key. + One ``(parent_table, attr_map)`` entry per foreign key. Not used here. Raises diff --git a/src/datajoint/declare.py b/src/datajoint/declare.py index 524cb22ec..24d76520e 100644 --- a/src/datajoint/declare.py +++ b/src/datajoint/declare.py @@ -204,7 +204,7 @@ def compile_foreign_key( fk_attribute_map: dict[str, tuple[str, str]] | None = None, fk_index_candidates: list[list[str]] | None = None, column_comments: dict[str, str] | None = None, - foreign_keys: list[tuple[str, dict[str, str], bool]] | None = None, + foreign_keys: list[tuple[str, dict[str, str]]] | None = None, ) -> None: """ Parse a foreign key line and update declaration components. @@ -241,7 +241,7 @@ def compile_foreign_key( comment, with its core-type marker (e.g. ``:int32:``), is added so the child keeps the parent's declared type. Updated in place. foreign_keys : list, optional - One ``(parent_table, attr_map, nullable)`` entry per foreign key, where + One ``(parent_table, attr_map)`` entry per foreign key, where ``attr_map`` maps each child attribute to the parent attribute it references. Unlike ``fk_attribute_map``, it keeps every reference when several share an attribute. Updated in place. @@ -335,7 +335,7 @@ def compile_foreign_key( ) if foreign_keys is not None: attr_map = {attr: ref.heading[attr].original_name for attr in ref.primary_key} - foreign_keys.append((ref_table_name, attr_map, is_nullable)) + foreign_keys.append((ref_table_name, attr_map)) # Supporting index on the foreign-key columns. # @@ -397,7 +397,7 @@ def prepare_declare( *, table_name: str | None = None, config=None, - foreign_keys: list[tuple[str, dict[str, str], bool]] | None = None, + foreign_keys: list[tuple[str, dict[str, str]]] | None = None, ) -> tuple[str, list[str], list[str], list[str], list[str], list[str], dict[str, tuple[str, str]], dict[str, str]]: """ Parse a table definition into its components. @@ -423,7 +423,7 @@ def prepare_declare( config : Config, optional Read for ``jobs.add_job_metadata`` and ``provenance.capture``. foreign_keys : list, optional - Receives one ``(parent_table, attr_map, nullable)`` entry per foreign + Receives one ``(parent_table, attr_map)`` entry per foreign key; see :func:`compile_foreign_key`. Updated in place. Returns @@ -559,7 +559,7 @@ def declare( adapter, *, config=None, - foreign_keys: list[tuple[str, dict[str, str], bool]] | None = None, + foreign_keys: list[tuple[str, dict[str, str]]] | None = None, ) -> tuple[str, list[str], list[str], dict[str, tuple[str, str]], list[str], list[str]]: r""" Parse a definition and generate SQL CREATE TABLE statement. @@ -577,7 +577,7 @@ def declare( config : Config, optional Configuration object. If None, falls back to global config. foreign_keys : list, optional - Receives one ``(parent_table, attr_map, nullable)`` entry per foreign + Receives one ``(parent_table, attr_map)`` entry per foreign key; see :func:`compile_foreign_key`. Updated in place. Returns diff --git a/src/datajoint/dependencies.py b/src/datajoint/dependencies.py index 164834349..76f21946a 100644 --- a/src/datajoint/dependencies.py +++ b/src/datajoint/dependencies.py @@ -44,44 +44,118 @@ def extract_master(part_table: str) -> str | None: CHAIN_LEGEND = "each table references the one before it or is the master of the part before it" -def group_graph(graph: nx.MultiDiGraph) -> nx.MultiDiGraph: +def owned_parts(graph: nx.MultiDiGraph) -> set[str]: """ - Copy a dependency graph and add an edge from each part table to its master. + Return the part tables that are owned by their masters. - A master's rows are inserted and deleted together with its parts' rows, so a - master depends on everything its parts depend on. With the added edges, - ``nx.ancestors`` gives a table's ancestors where reaching a master also - reaches all of its parts, and ``nx.descendants`` gives the converse, where - reaching a part also reaches its master. These are the same dependencies that - :func:`topo_sort` orders. + A part is owned by its master when a foreign key in the part's primary key + references the master, or references another owned part of the same master. + The reference may rename the attributes. Only owned parts form a group with + their master; every other part (one with no reference to its master, or only a + secondary or nullable one, or only one through a part that is not owned) is + treated as a separate table. - The added edges form a cycle with each part that references its master, so - the result is for reachability only, not for sorting. + Each foreign key is its own edge, renamed ones included, and carries a + ``primary`` flag that is True when all its attributes are in the referencing + table's primary key. An edge without the flag, as in a graph built by hand, + counts as primary. Parameters ---------- graph : nx.MultiDiGraph Dependency graph. + Returns + ------- + set[str] + Full names of the owned parts. + """ + masters = {extract_master(node) for node in graph.nodes} & set(graph.nodes) + owned = set() + for master in masters: + frontier = [master] + while frontier: + for _, child, props in graph.out_edges(frontier.pop(), data=True): + if child not in owned and extract_master(child) == master and props.get("primary", True): + owned.add(child) + frontier.append(child) + return owned + + +def _in_group(table: str, master: str, owned: set[str]) -> bool: + """Whether ``table`` is ``master`` or one of its owned parts.""" + return table == master or (table in owned and extract_master(table) == master) + + +def is_owned(graph: nx.MultiDiGraph, part: str, references: list[tuple[str, bool]]) -> bool: + """ + Tell whether a new part table will be owned by its master (see :func:`owned_parts`). + + Parameters + ---------- + graph : nx.MultiDiGraph + Dependency graph of the declared tables, which does not include ``part``. + part : str + Full name of the part table being declared. + references : list[tuple[str, bool]] + One ``(parent_table, primary)`` pair per foreign key of the part, where + ``primary`` is True when all the foreign key's attributes are in the + part's primary key. + + Returns + ------- + bool + True if a primary-key reference goes to the master or to an owned part of it. + """ + master = extract_master(part) + if master not in graph: + return False + owned = owned_parts(graph) + return any(primary and _in_group(parent, master, owned) for parent, primary in references) + + +def group_graph(graph: nx.MultiDiGraph, owned: set[str] | None = None) -> nx.MultiDiGraph: + """ + Copy a dependency graph and add an edge from each owned part to its master. + + A master's rows are inserted and deleted together with its owned parts' rows, + so a master depends on everything its owned parts depend on. With the added + edges, ``nx.ancestors`` gives a table's ancestors where reaching a master also + reaches all of its owned parts, and ``nx.descendants`` gives the converse, + where reaching an owned part also reaches its master. These are the + dependencies that :func:`topo_sort` orders. Parts that are not owned are + treated as separate tables. + + The added edges form a cycle with each owned part, so the result is for + reachability only, not for sorting. + + Parameters + ---------- + graph : nx.MultiDiGraph + Dependency graph. + owned : set[str], optional + The owned parts, if already computed with :func:`owned_parts`. + Returns ------- nx.MultiDiGraph - A copy of ``graph`` with an edge from each part to its master. + A copy of ``graph`` with an edge from each owned part to its master. """ + owned = owned_parts(graph) if owned is None else owned graph = nx.MultiDiGraph(graph) for part in list(graph.nodes): - if (master := extract_master(part)) in graph: - graph.add_edge(part, master, key="master") + if part in owned: + graph.add_edge(part, extract_master(part), key="master") return graph def group_ancestors(graph: nx.MultiDiGraph, table: str) -> set[str]: """ - Return a table and its ancestors, where reaching a master reaches all its parts. + Return a table and its ancestors, where reaching a master reaches all its owned parts. The closure starts from ``table`` and adds the parents of every table in it, - and all the parts of every master in it, until nothing new is added. A part - reaches its own master through its reference to the master (``-> master``). + and all the owned parts of every master in it, until nothing new is added. An + owned part reaches its master through its own references. Parameters ---------- @@ -100,24 +174,26 @@ def group_ancestors(graph: nx.MultiDiGraph, table: str) -> set[str]: def group_cycle(graph: nx.MultiDiGraph, part: str, parents: list[str]) -> list[str] | None: """ - Find the cycle that a new part table's references would close. + Find the cycle that the references of a new owned part would close. - A part may reference its master, another part of the same master, or a table - outside its master-part group. A reference to a table outside the group closes - a cycle when that table already depends on the part's master (see - :func:`group_ancestors`): the group would then depend on itself, and - :func:`topo_sort` could no longer order its tables. + An owned part may reference its master, an owned part of the same master, or + a table outside its master-part group, which includes the master's parts that + are not owned. A reference to a table outside the group closes a cycle when + that table already depends on the part's master (see :func:`group_ancestors`): + the group would then depend on itself, and :func:`topo_sort` could no longer + order its tables. - A new table has no children yet, so only a new part's references to tables - outside its group can close such a cycle, and checking them when the part is - declared keeps every cycle out. + A new table has no children yet, so only the references of a new owned part + to tables outside its group can close such a cycle. A part that is not owned + is a separate table and closes none (see :func:`is_owned`). Checking owned + parts when they are declared therefore keeps every cycle out. Parameters ---------- graph : nx.MultiDiGraph Dependency graph of the declared tables, which does not include ``part``. part : str - Full name of the part table being declared. + Full name of the owned part table being declared. parents : list[str] Full names of the tables the part references. @@ -130,10 +206,11 @@ def group_cycle(graph: nx.MultiDiGraph, part: str, parents: list[str]) -> list[s master = extract_master(part) if master not in graph: return None - outside = [p for p in parents if p in graph and master not in (p, extract_master(p))] + owned = owned_parts(graph) + outside = [p for p in parents if p in graph and not _in_group(p, master, owned)] if not outside: return None - extended = group_graph(graph) + extended = group_graph(graph, owned) for parent in outside: try: path = nx.shortest_path(extended, master, parent) @@ -190,7 +267,11 @@ def _cycle_error(graph: nx.MultiDiGraph) -> DataJointError: def topo_sort(graph: nx.MultiDiGraph, nodes=None) -> list[str]: """ - Topological sort keeping part tables with their masters. + Topological sort keeping owned part tables with their masters. + + Each owned part (see :func:`owned_parts`) is placed right after its master, so + the master is ordered after every table its owned parts reference. Parts that + are not owned are sorted as separate tables. Parameters ---------- @@ -205,14 +286,15 @@ def topo_sort(graph: nx.MultiDiGraph, nodes=None) -> list[str]: Returns ------- list[str] - Table names in topological order with parts following masters. + Table names in topological order with owned parts following their masters. Raises ------ DataJointError If the dependencies of the tables to sort form a cycle, for example one - that runs through a master and its parts. + that runs through a master and its owned parts. """ + owned = owned_parts(graph) if nodes is None: graph = nx.MultiDiGraph(graph) # make a mutable copy else: @@ -225,15 +307,14 @@ def topo_sort(graph: nx.MultiDiGraph, nodes=None) -> list[str]: subgraph.add_edges_from((u, v) for u, v in graph.edges(keep) if v in keep) graph = subgraph - # Add parts' dependencies to their masters' dependencies + # Add owned parts' dependencies to their masters' dependencies # to ensure correct topological ordering of the masters. + # Parts that are not owned are sorted as separate tables. for part in graph: - # find the part's master - if (master := extract_master(part)) in graph: - for edge in graph.in_edges(part): - parent = edge[0] - if master not in (parent, extract_master(parent)): - # if parent is neither master nor part of master + if part in owned and (master := extract_master(part)) in graph: + for parent, _ in graph.in_edges(part): + if not _in_group(parent, master, owned): + # if parent is neither master nor owned part of master graph.add_edge(parent, master, part=part) try: sorted_nodes = list(nx.topological_sort(graph)) @@ -245,7 +326,7 @@ def topo_sort(graph: nx.MultiDiGraph, nodes=None) -> list[str]: placed = set() while pos > 1: part = sorted_nodes[pos] - if (master := extract_master(part)) not in graph or part in placed: + if part not in owned or (master := extract_master(part)) not in graph or part in placed: pos -= 1 else: placed.add(part) diff --git a/src/datajoint/diagram.py b/src/datajoint/diagram.py index db8a59a82..3c749765c 100644 --- a/src/datajoint/diagram.py +++ b/src/datajoint/diagram.py @@ -24,7 +24,7 @@ import networkx as nx from .condition import AndList -from .dependencies import extract_master, group_graph, topo_sort +from .dependencies import extract_master, topo_sort from .errors import DataJointError from .table import Table, lookup_class_name from .user_tables import Computed, Imported, Lookup, Manual, Part, _get_tier @@ -781,12 +781,18 @@ def _propagate_restrictions(self, start_node, mode, part_integrity="enforce"): # Only propagate through descendants of start_node allowed_nodes = {start_node} | set(nx.descendants(self, start_node)) - # Sort only the tables propagation can reach: part_integrity="cascade" also - # reaches the master of every part it reaches, and the master's descendants. + # Sort only the tables propagation can reach. With part_integrity="cascade", + # propagation also reaches the master of every part it reaches (matched by + # name, as in _propagate_part_to_master), and the master's descendants. + reachable = set(allowed_nodes) if part_integrity == "cascade" and mode == "cascade": - reachable = {start_node} | nx.descendants(group_graph(self), start_node) - else: - reachable = allowed_nodes + pending = list(reachable) + while pending: + master = extract_master(pending.pop()) + if master in self.nodes and master not in reachable: + added = {master} | nx.descendants(self, master) + pending.extend(added - reachable) + reachable |= added sorted_nodes = topo_sort(self, reachable) propagated_edges = set() # Dedup upward part→master walks by (part, master) PAIR, not by master diff --git a/src/datajoint/table.py b/src/datajoint/table.py index bea6c0a8c..d637cfb2b 100644 --- a/src/datajoint/table.py +++ b/src/datajoint/table.py @@ -219,7 +219,7 @@ def _declare_check(self, primary_key, fk_attribute_map, foreign_keys): fk_attribute_map : dict Dict mapping child_attr -> (parent_table, parent_attr). foreign_keys : list - One ``(parent_table, attr_map, nullable)`` entry per foreign key, + One ``(parent_table, attr_map)`` entry per foreign key, where ``attr_map`` maps child_attr -> parent_attr. """ pass # Default: no validation diff --git a/src/datajoint/user_tables.py b/src/datajoint/user_tables.py index 8bc158a0a..dd07f113d 100644 --- a/src/datajoint/user_tables.py +++ b/src/datajoint/user_tables.py @@ -6,7 +6,7 @@ import warnings from .autopopulate import AutoPopulate -from .dependencies import CHAIN_LEGEND, extract_master, group_cycle +from .dependencies import CHAIN_LEGEND, extract_master, group_cycle, is_owned from .errors import DataJointError from .table import Table from .utils import from_camel_case @@ -220,15 +220,16 @@ def _declare_check(self, primary_key, fk_attribute_map, foreign_keys): """ Check the part's references before the part is created. - A part may reference its master, another part of the same master, or a - table outside its master-part group that does not depend on its master. - A reference to a table that does depend on the master would make the group - depend on itself, and DataJoint could no longer order its tables for - delete, drop or trace, so the declaration is refused. + A part is owned by its master when a foreign key in its primary key + references the master, or references another owned part of the same + master (see :func:`~datajoint.dependencies.owned_parts`). An owned part + forms one group with its master. It may reference its master, an owned + part of the same master, or a table outside the group that does not depend + on the master. A reference to a table that does depend on the master would + make the group depend on itself, and DataJoint could no longer order its + tables for delete, drop or trace, so the declaration is refused. - A part that does not inherit its master's primary key, unrenamed and - non-nullable, through a reference to the master or to another part of it, - gets a warning. + A part that is not owned is treated as a separate table and gets a warning. Parameters ---------- @@ -237,7 +238,7 @@ def _declare_check(self, primary_key, fk_attribute_map, foreign_keys): fk_attribute_map : dict Dict mapping child_attr -> (parent_table, parent_attr). foreign_keys : list - One ``(parent_table, attr_map, nullable)`` entry per foreign key. + One ``(parent_table, attr_map)`` entry per foreign key. Raises ------ @@ -247,9 +248,12 @@ def _declare_check(self, primary_key, fk_attribute_map, foreign_keys): super()._declare_check(primary_key, fk_attribute_map, foreign_keys) part = self.full_table_name master = extract_master(part) - parents = [parent for parent, _, _ in foreign_keys] + references = [(parent, set(attr_map) <= set(primary_key)) for parent, attr_map in foreign_keys] + parents = [parent for parent, _ in references] - if any(master not in (parent, extract_master(parent)) for parent in parents): + if all(parent == master for parent in parents): + owned = any(primary for _, primary in references) + else: # Reload: the graph may predate tables declared or dropped since. A # cycle runs through the ancestors of the referenced tables, so their # schemas and everything upstream of them is enough. @@ -257,27 +261,30 @@ def _declare_check(self, primary_key, fk_attribute_map, foreign_keys): dependencies.clear() split = self.connection.adapter.split_full_table_name dependencies.load_all_upstream({self.database} | {split(parent)[0] for parent in parents}) - if cycle := group_cycle(dependencies, part, parents): + owned = is_owned(dependencies, part, references) + if owned and (cycle := group_cycle(dependencies, part, parents)): parent = cycle[-3] # the cycle ends with the referenced table, the part, and the master + not_owned = ( + f" {parent} is a part of {master} that the master does not own: it has no primary-key reference" + " to the master, either directly or through another part that has one, so it counts as a" + " separate table." + if extract_master(parent) == master + else "" + ) raise DataJointError( f"Part table {part} cannot reference {parent}, which depends on the part's master {master}. " f"The reference would close the cycle {' -> '.join(cycle)} ({CHAIN_LEGEND}), and DataJoint " - "could no longer order these tables for delete, drop or trace. A part table may reference its " - "master, another part of the same master, or a table that does not depend on its master." + f"could no longer order these tables for delete, drop or trace.{not_owned} A part table may " + "reference its master, a part that its master owns, or a table that does not depend on its master." ) - master_key = self._master.primary_key - if master_key and not any( - not nullable and all(attr_map.get(attr) == attr for attr in master_key) - for parent, attr_map, nullable in foreign_keys - if master in (parent, extract_master(parent)) - ): + if not owned: warnings.warn( - f"Part table {part} does not inherit the primary key of its master {master} " - f"({', '.join(master_key)}), unrenamed, through a non-nullable reference to the master or to another " - "part of it. Without that key its rows cannot be matched to master rows by key, and operations that " - "keep a master and its parts together may treat it as a separate table. " - "Reference the master with `-> master` unless this is intended.", + f"Part table {part} is not owned by its master {master}: it has no primary-key reference to the " + "master, either directly or through another part that has one. DataJoint sorts it as a separate " + "table, and from DataJoint 2.4 delete and trace will also treat it as a separate table rather than " + "as a part of its master. To make it a part of the master, reference the master in its primary key " + "(`-> master` above the `---` line).", UserWarning, stacklevel=2, ) diff --git a/tests/integration/test_group_cycle.py b/tests/integration/test_group_cycle.py index f08c17827..7efa0dc68 100644 --- a/tests/integration/test_group_cycle.py +++ b/tests/integration/test_group_cycle.py @@ -1,8 +1,10 @@ """ Integration tests for the master-part group cycle rule. -A part may reference its master, another part of the same master, or a table -that does not depend on its master. A reference to a table that depends on the +A part is owned by its master when a foreign key in its primary key references +the master, or another owned part of the same master. An owned part may +reference its master, another owned part of the same master, or a table that +does not depend on its master. A reference to a table that depends on the master would make the master-part group depend on itself, and DataJoint could no longer order the tables for delete, drop or trace, so the declaration is refused before any DDL runs. @@ -62,8 +64,8 @@ def regrow(master, **parts): return type(master.__name__, master.__bases__, namespace) -def master_key_warnings(record): - return [w for w in record if "does not inherit the primary key of its master" in str(w.message)] +def not_owned_warnings(record): + return [w for w in record if "is not owned by its master" in str(w.message)] # --------------------------------------------------------------------------- @@ -257,7 +259,7 @@ class Link(dj.Part): -> Other.Item """ - assert not master_key_warnings(record) + assert not not_owned_warnings(record) assert {Session.Channel.table_name, Session.Spike.table_name, Session.Link.table_name} <= set(schema.list_tables()) Other.insert1({"other_id": 1}) @@ -369,10 +371,104 @@ class UnrelatedChild(dj.Manual): # --------------------------------------------------------------------------- -# A part that does not inherit its master's primary key +# Parts that their masters do not own # --------------------------------------------------------------------------- +def test_owned_part_referencing_a_sibling_that_is_not_owned_is_refused(schema_by_backend): + """ + Loose references the master only outside its primary key, so it is a + separate table that depends on the master. Summary is owned and references + Loose, which would make the master's group depend on itself. + """ + schema = schema_by_backend + + @schema + class Master(dj.Manual): + definition = """ + master_id : int32 + """ + + with pytest.warns(UserWarning, match="is not owned by its master"): + schema( + regrow( + Master, + Loose=""" + loose_id : int32 + --- + -> [nullable] master + """, + ) + ) + + Grown = regrow( + Master, + Loose=""" + loose_id : int32 + --- + -> [nullable] master + """, + Summary=""" + -> master + -> master.Loose + """, + ) + with pytest.raises(DataJointError) as error: + schema(Grown) + master, loose, summary = Master.full_table_name, Grown.Loose.full_table_name, Grown.Summary.full_table_name + message = str(error.value) + assert f"Part table {summary} cannot reference {loose}, which depends on the part's master {master}" in message + assert f"{master} -> {loose} -> {summary} -> {master}" in message + assert f"{loose} is a part of {master} that the master does not own" in message + assert not Grown.Summary().is_declared + assert set(schema.list_tables()) == {Master.table_name, Grown.Loose.table_name} + + +def test_part_with_nullable_master_reference_is_a_separate_table(schema_by_backend): + """ + Annotation references X, which references the master, and references the + master itself only through a nullable foreign key. It is not owned, so it is + sorted as a separate table after X; it is not refused and causes no cycle. + """ + schema = schema_by_backend + + @schema + class Master(dj.Manual): + definition = """ + master_id : int32 + """ + + @schema + class X(dj.Manual): + definition = """ + -> Master + x_id : int32 + """ + + Grown = regrow( + Master, + Annotation=""" + -> X + note_id : int32 + --- + -> [nullable] master.proj(reviewed_in="master_id") + """, + ) + with pytest.warns(UserWarning, match="is not owned by its master") as record: + schema(Grown) + message = str(not_owned_warnings(record)[0].message) + assert Grown.Annotation.full_table_name in message and Master.full_table_name in message + assert "from DataJoint 2.4 delete and trace will also treat it as a separate table" in message + + tables = schema.list_tables() + assert tables.index(Master.table_name) < tables.index(X.table_name) < tables.index(Grown.Annotation.table_name) + Master.insert1({"master_id": 1}) + X.insert1({"master_id": 1, "x_id": 1}) + Grown.Annotation.insert1({"master_id": 1, "x_id": 1, "note_id": 1, "reviewed_in": 1}) + Master.delete() + assert len(X()) == len(Grown.Annotation()) == 0 + + @pytest.mark.parametrize( "definition", [ @@ -380,18 +476,19 @@ class UnrelatedChild(dj.Manual): part_id : int32 """, """ - -> master.proj(other_id="master_id") part_id : int32 + --- + -> [nullable] master """, """ part_id : int32 --- - -> [nullable] master + -> master """, ], - ids=["no-reference", "renamed", "nullable"], + ids=["no-reference", "nullable", "secondary"], ) -def test_part_without_master_key_warns(schema_by_backend, definition): +def test_part_not_owned_warns(schema_by_backend, definition): schema = schema_by_backend @schema @@ -400,16 +497,15 @@ class Master(dj.Manual): master_id : int32 """ - with pytest.warns(UserWarning, match="does not inherit the primary key of its master") as record: + with pytest.warns(UserWarning, match="is not owned by its master") as record: schema(regrow(Master, Part=definition)) - assert Master.full_table_name in str(master_key_warnings(record)[0].message) + assert Master.full_table_name in str(not_owned_warnings(record)[0].message) -def test_part_with_master_key_through_secondary_reference_does_not_warn(schema_by_backend): +def test_part_owned_only_through_a_sibling_that_is_not_owned_warns(schema_by_backend): schema = schema_by_backend - with warnings.catch_warnings(record=True) as record: - warnings.simplefilter("always") + with pytest.warns(UserWarning, match="is not owned by its master") as record: @schema class Master(dj.Manual): @@ -417,11 +513,56 @@ class Master(dj.Manual): master_id : int32 """ - class Part(dj.Part): + class Loose(dj.Part): definition = """ - part_id : int32 + loose_id : int32 --- -> master """ - assert not master_key_warnings(record) + class Child(dj.Part): + definition = """ + -> master.Loose + child_id : int32 + """ + + warned = {str(w.message).split()[2] for w in not_owned_warnings(record)} + assert warned == {Master.Loose.full_table_name, Master.Child.full_table_name} + + +@pytest.mark.parametrize( + "parts", + [ + { + "Renamed": """ + -> master.proj(other_id="master_id") + part_id : int32 + """ + }, + { # the renamed part-of-part case from #1429 + "PartA": """ + -> master + part_a_id : int32 + """, + "PartB": """ + -> Master.PartA.proj(src_master='master_id', src_part='part_a_id') + part_b_id : int32 + """, + }, + ], + ids=["renamed-master", "renamed-sibling"], +) +def test_owned_parts_with_renamed_references_do_not_warn(schema_by_backend, parts): + schema = schema_by_backend + + @schema + class Master(dj.Manual): + definition = """ + master_id : int32 + """ + + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + schema(regrow(Master, **parts)) + + assert not not_owned_warnings(record) diff --git a/tests/unit/test_group_cycle.py b/tests/unit/test_group_cycle.py index e8cc0f29d..b4737403c 100644 --- a/tests/unit/test_group_cycle.py +++ b/tests/unit/test_group_cycle.py @@ -1,10 +1,13 @@ """ Unit tests for the master-part group cycle rule, on hand-built dependency graphs. -A master and its parts form one group. ``topo_sort`` orders a master after -every table its parts reference, so it fails when the groups depend on each -other in a cycle. ``group_cycle`` finds the cycle a new part's references would -close, and the declaration of such a part is refused. +A part is owned by its master when a foreign key in its primary key references +the master, or another owned part of the same master. An owned part and its +master form one group; every other part is a separate table. ``topo_sort`` +orders a master after every table its owned parts reference, so it fails +exactly when the groups depend on each other in a cycle. ``group_cycle`` finds +the cycle a new owned part's references would close, and the declaration of +such a part is refused. """ import random @@ -12,7 +15,7 @@ import networkx as nx import pytest -from datajoint.dependencies import group_ancestors, group_cycle, topo_sort +from datajoint.dependencies import extract_master, group_ancestors, group_cycle, is_owned, owned_parts, topo_sort from datajoint.errors import DataJointError @@ -22,12 +25,66 @@ def t(name): def graph(*edges): - """Dependency graph from (parent, child) pairs of short table names.""" + """ + Dependency graph from (parent, child) pairs of short table names. + + A third element False marks a reference that is not in the child's primary key. + """ g = nx.MultiDiGraph() - g.add_edges_from((t(parent), t(child)) for parent, child in edges) + for parent, child, *primary in edges: + g.add_edge(t(parent), t(child), primary=primary[0] if primary else True) return g +def sorts(g): + try: + topo_sort(g) + except DataJointError: + return False + return True + + +# --------------------------------------------------------------------------- +# owned_parts +# --------------------------------------------------------------------------- + + +def test_owned_parts(): + g = graph( + ("m", "m__direct"), # references the master in its primary key + ("m__direct", "m__chained"), # through an owned part (a rename is still one edge) + ("m", "m__secondary", False), # only a secondary or nullable reference + ("m__secondary", "m__via_plain"), # only through a part that is not owned + ("x", "m__none"), # no reference to the master + ("m", "m__parallel", False), # parallel references: one of them primary + ("m", "m__parallel"), + ("y", "n__orphan"), # its master is not in the graph + ) + assert owned_parts(g) == {t("m__direct"), t("m__chained"), t("m__parallel")} + + +def test_owned_parts_edge_without_flag_counts_as_primary(): + g = nx.MultiDiGraph([(t("m"), t("m__p"))]) + assert owned_parts(g) == {t("m__p")} + + +def test_owned_parts_does_not_cross_into_another_master(): + # n__q references m__p in its primary key, but m__p belongs to another master. + g = graph(("m", "m__p"), ("m__p", "n__q")) + g.add_node(t("n")) + assert owned_parts(g) == {t("m__p")} + + +def test_is_owned(): + g = graph(("m", "m__p"), ("m", "m__s", False)) + assert is_owned(g, t("m__new"), [(t("m"), True)]) + assert is_owned(g, t("m__new"), [(t("m__p"), True)]) + assert not is_owned(g, t("m__new"), [(t("m"), False)]) + assert not is_owned(g, t("m__new"), [(t("m__s"), True)]) + assert not is_owned(g, t("m__new"), []) + assert not is_owned(g, t("z__new"), [(t("m"), True)]) + + # --------------------------------------------------------------------------- # group_ancestors # --------------------------------------------------------------------------- @@ -39,23 +96,22 @@ def test_group_ancestors_plain_chain(): assert group_ancestors(g, t("a")) == {t("a")} -def test_group_ancestors_master_brings_its_parts_and_their_parents(): - # x is referenced by a part of m, so everything downstream of m depends on x. +def test_group_ancestors_master_brings_its_owned_parts_and_their_parents(): + # x is referenced by an owned part of m, so everything downstream of m depends on x. g = graph(("m", "m__p"), ("x", "m__p"), ("m", "y")) assert group_ancestors(g, t("y")) == {t("y"), t("m"), t("m__p"), t("x")} -def test_group_ancestors_part_reaches_master_through_its_reference(): +def test_group_ancestors_owned_part_reaches_master_through_its_reference(): g = graph(("m", "m__p"), ("x", "m__q"), ("m", "m__q")) - # m__p references m, and m brings its other part m__q and its parent x. assert group_ancestors(g, t("m__p")) == {t("m__p"), t("m"), t("m__q"), t("x")} -def test_group_ancestors_part_without_master_reference_is_plain(): - # m__p does not reference m: reaching it does not reach its master. - g = graph(("x", "m__p"), ("y", "m")) - g.add_node(t("m")) - assert group_ancestors(g, t("m__p")) == {t("m__p"), t("x")} +def test_group_ancestors_part_that_is_not_owned_is_plain(): + # m__p references m only outside its primary key: m does not bring it in. + g = graph(("m", "m__p", False), ("x", "m__p"), ("m", "y")) + assert group_ancestors(g, t("y")) == {t("y"), t("m")} + assert group_ancestors(g, t("m__p")) == {t("m__p"), t("m"), t("x")} def test_group_ancestors_across_two_groups(): @@ -82,23 +138,34 @@ def test_group_cycle_two_groups(): def test_group_cycle_through_a_master_downstream(): - # m -> y -> n__q, a part of n; n -> z. A part of m referencing z closes a cycle. + # m -> y -> n__q, an owned part of n; n -> z. A part of m referencing z closes a cycle. g = graph(("m", "y"), ("y", "n__q"), ("n", "n__q"), ("n", "z")) assert group_cycle(g, t("m__p"), [t("m"), t("z")]) == [t("m"), t("y"), t("n__q"), t("n"), t("z"), t("m__p"), t("m")] +def test_group_cycle_not_through_a_part_that_is_not_owned(): + # Same shape, but n__q references n outside its primary key: it is a separate + # table, n does not depend on y, and the reference to z closes no cycle. + g = graph(("m", "y"), ("y", "n__q"), ("n", "n__q", False), ("n", "z")) + assert group_cycle(g, t("m__p"), [t("m"), t("z")]) is None + g.add_edge(t("z"), t("m__p")) + g.add_edge(t("m"), t("m__p")) + assert sorts(g) + + @pytest.mark.parametrize( "parents", [ ["m"], # its master - ["m", "m__s"], # a sibling part + ["m", "m__s"], # an owned sibling part ["m", "u"], # an unrelated upstream table ["m", "y"], # a table downstream of another group - ["m__s"], # a sibling alone + ["m__s"], # an owned sibling alone + ["m", "m__loose"], # a sibling that is not owned, but does not depend on m ], ) def test_group_cycle_allowed(parents): - g = graph(("m", "m__s"), ("u", "m"), ("n", "n__q"), ("n", "y"), ("u", "y")) + g = graph(("m", "m__s"), ("u", "m"), ("n", "n__q"), ("n", "y"), ("u", "y"), ("u", "m__loose")) assert group_cycle(g, t("m__p"), [t(p) for p in parents]) is None @@ -112,6 +179,43 @@ def test_group_cycle_master_not_in_graph(): assert group_cycle(g, t("m__p"), [t("b")]) is None +# --------------------------------------------------------------------------- +# The two counterexamples to grouping parts by name +# --------------------------------------------------------------------------- + + +def test_part_without_master_reference_is_sorted_as_a_plain_table(): + """ + m; x references m; part m__p references x but not m. + + Grouped by name, m__p would make m come after x (a cycle), and keeping m__p + next to m would put it before x. m__p is not owned, so it is a plain table. + """ + g = graph(("m", "x"), ("x", "m__p")) + assert topo_sort(g) == [t("m"), t("x"), t("m__p")] + assert not is_owned(graph(("m", "x")), t("m__p"), [(t("x"), True)]) + + +def test_owned_part_referencing_a_sibling_that_is_not_owned_closes_a_cycle(): + """ + m; m__p1 references m only through a nullable foreign key (not owned); + m__p2 is owned and references m and m__p1. + + m__p1 is a separate table that depends on m, so m__p2's reference to it + makes m's group depend on itself. + """ + declared = graph(("m", "m__p1", False)) + references = [(t("m"), True), (t("m__p1"), True)] + assert is_owned(declared, t("m__p2"), references) + assert group_cycle(declared, t("m__p2"), [t("m"), t("m__p1")]) == [t("m"), t("m__p1"), t("m__p2"), t("m")] + + legacy = graph(("m", "m__p1", False), ("m", "m__p2"), ("m__p1", "m__p2")) + with pytest.raises(DataJointError) as error: + topo_sort(legacy) + assert f"{t('m')} -> {t('m__p1')} -> {t('m__p2')} -> {t('m')}" in str(error.value) + assert f"part table {t('m__p2')} references a table that depends on its own master" in str(error.value) + + # --------------------------------------------------------------------------- # topo_sort # --------------------------------------------------------------------------- @@ -157,75 +261,142 @@ def test_topo_sort_nodes_keep_order_through_tables_left_out(): assert topo_sort(g, []) == [] -def test_topo_sort_nodes_keep_parts_after_masters(): +def test_topo_sort_nodes_keep_owned_parts_after_masters(): g = graph(("a", "m"), ("a", "z"), ("m", "m__part"), ("z", "m__part")) assert topo_sort(g, [t("m"), t("m__part"), t("z")]) == [t("z"), t("m"), t("m__part")] +def test_topo_sort_nodes_judge_ownership_on_the_whole_graph(): + # m__q is owned through m__p; sorting m and m__q alone still keeps m__q with m. + g = graph(("a", "m"), ("m", "m__p"), ("m__p", "m__q"), ("z", "m__q"), ("a", "z")) + assert topo_sort(g, [t("m"), t("m__q"), t("z")]) == [t("z"), t("m"), t("m__q")] + + # --------------------------------------------------------------------------- -# Exactness: refusing what group_cycle finds keeps topo_sort working, and -# refuses nothing topo_sort could sort. +# Randomized checks # --------------------------------------------------------------------------- -def sorts(g): - try: - topo_sort(g) - except DataJointError: - return False - return True +def random_tables(rng, count): + """ + Yield random table declarations in order: (name, kind, references). + + ``kind`` is "plain", "master" or "part"; ``references`` is a list of + (parent, primary) pairs to tables declared earlier. A part references its + master (if declared) in most cases, in its primary key or not. Some parts are + declared before their master, as one left behind by a dropped master would be. + """ + declared, masters, orphans = [], [], [] + for i in range(count): + kind = rng.choice(["plain", "master", "part", "part", "part"]) + master = None + if kind == "part" and masters and rng.random() < 0.9: + master = rng.choice(masters) + elif kind == "part": + master = t(f"m{i}") # its master is not declared yet + orphans.append(master) + if kind == "part": + table = t(f"{master[5:-1]}__p{i}") + elif kind == "master" and orphans and rng.random() < 0.5: + table = orphans.pop() + else: + table = t(f"{'m' if kind == 'master' else 't'}{i}") + parents = rng.sample(declared, k=min(len(declared), rng.randint(0, 3))) + if master in declared and rng.random() < 0.8: + parents.append(master) + references = [(parent, rng.random() < 0.7) for parent in parents] + if kind == "master": + masters.append(table) + declared.append(table) + yield table, kind, references + + +def quotient_is_acyclic(g): + """ + Reference check, independent of ``owned_parts``: contract each master with + its owned parts and test the result for cycles. + """ + owned, changed = set(), True + while changed: # fixed point over all edges + changed = False + for parent, child, primary in g.edges(data="primary", default=True): + master = extract_master(child) + if ( + child not in owned + and master in g + and primary + and (parent == master or parent in owned and extract_master(parent) == master) + ): + owned.add(child) + changed = True + group = {node: extract_master(node) if node in owned else node for node in g.nodes} + quotient = nx.DiGraph() + quotient.add_nodes_from(set(group.values())) + quotient.add_edges_from((group[u], group[v]) for u, v in g.edges() if group[u] != group[v]) + return nx.is_directed_acyclic_graph(quotient), owned + + +def test_topo_sort_succeeds_iff_the_group_quotient_is_acyclic(): + """ + On random dependency graphs, topo_sort succeeds exactly when the graph with + each master and its owned parts contracted to one node has no cycle. When it + succeeds, every table follows the tables it references, and each master is + followed directly by its owned parts. + """ + outcomes = {True: 0, False: 0} + for seed in range(400): + rng = random.Random(seed) + g = nx.MultiDiGraph() + for table, _, references in random_tables(rng, 12): + g.add_node(table) + for parent, primary in references: + g.add_edge(parent, table, primary=primary) + acyclic, owned = quotient_is_acyclic(g) + assert sorts(g) == acyclic, seed + outcomes[acyclic] += 1 + if acyclic: + order = topo_sort(g) + position = {node: i for i, node in enumerate(order)} + assert sorted(order) == sorted(g.nodes) + assert all(position[u] < position[v] for u, v in g.edges()), seed + for part in owned: + master = extract_master(part) + between = order[position[master] + 1 : position[part]] + assert all(node in owned and extract_master(node) == master for node in between), seed + assert all(outcomes.values()), outcomes def test_group_cycle_is_exact(): """ Declare random tables in order, each referencing random earlier tables. - A new part is refused exactly when adding it would make topo_sort fail. No - other declaration ever makes topo_sort fail: not a plain table, not a - master, and not a master declared after a part of it that does not - reference it (as a part left behind by a dropped master would be). + A new owned part is refused exactly when adding it would make topo_sort fail. + No other declaration ever makes topo_sort fail: not a plain table, not a + master, not a part that is not owned, and not a master declared after a part + of its own. """ - counts = {"refused": 0, "accepted": 0, "late masters": 0} + counts = {"refused": 0, "owned": 0, "not owned": 0} for seed in range(300): rng = random.Random(seed) g = nx.MultiDiGraph() - masters, orphans = [], [] - for i in range(14): - existing = list(g) - kind = rng.choice(["plain", "master", "part", "part", "part"]) - if kind == "part" and masters and rng.random() < 0.9: - master = rng.choice(masters) - elif kind == "part": - master = t(f"m{i}") # its master is not declared yet - orphans.append(master) - if kind == "part": - table = t(f"{master[5:-1]}__p{i}") - elif kind == "master" and orphans and rng.random() < 0.5: - table = orphans.pop() - counts["late masters"] += 1 - else: - table = t(f"{'m' if kind == 'master' else 't'}{i}") - if table in g: - continue - parents = rng.sample(existing, k=min(len(existing), rng.randint(0, 3))) - if kind == "part" and master in g and rng.random() < 0.8: - parents.append(master) # `-> master`, usually - + for table, kind, references in random_tables(rng, 14): + if table in g or any(parent not in g for parent, _ in references): + continue # skip a table whose parent was refused candidate = nx.MultiDiGraph(g) candidate.add_node(table) - candidate.add_edges_from((parent, table) for parent in parents) + for parent, primary in references: + candidate.add_edge(parent, table, primary=primary) - if kind == "part": - cycle = group_cycle(g, table, parents) - assert (cycle is None) == sorts(candidate), (seed, table, parents, cycle) + if kind == "part" and is_owned(g, table, references): + cycle = group_cycle(g, table, [parent for parent, _ in references]) + assert (cycle is None) == sorts(candidate), (seed, table, references, cycle) if cycle is not None: counts["refused"] += 1 continue - counts["accepted"] += 1 + counts["owned"] += 1 else: - assert sorts(candidate), (seed, table, parents) - if kind == "master": - masters.append(table) + assert sorts(candidate), (seed, table, references) + counts["not owned"] += kind == "part" g = candidate assert sorts(g) # the random declarations exercise every case