Merge pull request #855 from tiran/integrate-tracking-topo

feat: implement DependencyGraph.get_build_topology

Author: mergify[bot]

src/fromager/commands/graph.py

@@ -565,3 +565,43 @@ def migrate_graph(
             graph.serialize(f)
     else:
         graph.serialize(sys.stdout)
+
+
+@graph.command()
+@click.argument(
+    "graph-file",
+    type=clickext.ClickPath(),
+)
+@click.pass_obj
+def build_graph(
+    wkctx: context.WorkContext,
+    graph_file: pathlib.Path,
+):
+    """Print build graph steps for parallel-build
+
+    The build-graph command takes a graph.json file and analyzes in which
+    order parallel build is going to build the wheels. It also shows which
+    wheels are recognized as build dependencies or exclusive builds.
+    """
+    graph = DependencyGraph.from_file(graph_file)
+    topo = graph.get_build_topology(context=wkctx)
+    topo.prepare()
+
+    def n2s(nodes: typing.Iterable[DependencyNode]) -> str:
+        return ", ".join(sorted(node.key for node in nodes))
+
+    print(f"Build dependencies ({len(topo.dependency_nodes)}):")
+    print(n2s(topo.dependency_nodes), "\n")
+    if topo.exclusive_nodes:
+        print(f"Exclusive builds ({len(topo.exclusive_nodes)}):")
+        print(n2s(topo.exclusive_nodes), "\n")
+
+    print("Build rounds:")
+    rounds: int = 0
+    while topo.is_active():
+        rounds += 1
+        nodes_to_build = topo.get_available()
+        print(f"{rounds}.", n2s(nodes_to_build))
+        topo.done(*nodes_to_build)
+
+    print(f"\nBuilding {len(graph)} packages in {rounds} rounds.")

src/fromager/dependency_graph.py

@@ -15,6 +15,9 @@
 from .read import open_file_or_url
 from .requirements_file import RequirementType
 
+if typing.TYPE_CHECKING:
+    from .context import WorkContext
+
 logger = logging.getLogger(__name__)
 
 ROOT = ""
@@ -67,6 +70,12 @@ def __post_init__(self) -> None:
                 self, "key", f"{self.canonicalized_name}=={self.version}"
             )
 
+    @property
+    def requirement(self) -> Requirement:
+        if self.canonicalized_name == ROOT:
+            raise RuntimeError("root node is not a requirement")
+        return Requirement(self.key)
+
     def add_child(
         self,
         child: DependencyNode,
@@ -249,6 +258,10 @@ def clear(self) -> None:
         self.nodes.clear()
         self.nodes[ROOT] = DependencyNode.construct_root_node()
 
+    def __len__(self) -> int:
+        # exclude ROOT
+        return len(self.nodes) - 1
+
     def _to_dict(self):
         raw_graph = {}
         stack = [self.nodes[ROOT]]
@@ -382,6 +395,22 @@ def _depth_first_traversal(
                 edge.destination_node.children, visited, match_dep_types
             )
 
+    def get_build_topology(self, context: WorkContext) -> TrackingTopologicalSorter:
+        """Create build topology graph
+
+        The build topology contains all nodes of the graph (except ROOT).
+        Each node depends on its build dependencies, but not on its
+        installation dependencies.
+        """
+        topo = TrackingTopologicalSorter()
+        for node in self.get_all_nodes():
+            if node.key == ROOT:
+                continue
+            pbi = context.package_build_info(node.canonicalized_name)
+            build_req = tuple(node.iter_build_requirements())
+            topo.add(node, *build_req, exclusive=pbi.exclusive_build)
+        return topo
+
 
 class TrackingTopologicalSorter:
     """A thread-safe topological sorter that tracks nodes in progress

tests/conftest.py

@@ -7,13 +7,19 @@
 from fromager import context, packagesettings
 
 TESTDATA_PATH = pathlib.Path(__file__).parent.absolute() / "testdata"
+E2E_PATH = pathlib.Path(__file__).parent.parent.absolute() / "e2e"
 
 
 @pytest.fixture
 def testdata_path() -> typing.Generator[pathlib.Path, None, None]:
     yield TESTDATA_PATH
 
 
+@pytest.fixture
+def e2e_path() -> typing.Generator[pathlib.Path, None, None]:
+    yield E2E_PATH
+
+
 @pytest.fixture
 def tmp_context(tmp_path: pathlib.Path) -> context.WorkContext:
     patches_dir = tmp_path / "overrides/patches"

tests/test_commands_graph.py

@@ -0,0 +1,13 @@
+import pathlib
+
+from click.testing import CliRunner
+
+from fromager.__main__ import main as fromager
+
+
+def test_fromager_version(cli_runner: CliRunner, e2e_path: pathlib.Path) -> None:
+    graph_json = e2e_path / "build-parallel" / "graph.json"
+    result = cli_runner.invoke(fromager, ["graph", "build-graph", str(graph_json)])
+    assert result.exit_code == 0
+    assert "1. flit-core==3.12.0, setuptools==80.8.0" in result.stdout
+    assert "Building 16 packages in 4 rounds" in result.stdout

tests/test_dependency_graph.py

@@ -1,13 +1,19 @@
 import dataclasses
 import graphlib
+import pathlib
 import typing
 
 import pytest
 from packaging.requirements import Requirement
 from packaging.utils import canonicalize_name
 from packaging.version import Version
 
-from fromager.dependency_graph import DependencyNode, TrackingTopologicalSorter
+from fromager import context
+from fromager.dependency_graph import (
+    DependencyGraph,
+    DependencyNode,
+    TrackingTopologicalSorter,
+)
 from fromager.requirements_file import RequirementType
 
 
@@ -54,6 +60,8 @@ def test_dependencynode_dataclass() -> None:
         repr(a)
         == "DependencyNode(canonicalized_name='a', version=<Version('1.0')>, download_url='', pre_built=False, constraint=None)"
     )
+    assert a.requirement == Requirement("a==1.0")
+
     with pytest.raises(dataclasses.FrozenInstanceError):
         a.version = Version("2.0")  # type: ignore[misc]
     with pytest.raises((TypeError, AttributeError)):
@@ -63,6 +71,8 @@ def test_dependencynode_dataclass() -> None:
     assert root.canonicalized_name == ""
     assert root.version == Version("0.0")
     assert root.key == ""
+    with pytest.raises(RuntimeError):
+        assert root.requirement
 
 
 def test_iter_requirements() -> None:
@@ -295,3 +305,89 @@ def test_tracking_topology_sorter_not_active_error() -> None:
     with pytest.raises(ValueError) as excinfo:
         topo.get_available()
     assert "topology is not active" in str(excinfo.value)
+
+
+def node2str(nodes: set[DependencyNode]) -> set[str]:
+    return {node.key for node in nodes}
+
+
+def test_e2e_parallel_graph(
+    tmp_context: context.WorkContext, e2e_path: pathlib.Path
+) -> None:
+    graph = DependencyGraph.from_file(e2e_path / "build-parallel" / "graph.json")
+    assert len(graph) == 16
+
+    topo = graph.get_build_topology(tmp_context)
+    assert node2str(topo.dependency_nodes) == {
+        "cython==3.1.1",
+        "flit-core==3.12.0",
+        "packaging==25.0",
+        "setuptools-scm==8.3.1",
+        "setuptools==80.8.0",
+        "wheel==0.46.1",
+    }
+
+    steps = [node2str(batch) for batch in topo.static_batches()]
+    assert steps == [
+        {
+            "flit-core==3.12.0",
+            "setuptools==80.8.0",
+        },
+        {
+            "cython==3.1.1",
+            "imapclient==3.0.1",
+            "jinja2==3.1.6",
+            "markupsafe==3.0.2",
+            "more-itertools==10.7.0",
+            "packaging==25.0",
+        },
+        {
+            "setuptools-scm==8.3.1",
+            "wheel==0.46.1",
+        },
+        {
+            "imapautofiler==1.14.0",
+            "jaraco-classes==3.4.0",
+            "jaraco-context==6.0.1",
+            "jaraco-functools==4.1.0",
+            "keyring==25.6.0",
+            "pyyaml==6.0.2",
+        },
+    ]
+
+    # same graph, but mark cython as exclusive
+    topo = graph.get_build_topology(tmp_context)
+    node = graph.nodes["cython==3.1.1"]
+    topo.add(node, exclusive=True)
+
+    steps = [node2str(batch) for batch in topo.static_batches()]
+    assert steps == [
+        {
+            "flit-core==3.12.0",
+            "setuptools==80.8.0",
+        },
+        {
+            "imapclient==3.0.1",
+            "jinja2==3.1.6",
+            "markupsafe==3.0.2",
+            "more-itertools==10.7.0",
+            "packaging==25.0",
+        },
+        {
+            "setuptools-scm==8.3.1",
+            "wheel==0.46.1",
+        },
+        {
+            "imapautofiler==1.14.0",
+            "jaraco-classes==3.4.0",
+            "jaraco-context==6.0.1",
+            "jaraco-functools==4.1.0",
+            "keyring==25.6.0",
+        },
+        {
+            "cython==3.1.1",
+        },
+        {
+            "pyyaml==6.0.2",
+        },
+    ]