build-parallel: Add exclusive_build option for per-package exclusive builds

- Added exclusive_build (bool) to BuildOptions; defaults to False.
- When set to True, the package is built alone, not in parallel with others.
- Updated build_parallel logic and all relevant docstrings and config examples.
- Replaced the previous parallel_builds enum with exclusive_build.
- Updated the end-to-end test to cover exclusive build behavior.

Author: dhellmann

e2e/build-parallel/cython.yaml

@@ -0,0 +1,2 @@
+build_options:
+  exclusive_build: True

e2e/build-parallel-graph.json e2e/build-parallel/graph.jsone2e/build-parallel-graph.json renamed to e2e/build-parallel/graph.json

@@ -15,34 +15,42 @@ VERSION="1.14.0"
 #     --sdists-repo="$OUTDIR/sdists-repo" \
 #     --wheels-repo="$OUTDIR/wheels-repo" \
 #     --work-dir="$OUTDIR/work-dir" \
-#     --settings-dir="$SCRIPTDIR/changelog_settings" \
+#     --settings-dir="$SCRIPTDIR/build-parallel" \
 #     bootstrap "${DIST}==${VERSION}"
 
 # Save the build order file but remove everything else.
 # cp "$OUTDIR/work-dir/graph.json" "$OUTDIR/"
 
 # Copy the cached graph file to the working directory
-cp "$SCRIPTDIR/build-parallel-graph.json" "$OUTDIR/graph.json"
+cp "$SCRIPTDIR/build-parallel/graph.json" "$OUTDIR/graph.json"
 
 # Build everything a first time
 log="$OUTDIR/build-logs/${DIST}-build.log"
 fromager \
-    --debug \
     --log-file "$log" \
     --work-dir "$OUTDIR/work-dir" \
     --sdists-repo "$OUTDIR/sdists-repo" \
     --wheels-repo "$OUTDIR/wheels-repo" \
-    --settings-dir="$SCRIPTDIR/changelog_settings" \
+    --settings-dir="$SCRIPTDIR/build-parallel" \
     build-parallel "$OUTDIR/graph.json"
 
+if ! grep -q "ready to build cython" "$log"; then
+  echo "Did not find message indicating build of cython would start" 1>&2
+  pass=false
+fi
+if ! grep -q "cython: requires exclusive build" "$log"; then
+  echo "Did not find message indicating build of cython would run on its own" 1>&2
+  pass=false
+fi
+
 # Rebuild everything even if it already exists
 log="$OUTDIR/build-logs/${DIST}-build.log"
 fromager \
     --log-file "$log" \
     --work-dir "$OUTDIR/work-dir" \
     --sdists-repo "$OUTDIR/sdists-repo" \
     --wheels-repo "$OUTDIR/wheels-repo" \
-    --settings-dir="$SCRIPTDIR/changelog_settings" \
+    --settings-dir="$SCRIPTDIR/build-parallel" \
     build-parallel --force "$OUTDIR/graph.json"
 
 find "$OUTDIR/wheels-repo/"
@@ -84,7 +92,7 @@ fromager \
     --work-dir "$OUTDIR/work-dir" \
     --sdists-repo "$OUTDIR/sdists-repo" \
     --wheels-repo "$OUTDIR/wheels-repo" \
-    --settings-dir="$SCRIPTDIR/changelog_settings" \
+    --settings-dir="$SCRIPTDIR/build-parallel" \
     build-parallel "$OUTDIR/graph.json"
 
 find "$OUTDIR/wheels-repo/"

e2e/test_build_parallel.sh

@@ -323,6 +323,7 @@ def _build_sdist(
         sdist_root_dir: pathlib.Path,
         build_env: build_environment.BuildEnvironment,
     ) -> pathlib.Path:
+        sdist_filename: pathlib.Path | None = None
         try:
             find_sdist_result = finders.find_sdist(
                 self.ctx, self.ctx.sdists_builds, req, str(resolved_version)
@@ -342,6 +343,12 @@ def _build_sdist(
                 )
         except Exception as err:
             logger.warning(f"failed to build source distribution: {err}")
+            # Re-raise the exception since we cannot continue without a sdist
+            raise
+
+        if sdist_filename is None:
+            raise RuntimeError(f"Failed to build or find sdist for {req}")
+
         return sdist_filename
 
     def _build_wheel(

src/fromager/bootstrapper.py

@@ -594,6 +594,21 @@ def build_parallel(
                 ", ".join(n.canonicalized_name for n in buildable_nodes),
             )
 
+            # Check if any buildable node requires exclusive build (exclusive_build == True)
+            exclusive_nodes = [
+                node
+                for node in buildable_nodes
+                if wkctx.settings.package_build_info(
+                    node.canonicalized_name
+                ).exclusive_build
+            ]
+            if exclusive_nodes:
+                # Only build the first exclusive node this round
+                buildable_nodes = [exclusive_nodes[0]]
+                logger.info(
+                    f"{exclusive_nodes[0].canonicalized_name}: requires exclusive build, running it alone this round."
+                )
+
             # Build up to max_workers nodes concurrently (or all if max_workers is None)
             with concurrent.futures.ThreadPoolExecutor(
                 max_workers=max_workers

src/fromager/commands/build.py

@@ -197,6 +197,9 @@ class BuildOptions(pydantic.BaseModel):
     0.5: assume each parallel job requires 512 MB virtual memory
     """
 
+    exclusive_build: bool = False
+    """If true, this package must be built on its own (not in parallel with other packages). Default: False."""
+
 
 class ProjectOverride(pydantic.BaseModel):
     """Override pyproject.toml settings
@@ -317,6 +320,11 @@ class PackageSettings(pydantic.BaseModel):
             sdist_server_url: https://sdist.test/egg
             include_sdists: true
             include_wheels: false
+        build_options:
+            build_ext_parallel: False
+            cpu_cores_per_job: 1
+            memory_per_job_gb: 1.0
+            exclusive_build: False
         variants:
             cpu:
                 env:
@@ -809,6 +817,10 @@ def git_options(self) -> GitOptions:
     def project_override(self) -> ProjectOverride:
         return self._ps.project_override
 
+    @property
+    def exclusive_build(self) -> bool:
+        return self._ps.build_options.exclusive_build
+
     def serialize(self, **kwargs) -> dict[str, typing.Any]:
         return self._ps.serialize(**kwargs)
 
@@ -847,6 +859,9 @@ def from_string(
         # ignore legacy settings
         parsed.pop("pre_built", None)
         parsed.pop("packages", None)
+        # Ensure changelog is correct type
+        if "changelog" in parsed and not isinstance(parsed["changelog"], dict):
+            parsed["changelog"] = {}
         try:
             return cls(**parsed)
         except Exception as err:

src/fromager/packagesettings.py

@@ -31,6 +31,7 @@
         "build_ext_parallel": True,
         "cpu_cores_per_job": 4,
         "memory_per_job_gb": 4.0,
+        "exclusive_build": False,
     },
     "changelog": {
         Version("1.0.1"): ["fixed bug"],
@@ -95,6 +96,7 @@
         "build_ext_parallel": False,
         "cpu_cores_per_job": 1,
         "memory_per_job_gb": 1.0,
+        "exclusive_build": False,
     },
     "changelog": {},
     "config_settings": {},
@@ -595,3 +597,36 @@ def test_package_build_info_git_options(testdata_context: context.WorkContext):
     custom_settings = PackageSettings.from_string("custom-pkg", settings_yaml)
     assert custom_settings.git_options.submodules is True
     assert custom_settings.git_options.submodule_paths == ["vendor/lib"]
+
+
+def test_package_build_info_exclusive_build(testdata_context: context.WorkContext):
+    """Test that PackageBuildInfo correctly exposes exclusive_build from build_options."""
+    # Test default package (should have exclusive_build=False by default)
+    req = Requirement("test-empty-pkg==1.0.0")
+    pbi = testdata_context.package_build_info(req)
+    assert pbi.exclusive_build is False
+
+    # Test creating a package settings with exclusive_build=True
+    settings_yaml = """
+build_options:
+  exclusive_build: true
+"""
+    custom_settings = PackageSettings.from_string("exclusive-pkg", settings_yaml)
+    assert custom_settings.build_options.exclusive_build is True
+
+    # Test PackageBuildInfo properly accesses it through build_options
+    import pathlib
+
+    from fromager.packagesettings import Settings, SettingsFile
+
+    # Create a temporary Settings object to test with
+    settings = Settings(
+        settings=SettingsFile(),
+        package_settings=[custom_settings],
+        variant="cpu",
+        patches_dir=pathlib.Path("/tmp"),
+        max_jobs=1,
+    )
+
+    custom_pbi = settings.package_build_info("exclusive-pkg")
+    assert custom_pbi.exclusive_build is True