concurrency updates for wheel output handling

Make the build output directory optionally thread-specific and move
the handling for updating the simple server index into the build
threads. That required moving similar code for build sequence for the
pre-built downloaded wheel use case, which in turn required updating
some expected output in test scripts.

Signed-off-by: Doug Hellmann <dhellmann@redhat.com>

Author: dhellmann

e2e/test_build.sh

@@ -46,7 +46,7 @@ fromager \
     build "${DIST}" "${VERSION}" "https://pypi.org/simple"
 
 EXPECTED_FILES="
-wheels-repo/build/stevedore-5.2.0-0-py3-none-any.whl
+wheels-repo/downloads/stevedore-5.2.0-0-py3-none-any.whl
 sdists-repo/downloads/stevedore-5.2.0.tar.gz
 sdists-repo/builds/stevedore-5.2.0.tar.gz
 sdists-repo/builds/test-output-file.txt

e2e/test_build_settings.sh

@@ -39,7 +39,7 @@ fromager \
     build "${DIST}" "${VERSION}" "unreachable_url"
 
 EXPECTED_FILES="
-wheels-repo/build/stevedore-5.2.0-0-py3-none-any.whl
+wheels-repo/downloads/stevedore-5.2.0-0-py3-none-any.whl
 sdists-repo/downloads/stevedore-custom-5.2.0.tar.gz
 sdists-repo/builds/stevedore-5.2.0.tar.gz
 sdists-repo/builds/test-output-file.txt

src/fromager/commands/build.py

@@ -224,6 +224,11 @@ def build_sequence(
                     dist_version=str(resolved_version),
                     wheel_filename=wheel_filename,
                 )
+                server.update_wheel_mirror(wkctx)
+                # After we update the wheel mirror, the built file has
+                # moved to a new directory.
+                wheel_filename = wkctx.wheels_downloads / wheel_filename.name
+
             else:
                 logger.info(
                     "%s: building %s==%s", dist_name, dist_name, resolved_version
@@ -232,10 +237,6 @@ def build_sequence(
                     wkctx, resolved_version, req, source_download_url
                 )
 
-            server.update_wheel_mirror(wkctx)
-            # After we update the wheel mirror, the built file has
-            # moved to a new directory.
-            wheel_filename = wkctx.wheels_downloads / wheel_filename.name
             entries.append(
                 BuildSequenceEntry(
                     dist_name,
@@ -422,6 +423,12 @@ def _build(
         wheel_filename=wheel_filename,
     )
 
+    server.update_wheel_mirror(wkctx)
+
+    # After we update the wheel mirror, the built file has
+    # moved to a new directory.
+    wheel_filename = wkctx.wheels_downloads / wheel_filename.name
+
     return wheel_filename
 
 
@@ -518,6 +525,8 @@ def build_parallel(
     parallel. Use --jobs to limit the number of concurrent builds.
 
     """
+    wkctx.enable_parallel_builds()
+
     server.start_wheel_server(wkctx)
     wheel_server_urls = [wkctx.wheel_server_url]
     if cache_wheel_server_url:
@@ -604,10 +613,6 @@ def build_parallel(
                 for node, future in zip(buildable_nodes, futures, strict=True):
                     try:
                         wheel_filename = future.result()
-                        server.update_wheel_mirror(wkctx)
-                        # After we update the wheel mirror, the built file has
-                        # moved to a new directory.
-                        wheel_filename = wkctx.wheels_downloads / wheel_filename.name
                         entries.append(
                             BuildSequenceEntry(
                                 node.canonicalized_name,

src/fromager/context.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import pathlib
+import threading
 from urllib.parse import urlparse
 
 from packaging.requirements import Requirement
@@ -52,7 +53,7 @@ def __init__(
         self.sdists_downloads = self.sdists_repo / "downloads"
         self.sdists_builds = self.sdists_repo / "builds"
         self.wheels_repo = pathlib.Path(wheels_repo).absolute()
-        self.wheels_build = self.wheels_repo / "build"
+        self.wheels_build_base = self.wheels_repo / "build"
         self.wheels_downloads = self.wheels_repo / "downloads"
         self.wheels_prebuilt = self.wheels_repo / "prebuilt"
         self.wheel_server_dir = self.wheels_repo / "simple"
@@ -74,6 +75,25 @@ def __init__(
         )
         self.time_description_store: dict[str, str] = {}
 
+        self._parallel_builds = False
+
+    def enable_parallel_builds(self) -> None:
+        self._parallel_builds = True
+
+    def disable_parallel_builds(self) -> None:
+        self._parallel_builds = False
+
+    @property
+    def wheels_build(self) -> pathlib.Path:
+        # when parallel builds are enabled, return a path that is unique for the
+        # current thread to avoid collisions when creating output files
+        if self._parallel_builds:
+            thread_path = self.wheels_build_base / f"{threading.get_native_id()}"
+            thread_path.mkdir(parents=True, exist_ok=True)
+            return thread_path
+        else:
+            return self.wheels_build_base
+
     @property
     def pip_wheel_server_args(self) -> list[str]:
         args = ["--index-url", self.wheel_server_url]