PERF Run faster on Linux

.github/workflows/test.yml

@@ -221,6 +221,26 @@ jobs:
             CC_OUTER_LOOP: "gcc"
             CC_INNER_LOOP: "gcc"
 
+          # Python 3.14, free-threaded
+          - name: py314_freethreaded_pip_openblas
+            os: ubuntu-latest
+            PACKAGER: "conda-forge"
+            BLAS: "openblas"
+            PYTHON_VERSION: "3.14"
+            FREETHREADING: "1"
+            CC_OUTER_LOOP: "gcc"
+            CC_INNER_LOOP: "gcc"
+
+          # Ubuntu 24.04, which has glibc with different dl_iterate_phdr than
+          # latest Ubuntus.
+          - name: ubuntu24_04
+            os: ubuntu-24.04
+            PACKAGER: "conda-forge"
+            BLAS: "openblas"
+            PYTHON_VERSION: "3.14"
+            CC_OUTER_LOOP: "gcc"
+            CC_INNER_LOOP: "gcc"
+
     env: ${{ matrix }}
 
     runs-on: ${{ matrix.os }}

CHANGES.md

@@ -1,7 +1,10 @@
 3.8.0 (under development)
 =========================
 
-TODO: update me.
+- Run faster on Linux, where the new `/proc/self/maps` mechanism in 3.7.0 added
+  quite a bit of overhead.
+  https://github.com/joblib/threadpoolctl/pull/250
+
 
 3.7.0 (2026-09-15)
 ==================

conftest.py

@@ -2,4 +2,5 @@
     "tests/empirical_scope_observation.py",
     "tests/_openmp_test_helper.py",
     "tests/_limit_blas.py",
+    "tests/_dl_iterate_phdr_deadlock.py",
 ]

continuous_integration/install.sh

@@ -41,7 +41,9 @@ make_conda() {
         fi
     fi
 
-    if [[ "$PYTHON_VERSION" == "*" ]]; then
+    if [[ "$FREETHREADING" == "1" ]]; then
+        TO_INSTALL="$TO_INSTALL python-freethreading"
+    elif [[ "$PYTHON_VERSION" == "*" ]]; then
         # Avoid installing free-threaded python
         TO_INSTALL="$TO_INSTALL python-gil"
     fi

tests/_dl_iterate_phdr_deadlock.py

@@ -0,0 +1,72 @@
+"""Try to trigger deadlock via dl_iterate_phdr."""
+
+import ctypes
+import os
+import sys
+from threading import Thread
+
+from threadpoolctl import threadpool_limits
+
+
+def create_controllers(done):
+    for _ in range(100):
+        # May use dl_iterate_phdr() on Linux:
+        limiter = threadpool_limits()
+        # dlopen() of some shared libraries like to be installed:
+        loaded = False
+        # Common libraries on Ubuntu:
+        for name in [
+            "libncurses.so.6",
+            "libncursesw.so.6",
+            "libgmp.so.10",
+            "libssl.so.3",
+            "libcrypt.so.1",
+        ]:
+            try:
+                dll = ctypes.CDLL(name)
+                del dll
+                loaded = True
+            except OSError:
+                pass
+        if not loaded:
+            # Couldn't do an dlopen()s.
+            os._exit(7)
+
+        # Imports, which also do dlopen():
+        try:
+            import numpy  # also gets us BLAS
+        except ImportError:
+            pass
+        import _pickle
+
+        try:
+            import tests._openmp_test_helper.nested_prange_blas
+        except ImportError:
+            pass
+
+        del limiter
+
+    done.append(True)
+
+
+def main():
+    threads = []
+    done = []
+
+    for _ in range(os.cpu_count() * 4):
+        t = Thread(target=create_controllers, args=(done,))
+        threads.append(t)
+        t.start()
+
+    for t in threads:
+        t.join()
+
+    if len(done) != os.cpu_count() * 4:
+        sys.exit(1)
+
+    # Special success exist code:
+    sys.exit(17)
+
+
+if __name__ == "__main__":
+    main()

tests/_openmp_test_helper/nested_prange_blas.pyx

@@ -1,3 +1,5 @@
+# cython: freethreading_compatible = True
+
 cimport openmp
 from cython.parallel import parallel, prange
 

tests/_openmp_test_helper/nested_prange_blas_custom.pyx

@@ -1,3 +1,5 @@
+# cython: freethreading_compatible = True
+
 cimport openmp
 from cython.parallel import parallel, prange
 

tests/_openmp_test_helper/openmp_helpers_inner.pyx

@@ -1,3 +1,5 @@
+# cython: freethreading_compatible = True
+
 cimport openmp
 from cython.parallel import prange
 

tests/_openmp_test_helper/openmp_helpers_outer.pyx

@@ -1,3 +1,5 @@
+# cython: freethreading_compatible = True
+
 cimport openmp
 from cython.parallel import prange
 from openmp_helpers_inner cimport inner_openmp_loop

tests/test_threadpoolctl.py

@@ -1121,43 +1121,36 @@ def test_conda_blas_detection_after_import(module):
 
 
 def test_controller_parallelism_no_deadlocks():
-    """Creating a controller in parallel to itself does not cause deadlocks.
+    """
+    Creating a controller in parallel to itself and other operations loading
+    shared libraries does not cause deadlocks.
 
     Non-regression test for https://github.com/joblib/threadpoolctl/issues/239
-
-    Lacking the fixes from PR #243, this deadlocks on Conda environments, at
-    least, but possibly not on PyPI with Python from a Linux distro.
     """
-    if sys.platform != "linux" or not hasattr(ctypes.PyDLL(None), "backtrace"):
+    if sys.platform != "linux":
         pytest.skip("Testing glibc on Linux")
 
-    # Internally, backtrace() calls dl_iterate_phdr which can result in
-    # deadlocks if threadpoolctl is also using dl_iterate_phdr.
-    backtrace_gil = ctypes.PyDLL(None).backtrace
-    backtrace_gil.argtypes = [ctypes.c_void_p, ctypes.c_int]
-    backtrace_nogil = ctypes.CDLL(None).backtrace
-    backtrace_nogil.argtypes = [ctypes.c_void_p, ctypes.c_int]
-
-    def create_controllers():
-        buf = (ctypes.c_void_p * 20)()
-        for _ in range(100):
-            limiter = threadpool_limits()
-            backtrace_gil(buf, 20)
-            backtrace_nogil(buf, 20)
+    # Deadlock isn't always reliable, so run multiple times:
+    for _ in range(10):
+        process = subprocess.run(
+            [sys.executable, "-m", "tests._dl_iterate_phdr_deadlock"], timeout=10
+        )
 
-    threads = []
-    for _ in range(os.cpu_count() * 4):
-        t = Thread(target=create_controllers)
-        threads.append(t)
-        t.start()
+        if process.returncode == 7:
+            # Special code indicating it couldn't load any shared libraries.
+            pytest.skip("Couldn't find any of the exected shared libraries")
 
-    for t in threads:
-        t.join()
+        # Special code indicating success:
+        assert process.returncode == 17
 
 
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"),
-    reason="ctypes.util is only avoided on Linux (#225)",
+    reason="ctypes.util is only avoided on Linux (#225) in Python 3.14",
+)
+@pytest.mark.skipif(
+    sys.version_info[:2] >= (3, 15),
+    reason="Python 3.15 shouldn't have the issue in #225",
 )
 def test_linux_does_not_import_ctypes_util():
     # ctypes.util on CPython 3.14 Linux allocates a process-lifetime CFUNCTYPE

threadpoolctl.py

@@ -24,6 +24,23 @@
 from functools import lru_cache
 from contextlib import ContextDecorator
 
+# ctypes.util is not imported on Linux: on CPython 3.14 it allocates a
+# process-lifetime CFUNCTYPE callback that is not fork-safe with some libffi
+# builds (#225). dllist also uses dl_iterate_phdr internally (#239), which we
+# already avoid on Linux via /proc/self/maps on older versions of Python.
+dllist = None
+if sys.platform != "emscripten" and (
+    # Python 3.15 doesn't have the CFUNCTYPE anymore:
+    sys.platform == "linux"
+    and sys.version_info[:2] >= (3, 15)
+):
+    try:
+        from ctypes.util import dllist
+    except ImportError:
+        # CPython before 3.14 does not provide dll inspection.
+        dllist = None
+
+
 __version__ = "3.8.0.dev0"
 __all__ = [
     "threadpool_limits",
@@ -57,6 +74,14 @@
 _SYSTEM_UINT_HALF = ctypes.c_uint32 if sys.maxsize > 2**32 else ctypes.c_uint16
 
 
+_HAS_PROCFS = (
+    # Only available on Linux:
+    sys.platform == "linux"
+    # Make sure /proc is mounted:
+    and os.path.exists("/proc/self")
+)
+
+
 class _dl_phdr_info(ctypes.Structure):
     _fields_ = [
         ("dlpi_addr", _SYSTEM_UINT),  # Base address of object
@@ -1125,24 +1150,31 @@ def __len__(self):
 
     def _load_libraries(self):
         """Loop through loaded shared libraries and store the supported ones"""
-        # ctypes.util is not imported on Linux: on CPython 3.14 it allocates a
-        # process-lifetime CFUNCTYPE callback that is not fork-safe with some
-        # libffi builds (#225). dllist also uses dl_iterate_phdr internally
-        # (#239), which we already avoid on Linux via /proc/self/maps.
-        dllist = None
-        if sys.platform not in ("linux", "emscripten"):
-            try:
-                from ctypes.util import dllist
-            except ImportError:
-                # CPython before 3.14 does not provide dll inspection.
-                dllist = None
-
-        if sys.platform == "linux" and os.path.exists("/proc/self/maps"):
-            # On glibc, dl_iterate_phdr has an internal lock, and that plus
-            # calling back into Python and the need to (re)acquire the GIL
-            # results in deadlocks. To avoid that, use a Linux-specific
-            # mechanism that doesn't have these issues; since it's Linux, musl
-            # works fine too.
+        # On glibc 2.39 and earlier, dl_iterate_phdr has an internal lock, and
+        # that plus calling back into Python and the need to (re)acquire the
+        # GIL can result in deadlocks.
+        #
+        # On glibc 2.40 and later, dl has a read-write lock, and
+        # dl_iterate_phdr should only use the read version since it's not
+        # modifying anything. So you no longer get deadlocks purely from
+        # dl_iterate_phdr. You can still deadlock with dlopen() though.
+        #
+        # To avoid that deadlock, listing shared libraries can use a
+        # Linux-specific mechanism that doesn't have these issues
+        # (/proc/self/maps). Since it's the Linux kernel, musl works fine too.
+        #
+        # The downside is this mechanism is slower, so avoid it when safe
+        # alternatives are available. In particular, if dllist() is available
+        # (3.14+), written in C so no risk of GC part way (3.15+), and there is
+        # no GIL, no need to use /proc, since dllist() is faster. It's possible
+        # dllist() might work in GIL builds too but see
+        # https://github.com/python/cpython/issues/157573. So we should
+        # consider enabling it on GIL Python too once threadpoolctl supports
+        # 3.15.
+        if _HAS_PROCFS and not (
+            sys.version_info[:2] >= (3, 15)
+            and not getattr(sys, "_is_gil_enabled", lambda: True)()
+        ):
             self._find_libraries_with_linux()
         elif dllist is not None:
             # On Python 3.14+, this functionality is built-in. Once Python 3.13
@@ -1159,24 +1191,19 @@ def _load_libraries(self):
             # Non-Linux Unix platforms.
             self._find_libraries_with_dl_iterate_phdr()
 
+    _PATH_RE = re.compile(rb" (/[^\n]+\.so[^\n^/]*)\n", re.MULTILINE)
+
     def _find_libraries_with_linux(self):
         """Loop through loaded libraries and return binders on supported ones
 
         Uses a Linux-specific mechanism:
         https://man7.org/linux/man-pages/man5/proc_pid_maps.5.html
         """
-        with open("/proc/self/maps") as f:
+        with open("/proc/self/maps", "rb") as f:
             maps = f.read()
-        filepaths = set()
-        for line in maps.splitlines():
-            start_index = line.find("/")
-            if start_index == -1 or ".so" not in line:
-                continue
-            filepath = line[start_index:]
-            if os.path.exists(filepath):
-                filepaths.add(filepath)
-
+        filepaths = set(self._PATH_RE.findall(maps))
         for filepath in filepaths:
+            filepath = filepath.decode("utf-8")
             self._make_controller_from_path(filepath)
 
     def _find_libraries_with_python(self, dllist):
@@ -1599,9 +1626,13 @@ def _make_controller_from_path(self, filepath):
             # expected library (e.g. a library having a common prefix with one of the
             # our supported libraries). Otherwise, create and store the library
             # controller.
-            lib_controller = controller_class(
-                filepath=filepath, prefix=prefix, parent=self
-            )
+            try:
+                lib_controller = controller_class(
+                    filepath=filepath, prefix=prefix, parent=self
+                )
+            except OSError:
+                # Probably because we couldn't load the filepath as a CDLL.
+                continue
 
             if filepath in (lib.filepath for lib in self.lib_controllers):
                 # We already have a controller for this library.