diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 64304eae..a77b1c5f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -221,6 +221,26 @@ jobs: CC_OUTER_LOOP: "gcc" CC_INNER_LOOP: "gcc" + # Python 3.14, free-threaded + - name: py314_freethreaded_pip_openblas + os: ubuntu-latest + PACKAGER: "conda-forge" + BLAS: "openblas" + PYTHON_VERSION: "3.14" + FREETHREADING: "1" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + + # Ubuntu 24.04, which has glibc with different dl_iterate_phdr than + # latest Ubuntus. + - name: ubuntu24_04 + os: ubuntu-24.04 + PACKAGER: "conda-forge" + BLAS: "openblas" + PYTHON_VERSION: "3.14" + CC_OUTER_LOOP: "gcc" + CC_INNER_LOOP: "gcc" + env: ${{ matrix }} runs-on: ${{ matrix.os }} diff --git a/CHANGES.md b/CHANGES.md index a23ce6e0..2529e3bc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,10 @@ 3.8.0 (under development) ========================= -TODO: update me. +- Run faster on Linux, where the new `/proc/self/maps` mechanism in 3.7.0 added + quite a bit of overhead. + https://github.com/joblib/threadpoolctl/pull/250 + 3.7.0 (2026-09-15) ================== diff --git a/conftest.py b/conftest.py index 53baddff..11855652 100644 --- a/conftest.py +++ b/conftest.py @@ -2,4 +2,5 @@ "tests/empirical_scope_observation.py", "tests/_openmp_test_helper.py", "tests/_limit_blas.py", + "tests/_dl_iterate_phdr_deadlock.py", ] diff --git a/continuous_integration/install.sh b/continuous_integration/install.sh index cd9fc2d2..42c551ba 100755 --- a/continuous_integration/install.sh +++ b/continuous_integration/install.sh @@ -41,7 +41,9 @@ make_conda() { fi fi - if [[ "$PYTHON_VERSION" == "*" ]]; then + if [[ "$FREETHREADING" == "1" ]]; then + TO_INSTALL="$TO_INSTALL python-freethreading" + elif [[ "$PYTHON_VERSION" == "*" ]]; then # Avoid installing free-threaded python TO_INSTALL="$TO_INSTALL python-gil" fi diff --git a/tests/_dl_iterate_phdr_deadlock.py b/tests/_dl_iterate_phdr_deadlock.py new file mode 100644 index 00000000..f55af53e --- /dev/null +++ b/tests/_dl_iterate_phdr_deadlock.py @@ -0,0 +1,72 @@ +"""Try to trigger deadlock via dl_iterate_phdr.""" + +import ctypes +import os +import sys +from threading import Thread + +from threadpoolctl import threadpool_limits + + +def create_controllers(done): + for _ in range(100): + # May use dl_iterate_phdr() on Linux: + limiter = threadpool_limits() + # dlopen() of some shared libraries like to be installed: + loaded = False + # Common libraries on Ubuntu: + for name in [ + "libncurses.so.6", + "libncursesw.so.6", + "libgmp.so.10", + "libssl.so.3", + "libcrypt.so.1", + ]: + try: + dll = ctypes.CDLL(name) + del dll + loaded = True + except OSError: + pass + if not loaded: + # Couldn't do an dlopen()s. + os._exit(7) + + # Imports, which also do dlopen(): + try: + import numpy # also gets us BLAS + except ImportError: + pass + import _pickle + + try: + import tests._openmp_test_helper.nested_prange_blas + except ImportError: + pass + + del limiter + + done.append(True) + + +def main(): + threads = [] + done = [] + + for _ in range(os.cpu_count() * 4): + t = Thread(target=create_controllers, args=(done,)) + threads.append(t) + t.start() + + for t in threads: + t.join() + + if len(done) != os.cpu_count() * 4: + sys.exit(1) + + # Special success exist code: + sys.exit(17) + + +if __name__ == "__main__": + main() diff --git a/tests/_openmp_test_helper/nested_prange_blas.pyx b/tests/_openmp_test_helper/nested_prange_blas.pyx index af2a6693..ad3ef185 100644 --- a/tests/_openmp_test_helper/nested_prange_blas.pyx +++ b/tests/_openmp_test_helper/nested_prange_blas.pyx @@ -1,3 +1,5 @@ +# cython: freethreading_compatible = True + cimport openmp from cython.parallel import parallel, prange diff --git a/tests/_openmp_test_helper/nested_prange_blas_custom.pyx b/tests/_openmp_test_helper/nested_prange_blas_custom.pyx index 6c0a9771..d76291e7 100644 --- a/tests/_openmp_test_helper/nested_prange_blas_custom.pyx +++ b/tests/_openmp_test_helper/nested_prange_blas_custom.pyx @@ -1,3 +1,5 @@ +# cython: freethreading_compatible = True + cimport openmp from cython.parallel import parallel, prange diff --git a/tests/_openmp_test_helper/openmp_helpers_inner.pyx b/tests/_openmp_test_helper/openmp_helpers_inner.pyx index e7928d2f..566e7f4c 100644 --- a/tests/_openmp_test_helper/openmp_helpers_inner.pyx +++ b/tests/_openmp_test_helper/openmp_helpers_inner.pyx @@ -1,3 +1,5 @@ +# cython: freethreading_compatible = True + cimport openmp from cython.parallel import prange diff --git a/tests/_openmp_test_helper/openmp_helpers_outer.pyx b/tests/_openmp_test_helper/openmp_helpers_outer.pyx index 2c8a383c..9e668f5c 100644 --- a/tests/_openmp_test_helper/openmp_helpers_outer.pyx +++ b/tests/_openmp_test_helper/openmp_helpers_outer.pyx @@ -1,3 +1,5 @@ +# cython: freethreading_compatible = True + cimport openmp from cython.parallel import prange from openmp_helpers_inner cimport inner_openmp_loop diff --git a/tests/test_threadpoolctl.py b/tests/test_threadpoolctl.py index 35350110..75a6ac9a 100644 --- a/tests/test_threadpoolctl.py +++ b/tests/test_threadpoolctl.py @@ -1121,43 +1121,36 @@ def test_conda_blas_detection_after_import(module): def test_controller_parallelism_no_deadlocks(): - """Creating a controller in parallel to itself does not cause deadlocks. + """ + Creating a controller in parallel to itself and other operations loading + shared libraries does not cause deadlocks. Non-regression test for https://github.com/joblib/threadpoolctl/issues/239 - - Lacking the fixes from PR #243, this deadlocks on Conda environments, at - least, but possibly not on PyPI with Python from a Linux distro. """ - if sys.platform != "linux" or not hasattr(ctypes.PyDLL(None), "backtrace"): + if sys.platform != "linux": pytest.skip("Testing glibc on Linux") - # Internally, backtrace() calls dl_iterate_phdr which can result in - # deadlocks if threadpoolctl is also using dl_iterate_phdr. - backtrace_gil = ctypes.PyDLL(None).backtrace - backtrace_gil.argtypes = [ctypes.c_void_p, ctypes.c_int] - backtrace_nogil = ctypes.CDLL(None).backtrace - backtrace_nogil.argtypes = [ctypes.c_void_p, ctypes.c_int] - - def create_controllers(): - buf = (ctypes.c_void_p * 20)() - for _ in range(100): - limiter = threadpool_limits() - backtrace_gil(buf, 20) - backtrace_nogil(buf, 20) + # Deadlock isn't always reliable, so run multiple times: + for _ in range(10): + process = subprocess.run( + [sys.executable, "-m", "tests._dl_iterate_phdr_deadlock"], timeout=10 + ) - threads = [] - for _ in range(os.cpu_count() * 4): - t = Thread(target=create_controllers) - threads.append(t) - t.start() + if process.returncode == 7: + # Special code indicating it couldn't load any shared libraries. + pytest.skip("Couldn't find any of the exected shared libraries") - for t in threads: - t.join() + # Special code indicating success: + assert process.returncode == 17 @pytest.mark.skipif( not sys.platform.startswith("linux"), - reason="ctypes.util is only avoided on Linux (#225)", + reason="ctypes.util is only avoided on Linux (#225) in Python 3.14", +) +@pytest.mark.skipif( + sys.version_info[:2] >= (3, 15), + reason="Python 3.15 shouldn't have the issue in #225", ) def test_linux_does_not_import_ctypes_util(): # ctypes.util on CPython 3.14 Linux allocates a process-lifetime CFUNCTYPE diff --git a/threadpoolctl.py b/threadpoolctl.py index b8b4448d..177bb244 100644 --- a/threadpoolctl.py +++ b/threadpoolctl.py @@ -24,6 +24,23 @@ from functools import lru_cache from contextlib import ContextDecorator +# ctypes.util is not imported on Linux: on CPython 3.14 it allocates a +# process-lifetime CFUNCTYPE callback that is not fork-safe with some libffi +# builds (#225). dllist also uses dl_iterate_phdr internally (#239), which we +# already avoid on Linux via /proc/self/maps on older versions of Python. +dllist = None +if sys.platform != "emscripten" and ( + # Python 3.15 doesn't have the CFUNCTYPE anymore: + sys.platform == "linux" + and sys.version_info[:2] >= (3, 15) +): + try: + from ctypes.util import dllist + except ImportError: + # CPython before 3.14 does not provide dll inspection. + dllist = None + + __version__ = "3.8.0.dev0" __all__ = [ "threadpool_limits", @@ -57,6 +74,14 @@ _SYSTEM_UINT_HALF = ctypes.c_uint32 if sys.maxsize > 2**32 else ctypes.c_uint16 +_HAS_PROCFS = ( + # Only available on Linux: + sys.platform == "linux" + # Make sure /proc is mounted: + and os.path.exists("/proc/self") +) + + class _dl_phdr_info(ctypes.Structure): _fields_ = [ ("dlpi_addr", _SYSTEM_UINT), # Base address of object @@ -1125,24 +1150,31 @@ def __len__(self): def _load_libraries(self): """Loop through loaded shared libraries and store the supported ones""" - # ctypes.util is not imported on Linux: on CPython 3.14 it allocates a - # process-lifetime CFUNCTYPE callback that is not fork-safe with some - # libffi builds (#225). dllist also uses dl_iterate_phdr internally - # (#239), which we already avoid on Linux via /proc/self/maps. - dllist = None - if sys.platform not in ("linux", "emscripten"): - try: - from ctypes.util import dllist - except ImportError: - # CPython before 3.14 does not provide dll inspection. - dllist = None - - if sys.platform == "linux" and os.path.exists("/proc/self/maps"): - # On glibc, dl_iterate_phdr has an internal lock, and that plus - # calling back into Python and the need to (re)acquire the GIL - # results in deadlocks. To avoid that, use a Linux-specific - # mechanism that doesn't have these issues; since it's Linux, musl - # works fine too. + # On glibc 2.39 and earlier, dl_iterate_phdr has an internal lock, and + # that plus calling back into Python and the need to (re)acquire the + # GIL can result in deadlocks. + # + # On glibc 2.40 and later, dl has a read-write lock, and + # dl_iterate_phdr should only use the read version since it's not + # modifying anything. So you no longer get deadlocks purely from + # dl_iterate_phdr. You can still deadlock with dlopen() though. + # + # To avoid that deadlock, listing shared libraries can use a + # Linux-specific mechanism that doesn't have these issues + # (/proc/self/maps). Since it's the Linux kernel, musl works fine too. + # + # The downside is this mechanism is slower, so avoid it when safe + # alternatives are available. In particular, if dllist() is available + # (3.14+), written in C so no risk of GC part way (3.15+), and there is + # no GIL, no need to use /proc, since dllist() is faster. It's possible + # dllist() might work in GIL builds too but see + # https://github.com/python/cpython/issues/157573. So we should + # consider enabling it on GIL Python too once threadpoolctl supports + # 3.15. + if _HAS_PROCFS and not ( + sys.version_info[:2] >= (3, 15) + and not getattr(sys, "_is_gil_enabled", lambda: True)() + ): self._find_libraries_with_linux() elif dllist is not None: # On Python 3.14+, this functionality is built-in. Once Python 3.13 @@ -1159,24 +1191,19 @@ def _load_libraries(self): # Non-Linux Unix platforms. self._find_libraries_with_dl_iterate_phdr() + _PATH_RE = re.compile(rb" (/[^\n]+\.so[^\n^/]*)\n", re.MULTILINE) + def _find_libraries_with_linux(self): """Loop through loaded libraries and return binders on supported ones Uses a Linux-specific mechanism: https://man7.org/linux/man-pages/man5/proc_pid_maps.5.html """ - with open("/proc/self/maps") as f: + with open("/proc/self/maps", "rb") as f: maps = f.read() - filepaths = set() - for line in maps.splitlines(): - start_index = line.find("/") - if start_index == -1 or ".so" not in line: - continue - filepath = line[start_index:] - if os.path.exists(filepath): - filepaths.add(filepath) - + filepaths = set(self._PATH_RE.findall(maps)) for filepath in filepaths: + filepath = filepath.decode("utf-8") self._make_controller_from_path(filepath) def _find_libraries_with_python(self, dllist): @@ -1599,9 +1626,13 @@ def _make_controller_from_path(self, filepath): # expected library (e.g. a library having a common prefix with one of the # our supported libraries). Otherwise, create and store the library # controller. - lib_controller = controller_class( - filepath=filepath, prefix=prefix, parent=self - ) + try: + lib_controller = controller_class( + filepath=filepath, prefix=prefix, parent=self + ) + except OSError: + # Probably because we couldn't load the filepath as a CDLL. + continue if filepath in (lib.filepath for lib in self.lib_controllers): # We already have a controller for this library.