ThreadpoolController() raises an error intermittently with conda-forge OpenCV on Windows

[dugalh]

Hello

I am getting an intermittent OSError when a ThreadpoolController is created soon after importing the conda-forge OpenCV package on Windows. This reproduces the problem reliably on my PC:

import cv2
from threadpoolctl import ThreadpoolController

for _ in range(1000):
    ThreadpoolController()

This is the stack trace:

Traceback (most recent call last):
  File "C:\Temp\threadpoolctl_test.py", line 5, in <module>
    ThreadpoolController()
    ~~~~~~~~~~~~~~~~~~~~^^
  File "C:\Users\dugalh\miniconda3\envs\py313-geospatial\Lib\site-packages\threadpoolctl.py", line 818, in __init__
    self._load_libraries()
    ~~~~~~~~~~~~~~~~~~~~^^
  File "C:\Users\dugalh\miniconda3\envs\py313-geospatial\Lib\site-packages\threadpoolctl.py", line 972, in _load_libraries
    self._find_libraries_with_enum_process_module_ex()
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
  File "C:\Users\dugalh\miniconda3\envs\py313-geospatial\Lib\site-packages\threadpoolctl.py", line 1099, in _find_libraries_with_enum_process_module_ex
    raise OSError("GetModuleFileNameEx failed")
OSError: GetModuleFileNameEx failed

I have done some debugging and think the problem occurs when DLL(s) are unloaded between the EnumProcessModulesEx() and GetModuleFileNameExW() calls in ThreadpoolController._find_libraries_with_enum_process_module_ex(). Then GetModuleFileNameExW() fails when it is passed handle(s) of unloaded DLL(s).

In addition to the possibility of DLL(s) being unloaded between these calls, the EnumProcessModulesEx() docs say "If the module list in the target process is corrupted or not yet initialized, or if the module list changes during the function call as a result of DLLs being loaded or unloaded, EnumProcessModules may fail or return incorrect information."

An alternative to EnumProcessModulesEx() / GetModuleFileNameExW() is CreateToolhelp32Snapshot(). E.g. this function returns the currently loaded DLLs:

import ctypes

def _find_libraries_with_create_tool_help_snapshot():
    from ctypes import wintypes

    class MODULEENTRY32W(ctypes.Structure):
        # ctypes definition of:
        # https://learn.microsoft.com/en-us/windows/win32/api/tlhelp32/ns-tlhelp32-moduleentry32w
        _fields_ = [
            ('dwSize', wintypes.DWORD),
            ('th32ModuleID', wintypes.DWORD),
            ('th32ProcessID', wintypes.DWORD),
            ('GlblcntUsage', wintypes.DWORD),
            ('ProccntUsage', wintypes.DWORD),
            ('modBaseAddr', ctypes.POINTER(wintypes.BYTE)),
            ('modBaseSize', wintypes.DWORD),
            ('hModule', wintypes.HMODULE),
            ('szModule', wintypes.WCHAR * 256),
            ('szExePath', wintypes.WCHAR * wintypes.MAX_PATH),
        ]

    TH32CS_SNAPMODULE = 0x8
    TH32CS_SNAPMODULE32 = 0x10
    ERROR_BAD_LENGTH = 0x18
    INVALID_HANDLE_VALUE = wintypes.HANDLE(-1).value
    ERROR_NO_MORE_FILES = 0x12

    kernel_32 = ctypes.WinDLL("kernel32.dll", use_last_error=True)

    # ctype function definitions
    CreateToolhelp32Snapshot = kernel_32.CreateToolhelp32Snapshot
    CreateToolhelp32Snapshot.argtypes = [wintypes.DWORD, wintypes.DWORD]
    CreateToolhelp32Snapshot.restype = wintypes.HANDLE

    Module32FirstW = kernel_32.Module32FirstW
    Module32FirstW.argtypes = [wintypes.HANDLE, ctypes.POINTER(MODULEENTRY32W)]
    Module32FirstW.restype = wintypes.BOOL

    Module32NextW = kernel_32.Module32NextW
    Module32NextW.argtypes = [wintypes.HANDLE, ctypes.POINTER(MODULEENTRY32W)]
    Module32NextW.restype = wintypes.BOOL

    CloseHandle = kernel_32.CloseHandle
    CloseHandle.argtypes = [wintypes.HANDLE]
    CloseHandle.restype = wintypes.BOOL

    # take a snapshot of loaded libraries
    while True:
        snap_handle = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE | TH32CS_SNAPMODULE32, 0)
        if snap_handle == INVALID_HANDLE_VALUE:
            err = ctypes.get_last_error()
            if err == ERROR_BAD_LENGTH:
                # retry until it succeeds
                continue
            msg = ctypes.FormatError(err).strip()
            raise ctypes.WinError(err, f'CreateToolhelp32Snapshot failed: {msg}')
        break

    libraries = []
    try:
        # iterate through the snapshot, retrieving library paths
        lib_entry = MODULEENTRY32W()
        lib_entry.dwSize = ctypes.sizeof(MODULEENTRY32W)
        if not (res := Module32FirstW(snap_handle, ctypes.byref(lib_entry))):
            err = ctypes.get_last_error()
            msg = ctypes.FormatError(err).strip()
            raise ctypes.WinError(err, f'Module32FirstW failed: {msg}')

        while res:
            libraries.append(lib_entry.szExePath)
            res = Module32NextW(snap_handle, ctypes.byref(lib_entry))

        if err := ctypes.get_last_error() != ERROR_NO_MORE_FILES:
            msg = ctypes.FormatError(err).strip()
            raise ctypes.WinError(err, f'Module32NextW failed: {msg}')
    finally:
        CloseHandle(snap_handle)

    return libraries

I think this is more robust to DLLs being loaded/unloaded, but it fails if any of the loaded DLL path lengths are longer than MAX_PATH. That is not really an issue for me, but it could be for other users?