diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index b5da3bf9..b2810855 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -52,7 +52,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v7 with: - python-version: "3.13" + python-version: "3.15" architecture: x64 - name: Checkout uses: actions/checkout@v7 @@ -68,7 +68,7 @@ jobs: - uses: actions/checkout@v7 - uses: conda-incubator/setup-miniconda@v4 with: - python-version: "3.13" + python-version: "3.15" miniforge-version: latest channels: conda-forge conda-remove-defaults: "true" @@ -88,7 +88,7 @@ jobs: - uses: actions/checkout@v7 - uses: conda-incubator/setup-miniconda@v4 with: - python-version: '3.14' + python-version: '3.15' miniforge-version: latest channels: conda-forge conda-remove-defaults: "true" @@ -121,11 +121,11 @@ jobs: matrix: include: - operating-system: ubuntu-latest - python-version: '3.14' + python-version: '3.15' environment-file: .ci_support/environment-openmpi.yml - operating-system: ubuntu-latest - python-version: '3.14' + python-version: '3.15' environment-file: .ci_support/environment-mpich.yml steps: @@ -165,7 +165,7 @@ jobs: - name: Setup Mambaforge uses: conda-incubator/setup-miniconda@v4 with: - python-version: "3.13" + python-version: "3.14" miniforge-version: latest channels: conda-forge conda-remove-defaults: "true" @@ -190,7 +190,7 @@ jobs: - name: Setup Mambaforge uses: conda-incubator/setup-miniconda@v4 with: - python-version: "3.13" + python-version: "3.14" miniforge-version: latest channels: conda-forge conda-remove-defaults: "true" @@ -218,7 +218,7 @@ jobs: cat .ci_support/environment-mpich.yml - uses: conda-incubator/setup-miniconda@v3 with: - python-version: '3.13' + python-version: '3.14' miniforge-version: latest channels: conda-forge conda-remove-defaults: "true" @@ -249,7 +249,7 @@ jobs: cat .ci_support/environment-openmpi.yml - uses: conda-incubator/setup-miniconda@v4 with: - python-version: '3.13' + python-version: '3.14' miniforge-version: latest channels: conda-forge conda-remove-defaults: "true" @@ -295,7 +295,7 @@ jobs: run: sudo apt install -y mpich - uses: conda-incubator/setup-miniconda@v4 with: - python-version: '3.13' + python-version: '3.14' miniforge-version: latest channels: conda-forge conda-remove-defaults: "true" @@ -318,13 +318,16 @@ jobs: matrix: include: - operating-system: macos-latest - python-version: '3.14' + python-version: '3.15' - operating-system: ubuntu-24.04-arm - python-version: '3.14' + python-version: '3.15' - operating-system: ubuntu-22.04-arm - python-version: '3.14' + python-version: '3.15' + + - operating-system: ubuntu-latest + python-version: '3.15' - operating-system: ubuntu-latest python-version: '3.14' @@ -359,15 +362,18 @@ jobs: matrix: include: - operating-system: macos-latest - python-version: '3.14' + python-version: '3.15' - operating-system: ubuntu-latest - python-version: '3.14' + python-version: '3.15' - operating-system: ubuntu-24.04-arm - python-version: '3.14' + python-version: '3.15' - operating-system: ubuntu-22.04-arm + python-version: '3.15' + + - operating-system: ubuntu-latest python-version: '3.14' - operating-system: ubuntu-latest @@ -400,7 +406,7 @@ jobs: - uses: actions/checkout@v7 - uses: conda-incubator/setup-miniconda@v4 with: - python-version: "3.14" + python-version: "3.15" miniforge-version: latest channels: conda-forge conda-remove-defaults: "true" diff --git a/pyproject.toml b/pyproject.toml index ec8278c3..e477bde2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ authors = [ readme = "README.md" license = { file = "LICENSE" } keywords = ["high performance computing", "hpc", "task scheduler", "slurm", "flux-framework", "executor", "python", "parallel computing", "distributed computing", "scientific computing", "concurrent.futures", "mpi4py", "gpu"] -requires-python = ">3.9, <3.15" +requires-python = ">3.9, <3.16" classifiers = [ "Development Status :: 5 - Production/Stable", "Topic :: Scientific/Engineering :: Physics", @@ -28,6 +28,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", ] dependencies = [ "cloudpickle==3.1.2", diff --git a/src/executorlib/executor/base.py b/src/executorlib/executor/base.py index 72e25ffd..552f182f 100644 --- a/src/executorlib/executor/base.py +++ b/src/executorlib/executor/base.py @@ -1,15 +1,18 @@ import queue from abc import ABC +from collections.abc import Iterable, Iterator from concurrent.futures import ( Executor as FutureExecutor, ) from concurrent.futures import ( Future, ) -from typing import Callable, Optional +from typing import Any, Callable, Optional, TypeVar from executorlib.task_scheduler.base import TaskSchedulerBase +_T = TypeVar("_T") + class BaseExecutor(FutureExecutor, ABC): """ @@ -121,11 +124,12 @@ def submit( # type: ignore def map( self, - fn: Callable, - *iterables, + fn: Callable[..., _T], + *iterables: Iterable[Any], timeout: Optional[float] = None, chunksize: int = 1, - ): + buffersize: Optional[int] = None, + ) -> Iterator[_T]: """Returns an iterator equivalent to map(fn, iter). Args: @@ -149,9 +153,11 @@ def map( """ if self._is_active: return self._task_scheduler.map( - *([fn] + list(iterables)), + fn, + *iterables, timeout=timeout, chunksize=chunksize, + buffersize=buffersize, ) else: raise RuntimeError("cannot schedule new futures after shutdown") diff --git a/src/executorlib/task_scheduler/base.py b/src/executorlib/task_scheduler/base.py index fd58e958..34534950 100644 --- a/src/executorlib/task_scheduler/base.py +++ b/src/executorlib/task_scheduler/base.py @@ -1,5 +1,7 @@ import contextlib import queue +import sys +from collections.abc import Iterable, Iterator from concurrent.futures import ( Executor as FutureExecutor, ) @@ -7,12 +9,14 @@ Future, ) from threading import Thread -from typing import Callable, Optional, Union +from typing import Any, Callable, Optional, TypeVar, Union from executorlib.standalone.inputcheck import check_resource_dict from executorlib.standalone.queue import cancel_items_in_queue from executorlib.standalone.serialize import cloudpickle_register +_T = TypeVar("_T") + def validate_resource_dict(resource_dict: dict): """ @@ -179,11 +183,12 @@ def submit( # type: ignore def map( self, - fn: Callable, - *iterables, + fn: Callable[..., _T], + *iterables: Iterable[Any], timeout: Optional[float] = None, chunksize: int = 1, - ): + buffersize: Optional[int] = None, + ) -> Iterator[_T]: """Returns an iterator equivalent to map(fn, iter). Args: @@ -212,6 +217,14 @@ def map( i.result() if isinstance(i, Future) else i for i in iterables ) + if sys.version_info >= (3, 14): + return super().map( + fn, + *iterables, + timeout=timeout, + chunksize=chunksize, + buffersize=buffersize, + ) return super().map(fn, *iterables, timeout=timeout, chunksize=chunksize) def shutdown(self, wait: bool = True, *, cancel_futures: bool = False): @@ -263,6 +276,10 @@ def __len__(self) -> int: queue_size = self._future_queue.qsize() return queue_size + def __bool__(self) -> bool: + """Return True regardless of the number of queued futures.""" + return True + def __del__(self): """ Clean-up the resources associated with the Executor.