diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 97989aa..81a6500 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -11,7 +11,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10"] + # 3.15 exercises the native PEP 810 lazy import path + python-version: ["3.10", "3.15-dev"] steps: - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 51d1ee2..96eaba0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,8 @@ jobs: "3.11", "3.12", "3.13", - "3.14-dev", + "3.14", + "3.15-dev", "pypy-3.9", "pypy-3.10", ] diff --git a/.gitignore b/.gitignore index bd4236a..49f9338 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,10 @@ lazy_loader.egg-info/ # Unit test / coverage reports .pytest_cache/ +.coverage +.coverage.* +coverage.xml +htmlcov/ # General .DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3da2b2e..fd777d7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,20 +23,20 @@ repos: - id: check-added-large-files - repo: https://github.com/rbubley/mirrors-prettier - rev: 39e2973981e6d2f9b6c543b0086a2d2393abdc89 # frozen: v3.9.4 + rev: 39e2973981e6d2f9b6c543b0086a2d2393abdc89 # frozen: v3.9.4 hooks: - id: prettier files: \.(html|md|yml|yaml|toml) args: [--prose-wrap=preserve] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: c59bba8fb259db0fec2bbb77ad8ba51ea7341b56 # frozen: v0.15.20 + rev: c59bba8fb259db0fec2bbb77ad8ba51ea7341b56 # frozen: v0.15.20 hooks: - id: ruff args: ["--fix", "--show-fixes", "--exit-non-zero-on-fix"] - id: ruff-format - repo: https://github.com/codespell-project/codespell - rev: "2ccb47ff45ad361a21071a7eedda4c37e6ae8c5a" # frozen: v2.4.2 + rev: "2ccb47ff45ad361a21071a7eedda4c37e6ae8c5a" # frozen: v2.4.2 hooks: - id: codespell diff --git a/README.md b/README.md index ab35b98..243fc24 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,24 @@ from .edges import (sobel, scharr, prewitt, roberts, Except that all subpackages (such as `rank`) and functions (such as `sobel`) are loaded upon access. +### Native lazy imports on Python 3.15+ + +On Python 3.15 and newer, `lazy.attach` and `lazy.attach_stub` use the +interpreter's own lazy imports ([PEP 810](https://peps.python.org/pep-0810/)) +when available. No code changes are needed, but two behaviors differ: + +- Attached names are bound in the package `__dict__` as lazy proxies from the + start, instead of appearing only once accessed. Code that walks + `vars(package)` rather than using `getattr` now sees proxy objects. +- An attached name that its submodule does not define raises `ImportError` on + access, where it used to raise `AttributeError`. Such a name is always a + mistake in the `submod_attrs` list or the stub file, which `hasattr` used to + hide by reporting the name as simply absent. + +`lazy.load` is unchanged on all versions, since PEP 810 proxies only resolve +when accessed through a module namespace. In code that only runs on 3.15+, a +plain `lazy import numpy` statement replaces it. + ### Type checkers Static type checkers and IDEs cannot infer type information from diff --git a/src/lazy_loader/__init__.py b/src/lazy_loader/__init__.py index b4c4c15..635e13e 100644 --- a/src/lazy_loader/__init__.py +++ b/src/lazy_loader/__init__.py @@ -50,6 +50,75 @@ def __setattr__(self, name, value): super().__setattr__(name, value) +# PEP 810 explicit lazy imports. The syntax is not available on every 3.15 +# build, so detect it rather than comparing version numbers. +try: + compile("lazy import sys", "", "exec") +except SyntaxError: + _NATIVE_LAZY_IMPORTS = False +else: + _NATIVE_LAZY_IMPORTS = True + + +def _attach_native(package_name, submodules, submod_attrs): + """Bind native lazy import proxies (PEP 810) in the package namespace. + + Names already bound in the package namespace are left untouched. Where + proxies cannot be bound, the names stay unbound and the caller's + ``__getattr__`` provides the lazy behavior instead. + """ + package = sys.modules.get(package_name) + if package is None: + # Not inside the package's import; cannot bind proxies in its + # namespace. + return + + # Since the names are embedded in generated import statements below, + # ensure they are identifiers and not arbitrary code. + names = [package_name, *submodules, *submod_attrs] + names.extend(attr for attrs in submod_attrs.values() for attr in attrs) + if not all(part.isidentifier() for name in names for part in name.split(".")): + return + + pkg_dict = vars(package) + + # Absolute imports, like the classic __getattr__ mechanism uses, so that + # no relative-import resolution (via __spec__ or __package__) is needed. + # `submodules` is a set, so sort it for a reproducible statement order; + # `submod_attrs` keeps its own order, under which a name listed for + # several modules resolves to the last one, as in __getattr__. + lines = [ + f"lazy from {package_name} import {name}" + for name in sorted(submodules) + if name not in pkg_dict + ] + for mod, attrs in submod_attrs.items(): + new_attrs = [a for a in attrs if a not in pkg_dict and a not in submodules] + if new_attrs: + lines.append( + f"lazy from {package_name}.{mod} import {', '.join(new_attrs)}" + ) + + if not lines: + return + + try: + code = compile( + "\n".join(lines), f"", "exec" + ) + except SyntaxError: + # A submodule or attribute name that is not expressible as import + # syntax (e.g., a reserved keyword). + return + + # exec() inserts __builtins__ into the namespace it is given; leave the + # package namespace as it was found. + had_builtins = "__builtins__" in pkg_dict + exec(code, pkg_dict) + if not had_builtins: + pkg_dict.pop("__builtins__", None) + + def attach(package_name, submodules=None, submod_attrs=None): """Attach lazily loaded submodules, functions, or other attributes. @@ -70,6 +139,9 @@ def attach(package_name, submodules=None, submod_attrs=None): __name__, ["mysubmodule", "anothersubmodule"], {"foo": ["someattr"]} ) + On Python 3.15 and newer, this delegates to the interpreter's native + lazy import mechanism (PEP 810) whenever possible. + Parameters ---------- package_name : str @@ -148,6 +220,14 @@ def __dir__(): if eager_import: for attr in set(attr_to_modules.keys()) | submodules: __getattr__(attr) + elif _NATIVE_LAZY_IMPORTS: + # On Python 3.15+, delegate to native lazy imports (PEP 810) where + # possible. The proxies are bound directly in the package namespace, + # so the returned __getattr__ is then only consulted for unknown + # names. If native binding is not possible (e.g. `package_name` is + # not an imported module), the classic __getattr__ mechanism above + # provides the lazy behavior as before. + _attach_native(package_name, submodules, submod_attrs) return __getattr__, __dir__, __all__.copy() diff --git a/tests/test_lazy_loader.py b/tests/test_lazy_loader.py index 3dc2913..da37662 100644 --- a/tests/test_lazy_loader.py +++ b/tests/test_lazy_loader.py @@ -178,10 +178,168 @@ def test_attach_same_module_and_attr_name(clean_fake_pkg, eager_import): assert isinstance(some_func, types.FunctionType) +NATIVE_LAZY_IMPORTS = lazy._NATIVE_LAZY_IMPORTS + + +def test_attach_native_proxies(clean_fake_pkg): + from tests import fake_pkg + + if NATIVE_LAZY_IMPORTS: + # Names are bound in the package namespace as native lazy proxies + assert "some_func" in vars(fake_pkg) + else: + # The classic mechanism leaves names unbound until first access + assert "some_func" not in vars(fake_pkg) + + # Either way, nothing is imported until first attribute access + assert "tests.fake_pkg.some_func" not in sys.modules + assert isinstance(fake_pkg.some_func, types.FunctionType) + assert "tests.fake_pkg.some_func" in sys.modules + + +def test_attach_native_keeps_existing_bindings(): + # A name already bound in the package namespace shadows the lazily + # attached one, on all Python versions. + name = "lazy_loader_test_existing_pkg" + mod = types.ModuleType(name) + mod.some_attr = "sentinel" + sys.modules[name] = mod + try: + getattr_, _, all_ = lazy.attach( + name, submod_attrs={"sub": ["some_attr", "other_attr"]} + ) + assert mod.some_attr == "sentinel" + assert all_ == ["other_attr", "some_attr"] + if NATIVE_LAZY_IMPORTS: + assert "other_attr" in vars(mod) + # Binding the proxies must not leave __builtins__ behind + assert "__builtins__" not in vars(mod) + # Unknown names raise AttributeError through the returned __getattr__ + with pytest.raises(AttributeError): + getattr_("unknown_attr") + finally: + del sys.modules[name] + + +def test_attach_rejects_non_identifier_names(): + # Names that are not identifiers must never reach the generated import + # statements of the native (PEP 810) path; the classic __getattr__ + # mechanism handles them as plain strings. + name = "lazy_loader_test_nonidentifier_pkg" + mod = types.ModuleType(name) + sys.modules[name] = mod + try: + evil = "nosuchmod import x\ninjected = 1\nlazy from victim.nosuchmod" + getattr_, _, _ = lazy.attach(name, submod_attrs={evil: ["x"]}) + assert "injected" not in vars(mod) + assert "x" not in vars(mod) + with pytest.raises(ImportError): + getattr_("x") + finally: + del sys.modules[name] + + +def test_attach_rejects_keyword_names(): + # Keywords are identifiers, but not valid in import statements, so the + # generated native (PEP 810) code fails to compile and attach() must fall + # back to the classic __getattr__ mechanism. + name = "lazy_loader_test_keyword_pkg" + mod = types.ModuleType(name) + sys.modules[name] = mod + try: + getattr_, _, all_ = lazy.attach(name, submod_attrs={"sub": ["class"]}) + assert all_ == ["class"] + assert "class" not in vars(mod) + with pytest.raises(ImportError): + getattr_("class") + finally: + del sys.modules[name] + + +def test_native_lazy_imports_detection_matches_syntax_support(): + # Detection must report on the interpreter in front of us, not on a + # version number: PEP 810 syntax is absent from 3.15.0a6 but present + # in 3.15.0rc2. + try: + compile("lazy import sys", "", "exec") + except SyntaxError: + assert not NATIVE_LAZY_IMPORTS + else: + assert NATIVE_LAZY_IMPORTS + + +def test_attach_falls_back_without_native_support(monkeypatch): + # Where the syntax is unavailable, attach() keeps the classic + # __getattr__ mechanism rather than binding anything up front. + monkeypatch.setattr(lazy, "_NATIVE_LAZY_IMPORTS", False) + name = "lazy_loader_test_disabled_pkg" + mod = types.ModuleType(name) + sys.modules[name] = mod + try: + lazy.attach(name, submod_attrs={"sub": ["some_attr"]}) + assert "some_attr" not in vars(mod) + finally: + del sys.modules[name] + + +def test_attach_falls_back_without_module(): + # attach() with a package name that is not in sys.modules cannot bind + # native proxies and must keep the classic __getattr__ mechanism. + getattr_, _, _ = lazy.attach( + "lazy_loader_test_not_a_module", submod_attrs={"sub": ["some_attr"]} + ) + with pytest.raises(ImportError): + getattr_("some_attr") + + +def test_attach_submodule_is_lazy(tmp_path, monkeypatch): + # Plain `submodules` go through `lazy from pkg import sub`, which resolves + # by looking `sub` up on `pkg` --- where the proxy being resolved is still + # bound. Check that this resolves to the submodule rather than to itself. + name = "lazy_loader_test_submodule_pkg" + pkg = tmp_path / name + pkg.mkdir() + (pkg / "__init__.py").write_text( + "import lazy_loader as lazy\n" + '__getattr__, __dir__, __all__ = lazy.attach(__name__, ["sub"])\n' + ) + (pkg / "sub.py").write_text("VALUE = 42\n") + monkeypatch.syspath_prepend(str(tmp_path)) + + mod = importlib.import_module(name) + try: + if NATIVE_LAZY_IMPORTS: + assert "sub" in vars(mod) + assert f"{name}.sub" not in sys.modules + assert mod.sub.VALUE == 42 + assert mod.sub is sys.modules[f"{name}.sub"] + finally: + for modname in [m for m in sys.modules if m.startswith(name)]: + del sys.modules[modname] + + +def test_attach_shadowing_submodule_stays_lazy(clean_fake_pkg): + # `x` is a function in `x/sub.py`, so importing `x.sub` makes the import + # machinery rebind `x` to the subpackage. Check the guard against that + # holds when the name is bound as a native proxy. + from tests import fake_pkg_submodule + + if NATIVE_LAZY_IMPORTS: + assert "x" in vars(fake_pkg_submodule) + assert isinstance(fake_pkg_submodule.x, types.FunctionType) + # Resolution must not leave the subpackage shadowing the function + assert isinstance(vars(fake_pkg_submodule)["x"], types.FunctionType) + + def test_attach_caches_resolved_attrs(clean_fake_pkg): from tests import fake_pkg - assert "aux_func" not in vars(fake_pkg) + if NATIVE_LAZY_IMPORTS: + # Bound as a native lazy proxy, which the interpreter reifies in place + assert "aux_func" in vars(fake_pkg) + assert "tests.fake_pkg.some_func" not in sys.modules + else: + assert "aux_func" not in vars(fake_pkg) aux_func = fake_pkg.aux_func # The resolved attribute is cached on the package, so later accesses # do not go through __getattr__ again