From 5c2b92e713340397391a1edfae259b8b1f820e59 Mon Sep 17 00:00:00 2001 From: Christian Tabedzki <35670232+tabedzki@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:54:26 -0400 Subject: [PATCH] fix: replace pkg_resources and otumat in plugin discovery pkg_resources was removed in setuptools 82, and otumat (archived) imports distutils, which Python 3.12+ only provides through setuptools. Use importlib.metadata for plugin discovery and inline otumat's hash_pkg and verify, so datajoint no longer depends on setuptools or otumat. The plugin system and its signature check are unchanged. Assisted-by: ClaudeCode:claude-opus-5-5 --- datajoint/plugin.py | 56 +++++++++++++++++----- pyproject.toml | 2 - tests/test_plugin.py | 112 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 153 insertions(+), 17 deletions(-) diff --git a/datajoint/plugin.py b/datajoint/plugin.py index 94070342d..b1fd3ca31 100644 --- a/datajoint/plugin.py +++ b/datajoint/plugin.py @@ -1,24 +1,55 @@ +import base64 +import hashlib import logging +from importlib.metadata import distribution, entry_points from pathlib import Path -import pkg_resources # requires setuptools<82 from cryptography.exceptions import InvalidSignature -from otumat import hash_pkg, verify +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import padding from .settings import config logger = logging.getLogger(__name__.split(".")[0]) +def _git_hash(text): + return hashlib.sha1(f"blob {len(text)}\0{text}".encode()).hexdigest() + + +def hash_pkg(*, pkgpath): + root = Path(pkgpath).absolute() + return _git_hash( + "".join( + f"100644 {_git_hash(path.read_text())} 0\t{path.relative_to(root.parent)}\n" + for path in sorted(root.rglob("*")) + if path.is_file() and "pycache" not in str(path) + ) + ) + + +def verify(*, pubkey_path, data, signature): + serialization.load_pem_public_key(Path(pubkey_path).read_bytes()).verify( + base64.b64decode(signature.encode()), + data.encode(), + padding.PSS( + mgf=padding.MGF1(hashes.SHA256()), salt_length=padding.PSS.MAX_LENGTH + ), + hashes.SHA256(), + ) + + def _update_error_stack(plugin_name): try: base_name = "datajoint" - base_meta = pkg_resources.get_distribution(base_name) - plugin_meta = pkg_resources.get_distribution(plugin_name) + base_meta = distribution(base_name) + plugin_meta = distribution(plugin_name) - data = hash_pkg(pkgpath=str(Path(plugin_meta.module_path, plugin_name))) - signature = plugin_meta.get_metadata(f"{plugin_name}.sig") - pubkey_path = str(Path(base_meta.egg_info, f"{base_name}.pub")) + data = hash_pkg(pkgpath=str(Path(plugin_meta.locate_file(""), plugin_name))) + signature = plugin_meta.read_text(f"{plugin_name}.sig") + if signature is None: + raise FileNotFoundError(f"{plugin_name}.sig") + pubkey_path = str(Path(base_meta._path).resolve() / f"{base_name}.pub") verify(pubkey_path=pubkey_path, data=data, signature=signature) logger.info(f"DataJoint verified plugin `{plugin_name}` detected.") return True @@ -28,17 +59,18 @@ def _update_error_stack(plugin_name): def _import_plugins(category): + eps = entry_points() + group = f"datajoint_plugins.{category}" + eps = eps.select(group=group) if hasattr(eps, "select") else eps.get(group, []) return { entry_point.name: dict( object=entry_point, - verified=_update_error_stack(entry_point.module_name.split(".")[0]), - ) - for entry_point in pkg_resources.iter_entry_points( - "datajoint_plugins.{}".format(category) + verified=_update_error_stack(entry_point.module.split(".")[0]), ) + for entry_point in eps if "plugin" not in config or category not in config["plugin"] - or entry_point.module_name.split(".")[0] in config["plugin"][category] + or entry_point.module.split(".")[0] in config["plugin"][category] } diff --git a/pyproject.toml b/pyproject.toml index 9ec0660db..22b010000 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,11 +18,9 @@ dependencies = [ "pydot", "minio>=7.0.0", "matplotlib", - "otumat", "faker", "cryptography", "urllib3", - "setuptools<82", # pkg_resources removed in 82.0.0 ] requires-python = ">=3.9,<4.0" authors = [ diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 7fd9aff22..b227a6f07 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -1,7 +1,14 @@ +import base64 +import os +import subprocess +import sys +from importlib.metadata import distribution from os import path -import pkg_resources import pytest +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import padding, rsa import datajoint.errors as djerr import datajoint.plugin as p @@ -10,8 +17,8 @@ @pytest.mark.skip(reason="marked for deprecation") def test_check_pubkey(): base_name = "datajoint" - base_meta = pkg_resources.get_distribution(base_name) - pubkey_meta = base_meta.get_metadata("{}.pub".format(base_name)) + base_meta = distribution(base_name) + pubkey_meta = base_meta.read_text("{}.pub".format(base_name)) with open( path.join(path.abspath(path.dirname(__file__)), "..", "datajoint.pub"), "r" @@ -60,3 +67,102 @@ def test_unverified_djerror(category="connection"): def test_unverified_djerror_type(): test_unverified_djerror(category="type") + + +def _fake_plugin(root, signature): + (root / "djfake").mkdir() + (root / "djfake" / "__init__.py").write_text("x = 1\n") + (root / "djfake-1.0.dist-info").mkdir() + (root / "djfake-1.0.dist-info" / "METADATA").write_text( + "Name: djfake\nVersion: 1.0\n" + ) + (root / "djfake-1.0.dist-info" / "entry_points.txt").write_text( + "[datajoint_plugins.connection]\nfake = djfake\n" + ) + if signature is not None: + (root / "djfake-1.0.dist-info" / "djfake.sig").write_text(signature) + + +@pytest.mark.parametrize("signature, verified", [("SIG", True), (None, False)]) +def test_import_plugins(tmp_path, monkeypatch, signature, verified): + _fake_plugin(tmp_path, signature) + monkeypatch.syspath_prepend(str(tmp_path)) + calls = {} + monkeypatch.setattr(p, "hash_pkg", lambda pkgpath: pkgpath) + monkeypatch.setattr(p, "verify", lambda **kwargs: calls.update(kwargs)) + plugins = p._import_plugins("connection") + assert plugins["fake"]["verified"] is verified + assert plugins["fake"]["object"].load().x == 1 + assert p._import_plugins("datatype") == {} + if verified: + assert calls["data"] == str(tmp_path / "djfake") + assert calls["signature"] == "SIG" + assert calls["pubkey_path"].endswith("datajoint.pub") + + +def test_import_without_pkg_resources_or_otumat(tmp_path): + _fake_plugin(tmp_path, "SIG") + code = ( + "import sys\n" + "class Block:\n" + " def find_spec(self, name, *args):\n" + " if name.split('.')[0] in ('pkg_resources', 'otumat', 'distutils'):\n" + " raise ImportError(name)\n" + "sys.meta_path.insert(0, Block())\n" + "import datajoint.plugin as p\n" + "print(p.connection_plugins['fake']['verified'])\n" + ) + env = dict( + os.environ, + PYTHONPATH=os.pathsep.join( + [str(tmp_path), path.dirname(path.dirname(__file__))] + ), + ) + result = subprocess.run( + [sys.executable, "-c", code], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + assert result.stdout.splitlines() == ["False"], result.stderr + + +def test_hash_pkg(tmp_path): + for name, text in { + "__init__.py": "", + "sub/data.txt": "café\n", + "__pycache__/a.pyc": "x", + }.items(): + (tmp_path / "pkg" / name).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / "pkg" / name).write_text(text) + assert ( + p.hash_pkg(pkgpath=str(tmp_path / "pkg")) + == "51ad1acd96ead2654ef409bb708c9c3b4b8c6333" + ) + assert ( + p.hash_pkg(pkgpath=str(tmp_path / "pkg" / "sub" / "missing")) + == "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" + ) + + +def test_verify(tmp_path): + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + (tmp_path / "datajoint.pub").write_bytes( + key.public_key().public_bytes( + serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo + ) + ) + pss = padding.PSS( + mgf=padding.MGF1(hashes.SHA256()), salt_length=padding.PSS.MAX_LENGTH + ) + signature = ( + base64.b64encode(key.sign(b"digest", pss, hashes.SHA256())).decode() + "\n" + ) + p.verify( + pubkey_path=str(tmp_path / "datajoint.pub"), data="digest", signature=signature + ) + with pytest.raises(InvalidSignature): + p.verify( + pubkey_path=str(tmp_path / "datajoint.pub"), data="", signature=signature + )