From 21f2e14e720f4dd9127fd254f15f6b07d8a34b0b Mon Sep 17 00:00:00 2001 From: AtomicGlance Date: Fri, 4 Sep 2026 09:33:39 +0330 Subject: [PATCH 1/9] Add subject ID discovery to Dandiset APIs --- dandi/dandiapi.py | 14 ++++++++++++++ dandi/dandiset.py | 17 +++++++++++++++++ dandi/tests/test_dandiapi.py | 23 +++++++++++++++++++++++ dandi/tests/test_dandiset.py | 14 ++++++++++++++ docs/source/modref/dandiapi.rst | 10 ++++++++++ docs/source/modref/dandiset.rst | 21 +++++++++++++++++++++ docs/source/modref/index.rst | 1 + 7 files changed, 100 insertions(+) create mode 100644 docs/source/modref/dandiset.rst diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index ccf17239b..251b7401d 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -1426,6 +1426,20 @@ def get_assets(self, order: str | None = None) -> Iterator[RemoteAsset]: f"No such version: {self.version_id!r} of Dandiset {self.identifier}" ) + def get_subject_ids(self) -> list[str]: + """Return sorted subject identifiers from top-level ``sub-*`` asset paths. + + The Archive API does not return directory objects, so asset records + are streamed in path order and only their paths are inspected. Asset + payloads and metadata are not downloaded. + """ + subject_ids = set[str]() + for asset in self.get_assets(order="path"): + parts = PurePosixPath(asset.path).parts + if len(parts) > 1 and parts[0].startswith("sub-") and len(parts[0]) > 4: + subject_ids.add(parts[0][4:]) + return sorted(subject_ids) + def get_asset(self, asset_id: str) -> RemoteAsset: """ Fetch the asset in this version of the Dandiset with the given asset diff --git a/dandi/dandiset.py b/dandi/dandiset.py index 22cbce892..a844c1f3f 100644 --- a/dandi/dandiset.py +++ b/dandi/dandiset.py @@ -168,6 +168,23 @@ def assets(self, allow_all: bool = False) -> AssetView: data[PurePosixPath(df.path)] = df return AssetView(data) + def get_subject_ids(self) -> list[str]: + """Return sorted subject identifiers from top-level ``sub-*`` directories. + + Only immediate children of the local Dandiset are inspected; files + inside subject directories are not read. + """ + return sorted( + { + path.name[4:] + for path in self.path_obj.iterdir() + if path.is_dir() + and not path.is_symlink() + and path.name.startswith("sub-") + and len(path.name) > 4 + } + ) + def metadata_file(self) -> DandisetMetadataFile: df = dandi_file(self._metadata_file_obj, dandiset_path=self.path) assert isinstance(df, DandisetMetadataFile) diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 8da28852e..58b59ad86 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -8,6 +8,7 @@ import random import re from shutil import rmtree +from types import SimpleNamespace from typing import Any import anys @@ -996,6 +997,28 @@ def _get_assets_with_path_prefix(prefix: str, **kw: Any) -> list[str]: ] +def test_remote_get_subject_ids(mocker: MockerFixture) -> None: + client = mocker.Mock(_instance_id="test") + dandiset = RemoteDandiset(client, "000001", version=DRAFT) + get_assets = mocker.patch.object( + dandiset, + "get_assets", + return_value=iter( + [ + SimpleNamespace(path="sub-mouse2/session/file.nwb"), + SimpleNamespace(path="sub-mouse1/file.nwb"), + SimpleNamespace(path="sub-mouse2/other/file.nwb"), + SimpleNamespace(path="sub-root.nwb"), + SimpleNamespace(path="other/file.nwb"), + SimpleNamespace(path="sub-/file.nwb"), + ] + ), + ) + + assert dandiset.get_subject_ids() == ["mouse1", "mouse2"] + get_assets.assert_called_once_with(order="path") + + def test_get_assets_by_glob(text_dandiset: SampleDandiset) -> None: assert sorted( asset.path for asset in text_dandiset.dandiset.get_assets_by_glob("*a*.txt") diff --git a/dandi/tests/test_dandiset.py b/dandi/tests/test_dandiset.py index 39653345a..8a0bf6ae2 100644 --- a/dandi/tests/test_dandiset.py +++ b/dandi/tests/test_dandiset.py @@ -1,3 +1,5 @@ +from pathlib import Path + from ..dandiset import Dandiset @@ -6,3 +8,15 @@ def test_get_dandiset_record() -> None: # Should have only header with "DO NOT EDIT" assert out.startswith("# DO NOT EDIT") assert "000000" in out + + +def test_get_subject_ids(tmp_path: Path) -> None: + (tmp_path / "dandiset.yaml").write_text("identifier: '000001'\n") + (tmp_path / "sub-mouse2").mkdir() + (tmp_path / "sub-mouse2" / "record.nwb").touch() + (tmp_path / "sub-mouse1").mkdir() + (tmp_path / "sub-").mkdir() + (tmp_path / "sub-root.nwb").touch() + (tmp_path / "subjects").mkdir() + + assert Dandiset(tmp_path).get_subject_ids() == ["mouse1", "mouse2"] diff --git a/docs/source/modref/dandiapi.rst b/docs/source/modref/dandiapi.rst index 8212711d0..46d6a17a0 100644 --- a/docs/source/modref/dandiapi.rst +++ b/docs/source/modref/dandiapi.rst @@ -36,6 +36,16 @@ be passed to functions of pynwb etc. You can see more usages of DANDI API to assist with data streaming at `PyNWB: Streaming NWB files `_. +To discover the subject labels represented by a remote Dandiset, use +``RemoteDandiset.get_subject_ids()``. It streams asset paths ordered by path +and does not download asset payloads or metadata: + +.. code-block:: python + + with DandiAPIClient() as client: + dandiset = client.get_dandiset("000001") + print(dandiset.get_subject_ids()) + Client ------ diff --git a/docs/source/modref/dandiset.rst b/docs/source/modref/dandiset.rst new file mode 100644 index 000000000..ea0e323bd --- /dev/null +++ b/docs/source/modref/dandiset.rst @@ -0,0 +1,21 @@ +.. module:: dandi.dandiset + +``dandi.dandiset`` +================== + +This module provides the local Dandiset API. A local Dandiset can report the +subject labels represented by its top-level ``sub-*`` directories without +opening or scanning the files within them: + +.. code-block:: python + + from dandi.dandiset import Dandiset + + dandiset = Dandiset("/data/my-dandiset") + print(dandiset.get_subject_ids()) + +.. autoclass:: Dandiset() + :members: + +.. autoclass:: AssetView() + :members: diff --git a/docs/source/modref/index.rst b/docs/source/modref/index.rst index 373b48502..ce215299c 100644 --- a/docs/source/modref/index.rst +++ b/docs/source/modref/index.rst @@ -32,6 +32,7 @@ Object-oriented interfaces to manipulate Dandisets and assets on a DANDI instanc .. toctree:: dandiarchive + dandiset Low-level user interfaces ========================= From 9a8f7590481a144feb66d078347bf43427b53557 Mon Sep 17 00:00:00 2001 From: AtomicGlance Date: Fri, 4 Sep 2026 16:36:38 +0330 Subject: [PATCH 2/9] Harden subject ID discovery contract --- dandi/consts.py | 6 ++++++ dandi/dandiapi.py | 12 ++++++++--- dandi/dandiset.py | 35 ++++++++++++++++++++------------- dandi/organize.py | 8 +++++--- dandi/tests/test_dandiapi.py | 22 +++++++++++++++++++++ dandi/tests/test_dandiset.py | 25 ++++++++++++++++++++++- dandi/utils.py | 16 ++++++++++++++- docs/source/modref/dandiapi.rst | 4 +++- docs/source/modref/dandiset.rst | 6 ++++-- 9 files changed, 109 insertions(+), 25 deletions(-) diff --git a/dandi/consts.py b/dandi/consts.py index baca5465c..1d6f618cf 100644 --- a/dandi/consts.py +++ b/dandi/consts.py @@ -14,6 +14,12 @@ from enum import Enum, StrEnum import os +# Labels used in DANDI-organized subject and session path components. Keep +# these expressions here so path discovery and path validation use the same +# syntax. +DANDI_LABEL_REGEX = r"[^_*\\/<>:|\"'?%@;.]+" +DANDI_SUBJECT_FOLDER_REGEX = rf"sub-{DANDI_LABEL_REGEX}" + #: A list of metadata fields which dandi extracts from .nwb files. #: Additional fields (such as ``number_of_*``) might be added by #: `get_metadata()` diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 251b7401d..510e8a12a 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -63,6 +63,7 @@ is_interactive, is_page2_url, joinurl, + parse_dandi_subject_dirname, ) if TYPE_CHECKING: @@ -1431,13 +1432,18 @@ def get_subject_ids(self) -> list[str]: The Archive API does not return directory objects, so asset records are streamed in path order and only their paths are inspected. Asset - payloads and metadata are not downloaded. + payloads and metadata are not downloaded. Runtime is linear in the + number of assets in the Dandiset. + + .. versionadded:: 0.79.0 """ subject_ids = set[str]() for asset in self.get_assets(order="path"): parts = PurePosixPath(asset.path).parts - if len(parts) > 1 and parts[0].startswith("sub-") and len(parts[0]) > 4: - subject_ids.add(parts[0][4:]) + if len(parts) > 1: + subject_id = parse_dandi_subject_dirname(parts[0]) + if subject_id is not None: + subject_ids.add(subject_id) return sorted(subject_ids) def get_asset(self, asset_id: str) -> RemoteAsset: diff --git a/dandi/dandiset.py b/dandi/dandiset.py index a844c1f3f..d3b378b9b 100644 --- a/dandi/dandiset.py +++ b/dandi/dandiset.py @@ -12,7 +12,13 @@ from . import get_logger from .consts import dandiset_metadata_file from .files import DandisetMetadataFile, LocalAsset, dandi_file, find_dandi_files -from .utils import find_parent_directory_containing, under_paths, yaml_dump, yaml_load +from .utils import ( + find_parent_directory_containing, + parse_dandi_subject_dirname, + under_paths, + yaml_dump, + yaml_load, +) if TYPE_CHECKING: from typing_extensions import Self @@ -169,21 +175,22 @@ def assets(self, allow_all: bool = False) -> AssetView: return AssetView(data) def get_subject_ids(self) -> list[str]: - """Return sorted subject identifiers from top-level ``sub-*`` directories. + """Return sorted IDs from populated top-level subject directories. - Only immediate children of the local Dandiset are inspected; files - inside subject directories are not read. + Directory entries are inspected, but file contents are not read. + Empty and symlinked subject directories are ignored so the result + describes subjects that can also be represented by remote assets. + + .. versionadded:: 0.79.0 """ - return sorted( - { - path.name[4:] - for path in self.path_obj.iterdir() - if path.is_dir() - and not path.is_symlink() - and path.name.startswith("sub-") - and len(path.name) > 4 - } - ) + subject_ids = set[str]() + for path in self.path_obj.iterdir(): + if not path.is_dir() or path.is_symlink(): + continue + subject_id = parse_dandi_subject_dirname(path.name) + if subject_id is not None and any(p.is_file() for p in path.rglob("*")): + subject_ids.add(subject_id) + return sorted(subject_ids) def metadata_file(self) -> DandisetMetadataFile: df = dandi_file(self._metadata_file_obj, dandiset_path=self.path) diff --git a/dandi/organize.py b/dandi/organize.py index 896702f8a..b15cb6141 100644 --- a/dandi/organize.py +++ b/dandi/organize.py @@ -27,7 +27,11 @@ import ruamel.yaml from . import get_logger -from .consts import dandi_layout_fields +from .consts import ( + DANDI_LABEL_REGEX as LABELREGEX, + DANDI_SUBJECT_FOLDER_REGEX as ORGANIZED_FOLDER_REGEX, + dandi_layout_fields, +) from .dandiset import Dandiset from .exceptions import OrganizeImpossibleError from .utils import ( @@ -1149,7 +1153,6 @@ def msg_(msg, n, cond=None): ) -LABELREGEX = r"[^_*\\/<>:|\"'?%@;.]+" ORGANIZED_FILENAME_REGEX = ( rf"sub-{LABELREGEX}" rf"(_ses-{LABELREGEX})?" @@ -1157,7 +1160,6 @@ def msg_(msg, n, cond=None): r"(_[a-z]+(\+[a-z]+)*)?" r"\.nwb" ) -ORGANIZED_FOLDER_REGEX = rf"sub-{LABELREGEX}" def validate_organized_path( diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 58b59ad86..9ecddd387 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -1011,6 +1011,7 @@ def test_remote_get_subject_ids(mocker: MockerFixture) -> None: SimpleNamespace(path="sub-root.nwb"), SimpleNamespace(path="other/file.nwb"), SimpleNamespace(path="sub-/file.nwb"), + SimpleNamespace(path="sub-bad_name/file.nwb"), ] ), ) @@ -1019,6 +1020,27 @@ def test_remote_get_subject_ids(mocker: MockerFixture) -> None: get_assets.assert_called_once_with(order="path") +def test_remote_get_subject_ids_empty(mocker: MockerFixture) -> None: + client = mocker.Mock(_instance_id="test") + dandiset = RemoteDandiset(client, "000001", version=DRAFT) + mocker.patch.object(dandiset, "get_assets", return_value=iter(())) + + assert dandiset.get_subject_ids() == [] + + +def test_remote_get_subject_ids_propagates_api_errors( + mocker: MockerFixture, +) -> None: + client = mocker.Mock(_instance_id="test") + dandiset = RemoteDandiset(client, "000001", version=DRAFT) + mocker.patch.object( + dandiset, "get_assets", side_effect=RuntimeError("pagination failed") + ) + + with pytest.raises(RuntimeError, match="pagination failed"): + dandiset.get_subject_ids() + + def test_get_assets_by_glob(text_dandiset: SampleDandiset) -> None: assert sorted( asset.path for asset in text_dandiset.dandiset.get_assets_by_glob("*a*.txt") diff --git a/dandi/tests/test_dandiset.py b/dandi/tests/test_dandiset.py index 8a0bf6ae2..6edff90d0 100644 --- a/dandi/tests/test_dandiset.py +++ b/dandi/tests/test_dandiset.py @@ -1,5 +1,7 @@ from pathlib import Path +from pytest_mock import MockerFixture + from ..dandiset import Dandiset @@ -10,13 +12,34 @@ def test_get_dandiset_record() -> None: assert "000000" in out -def test_get_subject_ids(tmp_path: Path) -> None: +def test_get_subject_ids(tmp_path: Path, mocker: MockerFixture) -> None: (tmp_path / "dandiset.yaml").write_text("identifier: '000001'\n") (tmp_path / "sub-mouse2").mkdir() (tmp_path / "sub-mouse2" / "record.nwb").touch() (tmp_path / "sub-mouse1").mkdir() + (tmp_path / "sub-mouse1" / "session").mkdir() + (tmp_path / "sub-mouse1" / "session" / "record.nwb").touch() (tmp_path / "sub-").mkdir() + (tmp_path / "sub-bad_name").mkdir() + (tmp_path / "sub-bad_name" / "record.nwb").touch() + (tmp_path / "sub-empty").mkdir() + linked = tmp_path / "sub-linked" + linked.mkdir() + (linked / "record.nwb").touch() (tmp_path / "sub-root.nwb").touch() (tmp_path / "subjects").mkdir() + original_is_symlink = Path.is_symlink + mocker.patch.object( + Path, + "is_symlink", + autospec=True, + side_effect=lambda path: path == linked or original_is_symlink(path), + ) assert Dandiset(tmp_path).get_subject_ids() == ["mouse1", "mouse2"] + + +def test_get_subject_ids_empty(tmp_path: Path) -> None: + (tmp_path / "dandiset.yaml").write_text("identifier: '000001'\n") + + assert Dandiset(tmp_path).get_subject_ids() == [] diff --git a/dandi/utils.py b/dandi/utils.py index cd6ea7afd..c002011c2 100644 --- a/dandi/utils.py +++ b/dandi/utils.py @@ -35,7 +35,12 @@ from yarl import URL from . import __version__, get_logger -from .consts import DandiInstance, known_instances, known_instances_rev +from .consts import ( + DANDI_SUBJECT_FOLDER_REGEX, + DandiInstance, + known_instances, + known_instances_rev, +) from .exceptions import BadCliVersionError, CliVersionTooOldError AnyPath = Union[str, Path] @@ -43,6 +48,15 @@ lgr = get_logger() + +def parse_dandi_subject_dirname(name: str) -> str | None: + """Return the ID encoded by a valid DANDI ``sub-*`` directory name.""" + + if re.fullmatch(DANDI_SUBJECT_FOLDER_REGEX, name) is None: + return None + return name[4:] + + _sys_excepthook = sys.excepthook # Just in case we ever need original one # diff --git a/docs/source/modref/dandiapi.rst b/docs/source/modref/dandiapi.rst index 46d6a17a0..5431ed382 100644 --- a/docs/source/modref/dandiapi.rst +++ b/docs/source/modref/dandiapi.rst @@ -38,7 +38,9 @@ You can see more usages of DANDI API to assist with data streaming at To discover the subject labels represented by a remote Dandiset, use ``RemoteDandiset.get_subject_ids()``. It streams asset paths ordered by path -and does not download asset payloads or metadata: +and does not download asset payloads or metadata. This is an ``O(number of +assets)`` operation because the Archive does not currently provide distinct +top-level path prefixes: .. code-block:: python diff --git a/docs/source/modref/dandiset.rst b/docs/source/modref/dandiset.rst index ea0e323bd..35e41a3e7 100644 --- a/docs/source/modref/dandiset.rst +++ b/docs/source/modref/dandiset.rst @@ -4,8 +4,10 @@ ================== This module provides the local Dandiset API. A local Dandiset can report the -subject labels represented by its top-level ``sub-*`` directories without -opening or scanning the files within them: +subject labels represented by populated, valid top-level ``sub-*`` directories. +It walks directory entries to establish that a subject contains a file, but it +does not open file contents. Empty and symlinked subject directories are +ignored: .. code-block:: python From 4ab8c90870b35e4328a294ae03d8f869419374ee Mon Sep 17 00:00:00 2001 From: AtomicGlance Date: Fri, 4 Sep 2026 16:55:57 +0330 Subject: [PATCH 3/9] Keep subject parsing change focused --- dandi/consts.py | 6 ------ dandi/organize.py | 8 +++----- dandi/utils.py | 11 ++++------- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/dandi/consts.py b/dandi/consts.py index 1d6f618cf..baca5465c 100644 --- a/dandi/consts.py +++ b/dandi/consts.py @@ -14,12 +14,6 @@ from enum import Enum, StrEnum import os -# Labels used in DANDI-organized subject and session path components. Keep -# these expressions here so path discovery and path validation use the same -# syntax. -DANDI_LABEL_REGEX = r"[^_*\\/<>:|\"'?%@;.]+" -DANDI_SUBJECT_FOLDER_REGEX = rf"sub-{DANDI_LABEL_REGEX}" - #: A list of metadata fields which dandi extracts from .nwb files. #: Additional fields (such as ``number_of_*``) might be added by #: `get_metadata()` diff --git a/dandi/organize.py b/dandi/organize.py index b15cb6141..896702f8a 100644 --- a/dandi/organize.py +++ b/dandi/organize.py @@ -27,11 +27,7 @@ import ruamel.yaml from . import get_logger -from .consts import ( - DANDI_LABEL_REGEX as LABELREGEX, - DANDI_SUBJECT_FOLDER_REGEX as ORGANIZED_FOLDER_REGEX, - dandi_layout_fields, -) +from .consts import dandi_layout_fields from .dandiset import Dandiset from .exceptions import OrganizeImpossibleError from .utils import ( @@ -1153,6 +1149,7 @@ def msg_(msg, n, cond=None): ) +LABELREGEX = r"[^_*\\/<>:|\"'?%@;.]+" ORGANIZED_FILENAME_REGEX = ( rf"sub-{LABELREGEX}" rf"(_ses-{LABELREGEX})?" @@ -1160,6 +1157,7 @@ def msg_(msg, n, cond=None): r"(_[a-z]+(\+[a-z]+)*)?" r"\.nwb" ) +ORGANIZED_FOLDER_REGEX = rf"sub-{LABELREGEX}" def validate_organized_path( diff --git a/dandi/utils.py b/dandi/utils.py index c002011c2..f6bcf6027 100644 --- a/dandi/utils.py +++ b/dandi/utils.py @@ -35,12 +35,7 @@ from yarl import URL from . import __version__, get_logger -from .consts import ( - DANDI_SUBJECT_FOLDER_REGEX, - DandiInstance, - known_instances, - known_instances_rev, -) +from .consts import DandiInstance, known_instances, known_instances_rev from .exceptions import BadCliVersionError, CliVersionTooOldError AnyPath = Union[str, Path] @@ -52,7 +47,9 @@ def parse_dandi_subject_dirname(name: str) -> str | None: """Return the ID encoded by a valid DANDI ``sub-*`` directory name.""" - if re.fullmatch(DANDI_SUBJECT_FOLDER_REGEX, name) is None: + # Match the label syntax used by ``dandi organize`` without importing that + # module, which imports this utility module itself. + if re.fullmatch(r"sub-[^_*\\/<>:|\"'?%@;.]+", name) is None: return None return name[4:] From c8369058dfb0b70759c75b35079576ea47976b9d Mon Sep 17 00:00:00 2001 From: AtomicGlance Date: Fri, 4 Sep 2026 20:13:13 +0330 Subject: [PATCH 4/9] Use path endpoint for remote subject discovery --- dandi/dandiapi.py | 28 +++++++++++--------- dandi/tests/test_dandiapi.py | 45 ++++++++++++++++++--------------- docs/source/modref/dandiapi.rst | 6 ++--- 3 files changed, 42 insertions(+), 37 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 510e8a12a..24be6a024 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -1430,21 +1430,25 @@ def get_assets(self, order: str | None = None) -> Iterator[RemoteAsset]: def get_subject_ids(self) -> list[str]: """Return sorted subject identifiers from top-level ``sub-*`` asset paths. - The Archive API does not return directory objects, so asset records - are streamed in path order and only their paths are inspected. Asset - payloads and metadata are not downloaded. Runtime is linear in the - number of assets in the Dandiset. + The Archive's path endpoint returns the immediate children of the + Dandiset root, so discovery does not require listing every asset. + Asset payloads and metadata are not downloaded. .. versionadded:: 0.79.0 """ - subject_ids = set[str]() - for asset in self.get_assets(order="path"): - parts = PurePosixPath(asset.path).parts - if len(parts) > 1: - subject_id = parse_dandi_subject_dirname(parts[0]) - if subject_id is not None: - subject_ids.add(subject_id) - return sorted(subject_ids) + try: + paths = self.client.paginate(f"{self.version_api_path}assets/paths/") + return sorted( + subject_id + for item in paths + if item["asset"] is None + if (subject_id := parse_dandi_subject_dirname(item["path"])) + is not None + ) + except HTTP404Error: + raise NotFoundError( + f"No such version: {self.version_id!r} of Dandiset {self.identifier}" + ) def get_asset(self, asset_id: str) -> RemoteAsset: """ diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 9ecddd387..292519f2b 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -8,7 +8,6 @@ import random import re from shutil import rmtree -from types import SimpleNamespace from typing import Any import anys @@ -41,7 +40,7 @@ VersionStatus, ) from ..download import download -from ..exceptions import NotFoundError, SchemaVersionError +from ..exceptions import HTTP404Error, NotFoundError, SchemaVersionError from ..files import GenericAsset, dandi_file from ..utils import list_paths @@ -1000,30 +999,27 @@ def _get_assets_with_path_prefix(prefix: str, **kw: Any) -> list[str]: def test_remote_get_subject_ids(mocker: MockerFixture) -> None: client = mocker.Mock(_instance_id="test") dandiset = RemoteDandiset(client, "000001", version=DRAFT) - get_assets = mocker.patch.object( - dandiset, - "get_assets", - return_value=iter( - [ - SimpleNamespace(path="sub-mouse2/session/file.nwb"), - SimpleNamespace(path="sub-mouse1/file.nwb"), - SimpleNamespace(path="sub-mouse2/other/file.nwb"), - SimpleNamespace(path="sub-root.nwb"), - SimpleNamespace(path="other/file.nwb"), - SimpleNamespace(path="sub-/file.nwb"), - SimpleNamespace(path="sub-bad_name/file.nwb"), - ] - ), + client.paginate.return_value = iter( + [ + {"path": "sub-mouse2", "asset": None}, + {"path": "sub-mouse1", "asset": None}, + {"path": "sub-root", "asset": {"asset_id": "a"}}, + {"path": "other", "asset": None}, + {"path": "sub-", "asset": None}, + {"path": "sub-bad_name", "asset": None}, + ] ) assert dandiset.get_subject_ids() == ["mouse1", "mouse2"] - get_assets.assert_called_once_with(order="path") + client.paginate.assert_called_once_with( + "/dandisets/000001/versions/draft/assets/paths/" + ) def test_remote_get_subject_ids_empty(mocker: MockerFixture) -> None: client = mocker.Mock(_instance_id="test") dandiset = RemoteDandiset(client, "000001", version=DRAFT) - mocker.patch.object(dandiset, "get_assets", return_value=iter(())) + client.paginate.return_value = iter(()) assert dandiset.get_subject_ids() == [] @@ -1033,14 +1029,21 @@ def test_remote_get_subject_ids_propagates_api_errors( ) -> None: client = mocker.Mock(_instance_id="test") dandiset = RemoteDandiset(client, "000001", version=DRAFT) - mocker.patch.object( - dandiset, "get_assets", side_effect=RuntimeError("pagination failed") - ) + client.paginate.side_effect = RuntimeError("pagination failed") with pytest.raises(RuntimeError, match="pagination failed"): dandiset.get_subject_ids() +def test_remote_get_subject_ids_missing_version(mocker: MockerFixture) -> None: + client = mocker.Mock(_instance_id="test") + dandiset = RemoteDandiset(client, "000001", version=DRAFT) + client.paginate.side_effect = HTTP404Error("not found") + + with pytest.raises(NotFoundError, match="No such version: 'draft'"): + dandiset.get_subject_ids() + + def test_get_assets_by_glob(text_dandiset: SampleDandiset) -> None: assert sorted( asset.path for asset in text_dandiset.dandiset.get_assets_by_glob("*a*.txt") diff --git a/docs/source/modref/dandiapi.rst b/docs/source/modref/dandiapi.rst index 5431ed382..5ee1e0c7b 100644 --- a/docs/source/modref/dandiapi.rst +++ b/docs/source/modref/dandiapi.rst @@ -37,10 +37,8 @@ You can see more usages of DANDI API to assist with data streaming at `PyNWB: Streaming NWB files `_. To discover the subject labels represented by a remote Dandiset, use -``RemoteDandiset.get_subject_ids()``. It streams asset paths ordered by path -and does not download asset payloads or metadata. This is an ``O(number of -assets)`` operation because the Archive does not currently provide distinct -top-level path prefixes: +``RemoteDandiset.get_subject_ids()``. It queries the Archive's efficient +top-level path endpoint and does not download asset payloads or metadata: .. code-block:: python From d7e4aa597e23e201e9b4525ff9c7131f5e19942e Mon Sep 17 00:00:00 2001 From: AtomicGlance Date: Tue, 15 Sep 2026 02:25:30 +0330 Subject: [PATCH 5/9] Expose Dandiset directory paths instead of subject-specific APIs --- dandi/consts.py | 11 ++ dandi/dandiapi.py | 151 +++++++++++++--- dandi/dandiset.py | 100 ++++++++--- dandi/organize.py | 17 +- dandi/tests/test_dandiapi.py | 50 +----- dandi/tests/test_dandiset.py | 37 ---- dandi/tests/test_dandiset_paths.py | 265 +++++++++++++++++++++++++++++ dandi/utils.py | 11 -- docs/source/modref/dandiapi.rst | 57 +++++-- docs/source/modref/dandiset.rst | 23 --- docs/source/modref/index.rst | 1 - 11 files changed, 537 insertions(+), 186 deletions(-) create mode 100644 dandi/tests/test_dandiset_paths.py delete mode 100644 docs/source/modref/dandiset.rst diff --git a/dandi/consts.py b/dandi/consts.py index baca5465c..82ee0b7cf 100644 --- a/dandi/consts.py +++ b/dandi/consts.py @@ -266,3 +266,14 @@ def urls(self) -> Iterator[str]: #: exFAT and some network filesystems truncate, and FAT rounds to a multiple #: of two seconds. See https://github.com/dandi/dandi-cli/issues/1907 MTIME_TOLERANCE = 2.0 + + +LABELREGEX = r"[^_*\\/<>:|\"'?%@;.]+" +ORGANIZED_FILENAME_REGEX = ( + rf"sub-{LABELREGEX}" + rf"(_ses-{LABELREGEX})?" + rf"(_(tis|slice|cell|desc|probe|obj)-{LABELREGEX})*" + r"(_[a-z]+(\+[a-z]+)*)?" + r"\.nwb" +) +ORGANIZED_FOLDER_REGEX = rf"sub-{LABELREGEX}" diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 24be6a024..96d6d67f3 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -15,7 +15,7 @@ from abc import ABC, abstractmethod from collections.abc import Callable, Iterable, Iterator, Sequence from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime from enum import Enum from fnmatch import fnmatchcase @@ -52,7 +52,7 @@ ) from .exceptions import HTTP404Error, NotFoundError, SchemaVersionError from .keyring_utils import keyring_lookup, keyring_save -from .misctypes import Digest, RemoteReadableAsset +from .misctypes import BasePath, Digest, RemoteReadableAsset from .utils import ( USER_AGENT, check_dandi_version, @@ -63,7 +63,6 @@ is_interactive, is_page2_url, joinurl, - parse_dandi_subject_dirname, ) if TYPE_CHECKING: @@ -1427,28 +1426,19 @@ def get_assets(self, order: str | None = None) -> Iterator[RemoteAsset]: f"No such version: {self.version_id!r} of Dandiset {self.identifier}" ) - def get_subject_ids(self) -> list[str]: - """Return sorted subject identifiers from top-level ``sub-*`` asset paths. + def get_path(self, path: str = "") -> RemoteDandisetPath: + """Return a lazy path for browsing this version's asset directories. - The Archive's path endpoint returns the immediate children of the - Dandiset root, so discovery does not require listing every asset. - Asset payloads and metadata are not downloaded. + Listing a directory uses the paginated `/assets/paths/` endpoint. + Listed children retain their type, recursive count and size, so reading + those properties makes no further requests. An arbitrary unlisted path + requires a listing of its parent to determine whether it exists. - .. versionadded:: 0.79.0 + Path objects cache listings; call this method again to see later changes. + + .. versionadded:: 0.80.0 """ - try: - paths = self.client.paginate(f"{self.version_api_path}assets/paths/") - return sorted( - subject_id - for item in paths - if item["asset"] is None - if (subject_id := parse_dandi_subject_dirname(item["path"])) - is not None - ) - except HTTP404Error: - raise NotFoundError( - f"No such version: {self.version_id!r} of Dandiset {self.identifier}" - ) + return RemoteDandisetPath(parts=(), dandiset=self) / path def get_asset(self, asset_id: str) -> RemoteAsset: """ @@ -2374,3 +2364,120 @@ class ZarrEntryServerData(BaseModel): last_modified: datetime = Field(alias="LastModified") etag: str = Field(alias="ETag") size: int = Field(alias="Size") + + +@dataclass +class RemoteDandisetPath(BasePath): + """A cached view of an asset or virtual directory in a Dandiset version. + + Zarr assets are leaves, just like blob assets. Their internal chunks are not + Dandiset children. Retrieving a full asset with `get_asset` makes a separate + request because the directory endpoint only supplies its identifier and URL. + + .. versionadded:: 0.80.0 + """ + + #: The Dandiset version containing this path. + dandiset: RemoteDandiset + _entry: dict[str, Any] | None = field(default=None, repr=False, compare=False) + _children: list[RemoteDandisetPath] | None = field( + default=None, repr=False, compare=False + ) + _missing: bool = field(default=False, repr=False, compare=False) + + def _get_subpath(self, name: str) -> RemoteDandisetPath: + if not name or "/" in name: + raise ValueError(f"Invalid path component: {name!r}") + if name == ".": + return self + if name == "..": + return self.parent + return type(self)(parts=(*self.parts, name), dandiset=self.dandiset) + + @property + def parent(self) -> RemoteDandisetPath: + return type(self)(parts=self.parts[:-1], dandiset=self.dandiset) + + def _list_entries(self, prefix: str) -> Iterator[dict[str, Any]]: + yield from self.dandiset.client.paginate( + f"{self.dandiset.version_api_path}assets/paths/", + params={"path_prefix": prefix}, + ) + + def _resolve(self) -> dict[str, Any] | None: + if self._entry is None and not self._missing: + try: + if self.is_root(): + self._load_children() + self._entry = { + "asset": None, + "aggregate_files": sum( + c.aggregate_files for c in self._children or [] + ), + "aggregate_size": sum(c.size for c in self._children or []), + } + else: + self._entry = next( + ( + e + for e in self._list_entries(str(self.parent)) + if e["path"] == str(self) + ), + None, + ) + except HTTP404Error: + self._missing = True + if self._entry is None: + self._missing = True + return self._entry + + def _require_entry(self) -> dict[str, Any]: + entry = self._resolve() + if entry is None: + raise NotFoundError(f"No such Dandiset path: {str(self)!r}") + return entry + + def exists(self) -> bool: + return self._resolve() is not None + + def is_file(self) -> bool: + entry = self._resolve() + return entry is not None and entry["asset"] is not None + + def is_dir(self) -> bool: + entry = self._resolve() + return entry is not None and entry["asset"] is None + + def _load_children(self) -> None: + if self._children is None: + self._children = [ + type(self)( + parts=tuple(entry["path"].split("/")), + dandiset=self.dandiset, + _entry=entry, + ) + for entry in self._list_entries(str(self)) + ] + + def iterdir(self) -> Iterator[RemoteDandisetPath]: + if self._require_entry()["asset"] is not None: + raise NotADirectoryError(str(self)) + self._load_children() + yield from self._children or [] + + @property + def aggregate_files(self) -> int: + """The recursive number of assets under this path (one for an asset).""" + return int(self._require_entry()["aggregate_files"]) + + @property + def size(self) -> int: + """The recursive size in bytes, as reported by the Archive.""" + return int(self._require_entry()["aggregate_size"]) + + def get_asset(self) -> RemoteAsset: + """Fetch the full asset record; directories raise IsADirectoryError.""" + asset = self._require_entry()["asset"] + if asset is None: + raise IsADirectoryError(str(self)) + return self.dandiset.get_asset(asset["asset_id"]) diff --git a/dandi/dandiset.py b/dandi/dandiset.py index d3b378b9b..f600a326f 100644 --- a/dandi/dandiset.py +++ b/dandi/dandiset.py @@ -1,4 +1,5 @@ """Classes/utilities for support of a dandiset""" + from __future__ import annotations from collections.abc import Iterable, Iterator @@ -11,14 +12,10 @@ from . import get_logger from .consts import dandiset_metadata_file +from .exceptions import NotFoundError from .files import DandisetMetadataFile, LocalAsset, dandi_file, find_dandi_files -from .utils import ( - find_parent_directory_containing, - parse_dandi_subject_dirname, - under_paths, - yaml_dump, - yaml_load, -) +from .misctypes import BasePath +from .utils import find_parent_directory_containing, under_paths, yaml_dump, yaml_load if TYPE_CHECKING: from typing_extensions import Self @@ -174,23 +171,17 @@ def assets(self, allow_all: bool = False) -> AssetView: data[PurePosixPath(df.path)] = df return AssetView(data) - def get_subject_ids(self) -> list[str]: - """Return sorted IDs from populated top-level subject directories. + def get_path(self, path: str = "") -> LocalDandisetPath: + """Browse a snapshot of the Dandiset's discoverable assets. - Directory entries are inspected, but file contents are not read. - Empty and symlinked subject directories are ignored so the result - describes subjects that can also be represented by remote assets. + Discovery runs once for the whole Dandiset and includes generic assets. + Empty and hidden directories are excluded by normal discovery rules; + Zarr directories are represented as single assets. Descendants share + the snapshot. Call this method again to refresh it. - .. versionadded:: 0.79.0 + .. versionadded:: 0.80.0 """ - subject_ids = set[str]() - for path in self.path_obj.iterdir(): - if not path.is_dir() or path.is_symlink(): - continue - subject_id = parse_dandi_subject_dirname(path.name) - if subject_id is not None and any(p.is_file() for p in path.rglob("*")): - subject_ids.add(subject_id) - return sorted(subject_ids) + return LocalDandisetPath(parts=(), assets=self.assets(allow_all=True)) / path def metadata_file(self) -> DandisetMetadataFile: df = dandi_file(self._metadata_file_obj, dandiset_path=self.path) @@ -216,3 +207,70 @@ def under_paths(self, paths: Iterable[str | PurePath]) -> Iterator[LocalAsset]: # contain '.' or '..' for p in under_paths(self.data.keys(), paths): yield self.data[p] + + +@dataclass +class LocalDandisetPath(BasePath): + """An asset or directory in a local Dandiset discovery snapshot. + + .. versionadded:: 0.80.0 + """ + + #: Shared discovery results, including generic assets. + assets: AssetView + + def _get_subpath(self, name: str) -> LocalDandisetPath: + if not name or "/" in name: + raise ValueError(f"Invalid path component: {name!r}") + if name == ".": + return self + if name == "..": + return self.parent + return type(self)(parts=(*self.parts, name), assets=self.assets) + + @property + def parent(self) -> LocalDandisetPath: + return type(self)(parts=self.parts[:-1], assets=self.assets) + + def _descendants(self) -> Iterator[LocalAsset]: + yield from self.assets.under_paths([PurePosixPath(str(self))]) + + def exists(self) -> bool: + return self.is_root() or next(self._descendants(), None) is not None + + def is_file(self) -> bool: + return PurePosixPath(str(self)) in self.assets.data + + def is_dir(self) -> bool: + return self.exists() and not self.is_file() + + def iterdir(self) -> Iterator[LocalDandisetPath]: + if not self.exists(): + raise NotFoundError(f"No such Dandiset path: {str(self)!r}") + if self.is_file(): + raise NotADirectoryError(str(self)) + names = {a.path.split("/")[len(self.parts)] for a in self._descendants()} + for name in sorted(names): + yield self / name + + @property + def aggregate_files(self) -> int: + """The recursive number of discoverable assets.""" + if not self.exists(): + raise NotFoundError(f"No such Dandiset path: {str(self)!r}") + return sum(1 for _ in self._descendants()) + + @property + def size(self) -> int: + """The total size in bytes of the assets below this path.""" + if not self.exists(): + raise NotFoundError(f"No such Dandiset path: {str(self)!r}") + return sum(a.size for a in self._descendants()) + + def get_asset(self) -> LocalAsset: + """Return the discovered asset; directories raise IsADirectoryError.""" + if not self.exists(): + raise NotFoundError(f"No such Dandiset path: {str(self)!r}") + if not self.is_file(): + raise IsADirectoryError(str(self)) + return self.assets.data[PurePosixPath(str(self))] diff --git a/dandi/organize.py b/dandi/organize.py index 896702f8a..f0095fc36 100644 --- a/dandi/organize.py +++ b/dandi/organize.py @@ -27,7 +27,11 @@ import ruamel.yaml from . import get_logger -from .consts import dandi_layout_fields +from .consts import ( + ORGANIZED_FILENAME_REGEX, + ORGANIZED_FOLDER_REGEX, + dandi_layout_fields, +) from .dandiset import Dandiset from .exceptions import OrganizeImpossibleError from .utils import ( @@ -1149,17 +1153,6 @@ def msg_(msg, n, cond=None): ) -LABELREGEX = r"[^_*\\/<>:|\"'?%@;.]+" -ORGANIZED_FILENAME_REGEX = ( - rf"sub-{LABELREGEX}" - rf"(_ses-{LABELREGEX})?" - rf"(_(tis|slice|cell|desc|probe|obj)-{LABELREGEX})*" - r"(_[a-z]+(\+[a-z]+)*)?" - r"\.nwb" -) -ORGANIZED_FOLDER_REGEX = rf"sub-{LABELREGEX}" - - def validate_organized_path( asset_path: str, filepath: Path, dandiset_path: Path ) -> list[ValidationResult]: diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 292519f2b..8da28852e 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -40,7 +40,7 @@ VersionStatus, ) from ..download import download -from ..exceptions import HTTP404Error, NotFoundError, SchemaVersionError +from ..exceptions import NotFoundError, SchemaVersionError from ..files import GenericAsset, dandi_file from ..utils import list_paths @@ -996,54 +996,6 @@ def _get_assets_with_path_prefix(prefix: str, **kw: Any) -> list[str]: ] -def test_remote_get_subject_ids(mocker: MockerFixture) -> None: - client = mocker.Mock(_instance_id="test") - dandiset = RemoteDandiset(client, "000001", version=DRAFT) - client.paginate.return_value = iter( - [ - {"path": "sub-mouse2", "asset": None}, - {"path": "sub-mouse1", "asset": None}, - {"path": "sub-root", "asset": {"asset_id": "a"}}, - {"path": "other", "asset": None}, - {"path": "sub-", "asset": None}, - {"path": "sub-bad_name", "asset": None}, - ] - ) - - assert dandiset.get_subject_ids() == ["mouse1", "mouse2"] - client.paginate.assert_called_once_with( - "/dandisets/000001/versions/draft/assets/paths/" - ) - - -def test_remote_get_subject_ids_empty(mocker: MockerFixture) -> None: - client = mocker.Mock(_instance_id="test") - dandiset = RemoteDandiset(client, "000001", version=DRAFT) - client.paginate.return_value = iter(()) - - assert dandiset.get_subject_ids() == [] - - -def test_remote_get_subject_ids_propagates_api_errors( - mocker: MockerFixture, -) -> None: - client = mocker.Mock(_instance_id="test") - dandiset = RemoteDandiset(client, "000001", version=DRAFT) - client.paginate.side_effect = RuntimeError("pagination failed") - - with pytest.raises(RuntimeError, match="pagination failed"): - dandiset.get_subject_ids() - - -def test_remote_get_subject_ids_missing_version(mocker: MockerFixture) -> None: - client = mocker.Mock(_instance_id="test") - dandiset = RemoteDandiset(client, "000001", version=DRAFT) - client.paginate.side_effect = HTTP404Error("not found") - - with pytest.raises(NotFoundError, match="No such version: 'draft'"): - dandiset.get_subject_ids() - - def test_get_assets_by_glob(text_dandiset: SampleDandiset) -> None: assert sorted( asset.path for asset in text_dandiset.dandiset.get_assets_by_glob("*a*.txt") diff --git a/dandi/tests/test_dandiset.py b/dandi/tests/test_dandiset.py index 6edff90d0..39653345a 100644 --- a/dandi/tests/test_dandiset.py +++ b/dandi/tests/test_dandiset.py @@ -1,7 +1,3 @@ -from pathlib import Path - -from pytest_mock import MockerFixture - from ..dandiset import Dandiset @@ -10,36 +6,3 @@ def test_get_dandiset_record() -> None: # Should have only header with "DO NOT EDIT" assert out.startswith("# DO NOT EDIT") assert "000000" in out - - -def test_get_subject_ids(tmp_path: Path, mocker: MockerFixture) -> None: - (tmp_path / "dandiset.yaml").write_text("identifier: '000001'\n") - (tmp_path / "sub-mouse2").mkdir() - (tmp_path / "sub-mouse2" / "record.nwb").touch() - (tmp_path / "sub-mouse1").mkdir() - (tmp_path / "sub-mouse1" / "session").mkdir() - (tmp_path / "sub-mouse1" / "session" / "record.nwb").touch() - (tmp_path / "sub-").mkdir() - (tmp_path / "sub-bad_name").mkdir() - (tmp_path / "sub-bad_name" / "record.nwb").touch() - (tmp_path / "sub-empty").mkdir() - linked = tmp_path / "sub-linked" - linked.mkdir() - (linked / "record.nwb").touch() - (tmp_path / "sub-root.nwb").touch() - (tmp_path / "subjects").mkdir() - original_is_symlink = Path.is_symlink - mocker.patch.object( - Path, - "is_symlink", - autospec=True, - side_effect=lambda path: path == linked or original_is_symlink(path), - ) - - assert Dandiset(tmp_path).get_subject_ids() == ["mouse1", "mouse2"] - - -def test_get_subject_ids_empty(tmp_path: Path) -> None: - (tmp_path / "dandiset.yaml").write_text("identifier: '000001'\n") - - assert Dandiset(tmp_path).get_subject_ids() == [] diff --git a/dandi/tests/test_dandiset_paths.py b/dandi/tests/test_dandiset_paths.py new file mode 100644 index 000000000..cf54aaf3c --- /dev/null +++ b/dandi/tests/test_dandiset_paths.py @@ -0,0 +1,265 @@ +from pathlib import Path +from typing import Any + +import pytest +import responses +from responses import matchers + +from .fixtures import SampleDandiset +from .test_files import mkpaths +from ..consts import DandiInstance +from ..dandiapi import DandiAPIClient, RemoteDandiset +from ..dandiset import Dandiset +from ..exceptions import NotFoundError + + +@pytest.mark.ai_generated +def test_local_path_tree(tmp_path: Path) -> None: + mkpaths( + tmp_path, + "dandiset.yaml", + "file.txt", + "sub-01/a.nwb", + "sub-01/b.nwb", + "record.zarr/chunk", + ".hidden/a.txt", + "empty/", + ) + (tmp_path / "file.txt").write_bytes(b"123") + root = Dandiset(tmp_path).get_path() + assert root.exists() and root.is_dir() and not root.is_file() + assert [p.name for p in root.iterdir()] == ["file.txt", "record.zarr", "sub-01"] + assert root.aggregate_files == 4 + assert root.size == 3 + assert root.parent == root + assert root / "." == root + assert (root / "sub-01/..") == root + assert (root / "sub-01").aggregate_files == 2 + assert [str(p) for p in (root / "sub-01").iterdir()] == [ + "sub-01/a.nwb", + "sub-01/b.nwb", + ] + assert (root / "file.txt").get_asset().path == "file.txt" + assert (root / "record.zarr").is_file() + assert not (root / "record.zarr/chunk").exists() + assert not (root / ".hidden").exists() + assert not (root / "empty").exists() + with pytest.raises(NotADirectoryError): + list((root / "record.zarr").iterdir()) + with pytest.raises(IsADirectoryError): + root.get_asset() + missing = root / "missing" + assert not missing.exists() and not missing.is_file() and not missing.is_dir() + for operation in ( + lambda: list(missing.iterdir()), + lambda: missing.size, + lambda: missing.aggregate_files, + missing.get_asset, + ): + with pytest.raises(NotFoundError): + operation() + with pytest.raises(ValueError, match="Absolute"): + root / "/etc" + with pytest.raises(ValueError): + root._get_subpath("") + with pytest.raises(ValueError): + root._get_subpath("a/b") + + +@pytest.mark.ai_generated +def test_local_empty_and_snapshot(tmp_path: Path) -> None: + mkpaths(tmp_path, "dandiset.yaml") + ds = Dandiset(tmp_path) + root = ds.get_path() + assert list(root.iterdir()) == [] + assert root.aggregate_files == root.size == 0 + mkpaths(tmp_path, "new.txt") + assert not (root / "new.txt").exists() + assert ds.get_path("new.txt").exists() + + +@pytest.mark.ai_generated +def test_local_symlink_directory(tmp_path: Path) -> None: + mkpaths(tmp_path, "dandiset.yaml", "target/a.txt") + try: + (tmp_path / "linked").symlink_to(tmp_path / "target", target_is_directory=True) + except OSError as exc: + pytest.skip(f"Cannot create directory symlink: {exc}") + assert not Dandiset(tmp_path).get_path("linked").exists() + + +def _entry(path: str, count: int, size: int, asset: Any = None) -> dict: + return { + "path": path, + "aggregate_files": count, + "aggregate_size": size, + "asset": asset, + } + + +@pytest.mark.ai_generated +@responses.activate +def test_remote_listing_pagination_and_cached_children() -> None: + url = "https://example.test/api/dandisets/000001/versions/draft/assets/paths/" + first = _entry( + "file.txt", 1, 5, {"asset_id": "test-id", "url": "https://example.test/blob"} + ) + directory = _entry("sub-01", 2, 9) + responses.get( + url, + json={"results": [first], "next": url + "?page=2"}, + match=[matchers.query_param_matcher({"path_prefix": ""})], + ) + responses.get( + url, + json={"results": [directory], "next": None}, + match=[matchers.query_param_matcher({"page": "2"})], + ) + responses.get( + url, + json={ + "results": [_entry("sub-01/a.nwb", 1, 9, {"asset_id": "other"})], + "next": None, + }, + match=[matchers.query_param_matcher({"path_prefix": "sub-01"})], + ) + with DandiAPIClient( + dandi_instance=DandiInstance( + name="test", gui="https://example.test", api="https://example.test/api/" + ) + ) as client: + root = RemoteDandiset( + client=client, identifier="000001", version="draft" + ).get_path() + responses.calls.reset() + children = list(root.iterdir()) + assert [str(p) for p in children] == ["file.txt", "sub-01"] + count = len(responses.calls) + assert count == 2 + assert children[0].is_file() and not children[0].is_dir() + assert children[1].is_dir() and not children[1].is_file() + assert children[0].size == 5 + assert children[1].aggregate_files == 2 + assert root.size == 14 and root.aggregate_files == 3 + assert list(root.iterdir()) == children + assert len(responses.calls) == count + nested = list(children[1].iterdir()) + assert nested[0].name == "a.nwb" + assert str(nested[0]) == "sub-01/a.nwb" + assert nested[0].size == 9 + assert len(responses.calls) == count + 1 + with pytest.raises(NotADirectoryError): + list(children[0].iterdir()) + with pytest.raises(IsADirectoryError): + children[1].get_asset() + assert root / "." == root and root / ".." == root + with pytest.raises(ValueError): + root._get_subpath("a/b") + with pytest.raises(ValueError): + root._get_subpath("") + + +@pytest.mark.ai_generated +@responses.activate +@pytest.mark.parametrize("status", [200, 404, 403]) +def test_remote_unlisted_path_errors(status: int) -> None: + import requests + + url = "https://example.test/api/dandisets/000001/versions/draft/assets/paths/" + responses.get(url, json={"results": [], "next": None}, status=status) + with DandiAPIClient( + dandi_instance=DandiInstance( + name="test", gui="https://example.test", api="https://example.test/api/" + ) + ) as client: + path = RemoteDandiset( + client=client, identifier="000001", version="draft" + ).get_path("missing") + if status == 403: + with pytest.raises(requests.HTTPError): + path.exists() + else: + assert not path.exists() and not path.is_file() and not path.is_dir() + with pytest.raises(NotFoundError): + path.get_asset() + with pytest.raises(NotFoundError): + list(path.iterdir()) + + +@pytest.mark.ai_generated +def test_path_listing_local_remote_parity(text_dandiset: SampleDandiset) -> None: + mkpaths(text_dandiset.dspath, "sub-01/session/a.txt", "sub-02/b.txt") + text_dandiset.upload() + local = Dandiset(text_dandiset.dspath).get_path() + remote = text_dandiset.dandiset.get_path() + + def describe(root: Any) -> dict: + return { + str(p): (p.is_file(), p.size, p.aggregate_files) for p in root.iterdir() + } + + assert describe(local) == describe(remote) + assert local.size == remote.size + assert local.aggregate_files == remote.aggregate_files == 6 + assert describe(local / "subdir2") == describe(remote / "subdir2") + asset = (remote / "file.txt").get_asset() + assert asset.path == "file.txt" + assert asset.size == (local / "file.txt").size + assert not (remote / "absent").exists() + + +@pytest.mark.ai_generated +@responses.activate +@pytest.mark.parametrize("kind", ["blob", "zarr"]) +def test_remote_unlisted_asset_can_be_fetched(kind: str) -> None: + base = "https://example.test/api/dandisets/000001/versions/draft/assets/" + name = "sample.zarr" if kind == "zarr" else "file.txt" + responses.get( + base + "paths/", + json={"results": [_entry(name, 1, 5, {"asset_id": "id"})], "next": None}, + ) + responses.get( + base + "id/info/", + json={ + "asset_id": "id", + "path": name, + "size": 5, + kind: "storage-id", + "created": "2026-01-01T00:00:00Z", + "modified": "2026-01-01T00:00:00Z", + }, + ) + with DandiAPIClient( + dandi_instance=DandiInstance( + name="test", gui="https://example.test", api="https://example.test/api/" + ) + ) as client: + path = RemoteDandiset(client, "000001", "draft").get_path(name) + assert path.exists() and path.is_file() and not path.is_dir() + assert path.size == 5 and path.aggregate_files == 1 + asset = path.get_asset() + assert asset.path == name and asset.size == 5 + + +@pytest.mark.ai_generated +@responses.activate +@pytest.mark.parametrize("status", [200, 404]) +def test_remote_empty_or_missing_root(status: int) -> None: + responses.get( + "https://example.test/api/dandisets/000001/versions/draft/assets/paths/", + json={"results": [], "next": None}, + status=status, + ) + with DandiAPIClient( + dandi_instance=DandiInstance( + name="test", gui="https://example.test", api="https://example.test/api/" + ) + ) as client: + root = RemoteDandiset(client, "000001", "draft").get_path() + assert root.exists() == (status == 200) + if status == 200: + assert list(root.iterdir()) == [] + assert root.size == root.aggregate_files == 0 + else: + with pytest.raises(NotFoundError): + list(root.iterdir()) diff --git a/dandi/utils.py b/dandi/utils.py index f6bcf6027..cd6ea7afd 100644 --- a/dandi/utils.py +++ b/dandi/utils.py @@ -43,17 +43,6 @@ lgr = get_logger() - -def parse_dandi_subject_dirname(name: str) -> str | None: - """Return the ID encoded by a valid DANDI ``sub-*`` directory name.""" - - # Match the label syntax used by ``dandi organize`` without importing that - # module, which imports this utility module itself. - if re.fullmatch(r"sub-[^_*\\/<>:|\"'?%@;.]+", name) is None: - return None - return name[4:] - - _sys_excepthook = sys.excepthook # Just in case we ever need original one # diff --git a/docs/source/modref/dandiapi.rst b/docs/source/modref/dandiapi.rst index 5ee1e0c7b..8713e97ea 100644 --- a/docs/source/modref/dandiapi.rst +++ b/docs/source/modref/dandiapi.rst @@ -36,16 +36,6 @@ be passed to functions of pynwb etc. You can see more usages of DANDI API to assist with data streaming at `PyNWB: Streaming NWB files `_. -To discover the subject labels represented by a remote Dandiset, use -``RemoteDandiset.get_subject_ids()``. It queries the Archive's efficient -top-level path endpoint and does not download asset payloads or metadata: - -.. code-block:: python - - with DandiAPIClient() as client: - dandiset = client.get_dandiset("000001") - print(dandiset.get_subject_ids()) - Client ------ @@ -59,6 +49,53 @@ Dandisets .. autoclass:: RemoteDandiset() +Browsing directories +^^^^^^^^^^^^^^^^^^^^ + +Use ``RemoteDandiset.get_path()`` to list one level without retrieving every asset: + +.. code-block:: python + + with DandiAPIClient() as client: + root = client.get_dandiset("000026", "draft").get_path() + for entry in root.iterdir(): + print(entry.name, entry.is_dir(), entry.aggregate_files, entry.size) + +The result follows the ``BasePath`` interface: ``/``, ``parent``, ``iterdir()``, +``exists()``, ``is_file()``, ``is_dir()`` and ``size``. Both blob and Zarr assets +are files in this tree; Zarr chunks are not children. Call ``entry.get_asset()`` +to retrieve the full asset record. + +Remote listing costs one paginated request sequence per directory. Listed +children already contain recursive sizes and counts, so inspecting those +properties does not fetch each asset. Resolving an arbitrary unlisted path +requires listing its parent. A full asset record requires an additional request. +Listings are cached on the path objects; create a new root to see later changes. +Authorization and server errors propagate to the caller. + +For a local Dandiset, ``Dandiset(directory).get_path()`` provides the same path +operations. It discovers all assets once using DANDI's existing discovery rules, +including generic files. Empty directories, dot-prefixed paths and directory +symlinks follow those rules; they are not additional assets. Metadata in +``dandiset.yaml`` is not part of the asset tree. Local sizes are calculated from +the files; remote sizes come from Archive aggregates. + +For organized Dandisets, path-derived subject IDs can be obtained as follows: + +.. code-block:: python + + import re + from dandi.consts import ORGANIZED_FOLDER_REGEX + + subjects = sorted(p.name[4:] for p in root.iterdir() + if p.is_dir() and re.fullmatch(ORGANIZED_FOLDER_REGEX, p.name)) + +This recipe does not inspect NWB metadata or BIDS participants tables. Directory +names do not necessarily describe every dataset's scientific subjects. + +.. autoclass:: RemoteDandisetPath() + :show-inheritance: + .. autoclass:: Version() :inherited-members: BaseModel :exclude-members: Config, JSON_EXCLUDE diff --git a/docs/source/modref/dandiset.rst b/docs/source/modref/dandiset.rst deleted file mode 100644 index 35e41a3e7..000000000 --- a/docs/source/modref/dandiset.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. module:: dandi.dandiset - -``dandi.dandiset`` -================== - -This module provides the local Dandiset API. A local Dandiset can report the -subject labels represented by populated, valid top-level ``sub-*`` directories. -It walks directory entries to establish that a subject contains a file, but it -does not open file contents. Empty and symlinked subject directories are -ignored: - -.. code-block:: python - - from dandi.dandiset import Dandiset - - dandiset = Dandiset("/data/my-dandiset") - print(dandiset.get_subject_ids()) - -.. autoclass:: Dandiset() - :members: - -.. autoclass:: AssetView() - :members: diff --git a/docs/source/modref/index.rst b/docs/source/modref/index.rst index ce215299c..373b48502 100644 --- a/docs/source/modref/index.rst +++ b/docs/source/modref/index.rst @@ -32,7 +32,6 @@ Object-oriented interfaces to manipulate Dandisets and assets on a DANDI instanc .. toctree:: dandiarchive - dandiset Low-level user interfaces ========================= From 5007b30a96db4a3708ea87b0c1b7d5157d4e5429 Mon Sep 17 00:00:00 2001 From: AtomicGlance Date: Tue, 15 Sep 2026 11:16:06 +0330 Subject: [PATCH 6/9] Fix directory browsing test fixtures --- dandi/tests/test_dandiset_paths.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/dandi/tests/test_dandiset_paths.py b/dandi/tests/test_dandiset_paths.py index cf54aaf3c..dfaff8416 100644 --- a/dandi/tests/test_dandiset_paths.py +++ b/dandi/tests/test_dandiset_paths.py @@ -59,7 +59,7 @@ def test_local_path_tree(tmp_path: Path) -> None: with pytest.raises(NotFoundError): operation() with pytest.raises(ValueError, match="Absolute"): - root / "/etc" + root.joinpath("/etc") with pytest.raises(ValueError): root._get_subpath("") with pytest.raises(ValueError): @@ -99,7 +99,9 @@ def _entry(path: str, count: int, size: int, asset: Any = None) -> dict: @pytest.mark.ai_generated @responses.activate -def test_remote_listing_pagination_and_cached_children() -> None: +def test_remote_listing_pagination_and_cached_children( + monkeypatch: pytest.MonkeyPatch, +) -> None: url = "https://example.test/api/dandisets/000001/versions/draft/assets/paths/" first = _entry( "file.txt", 1, 5, {"asset_id": "test-id", "url": "https://example.test/blob"} @@ -107,22 +109,24 @@ def test_remote_listing_pagination_and_cached_children() -> None: directory = _entry("sub-01", 2, 9) responses.get( url, - json={"results": [first], "next": url + "?page=2"}, + json={"count": 2, "results": [first], "next": url + "?path_prefix=&page=2"}, match=[matchers.query_param_matcher({"path_prefix": ""})], ) responses.get( url, - json={"results": [directory], "next": None}, - match=[matchers.query_param_matcher({"page": "2"})], + json={"count": 2, "results": [directory], "next": None}, + match=[matchers.query_param_matcher({"path_prefix": "", "page": "2"})], ) responses.get( url, json={ + "count": 1, "results": [_entry("sub-01/a.nwb", 1, 9, {"asset_id": "other"})], "next": None, }, match=[matchers.query_param_matcher({"path_prefix": "sub-01"})], ) + monkeypatch.setenv("DANDI_PAGINATION_DISABLE_FALLBACK", "1") with DandiAPIClient( dandi_instance=DandiInstance( name="test", gui="https://example.test", api="https://example.test/api/" @@ -189,6 +193,8 @@ def test_remote_unlisted_path_errors(status: int) -> None: @pytest.mark.ai_generated def test_path_listing_local_remote_parity(text_dandiset: SampleDandiset) -> None: mkpaths(text_dandiset.dspath, "sub-01/session/a.txt", "sub-02/b.txt") + (text_dandiset.dspath / "sub-01/session/a.txt").write_bytes(b"alpha\n") + (text_dandiset.dspath / "sub-02/b.txt").write_bytes(b"beta\n") text_dandiset.upload() local = Dandiset(text_dandiset.dspath).get_path() remote = text_dandiset.dandiset.get_path() From e280a596281b240fdbae41dbb59ffd1176c49f16 Mon Sep 17 00:00:00 2001 From: AtomicGlance Date: Tue, 15 Sep 2026 11:41:22 +0330 Subject: [PATCH 7/9] TEST: match empty path query across responses versions --- dandi/tests/test_dandiset_paths.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/dandi/tests/test_dandiset_paths.py b/dandi/tests/test_dandiset_paths.py index dfaff8416..328ab9b03 100644 --- a/dandi/tests/test_dandiset_paths.py +++ b/dandi/tests/test_dandiset_paths.py @@ -1,9 +1,9 @@ from pathlib import Path from typing import Any +from urllib.parse import parse_qsl, urlsplit import pytest import responses -from responses import matchers from .fixtures import SampleDandiset from .test_files import mkpaths @@ -97,6 +97,17 @@ def _entry(path: str, count: int, size: int, asset: Any = None) -> dict: } +def _query_matcher(expected: dict[str, str]): + """Match the raw query so empty values work with all supported responses versions.""" + + def match(request: Any) -> tuple[bool, str]: + actual = dict(parse_qsl(urlsplit(request.url).query, keep_blank_values=True)) + valid = actual == expected + return valid, f"Query parameters do not match: {actual!r} != {expected!r}" + + return match + + @pytest.mark.ai_generated @responses.activate def test_remote_listing_pagination_and_cached_children( @@ -110,12 +121,12 @@ def test_remote_listing_pagination_and_cached_children( responses.get( url, json={"count": 2, "results": [first], "next": url + "?path_prefix=&page=2"}, - match=[matchers.query_param_matcher({"path_prefix": ""})], + match=[_query_matcher({"path_prefix": ""})], ) responses.get( url, json={"count": 2, "results": [directory], "next": None}, - match=[matchers.query_param_matcher({"path_prefix": "", "page": "2"})], + match=[_query_matcher({"path_prefix": "", "page": "2"})], ) responses.get( url, @@ -124,7 +135,7 @@ def test_remote_listing_pagination_and_cached_children( "results": [_entry("sub-01/a.nwb", 1, 9, {"asset_id": "other"})], "next": None, }, - match=[matchers.query_param_matcher({"path_prefix": "sub-01"})], + match=[_query_matcher({"path_prefix": "sub-01"})], ) monkeypatch.setenv("DANDI_PAGINATION_DISABLE_FALLBACK", "1") with DandiAPIClient( From 4bf0e7fab97b936abd2505cb5f00eb988560c58e Mon Sep 17 00:00:00 2001 From: AtomicGlance Date: Tue, 15 Sep 2026 12:09:01 +0330 Subject: [PATCH 8/9] TEST: annotate query matcher return type --- dandi/tests/test_dandiset_paths.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dandi/tests/test_dandiset_paths.py b/dandi/tests/test_dandiset_paths.py index 328ab9b03..748d09dae 100644 --- a/dandi/tests/test_dandiset_paths.py +++ b/dandi/tests/test_dandiset_paths.py @@ -1,3 +1,4 @@ +from collections.abc import Callable from pathlib import Path from typing import Any from urllib.parse import parse_qsl, urlsplit @@ -97,7 +98,7 @@ def _entry(path: str, count: int, size: int, asset: Any = None) -> dict: } -def _query_matcher(expected: dict[str, str]): +def _query_matcher(expected: dict[str, str]) -> Callable[[Any], tuple[bool, str]]: """Match the raw query so empty values work with all supported responses versions.""" def match(request: Any) -> tuple[bool, str]: From 1be1ce2829e5400fcd93700ed6bda5e8663d04c9 Mon Sep 17 00:00:00 2001 From: AtomicGlance Date: Tue, 15 Sep 2026 21:27:35 +0330 Subject: [PATCH 9/9] DOC: remove unsupported subject path recipe --- dandi/consts.py | 11 ----------- dandi/organize.py | 17 ++++++++++++----- docs/source/modref/dandiapi.rst | 13 ------------- 3 files changed, 12 insertions(+), 29 deletions(-) diff --git a/dandi/consts.py b/dandi/consts.py index 82ee0b7cf..baca5465c 100644 --- a/dandi/consts.py +++ b/dandi/consts.py @@ -266,14 +266,3 @@ def urls(self) -> Iterator[str]: #: exFAT and some network filesystems truncate, and FAT rounds to a multiple #: of two seconds. See https://github.com/dandi/dandi-cli/issues/1907 MTIME_TOLERANCE = 2.0 - - -LABELREGEX = r"[^_*\\/<>:|\"'?%@;.]+" -ORGANIZED_FILENAME_REGEX = ( - rf"sub-{LABELREGEX}" - rf"(_ses-{LABELREGEX})?" - rf"(_(tis|slice|cell|desc|probe|obj)-{LABELREGEX})*" - r"(_[a-z]+(\+[a-z]+)*)?" - r"\.nwb" -) -ORGANIZED_FOLDER_REGEX = rf"sub-{LABELREGEX}" diff --git a/dandi/organize.py b/dandi/organize.py index f0095fc36..896702f8a 100644 --- a/dandi/organize.py +++ b/dandi/organize.py @@ -27,11 +27,7 @@ import ruamel.yaml from . import get_logger -from .consts import ( - ORGANIZED_FILENAME_REGEX, - ORGANIZED_FOLDER_REGEX, - dandi_layout_fields, -) +from .consts import dandi_layout_fields from .dandiset import Dandiset from .exceptions import OrganizeImpossibleError from .utils import ( @@ -1153,6 +1149,17 @@ def msg_(msg, n, cond=None): ) +LABELREGEX = r"[^_*\\/<>:|\"'?%@;.]+" +ORGANIZED_FILENAME_REGEX = ( + rf"sub-{LABELREGEX}" + rf"(_ses-{LABELREGEX})?" + rf"(_(tis|slice|cell|desc|probe|obj)-{LABELREGEX})*" + r"(_[a-z]+(\+[a-z]+)*)?" + r"\.nwb" +) +ORGANIZED_FOLDER_REGEX = rf"sub-{LABELREGEX}" + + def validate_organized_path( asset_path: str, filepath: Path, dandiset_path: Path ) -> list[ValidationResult]: diff --git a/docs/source/modref/dandiapi.rst b/docs/source/modref/dandiapi.rst index 8713e97ea..bb8b9439e 100644 --- a/docs/source/modref/dandiapi.rst +++ b/docs/source/modref/dandiapi.rst @@ -80,19 +80,6 @@ symlinks follow those rules; they are not additional assets. Metadata in ``dandiset.yaml`` is not part of the asset tree. Local sizes are calculated from the files; remote sizes come from Archive aggregates. -For organized Dandisets, path-derived subject IDs can be obtained as follows: - -.. code-block:: python - - import re - from dandi.consts import ORGANIZED_FOLDER_REGEX - - subjects = sorted(p.name[4:] for p in root.iterdir() - if p.is_dir() and re.fullmatch(ORGANIZED_FOLDER_REGEX, p.name)) - -This recipe does not inspect NWB metadata or BIDS participants tables. Directory -names do not necessarily describe every dataset's scientific subjects. - .. autoclass:: RemoteDandisetPath() :show-inheritance: