From 33ffac1e96464684ead0975046552d1b3e7b4bd0 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 28 Oct 2026 09:17:59 +0000 Subject: [PATCH 1/2] feat: patch NeXus files on the fly while loading Add `ess.reduce.nexus.patch`, a read-only view of an HDF5 file with a list of operations applied (set/delete attributes, remove, move, set dataset). ScippNexus reads through this view, so workflows see the fixed structure without copying or modifying the file. Data is read from the original datasets. Patches are opt-in via the `NeXusFilePatch[RunType]` workflow parameter, which defaults to an empty patch. Patches can be loaded from TOML files and, being plain lists, amended in memory. Co-Authored-By: Claude Opus 5.5 --- .../src/ess/reduce/nexus/__init__.py | 3 +- .../src/ess/reduce/nexus/_nexus_loader.py | 39 +- .../essreduce/src/ess/reduce/nexus/patch.py | 471 ++++++++++++++++++ .../essreduce/src/ess/reduce/nexus/types.py | 31 +- .../src/ess/reduce/nexus/workflow.py | 23 +- packages/essreduce/tests/nexus/patch_test.py | 284 +++++++++++ .../essreduce/tests/nexus/workflow_test.py | 2 +- 7 files changed, 831 insertions(+), 22 deletions(-) create mode 100644 packages/essreduce/src/ess/reduce/nexus/patch.py create mode 100644 packages/essreduce/tests/nexus/patch_test.py diff --git a/packages/essreduce/src/ess/reduce/nexus/__init__.py b/packages/essreduce/src/ess/reduce/nexus/__init__.py index e739a622f..8daf77ba7 100644 --- a/packages/essreduce/src/ess/reduce/nexus/__init__.py +++ b/packages/essreduce/src/ess/reduce/nexus/__init__.py @@ -12,7 +12,7 @@ The submodule :mod:`types` defines all domain types. """ -from . import types +from . import patch, types from ._nexus_loader import ( compute_component_position, compute_detector_position, @@ -39,5 +39,6 @@ 'load_from_path', 'open_component_group', 'open_nexus_file', + 'patch', 'types', ] diff --git a/packages/essreduce/src/ess/reduce/nexus/_nexus_loader.py b/packages/essreduce/src/ess/reduce/nexus/_nexus_loader.py index 606e4dc67..a13a25eaf 100644 --- a/packages/essreduce/src/ess/reduce/nexus/_nexus_loader.py +++ b/packages/essreduce/src/ess/reduce/nexus/_nexus_loader.py @@ -14,6 +14,7 @@ import scippnexus as snx from ..logging import get_logger +from .patch import open_patched_file from .types import ( Beamline, FilePath, @@ -24,6 +25,7 @@ NeXusGroup, NeXusLocationSpec, NeXusTransformation, + PatchedFilePath, RunType, ) @@ -129,7 +131,7 @@ def load_all_components( def load_metadata( - file_path: FilePath | NeXusFile | NeXusGroup, + file_path: FilePath | NeXusFile | NeXusGroup | PatchedFilePath, model: type[_Model], *, entry_name: NeXusEntryName | None = None, @@ -164,7 +166,7 @@ def compute_component_position(dg: sc.DataGroup) -> sc.DataGroup: def open_nexus_file( - file_path: FilePath | NeXusFile | NeXusGroup, + file_path: FilePath | NeXusFile | NeXusGroup | PatchedFilePath, definitions: Mapping | None | NoNewDefinitionsType = NoNewDefinitions, *, locking: bool | str | None | NoLockingIfNeededType = NoLockingIfNeeded, @@ -175,6 +177,8 @@ def open_nexus_file( ---------- file_path: Path of the file to open or a NeXus file or group handle. + If this is a :class:`ess.reduce.nexus.types.PatchedFilePath`, + the patch is applied to the file. definitions: If set, application definitions to use for the file. If ``file_path`` is a NeXus file or group, this must be unset or match @@ -266,26 +270,31 @@ def _attempt_to_open_without_locking( def _open_nexus_file_from_path( - file_path: FilePath, + file_path: FilePath | NeXusFile | PatchedFilePath, definitions: Mapping | None | NoNewDefinitionsType, **kwargs: object, ) -> AbstractContextManager[snx.Group]: - new_kwargs = ( - kwargs - | ({} if definitions is NoNewDefinitions else {'definitions': definitions}) - # ESS nexus files are always written in swmr mode. - # By default, try to open in swmr mode. - | ({'swmr': True} if 'swmr' not in kwargs else {}) - ) + defs = snx.base_definitions() if definitions is NoNewDefinitions else definitions + + def open_file(**h5_kwargs: object) -> AbstractContextManager[snx.Group]: + if isinstance(file_path, PatchedFilePath): + return open_patched_file( + file_path.path, file_path.patch, definitions=defs, **h5_kwargs + ) + return snx.File(file_path, definitions=defs, **h5_kwargs) + + # ESS nexus files are always written in swmr mode. + # By default, try to open in swmr mode. + new_kwargs = kwargs | ({'swmr': True} if 'swmr' not in kwargs else {}) try: - return snx.File(file_path, **new_kwargs) + return open_file(**new_kwargs) except OSError as e: # If the file type does not support swmr # and the caller did not explicitly request swmr # open the file without swmr. if 'swmr' not in kwargs: new_kwargs.pop('swmr') - return snx.File(file_path, **new_kwargs) + return open_file(**new_kwargs) raise e @@ -506,7 +515,7 @@ def compute_detector_position( def load_data( - file_path: FilePath | NeXusFile | NeXusGroup, + file_path: FilePath | NeXusFile | NeXusGroup | PatchedFilePath, selection: snx.typing.ScippIndex | slice = (), *, entry_name: NeXusEntryName | None = None, @@ -755,7 +764,9 @@ def _parse_monitor(group: snx.Group) -> NeXusMonitorInfo: ) -def read_nexus_file_info(file_path: FilePath | NeXusFile | NeXusGroup) -> NeXusFileInfo: +def read_nexus_file_info( + file_path: FilePath | NeXusFile | NeXusGroup | PatchedFilePath, +) -> NeXusFileInfo: """Opens and inspects a NeXus file, returning a summary of its contents.""" with open_nexus_file(file_path) as f: entry = _unique_child_group(f, snx.NXentry, None) diff --git a/packages/essreduce/src/ess/reduce/nexus/patch.py b/packages/essreduce/src/ess/reduce/nexus/patch.py new file mode 100644 index 000000000..90c856c3c --- /dev/null +++ b/packages/essreduce/src/ess/reduce/nexus/patch.py @@ -0,0 +1,471 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Scipp contributors (https://github.com/scipp) +"""Patches for fixing defects in NeXus files while loading them. + +A patch is a list of operations, applied in order to a read-only view of a file. +The file on disk is not modified and no data is copied: +Datasets are read from the original file when accessed. + +All paths in operations are absolute paths in the patched file, +i.e., they take preceding operations into account. +Patches apply to paths, not to objects: +If an object is reachable under multiple paths through hard links, +a patch for one path does not affect the others. +Soft links are resolved in the patched file. +As in HDF5, moving or removing the target of a soft link breaks the link. + +Patches are typically stored in TOML files and loaded with :func:`load_patch`: + +.. code-block:: toml + + # Explain the defect being fixed here. + [[op]] + kind = "move" + source = "/entry/instrument/detector/value" + target = "/entry/instrument/detector/transformations/motion" + + [[op]] + kind = "set_attrs" + path = "/entry/instrument/detector/transformations/motion" + attrs = { transformation_type = "translation", vector = [0.0, 0.0, 1.0] } + +A patch is a plain list, so it can be amended in memory before use: + +.. code-block:: python + + from ess.reduce.nexus import patch + + p = patch.load_patch('fix.toml') + [patch.Remove(path='/entry/broken')] + workflow[NeXusFilePatch[SampleRun]] = p +""" + +from __future__ import annotations + +import os +import posixpath +import uuid +from collections.abc import Iterable, Iterator, Mapping, Sequence +from contextlib import AbstractContextManager +from dataclasses import dataclass, replace +from typing import Any + +import h5py +import scippnexus as snx +import tomllib + + +@dataclass(frozen=True, kw_only=True) +class SetAttrs: + """Add attributes or replace existing ones.""" + + path: str + attrs: Mapping[str, Any] + + +@dataclass(frozen=True, kw_only=True) +class DeleteAttrs: + """Delete attributes. All attributes must exist.""" + + path: str + names: Sequence[str] + + +@dataclass(frozen=True, kw_only=True) +class Remove: + """Remove a group or dataset.""" + + path: str + + +@dataclass(frozen=True, kw_only=True) +class Move: + """Move or rename a group or dataset. The target must not exist.""" + + source: str + target: str + + +@dataclass(frozen=True, kw_only=True) +class SetDataset: + """Add a dataset or replace an existing dataset. + + The parent group must exist. + A replaced dataset does not keep its attributes. + """ + + path: str + value: Any + + +PatchOp = SetAttrs | DeleteAttrs | Remove | Move | SetDataset + +_op_kinds: dict[str, type[PatchOp]] = { + 'set_attrs': SetAttrs, + 'delete_attrs': DeleteAttrs, + 'remove': Remove, + 'move': Move, + 'set_dataset': SetDataset, +} + + +def load_patch(path: str | os.PathLike[str]) -> list[PatchOp]: + """Load a patch from a TOML file. + + The file contains an array of tables named ``op``. + Each table has a ``kind`` (one of ``set_attrs``, ``delete_attrs``, ``remove``, + ``move``, ``set_dataset``) and the fields of the corresponding operation class. + """ + with open(path, 'rb') as f: + content = tomllib.load(f) + if unknown := set(content) - {'op'}: + raise ValueError(f"Unknown top-level keys in NeXus patch {path}: {unknown}") + ops = [] + for entry in content.get('op', []): + fields = dict(entry) + kind = fields.pop('kind') + if (op := _op_kinds.get(kind)) is None: + raise ValueError( + f"Unknown operation kind '{kind}' in NeXus patch {path}, " + f"expected one of {list(_op_kinds)}" + ) + ops.append(op(**fields)) + return ops + + +def open_patched_file( + path: str | os.PathLike[str], + patch: Iterable[PatchOp], + *, + definitions: Mapping[str, type] | None, + **kwargs: Any, +) -> PatchedNeXusFile: + """Open a NeXus file and apply a patch. + + Parameters + ---------- + path: + Path of the file to open. + patch: + Operations to apply. + definitions: + Application definitions to use for the file, see :class:`scippnexus.File`. + kwargs: + Forwarded to :class:`h5py.File`. + + Returns + ------- + : + The patched file. + Can be used as a context manager to close the file. + """ + file = h5py.File(path, 'r', **kwargs) + try: + return PatchedNeXusFile(PatchedH5File(file, patch), definitions=definitions) + except BaseException: + file.close() + raise + + +class PatchedNeXusFile(snx.Group, AbstractContextManager['PatchedNeXusFile']): + """ScippNexus group for a patched file that closes the file on exit.""" + + def __init__( + self, file: PatchedH5File, *, definitions: Mapping[str, type] | None + ) -> None: + super().__init__(file, definitions=definitions) + self._patched_file = file + + def __exit__(self, *args: object) -> None: + self.close() + + def close(self) -> None: + self._patched_file.close() + + +class _PatchedAttrs(Mapping[str, Any]): + def __init__( + self, + base: h5py.AttributeManager, + new: h5py.AttributeManager | None, + deleted: set[str], + ) -> None: + self._base = base + self._new = new + self._deleted = deleted + + def __getitem__(self, name: str) -> Any: + if self._new is not None and name in self._new: + return self._new[name] + if name in self._deleted: + raise KeyError(name) + return self._base[name] + + def __iter__(self) -> Iterator[str]: + new = () if self._new is None else tuple(self._new) + yield from new + yield from (k for k in self._base if k not in self._deleted and k not in new) + + def __len__(self) -> int: + return sum(1 for _ in self) + + def get_id(self, name: bytes) -> Any: + """Used by ScippNexus to determine the encoding of string attributes.""" + if self._new is not None and name.decode() in self._new: + return self._new.get_id(name) + return self._base.get_id(name) + + +class _Node: + def __init__( + self, obj: h5py.Group | h5py.Dataset, path: str, file: PatchedH5File + ) -> None: + self._obj = obj + self._path = path + self._file = file + + @property + def name(self) -> str: + return self._path + + @property + def file(self) -> PatchedH5File: + return self._file + + @property + def parent(self) -> _Group: + return self._file[posixpath.dirname(self._path)] + + @property + def attrs(self) -> _PatchedAttrs: + return _PatchedAttrs( + self._obj.attrs, + self._file._new_attrs.get(self._path), + self._file._deleted_attrs.get(self._path, set()), + ) + + def __repr__(self) -> str: + return f'<{type(self).__name__} "{self._path}">' + + +class _Dataset(_Node): + def __getattr__(self, name: str) -> Any: + # shape, dtype, ndim, read_direct, asstr, ... of the underlying dataset. + return getattr(self._obj, name) + + def __getitem__(self, index: Any) -> Any: + return self._obj[index] + + def __len__(self) -> int: + return len(self._obj) + + +class _Group(_Node): + def _child_path(self, name: str) -> str: + return posixpath.join(self._path, name) + + def keys(self) -> list[str]: + names = [ + name + for name in self._obj + if not self._file._is_removed(self._child_path(name)) + ] + for path, obj in self._file._mounts.items(): + name = posixpath.basename(path) + if ( + obj is not None + and posixpath.dirname(path) == self._path + and name not in names + ): + names.append(name) + return names + + def __iter__(self) -> Iterator[str]: + return iter(self.keys()) + + def __len__(self) -> int: + return len(self.keys()) + + def items(self) -> list[tuple[str, _Group | _Dataset | None]]: + # As in h5py, dangling soft links yield None. + return [(name, self.get(name)) for name in self.keys()] + + def __contains__(self, name: str) -> bool: + try: + self[name] + except KeyError: + return False + return True + + def __getitem__(self, name: str) -> _Group | _Dataset: + path = posixpath.normpath(posixpath.join(self._path, name)) + if '/' in name: + obj = self._file._lookup(path) + else: + obj = self._file._child(self._obj, path) + return self._file._wrap(obj, path) + + def get(self, name: str) -> _Group | _Dataset | None: + try: + return self[name] + except KeyError: + return None + + +class PatchedH5File(_Group): + """Read-only view of an HDF5 file with a patch applied. + + Implements the subset of the :mod:`h5py` interface used by ScippNexus. + Closing this file also closes the underlying file. + """ + + def __init__(self, file: h5py.File, patch: Iterable[PatchOp]) -> None: + super().__init__(file, '/', self) + # Objects replacing or added to the original file, by path in the patched + # file. ``None`` marks removed objects. + self._mounts: dict[str, h5py.Group | h5py.Dataset | None] = {} + self._new_attrs: dict[str, h5py.AttributeManager] = {} + self._deleted_attrs: dict[str, set[str]] = {} + # New datasets and attributes are stored in an in-memory HDF5 file so they + # support the full h5py interface, e.g., string encodings and `asstr`. + self._scratch = h5py.File( + uuid.uuid4().hex, 'w', driver='core', backing_store=False + ) + for op in patch: + self._apply(op) + + @property + def parent(self) -> PatchedH5File: + return self + + def close(self) -> None: + self._scratch.close() + self._obj.close() + + def __getitem__(self, name: str) -> _Group | _Dataset: + if posixpath.normpath(name) == '/': + return self + return super().__getitem__(name) + + def _wrap(self, obj: h5py.Group | h5py.Dataset, path: str) -> _Group | _Dataset: + if isinstance(obj, h5py.Dataset): + return _Dataset(obj, path, self) + return _Group(obj, path, self) + + def _is_removed(self, path: str) -> bool: + return path in self._mounts and self._mounts[path] is None + + def _scratch_path(self, kind: str) -> str: + return f'{kind}/{len(self._scratch.require_group(kind))}' + + def _child(self, parent: h5py.Group, path: str) -> h5py.Group | h5py.Dataset: + """Return the child of ``parent`` with the given path in the patched file.""" + if self._is_removed(path): + raise KeyError(f"'{path}' was removed by a patch") + if (obj := self._mounts.get(path)) is not None: + return obj + name = posixpath.basename(path) + link = parent.get(name, getlink=True) + if link is None: + raise KeyError(f"'{path}' not found") + if isinstance(link, h5py.SoftLink): + return self._lookup( + posixpath.normpath(posixpath.join(posixpath.dirname(path), link.path)) + ) + return parent[name] + + def _lookup(self, path: str) -> h5py.Group | h5py.Dataset: + obj = self._obj + current = '/' + for name in path.strip('/').split('/'): + if not isinstance(obj, h5py.Group): + raise KeyError(f"'{path}' not found, '{current}' is not a group") + current = posixpath.join(current, name) + obj = self._child(obj, current) + return obj + + def _require(self, path: str) -> _Group | _Dataset: + try: + return self[path] + except KeyError: + raise KeyError( + f"Cannot apply patch, '{path}' does not exist in the patched file" + ) from None + + def _require_parent_group(self, path: str) -> None: + if not isinstance(self._require(posixpath.dirname(path)), _Group): + raise ValueError(f"Cannot apply patch, parent of '{path}' is not a group") + + def _drop(self, path: str) -> None: + """Forget patches of ``path`` and everything below it.""" + for patches in (self._mounts, self._new_attrs, self._deleted_attrs): + for key in [k for k in patches if _is_within(k, path)]: + del patches[key] + + def _apply(self, op: PatchOp) -> None: + match _normalize_paths(op): + case SetAttrs(path=path, attrs=attrs): + self._require(path) + if path not in self._new_attrs: + holder = self._scratch.create_group(self._scratch_path('attrs')) + self._new_attrs[path] = holder.attrs + self._new_attrs[path].update(attrs) + self._deleted_attrs.get(path, set()).difference_update(attrs) + case DeleteAttrs(path=path, names=names): + if missing := set(names) - set(self._require(path).attrs): + raise KeyError( + f"Cannot apply patch, attributes {missing} " + f"do not exist at '{path}'" + ) + new = self._new_attrs.get(path, {}) + for name in names: + if name in new: + del new[name] + self._deleted_attrs.setdefault(path, set()).update(names) + case Remove(path=path): + if path == '/': + raise ValueError("Cannot remove the root group") + self._require(path) + self._drop(path) + self._mounts[path] = None + case Move(source=source, target=target): + obj = self._require(source)._obj + if _is_within(target, source): + raise ValueError(f"Cannot move '{source}' into itself") + if target in self: + raise ValueError(f"Cannot move '{source}', '{target}' exists") + self._require_parent_group(target) + self._drop(target) + for patches in (self._mounts, self._new_attrs, self._deleted_attrs): + for key in [k for k in patches if _is_within(k, source)]: + patches[target + key[len(source) :]] = patches.pop(key) + self._mounts[target] = obj + self._mounts[source] = None + case SetDataset(path=path, value=value): + self._require_parent_group(path) + if isinstance(self.get(path), _Group): + raise ValueError(f"Cannot replace group '{path}' by a dataset") + self._drop(path) + self._mounts[path] = self._scratch.create_dataset( + self._scratch_path('datasets'), data=value + ) + case _: + raise TypeError(f"Unknown patch operation {op!r}") + + +def _normalize_paths(op: PatchOp) -> PatchOp: + paths = { + field: getattr(op, field) + for field in ('path', 'source', 'target') + if hasattr(op, field) + } + for path in paths.values(): + if not path.startswith('/'): + raise ValueError(f"Patch paths must be absolute, got '{path}'") + # Note that normpath preserves a leading '//'. + return replace( + op, **{k: posixpath.normpath('/' + v.lstrip('/')) for k, v in paths.items()} + ) + + +def _is_within(path: str, base: str) -> bool: + return path == base or path.startswith(base.rstrip('/') + '/') diff --git a/packages/essreduce/src/ess/reduce/nexus/types.py b/packages/essreduce/src/ess/reduce/nexus/types.py index ccf7d9286..c8f04ea92 100644 --- a/packages/essreduce/src/ess/reduce/nexus/types.py +++ b/packages/essreduce/src/ess/reduce/nexus/types.py @@ -10,6 +10,8 @@ from scippneutron import metadata as scn_meta from scippneutron.chopper import DiskChopper +from .patch import PatchOp + FilePath = NewType('FilePath', Path) """Full path to a NeXus file on disk.""" NeXusFile = NewType('NeXusFile', BinaryIO) @@ -24,6 +26,18 @@ NeXusGroup = NewType('NeXusGroup', snx.Group) """A ScippNexus group in an open file.""" + +@dataclass(frozen=True, kw_only=True) +class PatchedFilePath: + """Path to a NeXus file and a patch to apply when opening it. + + See :mod:`ess.reduce.nexus.patch`. + """ + + path: FilePath + patch: list[PatchOp] + + NeXusEntryName = NewType('NeXusEntryName', str) """Name of an entry in a NeXus file.""" NeXusSourceName = NewType('NeXusSourceName', str) @@ -277,6 +291,17 @@ class TimeInterval(Generic[RunType]): value: slice +class NeXusFilePatch(Generic[RunType]): + """Patch to apply to the NeXus file of a run when loading it. + + The default is an empty patch, i.e., the file is loaded as is. + See :mod:`ess.reduce.nexus.patch`. + """ + + def __new__(cls, x: list[PatchOp]) -> Any: + return x + + class TransformationTimeFilter(Generic[Component, RunType]): """Filter for time-dependent transformations.""" @@ -288,7 +313,7 @@ def __call__(self, transform: sc.DataArray) -> sc.Variable | sc.DataArray: ... @dataclass class NeXusFileSpec(Generic[RunType]): - value: FilePath | NeXusFile | NeXusGroup + value: FilePath | NeXusFile | NeXusGroup | PatchedFilePath @dataclass @@ -297,7 +322,7 @@ class NeXusLocationSpec: NeXus filename and optional parameters to identify (parts of) a component to load. """ - filename: FilePath | NeXusFile | NeXusGroup + filename: FilePath | NeXusFile | NeXusGroup | PatchedFilePath entry_name: NeXusEntryName | None = None component_name: str | None = None selection: snx.typing.ScippIndex | slice = () @@ -316,7 +341,7 @@ class NeXusAllLocationSpec: NeXus parameters to identify all components of a class to load. """ - filename: FilePath | NeXusFile | NeXusGroup + filename: FilePath | NeXusFile | NeXusGroup | PatchedFilePath entry_name: NeXusEntryName | None = None selection: snx.typing.ScippIndex | slice = () diff --git a/packages/essreduce/src/ess/reduce/nexus/workflow.py b/packages/essreduce/src/ess/reduce/nexus/workflow.py index e9c06c6d7..d5a1e02f9 100644 --- a/packages/essreduce/src/ess/reduce/nexus/workflow.py +++ b/packages/essreduce/src/ess/reduce/nexus/workflow.py @@ -18,6 +18,7 @@ from scippneutron.chopper import DiskChopper, extract_chopper_from_nexus from scippneutron.metadata import RadiationProbe, SourceType +from ..logging import get_logger from . import _nexus_loader as nexus from .types import ( COMPONENT_CONSTRAINTS, @@ -41,10 +42,12 @@ NeXusComponentLocationSpec, NeXusData, NeXusDataLocationSpec, + NeXusFilePatch, NeXusFileSpec, NeXusName, NeXusTransformation, NeXusTransformationChain, + PatchedFilePath, Position, PreopenNeXusFile, ProtonCharge, @@ -65,15 +68,28 @@ def file_path_to_file_spec( - filename: Filename[RunType], preopen: PreopenNeXusFile + filename: Filename[RunType], + preopen: PreopenNeXusFile, + patch: NeXusFilePatch[RunType], ) -> NeXusFileSpec[RunType]: + location = filename + if patch: + get_logger().info( + "Applying NeXus patch with %d operations to %s", len(patch), filename + ) + location = PatchedFilePath(path=filename, patch=patch) return NeXusFileSpec[RunType]( - nexus.open_nexus_file(filename, definitions=definitions) + nexus.open_nexus_file(location, definitions=definitions) if preopen - else filename + else location ) +def no_nexus_file_patch() -> NeXusFilePatch[RunType]: + """Load NeXus files without patching them.""" + return NeXusFilePatch[RunType]([]) + + def no_monitor_position_offset() -> MonitorPositionOffset[RunType, MonitorType]: return MonitorPositionOffset[RunType, MonitorType](no_offset) @@ -813,6 +829,7 @@ def load_source_metadata_from_nexus( _common_providers = ( gravity_vector_neg_y, file_path_to_file_spec, + no_nexus_file_patch, full_time_interval, component_spec_by_name, unique_component_spec, # after component_spec_by_name, partially overrides diff --git a/packages/essreduce/tests/nexus/patch_test.py b/packages/essreduce/tests/nexus/patch_test.py new file mode 100644 index 000000000..39de797e7 --- /dev/null +++ b/packages/essreduce/tests/nexus/patch_test.py @@ -0,0 +1,284 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Scipp contributors (https://github.com/scipp) +from pathlib import Path + +import h5py +import numpy as np +import pytest +import scipp as sc +import scipp.testing +import scippnexus as snx + +from ess.reduce.nexus import GenericNeXusWorkflow, patch +from ess.reduce.nexus.types import ( + EmptyDetector, + Filename, + NeXusDetectorName, + NeXusFilePatch, + PreopenNeXusFile, + SampleRun, +) + +DETECTOR = '/entry/instrument/detector' + + +@pytest.fixture +def nexus_file(tmp_path: Path) -> Path: + path = tmp_path / 'file.nxs' + with h5py.File(path, 'w') as f: + entry = f.create_group('entry') + entry.attrs['NX_class'] = 'NXentry' + entry['title'] = 'original title' + instrument = entry.create_group('instrument') + instrument.attrs['NX_class'] = 'NXinstrument' + detector = instrument.create_group('detector') + detector.attrs['NX_class'] = 'NXdetector' + detector.attrs['description'] = 'a detector' + detector['detector_number'] = np.arange(1, 5) + detector['data'] = np.ones(4) + for name in ('x_pixel_offset', 'y_pixel_offset', 'z_pixel_offset'): + detector.create_dataset(name, data=np.zeros(4)).attrs['units'] = 'm' + detector['depends_on'] = f'{DETECTOR}/transformations/offset' + transformations = detector.create_group('transformations') + transformations.attrs['NX_class'] = 'NXtransformations' + offset = transformations.create_dataset('offset', data=2.0) + offset.attrs['transformation_type'] = 'translation' + offset.attrs['vector'] = [0.0, 0.0, 1.0] + offset.attrs['units'] = 'm' + offset.attrs['depends_on'] = '.' + entry['number_link'] = h5py.SoftLink(f'{DETECTOR}/detector_number') + return path + + +def open_patched(path: Path, ops: list[patch.PatchOp]) -> patch.PatchedNeXusFile: + return patch.open_patched_file(path, ops, definitions=snx.base_definitions()) + + +def test_empty_patch_loads_same_as_unpatched_file(nexus_file: Path) -> None: + with snx.File(nexus_file) as f: + expected = f[DETECTOR][()] + with open_patched(nexus_file, []) as f: + sc.testing.assert_identical(f[DETECTOR][()], expected) + + +def test_patch_does_not_modify_file(nexus_file: Path) -> None: + ops = [ + patch.Remove(path='/entry/title'), + patch.SetAttrs(path=DETECTOR, attrs={'description': 'patched'}), + ] + with open_patched(nexus_file, ops) as f: + assert 'title' not in f['entry'] + with h5py.File(nexus_file) as f: + assert f['entry/title'][()] == b'original title' + assert f[DETECTOR].attrs['description'] == 'a detector' + + +def test_set_attrs_adds_and_replaces_attributes(nexus_file: Path) -> None: + ops = [ + patch.SetAttrs(path=DETECTOR, attrs={'description': 'patched', 'new': 1.5}), + ] + with open_patched(nexus_file, ops) as f: + attrs = f[DETECTOR].attrs + assert attrs['description'] == 'patched' + assert attrs['new'] == 1.5 + assert attrs['NX_class'] == 'NXdetector' + + +def test_set_attrs_on_dataset(nexus_file: Path) -> None: + ops = [patch.SetAttrs(path='/entry/title', attrs={'units': 'm'})] + with open_patched(nexus_file, ops) as f: + assert f['entry/title'].attrs['units'] == 'm' + + +def test_set_attrs_raises_if_path_does_not_exist(nexus_file: Path) -> None: + with pytest.raises(KeyError, match='/entry/missing'): + open_patched(nexus_file, [patch.SetAttrs(path='/entry/missing', attrs={})]) + + +def test_patch_paths_must_be_absolute(nexus_file: Path) -> None: + with pytest.raises(ValueError, match='absolute'): + open_patched(nexus_file, [patch.Remove(path='entry/title')]) + + +def test_patch_paths_are_normalized(nexus_file: Path) -> None: + ops = [ + patch.SetAttrs(path=f'/{DETECTOR}/', attrs={'x': 1}), + patch.SetAttrs(path=f'{DETECTOR}/../detector', attrs={'y': 2}), + ] + with open_patched(nexus_file, ops) as f: + assert f[DETECTOR].attrs['x'] == 1 + assert f[DETECTOR].attrs['y'] == 2 + + +def test_cannot_remove_root(nexus_file: Path) -> None: + with pytest.raises(ValueError, match='root'): + open_patched(nexus_file, [patch.Remove(path='/')]) + + +def test_delete_attrs(nexus_file: Path) -> None: + ops = [ + patch.SetAttrs(path=DETECTOR, attrs={'new': 1}), + patch.DeleteAttrs(path=DETECTOR, names=['description', 'new']), + ] + with open_patched(nexus_file, ops) as f: + assert set(f[DETECTOR].attrs) == {'NX_class'} + + +def test_delete_attrs_raises_if_attribute_does_not_exist(nexus_file: Path) -> None: + with pytest.raises(KeyError, match='missing'): + open_patched(nexus_file, [patch.DeleteAttrs(path=DETECTOR, names=['missing'])]) + + +def test_remove_group(nexus_file: Path) -> None: + ops = [patch.Remove(path=f'{DETECTOR}/transformations')] + with open_patched(nexus_file, ops) as f: + assert 'transformations' not in f[DETECTOR] + with pytest.raises(KeyError): + f[f'{DETECTOR}/transformations/offset'] + + +def test_remove_raises_if_path_does_not_exist(nexus_file: Path) -> None: + ops = [patch.Remove(path='/entry/title'), patch.Remove(path='/entry/title')] + with pytest.raises(KeyError, match='/entry/title'): + open_patched(nexus_file, ops) + + +def test_move_renames_group(nexus_file: Path) -> None: + ops = [patch.Move(source=DETECTOR, target='/entry/instrument/renamed')] + with open_patched(nexus_file, ops) as f: + instrument = f['entry/instrument'] + assert list(instrument) == ['renamed'] + renamed = instrument['renamed'] + assert renamed.name == '/entry/instrument/renamed' + assert renamed['transformations/offset'].name == ( + '/entry/instrument/renamed/transformations/offset' + ) + + +def test_move_keeps_patches_of_moved_objects(nexus_file: Path) -> None: + ops = [ + patch.SetAttrs(path=f'{DETECTOR}/transformations', attrs={'x': 1}), + patch.Remove(path=f'{DETECTOR}/detector_number'), + patch.Move(source=DETECTOR, target='/entry/detector'), + ] + with open_patched(nexus_file, ops) as f: + assert f['entry/detector/transformations'].attrs['x'] == 1 + assert 'detector_number' not in f['entry/detector'] + + +def test_move_raises_if_target_exists(nexus_file: Path) -> None: + ops = [patch.Move(source='/entry/title', target='/entry/instrument')] + with pytest.raises(ValueError, match='exists'): + open_patched(nexus_file, ops) + + +def test_move_onto_removed_path(nexus_file: Path) -> None: + ops = [ + patch.Remove(path='/entry/title'), + patch.Move(source=f'{DETECTOR}/depends_on', target='/entry/title'), + ] + with open_patched(nexus_file, ops) as f: + assert f['entry/title'][()] == f'{DETECTOR}/transformations/offset' + assert 'depends_on' not in f[DETECTOR] + + +def test_set_dataset_adds_dataset(nexus_file: Path) -> None: + ops = [ + patch.SetDataset(path='/entry/new', value=[1.0, 2.0]), + patch.SetAttrs(path='/entry/new', attrs={'units': 'mm'}), + ] + with open_patched(nexus_file, ops) as f: + sc.testing.assert_identical( + f['entry/new'][()], sc.array(dims=['dim_0'], values=[1.0, 2.0], unit='mm') + ) + + +def test_set_dataset_replaces_string_dataset(nexus_file: Path) -> None: + ops = [patch.SetDataset(path='/entry/title', value='patched title')] + with open_patched(nexus_file, ops) as f: + assert f['entry/title'][()] == 'patched title' + + +def test_set_dataset_raises_if_parent_does_not_exist(nexus_file: Path) -> None: + ops = [patch.SetDataset(path='/entry/missing/x', value=1)] + with pytest.raises(KeyError, match='/entry/missing'): + open_patched(nexus_file, ops) + + +def test_depends_on_resolves_in_patched_file(nexus_file: Path) -> None: + ops = [ + patch.Move( + source=f'{DETECTOR}/transformations/offset', + target=f'{DETECTOR}/transformations/moved', + ), + patch.SetAttrs(path=f'{DETECTOR}/transformations/moved', attrs={'units': 'cm'}), + patch.SetDataset( + path=f'{DETECTOR}/depends_on', value=f'{DETECTOR}/transformations/moved' + ), + ] + with open_patched(nexus_file, ops) as f: + detector = f[DETECTOR][()] + transformations = detector['depends_on'].transformations + assert list(transformations) == [f'{DETECTOR}/transformations/moved'] + assert transformations[f'{DETECTOR}/transformations/moved'].value.unit == 'cm' + + +def test_soft_link_resolves_in_patched_file(nexus_file: Path) -> None: + ops = [patch.SetDataset(path=f'{DETECTOR}/detector_number', value=[7, 8])] + with open_patched(nexus_file, ops) as f: + assert list(f['entry/number_link'][()].values) == [7, 8] + + +def test_load_patch_from_toml(tmp_path: Path) -> None: + path = tmp_path / 'patch.toml' + path.write_text( + ''' +[[op]] +kind = "set_attrs" +path = "/entry" +attrs = { vector = [0.0, 1.0], name = "x" } + +[[op]] +kind = "move" +source = "/a" +target = "/b" +''' + ) + assert patch.load_patch(path) == [ + patch.SetAttrs(path='/entry', attrs={'vector': [0.0, 1.0], 'name': 'x'}), + patch.Move(source='/a', target='/b'), + ] + + +def test_load_patch_raises_on_unknown_kind(tmp_path: Path) -> None: + path = tmp_path / 'patch.toml' + path.write_text('[[op]]\nkind = "rename"\npath = "/a"\n') + with pytest.raises(ValueError, match="'rename'"): + patch.load_patch(path) + + +def make_workflow(nexus_file: Path) -> GenericNeXusWorkflow: + wf = GenericNeXusWorkflow(run_types=[SampleRun], monitor_types=[]) + wf[Filename[SampleRun]] = nexus_file + wf[NeXusDetectorName] = 'detector' + return wf + + +@pytest.mark.parametrize('preopen', [False, True]) +def test_workflow_applies_patch(nexus_file: Path, preopen: bool) -> None: + wf = make_workflow(nexus_file) + wf[PreopenNeXusFile] = preopen + wf[NeXusFilePatch[SampleRun]] = [ + patch.SetAttrs(path=f'{DETECTOR}/transformations/offset', attrs={'units': 'cm'}) + ] + detector = wf.compute(EmptyDetector[SampleRun]) + sc.testing.assert_identical( + detector.coords['position'][0], sc.vector([0.0, 0.0, 0.02], unit='m') + ) + + +def test_workflow_does_not_patch_by_default(nexus_file: Path) -> None: + detector = make_workflow(nexus_file).compute(EmptyDetector[SampleRun]) + sc.testing.assert_identical( + detector.coords['position'][0], sc.vector([0.0, 0.0, 2.0], unit='m') + ) diff --git a/packages/essreduce/tests/nexus/workflow_test.py b/packages/essreduce/tests/nexus/workflow_test.py index 3b82e162a..204dc2283 100644 --- a/packages/essreduce/tests/nexus/workflow_test.py +++ b/packages/essreduce/tests/nexus/workflow_test.py @@ -236,7 +236,7 @@ def test_given_no_sample_load_nexus_sample_returns_group_with_origin_depends_on( loki_tutorial_sample_run_60250: Path, ) -> None: filespec = workflow.file_path_to_file_spec( - loki_tutorial_sample_run_60250, preopen=True + loki_tutorial_sample_run_60250, preopen=True, patch=[] ) spec = workflow.unique_component_spec(filespec) assert spec.filename['/entry'][snx.NXsample] == {} From cfe7693cd2e55f43e7215b7d0a011bcb9c5d335b Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 28 Oct 2026 09:18:06 +0000 Subject: [PATCH 2/2] feat: add LoKI patch for empty motion logs with TODO source Motion NXlogs of beamstop axes and the detector carriage were configured with a "TODO-" source prefix and are empty, while the motor readback was written to `/value`. The patch makes these files loadable with the standard workflows. It is also the first example of a tracked NeXus patch file. Co-Authored-By: Claude Opus 5.5 --- .../nexus_patches/todo_motion_sources.toml | 107 ++++++++++++++++++ .../esssans/tests/loki/nexus_patches_test.py | 57 ++++++++++ 2 files changed, 164 insertions(+) create mode 100644 packages/esssans/src/ess/loki/nexus_patches/todo_motion_sources.toml create mode 100644 packages/esssans/tests/loki/nexus_patches_test.py diff --git a/packages/esssans/src/ess/loki/nexus_patches/todo_motion_sources.toml b/packages/esssans/src/ess/loki/nexus_patches/todo_motion_sources.toml new file mode 100644 index 000000000..4f236839b --- /dev/null +++ b/packages/esssans/src/ess/loki/nexus_patches/todo_motion_sources.toml @@ -0,0 +1,107 @@ +# Motion NXlogs of LoKI beamstop axes and the detector carriage are empty. +# +# The file-writer configuration used a source name with a "TODO-" prefix for +# `/transformations/motion`, so no values were written. The motor +# readback (RBV) was written to the NXlog `/value` instead. +# This patch replaces each empty `motion` log by the readback log and copies the +# transformation attributes of the original `motion` log. +# +# Observed in run 336349 (2026-10-07). + +[[op]] +kind = "remove" +path = "/entry/instrument/beamstop_x/transformations/motion" + +[[op]] +kind = "move" +source = "/entry/instrument/beamstop_x/value" +target = "/entry/instrument/beamstop_x/transformations/motion" + +[[op]] +kind = "set_attrs" +path = "/entry/instrument/beamstop_x/transformations/motion" +attrs = { depends_on = "/entry/instrument/beamstop_x/transformations/distance", transformation_type = "translation", vector = [0.0, 0.0, 1.0] } + +[[op]] +kind = "remove" +path = "/entry/instrument/beamstop_y/transformations/motion" + +[[op]] +kind = "move" +source = "/entry/instrument/beamstop_y/value" +target = "/entry/instrument/beamstop_y/transformations/motion" + +[[op]] +kind = "set_attrs" +path = "/entry/instrument/beamstop_y/transformations/motion" +attrs = { depends_on = "/entry/instrument/beamstop_x/transformations/motion", transformation_type = "translation", vector = [1.0, 0.0, 0.0], offset = [0.0, 0.0, 0.0], offset_units = "m" } + +[[op]] +kind = "remove" +path = "/entry/instrument/beamstop_z1/transformations/motion" + +[[op]] +kind = "move" +source = "/entry/instrument/beamstop_z1/value" +target = "/entry/instrument/beamstop_z1/transformations/motion" + +[[op]] +kind = "set_attrs" +path = "/entry/instrument/beamstop_z1/transformations/motion" +attrs = { depends_on = "/entry/instrument/beamstop_y/transformations/motion", transformation_type = "translation", vector = [0.0, 1.0, 0.0], offset = [0.0, 0.0, 0.0], offset_units = "m" } + +[[op]] +kind = "remove" +path = "/entry/instrument/beamstop_z2/transformations/motion" + +[[op]] +kind = "move" +source = "/entry/instrument/beamstop_z2/value" +target = "/entry/instrument/beamstop_z2/transformations/motion" + +[[op]] +kind = "set_attrs" +path = "/entry/instrument/beamstop_z2/transformations/motion" +attrs = { depends_on = "/entry/instrument/beamstop_y/transformations/motion", transformation_type = "translation", vector = [0.0, 1.0, 0.0], offset = [0.0, 0.0, 0.0], offset_units = "m" } + +[[op]] +kind = "remove" +path = "/entry/instrument/beamstop_z3/transformations/motion" + +[[op]] +kind = "move" +source = "/entry/instrument/beamstop_z3/value" +target = "/entry/instrument/beamstop_z3/transformations/motion" + +[[op]] +kind = "set_attrs" +path = "/entry/instrument/beamstop_z3/transformations/motion" +attrs = { depends_on = "/entry/instrument/beamstop_y/transformations/motion", transformation_type = "translation", vector = [0.0, 1.0, 0.0], offset = [0.0, 0.0, 0.0], offset_units = "m" } + +[[op]] +kind = "remove" +path = "/entry/instrument/beamstop_z4/transformations/motion" + +[[op]] +kind = "move" +source = "/entry/instrument/beamstop_z4/value" +target = "/entry/instrument/beamstop_z4/transformations/motion" + +[[op]] +kind = "set_attrs" +path = "/entry/instrument/beamstop_z4/transformations/motion" +attrs = { depends_on = "/entry/instrument/beamstop_y/transformations/motion", transformation_type = "translation", vector = [0.0, 1.0, 0.0], offset = [0.0, 0.0, 0.0], offset_units = "m" } + +[[op]] +kind = "remove" +path = "/entry/instrument/detector_carriage/transformations/motion" + +[[op]] +kind = "move" +source = "/entry/instrument/detector_carriage/value" +target = "/entry/instrument/detector_carriage/transformations/motion" + +[[op]] +kind = "set_attrs" +path = "/entry/instrument/detector_carriage/transformations/motion" +attrs = { depends_on = "/entry/instrument/detector_carriage/transformations/detector_carriage_zero", transformation_type = "translation", vector = [0.0, 0.0, 1.0], offset = [0.0, 0.0, 0.0], offset_units = "m" } diff --git a/packages/esssans/tests/loki/nexus_patches_test.py b/packages/esssans/tests/loki/nexus_patches_test.py new file mode 100644 index 000000000..2766d686f --- /dev/null +++ b/packages/esssans/tests/loki/nexus_patches_test.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Scipp contributors (https://github.com/scipp) +from importlib.resources import files +from pathlib import Path + +import pytest +import sciline +import scipp as sc +import scippnexus as snx + +from ess.reduce.nexus import GenericNeXusWorkflow, patch +from ess.reduce.nexus.types import ( + EmptyDetector, + EmptyMonitor, + Filename, + IncidentMonitor, + NeXusDetectorName, + NeXusFilePatch, + NeXusName, + NeXusTransformationChain, + SampleRun, +) + +CARRIAGE_MOTION = '/entry/instrument/detector_carriage/transformations/motion' + + +@pytest.fixture +def workflow(loki_broken_file: Path) -> sciline.Pipeline: + wf = GenericNeXusWorkflow(run_types=[SampleRun], monitor_types=[IncidentMonitor]) + wf[Filename[SampleRun]] = loki_broken_file + wf[NeXusDetectorName] = 'loki_detector_0' + wf[NeXusName[IncidentMonitor]] = 'beam_monitor_m4' + return wf + + +def test_broken_file_cannot_be_loaded_without_patch( + workflow: sciline.Pipeline, +) -> None: + with pytest.raises(ValueError, match='time-dependent'): + workflow.compute(EmptyDetector[SampleRun]) + + +def test_todo_motion_sources_patch_uses_motor_readback( + workflow: sciline.Pipeline, +) -> None: + workflow[NeXusFilePatch[SampleRun]] = patch.load_patch( + files('ess.loki') / 'nexus_patches' / 'todo_motion_sources.toml' + ) + chain = workflow.compute(NeXusTransformationChain[snx.NXdetector, SampleRun]) + motion = chain.transformations[CARRIAGE_MOTION].value + assert motion.sizes == {'time': 1} + assert sc.isclose(motion.data[0], sc.scalar(4999.96164649, unit='mm')) + + detector = workflow.compute(EmptyDetector[SampleRun]) + monitor = workflow.compute(EmptyMonitor[SampleRun, IncidentMonitor]) + assert sc.all(sc.isfinite(detector.coords['position'])).value + assert sc.all(sc.isfinite(monitor.coords['position'])).value