diff --git a/packages/essreduce/src/ess/reduce/nexus/__init__.py b/packages/essreduce/src/ess/reduce/nexus/__init__.py index e739a622f..8daf77ba7 100644 --- a/packages/essreduce/src/ess/reduce/nexus/__init__.py +++ b/packages/essreduce/src/ess/reduce/nexus/__init__.py @@ -12,7 +12,7 @@ The submodule :mod:`types` defines all domain types. """ -from . import types +from . import patch, types from ._nexus_loader import ( compute_component_position, compute_detector_position, @@ -39,5 +39,6 @@ 'load_from_path', 'open_component_group', 'open_nexus_file', + 'patch', 'types', ] diff --git a/packages/essreduce/src/ess/reduce/nexus/_nexus_loader.py b/packages/essreduce/src/ess/reduce/nexus/_nexus_loader.py index 606e4dc67..a13a25eaf 100644 --- a/packages/essreduce/src/ess/reduce/nexus/_nexus_loader.py +++ b/packages/essreduce/src/ess/reduce/nexus/_nexus_loader.py @@ -14,6 +14,7 @@ import scippnexus as snx from ..logging import get_logger +from .patch import open_patched_file from .types import ( Beamline, FilePath, @@ -24,6 +25,7 @@ NeXusGroup, NeXusLocationSpec, NeXusTransformation, + PatchedFilePath, RunType, ) @@ -129,7 +131,7 @@ def load_all_components( def load_metadata( - file_path: FilePath | NeXusFile | NeXusGroup, + file_path: FilePath | NeXusFile | NeXusGroup | PatchedFilePath, model: type[_Model], *, entry_name: NeXusEntryName | None = None, @@ -164,7 +166,7 @@ def compute_component_position(dg: sc.DataGroup) -> sc.DataGroup: def open_nexus_file( - file_path: FilePath | NeXusFile | NeXusGroup, + file_path: FilePath | NeXusFile | NeXusGroup | PatchedFilePath, definitions: Mapping | None | NoNewDefinitionsType = NoNewDefinitions, *, locking: bool | str | None | NoLockingIfNeededType = NoLockingIfNeeded, @@ -175,6 +177,8 @@ def open_nexus_file( ---------- file_path: Path of the file to open or a NeXus file or group handle. + If this is a :class:`ess.reduce.nexus.types.PatchedFilePath`, + the patch is applied to the file. definitions: If set, application definitions to use for the file. If ``file_path`` is a NeXus file or group, this must be unset or match @@ -266,26 +270,31 @@ def _attempt_to_open_without_locking( def _open_nexus_file_from_path( - file_path: FilePath, + file_path: FilePath | NeXusFile | PatchedFilePath, definitions: Mapping | None | NoNewDefinitionsType, **kwargs: object, ) -> AbstractContextManager[snx.Group]: - new_kwargs = ( - kwargs - | ({} if definitions is NoNewDefinitions else {'definitions': definitions}) - # ESS nexus files are always written in swmr mode. - # By default, try to open in swmr mode. - | ({'swmr': True} if 'swmr' not in kwargs else {}) - ) + defs = snx.base_definitions() if definitions is NoNewDefinitions else definitions + + def open_file(**h5_kwargs: object) -> AbstractContextManager[snx.Group]: + if isinstance(file_path, PatchedFilePath): + return open_patched_file( + file_path.path, file_path.patch, definitions=defs, **h5_kwargs + ) + return snx.File(file_path, definitions=defs, **h5_kwargs) + + # ESS nexus files are always written in swmr mode. + # By default, try to open in swmr mode. + new_kwargs = kwargs | ({'swmr': True} if 'swmr' not in kwargs else {}) try: - return snx.File(file_path, **new_kwargs) + return open_file(**new_kwargs) except OSError as e: # If the file type does not support swmr # and the caller did not explicitly request swmr # open the file without swmr. if 'swmr' not in kwargs: new_kwargs.pop('swmr') - return snx.File(file_path, **new_kwargs) + return open_file(**new_kwargs) raise e @@ -506,7 +515,7 @@ def compute_detector_position( def load_data( - file_path: FilePath | NeXusFile | NeXusGroup, + file_path: FilePath | NeXusFile | NeXusGroup | PatchedFilePath, selection: snx.typing.ScippIndex | slice = (), *, entry_name: NeXusEntryName | None = None, @@ -755,7 +764,9 @@ def _parse_monitor(group: snx.Group) -> NeXusMonitorInfo: ) -def read_nexus_file_info(file_path: FilePath | NeXusFile | NeXusGroup) -> NeXusFileInfo: +def read_nexus_file_info( + file_path: FilePath | NeXusFile | NeXusGroup | PatchedFilePath, +) -> NeXusFileInfo: """Opens and inspects a NeXus file, returning a summary of its contents.""" with open_nexus_file(file_path) as f: entry = _unique_child_group(f, snx.NXentry, None) diff --git a/packages/essreduce/src/ess/reduce/nexus/patch.py b/packages/essreduce/src/ess/reduce/nexus/patch.py new file mode 100644 index 000000000..90c856c3c --- /dev/null +++ b/packages/essreduce/src/ess/reduce/nexus/patch.py @@ -0,0 +1,471 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Scipp contributors (https://github.com/scipp) +"""Patches for fixing defects in NeXus files while loading them. + +A patch is a list of operations, applied in order to a read-only view of a file. +The file on disk is not modified and no data is copied: +Datasets are read from the original file when accessed. + +All paths in operations are absolute paths in the patched file, +i.e., they take preceding operations into account. +Patches apply to paths, not to objects: +If an object is reachable under multiple paths through hard links, +a patch for one path does not affect the others. +Soft links are resolved in the patched file. +As in HDF5, moving or removing the target of a soft link breaks the link. + +Patches are typically stored in TOML files and loaded with :func:`load_patch`: + +.. code-block:: toml + + # Explain the defect being fixed here. + [[op]] + kind = "move" + source = "/entry/instrument/detector/value" + target = "/entry/instrument/detector/transformations/motion" + + [[op]] + kind = "set_attrs" + path = "/entry/instrument/detector/transformations/motion" + attrs = { transformation_type = "translation", vector = [0.0, 0.0, 1.0] } + +A patch is a plain list, so it can be amended in memory before use: + +.. code-block:: python + + from ess.reduce.nexus import patch + + p = patch.load_patch('fix.toml') + [patch.Remove(path='/entry/broken')] + workflow[NeXusFilePatch[SampleRun]] = p +""" + +from __future__ import annotations + +import os +import posixpath +import uuid +from collections.abc import Iterable, Iterator, Mapping, Sequence +from contextlib import AbstractContextManager +from dataclasses import dataclass, replace +from typing import Any + +import h5py +import scippnexus as snx +import tomllib + + +@dataclass(frozen=True, kw_only=True) +class SetAttrs: + """Add attributes or replace existing ones.""" + + path: str + attrs: Mapping[str, Any] + + +@dataclass(frozen=True, kw_only=True) +class DeleteAttrs: + """Delete attributes. All attributes must exist.""" + + path: str + names: Sequence[str] + + +@dataclass(frozen=True, kw_only=True) +class Remove: + """Remove a group or dataset.""" + + path: str + + +@dataclass(frozen=True, kw_only=True) +class Move: + """Move or rename a group or dataset. The target must not exist.""" + + source: str + target: str + + +@dataclass(frozen=True, kw_only=True) +class SetDataset: + """Add a dataset or replace an existing dataset. + + The parent group must exist. + A replaced dataset does not keep its attributes. + """ + + path: str + value: Any + + +PatchOp = SetAttrs | DeleteAttrs | Remove | Move | SetDataset + +_op_kinds: dict[str, type[PatchOp]] = { + 'set_attrs': SetAttrs, + 'delete_attrs': DeleteAttrs, + 'remove': Remove, + 'move': Move, + 'set_dataset': SetDataset, +} + + +def load_patch(path: str | os.PathLike[str]) -> list[PatchOp]: + """Load a patch from a TOML file. + + The file contains an array of tables named ``op``. + Each table has a ``kind`` (one of ``set_attrs``, ``delete_attrs``, ``remove``, + ``move``, ``set_dataset``) and the fields of the corresponding operation class. + """ + with open(path, 'rb') as f: + content = tomllib.load(f) + if unknown := set(content) - {'op'}: + raise ValueError(f"Unknown top-level keys in NeXus patch {path}: {unknown}") + ops = [] + for entry in content.get('op', []): + fields = dict(entry) + kind = fields.pop('kind') + if (op := _op_kinds.get(kind)) is None: + raise ValueError( + f"Unknown operation kind '{kind}' in NeXus patch {path}, " + f"expected one of {list(_op_kinds)}" + ) + ops.append(op(**fields)) + return ops + + +def open_patched_file( + path: str | os.PathLike[str], + patch: Iterable[PatchOp], + *, + definitions: Mapping[str, type] | None, + **kwargs: Any, +) -> PatchedNeXusFile: + """Open a NeXus file and apply a patch. + + Parameters + ---------- + path: + Path of the file to open. + patch: + Operations to apply. + definitions: + Application definitions to use for the file, see :class:`scippnexus.File`. + kwargs: + Forwarded to :class:`h5py.File`. + + Returns + ------- + : + The patched file. + Can be used as a context manager to close the file. + """ + file = h5py.File(path, 'r', **kwargs) + try: + return PatchedNeXusFile(PatchedH5File(file, patch), definitions=definitions) + except BaseException: + file.close() + raise + + +class PatchedNeXusFile(snx.Group, AbstractContextManager['PatchedNeXusFile']): + """ScippNexus group for a patched file that closes the file on exit.""" + + def __init__( + self, file: PatchedH5File, *, definitions: Mapping[str, type] | None + ) -> None: + super().__init__(file, definitions=definitions) + self._patched_file = file + + def __exit__(self, *args: object) -> None: + self.close() + + def close(self) -> None: + self._patched_file.close() + + +class _PatchedAttrs(Mapping[str, Any]): + def __init__( + self, + base: h5py.AttributeManager, + new: h5py.AttributeManager | None, + deleted: set[str], + ) -> None: + self._base = base + self._new = new + self._deleted = deleted + + def __getitem__(self, name: str) -> Any: + if self._new is not None and name in self._new: + return self._new[name] + if name in self._deleted: + raise KeyError(name) + return self._base[name] + + def __iter__(self) -> Iterator[str]: + new = () if self._new is None else tuple(self._new) + yield from new + yield from (k for k in self._base if k not in self._deleted and k not in new) + + def __len__(self) -> int: + return sum(1 for _ in self) + + def get_id(self, name: bytes) -> Any: + """Used by ScippNexus to determine the encoding of string attributes.""" + if self._new is not None and name.decode() in self._new: + return self._new.get_id(name) + return self._base.get_id(name) + + +class _Node: + def __init__( + self, obj: h5py.Group | h5py.Dataset, path: str, file: PatchedH5File + ) -> None: + self._obj = obj + self._path = path + self._file = file + + @property + def name(self) -> str: + return self._path + + @property + def file(self) -> PatchedH5File: + return self._file + + @property + def parent(self) -> _Group: + return self._file[posixpath.dirname(self._path)] + + @property + def attrs(self) -> _PatchedAttrs: + return _PatchedAttrs( + self._obj.attrs, + self._file._new_attrs.get(self._path), + self._file._deleted_attrs.get(self._path, set()), + ) + + def __repr__(self) -> str: + return f'<{type(self).__name__} "{self._path}">' + + +class _Dataset(_Node): + def __getattr__(self, name: str) -> Any: + # shape, dtype, ndim, read_direct, asstr, ... of the underlying dataset. + return getattr(self._obj, name) + + def __getitem__(self, index: Any) -> Any: + return self._obj[index] + + def __len__(self) -> int: + return len(self._obj) + + +class _Group(_Node): + def _child_path(self, name: str) -> str: + return posixpath.join(self._path, name) + + def keys(self) -> list[str]: + names = [ + name + for name in self._obj + if not self._file._is_removed(self._child_path(name)) + ] + for path, obj in self._file._mounts.items(): + name = posixpath.basename(path) + if ( + obj is not None + and posixpath.dirname(path) == self._path + and name not in names + ): + names.append(name) + return names + + def __iter__(self) -> Iterator[str]: + return iter(self.keys()) + + def __len__(self) -> int: + return len(self.keys()) + + def items(self) -> list[tuple[str, _Group | _Dataset | None]]: + # As in h5py, dangling soft links yield None. + return [(name, self.get(name)) for name in self.keys()] + + def __contains__(self, name: str) -> bool: + try: + self[name] + except KeyError: + return False + return True + + def __getitem__(self, name: str) -> _Group | _Dataset: + path = posixpath.normpath(posixpath.join(self._path, name)) + if '/' in name: + obj = self._file._lookup(path) + else: + obj = self._file._child(self._obj, path) + return self._file._wrap(obj, path) + + def get(self, name: str) -> _Group | _Dataset | None: + try: + return self[name] + except KeyError: + return None + + +class PatchedH5File(_Group): + """Read-only view of an HDF5 file with a patch applied. + + Implements the subset of the :mod:`h5py` interface used by ScippNexus. + Closing this file also closes the underlying file. + """ + + def __init__(self, file: h5py.File, patch: Iterable[PatchOp]) -> None: + super().__init__(file, '/', self) + # Objects replacing or added to the original file, by path in the patched + # file. ``None`` marks removed objects. + self._mounts: dict[str, h5py.Group | h5py.Dataset | None] = {} + self._new_attrs: dict[str, h5py.AttributeManager] = {} + self._deleted_attrs: dict[str, set[str]] = {} + # New datasets and attributes are stored in an in-memory HDF5 file so they + # support the full h5py interface, e.g., string encodings and `asstr`. + self._scratch = h5py.File( + uuid.uuid4().hex, 'w', driver='core', backing_store=False + ) + for op in patch: + self._apply(op) + + @property + def parent(self) -> PatchedH5File: + return self + + def close(self) -> None: + self._scratch.close() + self._obj.close() + + def __getitem__(self, name: str) -> _Group | _Dataset: + if posixpath.normpath(name) == '/': + return self + return super().__getitem__(name) + + def _wrap(self, obj: h5py.Group | h5py.Dataset, path: str) -> _Group | _Dataset: + if isinstance(obj, h5py.Dataset): + return _Dataset(obj, path, self) + return _Group(obj, path, self) + + def _is_removed(self, path: str) -> bool: + return path in self._mounts and self._mounts[path] is None + + def _scratch_path(self, kind: str) -> str: + return f'{kind}/{len(self._scratch.require_group(kind))}' + + def _child(self, parent: h5py.Group, path: str) -> h5py.Group | h5py.Dataset: + """Return the child of ``parent`` with the given path in the patched file.""" + if self._is_removed(path): + raise KeyError(f"'{path}' was removed by a patch") + if (obj := self._mounts.get(path)) is not None: + return obj + name = posixpath.basename(path) + link = parent.get(name, getlink=True) + if link is None: + raise KeyError(f"'{path}' not found") + if isinstance(link, h5py.SoftLink): + return self._lookup( + posixpath.normpath(posixpath.join(posixpath.dirname(path), link.path)) + ) + return parent[name] + + def _lookup(self, path: str) -> h5py.Group | h5py.Dataset: + obj = self._obj + current = '/' + for name in path.strip('/').split('/'): + if not isinstance(obj, h5py.Group): + raise KeyError(f"'{path}' not found, '{current}' is not a group") + current = posixpath.join(current, name) + obj = self._child(obj, current) + return obj + + def _require(self, path: str) -> _Group | _Dataset: + try: + return self[path] + except KeyError: + raise KeyError( + f"Cannot apply patch, '{path}' does not exist in the patched file" + ) from None + + def _require_parent_group(self, path: str) -> None: + if not isinstance(self._require(posixpath.dirname(path)), _Group): + raise ValueError(f"Cannot apply patch, parent of '{path}' is not a group") + + def _drop(self, path: str) -> None: + """Forget patches of ``path`` and everything below it.""" + for patches in (self._mounts, self._new_attrs, self._deleted_attrs): + for key in [k for k in patches if _is_within(k, path)]: + del patches[key] + + def _apply(self, op: PatchOp) -> None: + match _normalize_paths(op): + case SetAttrs(path=path, attrs=attrs): + self._require(path) + if path not in self._new_attrs: + holder = self._scratch.create_group(self._scratch_path('attrs')) + self._new_attrs[path] = holder.attrs + self._new_attrs[path].update(attrs) + self._deleted_attrs.get(path, set()).difference_update(attrs) + case DeleteAttrs(path=path, names=names): + if missing := set(names) - set(self._require(path).attrs): + raise KeyError( + f"Cannot apply patch, attributes {missing} " + f"do not exist at '{path}'" + ) + new = self._new_attrs.get(path, {}) + for name in names: + if name in new: + del new[name] + self._deleted_attrs.setdefault(path, set()).update(names) + case Remove(path=path): + if path == '/': + raise ValueError("Cannot remove the root group") + self._require(path) + self._drop(path) + self._mounts[path] = None + case Move(source=source, target=target): + obj = self._require(source)._obj + if _is_within(target, source): + raise ValueError(f"Cannot move '{source}' into itself") + if target in self: + raise ValueError(f"Cannot move '{source}', '{target}' exists") + self._require_parent_group(target) + self._drop(target) + for patches in (self._mounts, self._new_attrs, self._deleted_attrs): + for key in [k for k in patches if _is_within(k, source)]: + patches[target + key[len(source) :]] = patches.pop(key) + self._mounts[target] = obj + self._mounts[source] = None + case SetDataset(path=path, value=value): + self._require_parent_group(path) + if isinstance(self.get(path), _Group): + raise ValueError(f"Cannot replace group '{path}' by a dataset") + self._drop(path) + self._mounts[path] = self._scratch.create_dataset( + self._scratch_path('datasets'), data=value + ) + case _: + raise TypeError(f"Unknown patch operation {op!r}") + + +def _normalize_paths(op: PatchOp) -> PatchOp: + paths = { + field: getattr(op, field) + for field in ('path', 'source', 'target') + if hasattr(op, field) + } + for path in paths.values(): + if not path.startswith('/'): + raise ValueError(f"Patch paths must be absolute, got '{path}'") + # Note that normpath preserves a leading '//'. + return replace( + op, **{k: posixpath.normpath('/' + v.lstrip('/')) for k, v in paths.items()} + ) + + +def _is_within(path: str, base: str) -> bool: + return path == base or path.startswith(base.rstrip('/') + '/') diff --git a/packages/essreduce/src/ess/reduce/nexus/types.py b/packages/essreduce/src/ess/reduce/nexus/types.py index ccf7d9286..c8f04ea92 100644 --- a/packages/essreduce/src/ess/reduce/nexus/types.py +++ b/packages/essreduce/src/ess/reduce/nexus/types.py @@ -10,6 +10,8 @@ from scippneutron import metadata as scn_meta from scippneutron.chopper import DiskChopper +from .patch import PatchOp + FilePath = NewType('FilePath', Path) """Full path to a NeXus file on disk.""" NeXusFile = NewType('NeXusFile', BinaryIO) @@ -24,6 +26,18 @@ NeXusGroup = NewType('NeXusGroup', snx.Group) """A ScippNexus group in an open file.""" + +@dataclass(frozen=True, kw_only=True) +class PatchedFilePath: + """Path to a NeXus file and a patch to apply when opening it. + + See :mod:`ess.reduce.nexus.patch`. + """ + + path: FilePath + patch: list[PatchOp] + + NeXusEntryName = NewType('NeXusEntryName', str) """Name of an entry in a NeXus file.""" NeXusSourceName = NewType('NeXusSourceName', str) @@ -277,6 +291,17 @@ class TimeInterval(Generic[RunType]): value: slice +class NeXusFilePatch(Generic[RunType]): + """Patch to apply to the NeXus file of a run when loading it. + + The default is an empty patch, i.e., the file is loaded as is. + See :mod:`ess.reduce.nexus.patch`. + """ + + def __new__(cls, x: list[PatchOp]) -> Any: + return x + + class TransformationTimeFilter(Generic[Component, RunType]): """Filter for time-dependent transformations.""" @@ -288,7 +313,7 @@ def __call__(self, transform: sc.DataArray) -> sc.Variable | sc.DataArray: ... @dataclass class NeXusFileSpec(Generic[RunType]): - value: FilePath | NeXusFile | NeXusGroup + value: FilePath | NeXusFile | NeXusGroup | PatchedFilePath @dataclass @@ -297,7 +322,7 @@ class NeXusLocationSpec: NeXus filename and optional parameters to identify (parts of) a component to load. """ - filename: FilePath | NeXusFile | NeXusGroup + filename: FilePath | NeXusFile | NeXusGroup | PatchedFilePath entry_name: NeXusEntryName | None = None component_name: str | None = None selection: snx.typing.ScippIndex | slice = () @@ -316,7 +341,7 @@ class NeXusAllLocationSpec: NeXus parameters to identify all components of a class to load. """ - filename: FilePath | NeXusFile | NeXusGroup + filename: FilePath | NeXusFile | NeXusGroup | PatchedFilePath entry_name: NeXusEntryName | None = None selection: snx.typing.ScippIndex | slice = () diff --git a/packages/essreduce/src/ess/reduce/nexus/workflow.py b/packages/essreduce/src/ess/reduce/nexus/workflow.py index e9c06c6d7..d5a1e02f9 100644 --- a/packages/essreduce/src/ess/reduce/nexus/workflow.py +++ b/packages/essreduce/src/ess/reduce/nexus/workflow.py @@ -18,6 +18,7 @@ from scippneutron.chopper import DiskChopper, extract_chopper_from_nexus from scippneutron.metadata import RadiationProbe, SourceType +from ..logging import get_logger from . import _nexus_loader as nexus from .types import ( COMPONENT_CONSTRAINTS, @@ -41,10 +42,12 @@ NeXusComponentLocationSpec, NeXusData, NeXusDataLocationSpec, + NeXusFilePatch, NeXusFileSpec, NeXusName, NeXusTransformation, NeXusTransformationChain, + PatchedFilePath, Position, PreopenNeXusFile, ProtonCharge, @@ -65,15 +68,28 @@ def file_path_to_file_spec( - filename: Filename[RunType], preopen: PreopenNeXusFile + filename: Filename[RunType], + preopen: PreopenNeXusFile, + patch: NeXusFilePatch[RunType], ) -> NeXusFileSpec[RunType]: + location = filename + if patch: + get_logger().info( + "Applying NeXus patch with %d operations to %s", len(patch), filename + ) + location = PatchedFilePath(path=filename, patch=patch) return NeXusFileSpec[RunType]( - nexus.open_nexus_file(filename, definitions=definitions) + nexus.open_nexus_file(location, definitions=definitions) if preopen - else filename + else location ) +def no_nexus_file_patch() -> NeXusFilePatch[RunType]: + """Load NeXus files without patching them.""" + return NeXusFilePatch[RunType]([]) + + def no_monitor_position_offset() -> MonitorPositionOffset[RunType, MonitorType]: return MonitorPositionOffset[RunType, MonitorType](no_offset) @@ -813,6 +829,7 @@ def load_source_metadata_from_nexus( _common_providers = ( gravity_vector_neg_y, file_path_to_file_spec, + no_nexus_file_patch, full_time_interval, component_spec_by_name, unique_component_spec, # after component_spec_by_name, partially overrides diff --git a/packages/essreduce/tests/nexus/patch_test.py b/packages/essreduce/tests/nexus/patch_test.py new file mode 100644 index 000000000..39de797e7 --- /dev/null +++ b/packages/essreduce/tests/nexus/patch_test.py @@ -0,0 +1,284 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Scipp contributors (https://github.com/scipp) +from pathlib import Path + +import h5py +import numpy as np +import pytest +import scipp as sc +import scipp.testing +import scippnexus as snx + +from ess.reduce.nexus import GenericNeXusWorkflow, patch +from ess.reduce.nexus.types import ( + EmptyDetector, + Filename, + NeXusDetectorName, + NeXusFilePatch, + PreopenNeXusFile, + SampleRun, +) + +DETECTOR = '/entry/instrument/detector' + + +@pytest.fixture +def nexus_file(tmp_path: Path) -> Path: + path = tmp_path / 'file.nxs' + with h5py.File(path, 'w') as f: + entry = f.create_group('entry') + entry.attrs['NX_class'] = 'NXentry' + entry['title'] = 'original title' + instrument = entry.create_group('instrument') + instrument.attrs['NX_class'] = 'NXinstrument' + detector = instrument.create_group('detector') + detector.attrs['NX_class'] = 'NXdetector' + detector.attrs['description'] = 'a detector' + detector['detector_number'] = np.arange(1, 5) + detector['data'] = np.ones(4) + for name in ('x_pixel_offset', 'y_pixel_offset', 'z_pixel_offset'): + detector.create_dataset(name, data=np.zeros(4)).attrs['units'] = 'm' + detector['depends_on'] = f'{DETECTOR}/transformations/offset' + transformations = detector.create_group('transformations') + transformations.attrs['NX_class'] = 'NXtransformations' + offset = transformations.create_dataset('offset', data=2.0) + offset.attrs['transformation_type'] = 'translation' + offset.attrs['vector'] = [0.0, 0.0, 1.0] + offset.attrs['units'] = 'm' + offset.attrs['depends_on'] = '.' + entry['number_link'] = h5py.SoftLink(f'{DETECTOR}/detector_number') + return path + + +def open_patched(path: Path, ops: list[patch.PatchOp]) -> patch.PatchedNeXusFile: + return patch.open_patched_file(path, ops, definitions=snx.base_definitions()) + + +def test_empty_patch_loads_same_as_unpatched_file(nexus_file: Path) -> None: + with snx.File(nexus_file) as f: + expected = f[DETECTOR][()] + with open_patched(nexus_file, []) as f: + sc.testing.assert_identical(f[DETECTOR][()], expected) + + +def test_patch_does_not_modify_file(nexus_file: Path) -> None: + ops = [ + patch.Remove(path='/entry/title'), + patch.SetAttrs(path=DETECTOR, attrs={'description': 'patched'}), + ] + with open_patched(nexus_file, ops) as f: + assert 'title' not in f['entry'] + with h5py.File(nexus_file) as f: + assert f['entry/title'][()] == b'original title' + assert f[DETECTOR].attrs['description'] == 'a detector' + + +def test_set_attrs_adds_and_replaces_attributes(nexus_file: Path) -> None: + ops = [ + patch.SetAttrs(path=DETECTOR, attrs={'description': 'patched', 'new': 1.5}), + ] + with open_patched(nexus_file, ops) as f: + attrs = f[DETECTOR].attrs + assert attrs['description'] == 'patched' + assert attrs['new'] == 1.5 + assert attrs['NX_class'] == 'NXdetector' + + +def test_set_attrs_on_dataset(nexus_file: Path) -> None: + ops = [patch.SetAttrs(path='/entry/title', attrs={'units': 'm'})] + with open_patched(nexus_file, ops) as f: + assert f['entry/title'].attrs['units'] == 'm' + + +def test_set_attrs_raises_if_path_does_not_exist(nexus_file: Path) -> None: + with pytest.raises(KeyError, match='/entry/missing'): + open_patched(nexus_file, [patch.SetAttrs(path='/entry/missing', attrs={})]) + + +def test_patch_paths_must_be_absolute(nexus_file: Path) -> None: + with pytest.raises(ValueError, match='absolute'): + open_patched(nexus_file, [patch.Remove(path='entry/title')]) + + +def test_patch_paths_are_normalized(nexus_file: Path) -> None: + ops = [ + patch.SetAttrs(path=f'/{DETECTOR}/', attrs={'x': 1}), + patch.SetAttrs(path=f'{DETECTOR}/../detector', attrs={'y': 2}), + ] + with open_patched(nexus_file, ops) as f: + assert f[DETECTOR].attrs['x'] == 1 + assert f[DETECTOR].attrs['y'] == 2 + + +def test_cannot_remove_root(nexus_file: Path) -> None: + with pytest.raises(ValueError, match='root'): + open_patched(nexus_file, [patch.Remove(path='/')]) + + +def test_delete_attrs(nexus_file: Path) -> None: + ops = [ + patch.SetAttrs(path=DETECTOR, attrs={'new': 1}), + patch.DeleteAttrs(path=DETECTOR, names=['description', 'new']), + ] + with open_patched(nexus_file, ops) as f: + assert set(f[DETECTOR].attrs) == {'NX_class'} + + +def test_delete_attrs_raises_if_attribute_does_not_exist(nexus_file: Path) -> None: + with pytest.raises(KeyError, match='missing'): + open_patched(nexus_file, [patch.DeleteAttrs(path=DETECTOR, names=['missing'])]) + + +def test_remove_group(nexus_file: Path) -> None: + ops = [patch.Remove(path=f'{DETECTOR}/transformations')] + with open_patched(nexus_file, ops) as f: + assert 'transformations' not in f[DETECTOR] + with pytest.raises(KeyError): + f[f'{DETECTOR}/transformations/offset'] + + +def test_remove_raises_if_path_does_not_exist(nexus_file: Path) -> None: + ops = [patch.Remove(path='/entry/title'), patch.Remove(path='/entry/title')] + with pytest.raises(KeyError, match='/entry/title'): + open_patched(nexus_file, ops) + + +def test_move_renames_group(nexus_file: Path) -> None: + ops = [patch.Move(source=DETECTOR, target='/entry/instrument/renamed')] + with open_patched(nexus_file, ops) as f: + instrument = f['entry/instrument'] + assert list(instrument) == ['renamed'] + renamed = instrument['renamed'] + assert renamed.name == '/entry/instrument/renamed' + assert renamed['transformations/offset'].name == ( + '/entry/instrument/renamed/transformations/offset' + ) + + +def test_move_keeps_patches_of_moved_objects(nexus_file: Path) -> None: + ops = [ + patch.SetAttrs(path=f'{DETECTOR}/transformations', attrs={'x': 1}), + patch.Remove(path=f'{DETECTOR}/detector_number'), + patch.Move(source=DETECTOR, target='/entry/detector'), + ] + with open_patched(nexus_file, ops) as f: + assert f['entry/detector/transformations'].attrs['x'] == 1 + assert 'detector_number' not in f['entry/detector'] + + +def test_move_raises_if_target_exists(nexus_file: Path) -> None: + ops = [patch.Move(source='/entry/title', target='/entry/instrument')] + with pytest.raises(ValueError, match='exists'): + open_patched(nexus_file, ops) + + +def test_move_onto_removed_path(nexus_file: Path) -> None: + ops = [ + patch.Remove(path='/entry/title'), + patch.Move(source=f'{DETECTOR}/depends_on', target='/entry/title'), + ] + with open_patched(nexus_file, ops) as f: + assert f['entry/title'][()] == f'{DETECTOR}/transformations/offset' + assert 'depends_on' not in f[DETECTOR] + + +def test_set_dataset_adds_dataset(nexus_file: Path) -> None: + ops = [ + patch.SetDataset(path='/entry/new', value=[1.0, 2.0]), + patch.SetAttrs(path='/entry/new', attrs={'units': 'mm'}), + ] + with open_patched(nexus_file, ops) as f: + sc.testing.assert_identical( + f['entry/new'][()], sc.array(dims=['dim_0'], values=[1.0, 2.0], unit='mm') + ) + + +def test_set_dataset_replaces_string_dataset(nexus_file: Path) -> None: + ops = [patch.SetDataset(path='/entry/title', value='patched title')] + with open_patched(nexus_file, ops) as f: + assert f['entry/title'][()] == 'patched title' + + +def test_set_dataset_raises_if_parent_does_not_exist(nexus_file: Path) -> None: + ops = [patch.SetDataset(path='/entry/missing/x', value=1)] + with pytest.raises(KeyError, match='/entry/missing'): + open_patched(nexus_file, ops) + + +def test_depends_on_resolves_in_patched_file(nexus_file: Path) -> None: + ops = [ + patch.Move( + source=f'{DETECTOR}/transformations/offset', + target=f'{DETECTOR}/transformations/moved', + ), + patch.SetAttrs(path=f'{DETECTOR}/transformations/moved', attrs={'units': 'cm'}), + patch.SetDataset( + path=f'{DETECTOR}/depends_on', value=f'{DETECTOR}/transformations/moved' + ), + ] + with open_patched(nexus_file, ops) as f: + detector = f[DETECTOR][()] + transformations = detector['depends_on'].transformations + assert list(transformations) == [f'{DETECTOR}/transformations/moved'] + assert transformations[f'{DETECTOR}/transformations/moved'].value.unit == 'cm' + + +def test_soft_link_resolves_in_patched_file(nexus_file: Path) -> None: + ops = [patch.SetDataset(path=f'{DETECTOR}/detector_number', value=[7, 8])] + with open_patched(nexus_file, ops) as f: + assert list(f['entry/number_link'][()].values) == [7, 8] + + +def test_load_patch_from_toml(tmp_path: Path) -> None: + path = tmp_path / 'patch.toml' + path.write_text( + ''' +[[op]] +kind = "set_attrs" +path = "/entry" +attrs = { vector = [0.0, 1.0], name = "x" } + +[[op]] +kind = "move" +source = "/a" +target = "/b" +''' + ) + assert patch.load_patch(path) == [ + patch.SetAttrs(path='/entry', attrs={'vector': [0.0, 1.0], 'name': 'x'}), + patch.Move(source='/a', target='/b'), + ] + + +def test_load_patch_raises_on_unknown_kind(tmp_path: Path) -> None: + path = tmp_path / 'patch.toml' + path.write_text('[[op]]\nkind = "rename"\npath = "/a"\n') + with pytest.raises(ValueError, match="'rename'"): + patch.load_patch(path) + + +def make_workflow(nexus_file: Path) -> GenericNeXusWorkflow: + wf = GenericNeXusWorkflow(run_types=[SampleRun], monitor_types=[]) + wf[Filename[SampleRun]] = nexus_file + wf[NeXusDetectorName] = 'detector' + return wf + + +@pytest.mark.parametrize('preopen', [False, True]) +def test_workflow_applies_patch(nexus_file: Path, preopen: bool) -> None: + wf = make_workflow(nexus_file) + wf[PreopenNeXusFile] = preopen + wf[NeXusFilePatch[SampleRun]] = [ + patch.SetAttrs(path=f'{DETECTOR}/transformations/offset', attrs={'units': 'cm'}) + ] + detector = wf.compute(EmptyDetector[SampleRun]) + sc.testing.assert_identical( + detector.coords['position'][0], sc.vector([0.0, 0.0, 0.02], unit='m') + ) + + +def test_workflow_does_not_patch_by_default(nexus_file: Path) -> None: + detector = make_workflow(nexus_file).compute(EmptyDetector[SampleRun]) + sc.testing.assert_identical( + detector.coords['position'][0], sc.vector([0.0, 0.0, 2.0], unit='m') + ) diff --git a/packages/essreduce/tests/nexus/workflow_test.py b/packages/essreduce/tests/nexus/workflow_test.py index 3b82e162a..204dc2283 100644 --- a/packages/essreduce/tests/nexus/workflow_test.py +++ b/packages/essreduce/tests/nexus/workflow_test.py @@ -236,7 +236,7 @@ def test_given_no_sample_load_nexus_sample_returns_group_with_origin_depends_on( loki_tutorial_sample_run_60250: Path, ) -> None: filespec = workflow.file_path_to_file_spec( - loki_tutorial_sample_run_60250, preopen=True + loki_tutorial_sample_run_60250, preopen=True, patch=[] ) spec = workflow.unique_component_spec(filespec) assert spec.filename['/entry'][snx.NXsample] == {} diff --git a/packages/esssans/src/ess/loki/nexus_patches/todo_motion_sources.toml b/packages/esssans/src/ess/loki/nexus_patches/todo_motion_sources.toml new file mode 100644 index 000000000..4f236839b --- /dev/null +++ b/packages/esssans/src/ess/loki/nexus_patches/todo_motion_sources.toml @@ -0,0 +1,107 @@ +# Motion NXlogs of LoKI beamstop axes and the detector carriage are empty. +# +# The file-writer configuration used a source name with a "TODO-" prefix for +# `/transformations/motion`, so no values were written. The motor +# readback (RBV) was written to the NXlog `/value` instead. +# This patch replaces each empty `motion` log by the readback log and copies the +# transformation attributes of the original `motion` log. +# +# Observed in run 336349 (2026-10-07). + +[[op]] +kind = "remove" +path = "/entry/instrument/beamstop_x/transformations/motion" + +[[op]] +kind = "move" +source = "/entry/instrument/beamstop_x/value" +target = "/entry/instrument/beamstop_x/transformations/motion" + +[[op]] +kind = "set_attrs" +path = "/entry/instrument/beamstop_x/transformations/motion" +attrs = { depends_on = "/entry/instrument/beamstop_x/transformations/distance", transformation_type = "translation", vector = [0.0, 0.0, 1.0] } + +[[op]] +kind = "remove" +path = "/entry/instrument/beamstop_y/transformations/motion" + +[[op]] +kind = "move" +source = "/entry/instrument/beamstop_y/value" +target = "/entry/instrument/beamstop_y/transformations/motion" + +[[op]] +kind = "set_attrs" +path = "/entry/instrument/beamstop_y/transformations/motion" +attrs = { depends_on = "/entry/instrument/beamstop_x/transformations/motion", transformation_type = "translation", vector = [1.0, 0.0, 0.0], offset = [0.0, 0.0, 0.0], offset_units = "m" } + +[[op]] +kind = "remove" +path = "/entry/instrument/beamstop_z1/transformations/motion" + +[[op]] +kind = "move" +source = "/entry/instrument/beamstop_z1/value" +target = "/entry/instrument/beamstop_z1/transformations/motion" + +[[op]] +kind = "set_attrs" +path = "/entry/instrument/beamstop_z1/transformations/motion" +attrs = { depends_on = "/entry/instrument/beamstop_y/transformations/motion", transformation_type = "translation", vector = [0.0, 1.0, 0.0], offset = [0.0, 0.0, 0.0], offset_units = "m" } + +[[op]] +kind = "remove" +path = "/entry/instrument/beamstop_z2/transformations/motion" + +[[op]] +kind = "move" +source = "/entry/instrument/beamstop_z2/value" +target = "/entry/instrument/beamstop_z2/transformations/motion" + +[[op]] +kind = "set_attrs" +path = "/entry/instrument/beamstop_z2/transformations/motion" +attrs = { depends_on = "/entry/instrument/beamstop_y/transformations/motion", transformation_type = "translation", vector = [0.0, 1.0, 0.0], offset = [0.0, 0.0, 0.0], offset_units = "m" } + +[[op]] +kind = "remove" +path = "/entry/instrument/beamstop_z3/transformations/motion" + +[[op]] +kind = "move" +source = "/entry/instrument/beamstop_z3/value" +target = "/entry/instrument/beamstop_z3/transformations/motion" + +[[op]] +kind = "set_attrs" +path = "/entry/instrument/beamstop_z3/transformations/motion" +attrs = { depends_on = "/entry/instrument/beamstop_y/transformations/motion", transformation_type = "translation", vector = [0.0, 1.0, 0.0], offset = [0.0, 0.0, 0.0], offset_units = "m" } + +[[op]] +kind = "remove" +path = "/entry/instrument/beamstop_z4/transformations/motion" + +[[op]] +kind = "move" +source = "/entry/instrument/beamstop_z4/value" +target = "/entry/instrument/beamstop_z4/transformations/motion" + +[[op]] +kind = "set_attrs" +path = "/entry/instrument/beamstop_z4/transformations/motion" +attrs = { depends_on = "/entry/instrument/beamstop_y/transformations/motion", transformation_type = "translation", vector = [0.0, 1.0, 0.0], offset = [0.0, 0.0, 0.0], offset_units = "m" } + +[[op]] +kind = "remove" +path = "/entry/instrument/detector_carriage/transformations/motion" + +[[op]] +kind = "move" +source = "/entry/instrument/detector_carriage/value" +target = "/entry/instrument/detector_carriage/transformations/motion" + +[[op]] +kind = "set_attrs" +path = "/entry/instrument/detector_carriage/transformations/motion" +attrs = { depends_on = "/entry/instrument/detector_carriage/transformations/detector_carriage_zero", transformation_type = "translation", vector = [0.0, 0.0, 1.0], offset = [0.0, 0.0, 0.0], offset_units = "m" } diff --git a/packages/esssans/tests/loki/nexus_patches_test.py b/packages/esssans/tests/loki/nexus_patches_test.py new file mode 100644 index 000000000..2766d686f --- /dev/null +++ b/packages/esssans/tests/loki/nexus_patches_test.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Scipp contributors (https://github.com/scipp) +from importlib.resources import files +from pathlib import Path + +import pytest +import sciline +import scipp as sc +import scippnexus as snx + +from ess.reduce.nexus import GenericNeXusWorkflow, patch +from ess.reduce.nexus.types import ( + EmptyDetector, + EmptyMonitor, + Filename, + IncidentMonitor, + NeXusDetectorName, + NeXusFilePatch, + NeXusName, + NeXusTransformationChain, + SampleRun, +) + +CARRIAGE_MOTION = '/entry/instrument/detector_carriage/transformations/motion' + + +@pytest.fixture +def workflow(loki_broken_file: Path) -> sciline.Pipeline: + wf = GenericNeXusWorkflow(run_types=[SampleRun], monitor_types=[IncidentMonitor]) + wf[Filename[SampleRun]] = loki_broken_file + wf[NeXusDetectorName] = 'loki_detector_0' + wf[NeXusName[IncidentMonitor]] = 'beam_monitor_m4' + return wf + + +def test_broken_file_cannot_be_loaded_without_patch( + workflow: sciline.Pipeline, +) -> None: + with pytest.raises(ValueError, match='time-dependent'): + workflow.compute(EmptyDetector[SampleRun]) + + +def test_todo_motion_sources_patch_uses_motor_readback( + workflow: sciline.Pipeline, +) -> None: + workflow[NeXusFilePatch[SampleRun]] = patch.load_patch( + files('ess.loki') / 'nexus_patches' / 'todo_motion_sources.toml' + ) + chain = workflow.compute(NeXusTransformationChain[snx.NXdetector, SampleRun]) + motion = chain.transformations[CARRIAGE_MOTION].value + assert motion.sizes == {'time': 1} + assert sc.isclose(motion.data[0], sc.scalar(4999.96164649, unit='mm')) + + detector = workflow.compute(EmptyDetector[SampleRun]) + monitor = workflow.compute(EmptyMonitor[SampleRun, IncidentMonitor]) + assert sc.all(sc.isfinite(detector.coords['position'])).value + assert sc.all(sc.isfinite(monitor.coords['position'])).value