From c4f414a6c18e9eeb8d16219b2595f886ef37c78f Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 12:17:08 +0200 Subject: [PATCH 001/107] test: add synthetic Akoya qptiff, FCS and CODEX processor fixture writers Co-Authored-By: Claude Opus 5.5 --- tests/_akoya_data.py | 318 +++++++++++++++++++++++++++++++++++ tests/test_akoya_fixtures.py | 73 ++++++++ 2 files changed, 391 insertions(+) create mode 100644 tests/_akoya_data.py create mode 100644 tests/test_akoya_fixtures.py diff --git a/tests/_akoya_data.py b/tests/_akoya_data.py new file mode 100644 index 00000000..1bf4f1b8 --- /dev/null +++ b/tests/_akoya_data.py @@ -0,0 +1,318 @@ +"""Synthetic Akoya (PhenoCycler-Fusion qptiff, CODEX processor) inputs for the reader tests.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, Literal + +import numpy as np +import pandas as pd +import tifffile + +if TYPE_CHECKING: + from pathlib import Path + +_XML_DECLARATION = '' + + +def qpi_xml( + *, + image_type: str, + name: str, + biomarker: str | None = None, + software: str = "Fusion 2.3.1", + description_version: int = 6, + exposure_us: int = 450000, + color: str = "255,255,0", + scan_profile: str | None = None, + extra: dict[str, str] | None = None, +) -> str: + """Build a ``PerkinElmer-QPI-ImageDescription`` document (CRLF line endings, utf-16 declaration).""" + fields = [ + ("DescriptionVersion", str(description_version)), + ("AcquisitionSoftware", software), + ("ImageType", image_type), + ("Identifier", "00000000-0000-0000-0000-000000000000"), + ("SlideID", "SLIDE1"), + ("IsUnmixedComponent", "False"), + ("ExposureTime", str(exposure_us)), + ("SignalUnits", "64"), + ("Name", name), + ("Color", color), + ] + if biomarker is not None: + fields.append(("Biomarker", biomarker)) + fields.append(("ScaleFactor", "2.0001")) + fields.extend((extra or {}).items()) + if scan_profile is not None: + fields.append(("ScanProfile", scan_profile)) + lines = [_XML_DECLARATION, ""] + lines += [f" <{tag}>{text}" for tag, text in fields] + lines.append("") + return "\r\n".join(lines) + + +def fusion_scan_profile_json(wells: list[list[tuple[str, str]]]) -> str: + """Build a Fusion ``ScanProfile`` JSON (formatVersion 2); each well is a list of ``(markerName, channel)``.""" + channels = sorted({channel for well in wells for _, channel in well}) + profile = { + "binning": 2, + "experimentDescription": { + "name": "synthetic", + "formatVersion": 2, + "tissueType": "HumanFFPE", + "channels": [{"name": c, "defaultExposure": 100} for c in channels], + "wells": [ + { + "wellName": f"A{i + 1}", + "items": [ + { + "id": "00000000-0000-0000-0000-000000000000", + "inventoryItemType": "Marker", + "markerName": marker, + "channel": channel, + "dye": channel, + "exposure": 100, + } + for marker, channel in well + ], + } + for i, well in enumerate(wells) + ], + }, + } + return json.dumps(profile, separators=(",", ":")) + + +def cim_experiment_v4_xml(cycles: list[list[str]], resolution_nm: float = 377.39) -> str: + """Build a CODEX Instrument Manager ``ExperimentV4`` document; ``cycles`` lists marker names per channel.""" + cycle_xml = [] + for i, markers in enumerate(cycles, start=1): + channels = "".join( + f"{j}{marker}" + f"{'Nuclear' if marker == 'DAPI' else 'Blank' if marker == 'BLANK' else 'Other'}" + f"100{358 + 100 * j}" + for j, marker in enumerate(markers, start=1) + ) + cycle_xml.append(f"{i}{channels}") + return ( + "" + "synthetic" + f"{len(cycles)}" + f"{resolution_nm}" + f"{''.join(cycle_xml)}" + "" + ) + + +def write_qptiff( + path: Path, + *, + biomarkers: list[str | None], + names: list[str] | None = None, + shape: tuple[int, int] = (256, 320), + dtype: str = "uint8", + levels: int = 2, + software: str = "Fusion 2.3.1", + description_version: int = 6, + scan_profile: str | None = None, + resolution_px_per_cm: float | None = 1e4 / 0.5069, + resolution_unit: str = "CENTIMETER", + stage_position_cm: tuple[float, float] | None = None, + label_page: bool = True, + rgb: bool = False, + bigtiff: bool = True, + tile: tuple[int, int] | None = (64, 64), + slide_id: str = "SLIDE1", +) -> np.ndarray: + """Write a synthetic QPTIFF and return the level-0 data, ``(c, y, x)`` or ``(y, x, 3)`` when ``rgb``. + + Page order mirrors real files: full-resolution pages, an RGB thumbnail, the reduced-resolution + groups (the last one stripped), an RGB overview and an RGB label. + """ + rng = np.random.default_rng(0) + high = np.iinfo(dtype).max + n_pages = 1 if rgb else len(biomarkers) + names = ( + names if names is not None else [b if b is not None else f"Channel {i + 1}" for i, b in enumerate(biomarkers)] + ) + resolution = None if resolution_px_per_cm is None else (resolution_px_per_cm, resolution_px_per_cm) + shape_c = (*shape, 3) if rgb else shape + + def description(image_type: str, name: str, biomarker: str | None = None, *, first: bool = False) -> str: + xml = qpi_xml( + image_type=image_type, + name=name, + biomarker=biomarker, + software=software, + description_version=description_version, + scan_profile=scan_profile if first else None, + ) + return xml.replace("SLIDE1", f"{slide_id}") + + def write_page( + tw: tifffile.TiffWriter, data: np.ndarray, desc: str, *, subfiletype: int = 0, **kwargs: Any + ) -> None: + tw.write( + data, + software="PerkinElmer-QPI", + description=desc, + resolution=resolution, + resolutionunit=resolution_unit if resolution is not None else None, + subfiletype=subfiletype, + photometric="rgb" if data.ndim == 3 else "minisblack", + metadata=None, + **kwargs, + ) + + def rgb_page(side: tuple[int, int] = (64, 64)) -> np.ndarray: + return rng.integers(0, 256, (*side, 3), dtype=np.uint8) + + extratags: list[Any] = [] + if stage_position_cm is not None: + for code, value in zip((286, 287), stage_position_cm, strict=True): + extratags.append((code, 5, 1, (round(value * 1e6), 1_000_000), False)) + + data = rng.integers(0, high, (shape_c if rgb else (n_pages, *shape)), dtype=dtype) + with tifffile.TiffWriter(path, bigtiff=bigtiff) as tw: + for i in range(n_pages): + page = data if rgb else data[i] + write_page( + tw, + page, + description("FullResolution", names[i], biomarkers[i], first=i == 0), + tile=tile, + extratags=extratags if i == 0 else None, + ) + write_page(tw, rgb_page(), description("Thumbnail", "Thumbnail")) + for level in range(1, levels): + side = (shape[0] // 2**level, shape[1] // 2**level) + is_last = level == levels - 1 + for i in range(n_pages): + page = rng.integers(0, high, (*side, 3) if rgb else side, dtype=dtype) + write_page( + tw, + page, + description("ReducedResolution", names[i], biomarkers[i]), + subfiletype=1, + tile=None if is_last else tile, + ) + write_page(tw, rgb_page(), description("Overview", "Overview")) + if label_page: + write_page(tw, rgb_page(), description("Label", "Label")) + return data + + +def write_fcs(path: Path, df: pd.DataFrame, marker_names: dict[str, str] | None = None) -> None: + """Write a minimal FCS 3.0 file: little-endian float32, ``$PnN`` = column name, ``$PnS`` = marker name.""" + marker_names = marker_names or {} + values = np.ascontiguousarray(df.to_numpy(dtype=" bytes: + pairs: list[tuple[str, str]] = [ + ("$BEGINDATA", f"{data_begin:>12}"), + ("$ENDDATA", f"{data_end:>12}"), + ("$PAR", str(n_par)), + ("$TOT", str(len(df))), + ("$DATATYPE", "F"), + ("$BYTEORD", "1,2,3,4"), + ("$MODE", "L"), + ("$NEXTDATA", "0"), + ] + for i, col in enumerate(df.columns, start=1): + pairs += [(f"$P{i}N", str(col)), (f"$P{i}B", "32"), (f"$P{i}E", "0,0"), (f"$P{i}R", "262144")] + if col in marker_names: + pairs.append((f"$P{i}S", marker_names[col])) + return ("/" + "/".join(f"{k}/{v}" for k, v in pairs) + "/").encode("ascii") + + text_start = 58 + text_len = len(text_segment(0, 0)) # offsets are fixed-width, so the length does not depend on their value + text_end = text_start + text_len - 1 + data_begin = text_end + 1 + data_end = data_begin + len(data) - 1 + header = f"FCS3.0 {text_start:>8}{text_end:>8}{data_begin:>8}{data_end:>8}{0:>8}{0:>8}".encode("ascii") + path.write_bytes(header.ljust(text_start) + text_segment(data_begin, data_end) + data) + + +def write_legacy_processor_tree( + root: Path, + *, + regions: int = 1, + cycles: int = 2, + channels: int = 4, + markers: list[str] | None = None, + shape: tuple[int, int] = (128, 160), + layout: Literal["per_marker", "hyperstack", "old_flat"] = "per_marker", + fcs: bool = True, + csv: bool = True, + n_cells: int = 50, +) -> dict[str, Any]: + """Write a CODEX Processor output tree and return ``{"markers", "images", "cells"}``. + + ``markers`` has one entry per (cycle, channel), cycle-major. ``images`` maps region name to + ``(n_markers, y, x)`` for ``per_marker`` and ``old_flat`` and to ``(cycles, channels, y, x)`` for + ``hyperstack``. ``cells`` is the table written for the last region. + """ + n_markers = cycles * channels + if markers is None: + markers = ["DAPI" if c == 0 else f"MARKER{i}" for i in range(n_markers) for c in [i % channels]] + if len(markers) != n_markers: + raise ValueError(f"expected {n_markers} markers (cycles * channels), got {len(markers)}") + rng = np.random.default_rng(0) + root.mkdir(parents=True, exist_ok=True) + processed = root / "processed_2020-01-01" + channel_cols = [f"cyc{c + 1:03d}_ch{ch + 1:03d}" for c in range(cycles) for ch in range(channels)] + images: dict[str, np.ndarray] = {} + cells = pd.DataFrame() + + if layout != "old_flat": + (root / "channelNames.txt").write_text("\n".join(markers) + "\n") + (root / "experiment.json").write_text( + json.dumps({"name": "synthetic", "numCycles": cycles, "numChannels": channels, "regions": regions}) + ) + + for r in range(1, regions + 1): + name = f"reg{r:03d}" + stack = rng.integers(0, 2**16, (n_markers, *shape), dtype=np.uint16) + cells = pd.DataFrame( + { + "cell_id": np.arange(1, n_cells + 1, dtype=float), + "region": float(r), + "tile_num": 1.0, + "x": rng.uniform(0, shape[1], n_cells), + "y": rng.uniform(0, shape[0], n_cells), + "z": 1.0, + "size": rng.uniform(10, 100, n_cells), + **{col: rng.uniform(0, 1000, n_cells) for col in channel_cols}, + } + ) + if layout == "per_marker": + images[name] = stack + image_dir = processed / "stitched" / name + image_dir.mkdir(parents=True, exist_ok=True) + for i, marker in enumerate(markers): + cyc, ch = divmod(i, channels) + tifffile.imwrite(image_dir / f"{name}_cyc{cyc + 1:03d}_ch{ch + 1:03d}_{marker}.tif", stack[i]) + table_dir = processed / "segm" / "segm-1" / "fcs" / "compensated" + table_stem = f"{name}_compensated" + elif layout == "hyperstack": + images[name] = stack.reshape(cycles, channels, *shape) + (processed / "stitched").mkdir(parents=True, exist_ok=True) + tifffile.imwrite( + processed / "stitched" / f"{name}_montage.tif", images[name], imagej=True, metadata={"axes": "TCYX"} + ) + table_dir = processed / "segm" / "segm-1" / "fcs" / "compensated" + table_stem = f"{name}_compensated" + else: + images[name] = stack + tifffile.imwrite(root / f"exp_{name}.tif", stack[0]) + table_dir = root + table_stem = f"exp_{name}" + table_dir.mkdir(parents=True, exist_ok=True) + if fcs: + write_fcs(table_dir / f"{table_stem}.fcs", cells, dict(zip(channel_cols, markers, strict=True))) + if csv and not (layout == "old_flat" and fcs): + cells.to_csv(table_dir / f"{table_stem}.csv", index=False) + return {"markers": markers, "images": images, "cells": cells} diff --git a/tests/test_akoya_fixtures.py b/tests/test_akoya_fixtures.py new file mode 100644 index 00000000..470d7e91 --- /dev/null +++ b/tests/test_akoya_fixtures.py @@ -0,0 +1,73 @@ +from pathlib import Path + +import numpy as np +import pandas as pd +import readfcs +import tifffile + +from tests._akoya_data import write_fcs, write_legacy_processor_tree, write_qptiff + + +def _page(tf: tifffile.TiffFile, index: int) -> tifffile.TiffPage: + page = tf.pages[index] + assert isinstance(page, tifffile.TiffPage) + return page + + +def test_write_qptiff_is_qpi(tmp_path: Path) -> None: + p = tmp_path / "x.qptiff" + data = write_qptiff(p, biomarkers=["DAPI", "CD3", "CD8"], levels=2) + with tifffile.TiffFile(p) as tf: + assert tf.is_qpi + s = tf.series[0] + assert s.shape == (3, 256, 320) + assert len(s.levels) == 2 + assert s.levels[1].shape == (3, 128, 160) + desc = _page(tf, 0).description + assert "DAPI" in desc + assert 'encoding="utf-16"' in desc + assert "CD3" in _page(tf, 1).description + assert data.shape == (3, 256, 320) + + +def test_write_qptiff_rgb_and_stage_position(tmp_path: Path) -> None: + p = tmp_path / "rgb.qptiff" + data = write_qptiff(p, biomarkers=[None], rgb=True, stage_position_cm=(1.5, 2.5), label_page=False, levels=1) + assert data.shape == (256, 320, 3) + with tifffile.TiffFile(p) as tf: + page = _page(tf, 0) + x, y = (page.tags[t].value for t in ("XPosition", "YPosition")) + assert x[0] / x[1] == 1.5 + assert y[0] / y[1] == 2.5 + + +def test_write_fcs_roundtrip(tmp_path: Path) -> None: + p = tmp_path / "x.fcs" + df = pd.DataFrame({"cell_id": [1.0, 2.0], "cyc001_ch001": [0.5, 1.5]}) + write_fcs(p, df, marker_names={"cyc001_ch001": "DAPI"}) + fcs = readfcs.ReadFCS(p) + # readfcs exposes events with positional integer columns; the $PnN names live in ``channels`` + assert list(fcs.channels["PnN"]) == list(df.columns) + assert fcs.data.shape == df.shape + assert "DAPI" in set(fcs.channels["PnS"]) + np.testing.assert_allclose(fcs.data.iloc[:, 1].to_numpy(), [0.5, 1.5]) + + +def test_legacy_tree_per_marker(tmp_path: Path) -> None: + info = write_legacy_processor_tree(tmp_path, markers=["DAPI", "CD3", "CD8", "CD4", "DAPI", "CD20", "CD68", "CD31"]) + assert (tmp_path / "processed_2020-01-01/stitched/reg001/reg001_cyc001_ch001_DAPI.tif").exists() + assert (tmp_path / "processed_2020-01-01/segm/segm-1/fcs/compensated/reg001_compensated.fcs").exists() + assert (tmp_path / "processed_2020-01-01/segm/segm-1/fcs/compensated/reg001_compensated.csv").exists() + assert len(info["markers"]) == 8 + assert info["images"]["reg001"].shape == (8, 128, 160) + assert len(info["cells"]) == 50 + + +def test_legacy_tree_other_layouts(tmp_path: Path) -> None: + info = write_legacy_processor_tree(tmp_path / "h", layout="hyperstack") + with tifffile.TiffFile(tmp_path / "h/processed_2020-01-01/stitched/reg001_montage.tif") as tf: + assert tf.series[0].shape == (2, 4, 128, 160) + assert info["images"]["reg001"].shape == (2, 4, 128, 160) + write_legacy_processor_tree(tmp_path / "f", layout="old_flat", fcs=False) + assert (tmp_path / "f/exp_reg001.csv").exists() + assert (tmp_path / "f/exp_reg001.tif").exists() From 95eeaca6b607e6bd516ad74dd43445a91e975c34 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 12:25:15 +0200 Subject: [PATCH 002/107] feat: add QPTIFF metadata and lazy image core for Akoya readers Parse PerkinElmer/Akoya QPTIFF page XML (channels by , Biomarker > Name), calibrated pixel size from XResolution, stage position, generation detection, and Fusion JSON / CIM ExperimentV4 scan-profile enrichment. Expose level 0 lazily via aszarr. Add PhenocyclerKeys. Co-Authored-By: Claude Opus 5.5 --- src/spatialdata_io/_constants/_constants.py | 20 + src/spatialdata_io/readers/_utils/_qptiff.py | 388 +++++++++++++++++++ tests/readers/test_qptiff.py | 191 +++++++++ 3 files changed, 599 insertions(+) create mode 100644 src/spatialdata_io/readers/_utils/_qptiff.py create mode 100644 tests/readers/test_qptiff.py diff --git a/src/spatialdata_io/_constants/_constants.py b/src/spatialdata_io/_constants/_constants.py index 36e2a88f..8d7fbf98 100644 --- a/src/spatialdata_io/_constants/_constants.py +++ b/src/spatialdata_io/_constants/_constants.py @@ -18,6 +18,26 @@ class CodexKeys(ModeEnum): IMAGE_TIF = ".tif" +@unique +class PhenocyclerKeys(ModeEnum): + """Keys for *Akoya PhenoCycler-Fusion* (and CIM QPTIFF) formatted dataset.""" + + # files and directories + QPTIFF_SUFFIX = ".qptiff" + ER_QPTIFF_SUFFIX = ".er.qptiff" + XPD_SUFFIX = ".xpd" + CHANNELS_CSV_SUFFIX = "channels.csv" + # elements + IMAGE_SUFFIX = "_image" + LABELS_SUFFIX = "_labels" + CELLS_SUFFIX = "_cells" + TABLE_KEY = "table" + # metadata + REGION_KEY = "region" + INSTANCE_KEY = "cell_id" + ATTRS_KEY = "akoya" + + class CurioKeys(ModeEnum): """Keys for *Curio* formatted dataset.""" diff --git a/src/spatialdata_io/readers/_utils/_qptiff.py b/src/spatialdata_io/readers/_utils/_qptiff.py new file mode 100644 index 00000000..baf97b06 --- /dev/null +++ b/src/spatialdata_io/readers/_utils/_qptiff.py @@ -0,0 +1,388 @@ +"""PerkinElmer/Akoya QPTIFF metadata and lazy level-0 image access, shared by the Akoya readers.""" + +from __future__ import annotations + +import json +import re +import warnings +import xml.etree.ElementTree as ET +from collections import Counter +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +import dask.array as da +import tifffile +import zarr + +from spatialdata_io._constants._constants import PhenocyclerKeys + +if TYPE_CHECKING: + from pathlib import Path + +__all__ = ["AkoyaChannel", "AkoyaScan", "qpi_pixel_size_um", "read_qptiff_image", "read_qptiff_metadata"] + +_FULL_RESOLUTION = "FullResolution" +_XML_DECLARATION = re.compile(r"^\s*<\?xml[^>]*\?>") +# micrometres per unit of the TIFF ResolutionUnit; XPosition/YPosition are expressed in the same unit +_UM_PER_RESOLUTION_UNIT = {tifffile.RESUNIT.CENTIMETER: 1e4, tifffile.RESUNIT.INCH: 25400.0} + + +@dataclass +class AkoyaChannel: + """One image channel of an Akoya acquisition. + + ``name`` is the marker identity before de-duplication (``Biomarker`` > ``Name`` > ``channel_{i}``). + ``cycle`` and ``channel`` (both 1-based) come from the scan profile and stay ``None`` when it does not + describe this channel. ``extra`` holds further JSON-serialisable metadata. + """ + + name: str + fluor: str | None = None + cycle: int | None = None + channel: int | None = None + exposure_ms: float | None = None + color: str | None = None + extra: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class AkoyaScan: + """Metadata of one Akoya image (a QPTIFF scan or a legacy CODEX Processor region). + + ``generation`` is one of ``"fusion1"``, ``"fusion2"``, ``"cim_qptiff"``, ``"qpi_other"`` or + ``"legacy_processor"``. ``pixel_size_um`` is the calibrated pixel size, ``nominal_pixel_size_um`` the one + the acquisition software reports. ``stage_position_um`` is the ``(x, y)`` stage position of the upper-left + corner. ``raw`` keeps the unparsed source metadata (JSON-serialisable). + """ + + path: Path + region: str + generation: str + software: str | None + slide_id: str | None + pixel_size_um: float | None + nominal_pixel_size_um: float | None + stage_position_um: tuple[float, float] | None + channels: list[AkoyaChannel] + rgb: bool + raw: dict[str, Any] + + +def read_qptiff_metadata(path: Path, region: str | None = None) -> AkoyaScan: + """Read the metadata of a PerkinElmer/Akoya QPTIFF. + + Channels are the ``FullResolution`` pages in file order; each page's own XML is authoritative for the + channel identity, while the page-0 ``ScanProfile`` (Fusion JSON or CIM ``ExperimentV4``) only adds cycle, + channel and reagent information. + + Parameters + ---------- + path + Path to the ``.qptiff`` file. + region + Region name; defaults to the file name without the ``.qptiff``/``.er.qptiff`` suffix. + + Returns + ------- + The scan metadata. + """ + with _open_qpi(path) as tf: + page0 = _page(tf, 0) + page_xml = [_parse_description(_page(tf, i).description) for i in range(len(tf.pages))] + full_res = [i for i, xml in enumerate(page_xml) if _text(xml, "ImageType") == _FULL_RESOLUTION] + if not full_res: + raise ValueError(f"{path} has no page with {_FULL_RESOLUTION}.") + rgb = len(full_res) == 1 and _page(tf, full_res[0]).samplesperpixel == 3 + pixel_size_um = qpi_pixel_size_um(page0) + stage_position_um = _stage_position_um(page0) + page0_description = page0.description + + root = page_xml[0] + software = _text(root, "AcquisitionSoftware") + experiment_v4 = root.find("ScanProfile/ExperimentV4") + scan_profile = _scan_profile(root) + channels = [_channel(page_xml[page_index], i, page_index) for i, page_index in enumerate(full_res)] + if isinstance(scan_profile, dict): + _enrich(channels, _fusion_profile_records(scan_profile)) + elif experiment_v4 is not None: + _enrich(channels, _experiment_v4_records(experiment_v4)) + + if pixel_size_um is None: + warnings.warn( + f"Could not determine the pixel size of {path}; pixel_size_um is None.", UserWarning, stacklevel=2 + ) + + return AkoyaScan( + path=path, + region=region if region is not None else _default_region(path), + generation=_generation(software, experiment_v4 is not None), + software=software, + slide_id=_text(root, "SlideID"), + pixel_size_um=pixel_size_um, + nominal_pixel_size_um=_nominal_pixel_size_um(root), + stage_position_um=stage_position_um, + channels=channels, + rgb=rgb, + raw={ + "page0_xml": page0_description, + "scan_profile": scan_profile, + "image_types": dict(Counter(_text(xml, "ImageType") or "unknown" for xml in page_xml)), + }, + ) + + +def read_qptiff_image(path: Path) -> da.Array: + """Lazily read the full-resolution image of a PerkinElmer/Akoya QPTIFF. + + The array keeps the native tile chunks of the file. The underlying ``TiffFile`` stays open for as long as + the returned array (its zarr store holds the file handle), so it is intentionally not closed here. + + Parameters + ---------- + path + Path to the ``.qptiff`` file. + + Returns + ------- + ``(c, y, x)`` array, or ``(y, x, 3)`` for a brightfield RGB scan. + """ + tf = _open_qpi(path) + full_res = [ + i + for i in range(len(tf.pages)) + if _text(_parse_description(_page(tf, i).description), "ImageType") == _FULL_RESOLUTION + ] + if not full_res: + tf.close() + raise ValueError(f"{path} has no page with {_FULL_RESOLUTION}.") + first = _page(tf, full_res[0]) + if len(full_res) == 1 and first.samplesperpixel == 3: + expected_shape: tuple[int, ...] = first.shape + else: + expected_shape = (len(full_res), *first.shape) + + image = da.from_zarr(zarr.open(tf.series[0].aszarr(level=0), mode="r")) + if image.shape == expected_shape: + return image + # tifffile groups pages into series by shape heuristics; fall back to the pages the XML says are full-res + pages = [da.from_zarr(zarr.open(_page(tf, i).aszarr(), mode="r")) for i in full_res] + return pages[0] if len(expected_shape) == first.ndim else da.stack(pages) + + +def qpi_pixel_size_um(page: tifffile.TiffPage) -> float | None: + """Return the pixel size of a QPTIFF page in micrometres, or ``None`` when it is unknown. + + The calibrated ``XResolution`` is used when ``ResolutionUnit`` is centimetre. Otherwise (e.g. the + meaningless 96 dpi written when the resolution is unknown) the nominal ``PixelSizeMicrons`` or the CIM + ``Resolution_nm`` from the page XML is returned, if present. + """ + unit = page.tags.get("ResolutionUnit") + resolution = page.tags.get("XResolution") + if unit is not None and resolution is not None and unit.value == tifffile.RESUNIT.CENTIMETER: + numerator, denominator = resolution.value + if numerator > 0 and denominator > 0: + return 1e4 * denominator / numerator + return _nominal_pixel_size_um(_parse_description(page.description)) + + +def _open_qpi(path: Path) -> tifffile.TiffFile: + if not path.is_file(): + raise FileNotFoundError(f"{path} does not exist or is not a file.") + tf = tifffile.TiffFile(path) + if not tf.is_qpi: + tf.close() + raise ValueError( + f"{path} is not a PerkinElmer/Akoya QPTIFF (Software tag does not start with 'PerkinElmer-QPI')." + ) + return tf + + +def _page(tf: tifffile.TiffFile, index: int) -> tifffile.TiffPage: + page = tf.pages[index] + if not isinstance(page, tifffile.TiffPage): # only frames when ``useframes`` is set, which we never do + raise TypeError(f"Expected a TiffPage at index {index}, got {type(page).__name__}.") + return page + + +def _parse_description(description: str) -> ET.Element: + # the declaration claims utf-16 but the tag is ASCII, which ElementTree rejects + return ET.fromstring(_XML_DECLARATION.sub("", description, count=1)) + + +def _text(element: ET.Element, tag: str) -> str | None: + """Stripped text of ``tag`` (a path relative to ``element``), ``None`` when missing or empty.""" + found = element.find(tag) + if found is None or found.text is None or not found.text.strip(): + return None + return found.text.strip() + + +def _float(text: str | None) -> float | None: + try: + return None if text is None else float(text) + except ValueError: + return None + + +def _bool(text: str | None) -> bool | None: + return None if text is None else text.lower() == "true" + + +def _default_region(path: Path) -> str: + for suffix in (PhenocyclerKeys.ER_QPTIFF_SUFFIX, PhenocyclerKeys.QPTIFF_SUFFIX): + if path.name.lower().endswith(suffix.value): + return path.name[: -len(suffix.value)] + return path.stem + + +def _generation(software: str | None, has_experiment_v4: bool) -> str: + software = software or "" + if software.startswith("Fusion 1"): + return "fusion1" + if software.startswith("Fusion 2"): + return "fusion2" + if "instrument manager" in software.lower() or has_experiment_v4: + return "cim_qptiff" + return "qpi_other" + + +def _nominal_pixel_size_um(root: ET.Element) -> float | None: + # Vectra/Opal: ScanProfile/.../ScanResolution/PixelSizeMicrons; CIM: ScanProfile/ExperimentV4/Resolution_nm + pixel_size = _float(_text(root, ".//PixelSizeMicrons")) + if pixel_size is not None: + return pixel_size + resolution_nm = _float(_text(root, ".//Resolution_nm")) + return None if resolution_nm is None else resolution_nm / 1000 + + +def _stage_position_um(page: tifffile.TiffPage) -> tuple[float, float] | None: + unit = page.tags.get("ResolutionUnit") + um_per_unit = _UM_PER_RESOLUTION_UNIT.get(unit.value) if unit is not None else None + positions = [page.tags.get(name) for name in ("XPosition", "YPosition")] + if um_per_unit is None or positions[0] is None or positions[1] is None: + return None + (x_num, x_den), (y_num, y_den) = positions[0].value, positions[1].value + if x_den == 0 or y_den == 0: + return None + return x_num / x_den * um_per_unit, y_num / y_den * um_per_unit + + +def _scan_profile(root: ET.Element) -> dict[str, Any] | str | None: + """Page-0 ``ScanProfile``: parsed JSON (Fusion PhenoCycler mode) or the XML string (Opal, Vectra, CIM).""" + element = root.find("ScanProfile") + if element is None: + return None + if len(element): + return "".join(ET.tostring(child, encoding="unicode") for child in element) + text = (element.text or "").strip() + if not text: + return None + try: + profile = json.loads(text) + except json.JSONDecodeError: + return text + return profile if isinstance(profile, dict) else text + + +def _channel(xml: ET.Element, index: int, page_index: int) -> AkoyaChannel: + biomarker = _text(xml, "Biomarker") + fluor = _text(xml, "Name") + exposure_us = _float(_text(xml, "ExposureTime")) + extra: dict[str, Any] = {"page_index": page_index} + for key, tag in ( + ("is_unmixed_component", "IsUnmixedComponent"), + ("autofluorescence_subtracted", "AutofluorescenceSubtracted"), + ): + value = _bool(_text(xml, tag)) + if value is not None: + extra[key] = value + return AkoyaChannel( + name=biomarker or fluor or f"channel_{index}", + fluor=fluor if biomarker else None, + exposure_ms=None if exposure_us is None else exposure_us / 1000, + color=_text(xml, "Color"), + extra=extra, + ) + + +def _fusion_profile_records(profile: dict[str, Any]) -> list[tuple[str, dict[str, Any]]]: + """``(marker, fields)`` per item of the Fusion ``experimentDescription.wells``; one well per cycle.""" + records = [] + wells = profile.get("experimentDescription", {}).get("wells", []) + for cycle, well in enumerate(wells, start=1): + for channel, item in enumerate(well.get("items", []), start=1): + # Fusion writes the string "None" for reagent fields that do not apply (e.g. DAPI) + extra = { + key: item[key] + for key in ("reporter", "barcode", "clone", "dye") + if item.get(key) not in (None, "", "None") + } + if item.get("channel"): + extra["channel_name"] = item["channel"] + extra["well"] = well.get("wellName") + exposure = item.get("exposure") + records.append( + ( + str(item.get("markerName")), + { + "cycle": cycle, + "channel": channel, + "exposure_ms": float(exposure) if isinstance(exposure, int | float) else None, + "extra": extra, + }, + ) + ) + return records + + +def _experiment_v4_records(experiment: ET.Element) -> list[tuple[str, dict[str, Any]]]: + """``(marker, fields)`` per ``Cycles/Cycle/Channels/Channel`` of a CIM ``ExperimentV4``.""" + records = [] + for cycle in experiment.iterfind("Cycles/Cycle"): + cycle_index = _float(_text(cycle, "Index")) + for channel in cycle.iterfind("Channels/Channel"): + channel_index = _float(_text(channel, "Index")) + extra: dict[str, Any] = {} + for key, tag in (("dye", "DyeName"), ("marker_class", "MarkerClass")): + value = _text(channel, tag) + if value is not None: + extra[key] = value + wavelength = _float(_text(channel, "Wavelength_nm")) + if wavelength is not None: + extra["wavelength_nm"] = wavelength + records.append( + ( + str(_text(channel, "MarkerName")), + { + "cycle": None if cycle_index is None else int(cycle_index), + "channel": None if channel_index is None else int(channel_index), + "exposure_ms": _float(_text(channel, "ExposureTime_ms")), + "extra": extra, + }, + ) + ) + return records + + +def _enrich(channels: list[AkoyaChannel], records: list[tuple[str, dict[str, Any]]]) -> None: + """Fill cycle/channel/extra from the scan profile, joined on (marker name, occurrence index). + + The page XML stays authoritative: channels are never reordered and the page exposure wins; channels + without a matching record are left unchanged. + """ + by_key: dict[tuple[str, int], dict[str, Any]] = {} + seen: Counter[str] = Counter() + for name, record in records: + by_key[(name, seen[name])] = record + seen[name] += 1 + seen.clear() + for channel in channels: + match = by_key.get((channel.name, seen[channel.name])) + seen[channel.name] += 1 + if match is None: + continue + channel.cycle = match["cycle"] + channel.channel = match["channel"] + if channel.exposure_ms is None: + channel.exposure_ms = match["exposure_ms"] + channel.extra.update({k: v for k, v in match["extra"].items() if k not in channel.extra}) diff --git a/tests/readers/test_qptiff.py b/tests/readers/test_qptiff.py new file mode 100644 index 00000000..182a3ea0 --- /dev/null +++ b/tests/readers/test_qptiff.py @@ -0,0 +1,191 @@ +import json +import warnings +from pathlib import Path + +import dask.array as da +import numpy as np +import pytest +import tifffile +from tests._akoya_data import cim_experiment_v4_xml, fusion_scan_profile_json, write_qptiff + +from spatialdata_io.readers._utils._qptiff import read_qptiff_image, read_qptiff_metadata + + +def test_fusion2_channels_from_biomarker(tmp_path: Path) -> None: + p = tmp_path / "scan.qptiff" + write_qptiff(p, biomarkers=["DAPI", "CD3", "DAPI"], names=["DAPI", "ATTO 550", "DAPI"]) + scan = read_qptiff_metadata(p) + assert [c.name for c in scan.channels] == ["DAPI", "CD3", "DAPI"] + assert scan.channels[1].fluor == "ATTO 550" + assert scan.generation == "fusion2" + assert scan.region == "scan" + assert scan.slide_id == "SLIDE1" + assert scan.software == "Fusion 2.3.1" + assert scan.channels[1].exposure_ms == pytest.approx(450.0) + assert scan.channels[1].color == "255,255,0" + + +def test_opal_mode_uses_name(tmp_path: Path) -> None: + p = tmp_path / "opal.qptiff" + names = ["DAPI", "Opal 480", "Opal 520", "Opal 570", "Opal 620", "Opal 690"] + write_qptiff(p, biomarkers=[None] * 6, names=names) + scan = read_qptiff_metadata(p) + assert [c.name for c in scan.channels] == names + assert all(c.fluor is None for c in scan.channels) + assert scan.generation == "fusion2" + + +def test_fusion1_generation_and_region_override(tmp_path: Path) -> None: + p = tmp_path / "scan.er.qptiff" + write_qptiff(p, biomarkers=["DAPI"], software="Fusion 1.0.5", dtype="uint16") + assert read_qptiff_metadata(p).region == "scan" + scan = read_qptiff_metadata(p, region="reg001") + assert scan.generation == "fusion1" + assert scan.region == "reg001" + + +def test_vectra_is_qpi_other(tmp_path: Path) -> None: + p = tmp_path / "vectra.qptiff" + write_qptiff(p, biomarkers=[None, None], names=["DAPI", "FITC"], software="VectraPolaris 1.0") + assert read_qptiff_metadata(p).generation == "qpi_other" + + +def test_pixel_size_from_xresolution(tmp_path: Path) -> None: + p = tmp_path / "x.qptiff" + write_qptiff(p, biomarkers=["DAPI"], resolution_px_per_cm=1e4 / 0.5069) + scan = read_qptiff_metadata(p) + assert scan.pixel_size_um == pytest.approx(0.5069, rel=1e-6) + assert scan.nominal_pixel_size_um is None + + +def test_96dpi_resolution_is_unknown(tmp_path: Path) -> None: + p = tmp_path / "x.qptiff" + write_qptiff(p, biomarkers=["DAPI"], resolution_px_per_cm=96, resolution_unit="INCH") + with pytest.warns(UserWarning, match="Could not determine the pixel size"): + scan = read_qptiff_metadata(p) + assert scan.pixel_size_um is None + + +def test_stage_position(tmp_path: Path) -> None: + p = tmp_path / "x.qptiff" + write_qptiff(p, biomarkers=["DAPI"], stage_position_cm=(1.5, 0.25)) + scan = read_qptiff_metadata(p) + assert scan.stage_position_um == pytest.approx((15000.0, 2500.0)) + write_qptiff(p, biomarkers=["DAPI"]) + assert read_qptiff_metadata(p).stage_position_um is None + + +def test_cim_without_label_page_counts_pages_by_imagetype(tmp_path: Path) -> None: + p = tmp_path / "cim.qptiff" + biomarkers: list[str | None] = ["DAPI", "CD3", "CD8", "DAPI", "CD20"] + profile = cim_experiment_v4_xml([["DAPI", "CD3", "CD8"], ["DAPI", "CD20"]]) + written = write_qptiff( + p, + biomarkers=biomarkers, + software="Codex Instrument Manager 1.30.0.12", + description_version=3, + scan_profile=profile, + levels=3, + label_page=False, + resolution_px_per_cm=1e4 / 0.37739, + ) + image = read_qptiff_image(p) + assert image.shape == (5, 256, 320) + assert np.array_equal(image.compute(), written) + scan = read_qptiff_metadata(p) + assert scan.generation == "cim_qptiff" + assert [c.name for c in scan.channels] == biomarkers + assert scan.channels[1].cycle == 1 + assert scan.channels[1].channel == 2 + assert scan.channels[1].exposure_ms is not None + assert scan.channels[1].extra["wavelength_nm"] == pytest.approx(558.0) + assert [c.cycle for c in scan.channels] == [1, 1, 1, 2, 2] + assert scan.nominal_pixel_size_um == pytest.approx(0.37739) + + +def test_fusion_scan_profile_enrichment(tmp_path: Path) -> None: + p = tmp_path / "x.qptiff" + wells = [[("DAPI", "DAPI"), ("CD3", "ATTO 550")], [("DAPI", "DAPI"), ("CD8", "AF 750")]] + write_qptiff(p, biomarkers=["DAPI", "CD3", "DAPI", "CD8"], scan_profile=fusion_scan_profile_json(wells)) + scan = read_qptiff_metadata(p) + assert [c.cycle for c in scan.channels] == [1, 1, 2, 2] + assert [c.channel for c in scan.channels] == [1, 2, 1, 2] + assert isinstance(scan.raw["scan_profile"], dict) + + +def test_fusion_reagent_placeholders_are_dropped(tmp_path: Path) -> None: + p = tmp_path / "x.qptiff" + profile = json.loads(fusion_scan_profile_json([[("DAPI", "DAPI"), ("CD3", "ATTO 550")]])) + items = profile["experimentDescription"]["wells"][0]["items"] + items[0].update(reporter="None", barcode="None", clone="None") + items[1].update(reporter="GN65", barcode="GN65", clone="XXX") + write_qptiff(p, biomarkers=["DAPI", "CD3"], scan_profile=json.dumps(profile)) + dapi, cd3 = read_qptiff_metadata(p).channels + assert not {"reporter", "barcode", "clone"} & dapi.extra.keys() + assert (cd3.extra["reporter"], cd3.extra["barcode"], cd3.extra["clone"]) == ("GN65", "GN65", "XXX") + assert cd3.extra["dye"] == "ATTO 550" + + +def test_enrichment_leaves_unmatched_channels_empty(tmp_path: Path) -> None: + p = tmp_path / "x.qptiff" + wells = [[("DAPI", "DAPI"), ("CD3", "ATTO 550")]] + write_qptiff(p, biomarkers=["DAPI", "CD3", "CD4"], scan_profile=fusion_scan_profile_json(wells)) + scan = read_qptiff_metadata(p) + assert [c.name for c in scan.channels] == ["DAPI", "CD3", "CD4"] + assert [c.cycle for c in scan.channels] == [1, 1, None] + + +@pytest.mark.parametrize(("dtype", "name"), [("uint8", "x.qptiff"), ("uint16", "x.er.qptiff")]) +def test_image_is_lazy_and_correct(tmp_path: Path, dtype: str, name: str) -> None: + p = tmp_path / name + written = write_qptiff(p, biomarkers=["DAPI", "CD3"], dtype=dtype) + arr = read_qptiff_image(p) + assert isinstance(arr, da.Array) + assert arr.chunksize != arr.shape + assert arr.dtype == np.dtype(dtype) + assert np.array_equal(arr.compute(), written) + + +def test_three_channel_not_rgb(tmp_path: Path) -> None: + p = tmp_path / "x.qptiff" + written = write_qptiff(p, biomarkers=["DAPI", "CD3", "CD8"]) + scan = read_qptiff_metadata(p) + assert scan.rgb is False + image = read_qptiff_image(p) + assert image.ndim == 3 + assert image.shape == (3, 256, 320) + assert np.array_equal(image.compute(), written) + + +def test_brightfield_rgb(tmp_path: Path) -> None: + p = tmp_path / "x.qptiff" + written = write_qptiff(p, biomarkers=[None], names=["BF"], rgb=True, software="VectraPolaris 1.0") + scan = read_qptiff_metadata(p) + assert scan.rgb is True + image = read_qptiff_image(p) + assert image.shape == (256, 320, 3) + assert np.array_equal(image.compute(), written) + + +def test_not_qpi_raises(tmp_path: Path) -> None: + p = tmp_path / "x.qptiff" + tifffile.imwrite(p, np.zeros((16, 16), dtype=np.uint8)) + with pytest.raises(ValueError, match="not a PerkinElmer"): + read_qptiff_metadata(p) + with pytest.raises(ValueError, match="not a PerkinElmer"): + read_qptiff_image(p) + + +def test_missing_file_raises(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + read_qptiff_metadata(tmp_path / "missing.qptiff") + + +def test_classic_tiff_and_stripped_level(tmp_path: Path) -> None: + p = tmp_path / "x.qptiff" + written = write_qptiff(p, biomarkers=["DAPI", "CD3"], bigtiff=False, tile=None) + with warnings.catch_warnings(): + warnings.simplefilter("error") + scan = read_qptiff_metadata(p) + assert len(scan.channels) == 2 + assert np.array_equal(read_qptiff_image(p).compute(), written) From 125176b5aab5f8bd56109db66a2b562127fa9da5 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 12:29:46 +0200 Subject: [PATCH 003/107] fix: never guess QPTIFF scan-profile enrichment for ambiguous channels Join profile records to pages positionally when the non-blank counts match and no pair contradicts the names, else by normalised name only for names with equal page and record counts. Skip blank and markerless profile records. Co-Authored-By: Claude Opus 5.5 --- src/spatialdata_io/readers/_utils/_qptiff.py | 80 +++++++++++++++----- tests/readers/test_qptiff.py | 40 ++++++++++ 2 files changed, 100 insertions(+), 20 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_qptiff.py b/src/spatialdata_io/readers/_utils/_qptiff.py index baf97b06..207cef54 100644 --- a/src/spatialdata_io/readers/_utils/_qptiff.py +++ b/src/spatialdata_io/readers/_utils/_qptiff.py @@ -311,6 +311,9 @@ def _fusion_profile_records(profile: dict[str, Any]) -> list[tuple[str, dict[str wells = profile.get("experimentDescription", {}).get("wells", []) for cycle, well in enumerate(wells, start=1): for channel, item in enumerate(well.get("items", []), start=1): + marker = item.get("markerName") + if not isinstance(marker, str) or _is_blank(marker): + continue # Fusion writes the string "None" for reagent fields that do not apply (e.g. DAPI) extra = { key: item[key] @@ -323,7 +326,7 @@ def _fusion_profile_records(profile: dict[str, Any]) -> list[tuple[str, dict[str exposure = item.get("exposure") records.append( ( - str(item.get("markerName")), + marker, { "cycle": cycle, "channel": channel, @@ -341,6 +344,9 @@ def _experiment_v4_records(experiment: ET.Element) -> list[tuple[str, dict[str, for cycle in experiment.iterfind("Cycles/Cycle"): cycle_index = _float(_text(cycle, "Index")) for channel in cycle.iterfind("Channels/Channel"): + marker = _text(channel, "MarkerName") + if marker is None or _is_blank(marker) or _text(channel, "MarkerClass") == "Blank": + continue channel_index = _float(_text(channel, "Index")) extra: dict[str, Any] = {} for key, tag in (("dye", "DyeName"), ("marker_class", "MarkerClass")): @@ -352,7 +358,7 @@ def _experiment_v4_records(experiment: ET.Element) -> list[tuple[str, dict[str, extra["wavelength_nm"] = wavelength records.append( ( - str(_text(channel, "MarkerName")), + marker, { "cycle": None if cycle_index is None else int(cycle_index), "channel": None if channel_index is None else int(channel_index), @@ -364,25 +370,59 @@ def _experiment_v4_records(experiment: ET.Element) -> list[tuple[str, dict[str, return records +def _is_blank(marker: str) -> bool: + """Whether a scan-profile marker is a blank/empty slot (Fusion ``--``, CIM ``BLANK``/``EMPTY``).""" + return marker.strip() in {"", "--"} or marker.strip().casefold() in {"blank", "empty"} + + +def _name_key(name: str) -> str: + # CIM sanitises page Biomarker names (``F4/80`` -> ``F4-80``), so compare alphanumerics only + return re.sub(r"[^0-9a-z]", "", name.casefold()) + + def _enrich(channels: list[AkoyaChannel], records: list[tuple[str, dict[str, Any]]]) -> None: - """Fill cycle/channel/extra from the scan profile, joined on (marker name, occurrence index). + """Fill cycle/channel/extra of ``channels`` from the non-blank scan-profile ``records``. - The page XML stays authoritative: channels are never reordered and the page exposure wins; channels - without a matching record are left unchanged. + Record ``k`` enriches page ``k`` when there are as many records as pages and no pair contradicts the names. + Otherwise channels are joined on the normalised name with occurrence index, but only for names that occur + as often in the pages as in the profile; any other channel is left unchanged rather than guessed. The page + XML stays authoritative: names and order never change and the page exposure wins. """ - by_key: dict[tuple[str, int], dict[str, Any]] = {} - seen: Counter[str] = Counter() - for name, record in records: - by_key[(name, seen[name])] = record - seen[name] += 1 - seen.clear() - for channel in channels: - match = by_key.get((channel.name, seen[channel.name])) - seen[channel.name] += 1 - if match is None: - continue - channel.cycle = match["cycle"] - channel.channel = match["channel"] + for channel, (_, record) in _positional_pairs(channels, records) or _name_pairs(channels, records): + channel.cycle = record["cycle"] + channel.channel = record["channel"] if channel.exposure_ms is None: - channel.exposure_ms = match["exposure_ms"] - channel.extra.update({k: v for k, v in match["extra"].items() if k not in channel.extra}) + channel.exposure_ms = record["exposure_ms"] + channel.extra.update({k: v for k, v in record["extra"].items() if k not in channel.extra}) + + +def _positional_pairs( + channels: list[AkoyaChannel], records: list[tuple[str, dict[str, Any]]] +) -> list[tuple[AkoyaChannel, tuple[str, dict[str, Any]]]]: + if len(channels) != len(records): + return [] + # Fusion orders the pages of a cycle differently from the well items; a page whose marker sits at another + # position in the profile means the orders differ and the positional pairing would be wrong + record_keys = {_name_key(name) for name, _ in records} + for channel, (name, _) in zip(channels, records, strict=True): + key = _name_key(channel.name) + if key != _name_key(name) and key in record_keys: + return [] + return list(zip(channels, records, strict=True)) + + +def _name_pairs( + channels: list[AkoyaChannel], records: list[tuple[str, dict[str, Any]]] +) -> list[tuple[AkoyaChannel, tuple[str, dict[str, Any]]]]: + channels_by_key: dict[str, list[AkoyaChannel]] = {} + records_by_key: dict[str, list[tuple[str, dict[str, Any]]]] = {} + for channel in channels: + channels_by_key.setdefault(_name_key(channel.name), []).append(channel) + for record in records: + records_by_key.setdefault(_name_key(record[0]), []).append(record) + pairs: list[tuple[AkoyaChannel, tuple[str, dict[str, Any]]]] = [] + for key, keyed_channels in channels_by_key.items(): + keyed_records = records_by_key.get(key, []) + if len(keyed_records) == len(keyed_channels): + pairs.extend(zip(keyed_channels, keyed_records, strict=True)) + return pairs diff --git a/tests/readers/test_qptiff.py b/tests/readers/test_qptiff.py index 182a3ea0..57297133 100644 --- a/tests/readers/test_qptiff.py +++ b/tests/readers/test_qptiff.py @@ -135,6 +135,46 @@ def test_enrichment_leaves_unmatched_channels_empty(tmp_path: Path) -> None: assert [c.cycle for c in scan.channels] == [1, 1, None] +def test_enrichment_ambiguous_name_left_unset(tmp_path: Path) -> None: + p = tmp_path / "x.qptiff" + wells = [[("DAPI", "DAPI"), ("CD3", "ATTO 550")], [("DAPI", "DAPI"), ("CD8", "AF 750")], [("DAPI", "DAPI")]] + write_qptiff(p, biomarkers=["DAPI", "CD3", "CD8"], scan_profile=fusion_scan_profile_json(wells)) + scan = read_qptiff_metadata(p) + assert [c.cycle for c in scan.channels] == [None, 1, 2] + assert scan.channels[0].channel is None + + +def test_enrichment_positional_when_counts_match(tmp_path: Path) -> None: + p = tmp_path / "cim.qptiff" + write_qptiff( + p, + biomarkers=["DAPI", "F4-80", "NK1"], + software="Codex Instrument Manager 1.30.0.12", + scan_profile=cim_experiment_v4_xml([["DAPI", "F4/80"], ["NK1.1", "BLANK"]]), + ) + scan = read_qptiff_metadata(p) + assert [c.name for c in scan.channels] == ["DAPI", "F4-80", "NK1"] + assert [(c.cycle, c.channel) for c in scan.channels] == [(1, 1), (1, 2), (2, 1)] + + +def test_enrichment_not_positional_when_orders_differ(tmp_path: Path) -> None: + p = tmp_path / "x.qptiff" + wells = [[("DAPI", "DAPI"), ("CD3", "ATTO 550"), ("CD8", "CY5"), ("CD20", "AF 750")]] + write_qptiff(p, biomarkers=["DAPI", "CD3", "CD20", "CD8"], scan_profile=fusion_scan_profile_json(wells)) + scan = read_qptiff_metadata(p) + assert [c.channel for c in scan.channels] == [1, 2, 4, 3] + + +def test_profile_record_without_marker_skipped(tmp_path: Path) -> None: + p = tmp_path / "x.qptiff" + profile = json.loads(fusion_scan_profile_json([[("DAPI", "DAPI"), ("CD3", "ATTO 550")]])) + del profile["experimentDescription"]["wells"][0]["items"][1]["markerName"] + write_qptiff(p, biomarkers=["DAPI", "CD3"], scan_profile=json.dumps(profile)) + dapi, cd3 = read_qptiff_metadata(p).channels + assert dapi.cycle == 1 + assert cd3.cycle is None + + @pytest.mark.parametrize(("dtype", "name"), [("uint8", "x.qptiff"), ("uint16", "x.er.qptiff")]) def test_image_is_lazy_and_correct(tmp_path: Path, dtype: str, name: str) -> None: p = tmp_path / name From d4ec43769c31ffbb800b565013ba0f3c3e945ff3 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 12:34:02 +0200 Subject: [PATCH 004/107] feat: add shared Akoya channel, transform and metadata helpers Co-Authored-By: Claude Opus 5.5 --- src/spatialdata_io/readers/_utils/_akoya.py | 138 ++++++++++++++++++++ tests/readers/test_akoya_utils.py | 125 ++++++++++++++++++ 2 files changed, 263 insertions(+) create mode 100644 src/spatialdata_io/readers/_utils/_akoya.py create mode 100644 tests/readers/test_akoya_utils.py diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py new file mode 100644 index 00000000..0baac371 --- /dev/null +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -0,0 +1,138 @@ +"""Channel, transformation, image and metadata helpers shared by the Akoya readers.""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import asdict +from typing import TYPE_CHECKING, Any + +from spatialdata._logging import logger +from spatialdata.models import Image2DModel +from spatialdata.transformations import BaseTransformation, Identity, Translation + +from spatialdata_io.readers._utils._utils import _initialize_raster_models_kwargs + +if TYPE_CHECKING: + import dask.array as da + from xarray import DataArray, DataTree + + from spatialdata_io.readers._utils._qptiff import AkoyaScan + +__all__ = [ + "apply_channel_names", + "build_image_element", + "dedupe_channel_names", + "region_transformations", + "scans_to_attrs", + "select_regions", +] + + +def dedupe_channel_names(names: Sequence[str]) -> list[str]: + """Make channel names unique as ``name``, ``name (2)``, ``name (3)``; the first occurrence is unchanged.""" + seen: Counter[str] = Counter() + deduped = [] + for name in names: + seen[name] += 1 + deduped.append(name if seen[name] == 1 else f"{name} ({seen[name]})") + return deduped + + +def apply_channel_names(scan: AkoyaScan, channel_names: Sequence[str] | None) -> list[str]: + """Return the channel names of ``scan``: the user override if given, else the de-duplicated metadata names.""" + if channel_names is None: + return dedupe_channel_names([channel.name for channel in scan.channels]) + if len(channel_names) != len(scan.channels): + raise ValueError( + f"`channel_names` has {len(channel_names)} entries, but region '{scan.region}' ({scan.path}) has " + f"{len(scan.channels)} channels: expected {len(scan.channels)}." + ) + return list(channel_names) + + +def region_transformations(scans: Sequence[AkoyaScan]) -> dict[str, BaseTransformation]: + """Compute the transformation to the ``"global"`` coordinate system of every region. + + Regions are placed with a pixel ``Translation`` derived from their stage positions only if every scan has a + stage position and a pixel size and all scans share the same slide. Otherwise all regions get ``Identity``. + """ + regions = [scan.region for scan in scans] + placeable = ( + len(scans) > 1 + and all(scan.stage_position_um is not None and scan.pixel_size_um for scan in scans) + and len({scan.slide_id for scan in scans}) == 1 + and scans[0].slide_id is not None + ) + if not placeable: + if len(scans) > 1: + logger.info( + "Regions %s have no common stage frame (missing stage position or pixel size, or different " + "slides) and are all placed at the origin of 'global', so they overlap. Use `regions=` to read a " + "single region or `spatialdata.transform` to arrange them.", + regions, + ) + return {region: Identity() for region in regions} + + positions = [scan.stage_position_um for scan in scans] + min_x = min(position[0] for position in positions if position is not None) + min_y = min(position[1] for position in positions if position is not None) + result: dict[str, BaseTransformation] = {} + for scan in scans: + x, y = scan.stage_position_um # type: ignore[misc] + pixel_size = scan.pixel_size_um + result[scan.region] = Translation([(x - min_x) / pixel_size, (y - min_y) / pixel_size], axes=("x", "y")) # type: ignore[operator] + return result + + +def build_image_element( + image: da.Array, + scan: AkoyaScan, + c_coords: list[str], + transformation: BaseTransformation, + image_models_kwargs: Mapping[str, Any], +) -> DataArray | DataTree: + """Parse a lazy ``(c, y, x)`` array, or ``(y, x, 3)`` for brightfield RGB scans, into an ``Image2DModel``.""" + image_models_kwargs, _ = _initialize_raster_models_kwargs(image_models_kwargs, {}) + if scan.rgb: + image = image.transpose(2, 0, 1) + c_coords = ["r", "g", "b"] + return Image2DModel.parse( + image, + dims=("c", "y", "x"), + c_coords=c_coords, + rgb=scan.rgb, + transformations={"global": transformation}, + **image_models_kwargs, + ) + + +def scans_to_attrs(scans: Sequence[AkoyaScan], channel_names: Mapping[str, list[str]]) -> dict[str, Any]: + """Build the JSON-serialisable per-region metadata stored in ``sdata.attrs["akoya"]``.""" + regions: dict[str, Any] = {} + for scan in scans: + names = channel_names[scan.region] + regions[scan.region] = { + "generation": scan.generation, + "software": scan.software, + "slide_id": scan.slide_id, + "pixel_size_um": scan.pixel_size_um, + "nominal_pixel_size_um": scan.nominal_pixel_size_um, + "stage_position_um": list(scan.stage_position_um) if scan.stage_position_um is not None else None, + "path": str(scan.path), + "channels": [ + {**asdict(channel), "name_deduped": name} for channel, name in zip(scan.channels, names, strict=True) + ], + "raw": scan.raw, + } + return {"regions": regions} + + +def select_regions(available: Sequence[str], regions: Sequence[str] | None) -> list[str]: + """Return the requested regions (all if ``None``), raising if one is not available.""" + if regions is None: + return list(available) + unknown = [region for region in regions if region not in available] + if unknown: + raise ValueError(f"Unknown region(s) {unknown}. Available regions: {list(available)}.") + return list(regions) diff --git a/tests/readers/test_akoya_utils.py b/tests/readers/test_akoya_utils.py new file mode 100644 index 00000000..3a4c978a --- /dev/null +++ b/tests/readers/test_akoya_utils.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import dask.array as da +import numpy as np +import pytest +from spatialdata.transformations import Identity, Translation, get_transformation + +from spatialdata_io.readers._utils._akoya import ( + apply_channel_names, + build_image_element, + dedupe_channel_names, + region_transformations, + scans_to_attrs, + select_regions, +) +from spatialdata_io.readers._utils._qptiff import AkoyaChannel, AkoyaScan + +_NO_PYRAMID = {"scale_factors": None, "chunks": (1, 8, 8)} + + +def _scan( + region: str = "reg1", + names: tuple[str, ...] = ("DAPI", "CD3", "CD8"), + stage: tuple[float, float] | None = None, + pixel_size_um: float | None = 0.5, + slide_id: str | None = "slide", + rgb: bool = False, +) -> AkoyaScan: + return AkoyaScan( + path=Path(f"{region}.qptiff"), + region=region, + generation="fusion2", + software="Fusion", + slide_id=slide_id, + pixel_size_um=pixel_size_um, + nominal_pixel_size_um=0.5, + stage_position_um=stage, + channels=[AkoyaChannel(name=n, cycle=1, extra={"k": [1, 2]}) for n in names], + rgb=rgb, + raw={"page0_xml": "", "scan_profile": None}, + ) + + +def test_dedupe_channel_names(): + assert dedupe_channel_names(["DAPI", "CD3", "DAPI", "DAPI"]) == ["DAPI", "CD3", "DAPI (2)", "DAPI (3)"] + + +def test_apply_channel_names_override_and_default(): + scan = _scan(names=("DAPI", "DAPI", "CD3")) + assert apply_channel_names(scan, None) == ["DAPI", "DAPI (2)", "CD3"] + assert apply_channel_names(scan, ["a", "b", "c"]) == ["a", "b", "c"] + + +def test_apply_channel_names_length_mismatch(): + with pytest.raises(ValueError, match="expected 3"): + apply_channel_names(_scan(), ["a", "b"]) + + +def test_region_transformations_identity_without_positions(): + result = region_transformations([_scan("a"), _scan("b")]) + assert all(isinstance(t, Identity) for t in result.values()) + assert set(result) == {"a", "b"} + + +def test_region_transformations_translation_same_slide(): + result = region_transformations([_scan("a", stage=(0.0, 0.0)), _scan("b", stage=(100.0, 50.0))]) + assert result["a"] == Translation([0.0, 0.0], axes=("x", "y")) + assert result["b"] == Translation([200.0, 100.0], axes=("x", "y")) + + +def test_region_transformations_identity_different_slides(): + result = region_transformations( + [_scan("a", stage=(0.0, 0.0), slide_id="s1"), _scan("b", stage=(100.0, 50.0), slide_id="s2")] + ) + assert all(isinstance(t, Identity) for t in result.values()) + + +def test_region_transformations_identity_missing_pixel_size(): + result = region_transformations([_scan("a", stage=(0.0, 0.0), pixel_size_um=None), _scan("b", stage=(1.0, 1.0))]) + assert all(isinstance(t, Identity) for t in result.values()) + + +def test_region_transformations_single_scan_identity(): + assert isinstance(region_transformations([_scan("a", stage=(5.0, 5.0))])["a"], Identity) + + +def test_scans_to_attrs_json_serialisable(): + scans = [_scan("a", stage=(1.0, 2.0)), _scan("b")] + names = {"a": ["DAPI", "CD3", "CD8"], "b": ["x", "y", "z"]} + attrs = scans_to_attrs(scans, names) + json.dumps(attrs) + entry = attrs["regions"]["a"] + assert entry["stage_position_um"] == [1.0, 2.0] + assert entry["path"] == "a.qptiff" + assert [c["name_deduped"] for c in attrs["regions"]["b"]["channels"]] == ["x", "y", "z"] + assert entry["channels"][0]["name"] == "DAPI" + + +def test_select_regions(): + assert select_regions(["a", "b"], None) == ["a", "b"] + assert select_regions(["a", "b"], ["b"]) == ["b"] + + +def test_select_regions_unknown_raises(): + with pytest.raises(ValueError, match="'c'.*Available.*a.*b"): + select_regions(["a", "b"], ["c"]) + + +def test_build_image_rgb_and_multiplex(): + scan = _scan() + img = da.from_array(np.random.default_rng(0).integers(0, 255, (3, 16, 16), dtype=np.uint8), chunks=(1, 8, 8)) + element = build_image_element(img, scan, ["DAPI", "CD3", "CD8"], Identity(), _NO_PYRAMID) + assert list(element.coords["c"].values) == ["DAPI", "CD3", "CD8"] + assert isinstance(get_transformation(element, "global"), Identity) + + rgb_scan = _scan(rgb=True) + rgb_img = da.from_array(np.zeros((16, 16, 3), dtype=np.uint8), chunks=(8, 8, 3)) + trans = Translation([4.0, 2.0], axes=("x", "y")) + element = build_image_element(rgb_img, rgb_scan, ["r", "g", "b"], trans, _NO_PYRAMID) + assert list(element.coords["c"].values) == ["r", "g", "b"] + assert element.shape == (3, 16, 16) + assert get_transformation(element, "global") == trans From 8c6a8d2e49c5b29c0b46021dc11598ce4552acfe Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 12:46:35 +0200 Subject: [PATCH 005/107] feat: add phenocycler reader for PhenoCycler-Fusion 1.x/2.x qptiff Co-Authored-By: Claude Opus 5.5 --- docs/api.md | 1 + src/spatialdata_io/__init__.py | 3 + src/spatialdata_io/__main__.py | 72 ++++++ src/spatialdata_io/readers/phenocycler.py | 267 ++++++++++++++++++++++ tests/test_cli_alignment.py | 1 + tests/test_phenocycler.py | 198 ++++++++++++++++ 6 files changed, 542 insertions(+) create mode 100644 src/spatialdata_io/readers/phenocycler.py create mode 100644 tests/test_phenocycler.py diff --git a/docs/api.md b/docs/api.md index 95ff1650..71dd3bc5 100644 --- a/docs/api.md +++ b/docs/api.md @@ -25,6 +25,7 @@ I/O for the `spatialdata` project. experimental.iss mcmicro merscope + phenocycler seqfish steinbock stereoseq diff --git a/src/spatialdata_io/__init__.py b/src/spatialdata_io/__init__.py index 1c10cc53..8c165b5b 100644 --- a/src/spatialdata_io/__init__.py +++ b/src/spatialdata_io/__init__.py @@ -13,6 +13,7 @@ "macsima": "spatialdata_io.readers.macsima", "mcmicro": "spatialdata_io.readers.mcmicro", "merscope": "spatialdata_io.readers.merscope", + "phenocycler": "spatialdata_io.readers.phenocycler", "seqfish": "spatialdata_io.readers.seqfish", "steinbock": "spatialdata_io.readers.steinbock", "stereoseq": "spatialdata_io.readers.stereoseq", @@ -38,6 +39,7 @@ "macsima", "mcmicro", "merscope", + "phenocycler", "seqfish", "steinbock", "stereoseq", @@ -88,6 +90,7 @@ def __dir__() -> list[str]: from spatialdata_io.readers.macsima import macsima from spatialdata_io.readers.mcmicro import mcmicro from spatialdata_io.readers.merscope import merscope + from spatialdata_io.readers.phenocycler import phenocycler from spatialdata_io.readers.seqfish import seqfish from spatialdata_io.readers.steinbock import steinbock from spatialdata_io.readers.stereoseq import stereoseq diff --git a/src/spatialdata_io/__main__.py b/src/spatialdata_io/__main__.py index f0a5f9f7..7fc3a559 100644 --- a/src/spatialdata_io/__main__.py +++ b/src/spatialdata_io/__main__.py @@ -357,6 +357,78 @@ def merscope_wrapper( sdata.write(output) +def _parse_list_param(value: str | None) -> list[str] | None: + return None if value is None else [item.strip() for item in value.split(",")] + + +@cli.command(name="phenocycler") +@click.option( + "--input", + "-i", + type=click.Path(exists=True), + help="Path to a .qptiff file or to a directory containing .qptiff files.", + required=True, +) +@click.option("--output", "-o", type=click.Path(), help="Path to the output.zarr file.", required=True) +@click.option( + "--regions", type=str, default=None, help="Comma-separated regions to read; all if omitted. [default: None]" +) +@click.option( + "--channel-names", + type=str, + default=None, + help="Comma-separated channel names overriding the ones in the qptiff. [default: None]", +) +@click.option("--cells-table", type=click.Path(exists=True), default=None, help="Path to a cell table. [default: None]") +@click.option("--cells-table-format", type=str, default="auto", help="Format of the cell table. [default: auto]") +@click.option("--labels", type=click.Path(exists=True), default=None, help="Path to a label mask. [default: None]") +@click.option( + "--image-models-kwargs", + type=str, + default="{}", + help="JSON string of keyword arguments passed to Image2DModel. [default: {}]", +) +@click.option( + "--labels-models-kwargs", + type=str, + default="{}", + help="JSON string of keyword arguments passed to Labels2DModel. [default: {}]", +) +@click.option( + "--cells-table-kwargs", + type=str, + default="{}", + help="JSON string of keyword arguments for reading the cell table. [default: {}]", +) +def phenocycler_wrapper( + input: str, + output: str, + regions: str | None = None, + channel_names: str | None = None, + cells_table: str | None = None, + cells_table_format: str = "auto", + labels: str | None = None, + image_models_kwargs: str = "{}", + labels_models_kwargs: str = "{}", + cells_table_kwargs: str = "{}", +) -> None: + """PhenoCycler-Fusion conversion to SpatialData.""" + from spatialdata_io.readers.phenocycler import phenocycler + + sdata = phenocycler( + input, + regions=_parse_list_param(regions), + channel_names=_parse_list_param(channel_names), + cells_table=cells_table, + cells_table_format=cells_table_format, + labels=labels, + image_models_kwargs=_parse_json_param(image_models_kwargs, "image_models_kwargs"), + labels_models_kwargs=_parse_json_param(labels_models_kwargs, "labels_models_kwargs"), + cells_table_kwargs=_parse_json_param(cells_table_kwargs, "cells_table_kwargs"), + ) + sdata.write(output) + + @cli.command(name="seqfish") @_input_output_click_options @click.option("--load-images", type=bool, default=True, help="Whether to load images. [default: True]") diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py new file mode 100644 index 00000000..6a8f6f5d --- /dev/null +++ b/src/spatialdata_io/readers/phenocycler.py @@ -0,0 +1,267 @@ +from __future__ import annotations + +import csv +import json +import re +import warnings +from pathlib import Path +from types import MappingProxyType +from typing import TYPE_CHECKING, Any + +from spatialdata import SpatialData +from spatialdata._logging import logger + +from spatialdata_io._constants._constants import PhenocyclerKeys +from spatialdata_io._docs import inject_docs +from spatialdata_io.readers._utils._akoya import ( + apply_channel_names, + build_image_element, + region_transformations, + scans_to_attrs, + select_regions, +) +from spatialdata_io.readers._utils._qptiff import ( + _fusion_profile_records, + _is_blank, + read_qptiff_image, + read_qptiff_metadata, +) +from spatialdata_io.readers._utils._utils import _set_reader_metadata + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + from spatialdata_io.readers._utils._qptiff import AkoyaScan + +__all__ = ["discover_qptiffs", "phenocycler"] + +_SCAN_SUFFIX = re.compile(r"_Scan\d+$") +# the Fusion writes per-cycle acquisition files to ``raw/images/phenocycler/`` (HuBMAP layout) +_ACQUISITION_DIR = ("raw", "images", "phenocycler") + + +@inject_docs(vx=PhenocyclerKeys) +def phenocycler( + path: str | Path, + regions: Sequence[str] | None = None, + channel_names: Sequence[str] | None = None, + cells_table: str | Path | Mapping[str, str | Path] | None = None, + cells_table_format: str = "auto", + labels: str | Path | Mapping[str, str | Path] | None = None, + image_models_kwargs: Mapping[str, Any] = MappingProxyType({}), + labels_models_kwargs: Mapping[str, Any] = MappingProxyType({}), + cells_table_kwargs: Mapping[str, Any] = MappingProxyType({}), +) -> SpatialData: + """Read *Akoya PhenoCycler-Fusion* 1.x/2.x data. + + This function reads the following files: + + - ``*{vx.QPTIFF_SUFFIX!r}`` / ``*{vx.ER_QPTIFF_SUFFIX!r}``: the scans, one region each. The images are read + lazily; when both ``X{vx.QPTIFF_SUFFIX}`` and ``X{vx.ER_QPTIFF_SUFFIX}`` exist, the extended-range + (higher bit depth) ``{vx.ER_QPTIFF_SUFFIX!r}`` is used. + - ``*{vx.XPD_SUFFIX!r}``: optional experiment protocol (JSON) next to a scan. + - ``*{vx.CHANNELS_CSV_SUFFIX!r}``: optional channel list next to a scan or, in the HuBMAP layout, in + ``lab_processed/images/``. + + Sidecar files are matched to a scan by file-name prefix and stored as metadata only: the channel names always + come from the per-page XML of the qptiff. A sidecar whose number of markers differs from the number of channels + raises a warning. Acquisition files (``*.raw.qptiff``, ``*.qptiff.raw``, ``*.qptiff.intermediate`` and anything + in ``raw/images/phenocycler/``) are ignored. + + The region name is the file name without the ``{vx.ER_QPTIFF_SUFFIX!r}``/``{vx.QPTIFF_SUFFIX!r}`` suffix and + without a trailing ``_Scan``, unless that makes two regions share a name. Each region is stored as the image + ``{vx.IMAGE_SUFFIX}``. Several regions scanned on the same slide are placed at their stage positions in + the ``"global"`` coordinate system; otherwise all regions sit at its origin. The per-region metadata is stored + in ``sdata.attrs[{vx.ATTRS_KEY!r}]``. + + .. seealso:: + + - `Akoya PhenoCycler-Fusion `_. + + Parameters + ---------- + path + Path to a ``{vx.QPTIFF_SUFFIX!r}`` file, or to a directory that is searched recursively for them. + regions + Regions to read. If `None`, all regions are read. + channel_names + Channel names to use instead of the ones stored in the qptiff; must have one entry per channel of every + region read. + cells_table + Not implemented yet. + cells_table_format + Not implemented yet. + labels + Not implemented yet. + image_models_kwargs + Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. + labels_models_kwargs + Not implemented yet. + cells_table_kwargs + Not implemented yet. + + Returns + ------- + :class:`spatialdata.SpatialData` + """ + if cells_table is not None or labels is not None: + raise NotImplementedError("Reading cell tables and labels with `phenocycler` is added in a later task.") + + path = Path(path) + qptiffs = discover_qptiffs(path) + selected = select_regions(list(qptiffs), regions) + + scans = [read_qptiff_metadata(qptiffs[region], region=region) for region in selected] + names = {} + for scan in scans: + names[scan.region] = apply_channel_names(scan, channel_names) + _read_sidecars(scan) + + transformations = region_transformations(scans) + images = { + f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}": build_image_element( + read_qptiff_image(scan.path), + scan, + names[scan.region], + transformations[scan.region], + image_models_kwargs, + ) + for scan in scans + } + sdata = SpatialData(images=images) + sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value] = scans_to_attrs(scans, names) + return _set_reader_metadata(sdata, "phenocycler") + + +def discover_qptiffs(path: Path) -> dict[str, Path]: + """Find the qptiff scans of a PhenoCycler-Fusion dataset. + + Parameters + ---------- + path + A ``.qptiff`` file, or a directory that is searched recursively. + + Returns + ------- + The scan file of every region, sorted by path. + """ + if not path.exists(): + raise FileNotFoundError(f"{path} does not exist.") + if path.is_file(): + return {_region(_base_name(path)): path} + + # one scan per (directory, base name), preferring the extended-range `.er.qptiff` + scans: dict[tuple[Path, str], Path] = {} + for qptiff in sorted(path.rglob(f"*{PhenocyclerKeys.QPTIFF_SUFFIX}")): + if not qptiff.is_file() or _is_acquisition_file(qptiff): + continue + key = (qptiff.parent, _base_name(qptiff)) + if key not in scans or _is_extended_range(qptiff): + scans[key] = qptiff + if not scans: + raise FileNotFoundError( + f"No .qptiff files found in {path} (searched recursively for '*.qptiff', excluding acquisition files)." + ) + + bases = [base for _, base in scans] + regions = [_region(base) for base in bases] + # stripping `_Scan` must not merge two scans into one region + regions = [base if regions.count(region) > 1 else region for base, region in zip(bases, regions, strict=True)] + if len(set(regions)) < len(regions): + duplicated = sorted({region for region in regions if regions.count(region) > 1}) + raise ValueError( + f"Several qptiff files in {path} map to the same region(s) {duplicated}. Pass the path of a single " + "qptiff file or of a directory that contains each region once." + ) + return dict(zip(regions, scans.values(), strict=True)) + + +def _base_name(path: Path) -> str: + """File name without the ``.er.qptiff``/``.qptiff`` suffix.""" + for suffix in (PhenocyclerKeys.ER_QPTIFF_SUFFIX, PhenocyclerKeys.QPTIFF_SUFFIX): + if path.name.lower().endswith(suffix): + return path.name[: -len(suffix)] + return path.stem + + +def _region(base_name: str) -> str: + return _SCAN_SUFFIX.sub("", base_name) or base_name + + +def _is_extended_range(path: Path) -> bool: + return path.name.lower().endswith(PhenocyclerKeys.ER_QPTIFF_SUFFIX) + + +def _is_acquisition_file(path: Path) -> bool: + if path.name.lower().endswith(".raw" + PhenocyclerKeys.QPTIFF_SUFFIX): + return True + parts = path.parts + n = len(_ACQUISITION_DIR) + return any(parts[i : i + n] == _ACQUISITION_DIR for i in range(len(parts) - n)) + + +def _read_sidecars(scan: AkoyaScan) -> None: + """Store the ``.xpd`` and ``*channels.csv`` sidecars of ``scan`` in ``scan.raw`` and check their channel count.""" + base = _base_name(scan.path) + xpd = _find_sidecar(scan.path, base, PhenocyclerKeys.XPD_SUFFIX) + protocol = _read_xpd(xpd) if xpd is not None else None + if xpd is not None and protocol is not None: + scan.raw["xpd"] = protocol + # an .xpd has the layout of the Fusion scan profile's `experimentDescription` + markers = [marker for marker, _ in _fusion_profile_records({"experimentDescription": protocol})] + _check_channel_count(scan, xpd, markers) + + channels_csv = _find_sidecar(scan.path, base, PhenocyclerKeys.CHANNELS_CSV_SUFFIX) + if channels_csv is not None: + with channels_csv.open(newline="") as f: + rows = list(csv.DictReader(f)) + scan.raw["channels_csv"] = rows + column = next((key for key in (rows[0] if rows else {}) if key.strip().casefold() == "marker"), None) + if column is not None: + markers = [row[column] for row in rows if row[column] and not _is_blank(row[column])] + _check_channel_count(scan, channels_csv, markers) + + +def _read_xpd(path: Path) -> dict[str, Any] | None: + try: + protocol = json.loads(path.read_text()) + except json.JSONDecodeError as e: + warnings.warn(f"Ignoring {path}: it is not valid JSON ({e}).", UserWarning, stacklevel=2) + return None + if not isinstance(protocol, dict): + warnings.warn(f"Ignoring {path}: it is not a JSON object.", UserWarning, stacklevel=2) + return None + return protocol + + +def _find_sidecar(qptiff: Path, base: str, suffix: str) -> Path | None: + """First file ending in ``suffix`` whose name starts with ``base``, next to the scan or in HuBMAP's processed dir.""" + pattern = re.compile(rf"{re.escape(base)}(?![0-9A-Za-z]).*{re.escape(suffix)}$") + directories = [qptiff.parent] + # HuBMAP keeps `raw/images/.qptiff` and `lab_processed/images/.qptiff.channels.csv` + for ancestor in qptiff.parents: + if (ancestor / "raw").is_dir() and qptiff.is_relative_to(ancestor / "raw"): + directories.append(ancestor / "lab_processed" / "images") + break + for directory in directories: + if not directory.is_dir(): + continue + matches = sorted(p for p in directory.iterdir() if p.is_file() and pattern.match(p.name)) + if matches: + if len(matches) > 1: + logger.info("Found several sidecar files %s for %s; using %s.", matches, qptiff, matches[0]) + return matches[0] + return None + + +def _check_channel_count(scan: AkoyaScan, sidecar: Path, markers: list[str]) -> None: + # the Fusion acquires DAPI in every cycle but stores it as one page, so each marker is counted once + n_markers = len(set(markers)) + n_channels = len({channel.name for channel in scan.channels}) + if n_markers != n_channels: + warnings.warn( + f"{sidecar.name} lists {n_markers} distinct markers, but {scan.path.name} has {n_channels} distinct " + f"channel names ({len(scan.channels)} channels); using the channel names from the qptiff.", + UserWarning, + stacklevel=2, + ) diff --git a/tests/test_cli_alignment.py b/tests/test_cli_alignment.py index f2d6e6b1..5631943a 100644 --- a/tests/test_cli_alignment.py +++ b/tests/test_cli_alignment.py @@ -43,6 +43,7 @@ ("spatialdata_io.readers.macsima", "macsima", "macsima_wrapper"), ("spatialdata_io.readers.mcmicro", "mcmicro", "mcmicro_wrapper"), ("spatialdata_io.readers.merscope", "merscope", "merscope_wrapper"), + ("spatialdata_io.readers.phenocycler", "phenocycler", "phenocycler_wrapper"), ("spatialdata_io.readers.seqfish", "seqfish", "seqfish_wrapper"), ("spatialdata_io.readers.steinbock", "steinbock", "steinbock_wrapper"), ("spatialdata_io.readers.stereoseq", "stereoseq", "stereoseq_wrapper"), diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py new file mode 100644 index 00000000..ebb6416d --- /dev/null +++ b/tests/test_phenocycler.py @@ -0,0 +1,198 @@ +"""Tests for the PhenoCycler-Fusion reader on synthetic qptiff files.""" + +from __future__ import annotations + +import json +import warnings +from pathlib import Path + +import dask.array as da +import numpy as np +import pytest +import tifffile +from spatialdata import read_zarr +from spatialdata.models import get_channel_names + +from spatialdata_io import phenocycler +from tests._akoya_data import write_qptiff + +_FAST = {"scale_factors": [2]} + + +def _level0(sdata, name): # type: ignore[no-untyped-def] + return sdata.images[name]["scale0"]["image"] + + +def test_read_single_file(tmp_path: Path) -> None: + path = tmp_path / "Tonsil_Scan1.qptiff" + data = write_qptiff(path, biomarkers=["DAPI", "CD3", "CD3"]) + + sdata = phenocycler(path, image_models_kwargs=_FAST) + + assert list(sdata.images) == ["Tonsil_image"] + assert get_channel_names(sdata.images["Tonsil_image"]) == ["DAPI", "CD3", "CD3 (2)"] + np.testing.assert_array_equal(_level0(sdata, "Tonsil_image").values, data) + assert sdata.attrs["akoya"]["regions"]["Tonsil"]["generation"] == "fusion2" + assert sdata.attrs["spatialdata_io_reader"] == "phenocycler" + + +def test_read_directory_multiple_scans_and_subset(tmp_path: Path) -> None: + write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) + write_qptiff(tmp_path / "B_Scan1.qptiff", biomarkers=["DAPI", "CD8"]) + + sdata = phenocycler(str(tmp_path), image_models_kwargs=_FAST) + assert sorted(sdata.images) == ["A_image", "B_image"] + assert sorted(sdata.attrs["akoya"]["regions"]) == ["A", "B"] + + subset = phenocycler(tmp_path, regions=["B"], image_models_kwargs=_FAST) + assert list(subset.images) == ["B_image"] + assert list(subset.attrs["akoya"]["regions"]) == ["B"] + + with pytest.raises(ValueError, match="Z"): + phenocycler(tmp_path, regions=["Z"]) + + +def test_region_collision_keeps_full_stem(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + write_qptiff(tmp_path / "S_Scan2.qptiff", biomarkers=["DAPI"]) + + sdata = phenocycler(tmp_path, image_models_kwargs=_FAST) + + assert sorted(sdata.images) == ["S_Scan1_image", "S_Scan2_image"] + + +def test_er_preferred(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI", "CD3"], dtype="uint8") + write_qptiff(tmp_path / "S_Scan1.er.qptiff", biomarkers=["DAPI", "CD3"], dtype="uint16") + + sdata = phenocycler(tmp_path, image_models_kwargs=_FAST) + + assert list(sdata.images) == ["S_image"] + assert _level0(sdata, "S_image").dtype == np.uint16 + assert sdata.attrs["akoya"]["regions"]["S"]["path"].endswith("S_Scan1.er.qptiff") + + +def test_ignores_acquisition_files(tmp_path: Path) -> None: + images = tmp_path / "raw" / "images" + acquisition = images / "phenocycler" + acquisition.mkdir(parents=True) + write_qptiff(images / "S_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) + write_qptiff(acquisition / "S_Scan1_Cycle1.qptiff.raw", biomarkers=["DAPI"]) + write_qptiff(acquisition / "S.raw.qptiff", biomarkers=["DAPI"]) + write_qptiff(acquisition / "S_Scan1_Cycle1.qptiff", biomarkers=["DAPI"]) + write_qptiff(images / "S_Scan1.qptiff.intermediate", biomarkers=["DAPI"]) + + sdata = phenocycler(tmp_path, image_models_kwargs=_FAST) + + assert list(sdata.images) == ["S_image"] + + +def test_sidecars_hubmap_layout(tmp_path: Path) -> None: + raw_images = tmp_path / "raw" / "images" + processed = tmp_path / "lab_processed" / "images" + raw_images.mkdir(parents=True) + processed.mkdir(parents=True) + write_qptiff(raw_images / "S_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) + wells = [ + {"wellName": "A1", "items": [{"markerName": "DAPI"}]}, + {"wellName": "A2", "items": [{"markerName": "DAPI"}, {"markerName": "CD3"}, {"markerName": "--"}]}, + ] + xpd = {"formatType": "Protein", "wells": wells} + (raw_images / "S_Scan1.xpd").write_text(json.dumps(xpd)) + (processed / "S_Scan1.qptiff.channels.csv").write_text("Channel,Marker\n1,DAPI\n2,CD3\n") + + # DAPI repeats per cycle and blanks do not count as channels, so the sidecars match the two pages + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + sdata = phenocycler(tmp_path, image_models_kwargs=_FAST) + + raw = sdata.attrs["akoya"]["regions"]["S"]["raw"] + assert raw["xpd"] == xpd + assert raw["channels_csv"] == [{"Channel": "1", "Marker": "DAPI"}, {"Channel": "2", "Marker": "CD3"}] + + +def test_sidecar_channel_mismatch_warns(tmp_path: Path) -> None: + path = tmp_path / "S_Scan1.qptiff" + write_qptiff(path, biomarkers=["DAPI", "CD3", "CD8"]) + xpd = {"wells": [{"wellName": "A1", "items": [{"markerName": "DAPI"}, {"markerName": "CD4"}]}]} + (tmp_path / "S_Scan1.xpd").write_text(json.dumps(xpd)) + + with pytest.warns(UserWarning, match="3") as record: + sdata = phenocycler(path, image_models_kwargs=_FAST) + + assert any("2" in str(warning.message) for warning in record) + assert get_channel_names(sdata.images["S_image"]) == ["DAPI", "CD3", "CD8"] + + +def test_channels_csv_mismatch_warns(tmp_path: Path) -> None: + path = tmp_path / "S_Scan1.qptiff" + write_qptiff(path, biomarkers=["DAPI", "CD3"]) + (tmp_path / "S_Scan1.qptiff.channels.csv").write_text("Marker\nDAPI\nCD3\nCD8\n") + + with pytest.warns(UserWarning, match="channels.csv"): + phenocycler(path, image_models_kwargs=_FAST) + + +def test_invalid_xpd_warns_and_is_ignored(tmp_path: Path) -> None: + path = tmp_path / "S_Scan1.qptiff" + write_qptiff(path, biomarkers=["DAPI"]) + (tmp_path / "S_Scan1.xpd").write_text("not json") + + with pytest.warns(UserWarning, match="not valid JSON"): + sdata = phenocycler(path, image_models_kwargs=_FAST) + + assert "xpd" not in sdata.attrs["akoya"]["regions"]["S"]["raw"] + + +def test_empty_dir_raises(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError, match="\\*.qptiff"): + phenocycler(str(tmp_path)) + + +def test_channel_names_override(tmp_path: Path) -> None: + path = tmp_path / "S_Scan1.qptiff" + write_qptiff(path, biomarkers=["DAPI", "CD3"]) + + sdata = phenocycler(path, channel_names=["nuclei", "T cells"], image_models_kwargs=_FAST) + assert get_channel_names(sdata.images["S_image"]) == ["nuclei", "T cells"] + channels = sdata.attrs["akoya"]["regions"]["S"]["channels"] + assert [channel["name_deduped"] for channel in channels] == ["nuclei", "T cells"] + + with pytest.raises(ValueError, match="expected 2"): + phenocycler(path, channel_names=["only one"]) + + +def test_cells_table_and_labels_not_implemented(tmp_path: Path) -> None: + path = tmp_path / "S_Scan1.qptiff" + write_qptiff(path, biomarkers=["DAPI"]) + + with pytest.raises(NotImplementedError): + phenocycler(path, cells_table=tmp_path / "cells.csv") + with pytest.raises(NotImplementedError): + phenocycler(path, labels=tmp_path / "mask.tif") + + +def test_write_roundtrip(tmp_path: Path) -> None: + path = tmp_path / "S_Scan1.qptiff" + write_qptiff(path, biomarkers=["DAPI", "CD3", "CD3"]) + sdata = phenocycler(path, image_models_kwargs=_FAST) + + sdata.write(tmp_path / "x.zarr") + read = read_zarr(tmp_path / "x.zarr") + + assert get_channel_names(read.images["S_image"]) == get_channel_names(sdata.images["S_image"]) + assert read.attrs["akoya"] == sdata.attrs["akoya"] + assert read.attrs["spatialdata_io_reader"] == "phenocycler" + + +def test_is_lazy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + path = tmp_path / "S_Scan1.qptiff" + write_qptiff(path, biomarkers=["DAPI", "CD3"]) + + def fail(*args: object, **kwargs: object) -> None: + raise AssertionError("full page decode during read") + + monkeypatch.setattr(tifffile.TiffPage, "asarray", fail) + sdata = phenocycler(path, image_models_kwargs=_FAST) + + assert isinstance(_level0(sdata, "S_image").data, da.Array) From 08b73f142a1a494923d26f4fd84946812522ae35 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 13:06:41 +0200 Subject: [PATCH 006/107] feat!: rewrite codex reader discovery and images for CODEX Processor and CIM outputs codex() now discovers every region of a CODEX Processor/Uploader tree (per-marker stitched tifs, hyperstacks named via ImageJ Labels or channelNames.txt) and CODEX Instrument Manager qptiffs, reads them lazily and stores one `_image` per region with metadata in sdata.attrs["akoya"]. Paths are resolved against the input instead of the cwd. Cell tables are discovered but not read yet, so codex() returns images only for now. BREAKING CHANGE: the image element is named `_image` instead of `images`. Co-Authored-By: Claude Opus 5.5 --- src/spatialdata_io/__main__.py | 63 ++- src/spatialdata_io/_constants/_constants.py | 3 + src/spatialdata_io/readers/_utils/_akoya.py | 18 + src/spatialdata_io/readers/codex.py | 447 +++++++++++++++++--- src/spatialdata_io/readers/phenocycler.py | 16 +- tests/test_codex.py | 167 ++++++++ 6 files changed, 643 insertions(+), 71 deletions(-) create mode 100644 tests/test_codex.py diff --git a/src/spatialdata_io/__main__.py b/src/spatialdata_io/__main__.py index 7fc3a559..205ddbe7 100644 --- a/src/spatialdata_io/__main__.py +++ b/src/spatialdata_io/__main__.py @@ -47,29 +47,86 @@ def _parse_json_param(value: str, param_name: str) -> dict[str, Any]: @cli.command(name="codex") -@_input_output_click_options +@click.option( + "--input", + "-i", + type=click.Path(exists=True), + help="Path to a CODEX output directory or to a single .qptiff/.tif image.", + required=True, +) +@click.option("--output", "-o", type=click.Path(), help="Path to the output.zarr file.", required=True) @click.option( "--fcs", type=bool, default=True, - help="Whether the .fcs file is provided if False a .csv file is expected. [default: True]", + help="Whether to prefer .fcs over .csv cell tables when both exist. [default: True]", +) +@click.option( + "--regions", type=str, default=None, help="Comma-separated regions to read; all if omitted. [default: None]" +) +@click.option( + "--channel-names", + type=str, + default=None, + help="Comma-separated channel names overriding the discovered ones. [default: None]", ) +@click.option("--cells-table", type=click.Path(exists=True), default=None, help="Path to a cell table. [default: None]") +@click.option("--cells-table-format", type=str, default="auto", help="Format of the cell table. [default: auto]") +@click.option("--labels", type=click.Path(exists=True), default=None, help="Path to a label mask. [default: None]") @click.option( "--imread-kwargs", type=str, default="{}", help="JSON string of keyword arguments passed to imread. [default: {}]", ) +@click.option( + "--image-models-kwargs", + type=str, + default="{}", + help="JSON string of keyword arguments passed to Image2DModel. [default: {}]", +) +@click.option( + "--labels-models-kwargs", + type=str, + default="{}", + help="JSON string of keyword arguments passed to Labels2DModel. [default: {}]", +) +@click.option( + "--cells-table-kwargs", + type=str, + default="{}", + help="JSON string of keyword arguments for reading the cell table. [default: {}]", +) def codex_wrapper( input: str, output: str, fcs: bool = True, + regions: str | None = None, + channel_names: str | None = None, + cells_table: str | None = None, + cells_table_format: str = "auto", + labels: str | None = None, imread_kwargs: str = "{}", + image_models_kwargs: str = "{}", + labels_models_kwargs: str = "{}", + cells_table_kwargs: str = "{}", ) -> None: """Codex conversion to SpatialData.""" from spatialdata_io.readers.codex import codex - sdata = codex(input, fcs=fcs, imread_kwargs=_parse_json_param(imread_kwargs, "imread_kwargs")) + sdata = codex( + input, + fcs=fcs, + regions=_parse_list_param(regions), + channel_names=_parse_list_param(channel_names), + cells_table=cells_table, + cells_table_format=cells_table_format, + labels=labels, + imread_kwargs=_parse_json_param(imread_kwargs, "imread_kwargs"), + image_models_kwargs=_parse_json_param(image_models_kwargs, "image_models_kwargs"), + labels_models_kwargs=_parse_json_param(labels_models_kwargs, "labels_models_kwargs"), + cells_table_kwargs=_parse_json_param(cells_table_kwargs, "cells_table_kwargs"), + ) sdata.write(output) diff --git a/src/spatialdata_io/_constants/_constants.py b/src/spatialdata_io/_constants/_constants.py index 8d7fbf98..cfbbd08a 100644 --- a/src/spatialdata_io/_constants/_constants.py +++ b/src/spatialdata_io/_constants/_constants.py @@ -10,6 +10,9 @@ class CodexKeys(ModeEnum): # files and directories FCS_FILE = ".fcs" FCS_FILE_CSV = ".csv" + CHANNEL_NAMES_FILE = "channelNames.txt" + EXPERIMENT_FILE = "experiment.json" + QPTIFF = ".qptiff" # metadata REGION_KEY = "region" INSTANCE_KEY = "cell_id" diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index 0baac371..82c5fec4 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json +import warnings from collections import Counter from collections.abc import Mapping, Sequence from dataclasses import asdict @@ -14,6 +16,8 @@ from spatialdata_io.readers._utils._utils import _initialize_raster_models_kwargs if TYPE_CHECKING: + from pathlib import Path + import dask.array as da from xarray import DataArray, DataTree @@ -23,6 +27,7 @@ "apply_channel_names", "build_image_element", "dedupe_channel_names", + "read_json_object", "region_transformations", "scans_to_attrs", "select_regions", @@ -136,3 +141,16 @@ def select_regions(available: Sequence[str], regions: Sequence[str] | None) -> l if unknown: raise ValueError(f"Unknown region(s) {unknown}. Available regions: {list(available)}.") return list(regions) + + +def read_json_object(path: Path) -> dict[str, Any] | None: + """Read a JSON object from ``path``; warn and return ``None`` if the file is not valid JSON or not an object.""" + try: + content = json.loads(path.read_text()) + except json.JSONDecodeError as e: + warnings.warn(f"Ignoring {path}: it is not valid JSON ({e}).", UserWarning, stacklevel=2) + return None + if not isinstance(content, dict): + warnings.warn(f"Ignoring {path}: it is not a JSON object.", UserWarning, stacklevel=2) + return None + return content diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 925ed077..2916c65f 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -1,41 +1,108 @@ from __future__ import annotations +import math import os import re +import warnings +from dataclasses import dataclass from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Any -import anndata as ad -import pandas as pd -import readfcs +import dask.array as da +import tifffile +import zarr from dask_image.imread import imread from spatialdata import SpatialData from spatialdata._logging import logger -from spatialdata.models import Image2DModel, ShapesModel, TableModel -from spatialdata_io._constants._constants import CodexKeys +from spatialdata_io._constants._constants import CodexKeys, PhenocyclerKeys from spatialdata_io._docs import inject_docs +from spatialdata_io.readers._utils._akoya import ( + apply_channel_names, + build_image_element, + read_json_object, + region_transformations, + scans_to_attrs, + select_regions, +) +from spatialdata_io.readers._utils._qptiff import AkoyaChannel, AkoyaScan, read_qptiff_image, read_qptiff_metadata from spatialdata_io.readers._utils._utils import _set_reader_metadata if TYPE_CHECKING: - from collections.abc import Mapping + from collections.abc import Mapping, Sequence -__all__ = ["codex"] +__all__ = ["LegacyLayout", "codex", "discover_legacy", "read_legacy_image"] + +_REGION = re.compile(r"reg(\d+)", re.IGNORECASE) +_PER_MARKER = re.compile(r"reg(\d+)_cyc(\d+)_ch(\d+)_(.+)\.tiff?$", re.IGNORECASE) +_MONTAGE = re.compile(r"reg\d+_montage\.tiff?$", re.IGNORECASE) +_TILE_STACK = re.compile(r"reg\d+_X\d+_Y\d+_Z\d+\.tiff?$", re.IGNORECASE) +# raw Keyence acquisition folders, e.g. `cyc001_reg001/1_00001_Z001_CH1.tif` +_RAW_CYCLE_DIR = re.compile(r"cyc\d+_reg\d+", re.IGNORECASE) +_TIFF_SUFFIXES = (".tif", ".tiff") +_TABLE_SUFFIXES = (CodexKeys.FCS_FILE.value, CodexKeys.FCS_FILE_CSV.value) +# deep enough for `/processed_/segm/segm-1/fcs/compensated/` +_MAX_DEPTH = 8 +# `channelNames.txt`/`experiment.json` sit in the experiment folder, up to `processed_/stitched/reg001/` above +_SIDECAR_ANCESTORS = 3 +_PATTERNS = ( + "'regNNN_cycCCC_chHHH_.tif(f)' (per-marker stitched images)", + "'regNNN_montage.tif', 'regNNN_X01_Y01_Z01.tif' or '*regNNN*.tif' (hyperstacks)", + f"'*{CodexKeys.QPTIFF}' (CODEX Instrument Manager)", + f"'*{CodexKeys.FCS_FILE}' or '*{CodexKeys.FCS_FILE_CSV}' (cell tables)", +) + + +@dataclass +class LegacyLayout: + """Files of a CODEX Processor/Uploader output tree or of CODEX Instrument Manager scans. + + ``images`` maps each region to its per-marker images (sorted by cycle and channel), to its single hyperstack + or to its single ``.qptiff``. ``tables`` maps each region, or ``"*"`` for a table without a region in its + file name, to its ``.fcs``/``.csv`` cell table. + """ + + images: dict[str, list[Path]] + tables: dict[str, Path] + channel_names_txt: Path | None + experiment_json: Path | None @inject_docs(vx=CodexKeys) def codex( path: str | Path, fcs: bool = True, + regions: Sequence[str] | None = None, + channel_names: Sequence[str] | None = None, + cells_table: str | Path | Mapping[str, str | Path] | None = None, + cells_table_format: str = "auto", + labels: str | Path | Mapping[str, str | Path] | None = None, imread_kwargs: Mapping[str, Any] = MappingProxyType({}), + image_models_kwargs: Mapping[str, Any] = MappingProxyType({}), + labels_models_kwargs: Mapping[str, Any] = MappingProxyType({}), + cells_table_kwargs: Mapping[str, Any] = MappingProxyType({}), ) -> SpatialData: - """Read *CODEX* formatted dataset. + """Read *CODEX* (CODEX Processor, CODEX Uploader and CODEX Instrument Manager) data. + + This function reads the following files, searching ``path`` recursively: - This function reads the following files: + - ``regNNN_cycCCC_chHHH_{vx.IMAGE_TIF}(f)``: per-marker stitched images of the CODEX Processor, + stacked lazily into one image per region in (cycle, channel) order; the channel names come from the file + names. + - ``regNNN_montage{vx.IMAGE_TIF}``, ``regNNN_X01_Y01_Z01{vx.IMAGE_TIF}`` or ``*regNNN*{vx.IMAGE_TIF}``: + hyperstacks (CODEX Uploader, older Processor versions). All axes except ``y`` and ``x`` are flattened + into channels, in file order. The channel names come from the ImageJ ``Labels``, else from + ``{vx.CHANNEL_NAMES_FILE!r}`` (one name per line, cycle-major), else they are ``cycCCC_chHHH``. + - ``*{vx.QPTIFF!r}``: scans of the CODEX Instrument Manager, read lazily with their channel metadata. + - ``{vx.EXPERIMENT_FILE!r}``: optional; the pixel size is taken from its ``xyResolution`` (nm). + - ``*{vx.FCS_FILE!r}`` / ``*{vx.FCS_FILE_CSV!r}``: cell tables, not read yet. - - ``_`{vx.FCS_FILE!r}```: Counts and metadata file. - - ``_`{vx.IMAGE_TIF!r}```: High resolution tif image. + Raw acquisition and intermediate folders (``cycCCC_regNNN/``, ``tiles/`` and, next to ``stitched/``, + ``bestFocus/``) are ignored. The region id ``regNNN`` is taken from the file names; a ``{vx.QPTIFF!r}`` without + one is named after its file. If a directory contains no such files, its top-level ``{vx.IMAGE_TIF}(f)`` files + are read as hyperstacks named after their files. Each region is stored as the image ``_image`` in the + ``"global"`` coordinate system and its metadata in ``sdata.attrs["akoya"]``. .. seealso:: @@ -44,60 +111,332 @@ def codex( Parameters ---------- path - Path to the directory containing the data. + Path to a CODEX output directory, or to a single image (``{vx.QPTIFF!r}`` or ``{vx.IMAGE_TIF}(f)``). fcs - Whether a .fcs file is provided. If False, a .csv file is expected. + Whether to prefer ``{vx.FCS_FILE!r}`` cell tables over ``{vx.FCS_FILE_CSV!r}`` ones when both exist. + regions + Regions to read, e.g. ``["reg001"]``. If `None`, all regions are read. + channel_names + Channel names to use instead of the discovered ones; must have one entry per channel of every region read. + cells_table + Not implemented yet. + cells_table_format + Not implemented yet. + labels + Not implemented yet. imread_kwargs - Keyword arguments passed to :func:`dask_image.imread.imread`. + Keyword arguments passed to :func:`dask_image.imread.imread`, which is only used for ``{vx.IMAGE_TIF}(f)`` + files that :mod:`tifffile` cannot read lazily (e.g. unsupported compression). + image_models_kwargs + Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. + labels_models_kwargs + Not implemented yet. + cells_table_kwargs + Not implemented yet. Returns ------- :class:`spatialdata.SpatialData` """ + if cells_table is not None or labels is not None: + raise NotImplementedError("Reading cell tables and labels with `codex` is added in a later task.") + path = Path(path) - patt = re.compile(".*.fcs") if fcs else re.compile(".*.csv") - path_files = [i for i in os.listdir(path) if patt.match(i)] - if path_files and CodexKeys.FCS_FILE or CodexKeys.FCS_FILE_CSV in patt.pattern: - df = ( - readfcs.ReadFCS(path / path_files[0]).data - if CodexKeys.FCS_FILE in path_files[0] - else pd.read_csv(path_files[0], header=0, index_col=None) + layout = discover_legacy(path, fcs) + selected = select_regions(list(layout.images), regions) + experiment = read_json_object(layout.experiment_json) if layout.experiment_json is not None else None + names_txt = _read_channel_names_txt(layout.channel_names_txt) if layout.channel_names_txt is not None else None + + scans: list[AkoyaScan] = [] + arrays: dict[str, da.Array] = {} + for region in selected: + files = layout.images[region] + if _is_qptiff(files[0]): + scans.append(read_qptiff_metadata(files[0], region=region)) + arrays[region] = read_qptiff_image(files[0]) + continue + arrays[region], channels = read_legacy_image(files, layout.channel_names_txt, imread_kwargs) + scans.append( + AkoyaScan( + path=files[0] if len(files) == 1 else files[0].parent, + region=region, + generation="legacy_processor", + software=None, + slide_id=None, + pixel_size_um=_pixel_size_um(experiment), + nominal_pixel_size_um=None, + stage_position_um=None, + channels=channels, + rgb=False, + raw={ + "files": [str(file) for file in files], + "experiment_json": experiment, + "channel_names_txt": names_txt, + }, + ) ) - else: - raise ValueError("Cannot determine data set. Expecting a file with format .fcs or .csv") - - adata = _codex_df_to_anndata(df) - assert isinstance(adata.obs, pd.DataFrame) - - xy = adata.obsm[CodexKeys.SPATIAL_KEY] - shapes = ShapesModel.parse(xy, geometry=0, radius=1, index=adata.obs[CodexKeys.INSTANCE_KEY]) - region = adata.obs[CodexKeys.REGION_KEY].unique()[0] - adata.obs[CodexKeys.REGION_KEY] = adata.obs[CodexKeys.REGION_KEY].astype("category") - table = TableModel.parse(adata, region=region, region_key=CodexKeys.REGION_KEY, instance_key=CodexKeys.INSTANCE_KEY) - - im_patt = re.compile(".*.tif") - path_files = [i for i in os.listdir(path) if im_patt.match(i)] - if path_files and CodexKeys.IMAGE_TIF in path_files[0]: - image = imread(path_files[0], **imread_kwargs) - images = { - "images": Image2DModel.parse( - image, - scale_factors=[2, 2], - rgb=None, + + names = {scan.region: apply_channel_names(scan, channel_names) for scan in scans} + transformations = region_transformations(scans) + images = { + f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}": build_image_element( + arrays[scan.region], scan, names[scan.region], transformations[scan.region], image_models_kwargs + ) + for scan in scans + } + sdata = SpatialData(images=images) + sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value] = scans_to_attrs(scans, names) + return _set_reader_metadata(sdata, "codex") + + +def discover_legacy(path: Path, fcs: bool) -> LegacyLayout: + """Find the images, cell tables and sidecar files of a CODEX dataset. + + Parameters + ---------- + path + A CODEX output directory, searched recursively in sorted order, or a single image or table file. + fcs + Whether to prefer ``.fcs`` over ``.csv`` tables of the same region. + + Returns + ------- + The discovered files. + """ + if not path.exists(): + raise FileNotFoundError(f"{path} does not exist.") + files = [path] if path.is_file() else _walk(path) + images = _discover_images(files) + if not images and path.is_dir(): + # the original (2019) layout: one stitched `.tif` next to its table, without a region in the file name + images = {file.stem: [file] for file in files if file.parent == path and _is_tiff(file)} + tables = _discover_tables(files, fcs) + if not images and not tables: + raise FileNotFoundError(f"No CODEX images or cell tables found in {path}. Expected: {'; '.join(_PATTERNS)}.") + return LegacyLayout( + images=dict(sorted(images.items())), + tables=dict(sorted(tables.items())), + channel_names_txt=_find_sidecar(path, files, CodexKeys.CHANNEL_NAMES_FILE.value), + experiment_json=_find_sidecar(path, files, CodexKeys.EXPERIMENT_FILE.value), + ) + + +def read_legacy_image( + files: list[Path], channel_names_txt: Path | None, imread_kwargs: Mapping[str, Any] +) -> tuple[da.Array, list[AkoyaChannel]]: + """Lazily read the image of one legacy CODEX region. + + Parameters + ---------- + files + The per-marker images of the region, or its single hyperstack, as returned by :func:`discover_legacy`. + channel_names_txt + Optional ``channelNames.txt`` naming the hyperstack channels when the file has no ImageJ ``Labels``. + imread_kwargs + Keyword arguments passed to :func:`dask_image.imread.imread` for files :mod:`tifffile` cannot read lazily. + + Returns + ------- + The ``(c, y, x)`` image and its channels. + """ + markers = [(match, file) for file in files if (match := _PER_MARKER.search(file.name)) is not None] + if len(markers) == len(files): + return _read_per_marker(markers, imread_kwargs) + if len(files) != 1: + raise ValueError(f"Expected per-marker images or a single hyperstack, got {[str(f) for f in files]}.") + + image, leading_shape, labels = _read_tiff(files[0], imread_kwargs) + n_planes = image.shape[0] + # the last non-spatial axis holds the channels of a cycle; ImageJ axis labels are unreliable (cycles as Z) + n_channels = leading_shape[-1] if leading_shape else 1 + names: list[str] | None = labels if labels is not None and len(labels) == n_planes else None + if names is None and channel_names_txt is not None: + names = _read_channel_names_txt(channel_names_txt) + if len(names) != n_planes: + warnings.warn( + f"{channel_names_txt} lists {len(names)} channel names, but {files[0]} has {n_planes} channels; " + "using 'cycCCC_chHHH' names instead.", + UserWarning, + stacklevel=2, ) - } - sdata = SpatialData(images=images, shapes={str(region): shapes}, tables={"table": table}) + names = None + channels = [] + for i in range(n_planes): + cycle, channel = i // n_channels + 1, i % n_channels + 1 + name = names[i] if names is not None else f"cyc{cycle:03d}_ch{channel:03d}" + channels.append(AkoyaChannel(name=name, cycle=cycle, channel=channel)) + return image, channels + + +def _read_per_marker( + markers: list[tuple[re.Match[str], Path]], imread_kwargs: Mapping[str, Any] +) -> tuple[da.Array, list[AkoyaChannel]]: + parsed = sorted((int(match.group(2)), int(match.group(3)), match.group(4), file) for match, file in markers) + + planes = [] + for _, _, _, file in parsed: + plane, _, _ = _read_tiff(file, imread_kwargs) + if plane.shape[0] != 1: + raise ValueError(f"{file} holds {plane.shape[0]} planes; expected a single 2-D marker image.") + planes.append(plane[0]) + if len({(plane.shape, plane.dtype) for plane in planes}) > 1: + found = {file.name: (plane.shape, str(plane.dtype)) for (*_, file), plane in zip(parsed, planes, strict=True)} + raise ValueError(f"The per-marker images of one region must share shape and dtype, got {found}.") + channels = [ + AkoyaChannel(name=marker, cycle=cycle, channel=channel, extra={"file": file.name}) + for cycle, channel, marker, file in parsed + ] + return da.stack(planes), channels + + +def _read_tiff(path: Path, imread_kwargs: Mapping[str, Any]) -> tuple[da.Array, tuple[int, ...], list[str] | None]: + """Lazily read a tiff as ``(c, y, x)``, all non-spatial axes flattened into ``c``. + + Returns the image, the shape of the flattened axes and the ImageJ ``Labels``, if any. The ``TiffFile`` stays + open for as long as the array (its zarr store holds the file handle). + """ + tf = tifffile.TiffFile(path) + series = tf.series[0] + try: + store = series.aszarr() + except ValueError as e: # e.g. a compression tifffile cannot decode chunk-wise + tf.close() + logger.info("Reading %s with dask_image.imread because tifffile cannot read it lazily: %s", path, e) + image = imread(str(path), **imread_kwargs) + axes = "Q" * (image.ndim - 2) + "YX" + labels = None else: - logger.warning("Cannot find .tif file. Will build spatialdata with shapes and table only.") - sdata = SpatialData(shapes={str(region): shapes}, tables={"table": table}) + image = da.from_zarr(zarr.open(store, mode="r")) + axes = series.axes + metadata = tf.imagej_metadata or {} + labels = list(metadata["Labels"]) if isinstance(metadata.get("Labels"), list | tuple) else None + image = da.moveaxis(image, [axes.index("Y"), axes.index("X")], [-2, -1]) + leading_shape = tuple(image.shape[:-2]) + return image.reshape((math.prod(leading_shape), *image.shape[-2:])), leading_shape, labels - return _set_reader_metadata(sdata, "codex") + +def _walk(root: Path) -> list[Path]: + """All files below ``root`` in sorted order, skipping raw acquisition and intermediate processing folders.""" + files: list[Path] = [] + for dirpath, dirnames, filenames in os.walk(root): + directory = Path(dirpath) + depth = len(directory.relative_to(root).parts) + has_stitched = any(name.casefold() == "stitched" for name in dirnames) + dirnames[:] = sorted( + name for name in dirnames if depth < _MAX_DEPTH and not _is_ignored_dir(name, has_stitched) + ) + files.extend(directory / name for name in sorted(filenames)) + return files + + +def _is_ignored_dir(name: str, has_stitched: bool) -> bool: + folded = name.casefold() + return ( + folded == "tiles" + or (folded == "bestfocus" and has_stitched) + or _RAW_CYCLE_DIR.fullmatch(name) is not None + or name.startswith(".") + ) + + +def _discover_images(files: Sequence[Path]) -> dict[str, list[Path]]: + per_marker: dict[str, dict[tuple[int, int], Path]] = {} + stacks: dict[str, list[Path]] = {} + qptiffs: dict[str, list[Path]] = {} + for file in files: + region = _region_id(file.name) + if _is_qptiff(file): + qptiffs.setdefault(region or file.stem, []).append(file) + elif _is_tiff(file) and region is not None: + if (match := _PER_MARKER.search(file.name)) is None: + stacks.setdefault(region, []).append(file) + continue + markers = per_marker.setdefault(region, {}) + key = (int(match.group(2)), int(match.group(3))) + if key in markers: + raise ValueError( + f"Found several images for cycle {key[0]}, channel {key[1]} of region '{region}': " + f"{markers[key]} and {file}. Pass the path of a directory that contains each region once." + ) + markers[key] = file + + images: dict[str, list[Path]] = {} + # the Processor's stitched outputs come first: cell tables refer to their pixel space + for region in sorted(per_marker.keys() | stacks.keys() | qptiffs.keys()): + if region in per_marker: + images[region] = [per_marker[region][key] for key in sorted(per_marker[region])] + elif region in stacks: + images[region] = [_single(region, _preferred_stacks(stacks[region]))] + else: + images[region] = [_single(region, qptiffs[region])] + return images + + +def _preferred_stacks(files: list[Path]) -> list[Path]: + for pattern in (_MONTAGE, _TILE_STACK): + matching = [file for file in files if pattern.search(file.name)] + if matching: + return matching + return files + + +def _single(region: str, files: list[Path]) -> Path: + if len(files) > 1: + raise ValueError( + f"Found several images for region '{region}': {[str(f) for f in files]}. Pass the path of a single " + "image or of a directory that contains each region once." + ) + return files[0] + + +def _discover_tables(files: Sequence[Path], fcs: bool) -> dict[str, Path]: + preferred = CodexKeys.FCS_FILE.value if fcs else CodexKeys.FCS_FILE_CSV.value + candidates: dict[str, list[Path]] = {} + for file in files: + if file.suffix.lower() in _TABLE_SUFFIXES: + candidates.setdefault(_region_id(file.name) or "*", []).append(file) + + def rank(file: Path) -> tuple[bool, bool, int, str]: + # the preferred format first, then compensated over uncompensated, then the shallowest + uncompensated = any("uncompensated" in part.casefold() for part in file.parts) + return file.suffix.lower() != preferred, uncompensated, len(file.parts), str(file) + + return {region: min(paths, key=rank) for region, paths in candidates.items()} + + +def _find_sidecar(path: Path, files: Sequence[Path], name: str) -> Path | None: + """The shallowest file called ``name`` in ``files``, else in the nearest ancestor of ``path`` holding one.""" + matches = sorted((file for file in files if file.name == name), key=lambda file: (len(file.parts), str(file))) + if matches: + return matches[0] + start = path.parent if path.is_file() else path + for directory in [start, *start.parents][: _SIDECAR_ANCESTORS + 1]: + if (directory / name).is_file(): + return directory / name + return None + + +def _read_channel_names_txt(path: Path) -> list[str]: + return [line.strip() for line in path.read_text().splitlines() if line.strip()] + + +def _pixel_size_um(experiment: Mapping[str, Any] | None) -> float | None: + """Pixel size in µm from the ``experiment.json`` of the CODEX Processor (nm), ``None`` when it is missing.""" + for key in ("xyResolution", "per_pixel_XY_resolution", "Resolution_nm"): + value = (experiment or {}).get(key) + if isinstance(value, int | float) and not isinstance(value, bool) and value > 0: + return value / 1000 + return None + + +def _region_id(name: str) -> str | None: + match = _REGION.search(name) + return f"reg{int(match.group(1)):03d}" if match else None + + +def _is_qptiff(path: Path) -> bool: + return path.suffix.lower() == CodexKeys.QPTIFF.value -def _codex_df_to_anndata(df: pd.DataFrame) -> ad.AnnData: - """Convert a codex formatted .fcs dataframe or .csv file to anndata.""" - adata = ad.AnnData(df.filter(regex="cyc.*")) - adata.obs = df[df.columns.drop(list(df.filter(regex="cyc.*")))] - adata.obsm[CodexKeys.SPATIAL_KEY] = df[["x", "y"]].values - adata.var_names_make_unique() - return adata +def _is_tiff(path: Path) -> bool: + return path.suffix.lower() in _TIFF_SUFFIXES diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 6a8f6f5d..2eb7893f 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -1,7 +1,6 @@ from __future__ import annotations import csv -import json import re import warnings from pathlib import Path @@ -16,6 +15,7 @@ from spatialdata_io.readers._utils._akoya import ( apply_channel_names, build_image_element, + read_json_object, region_transformations, scans_to_attrs, select_regions, @@ -204,7 +204,7 @@ def _read_sidecars(scan: AkoyaScan) -> None: """Store the ``.xpd`` and ``*channels.csv`` sidecars of ``scan`` in ``scan.raw`` and check their channel count.""" base = _base_name(scan.path) xpd = _find_sidecar(scan.path, base, PhenocyclerKeys.XPD_SUFFIX) - protocol = _read_xpd(xpd) if xpd is not None else None + protocol = read_json_object(xpd) if xpd is not None else None if xpd is not None and protocol is not None: scan.raw["xpd"] = protocol # an .xpd has the layout of the Fusion scan profile's `experimentDescription` @@ -222,18 +222,6 @@ def _read_sidecars(scan: AkoyaScan) -> None: _check_channel_count(scan, channels_csv, markers) -def _read_xpd(path: Path) -> dict[str, Any] | None: - try: - protocol = json.loads(path.read_text()) - except json.JSONDecodeError as e: - warnings.warn(f"Ignoring {path}: it is not valid JSON ({e}).", UserWarning, stacklevel=2) - return None - if not isinstance(protocol, dict): - warnings.warn(f"Ignoring {path}: it is not a JSON object.", UserWarning, stacklevel=2) - return None - return protocol - - def _find_sidecar(qptiff: Path, base: str, suffix: str) -> Path | None: """First file ending in ``suffix`` whose name starts with ``base``, next to the scan or in HuBMAP's processed dir.""" pattern = re.compile(rf"{re.escape(base)}(?![0-9A-Za-z]).*{re.escape(suffix)}$") diff --git a/tests/test_codex.py b/tests/test_codex.py new file mode 100644 index 00000000..c7c42f71 --- /dev/null +++ b/tests/test_codex.py @@ -0,0 +1,167 @@ +"""Tests for the CODEX reader on synthetic CODEX Processor/Uploader trees and CIM qptiff files.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import numpy as np +import pytest +import tifffile +from spatialdata.models import get_channel_names + +from spatialdata_io import codex +from spatialdata_io.readers._utils._akoya import dedupe_channel_names +from spatialdata_io.readers.codex import discover_legacy +from tests._akoya_data import cim_experiment_v4_xml, write_legacy_processor_tree, write_qptiff + +if TYPE_CHECKING: + from pathlib import Path + +_FAST = {"scale_factors": [2]} + + +def _level0(sdata, name): # type: ignore[no-untyped-def] + return sdata.images[name]["scale0"]["image"] + + +def test_codex_per_marker_images(tmp_path: Path) -> None: + data = write_legacy_processor_tree(tmp_path, regions=2, layout="per_marker") + + sdata = codex(tmp_path, image_models_kwargs=_FAST) + + assert sorted(sdata.images) == ["reg001_image", "reg002_image"] + for region in ("reg001", "reg002"): + image = sdata.images[f"{region}_image"] + assert get_channel_names(image) == dedupe_channel_names(data["markers"]) + np.testing.assert_array_equal(_level0(sdata, f"{region}_image").values, data["images"][region]) + attrs = sdata.attrs["akoya"]["regions"][region] + assert attrs["generation"] == "legacy_processor" + assert attrs["pixel_size_um"] is None + assert [(c["cycle"], c["channel"]) for c in attrs["channels"][:5]] == [(1, 1), (1, 2), (1, 3), (1, 4), (2, 1)] + assert sdata.attrs["spatialdata_io_reader"] == "codex" + + +def test_codex_hyperstack_channel_names_txt(tmp_path: Path) -> None: + data = write_legacy_processor_tree(tmp_path, cycles=2, channels=4, layout="hyperstack") + + sdata = codex(str(tmp_path), image_models_kwargs=_FAST) + + assert list(sdata.images) == ["reg001_image"] + image = _level0(sdata, "reg001_image") + assert image.shape[0] == 8 + assert get_channel_names(sdata.images["reg001_image"]) == dedupe_channel_names(data["markers"]) + np.testing.assert_array_equal(image.values, data["images"]["reg001"].reshape(8, *image.shape[1:])) + + +def test_codex_hyperstack_bad_channel_names_warns(tmp_path: Path) -> None: + data = write_legacy_processor_tree(tmp_path, cycles=2, channels=4, layout="hyperstack") + (tmp_path / "channelNames.txt").write_text("\n".join(data["markers"][:7]) + "\n") + + with pytest.warns(UserWarning, match="channelNames.txt"): + sdata = codex(tmp_path, image_models_kwargs=_FAST) + + names = get_channel_names(sdata.images["reg001_image"]) + assert names[:5] == ["cyc001_ch001", "cyc001_ch002", "cyc001_ch003", "cyc001_ch004", "cyc002_ch001"] + assert len(names) == 8 + + +def test_codex_hyperstack_imagej_labels_win(tmp_path: Path) -> None: + labels = [f"L{i}" for i in range(6)] + tifffile.imwrite( + tmp_path / "reg001_X01_Y01_Z01.tif", + np.zeros((3, 2, 32, 48), dtype=np.uint16), + imagej=True, + metadata={"axes": "ZCYX", "Labels": labels}, + ) + (tmp_path / "channelNames.txt").write_text("\n".join(f"other{i}" for i in range(6))) + + sdata = codex(tmp_path, image_models_kwargs=_FAST) + + assert get_channel_names(sdata.images["reg001_image"]) == labels + + +def test_codex_cim_qptiff(tmp_path: Path) -> None: + path = tmp_path / "experiment_reg001.qptiff" + data = write_qptiff( + path, + biomarkers=["DAPI", "CD3", "DAPI", "CD20"], + software="Codex Instrument Manager 1.30.0.12", + scan_profile=cim_experiment_v4_xml([["DAPI", "CD3"], ["DAPI", "CD20"]]), + label_page=False, + ) + + for source in (tmp_path, path): + sdata = codex(source, image_models_kwargs=_FAST) + assert list(sdata.images) == ["reg001_image"] + assert get_channel_names(sdata.images["reg001_image"]) == ["DAPI", "CD3", "DAPI (2)", "CD20"] + np.testing.assert_array_equal(_level0(sdata, "reg001_image").values, data) + assert sdata.attrs["akoya"]["regions"]["reg001"]["generation"] == "cim_qptiff" + + +def test_codex_old_flat_layout_image(tmp_path: Path) -> None: + data = write_legacy_processor_tree(tmp_path, layout="old_flat") + + sdata = codex(tmp_path, image_models_kwargs=_FAST) + + assert list(sdata.images) == ["reg001_image"] + # only the first plane of the stack is written to disk + np.testing.assert_array_equal(_level0(sdata, "reg001_image").values, data["images"]["reg001"][:1]) + + +def test_codex_empty_dir_raises(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError, match="reg"): + codex(str(tmp_path)) + + +def test_codex_ignores_raw_tiles(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, layout="per_marker") + expected = sorted(codex(tmp_path, image_models_kwargs=_FAST).images) + processed = tmp_path / "processed_2020-01-01" + plane = np.zeros((16, 16), dtype=np.uint16) + for raw in ( + tmp_path / "cyc001_reg001" / "1_00001_Z001_CH1.tif", + tmp_path / "cyc001_reg002" / "reg002_X01_Y01_Z01.tif", + processed / "tiles" / "reg002_X01_Y01" / "reg002_X01_Y01_t001_z001_c001.tif", + processed / "bestFocus" / "reg002_X01_Y01_Z08.tif", + ): + raw.parent.mkdir(parents=True, exist_ok=True) + tifffile.imwrite(raw, plane) + + assert sorted(codex(tmp_path, image_models_kwargs=_FAST).images) == expected + + +def test_codex_regions_channel_names_and_pixel_size(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, regions=2, cycles=1, channels=2, layout="per_marker") + (tmp_path / "experiment.json").write_text(json.dumps({"xyResolution": 377.5, "numChannels": 2})) + + sdata = codex(tmp_path, regions=["reg002"], channel_names=["a", "b"], image_models_kwargs=_FAST) + + assert list(sdata.images) == ["reg002_image"] + assert get_channel_names(sdata.images["reg002_image"]) == ["a", "b"] + region = sdata.attrs["akoya"]["regions"]["reg002"] + assert region["pixel_size_um"] == pytest.approx(0.3775) + assert region["raw"]["experiment_json"]["numChannels"] == 2 + with pytest.raises(ValueError, match="reg003"): + codex(tmp_path, regions=["reg003"]) + with pytest.raises(ValueError, match="channel_names"): + codex(tmp_path, channel_names=["a"]) + + +def test_discover_legacy_tables_prefer_requested_format(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, regions=2, layout="per_marker", fcs=True, csv=True) + + assert {region: path.suffix for region, path in discover_legacy(tmp_path, fcs=True).tables.items()} == { + "reg001": ".fcs", + "reg002": ".fcs", + } + assert {region: path.suffix for region, path in discover_legacy(tmp_path, fcs=False).tables.items()} == { + "reg001": ".csv", + "reg002": ".csv", + } + + +def test_codex_not_implemented_cells_table(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, layout="per_marker") + with pytest.raises(NotImplementedError): + codex(tmp_path, cells_table=tmp_path / "x.csv") From 042695fbf3e2f7919b68ee856fb118c92a76b7d9 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 13:23:30 +0200 Subject: [PATCH 007/107] fix: tile uncompressed CODEX tiffs and harden legacy discovery Memory-map uncompressed contiguous tiffs per chunk instead of reading one zarr chunk per plane, accept a single region-less .tif, keep only region-less CSVs that look like cell tables (error when ambiguous) and warn when imread_kwargs is not used. Co-Authored-By: Claude Opus 5.5 --- src/spatialdata_io/readers/codex.py | 171 ++++++++++++++++++++++------ tests/test_codex.py | 49 +++++++- 2 files changed, 185 insertions(+), 35 deletions(-) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 2916c65f..02b6d09f 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -4,14 +4,17 @@ import os import re import warnings +from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Any import dask.array as da +import numpy as np import tifffile import zarr +from dask.base import tokenize from dask_image.imread import imread from spatialdata import SpatialData from spatialdata._logging import logger @@ -27,10 +30,10 @@ select_regions, ) from spatialdata_io.readers._utils._qptiff import AkoyaChannel, AkoyaScan, read_qptiff_image, read_qptiff_metadata -from spatialdata_io.readers._utils._utils import _set_reader_metadata +from spatialdata_io.readers._utils._utils import _initialize_raster_models_kwargs, _set_reader_metadata if TYPE_CHECKING: - from collections.abc import Mapping, Sequence + from collections.abc import Sequence __all__ = ["LegacyLayout", "codex", "discover_legacy", "read_legacy_image"] @@ -41,6 +44,8 @@ # raw Keyence acquisition folders, e.g. `cyc001_reg001/1_00001_Z001_CH1.tif` _RAW_CYCLE_DIR = re.compile(r"cyc\d+_reg\d+", re.IGNORECASE) _TIFF_SUFFIXES = (".tif", ".tiff") +# marker columns of CODEX Processor tables: `cyc001_ch001` or `cyc001_ch001:DAPI1` +_MARKER_COLUMN = re.compile(r"cyc\d+_ch\d+", re.IGNORECASE) _TABLE_SUFFIXES = (CodexKeys.FCS_FILE.value, CodexKeys.FCS_FILE_CSV.value) # deep enough for `/processed_/segm/segm-1/fcs/compensated/
` _MAX_DEPTH = 8 @@ -155,7 +160,9 @@ def codex( scans.append(read_qptiff_metadata(files[0], region=region)) arrays[region] = read_qptiff_image(files[0]) continue - arrays[region], channels = read_legacy_image(files, layout.channel_names_txt, imread_kwargs) + arrays[region], channels = read_legacy_image( + files, layout.channel_names_txt, imread_kwargs, image_models_kwargs=image_models_kwargs + ) scans.append( AkoyaScan( path=files[0] if len(files) == 1 else files[0].parent, @@ -207,9 +214,10 @@ def discover_legacy(path: Path, fcs: bool) -> LegacyLayout: raise FileNotFoundError(f"{path} does not exist.") files = [path] if path.is_file() else _walk(path) images = _discover_images(files) - if not images and path.is_dir(): + if not images: # the original (2019) layout: one stitched `.tif` next to its table, without a region in the file name - images = {file.stem: [file] for file in files if file.parent == path and _is_tiff(file)} + top_level = files if path.is_file() else [file for file in files if file.parent == path] + images = {file.stem: [file] for file in top_level if _is_tiff(file)} tables = _discover_tables(files, fcs) if not images and not tables: raise FileNotFoundError(f"No CODEX images or cell tables found in {path}. Expected: {'; '.join(_PATTERNS)}.") @@ -222,7 +230,10 @@ def discover_legacy(path: Path, fcs: bool) -> LegacyLayout: def read_legacy_image( - files: list[Path], channel_names_txt: Path | None, imread_kwargs: Mapping[str, Any] + files: list[Path], + channel_names_txt: Path | None, + imread_kwargs: Mapping[str, Any], + image_models_kwargs: Mapping[str, Any] = MappingProxyType({}), ) -> tuple[da.Array, list[AkoyaChannel]]: """Lazily read the image of one legacy CODEX region. @@ -234,18 +245,25 @@ def read_legacy_image( Optional ``channelNames.txt`` naming the hyperstack channels when the file has no ImageJ ``Labels``. imread_kwargs Keyword arguments passed to :func:`dask_image.imread.imread` for files :mod:`tifffile` cannot read lazily. + image_models_kwargs + Keyword arguments for :class:`spatialdata.models.Image2DModel`; their ``chunks`` set the spatial chunks of + uncompressed files, which are memory-mapped. Returns ------- The ``(c, y, x)`` image and its channels. """ + chunks = _spatial_chunks(image_models_kwargs) markers = [(match, file) for file in files if (match := _PER_MARKER.search(file.name)) is not None] if len(markers) == len(files): - return _read_per_marker(markers, imread_kwargs) + image, channels, read_by_tifffile = _read_per_marker(markers, imread_kwargs, chunks) + _warn_unused_imread_kwargs(imread_kwargs, read_by_tifffile) + return image, channels if len(files) != 1: raise ValueError(f"Expected per-marker images or a single hyperstack, got {[str(f) for f in files]}.") - image, leading_shape, labels = _read_tiff(files[0], imread_kwargs) + image, leading_shape, labels, read_by_tifffile = _read_tiff(files[0], imread_kwargs, chunks) + _warn_unused_imread_kwargs(imread_kwargs, read_by_tifffile) n_planes = image.shape[0] # the last non-spatial axis holds the channels of a cycle; ImageJ axis labels are unreliable (cycles as Z) n_channels = leading_shape[-1] if leading_shape else 1 @@ -269,13 +287,15 @@ def read_legacy_image( def _read_per_marker( - markers: list[tuple[re.Match[str], Path]], imread_kwargs: Mapping[str, Any] -) -> tuple[da.Array, list[AkoyaChannel]]: + markers: list[tuple[re.Match[str], Path]], imread_kwargs: Mapping[str, Any], chunks: tuple[int, int] +) -> tuple[da.Array, list[AkoyaChannel], bool]: parsed = sorted((int(match.group(2)), int(match.group(3)), match.group(4), file) for match, file in markers) planes = [] + read_by_tifffile = False for _, _, _, file in parsed: - plane, _, _ = _read_tiff(file, imread_kwargs) + plane, _, _, lazy = _read_tiff(file, imread_kwargs, chunks) + read_by_tifffile |= lazy if plane.shape[0] != 1: raise ValueError(f"{file} holds {plane.shape[0]} planes; expected a single 2-D marker image.") planes.append(plane[0]) @@ -286,33 +306,88 @@ def _read_per_marker( AkoyaChannel(name=marker, cycle=cycle, channel=channel, extra={"file": file.name}) for cycle, channel, marker, file in parsed ] - return da.stack(planes), channels + return da.stack(planes), channels, read_by_tifffile -def _read_tiff(path: Path, imread_kwargs: Mapping[str, Any]) -> tuple[da.Array, tuple[int, ...], list[str] | None]: +def _read_tiff( + path: Path, imread_kwargs: Mapping[str, Any], chunks: tuple[int, int] +) -> tuple[da.Array, tuple[int, ...], list[str] | None, bool]: """Lazily read a tiff as ``(c, y, x)``, all non-spatial axes flattened into ``c``. - Returns the image, the shape of the flattened axes and the ImageJ ``Labels``, if any. The ``TiffFile`` stays - open for as long as the array (its zarr store holds the file handle). + Uncompressed contiguous data (e.g. ImageJ hyperstacks and per-marker images, often a single strip per plane) + is memory-mapped and split into ``chunks``, so a chunk never needs a whole plane in memory; other layouts keep + their native tiles via zarr. Returns the image, the shape of the flattened axes, the ImageJ ``Labels`` (if + any) and whether tifffile, rather than the :func:`dask_image.imread.imread` fallback, read the file. The + file handle stays open for as long as the array needs it. """ tf = tifffile.TiffFile(path) series = tf.series[0] - try: - store = series.aszarr() - except ValueError as e: # e.g. a compression tifffile cannot decode chunk-wise + axes = series.axes + metadata = tf.imagej_metadata or {} + labels = list(metadata["Labels"]) if isinstance(metadata.get("Labels"), list | tuple) else None + read_by_tifffile = True + if series.dataoffset is not None: tf.close() - logger.info("Reading %s with dask_image.imread because tifffile cannot read it lazily: %s", path, e) - image = imread(str(path), **imread_kwargs) - axes = "Q" * (image.ndim - 2) + "YX" - labels = None + source = _ContiguousTiff(path, series.dataoffset, series.shape, series.dtype) + tiles = dict.fromkeys(range(source.ndim), 1) | {axes.index("Y"): chunks[0], axes.index("X"): chunks[1]} + image = da.from_array(source, chunks=tuple(tiles[i] for i in range(source.ndim)), name=source.name) else: - image = da.from_zarr(zarr.open(store, mode="r")) - axes = series.axes - metadata = tf.imagej_metadata or {} - labels = list(metadata["Labels"]) if isinstance(metadata.get("Labels"), list | tuple) else None + try: + image = da.from_zarr(zarr.open(series.aszarr(), mode="r")) + except ValueError as e: # e.g. a compression tifffile cannot decode chunk-wise + tf.close() + logger.info("Reading %s with dask_image.imread because tifffile cannot read it lazily: %s", path, e) + image = imread(str(path), **imread_kwargs) + axes = "Q" * (image.ndim - 2) + "YX" + labels = None + read_by_tifffile = False image = da.moveaxis(image, [axes.index("Y"), axes.index("X")], [-2, -1]) leading_shape = tuple(image.shape[:-2]) - return image.reshape((math.prod(leading_shape), *image.shape[-2:])), leading_shape, labels + return image.reshape((math.prod(leading_shape), *image.shape[-2:])), leading_shape, labels, read_by_tifffile + + +class _ContiguousTiff: + """Array-like view of uncompressed contiguous tiff data, memory-mapped anew for every chunk. + + Handing an ``np.memmap`` to :func:`dask.array.from_array` would read the whole file (dask hashes and slices + numpy arrays eagerly), so dask only gets this lightweight object and a name derived from the file. + """ + + def __init__(self, path: Path, offset: int, shape: tuple[int, ...], dtype: np.dtype[Any]) -> None: + self.path = path + self.offset = offset + self.shape = shape + self.ndim = len(shape) + self._file_dtype = dtype + self.dtype = dtype.newbyteorder("=") + self.name = f"codex-tiff-{tokenize(str(path), path.stat().st_mtime_ns, offset, shape, str(dtype))}" + + def __getitem__(self, key: Any) -> np.ndarray: + data = np.memmap(self.path, dtype=self._file_dtype, mode="r", offset=self.offset, shape=self.shape) + return np.asarray(data[key], dtype=self.dtype) + + +def _spatial_chunks(image_models_kwargs: Mapping[str, Any]) -> tuple[int, int]: + """The ``(y, x)`` chunk size requested in ``image_models_kwargs``, with the reader default when unspecified.""" + default, _ = _initialize_raster_models_kwargs({}, {}) + chunks = _initialize_raster_models_kwargs(image_models_kwargs, {})[0]["chunks"] + if isinstance(chunks, int): + return chunks, chunks + if isinstance(chunks, Mapping) and isinstance(chunks.get("y"), int) and isinstance(chunks.get("x"), int): + return chunks["y"], chunks["x"] + if isinstance(chunks, list | tuple) and len(chunks) >= 2 and all(isinstance(c, int) for c in chunks[-2:]): + return chunks[-2], chunks[-1] + return default["chunks"][-2], default["chunks"][-1] + + +def _warn_unused_imread_kwargs(imread_kwargs: Mapping[str, Any], read_by_tifffile: bool) -> None: + if imread_kwargs and read_by_tifffile: + warnings.warn( + "`imread_kwargs` is ignored for files read lazily with tifffile; it is only used by the " + "dask_image.imread fallback for files tifffile cannot read lazily.", + UserWarning, + stacklevel=3, + ) def _walk(root: Path) -> list[Path]: @@ -393,15 +468,43 @@ def _discover_tables(files: Sequence[Path], fcs: bool) -> dict[str, Path]: preferred = CodexKeys.FCS_FILE.value if fcs else CodexKeys.FCS_FILE_CSV.value candidates: dict[str, list[Path]] = {} for file in files: - if file.suffix.lower() in _TABLE_SUFFIXES: - candidates.setdefault(_region_id(file.name) or "*", []).append(file) + if file.suffix.lower() not in _TABLE_SUFFIXES: + continue + region = _region_id(file.name) + # without a region in its name, a CSV may be any sidecar (e.g. an antibody list) + if region is None and not _is_cell_table(file): + continue + candidates.setdefault(region or "*", []).append(file) + + def rank(file: Path) -> tuple[bool, bool]: + # the preferred format first, then compensated over uncompensated + return file.suffix.lower() != preferred, any("uncompensated" in part.casefold() for part in file.parts) + + tables = {} + for region, paths in candidates.items(): + best = min(rank(path) for path in paths) + paths = [path for path in paths if rank(path) == best] + if region == "*" and len(paths) > 1: + raise ValueError( + f"Found several cell tables without a region in their file name: {[str(p) for p in paths]}. " + "Pass the one to read as `cells_table=`." + ) + tables[region] = min(paths, key=lambda path: (len(path.parts), str(path))) + return tables - def rank(file: Path) -> tuple[bool, bool, int, str]: - # the preferred format first, then compensated over uncompensated, then the shallowest - uncompensated = any("uncompensated" in part.casefold() for part in file.parts) - return file.suffix.lower() != preferred, uncompensated, len(file.parts), str(file) - return {region: min(paths, key=rank) for region, paths in candidates.items()} +def _is_cell_table(path: Path) -> bool: + """Whether ``path`` is an FCS file or a CSV whose header has CODEX cell-table columns.""" + if path.suffix.lower() == CodexKeys.FCS_FILE.value: + return True + with path.open(encoding="utf-8-sig", errors="replace") as f: + header = f.readline() + columns = [column.strip().strip('"').casefold() for column in header.split(",")] + return ( + bool({"cell_id", "cellid"} & set(columns)) + or any(_MARKER_COLUMN.match(column) for column in columns) + or {"x", "y"} <= set(columns) + ) def _find_sidecar(path: Path, files: Sequence[Path], name: str) -> Path | None: diff --git a/tests/test_codex.py b/tests/test_codex.py index c7c42f71..d3c18717 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -12,7 +12,7 @@ from spatialdata_io import codex from spatialdata_io.readers._utils._akoya import dedupe_channel_names -from spatialdata_io.readers.codex import discover_legacy +from spatialdata_io.readers.codex import discover_legacy, read_legacy_image from tests._akoya_data import cim_experiment_v4_xml, write_legacy_processor_tree, write_qptiff if TYPE_CHECKING: @@ -165,3 +165,50 @@ def test_codex_not_implemented_cells_table(tmp_path: Path) -> None: write_legacy_processor_tree(tmp_path, layout="per_marker") with pytest.raises(NotImplementedError): codex(tmp_path, cells_table=tmp_path / "x.csv") + + +@pytest.mark.parametrize("imagej", [False, True]) +def test_read_legacy_image_tiles_uncompressed_planes(tmp_path: Path, imagej: bool) -> None: + # uncompressed single-strip planes are one zarr chunk each; they must be memory-mapped and tiled instead + stack = np.random.default_rng(0).integers(0, 2**16, (2, 100, 130), dtype=np.uint16) + files = [tmp_path / f"reg001_cyc001_ch{i + 1:03d}_M{i}.tif" for i in range(2)] + for file, plane in zip(files, stack, strict=True): + tifffile.imwrite(file, plane, imagej=imagej, rowsperstrip=plane.shape[0]) + hyperstack = tmp_path / "reg002_montage.tif" + tifffile.imwrite(hyperstack, stack[None], imagej=True, metadata={"axes": "TCYX"}) + + for source in (files, [hyperstack]): + image, _ = read_legacy_image(source, None, {}, image_models_kwargs={"chunks": (1, 32, 64)}) + assert max(image.chunks[-2]) <= 32 + assert max(image.chunks[-1]) <= 64 + np.testing.assert_array_equal(image.compute(), stack) + + +def test_codex_single_tif_without_region(tmp_path: Path) -> None: + path = tmp_path / "Kidney.tif" + tifffile.imwrite(path, np.zeros((3, 32, 48), dtype=np.uint16), imagej=True, metadata={"axes": "CYX"}) + + sdata = codex(path, image_models_kwargs=_FAST) + + assert list(sdata.images) == ["Kidney_image"] + assert _level0(sdata, "Kidney_image").shape == (3, 32, 48) + + +def test_discover_legacy_regionless_table_ignores_sidecar_csv(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path / "exp", layout="per_marker", fcs=False, csv=False) + (tmp_path / "antibody_list.csv").write_text("marker,clone\nCD3,UCHT1\n") + cells = tmp_path / "exp" / "results" / "cells.csv" + cells.parent.mkdir() + cells.write_text("cell_id,x,y,cyc001_ch001:DAPI\n1,2.0,3.0,4.0\n") + + assert discover_legacy(tmp_path, fcs=True).tables == {"*": cells} + + (tmp_path / "other_cells.csv").write_text("x,y\n1.0,2.0\n") + with pytest.raises(ValueError, match="cells_table"): + discover_legacy(tmp_path, fcs=True) + + +def test_codex_imread_kwargs_warns_when_unused(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, layout="per_marker") + with pytest.warns(UserWarning, match="imread_kwargs"): + codex(tmp_path, imread_kwargs={"nframes": 1}, image_models_kwargs=_FAST) From 736c75f96be5d4247be48e4cb56ee0938f511e11 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 13:57:19 +0200 Subject: [PATCH 008/107] feat: read CODEX Processor FCS/CSV cell tables with marker names and all regions Marker names come from the FCS $PnS keywords or the Processor CSV header 'cycCCC_chHHH:'; tables of all regions are read relative to the given path and annotate one circles element per region. Co-Authored-By: Claude Opus 5.5 --- .../readers/_utils/_cell_tables.py | 271 ++++++++++++++++++ src/spatialdata_io/readers/codex.py | 52 +++- tests/readers/test_cell_tables.py | 142 +++++++++ tests/test_codex.py | 88 ++++++ 4 files changed, 546 insertions(+), 7 deletions(-) create mode 100644 src/spatialdata_io/readers/_utils/_cell_tables.py create mode 100644 tests/readers/test_cell_tables.py diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py new file mode 100644 index 00000000..c740e58e --- /dev/null +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -0,0 +1,271 @@ +"""Cell-table adapters shared by the Akoya readers: read a table into a `CellTable` and build the table elements.""" + +from __future__ import annotations + +import re +import warnings +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import anndata as ad +import numpy as np +import pandas as pd +import readfcs +from spatialdata.models import ShapesModel, TableModel +from spatialdata.transformations import Identity + +from spatialdata_io._constants._constants import CodexKeys, PhenocyclerKeys +from spatialdata_io.readers._utils._akoya import dedupe_channel_names + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping, Sequence + from pathlib import Path + + from geopandas import GeoDataFrame + from spatialdata.transformations import BaseTransformation + from xarray import DataArray, DataTree + +__all__ = ["CellTable", "build_cells", "read_codex_table"] + +# `cyc001_ch001` (FCS `$PnN`, plain CSV header) or the name part of the Processor CSV header `cyc001_ch001:DAPI1` +_MARKER_COLUMN = re.compile(r"cyc(\d+)_ch(\d+)", re.IGNORECASE) +# non-marker columns of CODEX Processor tables, for tables whose marker columns are named after the markers +_META_COLUMNS = frozenset( + {"cell_id", "region", "tile_num", "x", "y", "z", "x_tile", "y_tile", "size", "tsne_x", "tsne_y", "homogeneity"} +) + + +@dataclass +class CellTable: + """A cell table in reader-independent form. + + Attributes + ---------- + obs + The non-marker columns, indexed by the string instance ids. + X + The marker intensities, ``(n_cells, n_markers)`` float32. + var + Per-marker metadata indexed by the de-duplicated marker names: ``source_column`` and, for + ``cycCCC_chHHH`` columns, ``cycle`` and ``channel``. + region + The region id of every cell. + instance_id + The instance id of every cell (int64). + centroids_px + The ``(x, y)`` centroids in level-0 pixels of the region image. + radius_px + The radius of a circle with the cell's area, or `None` if the table has no area. + """ + + obs: pd.DataFrame + X: np.ndarray + var: pd.DataFrame + region: pd.Series + instance_id: pd.Series + centroids_px: np.ndarray + radius_px: np.ndarray | None + + def take(self, rows: np.ndarray) -> CellTable: + """The cells at the positional indices ``rows``.""" + return CellTable( + obs=self.obs.iloc[rows], + X=self.X[rows], + var=self.var, + region=self.region.iloc[rows], + instance_id=self.instance_id.iloc[rows], + centroids_px=self.centroids_px[rows], + radius_px=None if self.radius_px is None else self.radius_px[rows], + ) + + +def read_codex_table(path: Path, region: str | None = None) -> CellTable: + """Read a CODEX Processor ``.fcs`` or ``.csv`` cell table. + + Marker columns are the ``cycCCC_chHHH`` columns; their names come from the FCS ``$PnS`` keywords or from the + display part of the Processor CSV header ``cycCCC_chHHH:``. A table without such columns has one + marker column per numeric column that is not a known CODEX Processor column (``x``, ``size``, ...). + + Parameters + ---------- + path + The ``.fcs`` or ``.csv`` file. + region + The region id of all cells if the table has no ``region`` column. A numeric ``region`` column ``r`` is + mapped to the region id ``regRRR``. + + Returns + ------- + The cell table. + """ + if path.suffix.lower() == CodexKeys.FCS_FILE.value: + fcs = readfcs.ReadFCS(path) + df = fcs.data # positional columns, named by `$PnN` + names = list(fcs.channels["PnN"]) + labels = list(fcs.channels["PnS"]) if "PnS" in fcs.channels.columns else [""] * len(names) + else: + df = pd.read_csv(path, engine="pyarrow") + names = [str(column) for column in df.columns] + labels = [""] * len(names) + sources = names + # the Processor CSV header form `cyc001_ch001:DAPI1` (also `cell_id:cell_id`) + split = [name.partition(":") for name in names] + df.columns = pd.Index([name for name, _, _ in split]) + displays = [label or display or name for (name, _, display), label in zip(split, labels, strict=True)] + + marker_matches = [_MARKER_COLUMN.fullmatch(column) for column in df.columns] + if any(marker_matches): + is_marker = [match is not None for match in marker_matches] + else: + is_marker = [ + column.casefold() not in _META_COLUMNS and pd.api.types.is_numeric_dtype(df[column]) + for column in df.columns + ] + markers = [i for i, marker in enumerate(is_marker) if marker] + var = pd.DataFrame( + {"source_column": [sources[i] for i in markers]}, + index=pd.Index(dedupe_channel_names([displays[i] for i in markers])), + ) + if any(marker_matches): + var["cycle"] = [int(marker_matches[i].group(1)) for i in markers] # type: ignore[union-attr] + var["channel"] = [int(marker_matches[i].group(2)) for i in markers] # type: ignore[union-attr] + X = df.iloc[:, markers].to_numpy(dtype=np.float32) + obs = df.iloc[:, [i for i, marker in enumerate(is_marker) if not marker]] + + columns = {column.casefold(): column for column in obs.columns} + if "x" not in columns or "y" not in columns: + raise ValueError(f"{path} has no 'x' and 'y' centroid columns; found {list(obs.columns)}.") + centroids = obs[[columns["x"], columns["y"]]].to_numpy(dtype=np.float64) + radius = np.sqrt(obs[columns["size"]].to_numpy(dtype=np.float64) / np.pi) if "size" in columns else None + + instance_key = CodexKeys.INSTANCE_KEY.value + if instance_key in columns: + instance_id = obs[columns[instance_key]].astype(np.int64) + else: + # label images use 0 for the background, so instance ids start at 1 + instance_id = pd.Series(np.arange(1, len(obs) + 1, dtype=np.int64)) + instance_id.index = pd.Index(instance_id.astype(str)) + obs.index = instance_id.index + instance_id.name = instance_key + + region_key = CodexKeys.REGION_KEY.value + if region_key in columns: + regions = obs[columns[region_key]] + if pd.api.types.is_numeric_dtype(regions): + ids = {value: f"reg{int(value):03d}" for value in regions.unique()} + regions = regions.map(ids) + region_ids = regions.astype(str) + elif region is not None: + region_ids = pd.Series(region, index=obs.index) + else: + raise ValueError( + f"{path} has no '{region_key}' column and no region id in its file name, so its cells cannot be " + "assigned to a region." + ) + region_ids.name = region_key + return CellTable( + obs=obs, + X=X, + var=var, + region=region_ids, + instance_id=instance_id, + centroids_px=centroids, + radius_px=radius, + ) + + +def build_cells( + tables: Sequence[CellTable], + region_elements: Mapping[str, str], + labels: Mapping[str, DataArray | DataTree], + transformations: Mapping[str, BaseTransformation], +) -> tuple[dict[str, ad.AnnData], dict[str, GeoDataFrame]]: + """Build the table elements and, for regions without labels, circle shapes from cell tables. + + Parameters + ---------- + tables + The cell tables, each covering one or more regions. + region_elements + The element each region's cells annotate. Regions missing from it annotate ``_labels`` if they are + in ``labels``, else the circles ``_cells`` built here. + labels + The labels elements by region id; their regions get no circles. + transformations + The transformation to ``"global"`` of each region's image; missing regions get ``Identity``. + + Returns + ------- + The table elements, ``{"table": ...}`` if all tables have the same markers, else ``{"_table": ...}`` + per region, and the circle shapes by element name. The circles have the cells' area (radius 1 px without one). + """ + pieces = _split_by_region(tables) if len({tuple(table.var.index) for table in tables}) > 1 else None + + def element(region: str) -> str: + if region in region_elements: + return region_elements[region] + suffix = PhenocyclerKeys.LABELS_SUFFIX if region in labels else PhenocyclerKeys.CELLS_SUFFIX + return f"{region}{suffix}" + + shapes = {} + circle_regions = [] + for table in tables: + for region, rows in _region_rows(table).items(): + if region in labels: + continue + circle_regions.append(region) + radius = 1.0 if table.radius_px is None else table.radius_px[rows] + shapes[element(region)] = ShapesModel.parse( + table.centroids_px[rows], + geometry=0, + radius=radius, + index=pd.Index(table.instance_id.to_numpy()[rows]), + transformations={"global": transformations.get(region, Identity())}, + ) + if circle_regions: + warnings.warn( + f"No segmentation mask found for region(s) {sorted(circle_regions)}; cells are represented as circles.", + UserWarning, + stacklevel=3, + ) + + if pieces is None: + return {PhenocyclerKeys.TABLE_KEY.value: _to_anndata(tables, element)}, shapes + return {f"{region}_table": _to_anndata([piece], element) for region, piece in sorted(pieces.items())}, shapes + + +def _region_rows(table: CellTable) -> dict[str, np.ndarray]: + """The row indices of each region of ``table``, in order of first appearance.""" + codes, regions = pd.factorize(table.region) + return {str(region): np.flatnonzero(codes == i) for i, region in enumerate(regions)} + + +def _split_by_region(tables: Sequence[CellTable]) -> dict[str, CellTable]: + pieces = {} + for table in tables: + for region, rows in _region_rows(table).items(): + if region in pieces: + raise ValueError(f"Several cell tables contain region '{region}'.") + pieces[region] = table.take(rows) + return pieces + + +def _to_anndata(tables: Sequence[CellTable], element: Callable[[str], str]) -> ad.AnnData: + """Concatenate cell tables with the same markers into one table annotating the elements of their regions.""" + region_key, instance_key = CodexKeys.REGION_KEY.value, CodexKeys.INSTANCE_KEY.value + obs = pd.concat([table.obs for table in tables]) if len(tables) > 1 else tables[0].obs.copy() + regions = pd.concat([table.region for table in tables]) if len(tables) > 1 else tables[0].region + elements = regions.map({region: element(region) for region in regions.unique()}).to_numpy() + obs[region_key] = pd.Categorical(elements, categories=sorted(set(elements))) + obs[instance_key] = np.concatenate([table.instance_id.to_numpy() for table in tables]) + if not obs.index.is_unique: + obs.index = pd.Index(regions.to_numpy(dtype=object) + "_" + obs.index.to_numpy(dtype=object)) + X = np.concatenate([table.X for table in tables]) if len(tables) > 1 else tables[0].X + adata = ad.AnnData(X=X, obs=obs, var=tables[0].var.copy()) + adata.obsm[CodexKeys.SPATIAL_KEY.value] = np.concatenate([table.centroids_px for table in tables]) + return TableModel.parse( + adata, + region=list(obs[region_key].cat.categories), + region_key=region_key, + instance_key=instance_key, + ) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 02b6d09f..58cefff6 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -29,6 +29,7 @@ scans_to_attrs, select_regions, ) +from spatialdata_io.readers._utils._cell_tables import CellTable, build_cells, read_codex_table from spatialdata_io.readers._utils._qptiff import AkoyaChannel, AkoyaScan, read_qptiff_image, read_qptiff_metadata from spatialdata_io.readers._utils._utils import _initialize_raster_models_kwargs, _set_reader_metadata @@ -101,13 +102,22 @@ def codex( ``{vx.CHANNEL_NAMES_FILE!r}`` (one name per line, cycle-major), else they are ``cycCCC_chHHH``. - ``*{vx.QPTIFF!r}``: scans of the CODEX Instrument Manager, read lazily with their channel metadata. - ``{vx.EXPERIMENT_FILE!r}``: optional; the pixel size is taken from its ``xyResolution`` (nm). - - ``*{vx.FCS_FILE!r}`` / ``*{vx.FCS_FILE_CSV!r}``: cell tables, not read yet. + - ``*{vx.FCS_FILE!r}`` / ``*{vx.FCS_FILE_CSV!r}``: cell tables of the CODEX Processor, one per region (or + one for all regions), e.g. ``segm/segm-1/fcs/compensated/regNNN_compensated.fcs``. Compensated tables + are preferred over uncompensated ones. The ``cycCCC_chHHH`` marker columns are stored in ``X`` (dense + float32) under their marker names, taken from the FCS ``$PnS`` keywords or the CSV header + ``cycCCC_chHHH:``; all other columns go to ``obs``, the ``x``/``y`` pixel centroids to + ``obsm["spatial"]``. Raw acquisition and intermediate folders (``cycCCC_regNNN/``, ``tiles/`` and, next to ``stitched/``, ``bestFocus/``) are ignored. The region id ``regNNN`` is taken from the file names; a ``{vx.QPTIFF!r}`` without one is named after its file. If a directory contains no such files, its top-level ``{vx.IMAGE_TIF}(f)`` files are read as hyperstacks named after their files. Each region is stored as the image ``_image`` in the - ``"global"`` coordinate system and its metadata in ``sdata.attrs["akoya"]``. + ``"global"`` coordinate system and its metadata in ``sdata.attrs["akoya"]``. The cells of each region are stored + as circles ``_cells`` with the cells' area (``size`` column), annotated by one table ``"table"`` + (``obs["region"]`` names the circles, ``obs["cell_id"]`` is the instance key), or by one table + ``_table`` per region if the regions have different markers. The paths of the tables read are stored in + ``sdata.attrs["akoya"]["cell_tables"]``. .. seealso:: @@ -118,9 +128,10 @@ def codex( path Path to a CODEX output directory, or to a single image (``{vx.QPTIFF!r}`` or ``{vx.IMAGE_TIF}(f)``). fcs - Whether to prefer ``{vx.FCS_FILE!r}`` cell tables over ``{vx.FCS_FILE_CSV!r}`` ones when both exist. + Whether to prefer ``{vx.FCS_FILE!r}`` cell tables over ``{vx.FCS_FILE_CSV!r}`` ones when both exist. With + `False`, a region that only has a ``{vx.FCS_FILE!r}`` table is read from it, with a warning. regions - Regions to read, e.g. ``["reg001"]``. If `None`, all regions are read. + Regions to read, e.g. ``["reg001"]``, including only their cells. If `None`, all regions are read. channel_names Channel names to use instead of the discovered ones; must have one entry per channel of every region read. cells_table @@ -148,13 +159,13 @@ def codex( path = Path(path) layout = discover_legacy(path, fcs) - selected = select_regions(list(layout.images), regions) + selected = select_regions(sorted(layout.images.keys() | layout.tables.keys() - {"*"}), regions) experiment = read_json_object(layout.experiment_json) if layout.experiment_json is not None else None names_txt = _read_channel_names_txt(layout.channel_names_txt) if layout.channel_names_txt is not None else None scans: list[AkoyaScan] = [] arrays: dict[str, da.Array] = {} - for region in selected: + for region in (region for region in selected if region in layout.images): files = layout.images[region] if _is_qptiff(files[0]): scans.append(read_qptiff_metadata(files[0], region=region)) @@ -191,11 +202,38 @@ def codex( ) for scan in scans } - sdata = SpatialData(images=images) + table_files = {key: file for key, file in layout.tables.items() if key == "*" or key in selected} + cell_tables = _read_tables(table_files, layout, selected if regions is not None else None, fcs) + tables, shapes = build_cells(cell_tables, {}, {}, transformations) if cell_tables else ({}, {}) + sdata = SpatialData(images=images, shapes=shapes, tables=tables) sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value] = scans_to_attrs(scans, names) + sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value]["cell_tables"] = {key: str(file) for key, file in table_files.items()} return _set_reader_metadata(sdata, "codex") +def _read_tables( + files: Mapping[str, Path], layout: LegacyLayout, regions: Sequence[str] | None, fcs: bool +) -> list[CellTable]: + """Read the cell tables of ``files`` (by region, ``"*"`` for one without a region), keeping only ``regions``.""" + fallback = [str(file) for file in files.values() if file.suffix.lower() == CodexKeys.FCS_FILE.value] + if not fcs and fallback: + warnings.warn( + f"`fcs=False`, but no CSV cell table was found for {fallback}; reading the FCS table(s) instead.", + UserWarning, + stacklevel=3, + ) + # a table without a region column belongs to the region in its file name, else to the only image + only_image = next(iter(layout.images)) if len(layout.images) == 1 else None + tables = [] + for key, file in files.items(): + table = read_codex_table(file, region=key if key != "*" else only_image) + if regions is not None: + table = table.take(np.flatnonzero(table.region.isin(regions).to_numpy())) + if len(table.region): + tables.append(table) + return tables + + def discover_legacy(path: Path, fcs: bool) -> LegacyLayout: """Find the images, cell tables and sidecar files of a CODEX dataset. diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py new file mode 100644 index 00000000..fe43b51a --- /dev/null +++ b/tests/readers/test_cell_tables.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np +import pandas as pd +import pytest +from spatialdata.transformations import Identity, Translation, get_transformation +from tests._akoya_data import write_fcs + +from spatialdata_io.readers._utils._akoya import dedupe_channel_names +from spatialdata_io.readers._utils._cell_tables import build_cells, read_codex_table + +if TYPE_CHECKING: + from pathlib import Path + + +def _cells(n: int = 5, region: int = 1, markers: tuple[str, ...] = ("cyc001_ch001", "cyc001_ch002")) -> pd.DataFrame: + rng = np.random.default_rng(0) + return pd.DataFrame( + { + "cell_id": np.arange(1, n + 1, dtype=float), + "region": float(region), + "x": rng.uniform(0, 100, n), + "y": rng.uniform(0, 80, n), + "size": rng.uniform(10, 100, n), + **{marker: rng.uniform(0, 1000, n) for marker in markers}, + } + ) + + +def test_read_codex_fcs_marker_names_from_pns(tmp_path: Path) -> None: + cells = _cells(markers=("cyc001_ch001", "cyc001_ch002", "cyc002_ch001")) + path = tmp_path / "reg001_compensated.fcs" + write_fcs(path, cells, {"cyc001_ch001": "DAPI", "cyc001_ch002": "CD3", "cyc002_ch001": "DAPI"}) + + table = read_codex_table(path) + + assert list(table.var.index) == ["DAPI", "CD3", "DAPI (2)"] + assert list(table.var["cycle"]) == [1, 1, 2] + assert list(table.var["channel"]) == [1, 2, 1] + assert list(table.var["source_column"]) == ["cyc001_ch001", "cyc001_ch002", "cyc002_ch001"] + assert table.X.dtype == np.float32 + np.testing.assert_allclose(table.X, cells[["cyc001_ch001", "cyc001_ch002", "cyc002_ch001"]], rtol=1e-6) + assert "cyc001_ch001" not in table.obs.columns + assert list(table.obs.index) == ["1", "2", "3", "4", "5"] + assert list(table.region.unique()) == ["reg001"] + assert table.instance_id.dtype == np.int64 + np.testing.assert_allclose(table.centroids_px, cells[["x", "y"]], rtol=1e-6) + assert table.radius_px is not None + np.testing.assert_allclose(table.radius_px, np.sqrt(cells["size"] / np.pi), rtol=1e-6) + + +def test_read_codex_fcs_without_pns_keeps_column_names(tmp_path: Path) -> None: + path = tmp_path / "reg001.fcs" + write_fcs(path, _cells()) + + assert list(read_codex_table(path).var.index) == ["cyc001_ch001", "cyc001_ch002"] + + +def test_read_codex_csv_colon_header(tmp_path: Path) -> None: + path = tmp_path / "reg002.csv" + path.write_text( + "cell_id:cell_id,region:region,x:x,y:y,size:size,cyc001_ch001:DAPI1,cyc001_ch002:CD3\n" + "1,2,10,20,30,100.5,7\n" + "2,2,11,21,31,200.5,8\n" + ) + + table = read_codex_table(path) + + assert list(table.var.index) == ["DAPI1", "CD3"] + assert list(table.var["source_column"]) == ["cyc001_ch001:DAPI1", "cyc001_ch002:CD3"] + np.testing.assert_array_equal(table.X, np.array([[100.5, 7], [200.5, 8]], dtype=np.float32)) + assert list(table.obs.columns) == ["cell_id", "region", "x", "y", "size"] + assert list(table.region) == ["reg002", "reg002"] + np.testing.assert_array_equal(table.centroids_px, [[10, 20], [11, 21]]) + + +def test_read_codex_csv_marker_name_columns(tmp_path: Path) -> None: + path = tmp_path / "cells.csv" + path.write_text("cell_id,x,y,tile_num,CD3,CD8,label\n1,1.0,2.0,1,5.0,6.0,T\n") + + table = read_codex_table(path, region="Kidney") + + assert list(table.var.index) == ["CD3", "CD8"] + assert "cycle" not in table.var.columns + assert list(table.obs.columns) == ["cell_id", "x", "y", "tile_num", "label"] + assert list(table.region) == ["Kidney"] + assert table.radius_px is None + + +def test_read_codex_table_errors(tmp_path: Path) -> None: + no_xy = tmp_path / "a.csv" + no_xy.write_text("cell_id,cyc001_ch001\n1,2.0\n") + with pytest.raises(ValueError, match="'x' and 'y'"): + read_codex_table(no_xy) + no_region = tmp_path / "b.csv" + no_region.write_text("cell_id,x,y,cyc001_ch001\n1,1.0,2.0,2.0\n") + with pytest.raises(ValueError, match="region"): + read_codex_table(no_region) + + +def test_build_cells_single_table_and_circles(tmp_path: Path) -> None: + write_fcs(tmp_path / "reg001.fcs", _cells(n=3, region=1)) + write_fcs(tmp_path / "reg002.fcs", _cells(n=4, region=2)) + tables = [read_codex_table(tmp_path / f"reg00{r}.fcs") for r in (1, 2)] + translation = Translation([10, 0], axes=("x", "y")) + + with pytest.warns(UserWarning, match=r"No segmentation mask found for region.*circles") as record: + anndatas, shapes = build_cells(tables, {}, {}, {"reg001": Identity(), "reg002": translation}) + + assert len(record) == 1 + assert list(anndatas) == ["table"] + table = anndatas["table"] + assert table.n_obs == 7 + assert table.obs_names.is_unique + assert list(table.obs["region"].cat.categories) == ["reg001_cells", "reg002_cells"] + assert table.uns["spatialdata_attrs"]["region"] == ["reg001_cells", "reg002_cells"] + assert table.obs["cell_id"].dtype == np.int64 + np.testing.assert_allclose(np.asarray(table.obsm["spatial"])[:3], tables[0].centroids_px) + assert sorted(shapes) == ["reg001_cells", "reg002_cells"] + assert list(shapes["reg002_cells"].index) == [1, 2, 3, 4] + assert get_transformation(shapes["reg002_cells"], "global") == translation + + +def test_build_cells_different_panels_split_tables(tmp_path: Path) -> None: + write_fcs(tmp_path / "reg001.fcs", _cells(region=1, markers=("cyc001_ch001",))) + write_fcs(tmp_path / "reg002.fcs", _cells(region=2, markers=("cyc001_ch001", "cyc001_ch002"))) + tables = [read_codex_table(tmp_path / f"reg00{r}.fcs") for r in (1, 2)] + + with pytest.warns(UserWarning, match="circles"): + anndatas, _ = build_cells(tables, {}, {}, {}) + + assert sorted(anndatas) == ["reg001_table", "reg002_table"] + assert anndatas["reg002_table"].n_vars == 2 + assert anndatas["reg002_table"].uns["spatialdata_attrs"]["region"] == ["reg002_cells"] + + +def test_dedupe_matches_akoya_convention(tmp_path: Path) -> None: + write_fcs(tmp_path / "reg001.fcs", _cells(), {"cyc001_ch001": "X", "cyc001_ch002": "X"}) + + assert list(read_codex_table(tmp_path / "reg001.fcs").var.index) == dedupe_channel_names(["X", "X"]) diff --git a/tests/test_codex.py b/tests/test_codex.py index d3c18717..e303808a 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -212,3 +212,91 @@ def test_codex_imread_kwargs_warns_when_unused(tmp_path: Path) -> None: write_legacy_processor_tree(tmp_path, layout="per_marker") with pytest.warns(UserWarning, match="imread_kwargs"): codex(tmp_path, imread_kwargs={"nframes": 1}, image_models_kwargs=_FAST) + + +def test_codex_table_multi_region(tmp_path: Path) -> None: + data = write_legacy_processor_tree(tmp_path, regions=2, layout="per_marker", n_cells=50) + + with pytest.warns(UserWarning, match="circles"): + sdata = codex(tmp_path, image_models_kwargs=_FAST) + + assert list(sdata.tables) == ["table"] + table = sdata.tables["table"] + assert table.n_obs == 100 + assert list(table.obs["region"].cat.categories) == ["reg001_cells", "reg002_cells"] + assert list(table.var_names) == dedupe_channel_names(data["markers"]) + assert sorted(sdata.shapes) == ["reg001_cells", "reg002_cells"] + # the fixture returns the cells of the last region + radius = sdata.shapes["reg002_cells"]["radius"].to_numpy() + np.testing.assert_allclose(radius, np.sqrt(data["cells"]["size"] / np.pi), rtol=1e-6) + reg002 = table[table.obs["region"] == "reg002_cells"] + np.testing.assert_allclose(reg002.obsm["spatial"], data["cells"][["x", "y"]], rtol=1e-6) + assert sdata.attrs["akoya"]["cell_tables"]["reg002"].endswith("reg002_compensated.fcs") + + +@pytest.mark.parametrize("fcs", [True, False]) +def test_codex_old_flat_layout_regression(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fcs: bool) -> None: + # the original reader read the image (and the CSV) relative to the working directory + data = write_legacy_processor_tree(tmp_path / "data", layout="old_flat", fcs=fcs) + monkeypatch.chdir(tmp_path) + + with pytest.warns(UserWarning, match="circles"): + sdata = codex(tmp_path / "data", fcs=fcs, image_models_kwargs=_FAST) + + np.testing.assert_array_equal(_level0(sdata, "reg001_image").values, data["images"]["reg001"][:1]) + table = sdata.tables["table"] + assert table.n_obs == 50 + assert list(table.obs["region"].cat.categories) == ["reg001_cells"] + np.testing.assert_allclose(table.X, data["cells"].filter(regex="^cyc").to_numpy(), rtol=1e-6) + + +def test_codex_fcs_false_falls_back_to_fcs_with_warning(tmp_path: Path) -> None: + data = write_legacy_processor_tree(tmp_path, layout="per_marker", fcs=True, csv=False) + + with pytest.warns(UserWarning, match="CSV"): + sdata = codex(tmp_path, fcs=False, image_models_kwargs=_FAST) + + assert list(sdata.tables["table"].var_names) == dedupe_channel_names(data["markers"]) + + +def test_codex_without_tables_reads_images_only(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, layout="per_marker", fcs=False, csv=False) + + sdata = codex(tmp_path, image_models_kwargs=_FAST) + + assert list(sdata.images) == ["reg001_image"] + assert not sdata.tables + assert not sdata.shapes + + +def test_codex_table_float32_dense(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, layout="per_marker", fcs=False, csv=True) + + with pytest.warns(UserWarning, match="circles"): + sdata = codex(tmp_path, fcs=False, image_models_kwargs=_FAST) + + table = sdata.tables["table"] + assert isinstance(table.X, np.ndarray) + assert table.X.dtype == np.float32 + + +def test_codex_regions_subsets_tables(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, regions=2, layout="per_marker") + + with pytest.warns(UserWarning, match="circles"): + sdata = codex(tmp_path, regions=["reg002"], image_models_kwargs=_FAST) + + assert list(sdata.shapes) == ["reg002_cells"] + assert list(sdata.tables["table"].obs["region"].cat.categories) == ["reg002_cells"] + + +def test_codex_single_fcs_file(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, layout="per_marker") + fcs = tmp_path / "processed_2020-01-01" / "segm" / "segm-1" / "fcs" / "compensated" / "reg001_compensated.fcs" + + with pytest.warns(UserWarning, match="circles"): + sdata = codex(fcs) + + assert not sdata.images + assert list(sdata.shapes) == ["reg001_cells"] + assert sdata.tables["table"].n_obs == 50 From afd5f24fca28d72decc60ebbd48b539c9863100a Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 14:09:57 +0200 Subject: [PATCH 009/107] fix: skip unusable region-less CODEX tables and reject overlapping ones Region-less tables that cannot be read or assigned to a region are skipped with a warning instead of aborting the read; next to a single stem-named image they annotate it. Overlapping regions raise early, blank $PnS fall back to $PnN, and attrs only record tables that contributed cells. Co-Authored-By: Claude Opus 5.5 --- .../readers/_utils/_cell_tables.py | 20 +++++---- src/spatialdata_io/readers/codex.py | 45 ++++++++++++------- tests/readers/test_cell_tables.py | 17 +++++++ tests/test_codex.py | 43 +++++++++++++++++- 4 files changed, 100 insertions(+), 25 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index c740e58e..734e86aa 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -111,7 +111,7 @@ def read_codex_table(path: Path, region: str | None = None) -> CellTable: # the Processor CSV header form `cyc001_ch001:DAPI1` (also `cell_id:cell_id`) split = [name.partition(":") for name in names] df.columns = pd.Index([name for name, _, _ in split]) - displays = [label or display or name for (name, _, display), label in zip(split, labels, strict=True)] + displays = [label.strip() or display or name for (name, _, display), label in zip(split, labels, strict=True)] marker_matches = [_MARKER_COLUMN.fullmatch(column) for column in df.columns] if any(marker_matches): @@ -199,6 +199,7 @@ def build_cells( The table elements, ``{"table": ...}`` if all tables have the same markers, else ``{"_table": ...}`` per region, and the circle shapes by element name. The circles have the cells' area (radius 1 px without one). """ + _check_disjoint_regions(tables) pieces = _split_by_region(tables) if len({tuple(table.var.index) for table in tables}) > 1 else None def element(region: str) -> str: @@ -240,14 +241,17 @@ def _region_rows(table: CellTable) -> dict[str, np.ndarray]: return {str(region): np.flatnonzero(codes == i) for i, region in enumerate(regions)} -def _split_by_region(tables: Sequence[CellTable]) -> dict[str, CellTable]: - pieces = {} +def _check_disjoint_regions(tables: Sequence[CellTable]) -> None: + seen: set[str] = set() for table in tables: - for region, rows in _region_rows(table).items(): - if region in pieces: - raise ValueError(f"Several cell tables contain region '{region}'.") - pieces[region] = table.take(rows) - return pieces + regions = set(table.region.unique()) + if overlap := seen & regions: + raise ValueError(f"Several cell tables contain the cells of region(s) {sorted(overlap)}.") + seen |= regions + + +def _split_by_region(tables: Sequence[CellTable]) -> dict[str, CellTable]: + return {region: table.take(rows) for table in tables for region, rows in _region_rows(table).items()} def _to_anndata(tables: Sequence[CellTable], element: Callable[[str], str]) -> ad.AnnData: diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 58cefff6..904bc616 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -12,6 +12,7 @@ import dask.array as da import numpy as np +import pandas as pd import tifffile import zarr from dask.base import tokenize @@ -204,17 +205,21 @@ def codex( } table_files = {key: file for key, file in layout.tables.items() if key == "*" or key in selected} cell_tables = _read_tables(table_files, layout, selected if regions is not None else None, fcs) - tables, shapes = build_cells(cell_tables, {}, {}, transformations) if cell_tables else ({}, {}) + tables, shapes = build_cells(list(cell_tables.values()), {}, {}, transformations) if cell_tables else ({}, {}) sdata = SpatialData(images=images, shapes=shapes, tables=tables) sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value] = scans_to_attrs(scans, names) - sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value]["cell_tables"] = {key: str(file) for key, file in table_files.items()} + sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value]["cell_tables"] = {key: str(table_files[key]) for key in cell_tables} return _set_reader_metadata(sdata, "codex") def _read_tables( files: Mapping[str, Path], layout: LegacyLayout, regions: Sequence[str] | None, fcs: bool -) -> list[CellTable]: - """Read the cell tables of ``files`` (by region, ``"*"`` for one without a region), keeping only ``regions``.""" +) -> dict[str, CellTable]: + """Read the cell tables of ``files`` (by region, ``"*"`` for one without a region), keeping only ``regions``. + + Returns the tables that have cells of ``regions``. A table without a region in its file name that cannot be + read, or whose cells cannot be assigned to a region, is skipped with a warning. + """ fallback = [str(file) for file in files.values() if file.suffix.lower() == CodexKeys.FCS_FILE.value] if not fcs and fallback: warnings.warn( @@ -222,15 +227,26 @@ def _read_tables( UserWarning, stacklevel=3, ) - # a table without a region column belongs to the region in its file name, else to the only image only_image = next(iter(layout.images)) if len(layout.images) == 1 else None - tables = [] + tables = {} for key, file in files.items(): - table = read_codex_table(file, region=key if key != "*" else only_image) + if key != "*": + table = read_codex_table(file, region=key) + else: + # a table without a region in its file name may be any CSV: skip it rather than fail the whole read + try: + table = read_codex_table(file, region=only_image) + except ValueError as e: + warnings.warn(f"Ignoring the cell table {file}: {e}", UserWarning, stacklevel=3) + continue + # next to a single image named after its file (e.g. `Kidney.tif`), its cells belong to that image + values = table.region.unique() + if only_image is not None and len(values) == 1 and values[0] not in layout.images: + table.region = pd.Series(only_image, index=table.region.index, name=table.region.name) if regions is not None: table = table.take(np.flatnonzero(table.region.isin(regions).to_numpy())) if len(table.region): - tables.append(table) + tables[key] = table return tables @@ -524,8 +540,8 @@ def rank(file: Path) -> tuple[bool, bool]: paths = [path for path in paths if rank(path) == best] if region == "*" and len(paths) > 1: raise ValueError( - f"Found several cell tables without a region in their file name: {[str(p) for p in paths]}. " - "Pass the one to read as `cells_table=`." + f"Found several cell tables without a region in their file name: {[str(p) for p in paths]}; " + "the reader cannot choose between them. Pass the path of a directory that contains only one." ) tables[region] = min(paths, key=lambda path: (len(path.parts), str(path))) return tables @@ -537,11 +553,10 @@ def _is_cell_table(path: Path) -> bool: return True with path.open(encoding="utf-8-sig", errors="replace") as f: header = f.readline() - columns = [column.strip().strip('"').casefold() for column in header.split(",")] - return ( - bool({"cell_id", "cellid"} & set(columns)) - or any(_MARKER_COLUMN.match(column) for column in columns) - or {"x", "y"} <= set(columns) + # the name part of the Processor header form `cyc001_ch001:DAPI1` + columns = {column.strip().strip('"').partition(":")[0].casefold() for column in header.split(",")} + return {"x", "y"} <= columns and ( + CodexKeys.INSTANCE_KEY.value in columns or any(_MARKER_COLUMN.fullmatch(column) for column in columns) ) diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index fe43b51a..ca444b9a 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -140,3 +140,20 @@ def test_dedupe_matches_akoya_convention(tmp_path: Path) -> None: write_fcs(tmp_path / "reg001.fcs", _cells(), {"cyc001_ch001": "X", "cyc001_ch002": "X"}) assert list(read_codex_table(tmp_path / "reg001.fcs").var.index) == dedupe_channel_names(["X", "X"]) + + +def test_read_codex_fcs_blank_pns_falls_back_to_pnn(tmp_path: Path) -> None: + write_fcs(tmp_path / "reg001.fcs", _cells(), {"cyc001_ch001": " ", "cyc001_ch002": "CD3"}) + + assert list(read_codex_table(tmp_path / "reg001.fcs").var.index) == ["cyc001_ch001", "CD3"] + + +@pytest.mark.parametrize("different_panels", [False, True]) +def test_build_cells_overlapping_regions_raise(tmp_path: Path, different_panels: bool) -> None: + write_fcs(tmp_path / "a.fcs", _cells(region=1)) + markers = ("cyc001_ch001",) if different_panels else ("cyc001_ch001", "cyc001_ch002") + write_fcs(tmp_path / "b.fcs", _cells(region=1, markers=markers)) + tables = [read_codex_table(tmp_path / name) for name in ("a.fcs", "b.fcs")] + + with pytest.raises(ValueError, match="Several cell tables"): + build_cells(tables, {}, {}, {}) diff --git a/tests/test_codex.py b/tests/test_codex.py index e303808a..56750950 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -203,8 +203,12 @@ def test_discover_legacy_regionless_table_ignores_sidecar_csv(tmp_path: Path) -> assert discover_legacy(tmp_path, fcs=True).tables == {"*": cells} - (tmp_path / "other_cells.csv").write_text("x,y\n1.0,2.0\n") - with pytest.raises(ValueError, match="cells_table"): + (tmp_path / "clusters.csv").write_text("cellid,cluster\n1,2\n") + (tmp_path / "xy.csv").write_text("x,y\n1.0,2.0\n") + assert discover_legacy(tmp_path, fcs=True).tables == {"*": cells} + + (tmp_path / "other_cells.csv").write_text("cell_id:cell_id,x:x,y:y\n1,1.0,2.0\n") + with pytest.raises(ValueError, match="cannot choose"): discover_legacy(tmp_path, fcs=True) @@ -300,3 +304,38 @@ def test_codex_single_fcs_file(tmp_path: Path) -> None: assert not sdata.images assert list(sdata.shapes) == ["reg001_cells"] assert sdata.tables["table"].n_obs == 50 + + +def test_codex_unreadable_regionless_table_is_skipped(tmp_path: Path) -> None: + tifffile.imwrite(tmp_path / "reg001_montage.tif", np.zeros((2, 16, 16), dtype=np.uint16)) + # a region-less table that is not assigned to a region: there are two images and no `region` column + tifffile.imwrite(tmp_path / "reg002_montage.tif", np.zeros((2, 16, 16), dtype=np.uint16)) + (tmp_path / "cells.csv").write_text("cell_id,x,y,cyc001_ch001\n1,1.0,2.0,3.0\n") + (tmp_path / "clusters.csv").write_text("cellid,cluster\n1,2\n") + + with pytest.warns(UserWarning, match="Ignoring the cell table"): + sdata = codex(tmp_path, image_models_kwargs=_FAST) + + assert sorted(sdata.images) == ["reg001_image", "reg002_image"] + assert not sdata.tables + assert sdata.attrs["akoya"]["cell_tables"] == {} + + +def test_codex_regionless_table_belongs_to_single_stem_image(tmp_path: Path) -> None: + tifffile.imwrite(tmp_path / "Kidney.tif", np.zeros((2, 16, 16), dtype=np.uint16)) + (tmp_path / "cells.csv").write_text("cell_id,region,x,y,size,cyc001_ch001\n1,1,1.0,2.0,3.0,4.0\n") + + for regions in (None, ["Kidney"]): + with pytest.warns(UserWarning, match="circles"): + sdata = codex(tmp_path, regions=regions, image_models_kwargs=_FAST) + assert list(sdata.tables["table"].obs["region"].cat.categories) == ["Kidney_cells"] + assert list(sdata.shapes) == ["Kidney_cells"] + assert sdata.attrs["akoya"]["cell_tables"] == {"*": str(tmp_path / "cells.csv")} + + +def test_codex_overlapping_tables_raise(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, layout="per_marker", fcs=False, csv=True) + (tmp_path / "all_cells.csv").write_text("cell_id,region,x,y,cyc001_ch001\n1,1,1.0,2.0,3.0\n") + + with pytest.raises(ValueError, match=r"Several cell tables contain the cells of region\(s\) \['reg001'\]"): + codex(tmp_path, fcs=False, image_models_kwargs=_FAST) From c2b81cf9474fccabed30cadfb5d75d15c4488b64 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 14:30:43 +0200 Subject: [PATCH 010/107] feat: ingest QuPath, mcquant and generic cell tables with mask labels in Akoya readers `cells_table`/`labels` now work in `codex()` and `phenocycler()`: a path or a {region: path} mapping (CLI: JSON). Masks become lazy `_labels` with the region image's transformation and the table annotates them; without a mask cells stay circles. A user `cells_table` replaces auto-discovered CODEX tables. A single table for several regions needs a region column (codex `region`, QuPath `Image`, or `cells_table_kwargs["region_column"]`). Different panels give per-region tables. SPRM/inForm are detected but raise NotImplementedError until a later release. The lazy tiff reader moves from codex.py to _akoya.py (now `aszarr(level=0)`, so pyramidal OME masks work) and is shared by images and masks. A `str` passed as `regions`/`channel_names` is wrapped into a list. Co-Authored-By: Claude Opus 5.5 --- src/spatialdata_io/__main__.py | 43 +- src/spatialdata_io/readers/_utils/_akoya.py | 79 ++- .../readers/_utils/_cell_tables.py | 483 +++++++++++++++++- src/spatialdata_io/readers/codex.py | 175 +++---- src/spatialdata_io/readers/phenocycler.py | 64 ++- tests/readers/test_cell_tables.py | 251 ++++++++- tests/test_codex.py | 50 +- tests/test_phenocycler.py | 199 +++++++- 8 files changed, 1201 insertions(+), 143 deletions(-) diff --git a/src/spatialdata_io/__main__.py b/src/spatialdata_io/__main__.py index 205ddbe7..565411a8 100644 --- a/src/spatialdata_io/__main__.py +++ b/src/spatialdata_io/__main__.py @@ -46,6 +46,13 @@ def _parse_json_param(value: str, param_name: str) -> dict[str, Any]: raise click.BadParameter(f"Invalid JSON for {param_name!r}: {e}") from e +def _parse_path_or_mapping_param(value: str | None, param_name: str) -> str | dict[str, Any] | None: + """A path, or a JSON mapping from region to path if ``value`` starts with ``{``.""" + if value is not None and value.lstrip().startswith("{"): + return _parse_json_param(value, param_name) + return value + + @cli.command(name="codex") @click.option( "--input", @@ -70,9 +77,19 @@ def _parse_json_param(value: str, param_name: str) -> dict[str, Any]: default=None, help="Comma-separated channel names overriding the discovered ones. [default: None]", ) -@click.option("--cells-table", type=click.Path(exists=True), default=None, help="Path to a cell table. [default: None]") +@click.option( + "--cells-table", + type=str, + default=None, + help='Path to a cell table, or a JSON mapping from region to path, e.g. \'{"A": "a.csv"}\'. [default: None]', +) @click.option("--cells-table-format", type=str, default="auto", help="Format of the cell table. [default: auto]") -@click.option("--labels", type=click.Path(exists=True), default=None, help="Path to a label mask. [default: None]") +@click.option( + "--labels", + type=str, + default=None, + help="Path to a segmentation mask, or a JSON mapping from region to path. [default: None]", +) @click.option( "--imread-kwargs", type=str, @@ -119,9 +136,9 @@ def codex_wrapper( fcs=fcs, regions=_parse_list_param(regions), channel_names=_parse_list_param(channel_names), - cells_table=cells_table, + cells_table=_parse_path_or_mapping_param(cells_table, "cells_table"), cells_table_format=cells_table_format, - labels=labels, + labels=_parse_path_or_mapping_param(labels, "labels"), imread_kwargs=_parse_json_param(imread_kwargs, "imread_kwargs"), image_models_kwargs=_parse_json_param(image_models_kwargs, "image_models_kwargs"), labels_models_kwargs=_parse_json_param(labels_models_kwargs, "labels_models_kwargs"), @@ -436,9 +453,19 @@ def _parse_list_param(value: str | None) -> list[str] | None: default=None, help="Comma-separated channel names overriding the ones in the qptiff. [default: None]", ) -@click.option("--cells-table", type=click.Path(exists=True), default=None, help="Path to a cell table. [default: None]") +@click.option( + "--cells-table", + type=str, + default=None, + help='Path to a cell table, or a JSON mapping from region to path, e.g. \'{"A": "a.csv"}\'. [default: None]', +) @click.option("--cells-table-format", type=str, default="auto", help="Format of the cell table. [default: auto]") -@click.option("--labels", type=click.Path(exists=True), default=None, help="Path to a label mask. [default: None]") +@click.option( + "--labels", + type=str, + default=None, + help="Path to a segmentation mask, or a JSON mapping from region to path. [default: None]", +) @click.option( "--image-models-kwargs", type=str, @@ -476,9 +503,9 @@ def phenocycler_wrapper( input, regions=_parse_list_param(regions), channel_names=_parse_list_param(channel_names), - cells_table=cells_table, + cells_table=_parse_path_or_mapping_param(cells_table, "cells_table"), cells_table_format=cells_table_format, - labels=labels, + labels=_parse_path_or_mapping_param(labels, "labels"), image_models_kwargs=_parse_json_param(image_models_kwargs, "image_models_kwargs"), labels_models_kwargs=_parse_json_param(labels_models_kwargs, "labels_models_kwargs"), cells_table_kwargs=_parse_json_param(cells_table_kwargs, "cells_table_kwargs"), diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index 82c5fec4..20882c56 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -3,12 +3,19 @@ from __future__ import annotations import json +import math import warnings from collections import Counter from collections.abc import Mapping, Sequence from dataclasses import asdict from typing import TYPE_CHECKING, Any +import dask.array as da +import numpy as np +import tifffile +import zarr +from dask.base import tokenize +from dask_image.imread import imread from spatialdata._logging import logger from spatialdata.models import Image2DModel from spatialdata.transformations import BaseTransformation, Identity, Translation @@ -18,7 +25,6 @@ if TYPE_CHECKING: from pathlib import Path - import dask.array as da from xarray import DataArray, DataTree from spatialdata_io.readers._utils._qptiff import AkoyaScan @@ -28,9 +34,11 @@ "build_image_element", "dedupe_channel_names", "read_json_object", + "read_tiff", "region_transformations", "scans_to_attrs", "select_regions", + "spatial_chunks", ] @@ -154,3 +162,72 @@ def read_json_object(path: Path) -> dict[str, Any] | None: warnings.warn(f"Ignoring {path}: it is not a JSON object.", UserWarning, stacklevel=2) return None return content + + +def read_tiff( + path: Path, imread_kwargs: Mapping[str, Any], chunks: tuple[int, int] +) -> tuple[da.Array, tuple[int, ...], list[str] | None, bool]: + """Lazily read a tiff as ``(c, y, x)``, all non-spatial axes flattened into ``c``. + + Uncompressed contiguous data (e.g. ImageJ hyperstacks and per-marker images, often a single strip per plane) + is memory-mapped and split into ``chunks``, so a chunk never needs a whole plane in memory; other layouts keep + their native tiles via zarr. Returns the image, the shape of the flattened axes, the ImageJ ``Labels`` (if + any) and whether tifffile, rather than the :func:`dask_image.imread.imread` fallback, read the file. The + file handle stays open for as long as the array needs it. + """ + tf = tifffile.TiffFile(path) + series = tf.series[0] + axes = series.axes + metadata = tf.imagej_metadata or {} + labels = list(metadata["Labels"]) if isinstance(metadata.get("Labels"), list | tuple) else None + read_by_tifffile = True + if series.dataoffset is not None: + tf.close() + source = _ContiguousTiff(path, series.dataoffset, series.shape, series.dtype) + tiles = dict.fromkeys(range(source.ndim), 1) | {axes.index("Y"): chunks[0], axes.index("X"): chunks[1]} + image = da.from_array(source, chunks=tuple(tiles[i] for i in range(source.ndim)), name=source.name) + else: + try: + image = da.from_zarr(zarr.open(series.aszarr(level=0), mode="r")) + except ValueError as e: # e.g. a compression tifffile cannot decode chunk-wise + tf.close() + logger.info("Reading %s with dask_image.imread because tifffile cannot read it lazily: %s", path, e) + image = imread(str(path), **imread_kwargs) + axes = "Q" * (image.ndim - 2) + "YX" + labels = None + read_by_tifffile = False + image = da.moveaxis(image, [axes.index("Y"), axes.index("X")], [-2, -1]) + leading_shape = tuple(image.shape[:-2]) + return image.reshape((math.prod(leading_shape), *image.shape[-2:])), leading_shape, labels, read_by_tifffile + + +class _ContiguousTiff: + """Array-like view of uncompressed contiguous tiff data, memory-mapped anew for every chunk. + + Handing an ``np.memmap`` to :func:`dask.array.from_array` would read the whole file (dask hashes and slices + numpy arrays eagerly), so dask only gets this lightweight object and a name derived from the file. + """ + + def __init__(self, path: Path, offset: int, shape: tuple[int, ...], dtype: np.dtype[Any]) -> None: + self.path = path + self.offset = offset + self.shape = shape + self.ndim = len(shape) + self._file_dtype = dtype + self.dtype = dtype.newbyteorder("=") + self.name = f"akoya-tiff-{tokenize(str(path), path.stat().st_mtime_ns, offset, shape, str(dtype))}" + + def __getitem__(self, key: Any) -> np.ndarray: + data = np.memmap(self.path, dtype=self._file_dtype, mode="r", offset=self.offset, shape=self.shape) + return np.asarray(data[key], dtype=self.dtype) + + +def spatial_chunks(chunks: Any, default: Sequence[int]) -> tuple[int, int]: + """The ``(y, x)`` sizes of a raster model ``chunks`` argument, ``default`` when it does not give plain sizes.""" + if isinstance(chunks, int): + return chunks, chunks + if isinstance(chunks, Mapping) and isinstance(chunks.get("y"), int) and isinstance(chunks.get("x"), int): + return chunks["y"], chunks["x"] + if isinstance(chunks, list | tuple) and len(chunks) >= 2 and all(isinstance(c, int) for c in chunks[-2:]): + return chunks[-2], chunks[-1] + return default[-2], default[-1] diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 734e86aa..610855af 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -4,28 +4,47 @@ import re import warnings +from collections.abc import Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING +from pathlib import Path +from typing import TYPE_CHECKING, Any import anndata as ad import numpy as np import pandas as pd import readfcs -from spatialdata.models import ShapesModel, TableModel +import tifffile +from spatialdata import sanitize_name, sanitize_table +from spatialdata._logging import logger +from spatialdata.models import Labels2DModel, ShapesModel, TableModel from spatialdata.transformations import Identity from spatialdata_io._constants._constants import CodexKeys, PhenocyclerKeys -from spatialdata_io.readers._utils._akoya import dedupe_channel_names +from spatialdata_io.readers._utils._akoya import dedupe_channel_names, read_tiff, spatial_chunks +from spatialdata_io.readers._utils._utils import _initialize_raster_models_kwargs if TYPE_CHECKING: - from collections.abc import Callable, Mapping, Sequence - from pathlib import Path + from collections.abc import Callable, Sequence from geopandas import GeoDataFrame from spatialdata.transformations import BaseTransformation from xarray import DataArray, DataTree -__all__ = ["CellTable", "build_cells", "read_codex_table"] +__all__ = [ + "CellTable", + "build_cells", + "detect_table_format", + "read_cell_table", + "read_codex_table", + "read_labels", + "read_region_labels", + "read_region_tables", + "resolve_per_region", +] + +TABLE_FORMATS = ("codex", "qupath", "mcquant", "sprm", "inform", "generic") +# the keys of `cells_table_kwargs`; see `read_cell_table` and `read_labels` +CELLS_TABLE_KWARGS = ("x", "y", "instance_id", "region_column", "markers", "coordinates_unit", "mask_channel") # `cyc001_ch001` (FCS `$PnN`, plain CSV header) or the name part of the Processor CSV header `cyc001_ch001:DAPI1` _MARKER_COLUMN = re.compile(r"cyc(\d+)_ch(\d+)", re.IGNORECASE) @@ -33,6 +52,11 @@ _META_COLUMNS = frozenset( {"cell_id", "region", "tile_num", "x", "y", "z", "x_tile", "y_tile", "size", "tsne_x", "tsne_y", "homogeneity"} ) +# QuPath measurement columns, e.g. `CD3: Cell: Mean`; the per-marker intensity is taken from the first compartment +_QUPATH_MARKER = re.compile(r"(.+): (Cell|Nucleus|Cytoplasm|Membrane): Mean") +_QUPATH_COMPARTMENTS = ("Cell", "Nucleus", "Cytoplasm", "Membrane") +_QUPATH_IMAGE = "Image" +_QUPATH_ID = "Object ID" @dataclass @@ -160,7 +184,7 @@ def read_codex_table(path: Path, region: str | None = None) -> CellTable: else: raise ValueError( f"{path} has no '{region_key}' column and no region id in its file name, so its cells cannot be " - "assigned to a region." + "assigned to a region; a table for several regions needs a region column." ) region_ids.name = region_key return CellTable( @@ -174,6 +198,448 @@ def read_codex_table(path: Path, region: str | None = None) -> CellTable: ) +def detect_table_format(path: Path) -> str: + """Detect the format of a cell table from its file type and header. + + Parameters + ---------- + path + The ``.fcs``, ``.csv``, ``.tsv`` or ``.txt`` (tab-separated) cell table. + + Returns + ------- + ``"codex"`` (FCS, or ``x``/``y`` with ``cell_id`` or ``cycCCC_chHHH`` columns), ``"qupath"`` (``Centroid X µm`` + or ``Object ID``), ``"mcquant"`` (``CellID`` and ``X_centroid``), ``"inform"`` (``Cell X Position``), + ``"sprm"`` (first column ``ID``), else ``"generic"``. + """ + if path.suffix.lower() == CodexKeys.FCS_FILE.value: + return "codex" + columns, _ = _read_header(path) + # the name part of the CODEX Processor header form `cyc001_ch001:DAPI1` + names = {column.partition(":")[0].casefold() for column in columns} + if {"x", "y"} <= names and ( + CodexKeys.INSTANCE_KEY.value in names or any(_MARKER_COLUMN.fullmatch(name) for name in names) + ): + return "codex" + if {"Centroid X µm", "Centroid X px", _QUPATH_ID} & set(columns): + return "qupath" + if {"CellID", "X_centroid"} <= set(columns): + return "mcquant" + if "Cell X Position" in columns: + return "inform" + if columns[:1] == ["ID"]: + return "sprm" + return "generic" + + +def read_cell_table( + path: Path, + fmt: str, + *, + pixel_size_um: float | None, + default_region: str | None, + cells_table_kwargs: Mapping[str, Any], +) -> CellTable: + """Read a cell table of any supported format. + + Parameters + ---------- + path + The cell table. + fmt + One of ``"auto"`` (see :func:`detect_table_format`), ``"codex"``, ``"qupath"``, ``"mcquant"`` or + ``"generic"``: + + - ``"codex"``: see :func:`read_codex_table`. + - ``"qupath"``: a QuPath measurement export. Centroids ``Centroid X µm``/``Centroid Y µm`` (divided by + ``pixel_size_um``) or ``Centroid X px``/``Centroid Y px``; instance ids from a numeric ``Object ID``, + else the row number (from 1); the region from the ``Image`` column. The markers are the + ``: : Mean`` columns, one per marker from the first of the ``Cell``, + ``Nucleus``, ``Cytoplasm`` and ``Membrane`` compartments (in ``var["compartment"]``). The radius + comes from ``Cell: Area µm^2`` (or ``px^2``). + - ``"mcquant"``: an MCMICRO ``mcquant`` table. Instance ids ``CellID``, pixel centroids + ``X_centroid``/``Y_centroid``, the markers are the columns between ``CellID`` and ``X_centroid``, the + radius comes from ``Area``. + - ``"generic"``: columns given by ``cells_table_kwargs``. + + The ``"sprm"`` and ``"inform"`` formats are detected but not read yet. + pixel_size_um + The pixel size of the region image, to convert µm centroids to pixels. + default_region + The region id of all cells if the table has no region column. + cells_table_kwargs + Optional keys: ``region_column`` (the column holding the region of each cell; QuPath default ``Image``), + and for ``"generic"`` tables ``x`` and ``y`` (required centroid columns), ``instance_id`` (else the row + number, from 1), ``markers`` (a list of columns or a regular expression matching the full column name; + default: all other numeric columns) and ``coordinates_unit`` (``"px"``, default, or ``"um"``). The key + ``mask_channel`` is used by :func:`read_labels`. + + Returns + ------- + The cell table. Its regions are the values of the region column, or ``default_region``. + """ + unknown = sorted(set(cells_table_kwargs) - set(CELLS_TABLE_KWARGS)) + if unknown: + raise ValueError(f"`cells_table_kwargs` has unknown key(s) {unknown}; expected some of {CELLS_TABLE_KWARGS}.") + if fmt == "auto": + fmt = detect_table_format(path) + logger.info("Reading %s as a %r cell table.", path, fmt) + if fmt in ("sprm", "inform"): + raise NotImplementedError(f"Reading {fmt!r} cell tables ({path}) is added in a later release.") + if fmt == "codex": + return read_codex_table(path, region=default_region) + if fmt not in TABLE_FORMATS: + raise ValueError(f"Unknown cell table format {fmt!r}; expected 'auto' or one of {TABLE_FORMATS}.") + + _, sep = _read_header(path) + df = pd.read_csv(path, sep=sep, engine="pyarrow") + df.columns = pd.Index([str(column) for column in df.columns]) + reader = {"qupath": _qupath_roles, "mcquant": _mcquant_roles, "generic": _generic_roles}[fmt] + roles = reader(path, df, cells_table_kwargs) + if roles.pop("in_um"): + if pixel_size_um is None: + raise ValueError( + f"{path} has centroids in µm, but no pixel size is known to convert them to pixels (the region's " + "pixel size is missing, or one table covers regions with different pixel sizes)." + ) + roles["px_per_unit"] = 1 / pixel_size_um + return _from_frame(path, df, default_region=default_region, **roles) + + +def _qupath_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: + unit = next((unit for unit in ("µm", "px") if f"Centroid X {unit}" in df.columns), None) + if unit is None: + raise ValueError(f"{path} has no 'Centroid X µm' or 'Centroid X px' column; found {list(df.columns)}.") + compartments: dict[str, dict[str, str]] = {} + for column in df.columns: + if (match := _QUPATH_MARKER.fullmatch(column)) is not None: + compartments.setdefault(match.group(1), {})[match.group(2)] = column + chosen = {marker: next(c for c in _QUPATH_COMPARTMENTS if c in found) for marker, found in compartments.items()} + markers = [compartments[marker][compartment] for marker, compartment in chosen.items()] + var = pd.DataFrame( + {"source_column": markers, "compartment": list(chosen.values())}, + index=pd.Index(dedupe_channel_names(list(chosen))), + ) + object_id = df.get(_QUPATH_ID) + area = f"Cell: Area {unit}^2" + return { + "markers": markers, + "var": var, + "x": f"Centroid X {unit}", + "y": f"Centroid Y {unit}", + "in_um": unit == "µm", + "instance_column": _QUPATH_ID if object_id is not None and pd.api.types.is_integer_dtype(object_id) else None, + "region_column": kwargs.get("region_column", _QUPATH_IMAGE if _QUPATH_IMAGE in df.columns else None), + "area_column": area if area in df.columns else None, + } + + +def _mcquant_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: + columns = list(df.columns) + if "CellID" not in columns or "X_centroid" not in columns: + raise ValueError(f"{path} has no 'CellID' and 'X_centroid' columns; found {columns}.") + markers = columns[columns.index("CellID") + 1 : columns.index("X_centroid")] + return { + "markers": markers, + "var": pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(markers))), + "x": "X_centroid", + "y": "Y_centroid", + "in_um": False, + "instance_column": "CellID", + "region_column": kwargs.get("region_column"), + "area_column": "Area" if "Area" in columns else None, + } + + +def _generic_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: + if "x" not in kwargs or "y" not in kwargs: + raise ValueError( + f"Reading {path} as a 'generic' cell table needs the centroid columns as `cells_table_kwargs` 'x' and " + f"'y'; found the columns {list(df.columns)}." + ) + unit = kwargs.get("coordinates_unit", "px") + if unit not in ("px", "um"): + raise ValueError(f"`cells_table_kwargs['coordinates_unit']` must be 'px' or 'um', got {unit!r}.") + roles = {kwargs["x"], kwargs["y"], kwargs.get("instance_id"), kwargs.get("region_column")} + selection = kwargs.get("markers") + if selection is None: + markers = [c for c in df.columns if c not in roles and pd.api.types.is_numeric_dtype(df[c])] + elif isinstance(selection, str): + markers = [c for c in df.columns if c not in roles and re.fullmatch(selection, c)] + else: + markers = [str(marker) for marker in selection] + return { + "markers": markers, + "var": pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(markers))), + "x": kwargs["x"], + "y": kwargs["y"], + "in_um": unit == "um", + "instance_column": kwargs.get("instance_id"), + "region_column": kwargs.get("region_column"), + "area_column": None, + } + + +def _from_frame( + path: Path, + df: pd.DataFrame, + *, + markers: list[str], + var: pd.DataFrame, + x: str, + y: str, + instance_column: str | None, + region_column: str | None, + area_column: str | None, + default_region: str | None, + px_per_unit: float = 1.0, +) -> CellTable: + """Build a `CellTable` from a data frame and the roles of its columns.""" + roles = [x, y, instance_column, region_column, area_column, *markers] + missing = [column for column in roles if column is not None and column not in df.columns] + if missing: + raise ValueError(f"{path} has no column(s) {missing}; found {list(df.columns)}.") + + if instance_column is not None: + instance_id = pd.Series(df[instance_column].to_numpy(dtype=np.int64)) + else: + # label images use 0 for the background, so instance ids start at 1 + instance_id = pd.Series(np.arange(1, len(df) + 1, dtype=np.int64)) + index = pd.Index(instance_id.astype(str)) + instance_id.index = index + instance_id.name = CodexKeys.INSTANCE_KEY.value + + if region_column is not None: + region = pd.Series(df[region_column].astype(str).to_numpy(), index=index) + elif default_region is not None: + region = pd.Series(default_region, index=index) + else: + raise ValueError( + f"{path} has no region column, so its cells cannot be assigned to regions: a table for several regions " + "needs a region column (`cells_table_kwargs['region_column']`). Alternatively, pass one table per " + "region as a mapping {region: path}." + ) + region.name = CodexKeys.REGION_KEY.value + + obs = df.drop(columns=markers) + obs.index = index + radius = None + if area_column is not None: + radius = np.sqrt(df[area_column].to_numpy(dtype=np.float64) * px_per_unit**2 / np.pi) + return CellTable( + obs=obs, + X=df[markers].to_numpy(dtype=np.float32), + var=var, + region=region, + instance_id=instance_id, + centroids_px=df[[x, y]].to_numpy(dtype=np.float64) * px_per_unit, + radius_px=radius, + ) + + +def _read_header(path: Path) -> tuple[list[str], str]: + """The column names of a delimited text table and its delimiter (tab if the header has one, else comma).""" + with path.open(encoding="utf-8-sig", errors="replace") as f: + header = f.readline().rstrip("\r\n") + sep = "\t" if "\t" in header else "," + return [column.strip().strip('"') for column in header.split(sep)], sep + + +def resolve_per_region(value: str | Path | Mapping[str, str | Path] | None, regions: Sequence[str]) -> dict[str, Path]: + """Resolve a ``cells_table`` or ``labels`` argument to one file per region. + + Parameters + ---------- + value + A path, a mapping from region id to path, or `None`. + regions + The regions read. + + Returns + ------- + ``{}`` for `None`; ``{region: path}`` for a path and a single region, else ``{"*": path}`` (a file for all + regions); the mapping, in the order of ``regions``. + """ + if value is None: + return {} + if isinstance(value, Mapping): + unknown = sorted(set(value) - set(regions)) + if unknown: + raise ValueError(f"Unknown region(s) {unknown}. Regions read: {list(regions)}.") + files = {region: Path(value[region]) for region in regions if region in value} + else: + files = {regions[0] if len(regions) == 1 else "*": Path(value)} + for file in files.values(): + if not file.is_file(): + raise FileNotFoundError(f"{file} does not exist or is not a file.") + return files + + +def read_region_tables( + files: Mapping[str, Path], + fmt: str, + *, + available: Sequence[str], + selected: Sequence[str], + pixel_sizes: Mapping[str, float | None], + cells_table_kwargs: Mapping[str, Any], +) -> list[CellTable]: + """Read user-supplied cell tables, as resolved by :func:`resolve_per_region`, and assign their cells to regions. + + A region column value names the region whose id it equals, else the longest region id it starts with followed + by a non-alphanumeric character (``A_Scan1.qptiff`` -> ``A``). A table given for one region holds the cells of + that region; if its region column matches no region, it holds only cells of that region. Cells of a table for + all regions (``"*"``) whose region matches no available region are dropped with a warning; cells of available + regions that are not read are dropped silently. + + Parameters + ---------- + files + The table of each region, or ``{"*": path}`` for one table for all regions. + fmt + The table format, see :func:`read_cell_table`. + available + All region ids of the dataset. + selected + The region ids read. + pixel_sizes + The pixel size in µm of each region with an image. + cells_table_kwargs + See :func:`read_cell_table`. + + Returns + ------- + The cell tables, one per file. + """ + tables = [] + for key, path in files.items(): + covered = selected if key == "*" else [key] + sizes = {pixel_sizes.get(region) for region in covered} + table = read_cell_table( + path, + fmt, + pixel_size_um=sizes.pop() if len(sizes) == 1 else None, + default_region=None if key == "*" else key, + cells_table_kwargs=cells_table_kwargs, + ) + tables.append(_assign_regions(table, path, key, available, selected)) + return tables + + +def _assign_regions( + table: CellTable, path: Path, key: str, available: Sequence[str], selected: Sequence[str] +) -> CellTable: + values = list(table.region.unique()) + matched = {value: _match_region(value, available) for value in values} + if key != "*" and all(region is None for region in matched.values()): + # the region column of a table for one region names its source image only + matched = dict.fromkeys(values, key) + unmatched = sorted(value for value, region in matched.items() if region is None) + if unmatched: + warnings.warn( + f"Ignoring the cells of {unmatched} in {path}: they match none of the regions {list(available)}.", + UserWarning, + stacklevel=4, + ) + table.region = table.region.map(matched) + keep = table.region.isin(selected if key == "*" else [key]).to_numpy() + if not keep.any(): + target = f"region '{key}'" if key != "*" else f"the regions read {list(selected)}" + raise ValueError(f"{path} has no cells of {target}; its region column holds {values}.") + return table.take(np.flatnonzero(keep)) + + +def _match_region(value: str, regions: Sequence[str]) -> str | None: + """The region named by ``value``: equal to it, else the longest one it starts with before a non-alphanumeric.""" + if value in regions: + return value + prefixes = [r for r in regions if value.startswith(r) and not value[len(r)].isalnum()] + return max(prefixes, key=len, default=None) + + +def read_labels( + path: Path, + labels_models_kwargs: Mapping[str, Any], + transformation: BaseTransformation, + *, + mask_channel: int | str | None = None, +) -> DataArray | DataTree: + """Lazily read a segmentation mask as a ``Labels2DModel``. + + Parameters + ---------- + path + A TIFF or OME-TIFF with integer instance ids (0 = background). + labels_models_kwargs + Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`; their ``chunks`` also set the + spatial chunks of uncompressed files, which are memory-mapped. + transformation + The transformation to ``"global"``, the one of the region's image. + mask_channel + The mask to read from a file with several: its index or OME channel name (e.g. ``"cells"``). If `None`, + the first one. + + Returns + ------- + The labels element. + """ + _, labels_models_kwargs = _initialize_raster_models_kwargs({}, labels_models_kwargs) + _, defaults = _initialize_raster_models_kwargs({}, {}) + chunks = spatial_chunks(labels_models_kwargs["chunks"], defaults["chunks"]) + image, _, _, _ = read_tiff(path, {}, chunks) + if not np.issubdtype(image.dtype, np.integer): + raise ValueError(f"{path} has dtype {image.dtype}; a segmentation mask must have an integer dtype.") + + n_masks = image.shape[0] + if isinstance(mask_channel, str): + names = _ome_channel_names(path) + if mask_channel not in names: + raise ValueError(f"{path} has no mask channel {mask_channel!r}; its OME channel names are {names}.") + index = names.index(mask_channel) + else: + index = 0 if mask_channel is None else mask_channel + if mask_channel is None and n_masks > 1: + logger.info("%s holds %d masks; reading the first. Set `mask_channel` to choose another.", path, n_masks) + if not 0 <= index < n_masks: + raise ValueError(f"{path} holds {n_masks} mask channel(s); `mask_channel` {mask_channel!r} is out of range.") + return Labels2DModel.parse( + image[index], dims=("y", "x"), transformations={"global": transformation}, **labels_models_kwargs + ) + + +def _ome_channel_names(path: Path) -> list[str]: + with tifffile.TiffFile(path) as tf: + if not tf.ome_metadata: + return [] + images = tifffile.xml2dict(tf.ome_metadata)["OME"]["Image"] + pixels = (images[0] if isinstance(images, list) else images)["Pixels"] + channels = pixels.get("Channel", []) + return [str(channel.get("Name", "")) for channel in (channels if isinstance(channels, list) else [channels])] + + +def read_region_labels( + files: Mapping[str, Path], + transformations: Mapping[str, BaseTransformation], + labels_models_kwargs: Mapping[str, Any], + cells_table_kwargs: Mapping[str, Any], +) -> dict[str, DataArray | DataTree]: + """Read the mask of each region, as resolved by :func:`resolve_per_region`, with the region's transformation.""" + if "*" in files: + raise ValueError( + f"A single labels mask {files['*']} was given for several regions; a mask belongs to one region image, " + "so pass a mapping {region: path}." + ) + return { + region: read_labels( + path, + labels_models_kwargs, + transformations.get(region, Identity()), + mask_channel=cells_table_kwargs.get("mask_channel"), + ) + for region, path in files.items() + } + + def build_cells( tables: Sequence[CellTable], region_elements: Mapping[str, str], @@ -267,6 +733,9 @@ def _to_anndata(tables: Sequence[CellTable], element: Callable[[str], str]) -> a X = np.concatenate([table.X for table in tables]) if len(tables) > 1 else tables[0].X adata = ad.AnnData(X=X, obs=obs, var=tables[0].var.copy()) adata.obsm[CodexKeys.SPATIAL_KEY.value] = np.concatenate([table.centroids_px for table in tables]) + # e.g. QuPath's `Centroid X µm` -> `Centroid_X_µm`: spatialdata only writes alphanumeric names and `_.-` + if any(sanitize_name(str(column), is_dataframe_column=True) != column for column in adata.obs.columns): + sanitize_table(adata, inplace=True) return TableModel.parse( adata, region=list(obs[region_key].cat.categories), diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 904bc616..30b5b55d 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -1,6 +1,5 @@ from __future__ import annotations -import math import os import re import warnings @@ -13,12 +12,7 @@ import dask.array as da import numpy as np import pandas as pd -import tifffile -import zarr -from dask.base import tokenize -from dask_image.imread import imread from spatialdata import SpatialData -from spatialdata._logging import logger from spatialdata_io._constants._constants import CodexKeys, PhenocyclerKeys from spatialdata_io._docs import inject_docs @@ -26,11 +20,21 @@ apply_channel_names, build_image_element, read_json_object, + read_tiff, region_transformations, scans_to_attrs, select_regions, + spatial_chunks, +) +from spatialdata_io.readers._utils._cell_tables import ( + CellTable, + build_cells, + detect_table_format, + read_codex_table, + read_region_labels, + read_region_tables, + resolve_per_region, ) -from spatialdata_io.readers._utils._cell_tables import CellTable, build_cells, read_codex_table from spatialdata_io.readers._utils._qptiff import AkoyaChannel, AkoyaScan, read_qptiff_image, read_qptiff_metadata from spatialdata_io.readers._utils._utils import _initialize_raster_models_kwargs, _set_reader_metadata @@ -46,8 +50,6 @@ # raw Keyence acquisition folders, e.g. `cyc001_reg001/1_00001_Z001_CH1.tif` _RAW_CYCLE_DIR = re.compile(r"cyc\d+_reg\d+", re.IGNORECASE) _TIFF_SUFFIXES = (".tif", ".tiff") -# marker columns of CODEX Processor tables: `cyc001_ch001` or `cyc001_ch001:DAPI1` -_MARKER_COLUMN = re.compile(r"cyc\d+_ch\d+", re.IGNORECASE) _TABLE_SUFFIXES = (CodexKeys.FCS_FILE.value, CodexKeys.FCS_FILE_CSV.value) # deep enough for `/processed_/segm/segm-1/fcs/compensated/
` _MAX_DEPTH = 8 @@ -117,8 +119,9 @@ def codex( ``"global"`` coordinate system and its metadata in ``sdata.attrs["akoya"]``. The cells of each region are stored as circles ``_cells`` with the cells' area (``size`` column), annotated by one table ``"table"`` (``obs["region"]`` names the circles, ``obs["cell_id"]`` is the instance key), or by one table - ``_table`` per region if the regions have different markers. The paths of the tables read are stored in - ``sdata.attrs["akoya"]["cell_tables"]``. + ``_table`` per region if the regions have different markers. A region with a segmentation mask in + ``labels`` gets the labels ``_labels`` instead of circles, and its cells annotate them. The paths of the + tables and masks read are stored in ``sdata.attrs["akoya"]["cell_tables"]`` and ``["labels"]``. .. seealso:: @@ -136,31 +139,44 @@ def codex( channel_names Channel names to use instead of the discovered ones; must have one entry per channel of every region read. cells_table - Not implemented yet. + A cell table to read instead of the discovered ones (explicit beats discovered): a path, or a mapping from + region id to path, e.g. ``{{"reg001": "reg001_cells.csv"}}``. A single path for several regions needs a + region column (``region`` for CODEX tables, ``Image`` for QuPath, else + ``cells_table_kwargs["region_column"]``). See ``cells_table_format`` for the formats. cells_table_format - Not implemented yet. + The format of ``cells_table``: ``"auto"`` (detected from the header), ``"codex"``, ``"qupath"`` (QuPath + measurements; µm centroids are converted with the region's pixel size), ``"mcquant"`` (MCMICRO) or + ``"generic"`` (columns given by ``cells_table_kwargs``). labels - Not implemented yet. + A segmentation mask (TIFF/OME-TIFF with integer instance ids) per region: a path for a single region, or a + mapping from region id to path. Each mask is stored lazily as the labels ``_labels`` + with the transformation of the region's image; the cells of the region annotate it instead of circles, and + their instance ids must be the mask values. imread_kwargs Keyword arguments passed to :func:`dask_image.imread.imread`, which is only used for ``{vx.IMAGE_TIF}(f)`` files that :mod:`tifffile` cannot read lazily (e.g. unsupported compression). image_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. labels_models_kwargs - Not implemented yet. + Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`. cells_table_kwargs - Not implemented yet. + Options for reading ``cells_table`` and ``labels``: ``region_column``; for ``"generic"`` tables ``x``, + ``y``, ``instance_id``, ``markers`` (list or regular expression) and ``coordinates_unit`` (``"px"`` or + ``"um"``); ``mask_channel`` (index or OME channel name of the mask in a multi-channel ``labels`` file). Returns ------- :class:`spatialdata.SpatialData` """ - if cells_table is not None or labels is not None: - raise NotImplementedError("Reading cell tables and labels with `codex` is added in a later task.") - + regions = [regions] if isinstance(regions, str) else regions + channel_names = [channel_names] if isinstance(channel_names, str) else channel_names path = Path(path) layout = discover_legacy(path, fcs) - selected = select_regions(sorted(layout.images.keys() | layout.tables.keys() - {"*"}), regions) + # regions that only have a discovered table exist only while the discovered tables are read + table_regions = layout.tables.keys() - {"*"} if cells_table is None else set() + selected = select_regions(sorted(layout.images.keys() | table_regions), regions) + table_files = resolve_per_region(cells_table, selected) + mask_files = resolve_per_region(labels, selected) experiment = read_json_object(layout.experiment_json) if layout.experiment_json is not None else None names_txt = _read_channel_names_txt(layout.channel_names_txt) if layout.channel_names_txt is not None else None @@ -203,12 +219,34 @@ def codex( ) for scan in scans } - table_files = {key: file for key, file in layout.tables.items() if key == "*" or key in selected} - cell_tables = _read_tables(table_files, layout, selected if regions is not None else None, fcs) - tables, shapes = build_cells(list(cell_tables.values()), {}, {}, transformations) if cell_tables else ({}, {}) - sdata = SpatialData(images=images, shapes=shapes, tables=tables) - sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value] = scans_to_attrs(scans, names) - sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value]["cell_tables"] = {key: str(table_files[key]) for key in cell_tables} + region_labels = read_region_labels(mask_files, transformations, labels_models_kwargs, cells_table_kwargs) + if table_files: + cell_tables = read_region_tables( + table_files, + cells_table_format, + available=list(layout.images), + selected=selected, + pixel_sizes={scan.region: scan.pixel_size_um for scan in scans}, + cells_table_kwargs=cells_table_kwargs, + ) + table_paths = {key: str(file) for key, file in table_files.items()} + else: + discovered = {key: file for key, file in layout.tables.items() if key == "*" or key in selected} + read = _read_tables(discovered, layout, selected if regions is not None else None, fcs) + cell_tables = list(read.values()) + table_paths = {key: str(discovered[key]) for key in read} + tables, shapes = build_cells(cell_tables, {}, region_labels, transformations) if cell_tables else ({}, {}) + sdata = SpatialData( + images=images, + labels={f"{region}{PhenocyclerKeys.LABELS_SUFFIX}": element for region, element in region_labels.items()}, + shapes=shapes, + tables=tables, + ) + attrs = scans_to_attrs(scans, names) + attrs["cell_tables"] = table_paths + if mask_files: + attrs["labels"] = {region: str(file) for region, file in mask_files.items()} + sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value] = attrs return _set_reader_metadata(sdata, "codex") @@ -316,7 +354,7 @@ def read_legacy_image( if len(files) != 1: raise ValueError(f"Expected per-marker images or a single hyperstack, got {[str(f) for f in files]}.") - image, leading_shape, labels, read_by_tifffile = _read_tiff(files[0], imread_kwargs, chunks) + image, leading_shape, labels, read_by_tifffile = read_tiff(files[0], imread_kwargs, chunks) _warn_unused_imread_kwargs(imread_kwargs, read_by_tifffile) n_planes = image.shape[0] # the last non-spatial axis holds the channels of a cycle; ImageJ axis labels are unreliable (cycles as Z) @@ -348,7 +386,7 @@ def _read_per_marker( planes = [] read_by_tifffile = False for _, _, _, file in parsed: - plane, _, _, lazy = _read_tiff(file, imread_kwargs, chunks) + plane, _, _, lazy = read_tiff(file, imread_kwargs, chunks) read_by_tifffile |= lazy if plane.shape[0] != 1: raise ValueError(f"{file} holds {plane.shape[0]} planes; expected a single 2-D marker image.") @@ -363,75 +401,11 @@ def _read_per_marker( return da.stack(planes), channels, read_by_tifffile -def _read_tiff( - path: Path, imread_kwargs: Mapping[str, Any], chunks: tuple[int, int] -) -> tuple[da.Array, tuple[int, ...], list[str] | None, bool]: - """Lazily read a tiff as ``(c, y, x)``, all non-spatial axes flattened into ``c``. - - Uncompressed contiguous data (e.g. ImageJ hyperstacks and per-marker images, often a single strip per plane) - is memory-mapped and split into ``chunks``, so a chunk never needs a whole plane in memory; other layouts keep - their native tiles via zarr. Returns the image, the shape of the flattened axes, the ImageJ ``Labels`` (if - any) and whether tifffile, rather than the :func:`dask_image.imread.imread` fallback, read the file. The - file handle stays open for as long as the array needs it. - """ - tf = tifffile.TiffFile(path) - series = tf.series[0] - axes = series.axes - metadata = tf.imagej_metadata or {} - labels = list(metadata["Labels"]) if isinstance(metadata.get("Labels"), list | tuple) else None - read_by_tifffile = True - if series.dataoffset is not None: - tf.close() - source = _ContiguousTiff(path, series.dataoffset, series.shape, series.dtype) - tiles = dict.fromkeys(range(source.ndim), 1) | {axes.index("Y"): chunks[0], axes.index("X"): chunks[1]} - image = da.from_array(source, chunks=tuple(tiles[i] for i in range(source.ndim)), name=source.name) - else: - try: - image = da.from_zarr(zarr.open(series.aszarr(), mode="r")) - except ValueError as e: # e.g. a compression tifffile cannot decode chunk-wise - tf.close() - logger.info("Reading %s with dask_image.imread because tifffile cannot read it lazily: %s", path, e) - image = imread(str(path), **imread_kwargs) - axes = "Q" * (image.ndim - 2) + "YX" - labels = None - read_by_tifffile = False - image = da.moveaxis(image, [axes.index("Y"), axes.index("X")], [-2, -1]) - leading_shape = tuple(image.shape[:-2]) - return image.reshape((math.prod(leading_shape), *image.shape[-2:])), leading_shape, labels, read_by_tifffile - - -class _ContiguousTiff: - """Array-like view of uncompressed contiguous tiff data, memory-mapped anew for every chunk. - - Handing an ``np.memmap`` to :func:`dask.array.from_array` would read the whole file (dask hashes and slices - numpy arrays eagerly), so dask only gets this lightweight object and a name derived from the file. - """ - - def __init__(self, path: Path, offset: int, shape: tuple[int, ...], dtype: np.dtype[Any]) -> None: - self.path = path - self.offset = offset - self.shape = shape - self.ndim = len(shape) - self._file_dtype = dtype - self.dtype = dtype.newbyteorder("=") - self.name = f"codex-tiff-{tokenize(str(path), path.stat().st_mtime_ns, offset, shape, str(dtype))}" - - def __getitem__(self, key: Any) -> np.ndarray: - data = np.memmap(self.path, dtype=self._file_dtype, mode="r", offset=self.offset, shape=self.shape) - return np.asarray(data[key], dtype=self.dtype) - - def _spatial_chunks(image_models_kwargs: Mapping[str, Any]) -> tuple[int, int]: """The ``(y, x)`` chunk size requested in ``image_models_kwargs``, with the reader default when unspecified.""" default, _ = _initialize_raster_models_kwargs({}, {}) chunks = _initialize_raster_models_kwargs(image_models_kwargs, {})[0]["chunks"] - if isinstance(chunks, int): - return chunks, chunks - if isinstance(chunks, Mapping) and isinstance(chunks.get("y"), int) and isinstance(chunks.get("x"), int): - return chunks["y"], chunks["x"] - if isinstance(chunks, list | tuple) and len(chunks) >= 2 and all(isinstance(c, int) for c in chunks[-2:]): - return chunks[-2], chunks[-1] - return default["chunks"][-2], default["chunks"][-1] + return spatial_chunks(chunks, default["chunks"][-2:]) def _warn_unused_imread_kwargs(imread_kwargs: Mapping[str, Any], read_by_tifffile: bool) -> None: @@ -526,7 +500,7 @@ def _discover_tables(files: Sequence[Path], fcs: bool) -> dict[str, Path]: continue region = _region_id(file.name) # without a region in its name, a CSV may be any sidecar (e.g. an antibody list) - if region is None and not _is_cell_table(file): + if region is None and detect_table_format(file) != "codex": continue candidates.setdefault(region or "*", []).append(file) @@ -547,19 +521,6 @@ def rank(file: Path) -> tuple[bool, bool]: return tables -def _is_cell_table(path: Path) -> bool: - """Whether ``path`` is an FCS file or a CSV whose header has CODEX cell-table columns.""" - if path.suffix.lower() == CodexKeys.FCS_FILE.value: - return True - with path.open(encoding="utf-8-sig", errors="replace") as f: - header = f.readline() - # the name part of the Processor header form `cyc001_ch001:DAPI1` - columns = {column.strip().strip('"').partition(":")[0].casefold() for column in header.split(",")} - return {"x", "y"} <= columns and ( - CodexKeys.INSTANCE_KEY.value in columns or any(_MARKER_COLUMN.fullmatch(column) for column in columns) - ) - - def _find_sidecar(path: Path, files: Sequence[Path], name: str) -> Path | None: """The shallowest file called ``name`` in ``files``, else in the nearest ancestor of ``path`` holding one.""" matches = sorted((file for file in files if file.name == name), key=lambda file: (len(file.parts), str(file))) diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 2eb7893f..f301601b 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -20,6 +20,12 @@ scans_to_attrs, select_regions, ) +from spatialdata_io.readers._utils._cell_tables import ( + build_cells, + read_region_labels, + read_region_tables, + resolve_per_region, +) from spatialdata_io.readers._utils._qptiff import ( _fusion_profile_records, _is_blank, @@ -74,6 +80,15 @@ def phenocycler( the ``"global"`` coordinate system; otherwise all regions sit at its origin. The per-region metadata is stored in ``sdata.attrs[{vx.ATTRS_KEY!r}]``. + A segmentation mask given in ``labels`` is stored lazily as the labels ``{vx.LABELS_SUFFIX}`` with the + transformation of the region's image. The cells of ``cells_table`` are stored in one table + ``{vx.TABLE_KEY!r}`` (marker intensities dense float32 in ``X``, pixel centroids in ``obsm["spatial"]``, + ``obs[{vx.REGION_KEY!r}]`` the annotated element, ``obs[{vx.INSTANCE_KEY!r}]`` the instance key), or in one + table ``_table`` per region if the regions have different markers. Cells annotate the labels of their + region, whose values must be their instance ids; without a mask they are stored as circles + ``{vx.CELLS_SUFFIX}``, with a warning. The paths of the tables and masks are stored in + ``sdata.attrs[{vx.ATTRS_KEY!r}]["cell_tables"]`` and ``["labels"]``. + .. seealso:: - `Akoya PhenoCycler-Fusion `_. @@ -88,28 +103,37 @@ def phenocycler( Channel names to use instead of the ones stored in the qptiff; must have one entry per channel of every region read. cells_table - Not implemented yet. + A cell table: a path, or a mapping from region id to path, e.g. ``{{"A": "A_cells.csv"}}``. A single path + for several regions needs a region column (``Image`` for QuPath, ``region`` for CODEX tables, else + ``cells_table_kwargs["region_column"]``); its values are matched to the region ids, e.g. + ``A_Scan1.qptiff`` to ``A``. cells_table_format - Not implemented yet. + The format of ``cells_table``: ``"auto"`` (detected from the header), ``"codex"`` (CODEX Processor), + ``"qupath"`` (QuPath measurements; µm centroids are converted with the region's pixel size), ``"mcquant"`` + (MCMICRO) or ``"generic"`` (columns given by ``cells_table_kwargs``). labels - Not implemented yet. + A segmentation mask (TIFF/OME-TIFF with integer instance ids) per region: a path for a single region, or a + mapping from region id to path. image_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. labels_models_kwargs - Not implemented yet. + Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`. cells_table_kwargs - Not implemented yet. + Options for reading ``cells_table`` and ``labels``: ``region_column``; for ``"generic"`` tables ``x``, + ``y``, ``instance_id``, ``markers`` (list or regular expression) and ``coordinates_unit`` (``"px"`` or + ``"um"``); ``mask_channel`` (index or OME channel name of the mask in a multi-channel ``labels`` file). Returns ------- :class:`spatialdata.SpatialData` """ - if cells_table is not None or labels is not None: - raise NotImplementedError("Reading cell tables and labels with `phenocycler` is added in a later task.") - + regions = [regions] if isinstance(regions, str) else regions + channel_names = [channel_names] if isinstance(channel_names, str) else channel_names path = Path(path) qptiffs = discover_qptiffs(path) selected = select_regions(list(qptiffs), regions) + table_files = resolve_per_region(cells_table, selected) + mask_files = resolve_per_region(labels, selected) scans = [read_qptiff_metadata(qptiffs[region], region=region) for region in selected] names = {} @@ -128,8 +152,28 @@ def phenocycler( ) for scan in scans } - sdata = SpatialData(images=images) - sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value] = scans_to_attrs(scans, names) + region_labels = read_region_labels(mask_files, transformations, labels_models_kwargs, cells_table_kwargs) + cell_tables = read_region_tables( + table_files, + cells_table_format, + available=list(qptiffs), + selected=selected, + pixel_sizes={scan.region: scan.pixel_size_um for scan in scans}, + cells_table_kwargs=cells_table_kwargs, + ) + tables, shapes = build_cells(cell_tables, {}, region_labels, transformations) if cell_tables else ({}, {}) + sdata = SpatialData( + images=images, + labels={f"{region}{PhenocyclerKeys.LABELS_SUFFIX}": element for region, element in region_labels.items()}, + shapes=shapes, + tables=tables, + ) + attrs = scans_to_attrs(scans, names) + if table_files: + attrs["cell_tables"] = {key: str(file) for key, file in table_files.items()} + if mask_files: + attrs["labels"] = {region: str(file) for region, file in mask_files.items()} + sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value] = attrs return _set_reader_metadata(sdata, "phenocycler") diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index ca444b9a..7a60288c 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -5,11 +5,20 @@ import numpy as np import pandas as pd import pytest +import tifffile +from spatialdata.models import get_axes_names from spatialdata.transformations import Identity, Translation, get_transformation from tests._akoya_data import write_fcs from spatialdata_io.readers._utils._akoya import dedupe_channel_names -from spatialdata_io.readers._utils._cell_tables import build_cells, read_codex_table +from spatialdata_io.readers._utils._cell_tables import ( + build_cells, + detect_table_format, + read_cell_table, + read_codex_table, + read_labels, + resolve_per_region, +) if TYPE_CHECKING: from pathlib import Path @@ -157,3 +166,243 @@ def test_build_cells_overlapping_regions_raise(tmp_path: Path, different_panels: with pytest.raises(ValueError, match="Several cell tables"): build_cells(tables, {}, {}, {}) + + +_QUPATH_HEADER = ( + "Image\tObject ID\tName\tClass\tCentroid X µm\tCentroid Y µm\tCell: Area µm^2\t" + "CD3: Nucleus: Mean\tCD3: Cell: Mean\tCD3: Cell: Median\tCD8: Nucleus: Mean" +) + + +def _write_qupath(path: Path, image: str = "S_Scan1.qptiff") -> None: + rows = [ + f"{image}\t0a1b\tcell\tTumor\t10\t5\t3.14159\t1\t2\t3\t4", + f"{image}\t0a1c\tcell\tStroma\t20\t15\t12.56637\t5\t6\t7\t8", + ] + path.write_text("\n".join([_QUPATH_HEADER, *rows]) + "\n", encoding="utf-8") + + +def _write_mcquant(path: Path, n: int = 5) -> pd.DataFrame: + rng = np.random.default_rng(1) + cells = pd.DataFrame( + { + "CellID": np.arange(1, n + 1), + "DAPI": rng.uniform(0, 100, n), + "CD3": rng.uniform(0, 100, n), + "X_centroid": rng.uniform(0, 60, n), + "Y_centroid": rng.uniform(0, 40, n), + "Area": rng.uniform(10, 50, n), + "Eccentricity": rng.uniform(0, 1, n), + } + ) + cells.to_csv(path, index=False) + return cells + + +@pytest.mark.parametrize( + ("header", "expected"), + [ + ("cell_id,region,x,y,size,cyc001_ch001:DAPI1\n", "codex"), + (_QUPATH_HEADER + "\n", "qupath"), + ("CellID,DAPI,CD3,X_centroid,Y_centroid,Area\n", "mcquant"), + ("ID,DAPI,CD3\n", "sprm"), + ("Path\tSample Name\tCell ID\tCell X Position\tCell Y Position\n", "inform"), + ("label,cx,cy,CD3\n", "generic"), + ], +) +def test_detect_formats(tmp_path: Path, header: str, expected: str) -> None: + path = tmp_path / "cells.csv" + path.write_text(header + "\n", encoding="utf-8") + + assert detect_table_format(path) == expected + + +def test_detect_fcs_is_codex(tmp_path: Path) -> None: + write_fcs(tmp_path / "cells.fcs", _cells()) + + assert detect_table_format(tmp_path / "cells.fcs") == "codex" + + +def test_qupath_um_to_px(tmp_path: Path) -> None: + path = tmp_path / "measurements.tsv" + _write_qupath(path) + + table = read_cell_table(path, "qupath", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) + + np.testing.assert_allclose(table.centroids_px, [[20, 10], [40, 30]]) + assert list(table.var.index) == ["CD3", "CD8"] + assert list(table.var["compartment"]) == ["Cell", "Nucleus"] + assert list(table.var["source_column"]) == ["CD3: Cell: Mean", "CD8: Nucleus: Mean"] + np.testing.assert_array_equal(table.X, np.array([[2, 4], [6, 8]], dtype=np.float32)) + # the other measurements, including the unused compartment, stay in obs + assert {"Class", "Object ID", "CD3: Nucleus: Mean", "CD3: Cell: Median"} <= set(table.obs.columns) + # Object IDs are UUIDs, so the instance ids are row numbers + assert list(table.instance_id) == [1, 2] + assert list(table.region) == ["S_Scan1.qptiff", "S_Scan1.qptiff"] + # areas of pi and 4 pi µm^2 at 0.5 µm/px are circles of radius 2 and 4 px + assert table.radius_px is not None + np.testing.assert_allclose(table.radius_px, [2, 4], rtol=1e-5) + + +def test_qupath_without_pixel_size_raises(tmp_path: Path) -> None: + path = tmp_path / "measurements.tsv" + _write_qupath(path) + + with pytest.raises(ValueError, match="pixel size"): + read_cell_table(path, "qupath", pixel_size_um=None, default_region="S", cells_table_kwargs={}) + + +def test_mcquant_columns(tmp_path: Path) -> None: + path = tmp_path / "cells.csv" + cells = _write_mcquant(path) + + table = read_cell_table(path, "auto", pixel_size_um=None, default_region="S", cells_table_kwargs={}) + + assert list(table.var.index) == ["DAPI", "CD3"] + np.testing.assert_allclose(table.X, cells[["DAPI", "CD3"]], rtol=1e-6) + np.testing.assert_allclose(table.centroids_px, cells[["X_centroid", "Y_centroid"]]) + assert list(table.instance_id) == [1, 2, 3, 4, 5] + assert list(table.region.unique()) == ["S"] + assert "Eccentricity" in table.obs.columns + assert table.radius_px is not None + np.testing.assert_allclose(table.radius_px, np.sqrt(cells["Area"] / np.pi)) + + +def test_mcquant_without_region_raises(tmp_path: Path) -> None: + path = tmp_path / "cells.csv" + _write_mcquant(path) + + with pytest.raises(ValueError, match="needs a region column"): + read_cell_table(path, "mcquant", pixel_size_um=None, default_region=None, cells_table_kwargs={}) + + +def test_generic_kwargs(tmp_path: Path) -> None: + path = tmp_path / "cells.csv" + path.write_text("label,cx,cy,sample,CD3,CD8,score\n7,2,4,A,1,2,0.5\n9,6,8,B,3,4,0.7\n") + kwargs = { + "x": "cx", + "y": "cy", + "instance_id": "label", + "region_column": "sample", + "markers": "CD.*", + "coordinates_unit": "um", + } + + table = read_cell_table(path, "generic", pixel_size_um=2.0, default_region=None, cells_table_kwargs=kwargs) + + np.testing.assert_allclose(table.centroids_px, [[1, 2], [3, 4]]) + assert list(table.var.index) == ["CD3", "CD8"] + assert list(table.instance_id) == [7, 9] + assert list(table.region) == ["A", "B"] + assert "score" in table.obs.columns + assert table.radius_px is None + + listed = read_cell_table( + path, + "generic", + pixel_size_um=None, + default_region="S", + cells_table_kwargs={"x": "cx", "y": "cy", "markers": ["CD8"]}, + ) + assert list(listed.var.index) == ["CD8"] + assert list(listed.instance_id) == [1, 2] + + +def test_generic_bad_kwargs_raise(tmp_path: Path) -> None: + path = tmp_path / "cells.csv" + path.write_text("cx,cy,CD3\n1,2,3\n") + + with pytest.raises(ValueError, match="'x'"): + read_cell_table(path, "generic", pixel_size_um=None, default_region="S", cells_table_kwargs={}) + with pytest.raises(ValueError, match="unknown"): + read_cell_table(path, "generic", pixel_size_um=None, default_region="S", cells_table_kwargs={"xx": "cx"}) + with pytest.raises(ValueError, match="nope"): + read_cell_table( + path, "generic", pixel_size_um=None, default_region="S", cells_table_kwargs={"x": "cx", "y": "nope"} + ) + + +@pytest.mark.parametrize("fmt", ["sprm", "inform"]) +def test_sprm_inform_not_implemented(tmp_path: Path, fmt: str) -> None: + path = tmp_path / "cells.csv" + path.write_text("ID,DAPI\n1,2\n") + + with pytest.raises(NotImplementedError, match="later release"): + read_cell_table(path, fmt, pixel_size_um=None, default_region="S", cells_table_kwargs={}) + + +def test_unknown_format_raises(tmp_path: Path) -> None: + path = tmp_path / "cells.csv" + path.write_text("ID,DAPI\n1,2\n") + + with pytest.raises(ValueError, match="qupath"): + read_cell_table(path, "excel", pixel_size_um=None, default_region="S", cells_table_kwargs={}) + + +def test_resolve_per_region(tmp_path: Path) -> None: + a, b = tmp_path / "a.csv", tmp_path / "b.csv" + a.touch() + b.touch() + + assert resolve_per_region(None, ["A"]) == {} + assert resolve_per_region(str(a), ["A"]) == {"A": a} + assert resolve_per_region(a, ["A", "B"]) == {"*": a} + assert resolve_per_region({"B": str(b), "A": a}, ["A", "B"]) == {"A": a, "B": b} + with pytest.raises(ValueError, match="'C'"): + resolve_per_region({"C": a}, ["A", "B"]) + with pytest.raises(FileNotFoundError, match="missing.csv"): + resolve_per_region(tmp_path / "missing.csv", ["A"]) + + +def test_read_labels_lazy_with_transformation(tmp_path: Path) -> None: + mask = np.zeros((40, 60), dtype=np.uint32) + mask[5:10, 5:10] = 1 + path = tmp_path / "mask.tif" + tifffile.imwrite(path, mask) + translation = Translation([3, 4], axes=("x", "y")) + + labels = read_labels(path, {"scale_factors": [2]}, translation) + + level0 = labels["scale0"]["image"] + assert get_axes_names(labels) == ("y", "x") + np.testing.assert_array_equal(level0.values, mask) + assert get_transformation(labels, "global") == translation + + +@pytest.mark.parametrize(("mask_channel", "expected"), [(None, 0), (1, 1), ("nuclei", 1)]) +def test_read_labels_ome_channels(tmp_path: Path, mask_channel: int | str | None, expected: int) -> None: + masks = np.stack([np.full((16, 24), i + 1, dtype=np.int32) for i in range(2)]) + path = tmp_path / "mask.ome.tif" + tifffile.imwrite(path, masks, metadata={"axes": "CYX", "Channel": {"Name": ["cells", "nuclei"]}}) + + labels = read_labels(path, {"scale_factors": None}, Identity(), mask_channel=mask_channel) + + np.testing.assert_array_equal(labels.values, masks[expected]) + + +def test_read_labels_errors(tmp_path: Path) -> None: + path = tmp_path / "mask.tif" + tifffile.imwrite(path, np.zeros((8, 8), dtype=np.float32)) + with pytest.raises(ValueError, match="integer"): + read_labels(path, {}, Identity()) + + tifffile.imwrite(path, np.zeros((2, 8, 8), dtype=np.uint16)) + with pytest.raises(ValueError, match="channel"): + read_labels(path, {}, Identity(), mask_channel=5) + + +def test_read_labels_pyramidal_ome_is_lazy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + mask = np.repeat(np.arange(64, dtype=np.uint16), 48).reshape(64, 48) + path = tmp_path / "mask.ome.tif" + with tifffile.TiffWriter(path) as tw: + tw.write(mask, subifds=1, tile=(16, 16), compression="zlib", metadata={"axes": "YX"}) + tw.write(mask[::2, ::2], subfiletype=1, tile=(16, 16), compression="zlib") + + def fail(*args: object, **kwargs: object) -> None: + raise AssertionError("full page decode during read") + + monkeypatch.setattr(tifffile.TiffPage, "asarray", fail) + labels = read_labels(path, {"scale_factors": None}, Identity()) + + assert labels.shape == (64, 48) + np.testing.assert_array_equal(labels.values, mask) diff --git a/tests/test_codex.py b/tests/test_codex.py index 56750950..1af8f2cb 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING import numpy as np +import pandas as pd import pytest import tifffile from spatialdata.models import get_channel_names @@ -161,10 +162,51 @@ def test_discover_legacy_tables_prefer_requested_format(tmp_path: Path) -> None: } -def test_codex_not_implemented_cells_table(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path, layout="per_marker") - with pytest.raises(NotImplementedError): - codex(tmp_path, cells_table=tmp_path / "x.csv") +def test_codex_user_cells_table_replaces_discovered(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, regions=2, layout="per_marker", n_cells=50) + pd.DataFrame( + {"CellID": [1, 2, 3], "CD3": [1.0, 2.0, 3.0], "X_centroid": [1.0, 2.0, 3.0], "Y_centroid": [4.0, 5.0, 6.0]} + ).to_csv(tmp_path / "mine.csv", index=False) + mask = np.zeros((128, 160), dtype=np.uint16) + mask[:3, :3] = [1, 2, 3] + tifffile.imwrite(tmp_path / "mask.tif", mask) + + sdata = codex( + tmp_path, + cells_table={"reg002": tmp_path / "mine.csv"}, + labels={"reg002": tmp_path / "mask.tif"}, + image_models_kwargs=_FAST, + labels_models_kwargs=_FAST, + ) + + table = sdata.tables["table"] + assert table.n_obs == 3 + assert list(table.var_names) == ["CD3"] + assert table.uns["spatialdata_attrs"]["region"] == ["reg002_labels"] + assert list(sdata.labels) == ["reg002_labels"] + assert len(sdata.shapes) == 0 + assert sdata.attrs["akoya"]["cell_tables"] == {"reg002": str(tmp_path / "mine.csv")} + + +def test_codex_labels_with_discovered_tables(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, regions=1, layout="per_marker", n_cells=5) + tifffile.imwrite(tmp_path / "mask.tif", np.zeros((128, 160), dtype=np.uint16)) + + sdata = codex(tmp_path, labels=tmp_path / "mask.tif", image_models_kwargs=_FAST, labels_models_kwargs=_FAST) + + assert list(sdata.labels) == ["reg001_labels"] + assert len(sdata.shapes) == 0 + assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["reg001_labels"] + + +def test_codex_str_regions_and_channel_names(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, regions=2, cycles=1, channels=1, layout="per_marker", n_cells=5) + + with pytest.warns(UserWarning, match="circles"): + sdata = codex(tmp_path, regions="reg002", channel_names="nuclei", image_models_kwargs=_FAST) + + assert list(sdata.images) == ["reg002_image"] + assert get_channel_names(sdata.images["reg002_image"]) == ["nuclei"] @pytest.mark.parametrize("imagej", [False, True]) diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index ebb6416d..b29a244d 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -8,12 +8,15 @@ import dask.array as da import numpy as np +import pandas as pd import pytest import tifffile +from click.testing import CliRunner from spatialdata import read_zarr from spatialdata.models import get_channel_names from spatialdata_io import phenocycler +from spatialdata_io.__main__ import phenocycler_wrapper from tests._akoya_data import write_qptiff _FAST = {"scale_factors": [2]} @@ -162,14 +165,200 @@ def test_channel_names_override(tmp_path: Path) -> None: phenocycler(path, channel_names=["only one"]) -def test_cells_table_and_labels_not_implemented(tmp_path: Path) -> None: +def _write_mcquant(path: Path, n: int = 5, markers: tuple[str, ...] = ("DAPI", "CD3")) -> None: + rng = np.random.default_rng(0) + cells = pd.DataFrame( + { + "CellID": np.arange(1, n + 1), + **{marker: rng.uniform(0, 100, n) for marker in markers}, + "X_centroid": rng.uniform(0, 300, n), + "Y_centroid": rng.uniform(0, 250, n), + "Area": rng.uniform(10, 50, n), + } + ) + cells.to_csv(path, index=False) + + +def _write_mask(path: Path, n: int = 5, shape: tuple[int, int] = (256, 320)) -> np.ndarray: + mask = np.zeros(shape, dtype=np.uint32) + for i in range(n): + mask[10 + 20 * i : 20 + 20 * i, 10:20] = i + 1 + tifffile.imwrite(path, mask) + return mask + + +def test_mcquant_with_mask_labels(tmp_path: Path) -> None: + path = tmp_path / "S_Scan1.qptiff" + write_qptiff(path, biomarkers=["DAPI", "CD3"], stage_position_cm=(1.0, 2.0)) + _write_mcquant(tmp_path / "cells.csv") + mask = _write_mask(tmp_path / "mask.tif") + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + sdata = phenocycler( + path, + cells_table=tmp_path / "cells.csv", + labels=tmp_path / "mask.tif", + image_models_kwargs=_FAST, + labels_models_kwargs=_FAST, + ) + + assert list(sdata.labels) == ["S_labels"] + assert len(sdata.shapes) == 0 + np.testing.assert_array_equal(sdata.labels["S_labels"]["scale0"]["image"].values, mask) + table = sdata.tables["table"] + assert set(table.obs["region"]) == {"S_labels"} + assert table.uns["spatialdata_attrs"]["region"] == ["S_labels"] + assert list(table.obs["cell_id"]) == [1, 2, 3, 4, 5] + assert list(table.var_names) == ["DAPI", "CD3"] + assert sdata.attrs["akoya"]["cell_tables"] == {"S": str(tmp_path / "cells.csv")} + assert sdata.attrs["akoya"]["labels"] == {"S": str(tmp_path / "mask.tif")} + + +def test_labels_without_cells_table(tmp_path: Path) -> None: path = tmp_path / "S_Scan1.qptiff" write_qptiff(path, biomarkers=["DAPI"]) + _write_mask(tmp_path / "mask.tif") + + sdata = phenocycler(path, labels=str(tmp_path / "mask.tif"), image_models_kwargs=_FAST) + + assert list(sdata.labels) == ["S_labels"] + assert len(sdata.tables) == 0 + + +def test_cells_table_without_mask_makes_circles(tmp_path: Path) -> None: + path = tmp_path / "S_Scan1.qptiff" + write_qptiff(path, biomarkers=["DAPI", "CD3"]) + _write_mcquant(tmp_path / "cells.csv") + + with pytest.warns(UserWarning, match="circles"): + sdata = phenocycler(path, cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) + + assert list(sdata.shapes) == ["S_cells"] + assert set(sdata.tables["table"].obs["region"]) == {"S_cells"} + - with pytest.raises(NotImplementedError): - phenocycler(path, cells_table=tmp_path / "cells.csv") - with pytest.raises(NotImplementedError): - phenocycler(path, labels=tmp_path / "mask.tif") +def test_phenocycler_cells_mapping_per_region(tmp_path: Path) -> None: + for region in ("A", "B"): + write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) + _write_mcquant(tmp_path / f"{region}.csv") + _write_mask(tmp_path / f"{region}_mask.tif") + + sdata = phenocycler( + tmp_path, + cells_table={"A": tmp_path / "A.csv", "B": str(tmp_path / "B.csv")}, + labels={"A": tmp_path / "A_mask.tif", "B": tmp_path / "B_mask.tif"}, + image_models_kwargs=_FAST, + labels_models_kwargs=_FAST, + ) + + assert list(sdata.tables) == ["table"] + table = sdata.tables["table"] + assert table.n_obs == 10 + assert table.obs_names.is_unique + assert table.uns["spatialdata_attrs"]["region"] == ["A_labels", "B_labels"] + assert sorted(sdata.labels) == ["A_labels", "B_labels"] + + +def test_mapping_with_unknown_region_raises(tmp_path: Path) -> None: + write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI"]) + _write_mcquant(tmp_path / "A.csv", markers=("DAPI",)) + + with pytest.raises(ValueError, match="'Z'"): + phenocycler(tmp_path, cells_table={"Z": tmp_path / "A.csv"}) + + +def test_single_table_many_regions_needs_region_column(tmp_path: Path) -> None: + for region in ("A", "B"): + write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) + _write_mcquant(tmp_path / "cells.csv") + + with pytest.raises(ValueError, match="needs a region column"): + phenocycler(tmp_path, cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) + + +def test_single_qupath_table_many_regions(tmp_path: Path) -> None: + for region in ("A", "B"): + write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) + header = "Image\tObject ID\tCentroid X µm\tCentroid Y µm\tCD3: Cell: Mean" + rows = ["A_Scan1.qptiff\tu1\t5\t5\t1", "B_Scan1.qptiff\tu2\t10\t10\t2", "C_Scan1.qptiff\tu3\t1\t1\t3"] + (tmp_path / "cells.tsv").write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") + + with pytest.warns(UserWarning, match="C_Scan1.qptiff"): + sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) + + table = sdata.tables["table"] + assert list(table.obs["region"]) == ["A_cells", "B_cells"] + np.testing.assert_allclose(np.asarray(table.obsm["spatial"]), np.array([[5, 5], [10, 10]]) / 0.5069) + # QuPath column names are not valid spatialdata names, so they are sanitized and the result can be written + assert {"Image", "Object_ID", "Centroid_X_µm"} <= set(table.obs.columns) + sdata.write(tmp_path / "x.zarr") + assert read_zarr(tmp_path / "x.zarr").tables["table"].n_obs == 2 + + +def test_different_panels_make_per_region_tables(tmp_path: Path) -> None: + write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) + write_qptiff(tmp_path / "B_Scan1.qptiff", biomarkers=["DAPI", "CD8", "CD4"]) + _write_mcquant(tmp_path / "A.csv", markers=("DAPI", "CD3")) + _write_mcquant(tmp_path / "B.csv", markers=("DAPI", "CD8", "CD4")) + + with pytest.warns(UserWarning, match="circles"): + sdata = phenocycler( + tmp_path, cells_table={"A": tmp_path / "A.csv", "B": tmp_path / "B.csv"}, image_models_kwargs=_FAST + ) + + assert sorted(sdata.tables) == ["A_table", "B_table"] + assert list(sdata.tables["B_table"].var_names) == ["DAPI", "CD8", "CD4"] + assert sdata.tables["A_table"].uns["spatialdata_attrs"]["region"] == ["A_cells"] + + +def test_single_mask_many_regions_raises(tmp_path: Path) -> None: + for region in ("A", "B"): + write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI"]) + _write_mask(tmp_path / "mask.tif") + + with pytest.raises(ValueError, match="mapping"): + phenocycler(tmp_path, labels=tmp_path / "mask.tif") + + +def test_cli_cells_table_and_labels_mappings(runner: CliRunner, tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) + _write_mcquant(tmp_path / "cells.csv") + _write_mask(tmp_path / "mask.tif") + output = tmp_path / "out.zarr" + + result = runner.invoke( + phenocycler_wrapper, + [ + "--input", + str(tmp_path / "S_Scan1.qptiff"), + "--output", + str(output), + "--cells-table", + str(tmp_path / "cells.csv"), + "--labels", + json.dumps({"S": str(tmp_path / "mask.tif")}), + "--image-models-kwargs", + json.dumps(_FAST), + "--labels-models-kwargs", + json.dumps(_FAST), + ], + ) + + assert result.exit_code == 0, result.output + sdata = read_zarr(output) + assert list(sdata.labels) == ["S_labels"] + assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["S_labels"] + + +def test_str_regions_and_channel_names(tmp_path: Path) -> None: + write_qptiff(tmp_path / "AB_Scan1.qptiff", biomarkers=["DAPI"]) + write_qptiff(tmp_path / "C_Scan1.qptiff", biomarkers=["DAPI"]) + + sdata = phenocycler(tmp_path, regions="AB", channel_names="nuclei", image_models_kwargs=_FAST) + + assert list(sdata.images) == ["AB_image"] + assert get_channel_names(sdata.images["AB_image"]) == ["nuclei"] def test_write_roundtrip(tmp_path: Path) -> None: From 3e1151b5ad92637d8d8dbea84dcc0cabe78d0c8e Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 14:45:57 +0200 Subject: [PATCH 011/107] fix: guard reserved obs keys, check masks and relax mapping keys in Akoya cell tables Source columns that clash case-insensitively with `region`/`cell_id` are kept as `source_` instead of crashing table validation. Masks whose shape differs from the region image, and masks paired with row-number instance ids, warn. Mapping keys of existing but unselected regions are ignored. QuPath tables without marker columns warn; malformed CLI mappings are a BadParameter. Co-Authored-By: Claude Opus 5.5 --- src/spatialdata_io/__main__.py | 9 +- .../readers/_utils/_cell_tables.py | 86 +++++++++++++--- src/spatialdata_io/readers/codex.py | 13 ++- src/spatialdata_io/readers/phenocycler.py | 10 +- tests/test_phenocycler.py | 99 +++++++++++++++++++ 5 files changed, 194 insertions(+), 23 deletions(-) diff --git a/src/spatialdata_io/__main__.py b/src/spatialdata_io/__main__.py index 565411a8..43e3b44b 100644 --- a/src/spatialdata_io/__main__.py +++ b/src/spatialdata_io/__main__.py @@ -48,9 +48,12 @@ def _parse_json_param(value: str, param_name: str) -> dict[str, Any]: def _parse_path_or_mapping_param(value: str | None, param_name: str) -> str | dict[str, Any] | None: """A path, or a JSON mapping from region to path if ``value`` starts with ``{``.""" - if value is not None and value.lstrip().startswith("{"): - return _parse_json_param(value, param_name) - return value + if value is None or not value.lstrip().startswith("{"): + return value + mapping = _parse_json_param(value, param_name) + if not isinstance(mapping, dict) or not all(isinstance(path, str) for path in mapping.values()): + raise click.BadParameter(f"{param_name!r} must be a path or a JSON object mapping regions to paths.") + return mapping @cli.command(name="codex") diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 610855af..e8decd53 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -18,6 +18,7 @@ from spatialdata._logging import logger from spatialdata.models import Labels2DModel, ShapesModel, TableModel from spatialdata.transformations import Identity +from xarray import DataTree from spatialdata_io._constants._constants import CodexKeys, PhenocyclerKeys from spatialdata_io.readers._utils._akoya import dedupe_channel_names, read_tiff, spatial_chunks @@ -28,10 +29,11 @@ from geopandas import GeoDataFrame from spatialdata.transformations import BaseTransformation - from xarray import DataArray, DataTree + from xarray import DataArray __all__ = [ "CellTable", + "raster_shape", "build_cells", "detect_table_format", "read_cell_table", @@ -80,6 +82,8 @@ class CellTable: The ``(x, y)`` centroids in level-0 pixels of the region image. radius_px The radius of a circle with the cell's area, or `None` if the table has no area. + row_instance_ids + Whether the instance ids are row numbers because the table has no instance id column. """ obs: pd.DataFrame @@ -89,6 +93,7 @@ class CellTable: instance_id: pd.Series centroids_px: np.ndarray radius_px: np.ndarray | None + row_instance_ids: bool = False def take(self, rows: np.ndarray) -> CellTable: """The cells at the positional indices ``rows``.""" @@ -100,6 +105,7 @@ def take(self, rows: np.ndarray) -> CellTable: instance_id=self.instance_id.iloc[rows], centroids_px=self.centroids_px[rows], radius_px=None if self.radius_px is None else self.radius_px[rows], + row_instance_ids=self.row_instance_ids, ) @@ -195,6 +201,7 @@ def read_codex_table(path: Path, region: str | None = None) -> CellTable: instance_id=instance_id, centroids_px=centroids, radius_px=radius, + row_instance_ids=instance_key not in columns, ) @@ -316,6 +323,12 @@ def _qupath_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> di compartments.setdefault(match.group(1), {})[match.group(2)] = column chosen = {marker: next(c for c in _QUPATH_COMPARTMENTS if c in found) for marker, found in compartments.items()} markers = [compartments[marker][compartment] for marker, compartment in chosen.items()] + if not markers: + warnings.warn( + f"{path} has no ': : Mean' columns, so its table has no marker intensities.", + UserWarning, + stacklevel=5, + ) var = pd.DataFrame( {"source_column": markers, "compartment": list(chosen.values())}, index=pd.Index(dedupe_channel_names(list(chosen))), @@ -434,6 +447,7 @@ def _from_frame( instance_id=instance_id, centroids_px=df[[x, y]].to_numpy(dtype=np.float64) * px_per_unit, radius_px=radius, + row_instance_ids=instance_column is None, ) @@ -445,7 +459,11 @@ def _read_header(path: Path) -> tuple[list[str], str]: return [column.strip().strip('"') for column in header.split(sep)], sep -def resolve_per_region(value: str | Path | Mapping[str, str | Path] | None, regions: Sequence[str]) -> dict[str, Path]: +def resolve_per_region( + value: str | Path | Mapping[str, str | Path] | None, + regions: Sequence[str], + available: Sequence[str] = (), +) -> dict[str, Path]: """Resolve a ``cells_table`` or ``labels`` argument to one file per region. Parameters @@ -454,6 +472,8 @@ def resolve_per_region(value: str | Path | Mapping[str, str | Path] | None, regi A path, a mapping from region id to path, or `None`. regions The regions read. + available + All regions of the dataset; mapping keys that are one of them but not read are ignored. Returns ------- @@ -463,9 +483,9 @@ def resolve_per_region(value: str | Path | Mapping[str, str | Path] | None, regi if value is None: return {} if isinstance(value, Mapping): - unknown = sorted(set(value) - set(regions)) + unknown = sorted(set(value) - set(regions) - set(available)) if unknown: - raise ValueError(f"Unknown region(s) {unknown}. Regions read: {list(regions)}.") + raise ValueError(f"Unknown region(s) {unknown}. Available regions: {list(available or regions)}.") files = {region: Path(value[region]) for region in regions if region in value} else: files = {regions[0] if len(regions) == 1 else "*": Path(value)} @@ -622,22 +642,41 @@ def read_region_labels( transformations: Mapping[str, BaseTransformation], labels_models_kwargs: Mapping[str, Any], cells_table_kwargs: Mapping[str, Any], + image_shapes: Mapping[str, tuple[int, int]], ) -> dict[str, DataArray | DataTree]: - """Read the mask of each region, as resolved by :func:`resolve_per_region`, with the region's transformation.""" + """Read the mask of each region, as resolved by :func:`resolve_per_region`, with the region's transformation. + + A mask whose ``(y, x)`` shape differs from ``image_shapes`` of its region raises a warning: its pixels would + not align with the image. + """ if "*" in files: raise ValueError( f"A single labels mask {files['*']} was given for several regions; a mask belongs to one region image, " "so pass a mapping {region: path}." ) - return { - region: read_labels( + labels = {} + for region, path in files.items(): + labels[region] = read_labels( path, labels_models_kwargs, transformations.get(region, Identity()), mask_channel=cells_table_kwargs.get("mask_channel"), ) - for region, path in files.items() - } + shape, image_shape = raster_shape(labels[region]), image_shapes.get(region) + if image_shape is not None and shape != tuple(image_shape): + warnings.warn( + f"The mask {path} has shape {shape}, but the image of region '{region}' has shape " + f"{tuple(image_shape)} (y, x); the labels will not align with the image.", + UserWarning, + stacklevel=3, + ) + return labels + + +def raster_shape(element: DataArray | DataTree) -> tuple[int, int]: + """The level-0 ``(y, x)`` shape of a raster element.""" + data = element["scale0"]["image"] if isinstance(element, DataTree) else element + return int(data.sizes["y"]), int(data.sizes["x"]) def build_cells( @@ -666,6 +705,16 @@ def build_cells( per region, and the circle shapes by element name. The circles have the cells' area (radius 1 px without one). """ _check_disjoint_regions(tables) + row_ids = sorted( + region for table in tables if table.row_instance_ids for region in table.region.unique() if region in labels + ) + if row_ids: + warnings.warn( + f"The cell table of region(s) {row_ids} has no instance id column, so its row numbers are used as " + "instance ids; they may not match the values of the mask.", + UserWarning, + stacklevel=3, + ) pieces = _split_by_region(tables) if len({tuple(table.var.index) for table in tables}) > 1 else None def element(region: str) -> str: @@ -724,10 +773,18 @@ def _to_anndata(tables: Sequence[CellTable], element: Callable[[str], str]) -> a """Concatenate cell tables with the same markers into one table annotating the elements of their regions.""" region_key, instance_key = CodexKeys.REGION_KEY.value, CodexKeys.INSTANCE_KEY.value obs = pd.concat([table.obs for table in tables]) if len(tables) > 1 else tables[0].obs.copy() + # the region and instance keys are set below; other columns that would clash with them (spatialdata names are + # case-insensitively unique, e.g. `Cell ID` -> `Cell_ID` vs `cell_id`) keep their values under `source_` + reserved = {region_key, instance_key} + obs = obs.drop(columns=[column for column in obs.columns if column in reserved]) + obs = obs.rename( + columns={ + column: f"source_{column}" + for column in obs.columns + if sanitize_name(str(column), is_dataframe_column=True).casefold() in reserved + } + ) regions = pd.concat([table.region for table in tables]) if len(tables) > 1 else tables[0].region - elements = regions.map({region: element(region) for region in regions.unique()}).to_numpy() - obs[region_key] = pd.Categorical(elements, categories=sorted(set(elements))) - obs[instance_key] = np.concatenate([table.instance_id.to_numpy() for table in tables]) if not obs.index.is_unique: obs.index = pd.Index(regions.to_numpy(dtype=object) + "_" + obs.index.to_numpy(dtype=object)) X = np.concatenate([table.X for table in tables]) if len(tables) > 1 else tables[0].X @@ -736,9 +793,12 @@ def _to_anndata(tables: Sequence[CellTable], element: Callable[[str], str]) -> a # e.g. QuPath's `Centroid X µm` -> `Centroid_X_µm`: spatialdata only writes alphanumeric names and `_.-` if any(sanitize_name(str(column), is_dataframe_column=True) != column for column in adata.obs.columns): sanitize_table(adata, inplace=True) + elements = regions.map({region: element(region) for region in regions.unique()}).to_numpy() + adata.obs[region_key] = pd.Categorical(elements, categories=sorted(set(elements))) + adata.obs[instance_key] = np.concatenate([table.instance_id.to_numpy() for table in tables]) return TableModel.parse( adata, - region=list(obs[region_key].cat.categories), + region=list(adata.obs[region_key].cat.categories), region_key=region_key, instance_key=instance_key, ) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 30b5b55d..341d1049 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -30,6 +30,7 @@ CellTable, build_cells, detect_table_format, + raster_shape, read_codex_table, read_region_labels, read_region_tables, @@ -174,9 +175,10 @@ def codex( layout = discover_legacy(path, fcs) # regions that only have a discovered table exist only while the discovered tables are read table_regions = layout.tables.keys() - {"*"} if cells_table is None else set() - selected = select_regions(sorted(layout.images.keys() | table_regions), regions) - table_files = resolve_per_region(cells_table, selected) - mask_files = resolve_per_region(labels, selected) + available = sorted(layout.images.keys() | table_regions) + selected = select_regions(available, regions) + table_files = resolve_per_region(cells_table, selected, available) + mask_files = resolve_per_region(labels, selected, available) experiment = read_json_object(layout.experiment_json) if layout.experiment_json is not None else None names_txt = _read_channel_names_txt(layout.channel_names_txt) if layout.channel_names_txt is not None else None @@ -219,7 +221,10 @@ def codex( ) for scan in scans } - region_labels = read_region_labels(mask_files, transformations, labels_models_kwargs, cells_table_kwargs) + image_shapes = {scan.region: raster_shape(images[f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}"]) for scan in scans} + region_labels = read_region_labels( + mask_files, transformations, labels_models_kwargs, cells_table_kwargs, image_shapes + ) if table_files: cell_tables = read_region_tables( table_files, diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index f301601b..0340016f 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -22,6 +22,7 @@ ) from spatialdata_io.readers._utils._cell_tables import ( build_cells, + raster_shape, read_region_labels, read_region_tables, resolve_per_region, @@ -132,8 +133,8 @@ def phenocycler( path = Path(path) qptiffs = discover_qptiffs(path) selected = select_regions(list(qptiffs), regions) - table_files = resolve_per_region(cells_table, selected) - mask_files = resolve_per_region(labels, selected) + table_files = resolve_per_region(cells_table, selected, list(qptiffs)) + mask_files = resolve_per_region(labels, selected, list(qptiffs)) scans = [read_qptiff_metadata(qptiffs[region], region=region) for region in selected] names = {} @@ -152,7 +153,10 @@ def phenocycler( ) for scan in scans } - region_labels = read_region_labels(mask_files, transformations, labels_models_kwargs, cells_table_kwargs) + image_shapes = {scan.region: raster_shape(images[f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}"]) for scan in scans} + region_labels = read_region_labels( + mask_files, transformations, labels_models_kwargs, cells_table_kwargs, image_shapes + ) cell_tables = read_region_tables( table_files, cells_table_format, diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index b29a244d..6d46c533 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -268,6 +268,93 @@ def test_mapping_with_unknown_region_raises(tmp_path: Path) -> None: phenocycler(tmp_path, cells_table={"Z": tmp_path / "A.csv"}) +def test_mapping_keys_of_unselected_regions_are_ignored(tmp_path: Path) -> None: + for region in ("A", "B"): + write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) + _write_mcquant(tmp_path / f"{region}.csv") + _write_mask(tmp_path / f"{region}_mask.tif") + + sdata = phenocycler( + tmp_path, + regions=["A"], + cells_table={"A": tmp_path / "A.csv", "B": tmp_path / "B.csv"}, + labels={"A": tmp_path / "A_mask.tif", "B": tmp_path / "B_mask.tif"}, + image_models_kwargs=_FAST, + labels_models_kwargs=_FAST, + ) + + assert list(sdata.labels) == ["A_labels"] + assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["A_labels"] + assert sdata.attrs["akoya"]["cell_tables"] == {"A": str(tmp_path / "A.csv")} + + +def test_mask_shape_mismatch_warns(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + _write_mask(tmp_path / "mask.tif", shape=(128, 160)) + + with pytest.warns(UserWarning, match=r"\(128, 160\).*\(256, 320\)"): + sdata = phenocycler(tmp_path, labels=tmp_path / "mask.tif", image_models_kwargs=_FAST) + + assert list(sdata.labels) == ["S_labels"] + + +def test_mask_with_row_number_instance_ids_warns(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + _write_mask(tmp_path / "mask.tif") + (tmp_path / "cells.csv").write_text("cx,cy,DAPI\n1,2,3\n4,5,6\n") + + with pytest.warns(UserWarning, match="row numbers"): + phenocycler( + tmp_path, + cells_table=tmp_path / "cells.csv", + cells_table_format="generic", + labels=tmp_path / "mask.tif", + image_models_kwargs=_FAST, + cells_table_kwargs={"x": "cx", "y": "cy"}, + ) + + +@pytest.mark.parametrize( + ("header", "kwargs"), + [ + ("Cell ID,X,Y,CD3", {"x": "X", "y": "Y", "instance_id": "Cell ID"}), + ("Region,X,Y,CD3", {"x": "X", "y": "Y", "region_column": "Region"}), + ("CELL_ID,region,X,Y,CD3", {"x": "X", "y": "Y", "region_column": "region", "markers": ["CD3"]}), + ], +) +def test_generic_columns_clashing_with_reserved_keys(tmp_path: Path, header: str, kwargs: dict[str, str]) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + rows = "\n".join( + ",".join("S" if col.casefold() == "region" else str(i + 1) for col in header.split(",")) for i in range(2) + ) + (tmp_path / "cells.csv").write_text(f"{header}\n{rows}\n") + + with pytest.warns(UserWarning, match="circles"): + sdata = phenocycler( + tmp_path, + cells_table=tmp_path / "cells.csv", + cells_table_format="generic", + image_models_kwargs=_FAST, + cells_table_kwargs=kwargs, + ) + + table = sdata.tables["table"] + assert list(table.obs["region"]) == ["S_cells", "S_cells"] + assert table.obs["cell_id"].dtype == np.int64 + sdata.write(tmp_path / "x.zarr") + assert read_zarr(tmp_path / "x.zarr").tables["table"].n_obs == 2 + + +def test_qupath_without_markers_warns(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + (tmp_path / "cells.tsv").write_text("Object ID\tCentroid X px\tCentroid Y px\nu1\t1\t2\n", encoding="utf-8") + + with pytest.warns(UserWarning, match="no marker intensities"): + sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) + + assert sdata.tables["table"].n_vars == 0 + + def test_single_table_many_regions_needs_region_column(tmp_path: Path) -> None: for region in ("A", "B"): write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) @@ -351,6 +438,18 @@ def test_cli_cells_table_and_labels_mappings(runner: CliRunner, tmp_path: Path) assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["S_labels"] +@pytest.mark.parametrize("value", ['{"S": 1}', "{not json"]) +def test_cli_bad_mapping_is_bad_parameter(runner: CliRunner, tmp_path: Path, value: str) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + + result = runner.invoke( + phenocycler_wrapper, ["--input", str(tmp_path), "--output", str(tmp_path / "o.zarr"), "--cells-table", value] + ) + + assert result.exit_code == 2 + assert "cells_table" in result.output + + def test_str_regions_and_channel_names(tmp_path: Path) -> None: write_qptiff(tmp_path / "AB_Scan1.qptiff", biomarkers=["DAPI"]) write_qptiff(tmp_path / "C_Scan1.qptiff", biomarkers=["DAPI"]) From 8ccd9ff84cbee23a04c94965958ea3267f1083d6 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 15:06:18 +0200 Subject: [PATCH 012/107] feat: rasterise QuPath cell polygons lazily into labels QuPath GeoJSON cell outlines (cells_table_kwargs['geojson'] or a sibling .geojson / _cells.geojson) become _labels annotated by the table. Features match the table by Object ID (UUID -> row instance id), else in row order. Chunk-wise map_blocks over an STRtree; higher id wins. Co-Authored-By: Claude Opus 5.5 --- .../readers/_utils/_cell_tables.py | 246 ++++++++++++++++- .../readers/_utils/_rasterize.py | 104 +++++++ src/spatialdata_io/readers/codex.py | 20 +- src/spatialdata_io/readers/phenocycler.py | 21 +- tests/readers/test_rasterize.py | 261 ++++++++++++++++++ tests/test_codex.py | 41 +++ 6 files changed, 676 insertions(+), 17 deletions(-) create mode 100644 src/spatialdata_io/readers/_utils/_rasterize.py create mode 100644 tests/readers/test_rasterize.py diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index e8decd53..ed9aec92 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import re import warnings from collections.abc import Mapping @@ -10,9 +11,11 @@ from typing import TYPE_CHECKING, Any import anndata as ad +import geopandas as gpd import numpy as np import pandas as pd import readfcs +import shapely import tifffile from spatialdata import sanitize_name, sanitize_table from spatialdata._logging import logger @@ -22,6 +25,7 @@ from spatialdata_io._constants._constants import CodexKeys, PhenocyclerKeys from spatialdata_io.readers._utils._akoya import dedupe_channel_names, read_tiff, spatial_chunks +from spatialdata_io.readers._utils._rasterize import rasterize_polygons_lazy from spatialdata_io.readers._utils._utils import _initialize_raster_models_kwargs if TYPE_CHECKING: @@ -36,9 +40,11 @@ "raster_shape", "build_cells", "detect_table_format", + "polygon_labels", "read_cell_table", "read_codex_table", "read_labels", + "read_qupath_polygons", "read_region_labels", "read_region_tables", "resolve_per_region", @@ -46,7 +52,17 @@ TABLE_FORMATS = ("codex", "qupath", "mcquant", "sprm", "inform", "generic") # the keys of `cells_table_kwargs`; see `read_cell_table` and `read_labels` -CELLS_TABLE_KWARGS = ("x", "y", "instance_id", "region_column", "markers", "coordinates_unit", "mask_channel") +CELLS_TABLE_KWARGS = ( + "x", + "y", + "instance_id", + "region_column", + "markers", + "coordinates_unit", + "mask_channel", + "geojson", + "geojson_units", +) # `cyc001_ch001` (FCS `$PnN`, plain CSV header) or the name part of the Processor CSV header `cyc001_ch001:DAPI1` _MARKER_COLUMN = re.compile(r"cyc(\d+)_ch(\d+)", re.IGNORECASE) @@ -59,6 +75,8 @@ _QUPATH_COMPARTMENTS = ("Cell", "Nucleus", "Cytoplasm", "Membrane") _QUPATH_IMAGE = "Image" _QUPATH_ID = "Object ID" +# the QuPath object types of cells; other features of a GeoJSON export (e.g. annotations) are not cells +_QUPATH_CELL_TYPES = frozenset({"cell", "detection", "PathCellObject", "PathDetectionObject"}) @dataclass @@ -84,6 +102,11 @@ class CellTable: The radius of a circle with the cell's area, or `None` if the table has no area. row_instance_ids Whether the instance ids are row numbers because the table has no instance id column. + polygons_px + The outline of every cell in level-0 pixels (`None` for a cell without one), or `None` if the table has no + polygons. + geojson + The file of ``polygons_px``. """ obs: pd.DataFrame @@ -94,6 +117,8 @@ class CellTable: centroids_px: np.ndarray radius_px: np.ndarray | None row_instance_ids: bool = False + polygons_px: np.ndarray | None = None + geojson: Path | None = None def take(self, rows: np.ndarray) -> CellTable: """The cells at the positional indices ``rows``.""" @@ -106,6 +131,8 @@ def take(self, rows: np.ndarray) -> CellTable: centroids_px=self.centroids_px[rows], radius_px=None if self.radius_px is None else self.radius_px[rows], row_instance_ids=self.row_instance_ids, + polygons_px=None if self.polygons_px is None else self.polygons_px[rows], + geojson=self.geojson, ) @@ -263,7 +290,9 @@ def read_cell_table( else the row number (from 1); the region from the ``Image`` column. The markers are the ``: : Mean`` columns, one per marker from the first of the ``Cell``, ``Nucleus``, ``Cytoplasm`` and ``Membrane`` compartments (in ``var["compartment"]``). The radius - comes from ``Cell: Area µm^2`` (or ``px^2``). + comes from ``Cell: Area µm^2`` (or ``px^2``). The cell outlines are read from a QuPath GeoJSON + export, see :func:`read_qupath_polygons`: ``cells_table_kwargs["geojson"]``, else + ``
.geojson`` or ``
_cells.geojson`` next to the table, if it exists. - ``"mcquant"``: an MCMICRO ``mcquant`` table. Instance ids ``CellID``, pixel centroids ``X_centroid``/``Y_centroid``, the markers are the columns between ``CellID`` and ``X_centroid``, the radius comes from ``Area``. @@ -278,8 +307,10 @@ def read_cell_table( Optional keys: ``region_column`` (the column holding the region of each cell; QuPath default ``Image``), and for ``"generic"`` tables ``x`` and ``y`` (required centroid columns), ``instance_id`` (else the row number, from 1), ``markers`` (a list of columns or a regular expression matching the full column name; - default: all other numeric columns) and ``coordinates_unit`` (``"px"``, default, or ``"um"``). The key - ``mask_channel`` is used by :func:`read_labels`. + default: all other numeric columns) and ``coordinates_unit`` (``"px"``, default, or ``"um"``); for + ``"qupath"`` tables ``geojson`` (the GeoJSON file of the cell outlines) and ``geojson_units`` (the unit of + its coordinates, ``"px"`` of the full-resolution image, default and what QuPath writes, or ``"um"``). The + key ``mask_channel`` is used by :func:`read_labels`. Returns ------- @@ -293,6 +324,8 @@ def read_cell_table( logger.info("Reading %s as a %r cell table.", path, fmt) if fmt in ("sprm", "inform"): raise NotImplementedError(f"Reading {fmt!r} cell tables ({path}) is added in a later release.") + if fmt != "qupath" and cells_table_kwargs.get("geojson") is not None: + raise ValueError(f"`cells_table_kwargs['geojson']` is only read with QuPath cell tables; {path} is {fmt!r}.") if fmt == "codex": return read_codex_table(path, region=default_region) if fmt not in TABLE_FORMATS: @@ -310,7 +343,119 @@ def read_cell_table( "pixel size is missing, or one table covers regions with different pixel sizes)." ) roles["px_per_unit"] = 1 / pixel_size_um - return _from_frame(path, df, default_region=default_region, **roles) + table = _from_frame(path, df, default_region=default_region, **roles) + if fmt == "qupath": + geojson = cells_table_kwargs.get("geojson") + geojson = _find_geojson(path) if geojson is None else Path(geojson) + if geojson is not None: + units = cells_table_kwargs.get("geojson_units", "px") + table.polygons_px = read_qupath_polygons(geojson, table, path, units=units, pixel_size_um=pixel_size_um) + table.geojson = geojson + return table + + +def _find_geojson(table_path: Path) -> Path | None: + """The GeoJSON next to a QuPath table: ``.geojson``, else ``_cells.geojson``. + + Other GeoJSON files, e.g. HuBMAP's ``*tissue-boundary.geojson`` tissue outline, are not cells. + """ + candidates = (table_path.with_name(f"{table_path.stem}{suffix}.geojson") for suffix in ("", "_cells")) + return next((candidate for candidate in candidates if candidate.is_file()), None) + + +def read_qupath_polygons( + geojson: Path, table: CellTable, table_path: Path, *, units: str = "px", pixel_size_um: float | None = None +) -> np.ndarray: + """Read the cell outlines of a QuPath GeoJSON export and match them to the cells of its measurement table. + + A feature belongs to the cell whose ``Object ID`` is the feature ``id`` (the object's UUID, QuPath >= 0.4); + features of other objects (e.g. annotations) are skipped. If the table has no ``Object ID`` column or the + features have no ids, the cell and detection features (``objectType``) are matched to the table rows in order, + and their number must equal the number of rows. + + Parameters + ---------- + geojson + A GeoJSON ``FeatureCollection`` or JSON array of features (QuPath *Export as GeoJSON*). + table + The cell table read from ``table_path``. + table_path + The QuPath measurement table, for error messages. + units + The unit of the coordinates: ``"px"`` (of the full-resolution image, what QuPath writes) or ``"um"``. + pixel_size_um + The pixel size, to convert ``"um"`` coordinates. + + Returns + ------- + The polygon of every row of ``table`` in pixels, `None` for rows without one. + """ + if units not in ("px", "um"): + raise ValueError(f"`cells_table_kwargs['geojson_units']` must be 'px' or 'um', got {units!r}.") + px_per_unit = 1.0 + if units == "um": + if pixel_size_um is None: + raise ValueError(f"{geojson} has coordinates in µm, but no pixel size is known to convert them to pixels.") + px_per_unit = 1 / pixel_size_um + if not geojson.is_file(): + raise FileNotFoundError(f"{geojson} does not exist or is not a file.") + ids, geometries, kinds = _read_geojson(geojson) + if px_per_unit != 1.0: + geometries = shapely.transform(geometries, lambda xy: xy * px_per_unit) + + polygons = np.full(len(table.obs), None, dtype=object) + if _QUPATH_ID in table.obs.columns and ids is not None: + object_ids = pd.Index(table.obs[_QUPATH_ID].astype(str)) + if not object_ids.is_unique: + raise ValueError(f"The '{_QUPATH_ID}' column of {table_path} is not unique.") + rows = object_ids.get_indexer(pd.Index(ids).astype(str)) + found = rows >= 0 + if not found.any(): + raise ValueError( + f"{geojson} does not hold the cells of {table_path}: none of the feature ids is an '{_QUPATH_ID}' " + "of the table." + ) + polygons[rows[found]] = geometries[found] + logger.info("Matched %d of the %d features of %s to cells by id.", found.sum(), len(geometries), geojson) + return polygons + + cells = geometries if kinds is None else geometries[np.isin(kinds, list(_QUPATH_CELL_TYPES))] + if len(cells) != len(polygons): + raise ValueError( + f"{geojson} has {len(cells)} cell polygons without ids, but {table_path} has {len(polygons)} rows; " + f"without ids ('{_QUPATH_ID}'), polygons are matched to cells in order." + ) + polygons[:] = cells + return polygons + + +def _read_geojson(path: Path) -> tuple[np.ndarray | None, np.ndarray, np.ndarray | None]: + """The ids (`None` if there are none), geometries and QuPath object types (`None` if unknown) of the features.""" + with path.open("rb") as f: + is_array = f.read(1024).lstrip(b"\xef\xbb\xbf \t\r\n").startswith(b"[") + if is_array: + # QuPath's default export without the FeatureCollection option, which GDAL does not read + features = json.loads(path.read_text(encoding="utf-8-sig")) + geometries = np.array( + [shapely.geometry.shape(f["geometry"]) if f.get("geometry") else None for f in features], dtype=object + ) + ids = [f.get("id") for f in features] + types = [(f.get("properties") or {}).get("objectType") for f in features] + else: + frame = gpd.read_file(path) + geometries = frame.geometry.to_numpy() + ids = list(frame["id"]) if "id" in frame.columns else [None] * len(frame) + types = list(frame["objectType"]) if "objectType" in frame.columns else [None] * len(frame) + if all(kind is None for kind in types) and any(i in _QUPATH_CELL_TYPES for i in ids): + # QuPath < 0.4 names the object type in the feature id, e.g. `PathCellObject` + types, ids = ids, [None] * len(ids) + has_ids = any(i is not None and not pd.isna(i) for i in ids) + has_types = any(kind is not None for kind in types) + return ( + np.asarray(ids, dtype=object) if has_ids else None, + geometries, + np.asarray(types, dtype=object) if has_types else None, + ) def _qupath_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: @@ -525,12 +670,19 @@ def read_region_tables( pixel_sizes The pixel size in µm of each region with an image. cells_table_kwargs - See :func:`read_cell_table`. + See :func:`read_cell_table`; ``geojson`` may also be a mapping from region id to path, like ``files``. Returns ------- The cell tables, one per file. """ + geojson = cells_table_kwargs.get("geojson") + geojson_files = resolve_per_region(geojson, selected, available) if geojson is not None else {} + if unknown := sorted(geojson_files.keys() - files.keys()): + raise ValueError( + f"`cells_table_kwargs['geojson']` is given for {unknown}, but the cell tables are given for " + f"{list(files)}; pass the GeoJSON files in the same form as the tables." + ) tables = [] for key, path in files.items(): covered = selected if key == "*" else [key] @@ -540,7 +692,7 @@ def read_region_tables( fmt, pixel_size_um=sizes.pop() if len(sizes) == 1 else None, default_region=None if key == "*" else key, - cells_table_kwargs=cells_table_kwargs, + cells_table_kwargs={**cells_table_kwargs, "geojson": geojson_files.get(key)}, ) tables.append(_assign_regions(table, path, key, available, selected)) return tables @@ -603,9 +755,7 @@ def read_labels( ------- The labels element. """ - _, labels_models_kwargs = _initialize_raster_models_kwargs({}, labels_models_kwargs) - _, defaults = _initialize_raster_models_kwargs({}, {}) - chunks = spatial_chunks(labels_models_kwargs["chunks"], defaults["chunks"]) + labels_models_kwargs, chunks = _labels_models_kwargs(labels_models_kwargs) image, _, _, _ = read_tiff(path, {}, chunks) if not np.issubdtype(image.dtype, np.integer): raise ValueError(f"{path} has dtype {image.dtype}; a segmentation mask must have an integer dtype.") @@ -627,6 +777,13 @@ def read_labels( ) +def _labels_models_kwargs(labels_models_kwargs: Mapping[str, Any]) -> tuple[dict[str, Any], tuple[int, int]]: + """``labels_models_kwargs`` with the reader defaults, and their ``(y, x)`` chunk size.""" + _, labels_models_kwargs = _initialize_raster_models_kwargs({}, labels_models_kwargs) + _, defaults = _initialize_raster_models_kwargs({}, {}) + return labels_models_kwargs, spatial_chunks(labels_models_kwargs["chunks"], defaults["chunks"]) + + def _ome_channel_names(path: Path) -> list[str]: with tifffile.TiffFile(path) as tf: if not tf.ome_metadata: @@ -673,6 +830,68 @@ def read_region_labels( return labels +def polygon_labels( + tables: Sequence[CellTable], + labels: Mapping[str, DataArray | DataTree], + image_shapes: Mapping[str, tuple[int, int]], + transformations: Mapping[str, BaseTransformation], + labels_models_kwargs: Mapping[str, Any], +) -> tuple[dict[str, DataArray | DataTree], dict[str, str]]: + """Lazily rasterise the cell polygons of each region into labels whose values are the cells' instance ids. + + Where cells overlap, the one with the higher instance id wins. Regions with a mask in ``labels`` keep it, with + a warning; regions without an image are skipped, their cells become circles. + + Parameters + ---------- + tables + The cell tables; those with ``polygons_px`` are rasterised. + labels + The labels read from masks, by region id. + image_shapes + The level-0 ``(y, x)`` shape of each region's image; the labels get the same shape. + transformations + The transformation to ``"global"`` of each region's image. + labels_models_kwargs + Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`; their ``chunks`` are the chunks + that are rasterised one at a time. + + Returns + ------- + The labels elements by region id, and the GeoJSON file of each. + """ + labels_models_kwargs, chunks = _labels_models_kwargs(labels_models_kwargs) + elements, sources = {}, {} + for table in tables: + if table.polygons_px is None: + continue + for region, rows in _region_rows(table).items(): + if region in labels: + row_ids = " Its cells' instance ids are row numbers and may not match the mask's values." + warnings.warn( + f"Region '{region}' has both a segmentation mask and the cell polygons {table.geojson}; the mask " + f"is used.{row_ids if table.row_instance_ids else ''}", + UserWarning, + stacklevel=3, + ) + continue + if region not in image_shapes: + logger.info("Region '%s' has no image, so its cell polygons are not rasterised.", region) + continue + rows = rows[pd.notna(table.polygons_px[rows])] + data = rasterize_polygons_lazy( + table.polygons_px[rows], table.instance_id.to_numpy()[rows], image_shapes[region], chunks + ) + elements[region] = Labels2DModel.parse( + data, + dims=("y", "x"), + transformations={"global": transformations.get(region, Identity())}, + **labels_models_kwargs, + ) + sources[region] = str(table.geojson) + return elements, sources + + def raster_shape(element: DataArray | DataTree) -> tuple[int, int]: """The level-0 ``(y, x)`` shape of a raster element.""" data = element["scale0"]["image"] if isinstance(element, DataTree) else element @@ -706,7 +925,12 @@ def build_cells( """ _check_disjoint_regions(tables) row_ids = sorted( - region for table in tables if table.row_instance_ids for region in table.region.unique() if region in labels + region + for table in tables + # labels rasterised from a table's polygons have its instance ids by construction + if table.row_instance_ids and table.polygons_px is None + for region in table.region.unique() + if region in labels ) if row_ids: warnings.warn( diff --git a/src/spatialdata_io/readers/_utils/_rasterize.py b/src/spatialdata_io/readers/_utils/_rasterize.py new file mode 100644 index 00000000..e40cdefd --- /dev/null +++ b/src/spatialdata_io/readers/_utils/_rasterize.py @@ -0,0 +1,104 @@ +"""Lazy, chunk-wise rasterisation of cell polygons into a labels array.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import dask.array as da +import numpy as np +import shapely +from dask.base import tokenize +from skimage.draw import polygon as draw_polygon + +if TYPE_CHECKING: + from collections.abc import Sequence + +__all__ = ["rasterize_polygons_lazy"] + +_MAX_ID = np.iinfo(np.uint32).max + + +def rasterize_polygons_lazy( + polygons: Sequence[shapely.Polygon | shapely.MultiPolygon], + ids: Sequence[int], + shape: tuple[int, int], + chunks: tuple[int, int] = (4096, 4096), + scale: float = 1.0, +) -> da.Array: + """Rasterise polygons into a lazy ``uint32`` labels array, one chunk at a time. + + A pixel ``(row, col)`` gets the id of a polygon that contains its centre ``(col + 0.5, row + 0.5)``: pixel + ``(0, 0)`` covers ``[0, 1) x [0, 1)``, as in QuPath. Holes are not part of a polygon. Where polygons overlap, + the one with the higher id wins, independently of their order. Only a spatial index of the polygons is built + here; each chunk draws just the polygons that intersect it when it is computed. + + Parameters + ---------- + polygons + The polygons, in pixel units after multiplying by ``scale``. + ids + The label of each polygon, between 1 and ``2**32 - 1``. + shape + The ``(y, x)`` shape of the labels. + chunks + The ``(y, x)`` chunk size. + scale + The number of pixels per polygon coordinate unit, e.g. ``1 / pixel_size_um`` for coordinates in µm. + + Returns + ------- + The ``(y, x)`` labels, 0 for the background. + """ + if len(polygons) != len(ids): + raise ValueError(f"Got {len(polygons)} polygons but {len(ids)} ids.") + labels = np.asarray(ids, dtype=np.int64) + if len(labels) and (labels.min() < 1 or labels.max() > _MAX_ID): + raise ValueError(f"Polygon ids must be between 1 and {_MAX_ID}; got {labels.min()} to {labels.max()}.") + # drawing in ascending id order lets the higher id win overlaps; the stable sort keeps ties in input order + order = np.argsort(labels, kind="stable") + geometries = np.asarray(polygons, dtype=object)[order] + if scale != 1.0: + geometries = shapely.transform(geometries, lambda xy: xy * scale) + labels = labels[order].astype(np.uint32) + template = da.empty(shape, chunks=chunks, dtype=np.uint32) + # dask would otherwise tokenize the geometries one by one; their vectorised WKB is much faster to hash + wkb = np.frombuffer(b"".join(shapely.to_wkb(geometries)), dtype=np.uint8) + token = tokenize(wkb, labels, template.shape, template.chunks) + return da.map_blocks( + _draw_block, + template, + dtype=np.uint32, + name=f"rasterize-polygons-{token}", + tree=shapely.STRtree(geometries), + labels=labels, + meta=np.empty((0, 0), dtype=np.uint32), + ) + + +def _draw_block( + block: np.ndarray, tree: shapely.STRtree, labels: np.ndarray, block_info: dict[Any, Any] | None = None +) -> np.ndarray: + (y0, y1), (x0, x1) = block_info[0]["array-location"] # type: ignore[index] + out = np.zeros(block.shape, dtype=np.uint32) + # the tree indices follow the ascending ids + for i in np.sort(tree.query(shapely.box(x0, y0, x1, y1))): + for part in shapely.get_parts(tree.geometries[i]): + if isinstance(part, shapely.Polygon) and not part.is_empty: + _draw_polygon(out, part, labels[i], y0, x0) + return out + + +def _draw_polygon(out: np.ndarray, polygon: shapely.Polygon, value: int, y0: int, x0: int) -> None: + """Set the pixels of ``out`` (the block at ``(y0, x0)``) whose centres are inside ``polygon`` to ``value``.""" + minx, miny, maxx, maxy = polygon.bounds + # the pixel window of the polygon in the block; pixel centres are at +0.5 + r0, c0 = max(int(np.floor(miny - 0.5)) - y0, 0), max(int(np.floor(minx - 0.5)) - x0, 0) + r1, c1 = min(int(np.ceil(maxy - 0.5)) + 1 - y0, out.shape[0]), min(int(np.ceil(maxx - 0.5)) + 1 - x0, out.shape[1]) + if r0 >= r1 or c0 >= c1: + return + inside = np.zeros((r1 - r0, c1 - c0), dtype=bool) + # holes are cleared from this polygon only, so cells lying in a hole keep their pixels + for ring, fill in [(polygon.exterior, True), *((ring, False) for ring in polygon.interiors)]: + xy = shapely.get_coordinates(ring) + inside[draw_polygon(xy[:, 1] - 0.5 - (y0 + r0), xy[:, 0] - 0.5 - (x0 + c0), shape=inside.shape)] = fill + out[r0:r1, c0:c1][inside] = value diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 341d1049..60b74082 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -30,6 +30,7 @@ CellTable, build_cells, detect_table_format, + polygon_labels, raster_shape, read_codex_table, read_region_labels, @@ -121,8 +122,11 @@ def codex( as circles ``_cells`` with the cells' area (``size`` column), annotated by one table ``"table"`` (``obs["region"]`` names the circles, ``obs["cell_id"]`` is the instance key), or by one table ``_table`` per region if the regions have different markers. A region with a segmentation mask in - ``labels`` gets the labels ``_labels`` instead of circles, and its cells annotate them. The paths of the - tables and masks read are stored in ``sdata.attrs["akoya"]["cell_tables"]`` and ``["labels"]``. + ``labels`` gets the labels ``_labels`` instead of circles, and its cells annotate them; so does a + region with the cell outlines of a QuPath GeoJSON export (see ``cells_table_kwargs``), rasterised lazily one + chunk at a time with the cells' instance ids (the higher id wins where cells overlap). The paths of the tables, + masks and GeoJSON files read are stored in ``sdata.attrs["akoya"]["cell_tables"]``, ``["labels"]`` and + ``["geojson"]``. .. seealso:: @@ -163,7 +167,11 @@ def codex( cells_table_kwargs Options for reading ``cells_table`` and ``labels``: ``region_column``; for ``"generic"`` tables ``x``, ``y``, ``instance_id``, ``markers`` (list or regular expression) and ``coordinates_unit`` (``"px"`` or - ``"um"``); ``mask_channel`` (index or OME channel name of the mask in a multi-channel ``labels`` file). + ``"um"``); ``mask_channel`` (index or OME channel name of the mask in a multi-channel ``labels`` file); + for QuPath tables ``geojson`` (the GeoJSON export of the cell outlines, a path or a mapping from region id + to path like ``cells_table``; default ``
.geojson`` or ``
_cells.geojson`` next to + the table) and ``geojson_units`` (``"px"`` of the full-resolution image, default and what QuPath writes, + or ``"um"``). Returns ------- @@ -240,6 +248,10 @@ def codex( read = _read_tables(discovered, layout, selected if regions is not None else None, fcs) cell_tables = list(read.values()) table_paths = {key: str(discovered[key]) for key in read} + polygons, geojson_files = polygon_labels( + cell_tables, region_labels, image_shapes, transformations, labels_models_kwargs + ) + region_labels |= polygons tables, shapes = build_cells(cell_tables, {}, region_labels, transformations) if cell_tables else ({}, {}) sdata = SpatialData( images=images, @@ -251,6 +263,8 @@ def codex( attrs["cell_tables"] = table_paths if mask_files: attrs["labels"] = {region: str(file) for region, file in mask_files.items()} + if geojson_files: + attrs["geojson"] = geojson_files sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value] = attrs return _set_reader_metadata(sdata, "codex") diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 0340016f..8774f43a 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -22,6 +22,7 @@ ) from spatialdata_io.readers._utils._cell_tables import ( build_cells, + polygon_labels, raster_shape, read_region_labels, read_region_tables, @@ -87,8 +88,12 @@ def phenocycler( ``obs[{vx.REGION_KEY!r}]`` the annotated element, ``obs[{vx.INSTANCE_KEY!r}]`` the instance key), or in one table ``_table`` per region if the regions have different markers. Cells annotate the labels of their region, whose values must be their instance ids; without a mask they are stored as circles - ``{vx.CELLS_SUFFIX}``, with a warning. The paths of the tables and masks are stored in - ``sdata.attrs[{vx.ATTRS_KEY!r}]["cell_tables"]`` and ``["labels"]``. + ``{vx.CELLS_SUFFIX}``, with a warning. The cell outlines of a QuPath GeoJSON export (see + ``cells_table_kwargs``) are rasterised lazily, one chunk at a time, into the labels of a region without a + mask; a feature's ``id`` is matched to the table's ``Object ID`` (else features and rows are matched in order), + the labels hold the cells' instance ids, and where cells overlap the higher instance id wins. The paths of the + tables, masks and GeoJSON files are stored in ``sdata.attrs[{vx.ATTRS_KEY!r}]["cell_tables"]``, ``["labels"]`` + and ``["geojson"]``. .. seealso:: @@ -122,7 +127,11 @@ def phenocycler( cells_table_kwargs Options for reading ``cells_table`` and ``labels``: ``region_column``; for ``"generic"`` tables ``x``, ``y``, ``instance_id``, ``markers`` (list or regular expression) and ``coordinates_unit`` (``"px"`` or - ``"um"``); ``mask_channel`` (index or OME channel name of the mask in a multi-channel ``labels`` file). + ``"um"``); ``mask_channel`` (index or OME channel name of the mask in a multi-channel ``labels`` file); + for QuPath tables ``geojson`` (the GeoJSON export of the cell outlines, a path or a mapping from region id + to path like ``cells_table``; default ``
.geojson`` or ``
_cells.geojson`` next to + the table) and ``geojson_units`` (``"px"`` of the full-resolution image, default and what QuPath writes, + or ``"um"``). Returns ------- @@ -165,6 +174,10 @@ def phenocycler( pixel_sizes={scan.region: scan.pixel_size_um for scan in scans}, cells_table_kwargs=cells_table_kwargs, ) + polygons, geojson_files = polygon_labels( + cell_tables, region_labels, image_shapes, transformations, labels_models_kwargs + ) + region_labels |= polygons tables, shapes = build_cells(cell_tables, {}, region_labels, transformations) if cell_tables else ({}, {}) sdata = SpatialData( images=images, @@ -177,6 +190,8 @@ def phenocycler( attrs["cell_tables"] = {key: str(file) for key, file in table_files.items()} if mask_files: attrs["labels"] = {region: str(file) for region, file in mask_files.items()} + if geojson_files: + attrs["geojson"] = geojson_files sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value] = attrs return _set_reader_metadata(sdata, "phenocycler") diff --git a/tests/readers/test_rasterize.py b/tests/readers/test_rasterize.py new file mode 100644 index 00000000..521cf057 --- /dev/null +++ b/tests/readers/test_rasterize.py @@ -0,0 +1,261 @@ +from __future__ import annotations + +import json +import warnings +from typing import TYPE_CHECKING, Any + +import dask.array as da +import numpy as np +import pytest +import shapely +import tifffile +from skimage.draw import polygon as draw_polygon +from spatialdata.transformations import get_transformation +from tests._akoya_data import write_qptiff + +from spatialdata_io import phenocycler +from spatialdata_io.readers._utils import _rasterize +from spatialdata_io.readers._utils._rasterize import rasterize_polygons_lazy + +if TYPE_CHECKING: + from pathlib import Path + + +def _eager_reference(polygons: list[shapely.Polygon], ids: list[int], shape: tuple[int, int]) -> np.ndarray: + """Draw on the full canvas at once, in ascending id order; pixel (r, c) is covered when (c + .5, r + .5) is.""" + labels = np.zeros(shape, dtype=np.uint32) + for i in np.argsort(ids, kind="stable"): + inside = np.zeros(shape, dtype=bool) + rings = [(polygons[i].exterior, True), *((ring, False) for ring in polygons[i].interiors)] + for ring, fill in rings: + xy = shapely.get_coordinates(ring) + inside[draw_polygon(xy[:, 1] - 0.5, xy[:, 0] - 0.5, shape=shape)] = fill + labels[inside] = ids[i] + return labels + + +def test_rasterize_matches_eager_reference() -> None: + polygons = [ + shapely.Polygon([(2.3, 3.1), (9.7, 2.2), (8.4, 11.6), (1.2, 9.9)]), + # crosses the chunk borders at x = 16 and y = 16 + shapely.Polygon([(12.2, 10.4), (27.6, 13.3), (24.1, 30.8), (11.7, 25.2)]), + # has a hole + shapely.Polygon( + [(30.1, 20.2), (48.3, 21.4), (47.6, 38.7), (31.4, 37.9)], + holes=[[(35.2, 25.3), (42.4, 25.1), (41.7, 32.6), (36.1, 33.3)]], + ), + ] + ids = [3, 1, 2] + + labels = rasterize_polygons_lazy(polygons, ids, shape=(40, 50), chunks=(16, 16)) + + assert labels.dtype == np.uint32 + assert labels.chunks == ((16, 16, 8), (16, 16, 16, 2)) + result = labels.compute() + np.testing.assert_array_equal(result, _eager_reference(polygons, ids, (40, 50))) + assert set(np.unique(result)) == {0, 1, 2, 3} + assert result[29, 38] == 0 # in the hole + + +def test_rasterize_pixel_convention_and_scale() -> None: + square = shapely.box(1, 2, 3, 5) # covers the pixels x 1..2, y 2..4 + + expected = np.zeros((8, 8), dtype=np.uint32) + expected[2:5, 1:3] = 7 + np.testing.assert_array_equal(rasterize_polygons_lazy([square], [7], shape=(8, 8)).compute(), expected) + # coordinates in µm with 0.5 µm pixels + half = shapely.box(0.5, 1, 1.5, 2.5) + np.testing.assert_array_equal(rasterize_polygons_lazy([half], [7], shape=(8, 8), scale=2).compute(), expected) + + +def test_rasterize_overlap_higher_id_wins() -> None: + left, right = shapely.box(0, 0, 6, 6), shapely.box(3, 3, 9, 9) + # a cell inside the hole of a higher-id ring keeps its pixels + ring = shapely.Polygon(shapely.box(10, 0, 20, 10).exterior, holes=[shapely.box(13, 3, 17, 7).exterior.coords]) + inner = shapely.box(14, 4, 16, 6) + + for order in ([0, 1, 2, 3], [3, 2, 1, 0]): + polygons = [[left, right, ring, inner][i] for i in order] + ids = [[5, 2, 9, 4][i] for i in order] + labels = rasterize_polygons_lazy(polygons, ids, shape=(10, 20), chunks=(4, 4)).compute() + assert labels[4, 4] == 5 + assert labels[1, 1] == 5 + assert labels[8, 8] == 2 + assert labels[5, 15] == 4 + assert labels[3, 13] == 0 + assert labels[1, 11] == 9 + + +def test_rasterize_multipolygon_and_empty() -> None: + multi = shapely.MultiPolygon([shapely.box(0, 0, 2, 2), shapely.box(5, 5, 7, 7)]) + + labels = rasterize_polygons_lazy([multi], [1], shape=(8, 8), chunks=(4, 4)).compute() + + assert labels[0, 0] == labels[6, 6] == 1 + assert labels.sum() == 8 + assert not rasterize_polygons_lazy([], [], shape=(5, 6)).compute().any() + + +def test_rasterize_errors() -> None: + with pytest.raises(ValueError, match="2 polygons but 1 ids"): + rasterize_polygons_lazy([shapely.box(0, 0, 1, 1)] * 2, [1], shape=(4, 4)) + with pytest.raises(ValueError, match="between 1 and"): + rasterize_polygons_lazy([shapely.box(0, 0, 1, 1)], [0], shape=(4, 4)) + + +def test_rasterize_is_lazy(monkeypatch: pytest.MonkeyPatch) -> None: + calls = [] + + def counting_polygon(*args: Any, **kwargs: Any) -> tuple[np.ndarray, np.ndarray]: + calls.append(1) + return draw_polygon(*args, **kwargs) + + monkeypatch.setattr(_rasterize, "draw_polygon", counting_polygon) + polygons = [shapely.box(i, i, i + 3, i + 3) for i in range(0, 60, 6)] + + labels = rasterize_polygons_lazy(polygons, list(range(1, 11)), shape=(64, 64), chunks=(16, 16)) + + assert not calls + labels.blocks[0, 0].compute() + # only the polygons of the chunk are drawn + assert len(calls) == 3 + assert labels.compute().max() == 10 + + +def _write_qupath_cells(path: Path, object_ids: list[str]) -> None: + header = "Image\tObject ID\tClass\tCentroid X µm\tCentroid Y µm\tCD3: Cell: Mean" + rows = [f"S_Scan1.qptiff\t{oid}\tTumor\t{5 + 20 * i}\t{5 + 20 * i}\t{i}" for i, oid in enumerate(object_ids)] + path.write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") + + +def _write_geojson(path: Path, features: list[dict], collection: bool = True) -> None: + path.write_text(json.dumps({"type": "FeatureCollection", "features": features} if collection else features)) + + +def _cell(object_id: str | None, x: float, y: float, object_type: str = "cell") -> dict: + feature = { + "type": "Feature", + "geometry": shapely.geometry.mapping(shapely.box(x, y, x + 8, y + 8)), + "properties": {"objectType": object_type, "classification": {"name": "Tumor"}}, + } + if object_id is not None: + feature["id"] = object_id + return feature + + +_FAST = {"scale_factors": [2]} + + +def test_qupath_geojson_to_labels(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"], stage_position_cm=(1.0, 2.0)) + uuids = ["5b1f-aa", "5b1f-bb", "5b1f-cc"] + _write_qupath_cells(tmp_path / "cells.tsv", uuids) + # in any order; the annotation has an id the table does not know + features = [_cell(uuids[2], 40, 40), _cell(uuids[0], 0, 0), _cell("tissue", 0, 0, "annotation")] + features.append(_cell(uuids[1], 20, 20)) + _write_geojson(tmp_path / "cells.geojson", features) + _write_geojson(tmp_path / "S-tissue-boundary.geojson", [_cell(None, 0, 0, "annotation")]) + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) + + assert list(sdata.labels) == ["S_labels"] + assert len(sdata.shapes) == 0 + labels = sdata.labels["S_labels"]["scale0"]["image"] + assert isinstance(labels.data, da.Array) + assert labels.shape == (256, 320) + assert get_transformation(sdata.labels["S_labels"]) == get_transformation(sdata.images["S_image"]) + table = sdata.tables["table"] + assert table.uns["spatialdata_attrs"]["region"] == ["S_labels"] + # the UUIDs are not integers, so the instance ids are the row numbers; the mask values follow them + assert list(table.obs["cell_id"]) == [1, 2, 3] + assert list(table.obs["Object_ID"]) == uuids + values = labels.values + assert values[4, 4] == 1 + assert values[24, 24] == 2 + assert values[44, 44] == 3 + assert set(np.unique(values)) == {0, 1, 2, 3} + assert sdata.attrs["akoya"]["geojson"] == {"S": str(tmp_path / "cells.geojson")} + + +@pytest.mark.parametrize("collection", [True, False]) +def test_qupath_geojson_by_order_explicit_path_in_um(tmp_path: Path, collection: bool) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + header = "Image\tCentroid X µm\tCentroid Y µm\tCD3: Cell: Mean" + rows = ["S_Scan1.qptiff\t1\t1\t1", "S_Scan1.qptiff\t2\t2\t2"] + (tmp_path / "cells.tsv").write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") + # without ids, the cells and detections follow the row order of the table; annotations are skipped + features = [_cell(None, 0, 0, "annotation"), _cell(None, 0, 0), _cell(None, 20, 20, "detection")] + _write_geojson(tmp_path / "polygons.json", features, collection=collection) + + sdata = phenocycler( + tmp_path, + cells_table=tmp_path / "cells.tsv", + image_models_kwargs=_FAST, + cells_table_kwargs={"geojson": tmp_path / "polygons.json", "geojson_units": "um"}, + ) + + values = sdata.labels["S_labels"]["scale0"]["image"].values + # 8 µm squares at 0.5069 µm per pixel + assert values[int(4 / 0.5069), int(4 / 0.5069)] == 1 + assert values[int(24 / 0.5069), int(24 / 0.5069)] == 2 + assert (values == 1).sum() == pytest.approx((8 / 0.5069) ** 2, rel=0.15) + + +def test_qupath_geojson_mask_wins(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + _write_qupath_cells(tmp_path / "cells.tsv", ["a", "b"]) + _write_geojson(tmp_path / "cells_cells.geojson", [_cell("a", 0, 0), _cell("b", 20, 20)]) + mask = np.zeros((256, 320), dtype=np.uint32) + mask[100:110, 100:110] = 1 + tifffile.imwrite(tmp_path / "mask.tif", mask) + + with pytest.warns(UserWarning, match="mask .* is used"): + sdata = phenocycler( + tmp_path, cells_table=tmp_path / "cells.tsv", labels=tmp_path / "mask.tif", image_models_kwargs=_FAST + ) + + np.testing.assert_array_equal(sdata.labels["S_labels"]["scale0"]["image"].values, mask) + + +def test_qupath_geojson_errors(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + _write_qupath_cells(tmp_path / "cells.tsv", ["a", "b"]) + _write_geojson(tmp_path / "other.geojson", [_cell("x", 0, 0), _cell("y", 20, 20)]) + + with pytest.raises(ValueError, match="none of the feature ids"): + phenocycler( + tmp_path, + cells_table=tmp_path / "cells.tsv", + image_models_kwargs=_FAST, + cells_table_kwargs={"geojson": tmp_path / "other.geojson"}, + ) + with pytest.raises(ValueError, match="geojson_units"): + phenocycler( + tmp_path, + cells_table=tmp_path / "cells.tsv", + image_models_kwargs=_FAST, + cells_table_kwargs={"geojson": tmp_path / "other.geojson", "geojson_units": "mm"}, + ) + with pytest.raises(FileNotFoundError, match="missing.geojson"): + phenocycler( + tmp_path, + cells_table=tmp_path / "cells.tsv", + image_models_kwargs=_FAST, + cells_table_kwargs={"geojson": tmp_path / "missing.geojson"}, + ) + (tmp_path / "cells.csv").write_text("CellID,DAPI,X_centroid,Y_centroid\n1,2,3,4\n") + with pytest.raises(ValueError, match="QuPath"): + phenocycler( + tmp_path, + cells_table=tmp_path / "cells.csv", + image_models_kwargs=_FAST, + cells_table_kwargs={"geojson": tmp_path / "other.geojson"}, + ) + + # without ids, the number of cells must match + (tmp_path / "noid.tsv").write_text("Image\tCentroid X px\tCentroid Y px\nS_Scan1.qptiff\t1\t1\n", encoding="utf-8") + _write_geojson(tmp_path / "noid.geojson", [_cell(None, 0, 0), _cell(None, 20, 20)]) + with pytest.raises(ValueError, match="2 cell polygons .* 1 rows"), pytest.warns(UserWarning, match="marker"): + phenocycler(tmp_path, cells_table=tmp_path / "noid.tsv", image_models_kwargs=_FAST) diff --git a/tests/test_codex.py b/tests/test_codex.py index 1af8f2cb..124ab511 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -199,6 +199,47 @@ def test_codex_labels_with_discovered_tables(tmp_path: Path) -> None: assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["reg001_labels"] +def test_codex_qupath_geojson_labels(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path / "tree", regions=2, layout="per_marker", n_cells=5) + header = "Image\tObject ID\tCentroid X px\tCentroid Y px\tCD3: Cell: Mean" + rows = ["reg002.tif\tu1\t5\t5\t1", "reg002.tif\tu2\t25\t25\t2"] + (tmp_path / "q.tsv").write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") + features = [ + {"type": "Feature", "id": oid, "geometry": {"type": "Polygon", "coordinates": [ring]}, "properties": {}} + for oid, ring in [ + ("u2", [[20, 20], [30, 20], [30, 30], [20, 30], [20, 20]]), + ("u1", [[0, 0], [9, 0], [9, 9], [0, 0]]), + ] + ] + (tmp_path / "polygons.geojson").write_text(json.dumps({"type": "FeatureCollection", "features": features})) + + sdata = codex( + tmp_path / "tree", + regions=["reg002"], + cells_table={"reg002": tmp_path / "q.tsv"}, + image_models_kwargs=_FAST, + cells_table_kwargs={"geojson": {"reg002": tmp_path / "polygons.geojson"}}, + ) + + assert list(sdata.labels) == ["reg002_labels"] + assert len(sdata.shapes) == 0 + values = sdata.labels["reg002_labels"]["scale0"]["image"].values + assert values.shape == (128, 160) + assert values[2, 7] == 1 # the triangle below the diagonal + assert values[7, 2] == 0 + assert values[25, 25] == 2 + assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["reg002_labels"] + assert sdata.attrs["akoya"]["geojson"] == {"reg002": str(tmp_path / "polygons.geojson")} + + with pytest.raises(ValueError, match="same form as the tables"): + codex( + tmp_path / "tree", + cells_table=tmp_path / "q.tsv", + image_models_kwargs=_FAST, + cells_table_kwargs={"geojson": {"reg002": tmp_path / "polygons.geojson"}}, + ) + + def test_codex_str_regions_and_channel_names(tmp_path: Path) -> None: write_legacy_processor_tree(tmp_path, regions=2, cycles=1, channels=1, layout="per_marker", n_cells=5) From 0d5cbb97d5defb8e28ec090a24599f52a0399a68 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 15:22:37 +0200 Subject: [PATCH 013/107] fix: reject QuPath polygons far from their cells' centroids Order or id matches are now checked: >5% of polygons more than 1 px from their row's centroid raise (wrong order, file or geojson_units). Duplicate feature ids raise; cells without a polygon warn. Co-Authored-By: Claude Opus 5.5 --- .../readers/_utils/_cell_tables.py | 52 ++++++++++++++++--- tests/readers/test_rasterize.py | 49 +++++++++++++++-- 2 files changed, 91 insertions(+), 10 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index ed9aec92..49eaca4b 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -77,6 +77,10 @@ _QUPATH_ID = "Object ID" # the QuPath object types of cells; other features of a GeoJSON export (e.g. annotations) are not cells _QUPATH_CELL_TYPES = frozenset({"cell", "detection", "PathCellObject", "PathDetectionObject"}) +# a matched polygon must lie within this distance (px) of its cell's centroid, else the match is considered wrong +_CENTROID_TOLERANCE_PX = 1.0 +# the largest fraction of matched polygons that may be far from their centroids before the match is rejected +_MAX_MISPLACED_FRACTION = 0.05 @dataclass @@ -373,6 +377,9 @@ def read_qupath_polygons( features have no ids, the cell and detection features (``objectType``) are matched to the table rows in order, and their number must equal the number of rows. + Every matched polygon must contain its cell's centroid or lie within 1 px of it; if more than 5% do not, the + polygons belong to other cells (or ``units`` is wrong) and a `ValueError` is raised. + Parameters ---------- geojson @@ -415,18 +422,49 @@ def read_qupath_polygons( f"{geojson} does not hold the cells of {table_path}: none of the feature ids is an '{_QUPATH_ID}' " "of the table." ) + if not pd.Index(rows[found]).is_unique: + raise ValueError(f"{geojson} has several features with the same id.") polygons[rows[found]] = geometries[found] logger.info("Matched %d of the %d features of %s to cells by id.", found.sum(), len(geometries), geojson) - return polygons + if found.sum() < len(polygons): + warnings.warn( + f"{len(polygons) - found.sum()} of the {len(polygons)} cells of {table_path} have no polygon in " + f"{geojson}; they are in the table but not in the labels.", + UserWarning, + stacklevel=5, + ) + else: + cells = geometries if kinds is None else geometries[np.isin(kinds, list(_QUPATH_CELL_TYPES))] + if len(cells) != len(polygons): + raise ValueError( + f"{geojson} has {len(cells)} cell polygons without ids, but {table_path} has {len(polygons)} rows; " + f"without ids ('{_QUPATH_ID}'), polygons are matched to cells in order." + ) + polygons[:] = cells + _check_centroids(polygons, table.centroids_px, geojson, table_path) + return polygons + - cells = geometries if kinds is None else geometries[np.isin(kinds, list(_QUPATH_CELL_TYPES))] - if len(cells) != len(polygons): +def _check_centroids(polygons: np.ndarray, centroids_px: np.ndarray, geojson: Path, table_path: Path) -> None: + """Raise if too many polygons are far from the centroids of the cells they were matched to.""" + matched = pd.notna(polygons) + if not matched.any(): + return + distance = shapely.distance(polygons[matched], shapely.points(centroids_px[matched])) + misplaced = int(np.count_nonzero(~(distance <= _CENTROID_TOLERANCE_PX))) + if misplaced > _MAX_MISPLACED_FRACTION * matched.sum(): raise ValueError( - f"{geojson} has {len(cells)} cell polygons without ids, but {table_path} has {len(polygons)} rows; " - f"without ids ('{_QUPATH_ID}'), polygons are matched to cells in order." + f"{misplaced} of the {matched.sum()} polygons of {geojson} are more than {_CENTROID_TOLERANCE_PX:g} px " + f"from the centroid of the cell of {table_path} they were matched to: the polygons are not those of " + "these cells, are in another order, or `cells_table_kwargs['geojson_units']` is wrong." + ) + if misplaced: + logger.info( + "%d polygons of %s are more than %g px from their cell's centroid.", + misplaced, + geojson, + _CENTROID_TOLERANCE_PX, ) - polygons[:] = cells - return polygons def _read_geojson(path: Path) -> tuple[np.ndarray | None, np.ndarray, np.ndarray | None]: diff --git a/tests/readers/test_rasterize.py b/tests/readers/test_rasterize.py index 521cf057..89dd6dcd 100644 --- a/tests/readers/test_rasterize.py +++ b/tests/readers/test_rasterize.py @@ -123,8 +123,9 @@ def counting_polygon(*args: Any, **kwargs: Any) -> tuple[np.ndarray, np.ndarray] def _write_qupath_cells(path: Path, object_ids: list[str]) -> None: - header = "Image\tObject ID\tClass\tCentroid X µm\tCentroid Y µm\tCD3: Cell: Mean" - rows = [f"S_Scan1.qptiff\t{oid}\tTumor\t{5 + 20 * i}\t{5 + 20 * i}\t{i}" for i, oid in enumerate(object_ids)] + # centroids in the 8 px squares `_cell(..., 20 * i, 20 * i)` + header = "Image\tObject ID\tClass\tCentroid X px\tCentroid Y px\tCD3: Cell: Mean" + rows = [f"S_Scan1.qptiff\t{oid}\tTumor\t{4 + 20 * i}\t{4 + 20 * i}\t{i}" for i, oid in enumerate(object_ids)] path.write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") @@ -183,7 +184,7 @@ def test_qupath_geojson_to_labels(tmp_path: Path) -> None: def test_qupath_geojson_by_order_explicit_path_in_um(tmp_path: Path, collection: bool) -> None: write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) header = "Image\tCentroid X µm\tCentroid Y µm\tCD3: Cell: Mean" - rows = ["S_Scan1.qptiff\t1\t1\t1", "S_Scan1.qptiff\t2\t2\t2"] + rows = ["S_Scan1.qptiff\t4\t4\t1", "S_Scan1.qptiff\t24\t24\t2"] (tmp_path / "cells.tsv").write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") # without ids, the cells and detections follow the row order of the table; annotations are skipped features = [_cell(None, 0, 0, "annotation"), _cell(None, 0, 0), _cell(None, 20, 20, "detection")] @@ -259,3 +260,45 @@ def test_qupath_geojson_errors(tmp_path: Path) -> None: _write_geojson(tmp_path / "noid.geojson", [_cell(None, 0, 0), _cell(None, 20, 20)]) with pytest.raises(ValueError, match="2 cell polygons .* 1 rows"), pytest.warns(UserWarning, match="marker"): phenocycler(tmp_path, cells_table=tmp_path / "noid.tsv", image_models_kwargs=_FAST) + + +def _write_order_table(path: Path) -> None: + header = "Image\tCentroid X px\tCentroid Y px\tCD3: Cell: Mean" + rows = ["S_Scan1.qptiff\t4\t4\t1", "S_Scan1.qptiff\t24\t24\t2"] + path.write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") + + +@pytest.mark.parametrize( + ("features", "match"), + [ + # without ids, polygons in another order than the rows + ([_cell(None, 20, 20), _cell(None, 0, 0)], "2 of the 2 polygons .* are more than 1 px"), + # ids, but no `Object ID` in the table: matched in order too + ([_cell("u2", 20, 20), _cell("u1", 0, 0)], "another order"), + # coordinates in µm read as pixels + ([_cell(None, 0, 0), _cell(None, 10, 10)], "geojson_units"), + ], +) +def test_qupath_geojson_polygons_far_from_centroids_raise(tmp_path: Path, features: list[dict], match: str) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + _write_order_table(tmp_path / "cells.tsv") + _write_geojson(tmp_path / "cells.geojson", features) + + with pytest.raises(ValueError, match=match): + phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) + + +def test_qupath_geojson_partial_and_duplicate_ids(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + _write_qupath_cells(tmp_path / "cells.tsv", ["a", "b", "c"]) + _write_geojson(tmp_path / "cells.geojson", [_cell("a", 0, 0), _cell("c", 40, 40)]) + + with pytest.warns(UserWarning, match="1 of the 3 cells .* have no polygon"): + sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) + + assert set(np.unique(sdata.labels["S_labels"]["scale0"]["image"].values)) == {0, 1, 3} + assert sdata.tables["table"].n_obs == 3 + + _write_geojson(tmp_path / "cells.geojson", [_cell("a", 0, 0), _cell("b", 20, 20), _cell("a", 40, 40)]) + with pytest.raises(ValueError, match="same id"): + phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) From a8634af390a79da2b9f402829d0292c06b832c51 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 15:37:07 +0200 Subject: [PATCH 014/107] feat: keep QuPath polygons and give unrelated Akoya regions their own coordinate systems QuPath cell outlines are stored as {region}_polygons besides the rasterised labels. Several scans of the same sample space are placed by stage position in "global"; unrelated regions (and regions without an image) each get a coordinate system named after the region; a single region stays in "global". Co-Authored-By: Claude Opus 5.5 --- src/spatialdata_io/_constants/_constants.py | 1 + src/spatialdata_io/readers/_utils/_akoya.py | 93 ++++++++++++------- .../readers/_utils/_cell_tables.py | 72 +++++++++++--- src/spatialdata_io/readers/codex.py | 43 +++++---- src/spatialdata_io/readers/phenocycler.py | 15 ++- tests/readers/test_akoya_utils.py | 58 ++++++++---- tests/readers/test_cell_tables.py | 16 ++-- tests/readers/test_rasterize.py | 26 +++++- tests/test_codex.py | 32 ++++++- tests/test_phenocycler.py | 35 +++++++ 10 files changed, 297 insertions(+), 94 deletions(-) diff --git a/src/spatialdata_io/_constants/_constants.py b/src/spatialdata_io/_constants/_constants.py index cfbbd08a..c2ffda3f 100644 --- a/src/spatialdata_io/_constants/_constants.py +++ b/src/spatialdata_io/_constants/_constants.py @@ -34,6 +34,7 @@ class PhenocyclerKeys(ModeEnum): IMAGE_SUFFIX = "_image" LABELS_SUFFIX = "_labels" CELLS_SUFFIX = "_cells" + POLYGONS_SUFFIX = "_polygons" TABLE_KEY = "table" # metadata REGION_KEY = "region" diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index 20882c56..6ece1607 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -6,7 +6,7 @@ import math import warnings from collections import Counter -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import asdict from typing import TYPE_CHECKING, Any @@ -64,48 +64,77 @@ def apply_channel_names(scan: AkoyaScan, channel_names: Sequence[str] | None) -> return list(channel_names) -def region_transformations(scans: Sequence[AkoyaScan]) -> dict[str, BaseTransformation]: - """Compute the transformation to the ``"global"`` coordinate system of every region. +def region_transformations( + scans: Sequence[AkoyaScan], regions_without_image: Iterable[str] = () +) -> dict[str, dict[str, BaseTransformation]]: + """Compute the coordinate systems and transformations of every region, by region id. - Regions are placed with a pixel ``Translation`` derived from their stage positions only if every scan has a - stage position and a pixel size and all scans share the same slide. Otherwise all regions get ``Identity``. + Akoya scans are already stitched, so scans of the same sample space are only placed: if there are several + scans and all have a stage position and a pixel size, the same slide and pixel sizes equal within 0.1%, each is + translated by its stage position (in pixels, relative to the top-left one) into the ``"global"`` coordinate + system. Otherwise the regions are unrelated and each gets its own coordinate system, named after the region, + with ``Identity``; a read of a single region uses ``"global"``. + + Parameters + ---------- + scans + The scans read. + regions_without_image + Regions read without an image (e.g. only their cells); they are unrelated to the scans. + + Returns + ------- + ``{region: {coordinate_system: transformation}}``, to be passed to every element of the region. """ - regions = [scan.region for scan in scans] - placeable = ( - len(scans) > 1 - and all(scan.stage_position_um is not None and scan.pixel_size_um for scan in scans) - and len({scan.slide_id for scan in scans}) == 1 - and scans[0].slide_id is not None - ) - if not placeable: - if len(scans) > 1: + imaged = [scan.region for scan in scans] + others = sorted(set(regions_without_image) - set(imaged)) + if len(imaged) + len(others) == 1: + return {region: {"global": Identity()} for region in [*imaged, *others]} + + result: dict[str, dict[str, BaseTransformation]] = {} + if _same_sample_space(scans): + min_x = min(scan.stage_position_um[0] for scan in scans) # type: ignore[index] + min_y = min(scan.stage_position_um[1] for scan in scans) # type: ignore[index] + for scan in scans: + x, y = scan.stage_position_um # type: ignore[misc] + pixel_size = scan.pixel_size_um + translation = [(x - min_x) / pixel_size, (y - min_y) / pixel_size] # type: ignore[operator] + result[scan.region] = {"global": Translation(translation, axes=("x", "y"))} + else: + if len(imaged) > 1: logger.info( - "Regions %s have no common stage frame (missing stage position or pixel size, or different " - "slides) and are all placed at the origin of 'global', so they overlap. Use `regions=` to read a " - "single region or `spatialdata.transform` to arrange them.", - regions, + "Regions %s do not share a sample space (missing stage position or pixel size, different slides " + "or pixel sizes), so each is stored in its own coordinate system, named after the region.", + imaged, ) - return {region: Identity() for region in regions} - - positions = [scan.stage_position_um for scan in scans] - min_x = min(position[0] for position in positions if position is not None) - min_y = min(position[1] for position in positions if position is not None) - result: dict[str, BaseTransformation] = {} - for scan in scans: - x, y = scan.stage_position_um # type: ignore[misc] - pixel_size = scan.pixel_size_um - result[scan.region] = Translation([(x - min_x) / pixel_size, (y - min_y) / pixel_size], axes=("x", "y")) # type: ignore[operator] - return result + result = {region: {region: Identity()} for region in imaged} + return result | {region: {region: Identity()} for region in others} + + +def _same_sample_space(scans: Sequence[AkoyaScan]) -> bool: + """Whether several scans can be placed by their stage positions in one pixel coordinate system.""" + if len(scans) < 2 or not all(scan.stage_position_um is not None and scan.pixel_size_um for scan in scans): + return False + slides = {scan.slide_id for scan in scans} + pixel_size = scans[0].pixel_size_um + return ( + len(slides) == 1 + and None not in slides + and all(math.isclose(scan.pixel_size_um, pixel_size, rel_tol=1e-3) for scan in scans) # type: ignore[arg-type] + ) def build_image_element( image: da.Array, scan: AkoyaScan, c_coords: list[str], - transformation: BaseTransformation, + transformations: Mapping[str, BaseTransformation], image_models_kwargs: Mapping[str, Any], ) -> DataArray | DataTree: - """Parse a lazy ``(c, y, x)`` array, or ``(y, x, 3)`` for brightfield RGB scans, into an ``Image2DModel``.""" + """Parse a lazy ``(c, y, x)`` array, or ``(y, x, 3)`` for brightfield RGB scans, into an ``Image2DModel``. + + ``transformations`` are the region's, see :func:`region_transformations`. + """ image_models_kwargs, _ = _initialize_raster_models_kwargs(image_models_kwargs, {}) if scan.rgb: image = image.transpose(2, 0, 1) @@ -115,7 +144,7 @@ def build_image_element( dims=("c", "y", "x"), c_coords=c_coords, rgb=scan.rgb, - transformations={"global": transformation}, + transformations=dict(transformations), **image_models_kwargs, ) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 49eaca4b..64f9d2b5 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -41,6 +41,7 @@ "build_cells", "detect_table_format", "polygon_labels", + "polygon_shapes", "read_cell_table", "read_codex_table", "read_labels", @@ -429,7 +430,7 @@ def read_qupath_polygons( if found.sum() < len(polygons): warnings.warn( f"{len(polygons) - found.sum()} of the {len(polygons)} cells of {table_path} have no polygon in " - f"{geojson}; they are in the table but not in the labels.", + f"{geojson}; they are in the table but not in the labels and polygons.", UserWarning, stacklevel=5, ) @@ -770,7 +771,7 @@ def _match_region(value: str, regions: Sequence[str]) -> str | None: def read_labels( path: Path, labels_models_kwargs: Mapping[str, Any], - transformation: BaseTransformation, + transformations: Mapping[str, BaseTransformation], *, mask_channel: int | str | None = None, ) -> DataArray | DataTree: @@ -783,8 +784,8 @@ def read_labels( labels_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`; their ``chunks`` also set the spatial chunks of uncompressed files, which are memory-mapped. - transformation - The transformation to ``"global"``, the one of the region's image. + transformations + The transformations of the region's image, by coordinate system. mask_channel The mask to read from a file with several: its index or OME channel name (e.g. ``"cells"``). If `None`, the first one. @@ -811,7 +812,7 @@ def read_labels( if not 0 <= index < n_masks: raise ValueError(f"{path} holds {n_masks} mask channel(s); `mask_channel` {mask_channel!r} is out of range.") return Labels2DModel.parse( - image[index], dims=("y", "x"), transformations={"global": transformation}, **labels_models_kwargs + image[index], dims=("y", "x"), transformations=dict(transformations), **labels_models_kwargs ) @@ -834,12 +835,12 @@ def _ome_channel_names(path: Path) -> list[str]: def read_region_labels( files: Mapping[str, Path], - transformations: Mapping[str, BaseTransformation], + transformations: Mapping[str, Mapping[str, BaseTransformation]], labels_models_kwargs: Mapping[str, Any], cells_table_kwargs: Mapping[str, Any], image_shapes: Mapping[str, tuple[int, int]], ) -> dict[str, DataArray | DataTree]: - """Read the mask of each region, as resolved by :func:`resolve_per_region`, with the region's transformation. + """Read the mask of each region, as resolved by :func:`resolve_per_region`, with the region's transformations. A mask whose ``(y, x)`` shape differs from ``image_shapes`` of its region raises a warning: its pixels would not align with the image. @@ -854,7 +855,7 @@ def read_region_labels( labels[region] = read_labels( path, labels_models_kwargs, - transformations.get(region, Identity()), + _transformations_of(transformations, region), mask_channel=cells_table_kwargs.get("mask_channel"), ) shape, image_shape = raster_shape(labels[region]), image_shapes.get(region) @@ -872,7 +873,7 @@ def polygon_labels( tables: Sequence[CellTable], labels: Mapping[str, DataArray | DataTree], image_shapes: Mapping[str, tuple[int, int]], - transformations: Mapping[str, BaseTransformation], + transformations: Mapping[str, Mapping[str, BaseTransformation]], labels_models_kwargs: Mapping[str, Any], ) -> tuple[dict[str, DataArray | DataTree], dict[str, str]]: """Lazily rasterise the cell polygons of each region into labels whose values are the cells' instance ids. @@ -889,7 +890,8 @@ def polygon_labels( image_shapes The level-0 ``(y, x)`` shape of each region's image; the labels get the same shape. transformations - The transformation to ``"global"`` of each region's image. + The transformations of each region's image, see :func:`region_transformations`; missing regions are in + ``"global"``. labels_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`; their ``chunks`` are the chunks that are rasterised one at a time. @@ -923,13 +925,54 @@ def polygon_labels( elements[region] = Labels2DModel.parse( data, dims=("y", "x"), - transformations={"global": transformations.get(region, Identity())}, + transformations=_transformations_of(transformations, region), **labels_models_kwargs, ) sources[region] = str(table.geojson) return elements, sources +def polygon_shapes( + tables: Sequence[CellTable], transformations: Mapping[str, Mapping[str, BaseTransformation]] +) -> dict[str, GeoDataFrame]: + """The cell polygons of each region as shapes indexed by the cells' instance ids, the ids of their labels. + + Cells without a polygon are left out. + + Parameters + ---------- + tables + The cell tables; those with ``polygons_px`` are kept. + transformations + The transformations of each region, see :func:`region_transformations`; missing regions are in + ``"global"``. + + Returns + ------- + The shapes elements by region id. + """ + shapes = {} + for table in tables: + if table.polygons_px is None: + continue + for region, rows in _region_rows(table).items(): + rows = rows[pd.notna(table.polygons_px[rows])] + if len(rows) == 0: + continue + geometry = gpd.GeoDataFrame( + geometry=list(table.polygons_px[rows]), index=pd.Index(table.instance_id.to_numpy()[rows]) + ) + shapes[region] = ShapesModel.parse(geometry, transformations=_transformations_of(transformations, region)) + return shapes + + +def _transformations_of( + transformations: Mapping[str, Mapping[str, BaseTransformation]], region: str +) -> dict[str, BaseTransformation]: + """A copy of the transformations of ``region``, ``Identity`` to ``"global"`` if it has none.""" + return dict(transformations.get(region, {"global": Identity()})) + + def raster_shape(element: DataArray | DataTree) -> tuple[int, int]: """The level-0 ``(y, x)`` shape of a raster element.""" data = element["scale0"]["image"] if isinstance(element, DataTree) else element @@ -940,7 +983,7 @@ def build_cells( tables: Sequence[CellTable], region_elements: Mapping[str, str], labels: Mapping[str, DataArray | DataTree], - transformations: Mapping[str, BaseTransformation], + transformations: Mapping[str, Mapping[str, BaseTransformation]], ) -> tuple[dict[str, ad.AnnData], dict[str, GeoDataFrame]]: """Build the table elements and, for regions without labels, circle shapes from cell tables. @@ -954,7 +997,8 @@ def build_cells( labels The labels elements by region id; their regions get no circles. transformations - The transformation to ``"global"`` of each region's image; missing regions get ``Identity``. + The transformations of each region, see :func:`region_transformations`; missing regions get ``Identity`` + to ``"global"``. Returns ------- @@ -998,7 +1042,7 @@ def element(region: str) -> str: geometry=0, radius=radius, index=pd.Index(table.instance_id.to_numpy()[rows]), - transformations={"global": transformations.get(region, Identity())}, + transformations=_transformations_of(transformations, region), ) if circle_regions: warnings.warn( diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 60b74082..8446b5ed 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -31,6 +31,7 @@ build_cells, detect_table_format, polygon_labels, + polygon_shapes, raster_shape, read_codex_table, read_region_labels, @@ -117,14 +118,18 @@ def codex( Raw acquisition and intermediate folders (``cycCCC_regNNN/``, ``tiles/`` and, next to ``stitched/``, ``bestFocus/``) are ignored. The region id ``regNNN`` is taken from the file names; a ``{vx.QPTIFF!r}`` without one is named after its file. If a directory contains no such files, its top-level ``{vx.IMAGE_TIF}(f)`` files - are read as hyperstacks named after their files. Each region is stored as the image ``_image`` in the - ``"global"`` coordinate system and its metadata in ``sdata.attrs["akoya"]``. The cells of each region are stored - as circles ``_cells`` with the cells' area (``size`` column), annotated by one table ``"table"`` - (``obs["region"]`` names the circles, ``obs["cell_id"]`` is the instance key), or by one table - ``_table`` per region if the regions have different markers. A region with a segmentation mask in + are read as hyperstacks named after their files. Each region is stored as the image ``_image`` and its + metadata in ``sdata.attrs["akoya"]``. A single region is stored in the ``"global"`` coordinate system. Several + CIM scans of the same sample space (same slide, with stage positions and equal pixel sizes) are placed at their + stage positions in ``"global"``; otherwise (e.g. CODEX Processor regions, which have no stage positions) each + region is stored in its own coordinate system, named after the region. All elements of a region share its + coordinate system. The cells of each region are stored as circles ``_cells`` with the cells' area + (``size`` column), annotated by one table ``"table"`` (``obs["region"]`` names the circles, ``obs["cell_id"]`` + is the instance key), or by one table ``_table`` per region if the regions have different markers. A region with a segmentation mask in ``labels`` gets the labels ``_labels`` instead of circles, and its cells annotate them; so does a region with the cell outlines of a QuPath GeoJSON export (see ``cells_table_kwargs``), rasterised lazily one - chunk at a time with the cells' instance ids (the higher id wins where cells overlap). The paths of the tables, + chunk at a time with the cells' instance ids (the higher id wins where cells overlap); the outlines are also kept + as the shapes ``_polygons`` in pixels, indexed by the same instance ids. The paths of the tables, masks and GeoJSON files read are stored in ``sdata.attrs["akoya"]["cell_tables"]``, ``["labels"]`` and ``["geojson"]``. @@ -222,17 +227,6 @@ def codex( ) names = {scan.region: apply_channel_names(scan, channel_names) for scan in scans} - transformations = region_transformations(scans) - images = { - f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}": build_image_element( - arrays[scan.region], scan, names[scan.region], transformations[scan.region], image_models_kwargs - ) - for scan in scans - } - image_shapes = {scan.region: raster_shape(images[f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}"]) for scan in scans} - region_labels = read_region_labels( - mask_files, transformations, labels_models_kwargs, cells_table_kwargs, image_shapes - ) if table_files: cell_tables = read_region_tables( table_files, @@ -248,11 +242,26 @@ def codex( read = _read_tables(discovered, layout, selected if regions is not None else None, fcs) cell_tables = list(read.values()) table_paths = {key: str(discovered[key]) for key in read} + # the regions of cells or masks without an image are unrelated to the scans + cell_regions = {region for table in cell_tables for region in table.region.unique()} + transformations = region_transformations(scans, cell_regions | (mask_files.keys() - {"*"})) + images = { + f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}": build_image_element( + arrays[scan.region], scan, names[scan.region], transformations[scan.region], image_models_kwargs + ) + for scan in scans + } + image_shapes = {scan.region: raster_shape(images[f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}"]) for scan in scans} + region_labels = read_region_labels( + mask_files, transformations, labels_models_kwargs, cells_table_kwargs, image_shapes + ) polygons, geojson_files = polygon_labels( cell_tables, region_labels, image_shapes, transformations, labels_models_kwargs ) region_labels |= polygons tables, shapes = build_cells(cell_tables, {}, region_labels, transformations) if cell_tables else ({}, {}) + for region, element in polygon_shapes(cell_tables, transformations).items(): + shapes[f"{region}{PhenocyclerKeys.POLYGONS_SUFFIX}"] = element sdata = SpatialData( images=images, labels={f"{region}{PhenocyclerKeys.LABELS_SUFFIX}": element for region, element in region_labels.items()}, diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 8774f43a..9b1b6b84 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -23,6 +23,7 @@ from spatialdata_io.readers._utils._cell_tables import ( build_cells, polygon_labels, + polygon_shapes, raster_shape, read_region_labels, read_region_tables, @@ -78,9 +79,11 @@ def phenocycler( The region name is the file name without the ``{vx.ER_QPTIFF_SUFFIX!r}``/``{vx.QPTIFF_SUFFIX!r}`` suffix and without a trailing ``_Scan``, unless that makes two regions share a name. Each region is stored as the image - ``{vx.IMAGE_SUFFIX}``. Several regions scanned on the same slide are placed at their stage positions in - the ``"global"`` coordinate system; otherwise all regions sit at its origin. The per-region metadata is stored - in ``sdata.attrs[{vx.ATTRS_KEY!r}]``. + ``{vx.IMAGE_SUFFIX}``. A single region is stored in the ``"global"`` coordinate system. Several regions + of the same sample space (scanned on the same slide, with stage positions and equal pixel sizes) are placed at + their stage positions in ``"global"``; otherwise the regions are unrelated and each is stored in its own + coordinate system, named after the region. All elements of a region share its coordinate system. The + per-region metadata is stored in ``sdata.attrs[{vx.ATTRS_KEY!r}]``. A segmentation mask given in ``labels`` is stored lazily as the labels ``{vx.LABELS_SUFFIX}`` with the transformation of the region's image. The cells of ``cells_table`` are stored in one table @@ -91,7 +94,9 @@ def phenocycler( ``{vx.CELLS_SUFFIX}``, with a warning. The cell outlines of a QuPath GeoJSON export (see ``cells_table_kwargs``) are rasterised lazily, one chunk at a time, into the labels of a region without a mask; a feature's ``id`` is matched to the table's ``Object ID`` (else features and rows are matched in order), - the labels hold the cells' instance ids, and where cells overlap the higher instance id wins. The paths of the + the labels hold the cells' instance ids, and where cells overlap the higher instance id wins. The outlines are + also kept as the shapes ``{vx.POLYGONS_SUFFIX}`` in pixels, indexed by the same instance ids; the table + annotates the labels. The paths of the tables, masks and GeoJSON files are stored in ``sdata.attrs[{vx.ATTRS_KEY!r}]["cell_tables"]``, ``["labels"]`` and ``["geojson"]``. @@ -179,6 +184,8 @@ def phenocycler( ) region_labels |= polygons tables, shapes = build_cells(cell_tables, {}, region_labels, transformations) if cell_tables else ({}, {}) + for region, element in polygon_shapes(cell_tables, transformations).items(): + shapes[f"{region}{PhenocyclerKeys.POLYGONS_SUFFIX}"] = element sdata = SpatialData( images=images, labels={f"{region}{PhenocyclerKeys.LABELS_SUFFIX}": element for region, element in region_labels.items()}, diff --git a/tests/readers/test_akoya_utils.py b/tests/readers/test_akoya_utils.py index 3a4c978a..a3ab2f9c 100644 --- a/tests/readers/test_akoya_utils.py +++ b/tests/readers/test_akoya_utils.py @@ -59,32 +59,54 @@ def test_apply_channel_names_length_mismatch(): apply_channel_names(_scan(), ["a", "b"]) -def test_region_transformations_identity_without_positions(): - result = region_transformations([_scan("a"), _scan("b")]) - assert all(isinstance(t, Identity) for t in result.values()) - assert set(result) == {"a", "b"} +def test_region_transformations_single_is_global(): + assert region_transformations([_scan("a", stage=(5.0, 5.0))]) == {"a": {"global": Identity()}} -def test_region_transformations_translation_same_slide(): +def test_region_transformations_shared_slide_translation(): result = region_transformations([_scan("a", stage=(0.0, 0.0)), _scan("b", stage=(100.0, 50.0))]) - assert result["a"] == Translation([0.0, 0.0], axes=("x", "y")) - assert result["b"] == Translation([200.0, 100.0], axes=("x", "y")) + assert result == { + "a": {"global": Translation([0.0, 0.0], axes=("x", "y"))}, + "b": {"global": Translation([200.0, 100.0], axes=("x", "y"))}, + } -def test_region_transformations_identity_different_slides(): +def test_region_transformations_unrelated_get_own_cs(): result = region_transformations( - [_scan("a", stage=(0.0, 0.0), slide_id="s1"), _scan("b", stage=(100.0, 50.0), slide_id="s2")] + [_scan("A", stage=(0.0, 0.0), slide_id="s1"), _scan("B", stage=(100.0, 50.0), slide_id="s2")] ) - assert all(isinstance(t, Identity) for t in result.values()) + assert result == {"A": {"A": Identity()}, "B": {"B": Identity()}} -def test_region_transformations_identity_missing_pixel_size(): - result = region_transformations([_scan("a", stage=(0.0, 0.0), pixel_size_um=None), _scan("b", stage=(1.0, 1.0))]) - assert all(isinstance(t, Identity) for t in result.values()) +def test_region_transformations_mixed_pixel_sizes_own_cs(): + result = region_transformations([_scan("a", stage=(0.0, 0.0)), _scan("b", stage=(1.0, 1.0), pixel_size_um=0.25)]) + assert result == {"a": {"a": Identity()}, "b": {"b": Identity()}} + # pixel sizes equal up to rounding share the sample space + result = region_transformations([_scan("a", stage=(0.0, 0.0)), _scan("b", stage=(1.0, 1.0), pixel_size_um=0.5004)]) + assert set(result["b"]) == {"global"} -def test_region_transformations_single_scan_identity(): - assert isinstance(region_transformations([_scan("a", stage=(5.0, 5.0))])["a"], Identity) +@pytest.mark.parametrize( + "scans", + [ + [_scan("a"), _scan("b")], + [_scan("a", stage=(0.0, 0.0), pixel_size_um=None), _scan("b", stage=(1.0, 1.0))], + [_scan("a", stage=(0.0, 0.0), slide_id=None), _scan("b", stage=(1.0, 1.0), slide_id=None)], + ], + ids=["no_stage_positions", "missing_pixel_size", "no_slide_id"], +) +def test_region_transformations_unplaceable_get_own_cs(scans: list[AkoyaScan]): + assert region_transformations(scans) == {"a": {"a": Identity()}, "b": {"b": Identity()}} + + +def test_region_transformations_regions_without_image(): + # a region read without an image (e.g. only its cells) is unrelated to the scans + shared = [_scan("a", stage=(0.0, 0.0)), _scan("b", stage=(100.0, 50.0))] + result = region_transformations(shared, ["c", "a"]) + assert set(result["a"]) == set(result["b"]) == {"global"} + assert result["c"] == {"c": Identity()} + assert region_transformations([_scan("a")], ["c"]) == {"a": {"a": Identity()}, "c": {"c": Identity()}} + assert region_transformations([], ["c"]) == {"c": {"global": Identity()}} def test_scans_to_attrs_json_serialisable(): @@ -112,14 +134,14 @@ def test_select_regions_unknown_raises(): def test_build_image_rgb_and_multiplex(): scan = _scan() img = da.from_array(np.random.default_rng(0).integers(0, 255, (3, 16, 16), dtype=np.uint8), chunks=(1, 8, 8)) - element = build_image_element(img, scan, ["DAPI", "CD3", "CD8"], Identity(), _NO_PYRAMID) + element = build_image_element(img, scan, ["DAPI", "CD3", "CD8"], {"global": Identity()}, _NO_PYRAMID) assert list(element.coords["c"].values) == ["DAPI", "CD3", "CD8"] assert isinstance(get_transformation(element, "global"), Identity) rgb_scan = _scan(rgb=True) rgb_img = da.from_array(np.zeros((16, 16, 3), dtype=np.uint8), chunks=(8, 8, 3)) trans = Translation([4.0, 2.0], axes=("x", "y")) - element = build_image_element(rgb_img, rgb_scan, ["r", "g", "b"], trans, _NO_PYRAMID) + element = build_image_element(rgb_img, rgb_scan, ["r", "g", "b"], {"reg1": trans}, _NO_PYRAMID) assert list(element.coords["c"].values) == ["r", "g", "b"] assert element.shape == (3, 16, 16) - assert get_transformation(element, "global") == trans + assert get_transformation(element, get_all=True) == {"reg1": trans} diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index 7a60288c..5039660a 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -116,7 +116,9 @@ def test_build_cells_single_table_and_circles(tmp_path: Path) -> None: translation = Translation([10, 0], axes=("x", "y")) with pytest.warns(UserWarning, match=r"No segmentation mask found for region.*circles") as record: - anndatas, shapes = build_cells(tables, {}, {}, {"reg001": Identity(), "reg002": translation}) + anndatas, shapes = build_cells( + tables, {}, {}, {"reg001": {"reg001": Identity()}, "reg002": {"reg002": translation}} + ) assert len(record) == 1 assert list(anndatas) == ["table"] @@ -129,7 +131,7 @@ def test_build_cells_single_table_and_circles(tmp_path: Path) -> None: np.testing.assert_allclose(np.asarray(table.obsm["spatial"])[:3], tables[0].centroids_px) assert sorted(shapes) == ["reg001_cells", "reg002_cells"] assert list(shapes["reg002_cells"].index) == [1, 2, 3, 4] - assert get_transformation(shapes["reg002_cells"], "global") == translation + assert get_transformation(shapes["reg002_cells"], get_all=True) == {"reg002": translation} def test_build_cells_different_panels_split_tables(tmp_path: Path) -> None: @@ -361,7 +363,7 @@ def test_read_labels_lazy_with_transformation(tmp_path: Path) -> None: tifffile.imwrite(path, mask) translation = Translation([3, 4], axes=("x", "y")) - labels = read_labels(path, {"scale_factors": [2]}, translation) + labels = read_labels(path, {"scale_factors": [2]}, {"global": translation}) level0 = labels["scale0"]["image"] assert get_axes_names(labels) == ("y", "x") @@ -375,7 +377,7 @@ def test_read_labels_ome_channels(tmp_path: Path, mask_channel: int | str | None path = tmp_path / "mask.ome.tif" tifffile.imwrite(path, masks, metadata={"axes": "CYX", "Channel": {"Name": ["cells", "nuclei"]}}) - labels = read_labels(path, {"scale_factors": None}, Identity(), mask_channel=mask_channel) + labels = read_labels(path, {"scale_factors": None}, {"global": Identity()}, mask_channel=mask_channel) np.testing.assert_array_equal(labels.values, masks[expected]) @@ -384,11 +386,11 @@ def test_read_labels_errors(tmp_path: Path) -> None: path = tmp_path / "mask.tif" tifffile.imwrite(path, np.zeros((8, 8), dtype=np.float32)) with pytest.raises(ValueError, match="integer"): - read_labels(path, {}, Identity()) + read_labels(path, {}, {"global": Identity()}) tifffile.imwrite(path, np.zeros((2, 8, 8), dtype=np.uint16)) with pytest.raises(ValueError, match="channel"): - read_labels(path, {}, Identity(), mask_channel=5) + read_labels(path, {}, {"global": Identity()}, mask_channel=5) def test_read_labels_pyramidal_ome_is_lazy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -402,7 +404,7 @@ def fail(*args: object, **kwargs: object) -> None: raise AssertionError("full page decode during read") monkeypatch.setattr(tifffile.TiffPage, "asarray", fail) - labels = read_labels(path, {"scale_factors": None}, Identity()) + labels = read_labels(path, {"scale_factors": None}, {"global": Identity()}) assert labels.shape == (64, 48) np.testing.assert_array_equal(labels.values, mask) diff --git a/tests/readers/test_rasterize.py b/tests/readers/test_rasterize.py index 89dd6dcd..06baa1f1 100644 --- a/tests/readers/test_rasterize.py +++ b/tests/readers/test_rasterize.py @@ -10,6 +10,7 @@ import shapely import tifffile from skimage.draw import polygon as draw_polygon +from spatialdata import read_zarr from spatialdata.transformations import get_transformation from tests._akoya_data import write_qptiff @@ -162,7 +163,7 @@ def test_qupath_geojson_to_labels(tmp_path: Path) -> None: sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) assert list(sdata.labels) == ["S_labels"] - assert len(sdata.shapes) == 0 + assert list(sdata.shapes) == ["S_polygons"] labels = sdata.labels["S_labels"]["scale0"]["image"] assert isinstance(labels.data, da.Array) assert labels.shape == (256, 320) @@ -180,6 +181,29 @@ def test_qupath_geojson_to_labels(tmp_path: Path) -> None: assert sdata.attrs["akoya"]["geojson"] == {"S": str(tmp_path / "cells.geojson")} +def test_qupath_geojson_keeps_polygons(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + _write_qupath_cells(tmp_path / "cells.tsv", ["a", "b", "c"]) + # cell "b" has no polygon: it is missing from the shapes and the labels + _write_geojson(tmp_path / "cells.geojson", [_cell("c", 40, 40), _cell("a", 0, 0)]) + + with pytest.warns(UserWarning, match="have no polygon"): + sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) + + assert list(sdata.shapes) == ["S_polygons"] + polygons = sdata.shapes["S_polygons"] + assert list(polygons.index) == [1, 3] + labels = sdata.labels["S_labels"]["scale0"]["image"].values + assert sorted(set(np.unique(labels)) - {0}) == list(polygons.index) + assert polygons.geometry.loc[3].equals(shapely.box(40, 40, 48, 48)) + assert get_transformation(polygons, get_all=True) == get_transformation(sdata.images["S_image"], get_all=True) + assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["S_labels"] + + sdata.write(tmp_path / "out.zarr") + reread = read_zarr(tmp_path / "out.zarr") + assert reread.shapes["S_polygons"].geometry.equals(polygons.geometry) + + @pytest.mark.parametrize("collection", [True, False]) def test_qupath_geojson_by_order_explicit_path_in_um(tmp_path: Path, collection: bool) -> None: write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) diff --git a/tests/test_codex.py b/tests/test_codex.py index 124ab511..f9f51c06 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import shutil from typing import TYPE_CHECKING import numpy as np @@ -10,6 +11,7 @@ import pytest import tifffile from spatialdata.models import get_channel_names +from spatialdata.transformations import Identity, get_transformation from spatialdata_io import codex from spatialdata_io.readers._utils._akoya import dedupe_channel_names @@ -222,7 +224,7 @@ def test_codex_qupath_geojson_labels(tmp_path: Path) -> None: ) assert list(sdata.labels) == ["reg002_labels"] - assert len(sdata.shapes) == 0 + assert list(sdata.shapes) == ["reg002_polygons"] values = sdata.labels["reg002_labels"]["scale0"]["image"].values assert values.shape == (128, 160) assert values[2, 7] == 1 # the triangle below the diagonal @@ -321,6 +323,34 @@ def test_codex_table_multi_region(tmp_path: Path) -> None: assert sdata.attrs["akoya"]["cell_tables"]["reg002"].endswith("reg002_compensated.fcs") +def test_codex_multi_region_own_coordinate_systems(tmp_path: Path) -> None: + # CODEX Processor regions have no stage positions: each gets its own coordinate system + write_legacy_processor_tree(tmp_path, regions=2, cycles=1, layout="per_marker", n_cells=5) + + with pytest.warns(UserWarning, match="circles"): + sdata = codex(tmp_path, image_models_kwargs=_FAST) + + assert sorted(sdata.coordinate_systems) == ["reg001", "reg002"] + for region in ("reg001", "reg002"): + for name in (f"{region}_image", f"{region}_cells"): + assert get_transformation(sdata[name], get_all=True) == {region: Identity()} + + single = codex(tmp_path, regions=["reg002"], image_models_kwargs=_FAST) + assert single.coordinate_systems == ["global"] + + +def test_codex_region_without_image_own_coordinate_system(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, regions=2, cycles=1, layout="per_marker", n_cells=5) + shutil.rmtree(tmp_path / "processed_2020-01-01" / "stitched" / "reg002") + + with pytest.warns(UserWarning, match="circles"): + sdata = codex(tmp_path, image_models_kwargs=_FAST) + + assert list(sdata.images) == ["reg001_image"] + assert get_transformation(sdata["reg001_image"], get_all=True) == {"reg001": Identity()} + assert get_transformation(sdata["reg002_cells"], get_all=True) == {"reg002": Identity()} + + @pytest.mark.parametrize("fcs", [True, False]) def test_codex_old_flat_layout_regression(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fcs: bool) -> None: # the original reader read the image (and the CSV) relative to the working directory diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 6d46c533..1350fd82 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -14,6 +14,7 @@ from click.testing import CliRunner from spatialdata import read_zarr from spatialdata.models import get_channel_names +from spatialdata.transformations import Identity, Translation, get_transformation from spatialdata_io import phenocycler from spatialdata_io.__main__ import phenocycler_wrapper @@ -55,6 +56,40 @@ def test_read_directory_multiple_scans_and_subset(tmp_path: Path) -> None: phenocycler(tmp_path, regions=["Z"]) +def test_phenocycler_unrelated_scans_own_coordinate_systems(tmp_path: Path) -> None: + for region, slide in (("A", "SLIDE1"), ("B", "SLIDE2")): + write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI", "CD3"], slide_id=slide) + _write_mcquant(tmp_path / f"{region}.csv") + _write_mask(tmp_path / "A_mask.tif") + + with pytest.warns(UserWarning, match="circles"): + sdata = phenocycler( + tmp_path, + cells_table={"A": tmp_path / "A.csv", "B": tmp_path / "B.csv"}, + labels={"A": tmp_path / "A_mask.tif"}, + image_models_kwargs=_FAST, + ) + + assert sorted(sdata.coordinate_systems) == ["A", "B"] + for name in ("A_image", "A_labels"): + assert get_transformation(sdata[name], get_all=True) == {"A": Identity()} + for name in ("B_image", "B_cells"): + assert get_transformation(sdata[name], get_all=True) == {"B": Identity()} + + +def test_phenocycler_same_slide_scans_placed_in_global(tmp_path: Path) -> None: + write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI"], stage_position_cm=(1.0, 2.0)) + write_qptiff(tmp_path / "B_Scan1.qptiff", biomarkers=["DAPI"], stage_position_cm=(1.01, 2.0)) + + sdata = phenocycler(tmp_path, image_models_kwargs=_FAST) + + assert sdata.coordinate_systems == ["global"] + assert get_transformation(sdata["A_image"], get_all=True) == {"global": Translation([0, 0], axes=("x", "y"))} + offset = get_transformation(sdata["B_image"], "global") + assert isinstance(offset, Translation) + np.testing.assert_allclose(offset.translation, [100 / 0.5069, 0], rtol=1e-3) + + def test_region_collision_keeps_full_stem(tmp_path: Path) -> None: write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) write_qptiff(tmp_path / "S_Scan2.qptiff", biomarkers=["DAPI"]) From 1bec2f83e58e21c16b6b7d7a95679430b694b4c8 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 15:49:11 +0200 Subject: [PATCH 015/107] feat: add HuBMAP SPRM and inForm cell-table adapters Co-Authored-By: Claude Opus 5.5 --- .../readers/_utils/_cell_tables.py | 153 ++++++++++++++++-- tests/readers/test_cell_tables.py | 141 +++++++++++++++- 2 files changed, 271 insertions(+), 23 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 64f9d2b5..d5311cfb 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -78,6 +78,15 @@ _QUPATH_ID = "Object ID" # the QuPath object types of cells; other features of a GeoJSON export (e.g. annotations) are not cells _QUPATH_CELL_TYPES = frozenset({"cell", "detection", "PathCellObject", "PathDetectionObject"}) +# inForm `*_cell_seg_data.txt` columns: `Cell X Position (microns)`, `Entire Cell Mean ()`, ... +_INFORM_POSITION = re.compile(r"Cell X Position(?: \((.*)\))?") +_INFORM_MARKER = re.compile(r"Entire Cell (.+) Mean(?: \(.*\))?") +_INFORM_AREA = re.compile(r"Entire Cell Area \((.*)\)") +_INFORM_REGION = "Sample Name" +_INFORM_ID = "Cell ID" +# HuBMAP SPRM outputs: `-cell_channel_mean.csv` with the centroids in `-cell_centers.csv` +_SPRM_MEANS_SUFFIX = "-cell_channel_mean.csv" +_SPRM_CENTERS_SUFFIX = "-cell_centers.csv" # a matched polygon must lie within this distance (px) of its cell's centroid, else the match is considered wrong _CENTROID_TOLERANCE_PX = 1.0 # the largest fraction of matched polygons that may be far from their centroids before the match is rejected @@ -102,7 +111,8 @@ class CellTable: instance_id The instance id of every cell (int64). centroids_px - The ``(x, y)`` centroids in level-0 pixels of the region image. + The ``(x, y)`` centroids in level-0 pixels of the region image, or `None` if the table has none (SPRM without + ``*-cell_centers.csv``). radius_px The radius of a circle with the cell's area, or `None` if the table has no area. row_instance_ids @@ -119,7 +129,7 @@ class CellTable: var: pd.DataFrame region: pd.Series instance_id: pd.Series - centroids_px: np.ndarray + centroids_px: np.ndarray | None radius_px: np.ndarray | None row_instance_ids: bool = False polygons_px: np.ndarray | None = None @@ -133,7 +143,7 @@ def take(self, rows: np.ndarray) -> CellTable: var=self.var, region=self.region.iloc[rows], instance_id=self.instance_id.iloc[rows], - centroids_px=self.centroids_px[rows], + centroids_px=None if self.centroids_px is None else self.centroids_px[rows], radius_px=None if self.radius_px is None else self.radius_px[rows], row_instance_ids=self.row_instance_ids, polygons_px=None if self.polygons_px is None else self.polygons_px[rows], @@ -248,7 +258,8 @@ def detect_table_format(path: Path) -> str: Returns ------- ``"codex"`` (FCS, or ``x``/``y`` with ``cell_id`` or ``cycCCC_chHHH`` columns), ``"qupath"`` (``Centroid X µm`` - or ``Object ID``), ``"mcquant"`` (``CellID`` and ``X_centroid``), ``"inform"`` (``Cell X Position``), + or ``Object ID``), ``"mcquant"`` (``CellID`` and ``X_centroid``), ``"inform"`` (``Cell X Position``, optionally + followed by its unit in parentheses), ``"sprm"`` (first column ``ID``), else ``"generic"``. """ if path.suffix.lower() == CodexKeys.FCS_FILE.value: @@ -264,7 +275,7 @@ def detect_table_format(path: Path) -> str: return "qupath" if {"CellID", "X_centroid"} <= set(columns): return "mcquant" - if "Cell X Position" in columns: + if any(_INFORM_POSITION.fullmatch(column) for column in columns): return "inform" if columns[:1] == ["ID"]: return "sprm" @@ -286,8 +297,8 @@ def read_cell_table( path The cell table. fmt - One of ``"auto"`` (see :func:`detect_table_format`), ``"codex"``, ``"qupath"``, ``"mcquant"`` or - ``"generic"``: + One of ``"auto"`` (see :func:`detect_table_format`), ``"codex"``, ``"qupath"``, ``"mcquant"``, + ``"sprm"``, ``"inform"`` or ``"generic"``: - ``"codex"``: see :func:`read_codex_table`. - ``"qupath"``: a QuPath measurement export. Centroids ``Centroid X µm``/``Centroid Y µm`` (divided by @@ -301,9 +312,19 @@ def read_cell_table( - ``"mcquant"``: an MCMICRO ``mcquant`` table. Instance ids ``CellID``, pixel centroids ``X_centroid``/``Y_centroid``, the markers are the columns between ``CellID`` and ``X_centroid``, the radius comes from ``Area``. + - ``"sprm"``: a HuBMAP SPRM ``-cell_channel_mean.csv`` (validated on synthetic data only). Instance + ids ``ID`` (the values of the mask, ``pipeline_output/mask/*.ome.tiff`` channel 0, passed as ``labels``), + one marker per remaining column. Pixel centroids come from ``-cell_centers.csv`` (``ID,x,y``) + next to the table, which must hold every cell; without it a `UserWarning` is raised and the table has + no ``obsm["spatial"]`` (the centroids are never computed from the mask), so its region needs a mask. + - ``"inform"``: a PerkinElmer/Akoya inForm ``*_cell_seg_data.txt`` (tab-separated; validated on synthetic + data only). Instance ids ``Cell ID``, centroids ``Cell X Position``/``Cell Y Position`` in the unit + named in the header, ``(microns)`` (divided by ``pixel_size_um``) or ``(pixels)``; pixels if the + header names none. The region comes from ``Sample Name``. The markers are the + ``Entire Cell Mean (...)`` columns; the radius comes from ``Entire Cell Area (...)`` if its + unit equals that of the positions. - ``"generic"``: columns given by ``cells_table_kwargs``. - The ``"sprm"`` and ``"inform"`` formats are detected but not read yet. pixel_size_um The pixel size of the region image, to convert µm centroids to pixels. default_region @@ -327,8 +348,6 @@ def read_cell_table( if fmt == "auto": fmt = detect_table_format(path) logger.info("Reading %s as a %r cell table.", path, fmt) - if fmt in ("sprm", "inform"): - raise NotImplementedError(f"Reading {fmt!r} cell tables ({path}) is added in a later release.") if fmt != "qupath" and cells_table_kwargs.get("geojson") is not None: raise ValueError(f"`cells_table_kwargs['geojson']` is only read with QuPath cell tables; {path} is {fmt!r}.") if fmt == "codex": @@ -339,7 +358,13 @@ def read_cell_table( _, sep = _read_header(path) df = pd.read_csv(path, sep=sep, engine="pyarrow") df.columns = pd.Index([str(column) for column in df.columns]) - reader = {"qupath": _qupath_roles, "mcquant": _mcquant_roles, "generic": _generic_roles}[fmt] + reader = { + "qupath": _qupath_roles, + "mcquant": _mcquant_roles, + "sprm": _sprm_roles, + "inform": _inform_roles, + "generic": _generic_roles, + }[fmt] roles = reader(path, df, cells_table_kwargs) if roles.pop("in_um"): if pixel_size_um is None: @@ -349,6 +374,8 @@ def read_cell_table( ) roles["px_per_unit"] = 1 / pixel_size_um table = _from_frame(path, df, default_region=default_region, **roles) + if fmt == "sprm": + table.centroids_px = _read_sprm_centers(path, table) if fmt == "qupath": geojson = cells_table_kwargs.get("geojson") geojson = _find_geojson(path) if geojson is None else Path(geojson) @@ -442,7 +469,8 @@ def read_qupath_polygons( f"without ids ('{_QUPATH_ID}'), polygons are matched to cells in order." ) polygons[:] = cells - _check_centroids(polygons, table.centroids_px, geojson, table_path) + if table.centroids_px is not None: + _check_centroids(polygons, table.centroids_px, geojson, table_path) return polygons @@ -548,6 +576,88 @@ def _mcquant_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> d } +def _sprm_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: + if "ID" not in df.columns: + raise ValueError(f"{path} has no 'ID' column; found {list(df.columns)}.") + if not pd.api.types.is_integer_dtype(df["ID"]): + raise ValueError(f"The 'ID' column of {path} must hold integer cell ids, got dtype {df['ID'].dtype}.") + region_column = kwargs.get("region_column") + markers = [c for c in df.columns if c not in ("ID", region_column)] + return { + "markers": markers, + "var": pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(markers))), + "x": None, + "y": None, + "in_um": False, + "instance_column": "ID", + "region_column": region_column, + "area_column": None, + } + + +def _read_sprm_centers(path: Path, table: CellTable) -> np.ndarray | None: + """The ``(x, y)`` pixel centroids of the cells of an SPRM table from its ``*-cell_centers.csv``, if it exists.""" + name = path.name + centers = path.with_name( + f"{name.removesuffix(_SPRM_MEANS_SUFFIX)}{_SPRM_CENTERS_SUFFIX}" + if name.endswith(_SPRM_MEANS_SUFFIX) + else f"{path.stem}{_SPRM_CENTERS_SUFFIX}" + ) + if not centers.is_file(): + warnings.warn( + f"No {centers.name} next to {path}, so the cells of this SPRM table have no centroids and the table has " + "no 'spatial' coordinates; the cells need a segmentation mask (`labels`).", + UserWarning, + stacklevel=4, + ) + return None + frame = pd.read_csv(centers) + if not {"ID", "x", "y"} <= set(frame.columns): + raise ValueError(f"{centers} must have the columns 'ID', 'x' and 'y'; found {list(frame.columns)}.") + if not frame["ID"].is_unique: + raise ValueError(f"The 'ID' column of {centers} is not unique.") + rows = pd.Index(frame["ID"]).get_indexer(pd.Index(table.instance_id)) + if (rows < 0).any(): + raise ValueError(f"{centers} has no centroid for {int((rows < 0).sum())} of the {len(rows)} cells of {path}.") + return frame[["x", "y"]].to_numpy(dtype=np.float64)[rows] + + +def _inform_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: + positions = [m for column in df.columns if (m := _INFORM_POSITION.fullmatch(column)) is not None] + y = next((c for c in df.columns if c.startswith("Cell Y Position")), None) + if not positions or y is None: + raise ValueError(f"{path} has no 'Cell X Position' and 'Cell Y Position' columns; found {list(df.columns)}.") + x = positions[0].group(0) + unit = (positions[0].group(1) or "pixels").casefold() + if unit not in ("microns", "pixels"): + raise ValueError(f"The unit '{unit}' of the column '{x}' of {path} must be 'microns' or 'pixels'.") + markers, names = [], [] + for column in df.columns: + if (match := _INFORM_MARKER.fullmatch(column)) is not None: + markers.append(column) + names.append(match.group(1)) + if not markers: + warnings.warn( + f"{path} has no 'Entire Cell Mean' columns, so its table has no marker intensities.", + UserWarning, + stacklevel=5, + ) + area = next((c for c in df.columns if (m := _INFORM_AREA.fullmatch(c)) is not None and m.group(1)), None) + if area is not None: + area_unit = "microns" if "micron" in area else "pixels" + area = area if area_unit == unit else None + return { + "markers": markers, + "var": pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(names))), + "x": x, + "y": y, + "in_um": unit == "microns", + "instance_column": _INFORM_ID if _INFORM_ID in df.columns else None, + "region_column": kwargs.get("region_column", _INFORM_REGION if _INFORM_REGION in df.columns else None), + "area_column": area, + } + + def _generic_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: if "x" not in kwargs or "y" not in kwargs: raise ValueError( @@ -583,8 +693,8 @@ def _from_frame( *, markers: list[str], var: pd.DataFrame, - x: str, - y: str, + x: str | None, + y: str | None, instance_column: str | None, region_column: str | None, area_column: str | None, @@ -629,7 +739,7 @@ def _from_frame( var=var, region=region, instance_id=instance_id, - centroids_px=df[[x, y]].to_numpy(dtype=np.float64) * px_per_unit, + centroids_px=None if x is None or y is None else df[[x, y]].to_numpy(dtype=np.float64) * px_per_unit, radius_px=radius, row_instance_ids=instance_column is None, ) @@ -1035,6 +1145,11 @@ def element(region: str) -> str: for region, rows in _region_rows(table).items(): if region in labels: continue + if table.centroids_px is None: + raise ValueError( + f"The cells of region '{region}' have no centroids and no segmentation mask, so they cannot be " + "placed; pass a mask as `labels` or a table with centroids." + ) circle_regions.append(region) radius = 1.0 if table.radius_px is None else table.radius_px[rows] shapes[element(region)] = ShapesModel.parse( @@ -1095,7 +1210,13 @@ def _to_anndata(tables: Sequence[CellTable], element: Callable[[str], str]) -> a obs.index = pd.Index(regions.to_numpy(dtype=object) + "_" + obs.index.to_numpy(dtype=object)) X = np.concatenate([table.X for table in tables]) if len(tables) > 1 else tables[0].X adata = ad.AnnData(X=X, obs=obs, var=tables[0].var.copy()) - adata.obsm[CodexKeys.SPATIAL_KEY.value] = np.concatenate([table.centroids_px for table in tables]) + centroids = [table.centroids_px for table in tables if table.centroids_px is not None] + if len(centroids) == len(tables): + adata.obsm[CodexKeys.SPATIAL_KEY.value] = np.concatenate(centroids) + elif centroids: + warnings.warn( + "Some cell tables have no centroids, so the table has no 'spatial' coordinates.", UserWarning, stacklevel=4 + ) # e.g. QuPath's `Centroid X µm` -> `Centroid_X_µm`: spatialdata only writes alphanumeric names and `_.-` if any(sanitize_name(str(column), is_dataframe_column=True) != column for column in adata.obs.columns): sanitize_table(adata, inplace=True) diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index 5039660a..1446c6c1 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -6,7 +6,7 @@ import pandas as pd import pytest import tifffile -from spatialdata.models import get_axes_names +from spatialdata.models import Labels2DModel, get_axes_names from spatialdata.transformations import Identity, Translation, get_transformation from tests._akoya_data import write_fcs @@ -55,6 +55,7 @@ def test_read_codex_fcs_marker_names_from_pns(tmp_path: Path) -> None: assert list(table.obs.index) == ["1", "2", "3", "4", "5"] assert list(table.region.unique()) == ["reg001"] assert table.instance_id.dtype == np.int64 + assert table.centroids_px is not None np.testing.assert_allclose(table.centroids_px, cells[["x", "y"]], rtol=1e-6) assert table.radius_px is not None np.testing.assert_allclose(table.radius_px, np.sqrt(cells["size"] / np.pi), rtol=1e-6) @@ -128,6 +129,7 @@ def test_build_cells_single_table_and_circles(tmp_path: Path) -> None: assert list(table.obs["region"].cat.categories) == ["reg001_cells", "reg002_cells"] assert table.uns["spatialdata_attrs"]["region"] == ["reg001_cells", "reg002_cells"] assert table.obs["cell_id"].dtype == np.int64 + assert tables[0].centroids_px is not None np.testing.assert_allclose(np.asarray(table.obsm["spatial"])[:3], tables[0].centroids_px) assert sorted(shapes) == ["reg001_cells", "reg002_cells"] assert list(shapes["reg002_cells"].index) == [1, 2, 3, 4] @@ -231,6 +233,7 @@ def test_qupath_um_to_px(tmp_path: Path) -> None: table = read_cell_table(path, "qupath", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) + assert table.centroids_px is not None np.testing.assert_allclose(table.centroids_px, [[20, 10], [40, 30]]) assert list(table.var.index) == ["CD3", "CD8"] assert list(table.var["compartment"]) == ["Cell", "Nucleus"] @@ -262,6 +265,7 @@ def test_mcquant_columns(tmp_path: Path) -> None: assert list(table.var.index) == ["DAPI", "CD3"] np.testing.assert_allclose(table.X, cells[["DAPI", "CD3"]], rtol=1e-6) + assert table.centroids_px is not None np.testing.assert_allclose(table.centroids_px, cells[["X_centroid", "Y_centroid"]]) assert list(table.instance_id) == [1, 2, 3, 4, 5] assert list(table.region.unique()) == ["S"] @@ -292,6 +296,7 @@ def test_generic_kwargs(tmp_path: Path) -> None: table = read_cell_table(path, "generic", pixel_size_um=2.0, default_region=None, cells_table_kwargs=kwargs) + assert table.centroids_px is not None np.testing.assert_allclose(table.centroids_px, [[1, 2], [3, 4]]) assert list(table.var.index) == ["CD3", "CD8"] assert list(table.instance_id) == [7, 9] @@ -324,13 +329,135 @@ def test_generic_bad_kwargs_raise(tmp_path: Path) -> None: ) -@pytest.mark.parametrize("fmt", ["sprm", "inform"]) -def test_sprm_inform_not_implemented(tmp_path: Path, fmt: str) -> None: - path = tmp_path / "cells.csv" - path.write_text("ID,DAPI\n1,2\n") +def _write_sprm(directory: Path, *, centers: bool = True) -> pd.DataFrame: + rng = np.random.default_rng(3) + means = pd.DataFrame({"ID": [1, 2, 4], "DAPI": rng.uniform(0, 1, 3), "CD3": rng.uniform(0, 1, 3)}) + means.to_csv(directory / "reg1.ome.tiff-cell_channel_mean.csv", index=False) + if centers: + # more cells than in the means table, in another order + pd.DataFrame({"ID": [4, 3, 2, 1], "x": [40.0, 30.0, 20.0, 10.0], "y": [4.0, 3.0, 2.0, 1.0]}).to_csv( + directory / "reg1.ome.tiff-cell_centers.csv", index=False + ) + return means + + +def test_sprm_adapter(tmp_path: Path) -> None: + means = _write_sprm(tmp_path) + path = tmp_path / "reg1.ome.tiff-cell_channel_mean.csv" + + assert detect_table_format(path) == "sprm" + table = read_cell_table(path, "auto", pixel_size_um=None, default_region="reg1", cells_table_kwargs={}) + + assert list(table.var.index) == ["DAPI", "CD3"] + np.testing.assert_allclose(table.X, means[["DAPI", "CD3"]], rtol=1e-6) + assert list(table.instance_id) == [1, 2, 4] + assert not table.row_instance_ids + assert list(table.region.unique()) == ["reg1"] + assert table.centroids_px is not None + assert table.centroids_px is not None + np.testing.assert_allclose(table.centroids_px, [[10, 1], [20, 2], [40, 4]]) + assert table.radius_px is None + + +def test_sprm_without_centers_warns_and_omits_spatial(tmp_path: Path) -> None: + _write_sprm(tmp_path, centers=False) + path = tmp_path / "reg1.ome.tiff-cell_channel_mean.csv" - with pytest.raises(NotImplementedError, match="later release"): - read_cell_table(path, fmt, pixel_size_um=None, default_region="S", cells_table_kwargs={}) + with pytest.warns(UserWarning, match="cell_centers"): + table = read_cell_table(path, "sprm", pixel_size_um=None, default_region="reg1", cells_table_kwargs={}) + assert table.centroids_px is None + + mask = np.zeros((6, 6), dtype=np.uint16) + mask[1:3, 1:3], mask[4, 4] = 1, 2 + labels = Labels2DModel.parse(mask, dims=("y", "x")) + sdata_tables, shapes = build_cells([table], {}, {"reg1": labels}, {}) + assert "spatial" not in sdata_tables["table"].obsm + assert shapes == {} + with pytest.raises(ValueError, match="centroids"): + build_cells([table], {}, {}, {}) + + +def test_sprm_centers_missing_cells_raise(tmp_path: Path) -> None: + _write_sprm(tmp_path) + pd.DataFrame({"ID": [1, 2], "x": [1.0, 2.0], "y": [1.0, 2.0]}).to_csv( + tmp_path / "reg1.ome.tiff-cell_centers.csv", index=False + ) + + with pytest.raises(ValueError, match="cell_centers"): + read_cell_table( + tmp_path / "reg1.ome.tiff-cell_channel_mean.csv", + "sprm", + pixel_size_um=None, + default_region="reg1", + cells_table_kwargs={}, + ) + + +def _write_inform(path: Path, unit: str | None, area_unit: str = "square microns") -> None: + suffix = "" if unit is None else f" ({unit})" + header = [ + "Sample Name", + "Cell ID", + f"Cell X Position{suffix}", + f"Cell Y Position{suffix}", + f"Entire Cell Area ({area_unit})", + "Entire Cell DAPI Mean (Normalized Counts, Total Weighting)", + "Entire Cell CD8 (Opal 520) Mean (Normalized Counts, Total Weighting)", + "Nucleus DAPI Mean (Normalized Counts, Total Weighting)", + "Phenotype", + ] + rows = [ + ["Scan1.im3", "1", "100", "50", "314.159", "1.5", "2.5", "9", "CD8+"], + ["Scan1.im3", "2", "200", "80", "78.54", "3.5", "4.5", "9", "other"], + ] + path.write_text("\n".join("\t".join(row) for row in [header, *rows]) + "\n", encoding="utf-8") + + +def test_inform_adapter_microns(tmp_path: Path) -> None: + path = tmp_path / "Scan1_cell_seg_data.txt" + _write_inform(path, "microns") + + assert detect_table_format(path) == "inform" + table = read_cell_table(path, "inform", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) + + assert table.centroids_px is not None + np.testing.assert_allclose(table.centroids_px, [[200, 100], [400, 160]]) + assert list(table.var.index) == ["DAPI", "CD8 (Opal 520)"] + np.testing.assert_allclose(table.X, [[1.5, 2.5], [3.5, 4.5]]) + assert list(table.instance_id) == [1, 2] + assert not table.row_instance_ids + assert list(table.region) == ["Scan1.im3"] * 2 + assert table.radius_px is not None + np.testing.assert_allclose(table.radius_px, np.sqrt(np.array([314.159, 78.54]) / 0.25 / np.pi)) + assert "Phenotype" in table.obs.columns + with pytest.raises(ValueError, match="pixel size"): + read_cell_table(path, "inform", pixel_size_um=None, default_region=None, cells_table_kwargs={}) + + +def test_inform_adapter_pixels(tmp_path: Path) -> None: + path = tmp_path / "Scan1_cell_seg_data.txt" + _write_inform(path, "pixels", area_unit="pixels") + + table = read_cell_table(path, "auto", pixel_size_um=None, default_region="S", cells_table_kwargs={}) + + assert table.centroids_px is not None + np.testing.assert_allclose(table.centroids_px, [[100, 50], [200, 80]]) + assert table.radius_px is not None + np.testing.assert_allclose(table.radius_px, np.sqrt(np.array([314.159, 78.54]) / np.pi)) + + +def test_inform_without_unit_is_pixels_and_renames_cell_id(tmp_path: Path) -> None: + path = tmp_path / "Scan1_cell_seg_data.txt" + _write_inform(path, None) + + table = read_cell_table(path, "inform", pixel_size_um=None, default_region=None, cells_table_kwargs={}) + assert table.centroids_px is not None + np.testing.assert_allclose(table.centroids_px, [[100, 50], [200, 80]]) + + # mixed units (px positions, µm areas) cannot give a radius + assert table.radius_px is None + elements, _ = build_cells([table], {}, {}, {}) + assert "source_Cell ID" in elements["table"].obs.columns or "source_Cell_ID" in elements["table"].obs.columns def test_unknown_format_raises(tmp_path: Path) -> None: From 814f037570bb41731f0122eed3074199f2755ff1 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 16:45:13 +0200 Subject: [PATCH 016/107] docs: document Akoya reader rewrite and add real-data tests Add CHANGELOG entries for phenocycler() and the breaking codex() rewrite, mention PhenoCycler-Fusion in the README, complete the reader docstrings (coordinate systems of image-less regions, kept polygons, source_ columns, sanitised obs names, sprm/inform formats), name the kept polygons in the mask-wins warning, and add tests on real public datasets that run only when AKOYA_TEST_DATA is set. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 24 +++ README.md | 2 +- .../readers/_utils/_cell_tables.py | 6 +- src/spatialdata_io/readers/codex.py | 40 ++-- src/spatialdata_io/readers/phenocycler.py | 33 ++-- tests/readers/test_rasterize.py | 3 +- tests/test_akoya_real_data.py | 176 ++++++++++++++++++ 7 files changed, 250 insertions(+), 34 deletions(-) create mode 100644 tests/test_akoya_real_data.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d298d1ad..e124b28d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,12 +22,36 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. and the raster cell/nucleus labels are reconstructed by rasterizing the boundary polygons. The cell feature matrix falls back to a `cell_feature_matrix/` MatrixMarket directory or `cell_feature_matrix.tar.gz` when `cell_feature_matrix.h5` is missing. +- `phenocycler()` reader (and `phenocycler` CLI command) for Akoya PhenoCycler-Fusion 1.x/2.x `.qptiff` / + `.er.qptiff` scans, including PhenoImager (Opal) mode: lazy level-0 image with a recomputed pyramid, channel names + from the per-page XML, structured and raw scan metadata in `sdata.attrs["akoya"]`, optional `.xpd` / + `*channels.csv` sidecars, and optional cell tables (QuPath with GeoJSON outlines, MCMICRO `mcquant`, HuBMAP SPRM, + inForm, CODEX Processor or generic CSV) with segmentation masks. ### Changed - Adopted the current `cookiecutter-scverse` template: `hatch`-managed test environments, `uv`-based CI and documentation builds, `mypy` type checking of `src` and `tests`, `biome`/`pyproject-fmt`/`zizmor` pre-commit hooks, and Dependabot updates. +- `codex()` was rewritten to read CODEX Processor, CODEX Uploader and CODEX Instrument Manager outputs. This changes + its output and signature: + - Element names are now `{region}_image`, `{region}_cells` (circles), `{region}_labels`, `{region}_polygons` and + `table` (previously `images` and `{region}` for the shapes). + - Every region is read (previously only the first file found); `regions=[...]` selects a subset. + - Marker names in `table.var_names` come from the FCS `$PnS` keywords or the CSV header `cycCCC_chHHH:` + (previously `cycCCC_chHHH`); the image channels are named too. + - The positional third parameter is now `regions` (previously `imread_kwargs`); pass `imread_kwargs` by keyword. + It is only used by the `dask_image.imread` fallback for TIFFs that `tifffile` cannot read lazily. + - A single region is stored in `"global"`. Several regions are placed at their stage positions in `"global"` only + when all of them share a sample space (same slide id, stage positions and pixel size); otherwise each region, + including multi-region reads of unrelated regions, gets its own coordinate system named after the region. + - New parameters `channel_names`, `cells_table`, `cells_table_format`, `labels`, `image_models_kwargs`, + `labels_models_kwargs` and `cells_table_kwargs`; the `codex` CLI command exposes them. + +### Fixed + +- `codex()` read the CSV table and the image relative to the current working directory instead of `path`, and an + operator-precedence bug in the table check made it ignore `fcs`. ### Removed diff --git a/README.md b/README.md index adb20e76..4e431c62 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ This package contains reader functions to load common spatial omics formats into - 10x Genomics Visium® - 10x Genomics Visium HD® - 10x Genomics Xenium® -- Akoya PhenoCycler® (formerly CODEX®) +- Akoya PhenoCycler® (formerly CODEX®): CODEX Processor/Uploader/Instrument Manager outputs and PhenoCycler-Fusion 1.x/2.x - Curio Seeker® - DBiT-seq - MCMICRO (output data) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index d5311cfb..ce77946e 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -989,7 +989,8 @@ def polygon_labels( """Lazily rasterise the cell polygons of each region into labels whose values are the cells' instance ids. Where cells overlap, the one with the higher instance id wins. Regions with a mask in ``labels`` keep it, with - a warning; regions without an image are skipped, their cells become circles. + a warning; regions without an image are skipped, their cells become circles. In both cases the polygons are + still kept as shapes, see :func:`polygon_shapes`. Parameters ---------- @@ -1020,7 +1021,8 @@ def polygon_labels( row_ids = " Its cells' instance ids are row numbers and may not match the mask's values." warnings.warn( f"Region '{region}' has both a segmentation mask and the cell polygons {table.geojson}; the mask " - f"is used.{row_ids if table.row_instance_ids else ''}", + f"is used for the labels and the polygons are kept as '{region}{PhenocyclerKeys.POLYGONS_SUFFIX}'." + f"{row_ids if table.row_instance_ids else ''}", UserWarning, stacklevel=3, ) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 8446b5ed..0e77a63c 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -119,19 +119,26 @@ def codex( ``bestFocus/``) are ignored. The region id ``regNNN`` is taken from the file names; a ``{vx.QPTIFF!r}`` without one is named after its file. If a directory contains no such files, its top-level ``{vx.IMAGE_TIF}(f)`` files are read as hyperstacks named after their files. Each region is stored as the image ``_image`` and its - metadata in ``sdata.attrs["akoya"]``. A single region is stored in the ``"global"`` coordinate system. Several - CIM scans of the same sample space (same slide, with stage positions and equal pixel sizes) are placed at their - stage positions in ``"global"``; otherwise (e.g. CODEX Processor regions, which have no stage positions) each - region is stored in its own coordinate system, named after the region. All elements of a region share its - coordinate system. The cells of each region are stored as circles ``_cells`` with the cells' area - (``size`` column), annotated by one table ``"table"`` (``obs["region"]`` names the circles, ``obs["cell_id"]`` - is the instance key), or by one table ``_table`` per region if the regions have different markers. A region with a segmentation mask in - ``labels`` gets the labels ``_labels`` instead of circles, and its cells annotate them; so does a - region with the cell outlines of a QuPath GeoJSON export (see ``cells_table_kwargs``), rasterised lazily one - chunk at a time with the cells' instance ids (the higher id wins where cells overlap); the outlines are also kept - as the shapes ``_polygons`` in pixels, indexed by the same instance ids. The paths of the tables, - masks and GeoJSON files read are stored in ``sdata.attrs["akoya"]["cell_tables"]``, ``["labels"]`` and - ``["geojson"]``. + metadata in ``sdata.attrs["akoya"]``. + + A single region is stored in the ``"global"`` coordinate system. Several CIM scans of the same sample space + (same slide, with stage positions and equal pixel sizes) are placed at their stage positions in ``"global"``; + otherwise (e.g. CODEX Processor regions, which have no stage positions) each region is stored in its own + coordinate system, named after the region. A region with cells or a mask but no image is unrelated to the + scans, so it also gets its own coordinate system. All elements of a region share its coordinate system. + + The cells of each region are stored as circles ``_cells`` with the cells' area (``size`` column), + annotated by one table ``"table"`` (``obs["region"]`` names the annotated element, ``obs["cell_id"]`` is the + instance key), or by one table ``_table`` per region if the regions have different markers. Columns of + the cell table that clash with ``region`` or ``cell_id`` are kept as ``source_``, and column names that + spatialdata cannot write (e.g. QuPath's ``Centroid X µm``) are sanitised (``Centroid_X_µm``). A region with a + segmentation mask in ``labels`` gets the labels ``_labels`` instead of circles, and its cells annotate + them; so does a region with the cell outlines of a QuPath GeoJSON export (see ``cells_table_kwargs``), + rasterised lazily one chunk at a time with the cells' instance ids (the higher id wins where cells overlap). + The outlines are always kept as the shapes ``_polygons`` in pixels, indexed by the same instance ids, + also when a mask is given for the region (the mask is used for the labels, with a warning) or the region has no + image (its cells are then circles). The paths of the tables, masks and GeoJSON files read are stored in + ``sdata.attrs["akoya"]["cell_tables"]``, ``["labels"]`` and ``["geojson"]``. .. seealso:: @@ -155,8 +162,9 @@ def codex( ``cells_table_kwargs["region_column"]``). See ``cells_table_format`` for the formats. cells_table_format The format of ``cells_table``: ``"auto"`` (detected from the header), ``"codex"``, ``"qupath"`` (QuPath - measurements; µm centroids are converted with the region's pixel size), ``"mcquant"`` (MCMICRO) or - ``"generic"`` (columns given by ``cells_table_kwargs``). + measurements; µm centroids are converted with the region's pixel size), ``"mcquant"`` (MCMICRO), ``"sprm"`` + (HuBMAP SPRM ``*-cell_channel_mean.csv``), ``"inform"`` (inForm ``*_cell_seg_data.txt``) or ``"generic"`` + (columns given by ``cells_table_kwargs``). labels A segmentation mask (TIFF/OME-TIFF with integer instance ids) per region: a path for a single region, or a mapping from region id to path. Each mask is stored lazily as the labels ``_labels`` @@ -164,7 +172,7 @@ def codex( their instance ids must be the mask values. imread_kwargs Keyword arguments passed to :func:`dask_image.imread.imread`, which is only used for ``{vx.IMAGE_TIF}(f)`` - files that :mod:`tifffile` cannot read lazily (e.g. unsupported compression). + files that ``tifffile`` cannot read lazily (e.g. unsupported compression). image_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. labels_models_kwargs diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 9b1b6b84..8fa3d353 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -82,23 +82,27 @@ def phenocycler( ``{vx.IMAGE_SUFFIX}``. A single region is stored in the ``"global"`` coordinate system. Several regions of the same sample space (scanned on the same slide, with stage positions and equal pixel sizes) are placed at their stage positions in ``"global"``; otherwise the regions are unrelated and each is stored in its own - coordinate system, named after the region. All elements of a region share its coordinate system. The - per-region metadata is stored in ``sdata.attrs[{vx.ATTRS_KEY!r}]``. + coordinate system, named after the region. A region with cells or a mask but no scan gets its own coordinate + system too. All elements of a region share its coordinate system. The per-region metadata is stored in + ``sdata.attrs[{vx.ATTRS_KEY!r}]``. A segmentation mask given in ``labels`` is stored lazily as the labels ``{vx.LABELS_SUFFIX}`` with the transformation of the region's image. The cells of ``cells_table`` are stored in one table ``{vx.TABLE_KEY!r}`` (marker intensities dense float32 in ``X``, pixel centroids in ``obsm["spatial"]``, ``obs[{vx.REGION_KEY!r}]`` the annotated element, ``obs[{vx.INSTANCE_KEY!r}]`` the instance key), or in one - table ``_table`` per region if the regions have different markers. Cells annotate the labels of their - region, whose values must be their instance ids; without a mask they are stored as circles - ``{vx.CELLS_SUFFIX}``, with a warning. The cell outlines of a QuPath GeoJSON export (see - ``cells_table_kwargs``) are rasterised lazily, one chunk at a time, into the labels of a region without a - mask; a feature's ``id`` is matched to the table's ``Object ID`` (else features and rows are matched in order), - the labels hold the cells' instance ids, and where cells overlap the higher instance id wins. The outlines are - also kept as the shapes ``{vx.POLYGONS_SUFFIX}`` in pixels, indexed by the same instance ids; the table - annotates the labels. The paths of the - tables, masks and GeoJSON files are stored in ``sdata.attrs[{vx.ATTRS_KEY!r}]["cell_tables"]``, ``["labels"]`` - and ``["geojson"]``. + table ``_table`` per region if the regions have different markers. Columns of the cell table that clash + with ``{vx.REGION_KEY!r}`` or ``{vx.INSTANCE_KEY!r}`` are kept as ``source_``, and column names that + spatialdata cannot write (e.g. QuPath's ``Centroid X µm``) are sanitised (``Centroid_X_µm``). Cells annotate + the labels of their region, whose values must be their instance ids; without a mask they are stored as circles + ``{vx.CELLS_SUFFIX}``, with a warning. + + The cell outlines of a QuPath GeoJSON export (see ``cells_table_kwargs``) are rasterised lazily, one chunk at a + time, into the labels of a region without a mask; a feature's ``id`` is matched to the table's ``Object ID`` + (else features and rows are matched in order), the labels hold the cells' instance ids, and where cells overlap + the higher instance id wins; the table annotates the labels. The outlines are always kept as the shapes + ``{vx.POLYGONS_SUFFIX}`` in pixels, indexed by the same instance ids, also when a mask is given for the + region (the mask is used for the labels, with a warning). The paths of the tables, masks and GeoJSON files are + stored in ``sdata.attrs[{vx.ATTRS_KEY!r}]["cell_tables"]``, ``["labels"]`` and ``["geojson"]``. .. seealso:: @@ -121,7 +125,8 @@ def phenocycler( cells_table_format The format of ``cells_table``: ``"auto"`` (detected from the header), ``"codex"`` (CODEX Processor), ``"qupath"`` (QuPath measurements; µm centroids are converted with the region's pixel size), ``"mcquant"`` - (MCMICRO) or ``"generic"`` (columns given by ``cells_table_kwargs``). + (MCMICRO), ``"sprm"`` (HuBMAP SPRM ``*-cell_channel_mean.csv``), ``"inform"`` (inForm + ``*_cell_seg_data.txt``) or ``"generic"`` (columns given by ``cells_table_kwargs``). labels A segmentation mask (TIFF/OME-TIFF with integer instance ids) per region: a path for a single region, or a mapping from region id to path. @@ -293,7 +298,7 @@ def _read_sidecars(scan: AkoyaScan) -> None: def _find_sidecar(qptiff: Path, base: str, suffix: str) -> Path | None: - """First file ending in ``suffix`` whose name starts with ``base``, next to the scan or in HuBMAP's processed dir.""" + """First file ending in ``suffix`` whose name starts with ``base``, next to the scan or in HuBMAP processed dir.""" pattern = re.compile(rf"{re.escape(base)}(?![0-9A-Za-z]).*{re.escape(suffix)}$") directories = [qptiff.parent] # HuBMAP keeps `raw/images/.qptiff` and `lab_processed/images/.qptiff.channels.csv` diff --git a/tests/readers/test_rasterize.py b/tests/readers/test_rasterize.py index 06baa1f1..b880c53e 100644 --- a/tests/readers/test_rasterize.py +++ b/tests/readers/test_rasterize.py @@ -236,12 +236,13 @@ def test_qupath_geojson_mask_wins(tmp_path: Path) -> None: mask[100:110, 100:110] = 1 tifffile.imwrite(tmp_path / "mask.tif", mask) - with pytest.warns(UserWarning, match="mask .* is used"): + with pytest.warns(UserWarning, match="mask .* is used for the labels and the polygons are kept as 'S_polygons'"): sdata = phenocycler( tmp_path, cells_table=tmp_path / "cells.tsv", labels=tmp_path / "mask.tif", image_models_kwargs=_FAST ) np.testing.assert_array_equal(sdata.labels["S_labels"]["scale0"]["image"].values, mask) + assert "S_polygons" in sdata.shapes def test_qupath_geojson_errors(tmp_path: Path) -> None: diff --git a/tests/test_akoya_real_data.py b/tests/test_akoya_real_data.py new file mode 100644 index 00000000..999446eb --- /dev/null +++ b/tests/test_akoya_real_data.py @@ -0,0 +1,176 @@ +"""Tests of the Akoya readers on real public datasets. + +They run only when the environment variable ``AKOYA_TEST_DATA`` points to a directory holding the files below (paths +relative to it). The files are public (Zenodo, Figshare, BioImage Archive, GitHub) but too large to download in CI. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + +import dask.array as da +import pytest +from spatialdata.models import get_channel_names + +from spatialdata_io import codex, phenocycler + +_ROOT = os.environ.get("AKOYA_TEST_DATA") + +pytestmark = [ + pytest.mark.skipif(_ROOT is None, reason="set AKOYA_TEST_DATA to the directory of the real Akoya datasets"), + # upstream noise unrelated to the readers: the open TIFF handles of lazy images, ome-zarr's deprecated `Scaler` + pytest.mark.filterwarnings("ignore::ResourceWarning", "ignore:Call to deprecated class Scaler"), +] + + +@dataclass(frozen=True) +class Scan: + """What a real dataset is expected to read as.""" + + path: str + reader: str + region: str + generation: str + n_channels: int + first_names: tuple[str, ...] + pixel_size_um: float | None + shape: tuple[int, int, int] | None = None + + +_SCANS = [ + Scan( + "figshare_25123538/AML2_183_Scan1.qptiff", + "phenocycler", + "AML2_183", + "fusion1", + 54, + ("DAPI", "NPM1C", "ASMA", "CD4"), + 0.5069, + (54, 10800, 7680), + ), + Scan( + "zenodo_7916784/DON126-B3A.qptiff", + "phenocycler", + "DON126-B3A", + "fusion1", + 28, + ("DAPI", "GCG", "CD45", "CD19"), + 0.5079, + ), + Scan( + "zenodo_16732565/VI_CODEX_12_Scan1.qptiff", + "phenocycler", + "VI_CODEX_12", + "fusion2", + 15, + ("DAPI", "GATA6", "S100a2", "LamininG"), + 0.5062, + ), + Scan( + "zenodo_14741083/EWS_Akoya_Slide6.qptiff", + "phenocycler", + "EWS_Akoya_Slide6", + "fusion2", + 42, + ("DAPI", "WGA", "Na/K ATPase", "CD11c"), + 0.5075, + ), + # Fusion 2.x in PhenoImager (Opal) mode: the names come from ``, there is no `` + Scan( + "bia_S-BIAD3467/Tonsil-ctrl_Scan1.qptiff", + "phenocycler", + "Tonsil-ctrl", + "fusion2", + 6, + ("DAPI", "Opal 690", "Opal 620", "Opal 780", "Opal 520", "Sample AF"), + 0.4994, + ), + Scan( + "figshare_31956057/20210825-ipLN-9-hpi-VSV.qptiff", + "codex", + "20210825-ipLN-9-hpi-VSV", + "cim_qptiff", + 37, + ("DAPI", "Runx2", "CXCR4", "VSV"), + 0.3774, + ), + # tifffile's QPI series builder finds only 3 of the 5 levels; the full-resolution shape must still be right + Scan( + "zenodo_22107401/Arlene_Tonsil_9x9.qptiff", + "codex", + "Arlene_Tonsil_9x9", + "cim_qptiff", + 43, + ("DAPI-01", "HLA-A", "CD20", "Vimentin"), + 0.3774, + (43, 9508, 12672), + ), + # CODEX Uploader montage, TCYX (21, 4, y, x); the names come from `channelNames.txt` + Scan( + "bia_S-BIAD859/stitched/6371v1", + "codex", + "reg001", + "legacy_processor", + 84, + ("HOECHST", "CollIV", "NaKATPase", "HLA-ABC"), + None, + (84, 3528, 3360), + ), + # CODEX Processor per-marker stitched images `reg001_cycCCC_chHHH_.tiff` + Scan( + "zenodo_7742474/human_pancreas_codex_zip_subset", + "codex", + "reg001", + "legacy_processor", + 2, + ("Hoechst", "CGC"), + None, + (2, 18440, 25816), + ), +] + + +@pytest.mark.parametrize("scan", _SCANS, ids=[scan.path.split("/")[-1] for scan in _SCANS]) +def test_real_scan(scan: Scan) -> None: + path = Path(_ROOT) / scan.path # type: ignore[arg-type] + if not path.exists(): + pytest.skip(f"{path} is missing") + + sdata = (phenocycler if scan.reader == "phenocycler" else codex)(path) + + image = sdata.images[f"{scan.region}_image"] + data = image["scale0"]["image"].data + assert isinstance(data, da.Array) + assert data.shape[0] == scan.n_channels + if scan.shape is not None: + assert data.shape == scan.shape + names = get_channel_names(image) + assert tuple(names[: len(scan.first_names)]) == scan.first_names + assert len(set(names)) == len(names) + metadata = sdata.attrs["akoya"]["regions"][scan.region] + assert metadata["generation"] == scan.generation + assert len(metadata["channels"]) == scan.n_channels + if scan.pixel_size_um is None: + assert metadata["pixel_size_um"] is None + else: + assert metadata["pixel_size_um"] == pytest.approx(scan.pixel_size_um, abs=1e-3) + assert sdata.attrs["spatialdata_io_reader"] == scan.reader + + +def test_real_codex_processor_csv() -> None: + # a 500-cell subset of a CODEX Processor CSV with the `cyc001_ch001:DAPI1` header (github.com/igordot/phenomenalist) + path = Path(_ROOT) / "github_codex_metadata/phenomenalist/tonsil-akoya-2018-500.csv" # type: ignore[arg-type] + if not path.exists(): + pytest.skip(f"{path} is missing") + + with pytest.warns(UserWarning, match="cells are represented as circles"): + sdata = codex(path) + + table = sdata.tables["table"] + assert table.shape == (500, 32) + assert list(table.var_names[:5]) == ["DAPI1", "Blank", "Blank (2)", "Blank (3)", "DAPI2"] + assert list(table.obs["region"].cat.categories) == ["reg001_cells"] + assert table.obsm["spatial"].shape == (500, 2) + assert len(sdata.shapes["reg001_cells"]) == 500 From 853509ec6288de8bc7096020e882b7de89255cd9 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 17:16:30 +0200 Subject: [PATCH 017/107] fix(codex): find channelNames.txt and experiment.json case-insensitively SPACEc and the HuBMAP codex-pipeline write `channelnames.txt`; an exact name match silently dropped the channel names. Co-Authored-By: Claude Opus 5.5 --- src/spatialdata_io/readers/codex.py | 22 +++++++++++++++++----- tests/test_codex.py | 15 +++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 0e77a63c..328c3041 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -107,7 +107,9 @@ def codex( into channels, in file order. The channel names come from the ImageJ ``Labels``, else from ``{vx.CHANNEL_NAMES_FILE!r}`` (one name per line, cycle-major), else they are ``cycCCC_chHHH``. - ``*{vx.QPTIFF!r}``: scans of the CODEX Instrument Manager, read lazily with their channel metadata. - - ``{vx.EXPERIMENT_FILE!r}``: optional; the pixel size is taken from its ``xyResolution`` (nm). + - ``{vx.EXPERIMENT_FILE!r}``: optional; the pixel size is taken from its ``xyResolution`` (nm). Like + ``{vx.CHANNEL_NAMES_FILE!r}``, it is found in ``path`` or up to three folders above it, ignoring the case + of the file name (e.g. ``channelnames.txt``). - ``*{vx.FCS_FILE!r}`` / ``*{vx.FCS_FILE_CSV!r}``: cell tables of the CODEX Processor, one per region (or one for all regions), e.g. ``segm/segm-1/fcs/compensated/regNNN_compensated.fcs``. Compensated tables are preferred over uncompensated ones. The ``cycCCC_chHHH`` marker columns are stored in ``X`` (dense @@ -558,14 +560,24 @@ def rank(file: Path) -> tuple[bool, bool]: def _find_sidecar(path: Path, files: Sequence[Path], name: str) -> Path | None: - """The shallowest file called ``name`` in ``files``, else in the nearest ancestor of ``path`` holding one.""" - matches = sorted((file for file in files if file.name == name), key=lambda file: (len(file.parts), str(file))) + """The shallowest file called ``name`` in ``files``, else in the nearest ancestor of ``path`` holding one. + + Names are compared case-insensitively: SPACEc and the HuBMAP codex-pipeline write ``channelnames.txt``. + """ + folded = name.casefold() + matches = sorted( + (file for file in files if file.name.casefold() == folded), key=lambda file: (len(file.parts), str(file)) + ) if matches: return matches[0] start = path.parent if path.is_file() else path for directory in [start, *start.parents][: _SIDECAR_ANCESTORS + 1]: - if (directory / name).is_file(): - return directory / name + try: + found = sorted(file for file in directory.iterdir() if file.name.casefold() == folded and file.is_file()) + except OSError: # e.g. an unreadable ancestor directory + continue + if found: + return found[0] return None diff --git a/tests/test_codex.py b/tests/test_codex.py index f9f51c06..d68b5fb7 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -57,6 +57,21 @@ def test_codex_hyperstack_channel_names_txt(tmp_path: Path) -> None: np.testing.assert_array_equal(image.values, data["images"]["reg001"].reshape(8, *image.shape[1:])) +@pytest.mark.parametrize("from_subdirectory", [False, True]) +def test_codex_sidecar_names_are_case_insensitive(tmp_path: Path, from_subdirectory: bool) -> None: + # SPACEc and the HuBMAP codex-pipeline write `channelnames.txt`; older exports `Experiment.json` + data = write_legacy_processor_tree(tmp_path, cycles=2, channels=4, layout="hyperstack") + (tmp_path / "channelNames.txt").rename(tmp_path / "channelnames.txt") + (tmp_path / "experiment.json").unlink() + (tmp_path / "Experiment.json").write_text(json.dumps({"xyResolution": 377.5})) + source = tmp_path / "processed_2020-01-01" / "stitched" if from_subdirectory else tmp_path + + sdata = codex(source, image_models_kwargs=_FAST) + + assert get_channel_names(sdata.images["reg001_image"]) == dedupe_channel_names(data["markers"]) + assert sdata.attrs["akoya"]["regions"]["reg001"]["pixel_size_um"] == pytest.approx(0.3775) + + def test_codex_hyperstack_bad_channel_names_warns(tmp_path: Path) -> None: data = write_legacy_processor_tree(tmp_path, cycles=2, channels=4, layout="hyperstack") (tmp_path / "channelNames.txt").write_text("\n".join(data["markers"][:7]) + "\n") From 183685a5b60d9dce7558c00a16b7885c45fc00e0 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 17:19:27 +0200 Subject: [PATCH 018/107] fix: keep cell-table columns named region or cell_id as source_ Columns named exactly like the region or instance key were dropped even when they were not read as the regions or instance ids, contradicting the reader docs. Only the column an adapter reads as the key is now dropped. Co-Authored-By: Claude Opus 5.5 --- .../readers/_utils/_cell_tables.py | 19 +++++++--- tests/readers/test_cell_tables.py | 5 ++- tests/test_phenocycler.py | 38 +++++++++++++++++++ 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index ce77946e..bdef00f8 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -236,7 +236,7 @@ def read_codex_table(path: Path, region: str | None = None) -> CellTable: ) region_ids.name = region_key return CellTable( - obs=obs, + obs=_drop_key_columns(obs, columns.get(instance_key), columns.get(region_key)), X=X, var=var, region=region_ids, @@ -728,7 +728,7 @@ def _from_frame( ) region.name = CodexKeys.REGION_KEY.value - obs = df.drop(columns=markers) + obs = _drop_key_columns(df.drop(columns=markers), instance_column, region_column) obs.index = index radius = None if area_column is not None: @@ -745,6 +745,15 @@ def _from_frame( ) +def _drop_key_columns(obs: pd.DataFrame, instance_column: str | None, region_column: str | None) -> pd.DataFrame: + """Drop the instance id and region columns if they are named like the keys whose values replace them. + + Any other column named like a key keeps its values as ``source_`` in the table element. + """ + consumed = {CodexKeys.INSTANCE_KEY.value: instance_column, CodexKeys.REGION_KEY.value: region_column} + return obs.drop(columns=[key for key, column in consumed.items() if column == key]) + + def _read_header(path: Path) -> tuple[list[str], str]: """The column names of a delimited text table and its delimiter (tab if the header has one, else comma).""" with path.open(encoding="utf-8-sig", errors="replace") as f: @@ -1196,10 +1205,10 @@ def _to_anndata(tables: Sequence[CellTable], element: Callable[[str], str]) -> a """Concatenate cell tables with the same markers into one table annotating the elements of their regions.""" region_key, instance_key = CodexKeys.REGION_KEY.value, CodexKeys.INSTANCE_KEY.value obs = pd.concat([table.obs for table in tables]) if len(tables) > 1 else tables[0].obs.copy() - # the region and instance keys are set below; other columns that would clash with them (spatialdata names are - # case-insensitively unique, e.g. `Cell ID` -> `Cell_ID` vs `cell_id`) keep their values under `source_` + # the region and instance keys are set below; the adapters dropped the columns they were read from, and other + # columns that would clash with them (also case-insensitively, as spatialdata requires, e.g. `Cell ID` -> + # `Cell_ID` vs `cell_id`) keep their values under `source_` reserved = {region_key, instance_key} - obs = obs.drop(columns=[column for column in obs.columns if column in reserved]) obs = obs.rename( columns={ column: f"source_{column}" diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index 1446c6c1..a3cc48ef 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -81,7 +81,8 @@ def test_read_codex_csv_colon_header(tmp_path: Path) -> None: assert list(table.var.index) == ["DAPI1", "CD3"] assert list(table.var["source_column"]) == ["cyc001_ch001:DAPI1", "cyc001_ch002:CD3"] np.testing.assert_array_equal(table.X, np.array([[100.5, 7], [200.5, 8]], dtype=np.float32)) - assert list(table.obs.columns) == ["cell_id", "region", "x", "y", "size"] + # the columns read as the instance ids and regions are replaced by the keys of the table element + assert list(table.obs.columns) == ["x", "y", "size"] assert list(table.region) == ["reg002", "reg002"] np.testing.assert_array_equal(table.centroids_px, [[10, 20], [11, 21]]) @@ -94,7 +95,7 @@ def test_read_codex_csv_marker_name_columns(tmp_path: Path) -> None: assert list(table.var.index) == ["CD3", "CD8"] assert "cycle" not in table.var.columns - assert list(table.obs.columns) == ["cell_id", "x", "y", "tile_num", "label"] + assert list(table.obs.columns) == ["x", "y", "tile_num", "label"] assert list(table.region) == ["Kidney"] assert table.radius_px is None diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 1350fd82..c19553c5 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -380,6 +380,44 @@ def test_generic_columns_clashing_with_reserved_keys(tmp_path: Path, header: str assert read_zarr(tmp_path / "x.zarr").tables["table"].n_obs == 2 +@pytest.mark.parametrize( + ("header", "row", "fmt", "kwargs"), + [ + ( + "CELL,X,Y,CD3,region,cell_id", + "{i},1,2,3,other,{source}", + "generic", + {"x": "X", "y": "Y", "instance_id": "CELL"}, + ), + ("CellID,DAPI,CD3,X_centroid,Y_centroid,Area,region,cell_id", "{i},1,2,3,4,5,other,{source}", "mcquant", {}), + ], +) +def test_extra_region_and_cell_id_columns_are_kept( + tmp_path: Path, header: str, row: str, fmt: str, kwargs: dict[str, object] +) -> None: + # columns named like the region and instance keys, but not read as them, keep their values as `source_` + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + rows = "\n".join(row.format(i=i, source=100 + i) for i in (1, 2)) + (tmp_path / "cells.csv").write_text(f"{header}\n{rows}\n") + if fmt == "generic": + kwargs = {**kwargs, "markers": ["CD3"]} + + with pytest.warns(UserWarning, match="circles"): + sdata = phenocycler( + tmp_path, + cells_table=tmp_path / "cells.csv", + cells_table_format=fmt, + image_models_kwargs=_FAST, + cells_table_kwargs=kwargs, + ) + + obs = sdata.tables["table"].obs + assert list(obs["region"]) == ["S_cells", "S_cells"] + assert list(obs["cell_id"]) == [1, 2] + assert list(obs["source_region"]) == ["other", "other"] + assert list(obs["source_cell_id"]) == [101, 102] + + def test_qupath_without_markers_warns(tmp_path: Path) -> None: write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) (tmp_path / "cells.tsv").write_text("Object ID\tCentroid X px\tCentroid Y px\nu1\t1\t2\n", encoding="utf-8") From 82a2a72da21d3d1ef2eb83b1a3b26b24bff09c5a Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 17:21:48 +0200 Subject: [PATCH 019/107] fix: reject a mapping or an empty sequence as regions `imread_kwargs` used to be the third positional parameter of `codex()`; passing it positionally now raises a TypeError pointing to the keyword instead of being treated as a region list. An empty `regions` raises a ValueError and duplicate regions are read once. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++- src/spatialdata_io/readers/_utils/_akoya.py | 28 ++++++++++++++++++--- src/spatialdata_io/readers/codex.py | 3 ++- src/spatialdata_io/readers/phenocycler.py | 3 ++- tests/readers/test_akoya_utils.py | 14 +++++++++++ tests/test_codex.py | 13 ++++++++++ 6 files changed, 59 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e124b28d..bc363ce4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,9 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. - Marker names in `table.var_names` come from the FCS `$PnS` keywords or the CSV header `cycCCC_chHHH:` (previously `cycCCC_chHHH`); the image channels are named too. - The positional third parameter is now `regions` (previously `imread_kwargs`); pass `imread_kwargs` by keyword. - It is only used by the `dask_image.imread` fallback for TIFFs that `tifffile` cannot read lazily. + Code that still passes `imread_kwargs` positionally gets a `TypeError` (a mapping is not a valid `regions`) + rather than a silently different read; an empty `regions` raises a `ValueError`. `imread_kwargs` is only used by + the `dask_image.imread` fallback for TIFFs that `tifffile` cannot read lazily. - A single region is stored in `"global"`. Several regions are placed at their stage positions in `"global"` only when all of them share a sample space (same slide id, stage positions and pixel size); otherwise each region, including multi-region reads of unrelated regions, gets its own coordinate system named after the region. diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index 6ece1607..3dd339b9 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -33,6 +33,7 @@ "apply_channel_names", "build_image_element", "dedupe_channel_names", + "normalize_regions", "read_json_object", "read_tiff", "region_transformations", @@ -170,14 +171,35 @@ def scans_to_attrs(scans: Sequence[AkoyaScan], channel_names: Mapping[str, list[ return {"regions": regions} -def select_regions(available: Sequence[str], regions: Sequence[str] | None) -> list[str]: - """Return the requested regions (all if ``None``), raising if one is not available.""" +def normalize_regions(regions: str | Sequence[str] | None) -> list[str] | None: + """Validate the ``regions`` argument of a reader: a region id or a sequence of them, without duplicates. + + Raises a `TypeError` for a mapping, e.g. ``imread_kwargs`` passed positionally to ``codex()``, whose third + parameter used to be ``imread_kwargs``, and a `ValueError` for an empty sequence. + """ + if regions is None: + return None + if isinstance(regions, str): + return [regions] + if isinstance(regions, Mapping): + raise TypeError( + f"`regions` must be a sequence of region ids, got {type(regions).__name__}; `imread_kwargs` is no " + "longer the third positional argument of `codex()` — pass it by keyword." + ) + if not regions: + raise ValueError("`regions` is empty; pass `None` to read all regions.") + return list(dict.fromkeys(regions)) + + +def select_regions(available: Sequence[str], regions: str | Sequence[str] | None) -> list[str]: + """Return the requested regions (all if ``None``), raising if one is not available; see :func:`normalize_regions`.""" + regions = normalize_regions(regions) if regions is None: return list(available) unknown = [region for region in regions if region not in available] if unknown: raise ValueError(f"Unknown region(s) {unknown}. Available regions: {list(available)}.") - return list(regions) + return regions def read_json_object(path: Path) -> dict[str, Any] | None: diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 328c3041..648615d8 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -19,6 +19,7 @@ from spatialdata_io.readers._utils._akoya import ( apply_channel_names, build_image_element, + normalize_regions, read_json_object, read_tiff, region_transformations, @@ -192,7 +193,7 @@ def codex( ------- :class:`spatialdata.SpatialData` """ - regions = [regions] if isinstance(regions, str) else regions + regions = normalize_regions(regions) channel_names = [channel_names] if isinstance(channel_names, str) else channel_names path = Path(path) layout = discover_legacy(path, fcs) diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 8fa3d353..3a676f8c 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -15,6 +15,7 @@ from spatialdata_io.readers._utils._akoya import ( apply_channel_names, build_image_element, + normalize_regions, read_json_object, region_transformations, scans_to_attrs, @@ -147,7 +148,7 @@ def phenocycler( ------- :class:`spatialdata.SpatialData` """ - regions = [regions] if isinstance(regions, str) else regions + regions = normalize_regions(regions) channel_names = [channel_names] if isinstance(channel_names, str) else channel_names path = Path(path) qptiffs = discover_qptiffs(path) diff --git a/tests/readers/test_akoya_utils.py b/tests/readers/test_akoya_utils.py index a3ab2f9c..b40ecec7 100644 --- a/tests/readers/test_akoya_utils.py +++ b/tests/readers/test_akoya_utils.py @@ -126,6 +126,20 @@ def test_select_regions(): assert select_regions(["a", "b"], ["b"]) == ["b"] +def test_select_regions_deduplicates(): + assert select_regions(["a", "b"], ["b", "a", "b"]) == ["b", "a"] + assert select_regions(["a", "b"], "b") == ["b"] + + +@pytest.mark.parametrize( + ("regions", "error", "match"), + [({}, TypeError, "pass it by keyword"), ({"a": 1}, TypeError, "sequence of region ids"), ([], ValueError, "empty")], +) +def test_select_regions_invalid_raises(regions: object, error: type[Exception], match: str): + with pytest.raises(error, match=match): + select_regions(["a", "b"], regions) # type: ignore[arg-type] + + def test_select_regions_unknown_raises(): with pytest.raises(ValueError, match="'c'.*Available.*a.*b"): select_regions(["a", "b"], ["c"]) diff --git a/tests/test_codex.py b/tests/test_codex.py index d68b5fb7..ed284d71 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -267,6 +267,19 @@ def test_codex_str_regions_and_channel_names(tmp_path: Path) -> None: assert get_channel_names(sdata.images["reg002_image"]) == ["nuclei"] +def test_codex_regions_validation(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, regions=2, cycles=1, channels=1, layout="per_marker", fcs=False, csv=False) + + # `imread_kwargs` used to be the third positional parameter + with pytest.raises(TypeError, match="imread_kwargs.*pass it by keyword"): + codex(tmp_path, True, {}) # type: ignore[arg-type] + with pytest.raises(ValueError, match="empty"): + codex(tmp_path, regions=[]) + + sdata = codex(tmp_path, regions=["reg002", "reg002"], image_models_kwargs=_FAST) + assert list(sdata.images) == ["reg002_image"] + + @pytest.mark.parametrize("imagej", [False, True]) def test_read_legacy_image_tiles_uncompressed_planes(tmp_path: Path, imagej: bool) -> None: # uncompressed single-strip planes are one zarr chunk each; they must be memory-mapped and tiled instead From a8d0358c7462ff72d6201dee35b8bd2679937184 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 17:25:50 +0200 Subject: [PATCH 020/107] fix(phenocycler): ignore malformed or non-UTF-8 sidecars with a warning A UTF-16 .xpd, wells or items that are not JSON objects, or a non-UTF-8 channels.csv aborted the whole read. Sidecars are metadata only: they are now ignored with a UserWarning, byte order marks are accepted and an .xpd without wells no longer warns about 0 markers. Co-Authored-By: Claude Opus 5.5 --- src/spatialdata_io/readers/_utils/_akoya.py | 9 ++-- src/spatialdata_io/readers/_utils/_qptiff.py | 20 +++++-- src/spatialdata_io/readers/phenocycler.py | 25 +++++++-- tests/test_phenocycler.py | 56 ++++++++++++++++++++ 4 files changed, 98 insertions(+), 12 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index 3dd339b9..b9d673ad 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -203,10 +203,13 @@ def select_regions(available: Sequence[str], regions: str | Sequence[str] | None def read_json_object(path: Path) -> dict[str, Any] | None: - """Read a JSON object from ``path``; warn and return ``None`` if the file is not valid JSON or not an object.""" + """Read a JSON object from ``path``; warn and return ``None`` if the file is not valid UTF-8 JSON or not an object. + + A UTF-8 byte order mark, as written by some Windows tools, is ignored. + """ try: - content = json.loads(path.read_text()) - except json.JSONDecodeError as e: + content = json.loads(path.read_text(encoding="utf-8-sig")) + except (json.JSONDecodeError, UnicodeDecodeError) as e: warnings.warn(f"Ignoring {path}: it is not valid JSON ({e}).", UserWarning, stacklevel=2) return None if not isinstance(content, dict): diff --git a/src/spatialdata_io/readers/_utils/_qptiff.py b/src/spatialdata_io/readers/_utils/_qptiff.py index 207cef54..a4858783 100644 --- a/src/spatialdata_io/readers/_utils/_qptiff.py +++ b/src/spatialdata_io/readers/_utils/_qptiff.py @@ -306,11 +306,15 @@ def _channel(xml: ET.Element, index: int, page_index: int) -> AkoyaChannel: def _fusion_profile_records(profile: dict[str, Any]) -> list[tuple[str, dict[str, Any]]]: - """``(marker, fields)`` per item of the Fusion ``experimentDescription.wells``; one well per cycle.""" + """``(marker, fields)`` per item of the Fusion ``experimentDescription.wells``; one well per cycle. + + Wells and items that are not JSON objects, and ``wells``/``items`` that are not lists, are skipped. + """ records = [] - wells = profile.get("experimentDescription", {}).get("wells", []) - for cycle, well in enumerate(wells, start=1): - for channel, item in enumerate(well.get("items", []), start=1): + description = profile.get("experimentDescription") + wells = description.get("wells") if isinstance(description, dict) else None + for cycle, well in enumerate(_objects(wells), start=1): + for channel, item in enumerate(_objects(well.get("items")), start=1): marker = item.get("markerName") if not isinstance(marker, str) or _is_blank(marker): continue @@ -338,6 +342,14 @@ def _fusion_profile_records(profile: dict[str, Any]) -> list[tuple[str, dict[str return records +def _objects(value: Any) -> list[dict[str, Any]]: + """The entries of ``value`` if it is a list, else none; entries that are not objects become empty objects. + + They are kept as empty objects so that the cycle and channel indices still count every entry. + """ + return [entry if isinstance(entry, dict) else {} for entry in value] if isinstance(value, list) else [] + + def _experiment_v4_records(experiment: ET.Element) -> list[tuple[str, dict[str, Any]]]: """``(marker, fields)`` per ``Cycles/Cycle/Channels/Channel`` of a CIM ``ExperimentV4``.""" records = [] diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 3a676f8c..70a8da85 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -277,7 +277,10 @@ def _is_acquisition_file(path: Path) -> bool: def _read_sidecars(scan: AkoyaScan) -> None: - """Store the ``.xpd`` and ``*channels.csv`` sidecars of ``scan`` in ``scan.raw`` and check their channel count.""" + """Store the ``.xpd`` and ``*channels.csv`` sidecars of ``scan`` in ``scan.raw`` and check their channel count. + + The sidecars are metadata only, so one that cannot be read is ignored with a warning. + """ base = _base_name(scan.path) xpd = _find_sidecar(scan.path, base, PhenocyclerKeys.XPD_SUFFIX) protocol = read_json_object(xpd) if xpd is not None else None @@ -285,14 +288,26 @@ def _read_sidecars(scan: AkoyaScan) -> None: scan.raw["xpd"] = protocol # an .xpd has the layout of the Fusion scan profile's `experimentDescription` markers = [marker for marker, _ in _fusion_profile_records({"experimentDescription": protocol})] - _check_channel_count(scan, xpd, markers) + if markers: + _check_channel_count(scan, xpd, markers) + elif protocol.get("wells") is not None: + warnings.warn( + f"{xpd.name} has no readable markers in its 'wells', so its channel count is not checked.", + UserWarning, + stacklevel=3, + ) channels_csv = _find_sidecar(scan.path, base, PhenocyclerKeys.CHANNELS_CSV_SUFFIX) if channels_csv is not None: - with channels_csv.open(newline="") as f: - rows = list(csv.DictReader(f)) + try: + with channels_csv.open(encoding="utf-8-sig", newline="") as f: + rows = list(csv.DictReader(f)) + except (UnicodeDecodeError, csv.Error) as e: + warnings.warn(f"Ignoring {channels_csv}: it is not a UTF-8 CSV file ({e}).", UserWarning, stacklevel=3) + return scan.raw["channels_csv"] = rows - column = next((key for key in (rows[0] if rows else {}) if key.strip().casefold() == "marker"), None) + header = rows[0] if rows else {} + column = next((key for key in header if isinstance(key, str) and key.strip().casefold() == "marker"), None) if column is not None: markers = [row[column] for row in rows if row[column] and not _is_blank(row[column])] _check_channel_count(scan, channels_csv, markers) diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index c19553c5..25ce324c 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -182,6 +182,62 @@ def test_invalid_xpd_warns_and_is_ignored(tmp_path: Path) -> None: assert "xpd" not in sdata.attrs["akoya"]["regions"]["S"]["raw"] +@pytest.mark.parametrize( + ("content", "match"), + [ + (json.dumps({"wells": [{"items": ["DAPI"]}]}).encode(), "no readable markers"), + (json.dumps({"wells": "x"}).encode(), "no readable markers"), + (json.dumps({"wells": [{"items": [{"markerName": "DAPI"}]}]}).encode("utf-16"), "not valid"), + ], + ids=["items-not-objects", "wells-not-a-list", "utf-16"], +) +def test_malformed_xpd_warns_and_image_is_read(tmp_path: Path, content: bytes, match: str) -> None: + path = tmp_path / "S_Scan1.qptiff" + data = write_qptiff(path, biomarkers=["DAPI"]) + (tmp_path / "S_Scan1.xpd").write_bytes(content) + + with pytest.warns(UserWarning, match=match) as record: + sdata = phenocycler(path, image_models_kwargs=_FAST) + + assert not any("distinct markers" in str(warning.message) for warning in record) + np.testing.assert_array_equal(_level0(sdata, "S_image").values, data) + + +def test_xpd_without_wells_is_kept_without_warning(tmp_path: Path) -> None: + path = tmp_path / "S_Scan1.qptiff" + write_qptiff(path, biomarkers=["DAPI"]) + # a UTF-8 byte order mark, as Windows tools write it + (tmp_path / "S_Scan1.xpd").write_text(json.dumps({"formatType": "Protein"}), encoding="utf-8-sig") + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + sdata = phenocycler(path, image_models_kwargs=_FAST) + + assert sdata.attrs["akoya"]["regions"]["S"]["raw"]["xpd"] == {"formatType": "Protein"} + + +def test_channels_csv_with_byte_order_mark(tmp_path: Path) -> None: + path = tmp_path / "S_Scan1.qptiff" + write_qptiff(path, biomarkers=["DAPI", "CD3"]) + (tmp_path / "S_Scan1.qptiff.channels.csv").write_text("Marker\nDAPI\nCD3\nCD8\n", encoding="utf-8-sig") + + with pytest.warns(UserWarning, match="channels.csv lists 3 distinct markers"): + sdata = phenocycler(path, image_models_kwargs=_FAST) + + assert sdata.attrs["akoya"]["regions"]["S"]["raw"]["channels_csv"][0] == {"Marker": "DAPI"} + + +def test_unreadable_channels_csv_warns_and_is_ignored(tmp_path: Path) -> None: + path = tmp_path / "S_Scan1.qptiff" + write_qptiff(path, biomarkers=["DAPI"]) + (tmp_path / "S_Scan1.qptiff.channels.csv").write_bytes("Marker\nDAPI\n".encode("utf-16")) + + with pytest.warns(UserWarning, match="Ignoring"): + sdata = phenocycler(path, image_models_kwargs=_FAST) + + assert "channels_csv" not in sdata.attrs["akoya"]["regions"]["S"]["raw"] + + def test_empty_dir_raises(tmp_path: Path) -> None: with pytest.raises(FileNotFoundError, match="\\*.qptiff"): phenocycler(str(tmp_path)) From 6eb8b2783a3bad4c54eaa46ef115c5358eb08056 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 17:35:00 +0200 Subject: [PATCH 021/107] fix: make lazy qptiff, compressed-TIFF and mask arrays picklable The dask arrays held tifffile's zarr store and its open file handle, so they could not be pickled (dask.distributed, process schedulers) and left unclosed files behind (ResourceWarning). A small array-like now keeps only the path and the series or page, opens the file on the first chunk read in each process and closes it with the object; native tile chunks are kept. Co-Authored-By: Claude Opus 5.5 --- src/spatialdata_io/readers/_utils/_akoya.py | 109 +++++++++++++++++-- src/spatialdata_io/readers/_utils/_qptiff.py | 42 +++---- tests/readers/test_akoya_utils.py | 18 +++ tests/readers/test_qptiff.py | 33 ++++++ tests/test_akoya_real_data.py | 8 +- 5 files changed, 177 insertions(+), 33 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index b9d673ad..dc7671c1 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -4,6 +4,7 @@ import json import math +import threading import warnings from collections import Counter from collections.abc import Iterable, Mapping, Sequence @@ -33,6 +34,7 @@ "apply_channel_names", "build_image_element", "dedupe_channel_names", + "lazy_tiff_array", "normalize_regions", "read_json_object", "read_tiff", @@ -227,24 +229,22 @@ def read_tiff( is memory-mapped and split into ``chunks``, so a chunk never needs a whole plane in memory; other layouts keep their native tiles via zarr. Returns the image, the shape of the flattened axes, the ImageJ ``Labels`` (if any) and whether tifffile, rather than the :func:`dask_image.imread.imread` fallback, read the file. The - file handle stays open for as long as the array needs it. + array opens the file only when its chunks are read and can be pickled, e.g. for ``dask.distributed``. """ - tf = tifffile.TiffFile(path) - series = tf.series[0] - axes = series.axes - metadata = tf.imagej_metadata or {} + with tifffile.TiffFile(path) as tf: + series = tf.series[0] + axes, offset, shape, dtype = series.axes, series.dataoffset, series.shape, series.dtype + metadata = tf.imagej_metadata or {} labels = list(metadata["Labels"]) if isinstance(metadata.get("Labels"), list | tuple) else None read_by_tifffile = True - if series.dataoffset is not None: - tf.close() - source = _ContiguousTiff(path, series.dataoffset, series.shape, series.dtype) + if offset is not None: + source = _ContiguousTiff(path, offset, shape, dtype) tiles = dict.fromkeys(range(source.ndim), 1) | {axes.index("Y"): chunks[0], axes.index("X"): chunks[1]} image = da.from_array(source, chunks=tuple(tiles[i] for i in range(source.ndim)), name=source.name) else: try: - image = da.from_zarr(zarr.open(series.aszarr(level=0), mode="r")) + image = lazy_tiff_array(path, series=0) except ValueError as e: # e.g. a compression tifffile cannot decode chunk-wise - tf.close() logger.info("Reading %s with dask_image.imread because tifffile cannot read it lazily: %s", path, e) image = imread(str(path), **imread_kwargs) axes = "Q" * (image.ndim - 2) + "YX" @@ -276,6 +276,95 @@ def __getitem__(self, key: Any) -> np.ndarray: return np.asarray(data[key], dtype=self.dtype) +def lazy_tiff_array(path: Path, *, series: int = 0, page: int | None = None) -> da.Array: + """Lazily read the level 0 of a tiff series, or one page, with the native tile chunks of the file. + + The array can be pickled (e.g. for ``dask.distributed`` or process-based schedulers): its chunks are read by a + :class:`_TiffZarrSource`, which opens the file in each process that reads them. + + Parameters + ---------- + path + The tiff file. + series + The index of the series to read, if ``page`` is `None`. + page + The index of the page to read instead of a series. + + Returns + ------- + The array, with the shape of the series or page. + """ + source = _TiffZarrSource(path, series=series, page=page) + return da.from_array( + source, chunks=source.chunks, name=source.name, meta=np.empty((0,) * source.ndim, dtype=source.dtype) + ) + + +class _TiffZarrSource: + """Picklable array-like view of a tiff series (level 0) or page, read chunk-wise through tifffile's zarr store. + + Neither the ``TiffFile`` nor its zarr store are pickled: they are opened on the first read in each process and + closed with the object, so dask graphs built on it work across processes and leave no file handle unclosed. + """ + + def __init__(self, path: Path, *, series: int = 0, page: int | None = None) -> None: + self.path = path + self.series = series + self.page = page + self._lock = threading.Lock() + self._tiff: tifffile.TiffFile | None = None + self._array: zarr.Array | None = None + try: + array = self._open() + self.shape = tuple(array.shape) + self.chunks = tuple(array.chunks) + self.dtype = array.dtype + finally: + # the metadata is all a lazy read needs; the file is opened again once chunks are read + self.close() + self.ndim = len(self.shape) + token = tokenize(str(path), path.stat().st_mtime_ns, series, page, self.shape, str(self.dtype)) + self.name = f"akoya-tiff-zarr-{token}" + + def _open(self) -> zarr.Array: + with self._lock: + if self._array is not None: + return self._array + tiff = tifffile.TiffFile(self.path) + try: + if self.page is not None: + store = tiff.pages[self.page].aszarr() + else: + store = tiff.series[self.series].aszarr(level=0) + array = zarr.open_array(store, mode="r") + except BaseException: + tiff.close() + raise + self._tiff, self._array = tiff, array + return array + + def __getitem__(self, key: Any) -> np.ndarray: + return np.asarray(self._open()[key]) + + def close(self) -> None: + """Close the file; it is opened again by the next read.""" + with self._lock: + if self._tiff is not None: + self._tiff.close() + self._tiff, self._array = None, None + + def __getstate__(self) -> dict[str, Any]: + return {key: value for key, value in self.__dict__.items() if key not in ("_lock", "_tiff", "_array")} + + def __setstate__(self, state: dict[str, Any]) -> None: + self.__dict__.update(state, _lock=threading.Lock(), _tiff=None, _array=None) + + def __del__(self) -> None: + if getattr(self, "_tiff", None) is not None: + self.close() + + def spatial_chunks(chunks: Any, default: Sequence[int]) -> tuple[int, int]: """The ``(y, x)`` sizes of a raster model ``chunks`` argument, ``default`` when it does not give plain sizes.""" if isinstance(chunks, int): diff --git a/src/spatialdata_io/readers/_utils/_qptiff.py b/src/spatialdata_io/readers/_utils/_qptiff.py index a4858783..f37984a4 100644 --- a/src/spatialdata_io/readers/_utils/_qptiff.py +++ b/src/spatialdata_io/readers/_utils/_qptiff.py @@ -12,9 +12,9 @@ import dask.array as da import tifffile -import zarr from spatialdata_io._constants._constants import PhenocyclerKeys +from spatialdata_io.readers._utils._akoya import lazy_tiff_array if TYPE_CHECKING: from pathlib import Path @@ -134,8 +134,8 @@ def read_qptiff_metadata(path: Path, region: str | None = None) -> AkoyaScan: def read_qptiff_image(path: Path) -> da.Array: """Lazily read the full-resolution image of a PerkinElmer/Akoya QPTIFF. - The array keeps the native tile chunks of the file. The underlying ``TiffFile`` stays open for as long as - the returned array (its zarr store holds the file handle), so it is intentionally not closed here. + The array keeps the native tile chunks of the file, opens the file only when its chunks are read and can be + pickled, e.g. for ``dask.distributed``. Parameters ---------- @@ -146,27 +146,27 @@ def read_qptiff_image(path: Path) -> da.Array: ------- ``(c, y, x)`` array, or ``(y, x, 3)`` for a brightfield RGB scan. """ - tf = _open_qpi(path) - full_res = [ - i - for i in range(len(tf.pages)) - if _text(_parse_description(_page(tf, i).description), "ImageType") == _FULL_RESOLUTION - ] - if not full_res: - tf.close() - raise ValueError(f"{path} has no page with {_FULL_RESOLUTION}.") - first = _page(tf, full_res[0]) - if len(full_res) == 1 and first.samplesperpixel == 3: - expected_shape: tuple[int, ...] = first.shape - else: - expected_shape = (len(full_res), *first.shape) - - image = da.from_zarr(zarr.open(tf.series[0].aszarr(level=0), mode="r")) + with _open_qpi(path) as tf: + full_res = [ + i + for i in range(len(tf.pages)) + if _text(_parse_description(_page(tf, i).description), "ImageType") == _FULL_RESOLUTION + ] + if not full_res: + raise ValueError(f"{path} has no page with {_FULL_RESOLUTION}.") + first = _page(tf, full_res[0]) + page_shape = first.shape + if len(full_res) == 1 and first.samplesperpixel == 3: + expected_shape: tuple[int, ...] = page_shape + else: + expected_shape = (len(full_res), *page_shape) + + image = lazy_tiff_array(path, series=0) if image.shape == expected_shape: return image # tifffile groups pages into series by shape heuristics; fall back to the pages the XML says are full-res - pages = [da.from_zarr(zarr.open(_page(tf, i).aszarr(), mode="r")) for i in full_res] - return pages[0] if len(expected_shape) == first.ndim else da.stack(pages) + pages = [lazy_tiff_array(path, page=i) for i in full_res] + return pages[0] if len(expected_shape) == len(page_shape) else da.stack(pages) def qpi_pixel_size_um(page: tifffile.TiffPage) -> float | None: diff --git a/tests/readers/test_akoya_utils.py b/tests/readers/test_akoya_utils.py index b40ecec7..5736a60d 100644 --- a/tests/readers/test_akoya_utils.py +++ b/tests/readers/test_akoya_utils.py @@ -1,17 +1,20 @@ from __future__ import annotations import json +import pickle from pathlib import Path import dask.array as da import numpy as np import pytest +import tifffile from spatialdata.transformations import Identity, Translation, get_transformation from spatialdata_io.readers._utils._akoya import ( apply_channel_names, build_image_element, dedupe_channel_names, + read_tiff, region_transformations, scans_to_attrs, select_regions, @@ -145,6 +148,21 @@ def test_select_regions_unknown_raises(): select_regions(["a", "b"], ["c"]) +@pytest.mark.parametrize("compression", [None, "zlib"]) +def test_read_tiff_is_picklable(tmp_path: Path, compression: str | None) -> None: + # compressed tiles are read through tifffile's zarr store, uncompressed data is memory-mapped + path = tmp_path / "x.tif" + data = np.arange(2 * 64 * 96, dtype=np.uint16).reshape(2, 64, 96) + tifffile.imwrite(path, data, compression=compression, tile=(32, 32) if compression else None) + + image, _, _, read_by_tifffile = read_tiff(path, {}, (16, 16)) + + assert read_by_tifffile + restored = pickle.loads(pickle.dumps(image)) + np.testing.assert_array_equal(restored[0, :8, :8].compute(), data[0, :8, :8]) + np.testing.assert_array_equal(restored.compute(), data) + + def test_build_image_rgb_and_multiplex(): scan = _scan() img = da.from_array(np.random.default_rng(0).integers(0, 255, (3, 16, 16), dtype=np.uint8), chunks=(1, 8, 8)) diff --git a/tests/readers/test_qptiff.py b/tests/readers/test_qptiff.py index 57297133..d349d9bb 100644 --- a/tests/readers/test_qptiff.py +++ b/tests/readers/test_qptiff.py @@ -1,4 +1,6 @@ +import gc import json +import pickle import warnings from pathlib import Path @@ -8,6 +10,7 @@ import tifffile from tests._akoya_data import cim_experiment_v4_xml, fusion_scan_profile_json, write_qptiff +from spatialdata_io.readers._utils import _qptiff from spatialdata_io.readers._utils._qptiff import read_qptiff_image, read_qptiff_metadata @@ -186,6 +189,36 @@ def test_image_is_lazy_and_correct(tmp_path: Path, dtype: str, name: str) -> Non assert np.array_equal(arr.compute(), written) +def test_image_is_picklable_and_closes_its_file(tmp_path: Path) -> None: + # dask.distributed and process-based schedulers pickle the graph, so it must not hold open file handles + p = tmp_path / "x.qptiff" + written = write_qptiff(p, biomarkers=["DAPI", "CD3"]) + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always", ResourceWarning) + arr = read_qptiff_image(p) + restored = pickle.loads(pickle.dumps(arr)) + np.testing.assert_array_equal(restored[0, :8, :8].compute(), written[0, :8, :8]) + np.testing.assert_array_equal(arr.compute(), written) + del arr, restored + gc.collect() + assert not [warning for warning in record if issubclass(warning.category, ResourceWarning)] + + +def test_image_pages_are_picklable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + # the per-page fallback, used when tifffile's series do not match the full-resolution pages of the XML + p = tmp_path / "x.qptiff" + written = write_qptiff(p, biomarkers=["DAPI", "CD3"]) + real = _qptiff.lazy_tiff_array + + def wrong_series(path: Path, *, series: int = 0, page: int | None = None) -> da.Array: + return real(path, series=series, page=page) if page is not None else da.zeros((1, 1, 1)) + + monkeypatch.setattr(_qptiff, "lazy_tiff_array", wrong_series) + arr = read_qptiff_image(p) + assert arr.shape == written.shape + np.testing.assert_array_equal(pickle.loads(pickle.dumps(arr))[1, :8, :8].compute(), written[1, :8, :8]) + + def test_three_channel_not_rgb(tmp_path: Path) -> None: p = tmp_path / "x.qptiff" written = write_qptiff(p, biomarkers=["DAPI", "CD3", "CD8"]) diff --git a/tests/test_akoya_real_data.py b/tests/test_akoya_real_data.py index 999446eb..f39f7e62 100644 --- a/tests/test_akoya_real_data.py +++ b/tests/test_akoya_real_data.py @@ -20,8 +20,12 @@ pytestmark = [ pytest.mark.skipif(_ROOT is None, reason="set AKOYA_TEST_DATA to the directory of the real Akoya datasets"), - # upstream noise unrelated to the readers: the open TIFF handles of lazy images, ome-zarr's deprecated `Scaler` - pytest.mark.filterwarnings("ignore::ResourceWarning", "ignore:Call to deprecated class Scaler"), + # upstream noise unrelated to the readers: ome-zarr's deprecated `Scaler`; unclosed files must not be left behind + pytest.mark.filterwarnings( + "ignore:Call to deprecated class Scaler", + "error::ResourceWarning", + "error::pytest.PytestUnraisableExceptionWarning", + ), ] From a32a8e33932be6ee298f8e209f688537f28c8140 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 17:37:10 +0200 Subject: [PATCH 022/107] fix(codex): ignore region-named CSVs that are not CODEX cell tables `reg001_antibodies.csv` or a HuBMAP SPRM `reg001_...-cell_channel_mean.csv` was discovered as the region's cell table and aborted the read. Discovery now keeps only CSVs detected as CODEX Processor tables, as it already did for CSVs without a region in their name. Co-Authored-By: Claude Opus 5.5 --- src/spatialdata_io/readers/codex.py | 11 ++++++----- tests/test_codex.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 648615d8..66545951 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -116,7 +116,8 @@ def codex( are preferred over uncompensated ones. The ``cycCCC_chHHH`` marker columns are stored in ``X`` (dense float32) under their marker names, taken from the FCS ``$PnS`` keywords or the CSV header ``cycCCC_chHHH:``; all other columns go to ``obs``, the ``x``/``y`` pixel centroids to - ``obsm["spatial"]``. + ``obsm["spatial"]``. Other CSV files (without ``x``/``y`` and ``cell_id`` or ``cycCCC_chHHH`` columns, + e.g. antibody lists or HuBMAP SPRM outputs) are ignored; pass them as ``cells_table`` to read them. Raw acquisition and intermediate folders (``cycCCC_regNNN/``, ``tiles/`` and, next to ``stitched/``, ``bestFocus/``) are ignored. The region id ``regNNN`` is taken from the file names; a ``{vx.QPTIFF!r}`` without @@ -537,11 +538,11 @@ def _discover_tables(files: Sequence[Path], fcs: bool) -> dict[str, Path]: for file in files: if file.suffix.lower() not in _TABLE_SUFFIXES: continue - region = _region_id(file.name) - # without a region in its name, a CSV may be any sidecar (e.g. an antibody list) - if region is None and detect_table_format(file) != "codex": + # a CSV may be any sidecar, also with a region in its name (e.g. `reg001_antibodies.csv` or a HuBMAP SPRM + # `reg001_...-cell_channel_mean.csv`); only CODEX Processor tables are discovered + if detect_table_format(file) != "codex": continue - candidates.setdefault(region or "*", []).append(file) + candidates.setdefault(_region_id(file.name) or "*", []).append(file) def rank(file: Path) -> tuple[bool, bool]: # the preferred format first, then compensated over uncompensated diff --git a/tests/test_codex.py b/tests/test_codex.py index ed284d71..0b1dbda6 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -325,6 +325,24 @@ def test_discover_legacy_regionless_table_ignores_sidecar_csv(tmp_path: Path) -> discover_legacy(tmp_path, fcs=True) +@pytest.mark.parametrize("with_table", [True, False]) +def test_codex_region_named_non_cell_csv_is_ignored(tmp_path: Path, with_table: bool) -> None: + data = write_legacy_processor_tree(tmp_path, cycles=1, channels=2, fcs=False, csv=with_table, n_cells=5) + (tmp_path / "reg001_antibodies.csv").write_text("Antibody,Clone\nCD3,UCHT1\n") + # a HuBMAP SPRM output of the same region, which is only read when given as `cells_table` + (tmp_path / "reg001_expr-cell_channel_mean.csv").write_text("ID,CD3\n1,2.0\n") + + if with_table: + with pytest.warns(UserWarning, match="circles"): + sdata = codex(tmp_path, image_models_kwargs=_FAST) + assert sdata.tables["table"].n_obs == len(data["cells"]) + assert sdata.attrs["akoya"]["cell_tables"]["reg001"].endswith("reg001_compensated.csv") + else: + sdata = codex(tmp_path, image_models_kwargs=_FAST) + assert not sdata.tables + assert list(sdata.images) == ["reg001_image"] + + def test_codex_imread_kwargs_warns_when_unused(tmp_path: Path) -> None: write_legacy_processor_tree(tmp_path, layout="per_marker") with pytest.warns(UserWarning, match="imread_kwargs"): From 90f15db5b57fb25d50c58876e050d62cf9d74dbc Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 17:43:10 +0200 Subject: [PATCH 023/107] =?UTF-8?q?fix:=20convert=20=C2=B5m=20cell=20coord?= =?UTF-8?q?inates=20with=20each=20region's=20own=20pixel=20size?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A QuPath (or inForm, generic) table for several scans with pixel sizes that differ in the 4th decimal could not be read: it required one pixel size for all regions. Each cell's centroid, area and GeoJSON outline are now converted with the pixel size of the region its region column names. Co-Authored-By: Claude Opus 5.5 --- .../readers/_utils/_cell_tables.py | 90 ++++++++++++++----- tests/readers/test_rasterize.py | 20 +++++ tests/test_phenocycler.py | 21 +++++ 3 files changed, 108 insertions(+), 23 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index bdef00f8..f7e52d83 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -7,6 +7,7 @@ import warnings from collections.abc import Mapping from dataclasses import dataclass +from functools import partial from pathlib import Path from typing import TYPE_CHECKING, Any @@ -286,7 +287,7 @@ def read_cell_table( path: Path, fmt: str, *, - pixel_size_um: float | None, + pixel_size_um: float | Mapping[str, float | None] | None, default_region: str | None, cells_table_kwargs: Mapping[str, Any], ) -> CellTable: @@ -326,7 +327,9 @@ def read_cell_table( - ``"generic"``: columns given by ``cells_table_kwargs``. pixel_size_um - The pixel size of the region image, to convert µm centroids to pixels. + The pixel size of the region image, to convert µm centroids to pixels; for a table of several regions, the + pixel size of each region, by region id: every cell is converted with the pixel size of the region its + region column names (see :func:`read_region_tables`), cells of other regions get NaN coordinates. default_region The region id of all cells if the table has no region column. cells_table_kwargs @@ -366,13 +369,17 @@ def read_cell_table( "generic": _generic_roles, }[fmt] roles = reader(path, df, cells_table_kwargs) + units = cells_table_kwargs.get("geojson_units", "px") + row_pixel_sizes = None + if roles["in_um"] or (fmt == "qupath" and units == "um"): + row_pixel_sizes = _row_pixel_sizes(path, df, roles["region_column"], pixel_size_um) if roles.pop("in_um"): - if pixel_size_um is None: + if row_pixel_sizes is None: raise ValueError( f"{path} has centroids in µm, but no pixel size is known to convert them to pixels (the region's " - "pixel size is missing, or one table covers regions with different pixel sizes)." + "pixel size is missing)." ) - roles["px_per_unit"] = 1 / pixel_size_um + roles["px_per_unit"] = 1 / row_pixel_sizes table = _from_frame(path, df, default_region=default_region, **roles) if fmt == "sprm": table.centroids_px = _read_sprm_centers(path, table) @@ -380,12 +387,36 @@ def read_cell_table( geojson = cells_table_kwargs.get("geojson") geojson = _find_geojson(path) if geojson is None else Path(geojson) if geojson is not None: - units = cells_table_kwargs.get("geojson_units", "px") - table.polygons_px = read_qupath_polygons(geojson, table, path, units=units, pixel_size_um=pixel_size_um) + table.polygons_px = read_qupath_polygons(geojson, table, path, units=units, pixel_size_um=row_pixel_sizes) table.geojson = geojson return table +def _row_pixel_sizes( + path: Path, df: pd.DataFrame, region_column: str | None, pixel_size_um: float | Mapping[str, float | None] | None +) -> float | np.ndarray | None: + """The pixel size of every row of ``df``, or one for all rows; `None` if it is unknown. + + For pixel sizes by region id, each row gets that of the region its region column names, NaN if it names none of + them (its cells are not read). Raises if a region named by the table has no pixel size. + """ + if not isinstance(pixel_size_um, Mapping): + return pixel_size_um + if region_column is None or region_column not in df.columns: + sizes = set(pixel_size_um.values()) + return sizes.pop() if len(sizes) == 1 else None + values = df[region_column].astype(str) + regions = {value: _match_region(value, list(pixel_size_um)) for value in values.unique()} + unknown = sorted({region for region in regions.values() if region is not None and pixel_size_um[region] is None}) + if unknown: + raise ValueError( + f"{path} has coordinates in µm, but the pixel size of region(s) {unknown} is unknown, so they cannot be " + "converted to pixels." + ) + sizes_by_value = {value: np.nan if region is None else pixel_size_um[region] for value, region in regions.items()} + return values.map(sizes_by_value).to_numpy(dtype=np.float64) + + def _find_geojson(table_path: Path) -> Path | None: """The GeoJSON next to a QuPath table: ``.geojson``, else ``_cells.geojson``. @@ -396,7 +427,12 @@ def _find_geojson(table_path: Path) -> Path | None: def read_qupath_polygons( - geojson: Path, table: CellTable, table_path: Path, *, units: str = "px", pixel_size_um: float | None = None + geojson: Path, + table: CellTable, + table_path: Path, + *, + units: str = "px", + pixel_size_um: float | np.ndarray | None = None, ) -> np.ndarray: """Read the cell outlines of a QuPath GeoJSON export and match them to the cells of its measurement table. @@ -419,7 +455,7 @@ def read_qupath_polygons( units The unit of the coordinates: ``"px"`` (of the full-resolution image, what QuPath writes) or ``"um"``. pixel_size_um - The pixel size, to convert ``"um"`` coordinates. + The pixel size, to convert ``"um"`` coordinates, or the pixel size of every row of ``table``. Returns ------- @@ -427,16 +463,11 @@ def read_qupath_polygons( """ if units not in ("px", "um"): raise ValueError(f"`cells_table_kwargs['geojson_units']` must be 'px' or 'um', got {units!r}.") - px_per_unit = 1.0 - if units == "um": - if pixel_size_um is None: - raise ValueError(f"{geojson} has coordinates in µm, but no pixel size is known to convert them to pixels.") - px_per_unit = 1 / pixel_size_um + if units == "um" and pixel_size_um is None: + raise ValueError(f"{geojson} has coordinates in µm, but no pixel size is known to convert them to pixels.") if not geojson.is_file(): raise FileNotFoundError(f"{geojson} does not exist or is not a file.") ids, geometries, kinds = _read_geojson(geojson) - if px_per_unit != 1.0: - geometries = shapely.transform(geometries, lambda xy: xy * px_per_unit) polygons = np.full(len(table.obs), None, dtype=object) if _QUPATH_ID in table.obs.columns and ids is not None: @@ -469,14 +500,25 @@ def read_qupath_polygons( f"without ids ('{_QUPATH_ID}'), polygons are matched to cells in order." ) polygons[:] = cells + if units == "um": + _scale_polygons(polygons, 1 / np.broadcast_to(np.asarray(pixel_size_um, dtype=np.float64), polygons.shape)) if table.centroids_px is not None: _check_centroids(polygons, table.centroids_px, geojson, table_path) return polygons +def _scale_polygons(polygons: np.ndarray, factors: np.ndarray) -> None: + """Scale every polygon by its factor in place; polygons with a NaN factor (cells that are not read) are dropped.""" + polygons[np.isnan(factors)] = None + for factor in np.unique(factors[~np.isnan(factors)]): + rows = np.flatnonzero((factors == factor) & pd.notna(polygons)) + polygons[rows] = shapely.transform(polygons[rows], partial(np.multiply, factor)) + + def _check_centroids(polygons: np.ndarray, centroids_px: np.ndarray, geojson: Path, table_path: Path) -> None: """Raise if too many polygons are far from the centroids of the cells they were matched to.""" - matched = pd.notna(polygons) + # cells of regions that are not read have no pixel coordinates + matched = pd.notna(polygons) & np.isfinite(centroids_px).all(axis=1) if not matched.any(): return distance = shapely.distance(polygons[matched], shapely.points(centroids_px[matched])) @@ -699,9 +741,9 @@ def _from_frame( region_column: str | None, area_column: str | None, default_region: str | None, - px_per_unit: float = 1.0, + px_per_unit: float | np.ndarray = 1.0, ) -> CellTable: - """Build a `CellTable` from a data frame and the roles of its columns.""" + """Build a `CellTable` from a data frame and the roles of its columns; ``px_per_unit`` may be given per row.""" roles = [x, y, instance_column, region_column, area_column, *markers] missing = [column for column in roles if column is not None and column not in df.columns] if missing: @@ -739,7 +781,9 @@ def _from_frame( var=var, region=region, instance_id=instance_id, - centroids_px=None if x is None or y is None else df[[x, y]].to_numpy(dtype=np.float64) * px_per_unit, + centroids_px=None + if x is None or y is None + else df[[x, y]].to_numpy(dtype=np.float64) * np.reshape(px_per_unit, (-1, 1)), radius_px=radius, row_instance_ids=instance_column is None, ) @@ -843,12 +887,12 @@ def read_region_tables( ) tables = [] for key, path in files.items(): - covered = selected if key == "*" else [key] - sizes = {pixel_sizes.get(region) for region in covered} + # a table for several regions converts the µm coordinates of each cell with the pixel size of its region + size = {region: pixel_sizes.get(region) for region in selected} if key == "*" else pixel_sizes.get(key) table = read_cell_table( path, fmt, - pixel_size_um=sizes.pop() if len(sizes) == 1 else None, + pixel_size_um=size, default_region=None if key == "*" else key, cells_table_kwargs={**cells_table_kwargs, "geojson": geojson_files.get(key)}, ) diff --git a/tests/readers/test_rasterize.py b/tests/readers/test_rasterize.py index b880c53e..5ad442ec 100644 --- a/tests/readers/test_rasterize.py +++ b/tests/readers/test_rasterize.py @@ -228,6 +228,26 @@ def test_qupath_geojson_by_order_explicit_path_in_um(tmp_path: Path, collection: assert (values == 1).sum() == pytest.approx((8 / 0.5069) ** 2, rel=0.15) +def test_qupath_geojson_in_um_regions_with_different_pixel_sizes(tmp_path: Path) -> None: + sizes = {"A": 0.5, "B": 0.25} + for region, size in sizes.items(): + write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI"], resolution_px_per_cm=1e4 / size) + header = "Image\tCentroid X µm\tCentroid Y µm\tCD3: Cell: Mean" + rows = ["A_Scan1.qptiff\t4\t4\t1", "B_Scan1.qptiff\t24\t24\t2"] + (tmp_path / "cells.tsv").write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") + _write_geojson(tmp_path / "polygons.json", [_cell(None, 0, 0), _cell(None, 20, 20)]) + + sdata = phenocycler( + tmp_path, + cells_table=tmp_path / "cells.tsv", + image_models_kwargs=_FAST, + cells_table_kwargs={"geojson": tmp_path / "polygons.json", "geojson_units": "um"}, + ) + + assert sdata.shapes["A_polygons"].geometry.iloc[0].equals(shapely.box(0, 0, 16, 16)) + assert sdata.shapes["B_polygons"].geometry.iloc[0].equals(shapely.box(80, 80, 112, 112)) + + def test_qupath_geojson_mask_wins(tmp_path: Path) -> None: write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) _write_qupath_cells(tmp_path / "cells.tsv", ["a", "b"]) diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 25ce324c..c8938b34 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -512,6 +512,27 @@ def test_single_qupath_table_many_regions(tmp_path: Path) -> None: assert read_zarr(tmp_path / "x.zarr").tables["table"].n_obs == 2 +def test_single_qupath_table_regions_with_different_pixel_sizes(tmp_path: Path) -> None: + # scans of the same instrument differ in the 4th decimal; each cell is converted with its own scan's pixel size + sizes = {"A": 0.5069, "B": 0.5079} + for region, size in sizes.items(): + write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI", "CD3"], resolution_px_per_cm=1e4 / size) + header = "Image\tObject ID\tCentroid X µm\tCentroid Y µm\tCell: Area µm^2\tCD3: Cell: Mean" + rows = ["A_Scan1.qptiff\tu1\t5\t6\t3.14159\t1", "B_Scan1.qptiff\tu2\t10\t20\t3.14159\t2"] + (tmp_path / "cells.tsv").write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") + + with pytest.warns(UserWarning, match="circles"): + sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) + + table = sdata.tables["table"] + expected = np.array([[5 / sizes["A"], 6 / sizes["A"]], [10 / sizes["B"], 20 / sizes["B"]]]) + np.testing.assert_allclose(np.asarray(table.obsm["spatial"]), expected) + for region, (x, y) in zip(sizes, expected, strict=True): + circles = sdata.shapes[f"{region}_cells"] + np.testing.assert_allclose([circles.geometry.x.iloc[0], circles.geometry.y.iloc[0]], [x, y]) + np.testing.assert_allclose(circles["radius"].iloc[0], 1 / sizes[region], rtol=1e-5) + + def test_different_panels_make_per_region_tables(tmp_path: Path) -> None: write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) write_qptiff(tmp_path / "B_Scan1.qptiff", biomarkers=["DAPI", "CD8", "CD4"]) From b0ed75d6f61fd5b8e31eb95f17db40b01ccf3ab2 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 17:45:10 +0200 Subject: [PATCH 024/107] fix(phenocycler): find .QPTIFF scans and datasets below raw/images/phenocycler Discovery matched the suffix case-sensitively, and the acquisition-folder check looked at the absolute path, so a dataset stored below a `raw/images/phenocycler/` folder was ignored entirely. Only the folders below the searched directory are now checked. Co-Authored-By: Claude Opus 5.5 --- src/spatialdata_io/readers/phenocycler.py | 11 ++++++----- tests/test_phenocycler.py | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 70a8da85..98b8ab0b 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -76,7 +76,7 @@ def phenocycler( Sidecar files are matched to a scan by file-name prefix and stored as metadata only: the channel names always come from the per-page XML of the qptiff. A sidecar whose number of markers differs from the number of channels raises a warning. Acquisition files (``*.raw.qptiff``, ``*.qptiff.raw``, ``*.qptiff.intermediate`` and anything - in ``raw/images/phenocycler/``) are ignored. + in a ``raw/images/phenocycler/`` folder below ``path``) are ignored. File suffixes are matched ignoring case. The region name is the file name without the ``{vx.ER_QPTIFF_SUFFIX!r}``/``{vx.QPTIFF_SUFFIX!r}`` suffix and without a trailing ``_Scan``, unless that makes two regions share a name. Each region is stored as the image @@ -228,8 +228,8 @@ def discover_qptiffs(path: Path) -> dict[str, Path]: # one scan per (directory, base name), preferring the extended-range `.er.qptiff` scans: dict[tuple[Path, str], Path] = {} - for qptiff in sorted(path.rglob(f"*{PhenocyclerKeys.QPTIFF_SUFFIX}")): - if not qptiff.is_file() or _is_acquisition_file(qptiff): + for qptiff in sorted(path.rglob(f"*{PhenocyclerKeys.QPTIFF_SUFFIX}", case_sensitive=False)): + if not qptiff.is_file() or _is_acquisition_file(qptiff.relative_to(path)): continue key = (qptiff.parent, _base_name(qptiff)) if key not in scans or _is_extended_range(qptiff): @@ -269,11 +269,12 @@ def _is_extended_range(path: Path) -> bool: def _is_acquisition_file(path: Path) -> bool: + """Whether ``path``, relative to the searched directory, is a per-cycle acquisition file of the Fusion.""" if path.name.lower().endswith(".raw" + PhenocyclerKeys.QPTIFF_SUFFIX): return True - parts = path.parts + parts = tuple(part.casefold() for part in path.parent.parts) n = len(_ACQUISITION_DIR) - return any(parts[i : i + n] == _ACQUISITION_DIR for i in range(len(parts) - n)) + return any(parts[i : i + n] == _ACQUISITION_DIR for i in range(len(parts) - n + 1)) def _read_sidecars(scan: AkoyaScan) -> None: diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index c8938b34..1be0c3e8 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -125,6 +125,26 @@ def test_ignores_acquisition_files(tmp_path: Path) -> None: assert list(sdata.images) == ["S_image"] +def test_discovery_ignores_suffix_case(tmp_path: Path) -> None: + write_qptiff(tmp_path / "A_Scan1.QPTIFF", biomarkers=["DAPI"]) + write_qptiff(tmp_path / "B_Scan1.ER.QPTIFF", biomarkers=["DAPI"]) + + sdata = phenocycler(tmp_path, image_models_kwargs=_FAST) + + assert sorted(sdata.images) == ["A_image", "B_image"] + + +def test_dataset_below_an_acquisition_like_folder_is_read(tmp_path: Path) -> None: + # only the folders below `path` are checked for the HuBMAP acquisition folder `raw/images/phenocycler/` + root = tmp_path / "raw" / "images" / "phenocycler" / "project" + root.mkdir(parents=True) + write_qptiff(root / "S_Scan1.qptiff", biomarkers=["DAPI"]) + + sdata = phenocycler(root, image_models_kwargs=_FAST) + + assert list(sdata.images) == ["S_image"] + + def test_sidecars_hubmap_layout(tmp_path: Path) -> None: raw_images = tmp_path / "raw" / "images" processed = tmp_path / "lab_processed" / "images" From 77953f1b7953d947768b40eeecef00ec54220b67 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 17:47:59 +0200 Subject: [PATCH 025/107] fix(qptiff): do not pair scan-profile records by position across markers With pages [DAPI, CD3, CD4] and records [DAPI, CD3, DAPI], CD4 got the cycle of the second DAPI. Positional pairing is now skipped when a record names the marker of another page; the name join leaves such channels unset. Co-Authored-By: Claude Opus 5.5 --- src/spatialdata_io/readers/_utils/_qptiff.py | 8 +++++--- tests/readers/test_qptiff.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_qptiff.py b/src/spatialdata_io/readers/_utils/_qptiff.py index f37984a4..15fd48a6 100644 --- a/src/spatialdata_io/readers/_utils/_qptiff.py +++ b/src/spatialdata_io/readers/_utils/_qptiff.py @@ -414,11 +414,13 @@ def _positional_pairs( if len(channels) != len(records): return [] # Fusion orders the pages of a cycle differently from the well items; a page whose marker sits at another - # position in the profile means the orders differ and the positional pairing would be wrong + # position in the profile, or a record naming the marker of another page, means the orders differ and the + # positional pairing would be wrong + page_keys = {_name_key(channel.name) for channel in channels} record_keys = {_name_key(name) for name, _ in records} for channel, (name, _) in zip(channels, records, strict=True): - key = _name_key(channel.name) - if key != _name_key(name) and key in record_keys: + key, record_key = _name_key(channel.name), _name_key(name) + if key != record_key and (key in record_keys or record_key in page_keys): return [] return list(zip(channels, records, strict=True)) diff --git a/tests/readers/test_qptiff.py b/tests/readers/test_qptiff.py index d349d9bb..12c450ba 100644 --- a/tests/readers/test_qptiff.py +++ b/tests/readers/test_qptiff.py @@ -160,6 +160,17 @@ def test_enrichment_positional_when_counts_match(tmp_path: Path) -> None: assert [(c.cycle, c.channel) for c in scan.channels] == [(1, 1), (1, 2), (2, 1)] +def test_enrichment_not_positional_when_a_record_names_another_page(tmp_path: Path) -> None: + # the third record is DAPI, a marker of another page: pairing it with CD4 by position would be a guess + p = tmp_path / "x.qptiff" + wells = [[("DAPI", "DAPI"), ("CD3", "ATTO 550")], [("DAPI", "DAPI")]] + write_qptiff(p, biomarkers=["DAPI", "CD3", "CD4"], scan_profile=fusion_scan_profile_json(wells)) + scan = read_qptiff_metadata(p) + cd4 = scan.channels[2] + assert (cd4.cycle, cd4.channel) == (None, None) + assert (scan.channels[1].cycle, scan.channels[1].channel) == (1, 2) + + def test_enrichment_not_positional_when_orders_differ(tmp_path: Path) -> None: p = tmp_path / "x.qptiff" wells = [[("DAPI", "DAPI"), ("CD3", "ATTO 550"), ("CD8", "CY5"), ("CD20", "AF 750")]] From fc5fa05c05219fdabb168cd6bc64488380e89e89 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 17:51:05 +0200 Subject: [PATCH 026/107] fix: name the file when a region's cells share an instance id Duplicate instance ids within one region surfaced as TableModel's raw 'Instance key ... not unique' error. The adapters now raise a ValueError naming the table, the region and the duplicated ids. Co-Authored-By: Claude Opus 5.5 --- src/spatialdata_io/readers/_utils/_cell_tables.py | 15 +++++++++++++++ tests/test_codex.py | 10 ++++++++++ tests/test_phenocycler.py | 8 ++++++++ 3 files changed, 33 insertions(+) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index f7e52d83..4884be01 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -236,6 +236,7 @@ def read_codex_table(path: Path, region: str | None = None) -> CellTable: "assigned to a region; a table for several regions needs a region column." ) region_ids.name = region_key + _check_unique_instance_ids(path, region_ids, instance_id) return CellTable( obs=_drop_key_columns(obs, columns.get(instance_key), columns.get(region_key)), X=X, @@ -769,6 +770,7 @@ def _from_frame( "region as a mapping {region: path}." ) region.name = CodexKeys.REGION_KEY.value + _check_unique_instance_ids(path, region, instance_id) obs = _drop_key_columns(df.drop(columns=markers), instance_column, region_column) obs.index = index @@ -789,6 +791,19 @@ def _from_frame( ) +def _check_unique_instance_ids(path: Path, region: pd.Series, instance_id: pd.Series) -> None: + """Raise if two cells of the same region have the same instance id: they could not be told apart.""" + regions, ids = region.to_numpy(), instance_id.to_numpy() + duplicated = pd.DataFrame({"region": regions, "id": ids}).duplicated().to_numpy() + if duplicated.any(): + first = regions[duplicated][0] + found = sorted(set(ids[duplicated & (regions == first)].tolist())) + raise ValueError( + f"{path} has {len(found)} duplicate instance id(s) {found[:5]} in region '{first}'; every cell of a " + "region needs its own instance id. Choose another instance id column, or remove the duplicate rows." + ) + + def _drop_key_columns(obs: pd.DataFrame, instance_column: str | None, region_column: str | None) -> pd.DataFrame: """Drop the instance id and region columns if they are named like the keys whose values replace them. diff --git a/tests/test_codex.py b/tests/test_codex.py index 0b1dbda6..7dce4c9f 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -492,6 +492,16 @@ def test_codex_regionless_table_belongs_to_single_stem_image(tmp_path: Path) -> assert sdata.attrs["akoya"]["cell_tables"] == {"*": str(tmp_path / "cells.csv")} +def test_codex_duplicate_cell_ids_raise(tmp_path: Path) -> None: + data = write_legacy_processor_tree(tmp_path, cycles=1, channels=1, fcs=False, n_cells=3) + table = next(tmp_path.rglob("reg001_compensated.csv")) + cells = data["cells"].assign(cell_id=[1.0, 1.0, 2.0]) + cells.to_csv(table, index=False) + + with pytest.raises(ValueError, match="reg001_compensated.csv has 1 duplicate instance id"): + codex(tmp_path, image_models_kwargs=_FAST) + + def test_codex_overlapping_tables_raise(tmp_path: Path) -> None: write_legacy_processor_tree(tmp_path, layout="per_marker", fcs=False, csv=True) (tmp_path / "all_cells.csv").write_text("cell_id,region,x,y,cyc001_ch001\n1,1,1.0,2.0,3.0\n") diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 1be0c3e8..3d4bf7e3 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -494,6 +494,14 @@ def test_extra_region_and_cell_id_columns_are_kept( assert list(obs["source_cell_id"]) == [101, 102] +def test_duplicate_instance_ids_raise(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + (tmp_path / "cells.csv").write_text("CellID,CD3,X_centroid,Y_centroid\n1,1,1,1\n2,1,2,2\n2,1,3,3\n") + + with pytest.raises(ValueError, match=r"cells\.csv has 1 duplicate instance id.*\[2\].*region 'S'"): + phenocycler(tmp_path, cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) + + def test_qupath_without_markers_warns(tmp_path: Path) -> None: write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) (tmp_path / "cells.tsv").write_text("Object ID\tCentroid X px\tCentroid Y px\nu1\t1\t2\n", encoding="utf-8") From 8da35f22eb5f6dab5cdf160061377a56d0630a58 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 17:51:53 +0200 Subject: [PATCH 027/107] test(codex): write and re-read a two-region CODEX Processor read Co-Authored-By: Claude Opus 5.5 --- tests/test_codex.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_codex.py b/tests/test_codex.py index 7dce4c9f..e755bbf7 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -10,6 +10,7 @@ import pandas as pd import pytest import tifffile +from spatialdata import read_zarr from spatialdata.models import get_channel_names from spatialdata.transformations import Identity, get_transformation @@ -508,3 +509,27 @@ def test_codex_overlapping_tables_raise(tmp_path: Path) -> None: with pytest.raises(ValueError, match=r"Several cell tables contain the cells of region\(s\) \['reg001'\]"): codex(tmp_path, fcs=False, image_models_kwargs=_FAST) + + +def test_codex_write_roundtrip(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path / "data", regions=2, cycles=1, channels=2, n_cells=5) + with pytest.warns(UserWarning, match="circles"): + sdata = codex(tmp_path / "data", image_models_kwargs=_FAST) + + sdata.write(tmp_path / "x.zarr") + read = read_zarr(tmp_path / "x.zarr") + + assert sorted(read.images) == ["reg001_image", "reg002_image"] + assert sorted(read.shapes) == ["reg001_cells", "reg002_cells"] + for region in ("reg001", "reg002"): + name = f"{region}_image" + assert get_channel_names(read.images[name]) == get_channel_names(sdata.images[name]) + np.testing.assert_array_equal(_level0(read, name).values, _level0(sdata, name).values) + assert get_transformation(read.images[name], get_all=True) == {region: Identity()} + assert get_transformation(read.shapes[f"{region}_cells"], get_all=True) == {region: Identity()} + table = read.tables["table"] + assert table.n_obs == 10 + assert list(table.obs["region"].cat.categories) == ["reg001_cells", "reg002_cells"] + assert list(table.var_names) == list(sdata.tables["table"].var_names) + assert read.attrs["akoya"] == sdata.attrs["akoya"] + assert read.attrs["spatialdata_io_reader"] == "codex" From f09f09b10a2142e8ada20076a079df639b0aa3ce Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 17:59:31 +0200 Subject: [PATCH 028/107] refactor: share the Akoya element assembly and qptiff region ids `codex()` and `phenocycler()` duplicated the images -> labels -> polygons -> cells -> SpatialData -> attrs assembly; it is now one helper. Both readers also name a qptiff without `regNNN` the same way (suffix and trailing `_Scan` stripped, `.er.qptiff` preferred). The discovery helpers of the reader modules are private and no longer exported. Co-Authored-By: Claude Opus 5.5 --- .../readers/_utils/_cell_tables.py | 95 ++++++++++++- src/spatialdata_io/readers/_utils/_qptiff.py | 57 +++++++- src/spatialdata_io/readers/codex.py | 93 ++++++------- src/spatialdata_io/readers/phenocycler.py | 128 ++++-------------- tests/test_codex.py | 34 +++-- 5 files changed, 236 insertions(+), 171 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 4884be01..ae0f8b0f 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -18,31 +18,42 @@ import readfcs import shapely import tifffile -from spatialdata import sanitize_name, sanitize_table +from spatialdata import SpatialData, sanitize_name, sanitize_table from spatialdata._logging import logger from spatialdata.models import Labels2DModel, ShapesModel, TableModel from spatialdata.transformations import Identity from xarray import DataTree from spatialdata_io._constants._constants import CodexKeys, PhenocyclerKeys -from spatialdata_io.readers._utils._akoya import dedupe_channel_names, read_tiff, spatial_chunks +from spatialdata_io.readers._utils._akoya import ( + build_image_element, + dedupe_channel_names, + read_tiff, + region_transformations, + scans_to_attrs, + spatial_chunks, +) from spatialdata_io.readers._utils._rasterize import rasterize_polygons_lazy from spatialdata_io.readers._utils._utils import _initialize_raster_models_kwargs if TYPE_CHECKING: from collections.abc import Callable, Sequence + import dask.array as da from geopandas import GeoDataFrame from spatialdata.transformations import BaseTransformation from xarray import DataArray + from spatialdata_io.readers._utils._qptiff import AkoyaScan + __all__ = [ "CellTable", - "raster_shape", "build_cells", + "build_spatialdata", "detect_table_format", "polygon_labels", "polygon_shapes", + "raster_shape", "read_cell_table", "read_codex_table", "read_labels", @@ -1241,6 +1252,84 @@ def element(region: str) -> str: return {f"{region}_table": _to_anndata([piece], element) for region, piece in sorted(pieces.items())}, shapes +def build_spatialdata( + scans: Sequence[AkoyaScan], + images: Mapping[str, da.Array], + channel_names: Mapping[str, list[str]], + cell_tables: Sequence[CellTable], + mask_files: Mapping[str, Path], + *, + table_paths: Mapping[str, str] | None, + image_models_kwargs: Mapping[str, Any], + labels_models_kwargs: Mapping[str, Any], + cells_table_kwargs: Mapping[str, Any], +) -> SpatialData: + """Assemble the regions of an Akoya read into a `SpatialData` with their metadata in ``attrs["akoya"]``. + + Each region gets the image ``_image``, the labels ``_labels`` from its mask or its cell polygons + (see :func:`polygon_labels`), the polygons ``_polygons`` and the cells (see :func:`build_cells`), all in + the coordinate systems of :func:`region_transformations`; regions with cells or a mask but no image are + unrelated to the scans. + + Parameters + ---------- + scans + The scans read. + images + The lazy image of each scan, by region id, see :func:`build_image_element`. + channel_names + The channel names of each scan, by region id. + cell_tables + The cell tables read. + mask_files + The mask of each region, as resolved by :func:`resolve_per_region`. + table_paths + The paths of the cell tables, stored in ``attrs["akoya"]["cell_tables"]`` unless `None`. + image_models_kwargs + Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. + labels_models_kwargs + Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`. + cells_table_kwargs + See :func:`read_labels` (``mask_channel``). + + Returns + ------- + The `SpatialData` object. + """ + cell_regions = {region for table in cell_tables for region in table.region.unique()} + transformations = region_transformations(scans, cell_regions | (mask_files.keys() - {"*"})) + image_elements = { + f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}": build_image_element( + images[scan.region], scan, channel_names[scan.region], transformations[scan.region], image_models_kwargs + ) + for scan in scans + } + image_shapes = { + scan.region: raster_shape(image_elements[f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}"]) for scan in scans + } + labels = read_region_labels(mask_files, transformations, labels_models_kwargs, cells_table_kwargs, image_shapes) + polygons, geojson_files = polygon_labels(cell_tables, labels, image_shapes, transformations, labels_models_kwargs) + labels |= polygons + tables, shapes = build_cells(cell_tables, {}, labels, transformations) if cell_tables else ({}, {}) + for region, element in polygon_shapes(cell_tables, transformations).items(): + shapes[f"{region}{PhenocyclerKeys.POLYGONS_SUFFIX}"] = element + sdata = SpatialData( + images=image_elements, + labels={f"{region}{PhenocyclerKeys.LABELS_SUFFIX}": element for region, element in labels.items()}, + shapes=shapes, + tables=tables, + ) + attrs = scans_to_attrs(scans, channel_names) + if table_paths is not None: + attrs["cell_tables"] = dict(table_paths) + if mask_files: + attrs["labels"] = {region: str(file) for region, file in mask_files.items()} + if geojson_files: + attrs["geojson"] = geojson_files + sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value] = attrs + return sdata + + def _region_rows(table: CellTable) -> dict[str, np.ndarray]: """The row indices of each region of ``table``, in order of first appearance.""" codes, regions = pd.factorize(table.region) diff --git a/src/spatialdata_io/readers/_utils/_qptiff.py b/src/spatialdata_io/readers/_utils/_qptiff.py index 15fd48a6..fcd7a302 100644 --- a/src/spatialdata_io/readers/_utils/_qptiff.py +++ b/src/spatialdata_io/readers/_utils/_qptiff.py @@ -17,11 +17,21 @@ from spatialdata_io.readers._utils._akoya import lazy_tiff_array if TYPE_CHECKING: + from collections.abc import Sequence from pathlib import Path -__all__ = ["AkoyaChannel", "AkoyaScan", "qpi_pixel_size_um", "read_qptiff_image", "read_qptiff_metadata"] +__all__ = [ + "AkoyaChannel", + "AkoyaScan", + "qpi_pixel_size_um", + "qptiff_base_name", + "qptiff_regions", + "read_qptiff_image", + "read_qptiff_metadata", +] _FULL_RESOLUTION = "FullResolution" +_SCAN_SUFFIX = re.compile(r"_Scan\d+$") _XML_DECLARATION = re.compile(r"^\s*<\?xml[^>]*\?>") # micrometres per unit of the TIFF ResolutionUnit; XPosition/YPosition are expressed in the same unit _UM_PER_RESOLUTION_UNIT = {tifffile.RESUNIT.CENTIMETER: 1e4, tifffile.RESUNIT.INCH: 25400.0} @@ -114,7 +124,7 @@ def read_qptiff_metadata(path: Path, region: str | None = None) -> AkoyaScan: return AkoyaScan( path=path, - region=region if region is not None else _default_region(path), + region=region if region is not None else qptiff_base_name(path), generation=_generation(software, experiment_v4 is not None), software=software, slide_id=_text(root, "SlideID"), @@ -228,13 +238,48 @@ def _bool(text: str | None) -> bool | None: return None if text is None else text.lower() == "true" -def _default_region(path: Path) -> str: - for suffix in (PhenocyclerKeys.ER_QPTIFF_SUFFIX, PhenocyclerKeys.QPTIFF_SUFFIX): - if path.name.lower().endswith(suffix.value): - return path.name[: -len(suffix.value)] +def qptiff_base_name(path: Path) -> str: + """The file name without the ``.er.qptiff``/``.qptiff`` suffix (in any case).""" + for suffix in (PhenocyclerKeys.ER_QPTIFF_SUFFIX.value, PhenocyclerKeys.QPTIFF_SUFFIX.value): + if path.name.lower().endswith(suffix): + return path.name[: -len(suffix)] return path.stem +def qptiff_regions(files: Sequence[Path]) -> dict[str, Path]: + """The region id of every qptiff scan among ``files``, the same for both Akoya readers. + + Files of the same scan (folder and base name) are read once, preferring the extended-range ``.er.qptiff``. + The region id is the base name without a trailing ``_Scan``, unless that makes two scans share a region id. + + Parameters + ---------- + files + The qptiff files, in the order of the result. + + Returns + ------- + The scan file of every region. + """ + scans: dict[tuple[Path, str], Path] = {} + for file in files: + key = (file.parent, qptiff_base_name(file)) + if key not in scans or file.name.lower().endswith(PhenocyclerKeys.ER_QPTIFF_SUFFIX.value): + scans[key] = file + bases = [base for _, base in scans] + stripped = [_SCAN_SUFFIX.sub("", base) or base for base in bases] + # stripping `_Scan` must not merge two scans into one region + regions = [base if stripped.count(region) > 1 else region for base, region in zip(bases, stripped, strict=True)] + if len(set(regions)) < len(regions): + duplicated = sorted({region for region in regions if regions.count(region) > 1}) + found = [str(file) for file, region in zip(scans.values(), regions, strict=True) if region in duplicated] + raise ValueError( + f"Several qptiff files map to the same region(s) {duplicated}: {found}. Pass the path of a single qptiff " + "file or of a directory that contains each region once." + ) + return dict(zip(regions, scans.values(), strict=True)) + + def _generation(software: str | None, has_experiment_v4: bool) -> str: software = software or "" if software.startswith("Fusion 1"): diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 66545951..54cb5fda 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -14,38 +14,37 @@ import pandas as pd from spatialdata import SpatialData -from spatialdata_io._constants._constants import CodexKeys, PhenocyclerKeys +from spatialdata_io._constants._constants import CodexKeys from spatialdata_io._docs import inject_docs from spatialdata_io.readers._utils._akoya import ( apply_channel_names, - build_image_element, normalize_regions, read_json_object, read_tiff, - region_transformations, - scans_to_attrs, select_regions, spatial_chunks, ) from spatialdata_io.readers._utils._cell_tables import ( CellTable, - build_cells, + build_spatialdata, detect_table_format, - polygon_labels, - polygon_shapes, - raster_shape, read_codex_table, - read_region_labels, read_region_tables, resolve_per_region, ) -from spatialdata_io.readers._utils._qptiff import AkoyaChannel, AkoyaScan, read_qptiff_image, read_qptiff_metadata +from spatialdata_io.readers._utils._qptiff import ( + AkoyaChannel, + AkoyaScan, + qptiff_regions, + read_qptiff_image, + read_qptiff_metadata, +) from spatialdata_io.readers._utils._utils import _initialize_raster_models_kwargs, _set_reader_metadata if TYPE_CHECKING: from collections.abc import Sequence -__all__ = ["LegacyLayout", "codex", "discover_legacy", "read_legacy_image"] +__all__ = ["codex"] _REGION = re.compile(r"reg(\d+)", re.IGNORECASE) _PER_MARKER = re.compile(r"reg(\d+)_cyc(\d+)_ch(\d+)_(.+)\.tiff?$", re.IGNORECASE) @@ -68,7 +67,7 @@ @dataclass -class LegacyLayout: +class _LegacyLayout: """Files of a CODEX Processor/Uploader output tree or of CODEX Instrument Manager scans. ``images`` maps each region to its per-marker images (sorted by cycle and channel), to its single hyperstack @@ -121,7 +120,9 @@ def codex( Raw acquisition and intermediate folders (``cycCCC_regNNN/``, ``tiles/`` and, next to ``stitched/``, ``bestFocus/``) are ignored. The region id ``regNNN`` is taken from the file names; a ``{vx.QPTIFF!r}`` without - one is named after its file. If a directory contains no such files, its top-level ``{vx.IMAGE_TIF}(f)`` files + one is named as :func:`spatialdata_io.phenocycler` names it: after its file, without the ``.er.qptiff`` or + ``{vx.QPTIFF!r}`` suffix and a trailing ``_Scan`` (unless that makes two regions share a name), and the + extended-range ``.er.qptiff`` of a scan is preferred. If a directory contains no such files, its top-level ``{vx.IMAGE_TIF}(f)`` files are read as hyperstacks named after their files. Each region is stored as the image ``_image`` and its metadata in ``sdata.attrs["akoya"]``. @@ -197,7 +198,7 @@ def codex( regions = normalize_regions(regions) channel_names = [channel_names] if isinstance(channel_names, str) else channel_names path = Path(path) - layout = discover_legacy(path, fcs) + layout = _discover_legacy(path, fcs) # regions that only have a discovered table exist only while the discovered tables are read table_regions = layout.tables.keys() - {"*"} if cells_table is None else set() available = sorted(layout.images.keys() | table_regions) @@ -215,7 +216,7 @@ def codex( scans.append(read_qptiff_metadata(files[0], region=region)) arrays[region] = read_qptiff_image(files[0]) continue - arrays[region], channels = read_legacy_image( + arrays[region], channels = _read_legacy_image( files, layout.channel_names_txt, imread_kwargs, image_models_kwargs=image_models_kwargs ) scans.append( @@ -254,44 +255,22 @@ def codex( read = _read_tables(discovered, layout, selected if regions is not None else None, fcs) cell_tables = list(read.values()) table_paths = {key: str(discovered[key]) for key in read} - # the regions of cells or masks without an image are unrelated to the scans - cell_regions = {region for table in cell_tables for region in table.region.unique()} - transformations = region_transformations(scans, cell_regions | (mask_files.keys() - {"*"})) - images = { - f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}": build_image_element( - arrays[scan.region], scan, names[scan.region], transformations[scan.region], image_models_kwargs - ) - for scan in scans - } - image_shapes = {scan.region: raster_shape(images[f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}"]) for scan in scans} - region_labels = read_region_labels( - mask_files, transformations, labels_models_kwargs, cells_table_kwargs, image_shapes - ) - polygons, geojson_files = polygon_labels( - cell_tables, region_labels, image_shapes, transformations, labels_models_kwargs + sdata = build_spatialdata( + scans, + arrays, + names, + cell_tables, + mask_files, + table_paths=table_paths, + image_models_kwargs=image_models_kwargs, + labels_models_kwargs=labels_models_kwargs, + cells_table_kwargs=cells_table_kwargs, ) - region_labels |= polygons - tables, shapes = build_cells(cell_tables, {}, region_labels, transformations) if cell_tables else ({}, {}) - for region, element in polygon_shapes(cell_tables, transformations).items(): - shapes[f"{region}{PhenocyclerKeys.POLYGONS_SUFFIX}"] = element - sdata = SpatialData( - images=images, - labels={f"{region}{PhenocyclerKeys.LABELS_SUFFIX}": element for region, element in region_labels.items()}, - shapes=shapes, - tables=tables, - ) - attrs = scans_to_attrs(scans, names) - attrs["cell_tables"] = table_paths - if mask_files: - attrs["labels"] = {region: str(file) for region, file in mask_files.items()} - if geojson_files: - attrs["geojson"] = geojson_files - sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value] = attrs return _set_reader_metadata(sdata, "codex") def _read_tables( - files: Mapping[str, Path], layout: LegacyLayout, regions: Sequence[str] | None, fcs: bool + files: Mapping[str, Path], layout: _LegacyLayout, regions: Sequence[str] | None, fcs: bool ) -> dict[str, CellTable]: """Read the cell tables of ``files`` (by region, ``"*"`` for one without a region), keeping only ``regions``. @@ -328,7 +307,7 @@ def _read_tables( return tables -def discover_legacy(path: Path, fcs: bool) -> LegacyLayout: +def _discover_legacy(path: Path, fcs: bool) -> _LegacyLayout: """Find the images, cell tables and sidecar files of a CODEX dataset. Parameters @@ -353,7 +332,7 @@ def discover_legacy(path: Path, fcs: bool) -> LegacyLayout: tables = _discover_tables(files, fcs) if not images and not tables: raise FileNotFoundError(f"No CODEX images or cell tables found in {path}. Expected: {'; '.join(_PATTERNS)}.") - return LegacyLayout( + return _LegacyLayout( images=dict(sorted(images.items())), tables=dict(sorted(tables.items())), channel_names_txt=_find_sidecar(path, files, CodexKeys.CHANNEL_NAMES_FILE.value), @@ -361,7 +340,7 @@ def discover_legacy(path: Path, fcs: bool) -> LegacyLayout: ) -def read_legacy_image( +def _read_legacy_image( files: list[Path], channel_names_txt: Path | None, imread_kwargs: Mapping[str, Any], @@ -372,7 +351,7 @@ def read_legacy_image( Parameters ---------- files - The per-marker images of the region, or its single hyperstack, as returned by :func:`discover_legacy`. + The per-marker images of the region, or its single hyperstack, as returned by :func:`_discover_legacy`. channel_names_txt Optional ``channelNames.txt`` naming the hyperstack channels when the file has no ImageJ ``Labels``. imread_kwargs @@ -486,10 +465,14 @@ def _discover_images(files: Sequence[Path]) -> dict[str, list[Path]]: per_marker: dict[str, dict[tuple[int, int], Path]] = {} stacks: dict[str, list[Path]] = {} qptiffs: dict[str, list[Path]] = {} + unnamed_qptiffs = [] for file in files: region = _region_id(file.name) if _is_qptiff(file): - qptiffs.setdefault(region or file.stem, []).append(file) + if region is None: + unnamed_qptiffs.append(file) + else: + qptiffs.setdefault(region, []).append(file) elif _is_tiff(file) and region is not None: if (match := _PER_MARKER.search(file.name)) is None: stacks.setdefault(region, []).append(file) @@ -503,6 +486,10 @@ def _discover_images(files: Sequence[Path]) -> dict[str, list[Path]]: ) markers[key] = file + # named like `phenocycler()` names them, e.g. `Tonsil_Scan1.er.qptiff` -> `Tonsil` + for region, file in qptiff_regions(unnamed_qptiffs).items(): + qptiffs.setdefault(region, []).append(file) + images: dict[str, list[Path]] = {} # the Processor's stitched outputs come first: cell tables refer to their pixel space for region in sorted(per_marker.keys() | stacks.keys() | qptiffs.keys()): diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 98b8ab0b..1f8c5b66 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -14,25 +14,16 @@ from spatialdata_io._docs import inject_docs from spatialdata_io.readers._utils._akoya import ( apply_channel_names, - build_image_element, normalize_regions, read_json_object, - region_transformations, - scans_to_attrs, select_regions, ) -from spatialdata_io.readers._utils._cell_tables import ( - build_cells, - polygon_labels, - polygon_shapes, - raster_shape, - read_region_labels, - read_region_tables, - resolve_per_region, -) +from spatialdata_io.readers._utils._cell_tables import build_spatialdata, read_region_tables, resolve_per_region from spatialdata_io.readers._utils._qptiff import ( _fusion_profile_records, _is_blank, + qptiff_base_name, + qptiff_regions, read_qptiff_image, read_qptiff_metadata, ) @@ -43,9 +34,8 @@ from spatialdata_io.readers._utils._qptiff import AkoyaScan -__all__ = ["discover_qptiffs", "phenocycler"] +__all__ = ["phenocycler"] -_SCAN_SUFFIX = re.compile(r"_Scan\d+$") # the Fusion writes per-cycle acquisition files to ``raw/images/phenocycler/`` (HuBMAP layout) _ACQUISITION_DIR = ("raw", "images", "phenocycler") @@ -151,7 +141,7 @@ def phenocycler( regions = normalize_regions(regions) channel_names = [channel_names] if isinstance(channel_names, str) else channel_names path = Path(path) - qptiffs = discover_qptiffs(path) + qptiffs = _discover_qptiffs(path) selected = select_regions(list(qptiffs), regions) table_files = resolve_per_region(cells_table, selected, list(qptiffs)) mask_files = resolve_per_region(labels, selected, list(qptiffs)) @@ -161,22 +151,6 @@ def phenocycler( for scan in scans: names[scan.region] = apply_channel_names(scan, channel_names) _read_sidecars(scan) - - transformations = region_transformations(scans) - images = { - f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}": build_image_element( - read_qptiff_image(scan.path), - scan, - names[scan.region], - transformations[scan.region], - image_models_kwargs, - ) - for scan in scans - } - image_shapes = {scan.region: raster_shape(images[f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}"]) for scan in scans} - region_labels = read_region_labels( - mask_files, transformations, labels_models_kwargs, cells_table_kwargs, image_shapes - ) cell_tables = read_region_tables( table_files, cells_table_format, @@ -185,87 +159,39 @@ def phenocycler( pixel_sizes={scan.region: scan.pixel_size_um for scan in scans}, cells_table_kwargs=cells_table_kwargs, ) - polygons, geojson_files = polygon_labels( - cell_tables, region_labels, image_shapes, transformations, labels_models_kwargs - ) - region_labels |= polygons - tables, shapes = build_cells(cell_tables, {}, region_labels, transformations) if cell_tables else ({}, {}) - for region, element in polygon_shapes(cell_tables, transformations).items(): - shapes[f"{region}{PhenocyclerKeys.POLYGONS_SUFFIX}"] = element - sdata = SpatialData( - images=images, - labels={f"{region}{PhenocyclerKeys.LABELS_SUFFIX}": element for region, element in region_labels.items()}, - shapes=shapes, - tables=tables, + sdata = build_spatialdata( + scans, + {scan.region: read_qptiff_image(scan.path) for scan in scans}, + names, + cell_tables, + mask_files, + table_paths={key: str(file) for key, file in table_files.items()} if table_files else None, + image_models_kwargs=image_models_kwargs, + labels_models_kwargs=labels_models_kwargs, + cells_table_kwargs=cells_table_kwargs, ) - attrs = scans_to_attrs(scans, names) - if table_files: - attrs["cell_tables"] = {key: str(file) for key, file in table_files.items()} - if mask_files: - attrs["labels"] = {region: str(file) for region, file in mask_files.items()} - if geojson_files: - attrs["geojson"] = geojson_files - sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value] = attrs return _set_reader_metadata(sdata, "phenocycler") -def discover_qptiffs(path: Path) -> dict[str, Path]: - """Find the qptiff scans of a PhenoCycler-Fusion dataset. +def _discover_qptiffs(path: Path) -> dict[str, Path]: + """Find the qptiff scans of a PhenoCycler-Fusion dataset: a ``.qptiff`` file, or a directory searched recursively. - Parameters - ---------- - path - A ``.qptiff`` file, or a directory that is searched recursively. - - Returns - ------- - The scan file of every region, sorted by path. + Returns the scan file of every region, sorted by path; see :func:`qptiff_regions`. """ if not path.exists(): raise FileNotFoundError(f"{path} does not exist.") if path.is_file(): - return {_region(_base_name(path)): path} - - # one scan per (directory, base name), preferring the extended-range `.er.qptiff` - scans: dict[tuple[Path, str], Path] = {} - for qptiff in sorted(path.rglob(f"*{PhenocyclerKeys.QPTIFF_SUFFIX}", case_sensitive=False)): - if not qptiff.is_file() or _is_acquisition_file(qptiff.relative_to(path)): - continue - key = (qptiff.parent, _base_name(qptiff)) - if key not in scans or _is_extended_range(qptiff): - scans[key] = qptiff - if not scans: + return qptiff_regions([path]) + files = [ + qptiff + for qptiff in sorted(path.rglob(f"*{PhenocyclerKeys.QPTIFF_SUFFIX}", case_sensitive=False)) + if qptiff.is_file() and not _is_acquisition_file(qptiff.relative_to(path)) + ] + if not files: raise FileNotFoundError( f"No .qptiff files found in {path} (searched recursively for '*.qptiff', excluding acquisition files)." ) - - bases = [base for _, base in scans] - regions = [_region(base) for base in bases] - # stripping `_Scan` must not merge two scans into one region - regions = [base if regions.count(region) > 1 else region for base, region in zip(bases, regions, strict=True)] - if len(set(regions)) < len(regions): - duplicated = sorted({region for region in regions if regions.count(region) > 1}) - raise ValueError( - f"Several qptiff files in {path} map to the same region(s) {duplicated}. Pass the path of a single " - "qptiff file or of a directory that contains each region once." - ) - return dict(zip(regions, scans.values(), strict=True)) - - -def _base_name(path: Path) -> str: - """File name without the ``.er.qptiff``/``.qptiff`` suffix.""" - for suffix in (PhenocyclerKeys.ER_QPTIFF_SUFFIX, PhenocyclerKeys.QPTIFF_SUFFIX): - if path.name.lower().endswith(suffix): - return path.name[: -len(suffix)] - return path.stem - - -def _region(base_name: str) -> str: - return _SCAN_SUFFIX.sub("", base_name) or base_name - - -def _is_extended_range(path: Path) -> bool: - return path.name.lower().endswith(PhenocyclerKeys.ER_QPTIFF_SUFFIX) + return qptiff_regions(files) def _is_acquisition_file(path: Path) -> bool: @@ -282,7 +208,7 @@ def _read_sidecars(scan: AkoyaScan) -> None: The sidecars are metadata only, so one that cannot be read is ignored with a warning. """ - base = _base_name(scan.path) + base = qptiff_base_name(scan.path) xpd = _find_sidecar(scan.path, base, PhenocyclerKeys.XPD_SUFFIX) protocol = read_json_object(xpd) if xpd is not None else None if xpd is not None and protocol is not None: diff --git a/tests/test_codex.py b/tests/test_codex.py index e755bbf7..57a1094b 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -14,9 +14,9 @@ from spatialdata.models import get_channel_names from spatialdata.transformations import Identity, get_transformation -from spatialdata_io import codex +from spatialdata_io import codex, phenocycler from spatialdata_io.readers._utils._akoya import dedupe_channel_names -from spatialdata_io.readers.codex import discover_legacy, read_legacy_image +from spatialdata_io.readers.codex import _discover_legacy, _read_legacy_image from tests._akoya_data import cim_experiment_v4_xml, write_legacy_processor_tree, write_qptiff if TYPE_CHECKING: @@ -118,6 +118,24 @@ def test_codex_cim_qptiff(tmp_path: Path) -> None: assert sdata.attrs["akoya"]["regions"]["reg001"]["generation"] == "cim_qptiff" +def test_codex_and_phenocycler_name_a_qptiff_alike(tmp_path: Path) -> None: + write_qptiff(tmp_path / "Tonsil_Scan1.qptiff", biomarkers=["DAPI"], dtype="uint8") + write_qptiff(tmp_path / "Tonsil_Scan1.er.qptiff", biomarkers=["DAPI"], dtype="uint16") + + for reader in (codex, phenocycler): + sdata = reader(tmp_path, image_models_kwargs=_FAST) + assert list(sdata.images) == ["Tonsil_image"] + assert sdata.attrs["akoya"]["regions"]["Tonsil"]["path"].endswith("Tonsil_Scan1.er.qptiff") + + +def test_reader_modules_export_only_the_readers() -> None: + from spatialdata_io.readers import codex as codex_module + from spatialdata_io.readers import phenocycler as phenocycler_module + + assert codex_module.__all__ == ["codex"] + assert phenocycler_module.__all__ == ["phenocycler"] + + def test_codex_old_flat_layout_image(tmp_path: Path) -> None: data = write_legacy_processor_tree(tmp_path, layout="old_flat") @@ -170,11 +188,11 @@ def test_codex_regions_channel_names_and_pixel_size(tmp_path: Path) -> None: def test_discover_legacy_tables_prefer_requested_format(tmp_path: Path) -> None: write_legacy_processor_tree(tmp_path, regions=2, layout="per_marker", fcs=True, csv=True) - assert {region: path.suffix for region, path in discover_legacy(tmp_path, fcs=True).tables.items()} == { + assert {region: path.suffix for region, path in _discover_legacy(tmp_path, fcs=True).tables.items()} == { "reg001": ".fcs", "reg002": ".fcs", } - assert {region: path.suffix for region, path in discover_legacy(tmp_path, fcs=False).tables.items()} == { + assert {region: path.suffix for region, path in _discover_legacy(tmp_path, fcs=False).tables.items()} == { "reg001": ".csv", "reg002": ".csv", } @@ -292,7 +310,7 @@ def test_read_legacy_image_tiles_uncompressed_planes(tmp_path: Path, imagej: boo tifffile.imwrite(hyperstack, stack[None], imagej=True, metadata={"axes": "TCYX"}) for source in (files, [hyperstack]): - image, _ = read_legacy_image(source, None, {}, image_models_kwargs={"chunks": (1, 32, 64)}) + image, _ = _read_legacy_image(source, None, {}, image_models_kwargs={"chunks": (1, 32, 64)}) assert max(image.chunks[-2]) <= 32 assert max(image.chunks[-1]) <= 64 np.testing.assert_array_equal(image.compute(), stack) @@ -315,15 +333,15 @@ def test_discover_legacy_regionless_table_ignores_sidecar_csv(tmp_path: Path) -> cells.parent.mkdir() cells.write_text("cell_id,x,y,cyc001_ch001:DAPI\n1,2.0,3.0,4.0\n") - assert discover_legacy(tmp_path, fcs=True).tables == {"*": cells} + assert _discover_legacy(tmp_path, fcs=True).tables == {"*": cells} (tmp_path / "clusters.csv").write_text("cellid,cluster\n1,2\n") (tmp_path / "xy.csv").write_text("x,y\n1.0,2.0\n") - assert discover_legacy(tmp_path, fcs=True).tables == {"*": cells} + assert _discover_legacy(tmp_path, fcs=True).tables == {"*": cells} (tmp_path / "other_cells.csv").write_text("cell_id:cell_id,x:x,y:y\n1,1.0,2.0\n") with pytest.raises(ValueError, match="cannot choose"): - discover_legacy(tmp_path, fcs=True) + _discover_legacy(tmp_path, fcs=True) @pytest.mark.parametrize("with_table", [True, False]) From 97b1051e2b1c07d270c7344c2f0fb502caf8122a Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 18:00:19 +0200 Subject: [PATCH 029/107] docs: describe the Akoya fixes in the changelog Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc363ce4..61e7155f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,9 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. `.er.qptiff` scans, including PhenoImager (Opal) mode: lazy level-0 image with a recomputed pyramid, channel names from the per-page XML, structured and raw scan metadata in `sdata.attrs["akoya"]`, optional `.xpd` / `*channels.csv` sidecars, and optional cell tables (QuPath with GeoJSON outlines, MCMICRO `mcquant`, HuBMAP SPRM, - inForm, CODEX Processor or generic CSV) with segmentation masks. + inForm, CODEX Processor or generic CSV) with segmentation masks. The lazy images and masks can be pickled (e.g. for + `dask.distributed`): they open their file in each process on the first chunk read. A cell table for several + regions converts µm coordinates with each region's own pixel size. ### Changed @@ -49,6 +51,10 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. including multi-region reads of unrelated regions, gets its own coordinate system named after the region. - New parameters `channel_names`, `cells_table`, `cells_table_format`, `labels`, `image_models_kwargs`, `labels_models_kwargs` and `cells_table_kwargs`; the `codex` CLI command exposes them. + - `channelNames.txt` and `experiment.json` are found in any case (e.g. SPACEc's `channelnames.txt`); CSV files + that are not CODEX Processor cell tables are ignored, also when their name holds a region id. + - A CIM `.qptiff` without `regNNN` in its name is named as `phenocycler()` names it (without the suffix and a + trailing `_Scan`), and its `.er.qptiff` is preferred. ### Fixed From 2bb33a07506151bb3b37f21879d9b4cf013f17c8 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 19:42:29 +0200 Subject: [PATCH 030/107] fix: keep codex positional signature backward compatible codex(path, fcs, imread_kwargs) is restored; all newer parameters are keyword-only, as is everything after path in phenocycler(). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++++---- src/spatialdata_io/readers/_utils/_akoya.py | 8 ++------ src/spatialdata_io/readers/codex.py | 9 +++++---- src/spatialdata_io/readers/phenocycler.py | 1 + tests/readers/test_akoya_utils.py | 6 +++++- tests/test_codex.py | 17 ++++++++++++++--- tests/test_phenocycler.py | 6 ++++++ 7 files changed, 37 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61e7155f..f8df51df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,10 +42,10 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. - Every region is read (previously only the first file found); `regions=[...]` selects a subset. - Marker names in `table.var_names` come from the FCS `$PnS` keywords or the CSV header `cycCCC_chHHH:` (previously `cycCCC_chHHH`); the image channels are named too. - - The positional third parameter is now `regions` (previously `imread_kwargs`); pass `imread_kwargs` by keyword. - Code that still passes `imread_kwargs` positionally gets a `TypeError` (a mapping is not a valid `regions`) - rather than a silently different read; an empty `regions` raises a `ValueError`. `imread_kwargs` is only used by - the `dask_image.imread` fallback for TIFFs that `tifffile` cannot read lazily. + - The positional signature `codex(path, fcs, imread_kwargs)` is unchanged; all new parameters (`regions`, + `channel_names`, `cells_table`, `cells_table_format`, `labels` and the `*_models_kwargs` / `cells_table_kwargs`) + are keyword-only. An empty `regions` raises a `ValueError`. `imread_kwargs` is only used by the + `dask_image.imread` fallback for TIFFs that `tifffile` cannot read lazily. - A single region is stored in `"global"`. Several regions are placed at their stage positions in `"global"` only when all of them share a sample space (same slide id, stage positions and pixel size); otherwise each region, including multi-region reads of unrelated regions, gets its own coordinate system named after the region. diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index dc7671c1..d9885a06 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -176,18 +176,14 @@ def scans_to_attrs(scans: Sequence[AkoyaScan], channel_names: Mapping[str, list[ def normalize_regions(regions: str | Sequence[str] | None) -> list[str] | None: """Validate the ``regions`` argument of a reader: a region id or a sequence of them, without duplicates. - Raises a `TypeError` for a mapping, e.g. ``imread_kwargs`` passed positionally to ``codex()``, whose third - parameter used to be ``imread_kwargs``, and a `ValueError` for an empty sequence. + Raises a `TypeError` for a mapping and a `ValueError` for an empty sequence. """ if regions is None: return None if isinstance(regions, str): return [regions] if isinstance(regions, Mapping): - raise TypeError( - f"`regions` must be a sequence of region ids, got {type(regions).__name__}; `imread_kwargs` is no " - "longer the third positional argument of `codex()` — pass it by keyword." - ) + raise TypeError(f"`regions` must be a sequence of region ids, got {type(regions).__name__}.") if not regions: raise ValueError("`regions` is empty; pass `None` to read all regions.") return list(dict.fromkeys(regions)) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 54cb5fda..5bdb249b 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -85,12 +85,13 @@ class _LegacyLayout: def codex( path: str | Path, fcs: bool = True, + imread_kwargs: Mapping[str, Any] = MappingProxyType({}), + *, regions: Sequence[str] | None = None, channel_names: Sequence[str] | None = None, cells_table: str | Path | Mapping[str, str | Path] | None = None, cells_table_format: str = "auto", labels: str | Path | Mapping[str, str | Path] | None = None, - imread_kwargs: Mapping[str, Any] = MappingProxyType({}), image_models_kwargs: Mapping[str, Any] = MappingProxyType({}), labels_models_kwargs: Mapping[str, Any] = MappingProxyType({}), cells_table_kwargs: Mapping[str, Any] = MappingProxyType({}), @@ -156,6 +157,9 @@ def codex( fcs Whether to prefer ``{vx.FCS_FILE!r}`` cell tables over ``{vx.FCS_FILE_CSV!r}`` ones when both exist. With `False`, a region that only has a ``{vx.FCS_FILE!r}`` table is read from it, with a warning. + imread_kwargs + Keyword arguments passed to :func:`dask_image.imread.imread`, which is only used for ``{vx.IMAGE_TIF}(f)`` + files that ``tifffile`` cannot read lazily (e.g. unsupported compression). regions Regions to read, e.g. ``["reg001"]``, including only their cells. If `None`, all regions are read. channel_names @@ -175,9 +179,6 @@ def codex( mapping from region id to path. Each mask is stored lazily as the labels ``_labels`` with the transformation of the region's image; the cells of the region annotate it instead of circles, and their instance ids must be the mask values. - imread_kwargs - Keyword arguments passed to :func:`dask_image.imread.imread`, which is only used for ``{vx.IMAGE_TIF}(f)`` - files that ``tifffile`` cannot read lazily (e.g. unsupported compression). image_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. labels_models_kwargs diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 1f8c5b66..e8b1e1a9 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -43,6 +43,7 @@ @inject_docs(vx=PhenocyclerKeys) def phenocycler( path: str | Path, + *, regions: Sequence[str] | None = None, channel_names: Sequence[str] | None = None, cells_table: str | Path | Mapping[str, str | Path] | None = None, diff --git a/tests/readers/test_akoya_utils.py b/tests/readers/test_akoya_utils.py index 5736a60d..52943635 100644 --- a/tests/readers/test_akoya_utils.py +++ b/tests/readers/test_akoya_utils.py @@ -136,7 +136,11 @@ def test_select_regions_deduplicates(): @pytest.mark.parametrize( ("regions", "error", "match"), - [({}, TypeError, "pass it by keyword"), ({"a": 1}, TypeError, "sequence of region ids"), ([], ValueError, "empty")], + [ + ({}, TypeError, "sequence of region ids"), + ({"a": 1}, TypeError, "sequence of region ids"), + ([], ValueError, "empty"), + ], ) def test_select_regions_invalid_raises(regions: object, error: type[Exception], match: str): with pytest.raises(error, match=match): diff --git a/tests/test_codex.py b/tests/test_codex.py index 57a1094b..fe7d8312 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -289,9 +289,8 @@ def test_codex_str_regions_and_channel_names(tmp_path: Path) -> None: def test_codex_regions_validation(tmp_path: Path) -> None: write_legacy_processor_tree(tmp_path, regions=2, cycles=1, channels=1, layout="per_marker", fcs=False, csv=False) - # `imread_kwargs` used to be the third positional parameter - with pytest.raises(TypeError, match="imread_kwargs.*pass it by keyword"): - codex(tmp_path, True, {}) # type: ignore[arg-type] + with pytest.raises(TypeError, match="sequence of region ids"): + codex(tmp_path, regions={}) # type: ignore[arg-type] with pytest.raises(ValueError, match="empty"): codex(tmp_path, regions=[]) @@ -551,3 +550,15 @@ def test_codex_write_roundtrip(tmp_path: Path) -> None: assert list(table.var_names) == list(sdata.tables["table"].var_names) assert read.attrs["akoya"] == sdata.attrs["akoya"] assert read.attrs["spatialdata_io_reader"] == "codex" + + +def test_codex_old_positional_call(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, regions=2, cycles=1, channels=1, layout="per_marker", fcs=False, csv=False) + + # `codex(path, fcs, imread_kwargs)` is the signature of earlier releases + for fcs in (True, False): + sdata = codex(tmp_path, fcs, {}) + assert sorted(sdata.images) == ["reg001_image", "reg002_image"] + + with pytest.warns(UserWarning, match="imread_kwargs"): + codex(tmp_path, True, {"nframes": 1}) diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 3d4bf7e3..bd9f77d7 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -662,3 +662,9 @@ def fail(*args: object, **kwargs: object) -> None: sdata = phenocycler(path, image_models_kwargs=_FAST) assert isinstance(_level0(sdata, "S_image").data, da.Array) + + +def test_phenocycler_keyword_only(tmp_path: Path) -> None: + write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI"]) + with pytest.raises(TypeError): + phenocycler(tmp_path, ["A"]) # type: ignore[misc] From 7fe106aa5842b0b695f86caefe8e03e0b56060ec Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 22:50:27 +0200 Subject: [PATCH 031/107] feat: auto-scale integer-downsampled Akoya masks A mask whose shape is the region image's divided by an integer factor f (exactly or rounded up) gets Scale([f, f]) before the region's transformation, so it aligns with the image; mask-pixel centroids and radii of mcquant, SPRM and generic px tables are scaled by f. Other shape mismatches still warn. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 + .../readers/_utils/_cell_tables.py | 97 ++++++++++++++++--- src/spatialdata_io/readers/codex.py | 6 +- src/spatialdata_io/readers/phenocycler.py | 5 +- tests/readers/test_cell_tables.py | 63 +++++++++++- tests/test_phenocycler.py | 4 +- 6 files changed, 158 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8df51df..892d3792 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,9 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. inForm, CODEX Processor or generic CSV) with segmentation masks. The lazy images and masks can be pickled (e.g. for `dask.distributed`): they open their file in each process on the first chunk read. A cell table for several regions converts µm coordinates with each region's own pixel size. +- The Akoya readers (`phenocycler()`, `codex()`) scale a segmentation mask downsampled from its region image by an + integer factor onto the image (a `Scale` before the region's transformation), and scale the mask-pixel centroids + and areas of `mcquant`, SPRM and generic pixel tables by the same factor; other shape mismatches still warn. ### Changed diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index ae0f8b0f..6a669d83 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -6,7 +6,7 @@ import re import warnings from collections.abc import Mapping -from dataclasses import dataclass +from dataclasses import dataclass, replace from functools import partial from pathlib import Path from typing import TYPE_CHECKING, Any @@ -21,7 +21,8 @@ from spatialdata import SpatialData, sanitize_name, sanitize_table from spatialdata._logging import logger from spatialdata.models import Labels2DModel, ShapesModel, TableModel -from spatialdata.transformations import Identity +from spatialdata.transformations import Identity, Scale, set_transformation +from spatialdata.transformations import Sequence as TransformationSequence from xarray import DataTree from spatialdata_io._constants._constants import CodexKeys, PhenocyclerKeys @@ -56,6 +57,7 @@ "raster_shape", "read_cell_table", "read_codex_table", + "mask_downsample_factor", "read_labels", "read_qupath_polygons", "read_region_labels", @@ -127,6 +129,10 @@ class CellTable: ``*-cell_centers.csv``). radius_px The radius of a circle with the cell's area, or `None` if the table has no area. + mask_pixels + Whether ``centroids_px`` and ``radius_px`` are in pixels of the segmentation mask (pixel coordinates of + mask-based pipelines: mcquant, SPRM, ``"generic"`` in ``"px"``) rather than of the image; they are scaled + to image pixels when the mask is a downsample of the image, see :func:`build_spatialdata`. row_instance_ids Whether the instance ids are row numbers because the table has no instance id column. polygons_px @@ -146,6 +152,7 @@ class CellTable: row_instance_ids: bool = False polygons_px: np.ndarray | None = None geojson: Path | None = None + mask_pixels: bool = False def take(self, rows: np.ndarray) -> CellTable: """The cells at the positional indices ``rows``.""" @@ -160,6 +167,7 @@ def take(self, rows: np.ndarray) -> CellTable: row_instance_ids=self.row_instance_ids, polygons_px=None if self.polygons_px is None else self.polygons_px[rows], geojson=self.geojson, + mask_pixels=self.mask_pixels, ) @@ -381,6 +389,7 @@ def read_cell_table( "generic": _generic_roles, }[fmt] roles = reader(path, df, cells_table_kwargs) + mask_pixels = fmt in {"mcquant", "sprm"} or (fmt == "generic" and not roles["in_um"]) units = cells_table_kwargs.get("geojson_units", "px") row_pixel_sizes = None if roles["in_um"] or (fmt == "qupath" and units == "um"): @@ -393,6 +402,7 @@ def read_cell_table( ) roles["px_per_unit"] = 1 / row_pixel_sizes table = _from_frame(path, df, default_region=default_region, **roles) + table.mask_pixels = mask_pixels if fmt == "sprm": table.centroids_px = _read_sprm_centers(path, table) if fmt == "qupath": @@ -1028,34 +1038,88 @@ def read_region_labels( labels_models_kwargs: Mapping[str, Any], cells_table_kwargs: Mapping[str, Any], image_shapes: Mapping[str, tuple[int, int]], -) -> dict[str, DataArray | DataTree]: +) -> tuple[dict[str, DataArray | DataTree], dict[str, int]]: """Read the mask of each region, as resolved by :func:`resolve_per_region`, with the region's transformations. - A mask whose ``(y, x)`` shape differs from ``image_shapes`` of its region raises a warning: its pixels would - not align with the image. + A mask whose ``(y, x)`` shape differs from ``image_shapes`` of its region is scaled onto the image if it is + downsampled by an integer factor, see :func:`mask_downsample_factor`: its transformations become + ``Sequence([Scale([f, f]), t])`` for each region transformation ``t``. Any other mismatch raises a warning: its + pixels would not align with the image. + + Returns + ------- + The labels elements by region id, and the downsampling factor of the scaled masks by region id. """ if "*" in files: raise ValueError( f"A single labels mask {files['*']} was given for several regions; a mask belongs to one region image, " "so pass a mapping {region: path}." ) - labels = {} + labels, factors = {}, {} for region, path in files.items(): + region_transformations = _transformations_of(transformations, region) labels[region] = read_labels( - path, - labels_models_kwargs, - _transformations_of(transformations, region), - mask_channel=cells_table_kwargs.get("mask_channel"), + path, labels_models_kwargs, region_transformations, mask_channel=cells_table_kwargs.get("mask_channel") ) shape, image_shape = raster_shape(labels[region]), image_shapes.get(region) - if image_shape is not None and shape != tuple(image_shape): + if image_shape is None or shape == tuple(image_shape): + continue + factor = mask_downsample_factor(shape, image_shape) + if factor is None: warnings.warn( f"The mask {path} has shape {shape}, but the image of region '{region}' has shape " f"{tuple(image_shape)} (y, x); the labels will not align with the image.", UserWarning, stacklevel=3, ) - return labels + continue + logger.info( + "The mask %s has shape %s, the image of region '%s' %s (y, x); the labels are scaled by %d to align.", + path, + shape, + region, + tuple(image_shape), + factor, + ) + scale = Scale([factor, factor], axes=("y", "x")) + set_transformation( + labels[region], + { + cs: TransformationSequence([scale, transformation]) + for cs, transformation in region_transformations.items() + }, + set_all=True, + ) + factors[region] = factor + return labels, factors + + +def mask_downsample_factor(mask_shape: tuple[int, int], image_shape: tuple[int, int]) -> int | None: + """The integer factor ``f > 1`` by which a mask of ``(y, x)`` shape ``mask_shape`` downsamples an image. + + The mask must be ``image_shape / f`` along both axes, exactly or rounded up (each image size within ``f - 1`` + pixels of ``f`` times the mask size); `None` if there is no such factor. + """ + candidates = {-(-image // mask) for image, mask in zip(image_shape, mask_shape, strict=True) if mask > 0} + for factor in sorted(candidates): + if factor > 1 and all(-(-image // factor) == mask for image, mask in zip(image_shape, mask_shape, strict=True)): + return factor + return None + + +def _to_image_pixels(tables: Sequence[CellTable], factors: Mapping[str, int]) -> list[CellTable]: + """``tables`` with mask-pixel centroids and radii scaled to image pixels by their region's mask factor.""" + scaled = [] + for table in tables: + if table.mask_pixels and factors: + row_factors = table.region.map(factors).fillna(1).to_numpy(dtype=np.float64) + table = replace( + table, + centroids_px=None if table.centroids_px is None else table.centroids_px * row_factors[:, None], + radius_px=None if table.radius_px is None else table.radius_px * row_factors, + ) + scaled.append(table) + return scaled def polygon_labels( @@ -1269,7 +1333,9 @@ def build_spatialdata( Each region gets the image ``_image``, the labels ``_labels`` from its mask or its cell polygons (see :func:`polygon_labels`), the polygons ``_polygons`` and the cells (see :func:`build_cells`), all in the coordinate systems of :func:`region_transformations`; regions with cells or a mask but no image are - unrelated to the scans. + unrelated to the scans. A mask downsampled from its region image by an integer factor is scaled onto the image + (see :func:`read_region_labels`), and the centroids and radii in mask pixels of its cells (``mask_pixels``, see + :class:`CellTable`) are scaled by the same factor, so that the table and circles are in image pixels. Parameters ---------- @@ -1307,7 +1373,10 @@ def build_spatialdata( image_shapes = { scan.region: raster_shape(image_elements[f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}"]) for scan in scans } - labels = read_region_labels(mask_files, transformations, labels_models_kwargs, cells_table_kwargs, image_shapes) + labels, factors = read_region_labels( + mask_files, transformations, labels_models_kwargs, cells_table_kwargs, image_shapes + ) + cell_tables = _to_image_pixels(cell_tables, factors) polygons, geojson_files = polygon_labels(cell_tables, labels, image_shapes, transformations, labels_models_kwargs) labels |= polygons tables, shapes = build_cells(cell_tables, {}, labels, transformations) if cell_tables else ({}, {}) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 5bdb249b..3f92200e 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -178,7 +178,11 @@ def codex( A segmentation mask (TIFF/OME-TIFF with integer instance ids) per region: a path for a single region, or a mapping from region id to path. Each mask is stored lazily as the labels ``_labels`` with the transformation of the region's image; the cells of the region annotate it instead of circles, and - their instance ids must be the mask values. + their instance ids must be the mask values. A mask whose shape is the region image's divided by an integer + factor ``f`` (rounded up) is scaled onto the image (``Scale([f, f])`` before the region's transformation), + and the pixel centroids and areas of ``"mcquant"``, ``"sprm"`` and ``"generic"`` (``"px"``) tables, which + are taken to be in mask pixels, are scaled by ``f`` to image pixels; any other shape mismatch raises a + warning. image_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. labels_models_kwargs diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index e8b1e1a9..0a6048e1 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -121,7 +121,10 @@ def phenocycler( ``*_cell_seg_data.txt``) or ``"generic"`` (columns given by ``cells_table_kwargs``). labels A segmentation mask (TIFF/OME-TIFF with integer instance ids) per region: a path for a single region, or a - mapping from region id to path. + mapping from region id to path. A mask whose shape is the region image's divided by an integer factor ``f`` + (rounded up) is scaled onto the image (``Scale([f, f])`` before the region's transformation), and the pixel + centroids and areas of ``"mcquant"``, ``"sprm"`` and ``"generic"`` (``"px"``) tables, which are taken to be + in mask pixels, are scaled by ``f`` to image pixels; any other shape mismatch raises a warning. image_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. labels_models_kwargs diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index a3cc48ef..cef56e08 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -1,15 +1,18 @@ from __future__ import annotations +import warnings from typing import TYPE_CHECKING import numpy as np import pandas as pd import pytest import tifffile +from spatialdata import get_extent, read_zarr from spatialdata.models import Labels2DModel, get_axes_names -from spatialdata.transformations import Identity, Translation, get_transformation -from tests._akoya_data import write_fcs +from spatialdata.transformations import Identity, Scale, Sequence, Translation, get_transformation +from tests._akoya_data import write_fcs, write_qptiff +from spatialdata_io import phenocycler from spatialdata_io.readers._utils._akoya import dedupe_channel_names from spatialdata_io.readers._utils._cell_tables import ( build_cells, @@ -536,3 +539,59 @@ def fail(*args: object, **kwargs: object) -> None: assert labels.shape == (64, 48) np.testing.assert_array_equal(labels.values, mask) + + +@pytest.mark.parametrize(("mask_shape", "factor"), [((32, 40), 2), ((22, 27), 3)]) +def test_mask_integer_downsample_autoscaled(tmp_path: Path, mask_shape: tuple[int, int], factor: int) -> None: + # a mask downsampled by an integer factor (exactly, or with its shape rounded up) is scaled onto the image + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"], shape=(64, 80), stage_position_cm=(1.0, 2.0)) + mask = np.zeros(mask_shape, dtype=np.uint32) + mask[2:6, 2:6], mask[10:14, 8:12] = 1, 2 + tifffile.imwrite(tmp_path / "mask.tif", mask) + cells = _write_mcquant(tmp_path / "cells.csv", n=2) + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + sdata = phenocycler( + tmp_path, + cells_table=tmp_path / "cells.csv", + labels=tmp_path / "mask.tif", + image_models_kwargs={"scale_factors": [2]}, + ) + + image_transformation = get_transformation(sdata.images["S_image"], "global") + labels_transformation = get_transformation(sdata.labels["S_labels"], "global") + assert isinstance(labels_transformation, Sequence) + assert labels_transformation.transformations == [ + Scale([factor, factor], axes=("y", "x")), + image_transformation, + ] + image_extent = get_extent(sdata.images["S_image"], coordinate_system="global") + labels_extent = get_extent(sdata.labels["S_labels"], coordinate_system="global") + for axis in ("x", "y"): + assert labels_extent[axis][0] == pytest.approx(image_extent[axis][0]) + assert labels_extent[axis][1] == pytest.approx(image_extent[axis][1], abs=factor - 1) + # mcquant centroids of the mask are in mask pixels, the table's are in image pixels + np.testing.assert_allclose( + sdata.tables["table"].obsm["spatial"], cells[["X_centroid", "Y_centroid"]].to_numpy() * factor + ) + + sdata.write(tmp_path / "x.zarr") + assert get_transformation(read_zarr(tmp_path / "x.zarr").labels["S_labels"], "global") == labels_transformation + + +def test_mask_non_integer_mismatch_warns(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"], shape=(64, 80)) + tifffile.imwrite(tmp_path / "mask.tif", np.zeros((48, 60), dtype=np.uint32)) + cells = _write_mcquant(tmp_path / "cells.csv", n=2) + + with pytest.warns(UserWarning, match=r"\(48, 60\).*\(64, 80\).*not align"): + sdata = phenocycler( + tmp_path, + cells_table=tmp_path / "cells.csv", + labels=tmp_path / "mask.tif", + image_models_kwargs={"scale_factors": [2]}, + ) + + assert get_transformation(sdata.labels["S_labels"], "global") == Identity() + np.testing.assert_allclose(sdata.tables["table"].obsm["spatial"], cells[["X_centroid", "Y_centroid"]].to_numpy()) diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index bd9f77d7..4a2318b9 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -401,9 +401,9 @@ def test_mapping_keys_of_unselected_regions_are_ignored(tmp_path: Path) -> None: def test_mask_shape_mismatch_warns(tmp_path: Path) -> None: write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - _write_mask(tmp_path / "mask.tif", shape=(128, 160)) + _write_mask(tmp_path / "mask.tif", shape=(200, 300)) - with pytest.warns(UserWarning, match=r"\(128, 160\).*\(256, 320\)"): + with pytest.warns(UserWarning, match=r"\(200, 300\).*\(256, 320\)"): sdata = phenocycler(tmp_path, labels=tmp_path / "mask.tif", image_models_kwargs=_FAST) assert list(sdata.labels) == ["S_labels"] From 67261299ee0af7e5447d18af0e88a1392fb0dd78 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 22:58:45 +0200 Subject: [PATCH 032/107] fix: document and test the Akoya mask scaling precisely Docs say centroids and circle radii (not areas) are scaled, obs keeps source columns, and how to opt out for generic px tables. Tests cover the composition with a stage translation, exact extents, and an inForm table whose image-pixel centroids stay unscaled. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +- .../readers/_utils/_cell_tables.py | 2 +- src/spatialdata_io/readers/codex.py | 10 ++- src/spatialdata_io/readers/phenocycler.py | 9 ++- tests/readers/test_cell_tables.py | 73 ++++++++++++++----- 5 files changed, 69 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 892d3792..0af9ee50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,8 +30,9 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. `dask.distributed`): they open their file in each process on the first chunk read. A cell table for several regions converts µm coordinates with each region's own pixel size. - The Akoya readers (`phenocycler()`, `codex()`) scale a segmentation mask downsampled from its region image by an - integer factor onto the image (a `Scale` before the region's transformation), and scale the mask-pixel centroids - and areas of `mcquant`, SPRM and generic pixel tables by the same factor; other shape mismatches still warn. + integer factor (exactly, or rounded up) onto the image (a `Scale` before the region's transformation), and scale + the centroids (`obsm["spatial"]`) and circle radii of `mcquant`, SPRM and generic pixel tables, taken to be in + mask pixels, by the same factor; the source columns in `obs` are kept as read. Other shape mismatches still warn. ### Changed diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 6a669d83..54e7f7c4 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -52,12 +52,12 @@ "build_cells", "build_spatialdata", "detect_table_format", + "mask_downsample_factor", "polygon_labels", "polygon_shapes", "raster_shape", "read_cell_table", "read_codex_table", - "mask_downsample_factor", "read_labels", "read_qupath_polygons", "read_region_labels", diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 3f92200e..23b7f5f0 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -179,10 +179,12 @@ def codex( mapping from region id to path. Each mask is stored lazily as the labels ``_labels`` with the transformation of the region's image; the cells of the region annotate it instead of circles, and their instance ids must be the mask values. A mask whose shape is the region image's divided by an integer - factor ``f`` (rounded up) is scaled onto the image (``Scale([f, f])`` before the region's transformation), - and the pixel centroids and areas of ``"mcquant"``, ``"sprm"`` and ``"generic"`` (``"px"``) tables, which - are taken to be in mask pixels, are scaled by ``f`` to image pixels; any other shape mismatch raises a - warning. + factor ``f`` (exactly, or rounded up) is scaled onto the image (``Scale([f, f])`` before the region's + transformation); any other shape mismatch raises a warning. The pixel centroids of ``"mcquant"``, + ``"sprm"`` and ``"generic"`` (``"px"``) tables are taken to be in mask pixels, so their centroids + (``obsm["spatial"]``) and circle radii are scaled by ``f`` to image pixels; the source columns in ``obs`` + are kept as read (mask pixels). To avoid this for a generic table in image pixels, use + ``coordinates_unit="um"`` or a full-resolution mask. image_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. labels_models_kwargs diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 0a6048e1..09f3d526 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -122,9 +122,12 @@ def phenocycler( labels A segmentation mask (TIFF/OME-TIFF with integer instance ids) per region: a path for a single region, or a mapping from region id to path. A mask whose shape is the region image's divided by an integer factor ``f`` - (rounded up) is scaled onto the image (``Scale([f, f])`` before the region's transformation), and the pixel - centroids and areas of ``"mcquant"``, ``"sprm"`` and ``"generic"`` (``"px"``) tables, which are taken to be - in mask pixels, are scaled by ``f`` to image pixels; any other shape mismatch raises a warning. + (exactly, or rounded up) is scaled onto the image (``Scale([f, f])`` before the region's transformation); + any other shape mismatch raises a warning. The pixel centroids of ``"mcquant"``, ``"sprm"`` and + ``"generic"`` (``"px"``) tables are taken to be in mask pixels, so their centroids (``obsm["spatial"]``) and + circle radii are scaled by ``f`` to image pixels; the source columns in ``obs`` are kept as read (mask + pixels). To avoid this for a generic table in image pixels, use ``coordinates_unit="um"`` or a + full-resolution mask. image_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. labels_models_kwargs diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index cef56e08..5689df72 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -543,41 +543,74 @@ def fail(*args: object, **kwargs: object) -> None: @pytest.mark.parametrize(("mask_shape", "factor"), [((32, 40), 2), ((22, 27), 3)]) def test_mask_integer_downsample_autoscaled(tmp_path: Path, mask_shape: tuple[int, int], factor: int) -> None: - # a mask downsampled by an integer factor (exactly, or with its shape rounded up) is scaled onto the image - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"], shape=(64, 80), stage_position_cm=(1.0, 2.0)) + # a mask downsampled by an integer factor (exactly, or with its shape rounded up) is scaled onto the image, + # before the stage translation of its region; the full-resolution mask of the other region is unaffected + image_shape = (64, 80) + write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI"], shape=image_shape, stage_position_cm=(1.0, 2.0)) + write_qptiff(tmp_path / "B_Scan1.qptiff", biomarkers=["DAPI"], shape=image_shape, stage_position_cm=(1.01, 2.0)) + tifffile.imwrite(tmp_path / "A_mask.tif", np.zeros(image_shape, dtype=np.uint32)) mask = np.zeros(mask_shape, dtype=np.uint32) mask[2:6, 2:6], mask[10:14, 8:12] = 1, 2 - tifffile.imwrite(tmp_path / "mask.tif", mask) - cells = _write_mcquant(tmp_path / "cells.csv", n=2) + tifffile.imwrite(tmp_path / "B_mask.tif", mask) + cells = _write_mcquant(tmp_path / "B.csv", n=2) with warnings.catch_warnings(): warnings.simplefilter("error", UserWarning) sdata = phenocycler( tmp_path, - cells_table=tmp_path / "cells.csv", - labels=tmp_path / "mask.tif", + cells_table={"B": tmp_path / "B.csv"}, + labels={"A": tmp_path / "A_mask.tif", "B": tmp_path / "B_mask.tif"}, image_models_kwargs={"scale_factors": [2]}, ) - image_transformation = get_transformation(sdata.images["S_image"], "global") - labels_transformation = get_transformation(sdata.labels["S_labels"], "global") + assert sdata.coordinate_systems == ["global"] + assert get_transformation(sdata.labels["A_labels"], get_all=True) == get_transformation( + sdata.images["A_image"], get_all=True + ) + image_transformation = get_transformation(sdata.images["B_image"], "global") + assert isinstance(image_transformation, Translation) + labels_transformation = get_transformation(sdata.labels["B_labels"], "global") assert isinstance(labels_transformation, Sequence) - assert labels_transformation.transformations == [ - Scale([factor, factor], axes=("y", "x")), - image_transformation, - ] - image_extent = get_extent(sdata.images["S_image"], coordinate_system="global") - labels_extent = get_extent(sdata.labels["S_labels"], coordinate_system="global") - for axis in ("x", "y"): + assert labels_transformation.transformations == [Scale([factor, factor], axes=("y", "x")), image_transformation] + image_extent = get_extent(sdata.images["B_image"], coordinate_system="global") + labels_extent = get_extent(sdata.labels["B_labels"], coordinate_system="global") + for axis, image_size, mask_size in zip(("y", "x"), image_shape, mask_shape, strict=True): assert labels_extent[axis][0] == pytest.approx(image_extent[axis][0]) - assert labels_extent[axis][1] == pytest.approx(image_extent[axis][1], abs=factor - 1) - # mcquant centroids of the mask are in mask pixels, the table's are in image pixels + # a mask rounded up overhangs the image by less than one mask pixel + expected_end = image_extent[axis][0] + mask_size * factor + assert labels_extent[axis][1] == pytest.approx(expected_end) + if image_size % mask_size == 0: + assert labels_extent[axis][1] == pytest.approx(image_extent[axis][1]) + # mcquant centroids of the mask are in mask pixels, the table's are in image pixels; obs keeps them as read + table = sdata.tables["table"] + np.testing.assert_allclose(table.obsm["spatial"], cells[["X_centroid", "Y_centroid"]].to_numpy() * factor) + np.testing.assert_allclose(table.obs["X_centroid"], cells["X_centroid"]) + + # writing may reorder the axes of the translation, so compare the composed affine maps + sdata.write(tmp_path / "x.zarr") + read = get_transformation(read_zarr(tmp_path / "x.zarr").labels["B_labels"], "global") + assert isinstance(read, Sequence) np.testing.assert_allclose( - sdata.tables["table"].obsm["spatial"], cells[["X_centroid", "Y_centroid"]].to_numpy() * factor + read.to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")), + labels_transformation.to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")), ) - sdata.write(tmp_path / "x.zarr") - assert get_transformation(read_zarr(tmp_path / "x.zarr").labels["S_labels"], "global") == labels_transformation + +def test_mask_downsample_keeps_image_pixel_centroids(tmp_path: Path) -> None: + # inForm centroids are in image pixels, so a downsampled mask does not scale them + write_qptiff(tmp_path / "Scan1.qptiff", biomarkers=["DAPI"], shape=(64, 80)) + tifffile.imwrite(tmp_path / "mask.tif", np.zeros((32, 40), dtype=np.uint32)) + _write_inform(tmp_path / "Scan1_cell_seg_data.txt", "pixels", area_unit="pixels") + + sdata = phenocycler( + tmp_path, + cells_table=tmp_path / "Scan1_cell_seg_data.txt", + labels=tmp_path / "mask.tif", + image_models_kwargs={"scale_factors": [2]}, + ) + + assert isinstance(get_transformation(sdata.labels[next(iter(sdata.labels))], "global"), Sequence) + np.testing.assert_allclose(sdata.tables["table"].obsm["spatial"], [[100, 50], [200, 80]]) def test_mask_non_integer_mismatch_warns(tmp_path: Path) -> None: From c6863a1ac8962852d006bb9301276492a682a447 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 23:02:46 +0200 Subject: [PATCH 033/107] feat: optionally read both 8-bit and extended-range qptiffs Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 ++ src/spatialdata_io/__main__.py | 8 ++++ .../readers/_utils/_cell_tables.py | 15 ++++++- src/spatialdata_io/readers/_utils/_qptiff.py | 30 ++++++++++++- src/spatialdata_io/readers/phenocycler.py | 39 +++++++++++++---- tests/test_phenocycler.py | 43 +++++++++++++++++++ 6 files changed, 126 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0af9ee50..fee5fbe7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,9 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. inForm, CODEX Processor or generic CSV) with segmentation masks. The lazy images and masks can be pickled (e.g. for `dask.distributed`): they open their file in each process on the first chunk read. A cell table for several regions converts µm coordinates with each region's own pixel size. +- `phenocycler(read_all_bit_depths=True)` (CLI `--read-all-bit-depths`) reads both the 8-bit `.qptiff` and the + extended-range `.er.qptiff` of a scan: the 8-bit one becomes the image `_8bit_image` with the transformations + of the `.er` region. By default only the extended-range file is read. - The Akoya readers (`phenocycler()`, `codex()`) scale a segmentation mask downsampled from its region image by an integer factor (exactly, or rounded up) onto the image (a `Scale` before the region's transformation), and scale the centroids (`obsm["spatial"]`) and circle radii of `mcquant`, SPRM and generic pixel tables, taken to be in diff --git a/src/spatialdata_io/__main__.py b/src/spatialdata_io/__main__.py index 43e3b44b..c9fa6dbb 100644 --- a/src/spatialdata_io/__main__.py +++ b/src/spatialdata_io/__main__.py @@ -469,6 +469,12 @@ def _parse_list_param(value: str | None) -> list[str] | None: default=None, help="Path to a segmentation mask, or a JSON mapping from region to path. [default: None]", ) +@click.option( + "--read-all-bit-depths/--no-read-all-bit-depths", + default=False, + help="Read both the 8-bit .qptiff and the extended-range .er.qptiff of a scan (the 8-bit one as region " + "'_8bit'). [default: no-read-all-bit-depths]", +) @click.option( "--image-models-kwargs", type=str, @@ -495,6 +501,7 @@ def phenocycler_wrapper( cells_table: str | None = None, cells_table_format: str = "auto", labels: str | None = None, + read_all_bit_depths: bool = False, image_models_kwargs: str = "{}", labels_models_kwargs: str = "{}", cells_table_kwargs: str = "{}", @@ -509,6 +516,7 @@ def phenocycler_wrapper( cells_table=_parse_path_or_mapping_param(cells_table, "cells_table"), cells_table_format=cells_table_format, labels=_parse_path_or_mapping_param(labels, "labels"), + read_all_bit_depths=read_all_bit_depths, image_models_kwargs=_parse_json_param(image_models_kwargs, "image_models_kwargs"), labels_models_kwargs=_parse_json_param(labels_models_kwargs, "labels_models_kwargs"), cells_table_kwargs=_parse_json_param(cells_table_kwargs, "cells_table_kwargs"), diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 54e7f7c4..a10ed67b 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -1327,6 +1327,7 @@ def build_spatialdata( image_models_kwargs: Mapping[str, Any], labels_models_kwargs: Mapping[str, Any], cells_table_kwargs: Mapping[str, Any], + duplicate_scans: Mapping[str, AkoyaScan] | None = None, ) -> SpatialData: """Assemble the regions of an Akoya read into a `SpatialData` with their metadata in ``attrs["akoya"]``. @@ -1357,6 +1358,10 @@ def build_spatialdata( Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`. cells_table_kwargs See :func:`read_labels` (``mask_channel``). + duplicate_scans + Further scans of the same physical scan as a region (e.g. its 8-bit file), by that region's id. Each is + stored as its own image ``_image`` with exactly the transformations of the region; their + images and channel names are looked up in ``images`` and ``channel_names`` by ``duplicate.region``. Returns ------- @@ -1370,6 +1375,14 @@ def build_spatialdata( ) for scan in scans } + for region, duplicate in (duplicate_scans or {}).items(): + image_elements[f"{duplicate.region}{PhenocyclerKeys.IMAGE_SUFFIX}"] = build_image_element( + images[duplicate.region], + duplicate, + channel_names[duplicate.region], + transformations[region], + image_models_kwargs, + ) image_shapes = { scan.region: raster_shape(image_elements[f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}"]) for scan in scans } @@ -1388,7 +1401,7 @@ def build_spatialdata( shapes=shapes, tables=tables, ) - attrs = scans_to_attrs(scans, channel_names) + attrs = scans_to_attrs([*scans, *(duplicate_scans or {}).values()], channel_names) if table_paths is not None: attrs["cell_tables"] = dict(table_paths) if mask_files: diff --git a/src/spatialdata_io/readers/_utils/_qptiff.py b/src/spatialdata_io/readers/_utils/_qptiff.py index fcd7a302..e6acf2dc 100644 --- a/src/spatialdata_io/readers/_utils/_qptiff.py +++ b/src/spatialdata_io/readers/_utils/_qptiff.py @@ -26,6 +26,7 @@ "qpi_pixel_size_um", "qptiff_base_name", "qptiff_regions", + "qptiff_regions_with_duplicates", "read_qptiff_image", "read_qptiff_metadata", ] @@ -246,6 +247,10 @@ def qptiff_base_name(path: Path) -> str: return path.stem +def _is_extended_range(path: Path) -> bool: + return path.name.lower().endswith(PhenocyclerKeys.ER_QPTIFF_SUFFIX.value) + + def qptiff_regions(files: Sequence[Path]) -> dict[str, Path]: """The region id of every qptiff scan among ``files``, the same for both Akoya readers. @@ -261,11 +266,31 @@ def qptiff_regions(files: Sequence[Path]) -> dict[str, Path]: ------- The scan file of every region. """ + return qptiff_regions_with_duplicates(files)[0] + + +def qptiff_regions_with_duplicates(files: Sequence[Path]) -> tuple[dict[str, Path], dict[str, Path]]: + """Like :func:`qptiff_regions`, and the 8-bit ``.qptiff`` of every region that also has an ``.er.qptiff``. + + Returns + ------- + The scan file of every region (the ``.er.qptiff`` if there is one), and the 8-bit file of the same scan by + region id for the regions that have both. + """ scans: dict[tuple[Path, str], Path] = {} + eight_bit: dict[tuple[Path, str], Path] = {} for file in files: key = (file.parent, qptiff_base_name(file)) - if key not in scans or file.name.lower().endswith(PhenocyclerKeys.ER_QPTIFF_SUFFIX.value): + if _is_extended_range(file): + if key in scans and not _is_extended_range(scans[key]): + eight_bit.setdefault(key, scans[key]) scans[key] = file + elif key not in scans: + scans[key] = file + else: + eight_bit.setdefault(key, file) + # an 8-bit file is only a duplicate if the scan is read from an extended-range file + eight_bit = {key: file for key, file in eight_bit.items() if _is_extended_range(scans[key])} bases = [base for _, base in scans] stripped = [_SCAN_SUFFIX.sub("", base) or base for base in bases] # stripping `_Scan` must not merge two scans into one region @@ -277,7 +302,8 @@ def qptiff_regions(files: Sequence[Path]) -> dict[str, Path]: f"Several qptiff files map to the same region(s) {duplicated}: {found}. Pass the path of a single qptiff " "file or of a directory that contains each region once." ) - return dict(zip(regions, scans.values(), strict=True)) + region_of = dict(zip(scans, regions, strict=True)) + return dict(zip(regions, scans.values(), strict=True)), {region_of[key]: file for key, file in eight_bit.items()} def _generation(software: str | None, has_experiment_v4: bool) -> str: diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 09f3d526..f7ff14e6 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -23,7 +23,7 @@ _fusion_profile_records, _is_blank, qptiff_base_name, - qptiff_regions, + qptiff_regions_with_duplicates, read_qptiff_image, read_qptiff_metadata, ) @@ -38,6 +38,8 @@ # the Fusion writes per-cycle acquisition files to ``raw/images/phenocycler/`` (HuBMAP layout) _ACQUISITION_DIR = ("raw", "images", "phenocycler") +# region id suffix of the 8-bit scan read next to its extended-range scan (``read_all_bit_depths``) +_EIGHT_BIT_SUFFIX = "_8bit" @inject_docs(vx=PhenocyclerKeys) @@ -49,6 +51,7 @@ def phenocycler( cells_table: str | Path | Mapping[str, str | Path] | None = None, cells_table_format: str = "auto", labels: str | Path | Mapping[str, str | Path] | None = None, + read_all_bit_depths: bool = False, image_models_kwargs: Mapping[str, Any] = MappingProxyType({}), labels_models_kwargs: Mapping[str, Any] = MappingProxyType({}), cells_table_kwargs: Mapping[str, Any] = MappingProxyType({}), @@ -59,7 +62,7 @@ def phenocycler( - ``*{vx.QPTIFF_SUFFIX!r}`` / ``*{vx.ER_QPTIFF_SUFFIX!r}``: the scans, one region each. The images are read lazily; when both ``X{vx.QPTIFF_SUFFIX}`` and ``X{vx.ER_QPTIFF_SUFFIX}`` exist, the extended-range - (higher bit depth) ``{vx.ER_QPTIFF_SUFFIX!r}`` is used. + (higher bit depth) ``{vx.ER_QPTIFF_SUFFIX!r}`` is used, unless ``read_all_bit_depths`` is set. - ``*{vx.XPD_SUFFIX!r}``: optional experiment protocol (JSON) next to a scan. - ``*{vx.CHANNELS_CSV_SUFFIX!r}``: optional channel list next to a scan or, in the HuBMAP layout, in ``lab_processed/images/``. @@ -128,6 +131,12 @@ def phenocycler( circle radii are scaled by ``f`` to image pixels; the source columns in ``obs`` are kept as read (mask pixels). To avoid this for a generic table in image pixels, use ``coordinates_unit="um"`` or a full-resolution mask. + read_all_bit_depths + If `True`, a scan that exists as both ``X{vx.QPTIFF_SUFFIX}`` (8-bit) and ``X{vx.ER_QPTIFF_SUFFIX}`` + (extended range, e.g. uint16) is read from both files: the extended-range file as region ``X`` and the + 8-bit file as the image ``X_8bit{vx.IMAGE_SUFFIX}`` (region ``X_8bit`` in ``sdata.attrs[{vx.ATTRS_KEY!r}]``). + Both are the same physical scan, so the 8-bit image has exactly the transformations of region ``X``. Cell + tables and masks annotate region ``X`` only. If `False`, only the extended-range file is read. image_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. labels_models_kwargs @@ -148,14 +157,24 @@ def phenocycler( regions = normalize_regions(regions) channel_names = [channel_names] if isinstance(channel_names, str) else channel_names path = Path(path) - qptiffs = _discover_qptiffs(path) + qptiffs, eight_bit = _discover_qptiffs(path) selected = select_regions(list(qptiffs), regions) table_files = resolve_per_region(cells_table, selected, list(qptiffs)) mask_files = resolve_per_region(labels, selected, list(qptiffs)) scans = [read_qptiff_metadata(qptiffs[region], region=region) for region in selected] + duplicates: dict[str, AkoyaScan] = {} + if read_all_bit_depths: + for region in selected: + if region in eight_bit: + duplicates[region] = read_qptiff_metadata(eight_bit[region], region=f"{region}{_EIGHT_BIT_SUFFIX}") + if clash := sorted({scan.region for scan in duplicates.values()} & set(qptiffs)): + raise ValueError( + f"Cannot read the 8-bit scans as region(s) {clash}: a scan with that region id exists. Rename it or " + "pass read_all_bit_depths=False." + ) names = {} - for scan in scans: + for scan in [*scans, *duplicates.values()]: names[scan.region] = apply_channel_names(scan, channel_names) _read_sidecars(scan) cell_tables = read_region_tables( @@ -168,7 +187,7 @@ def phenocycler( ) sdata = build_spatialdata( scans, - {scan.region: read_qptiff_image(scan.path) for scan in scans}, + {scan.region: read_qptiff_image(scan.path) for scan in [*scans, *duplicates.values()]}, names, cell_tables, mask_files, @@ -176,19 +195,21 @@ def phenocycler( image_models_kwargs=image_models_kwargs, labels_models_kwargs=labels_models_kwargs, cells_table_kwargs=cells_table_kwargs, + duplicate_scans=duplicates, ) return _set_reader_metadata(sdata, "phenocycler") -def _discover_qptiffs(path: Path) -> dict[str, Path]: +def _discover_qptiffs(path: Path) -> tuple[dict[str, Path], dict[str, Path]]: """Find the qptiff scans of a PhenoCycler-Fusion dataset: a ``.qptiff`` file, or a directory searched recursively. - Returns the scan file of every region, sorted by path; see :func:`qptiff_regions`. + Returns the scan file of every region, sorted by path, and the 8-bit file of the regions that also have an + extended-range one; see :func:`qptiff_regions_with_duplicates`. """ if not path.exists(): raise FileNotFoundError(f"{path} does not exist.") if path.is_file(): - return qptiff_regions([path]) + return qptiff_regions_with_duplicates([path]) files = [ qptiff for qptiff in sorted(path.rglob(f"*{PhenocyclerKeys.QPTIFF_SUFFIX}", case_sensitive=False)) @@ -198,7 +219,7 @@ def _discover_qptiffs(path: Path) -> dict[str, Path]: raise FileNotFoundError( f"No .qptiff files found in {path} (searched recursively for '*.qptiff', excluding acquisition files)." ) - return qptiff_regions(files) + return qptiff_regions_with_duplicates(files) def _is_acquisition_file(path: Path) -> bool: diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 4a2318b9..83085390 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -110,6 +110,49 @@ def test_er_preferred(tmp_path: Path) -> None: assert sdata.attrs["akoya"]["regions"]["S"]["path"].endswith("S_Scan1.er.qptiff") +def test_read_all_bit_depths(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI", "CD3"], dtype="uint8") + write_qptiff(tmp_path / "S_Scan1.er.qptiff", biomarkers=["DAPI", "CD3"], dtype="uint16") + + sdata = phenocycler(tmp_path, read_all_bit_depths=True, image_models_kwargs=_FAST) + + assert sorted(sdata.images) == ["S_8bit_image", "S_image"] + assert _level0(sdata, "S_image").dtype == np.uint16 + assert _level0(sdata, "S_8bit_image").dtype == np.uint8 + assert get_transformation(sdata["S_8bit_image"], get_all=True) == get_transformation(sdata["S_image"], get_all=True) + assert list(get_transformation(sdata["S_image"], get_all=True)) == ["global"] + assert sdata.attrs["akoya"]["regions"]["S"]["path"].endswith("S_Scan1.er.qptiff") + assert sdata.attrs["akoya"]["regions"]["S_8bit"]["path"].endswith("S_Scan1.qptiff") + + +def test_read_all_bit_depths_shares_stage_placement(tmp_path: Path) -> None: + for name, pos in (("A", (0.0, 0.0)), ("B", (0.1, 0.0))): + write_qptiff(tmp_path / f"{name}.qptiff", biomarkers=["DAPI"], dtype="uint8", stage_position_cm=pos) + write_qptiff(tmp_path / f"{name}.er.qptiff", biomarkers=["DAPI"], dtype="uint16", stage_position_cm=pos) + + sdata = phenocycler(tmp_path, read_all_bit_depths=True, regions=["B"], image_models_kwargs=_FAST) + + assert sorted(sdata.images) == ["B_8bit_image", "B_image"] + assert get_transformation(sdata["B_8bit_image"], get_all=True) == get_transformation(sdata["B_image"], get_all=True) + + +def test_read_all_bit_depths_without_pair_reads_one(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S.qptiff", biomarkers=["DAPI"], dtype="uint8") + + sdata = phenocycler(tmp_path, read_all_bit_depths=True, image_models_kwargs=_FAST) + + assert list(sdata.images) == ["S_image"] + + +def test_read_all_bit_depths_region_name_clash_raises(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S.qptiff", biomarkers=["DAPI"], dtype="uint8") + write_qptiff(tmp_path / "S.er.qptiff", biomarkers=["DAPI"], dtype="uint16") + write_qptiff(tmp_path / "S_8bit.qptiff", biomarkers=["DAPI"], dtype="uint8") + + with pytest.raises(ValueError, match="S_8bit"): + phenocycler(tmp_path, read_all_bit_depths=True, image_models_kwargs=_FAST) + + def test_ignores_acquisition_files(tmp_path: Path) -> None: images = tmp_path / "raw" / "images" acquisition = images / "phenocycler" From 38b0e4aa96b98a54e83d6e1974e9a4a2aa0753e0 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 23:09:55 +0200 Subject: [PATCH 034/107] feat: officially support Vectra, PhenoImager and inForm QPTIFFs Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++ README.md | 3 +- src/spatialdata_io/readers/_utils/_qptiff.py | 9 ++- src/spatialdata_io/readers/phenocycler.py | 13 +++- tests/_akoya_data.py | 11 ++- tests/test_akoya_real_data.py | 76 +++++++++++++++++++- tests/test_phenocycler.py | 58 +++++++++++++++ 7 files changed, 164 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fee5fbe7..7071850d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,10 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. integer factor (exactly, or rounded up) onto the image (a `Scale` before the region's transformation), and scale the centroids (`obsm["spatial"]`) and circle radii of `mcquant`, SPRM and generic pixel tables, taken to be in mask pixels, by the same factor; the source columns in `obs` are kept as read. Other shape mismatches still warn. +- `phenocycler()` officially supports the PerkinElmer/Akoya QPTIFFs of Vectra Polaris and PhenoImager HT (generation + `"qpi_other"`): Opal and unmixed (uint16) scans, brightfield RGB scans (channels `r`, `g`, `b`) and inForm component + data (float32). A single `.tif` file is accepted if it is a QPI file. Characters of a file name that are not valid in + an element name (e.g. the `[x,y]` of inForm tiles) are replaced by `_` in the region id. ### Changed diff --git a/README.md b/README.md index 4e431c62..01b3c57d 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ This package contains reader functions to load common spatial omics formats into - 10x Genomics Visium® - 10x Genomics Visium HD® - 10x Genomics Xenium® -- Akoya PhenoCycler® (formerly CODEX®): CODEX Processor/Uploader/Instrument Manager outputs and PhenoCycler-Fusion 1.x/2.x +- Akoya PhenoCycler® (formerly CODEX®): CODEX Processor/Uploader/Instrument Manager outputs and PhenoCycler-Fusion 1.x/2.x, + Vectra® Polaris™ / PhenoImager® HT QPTIFFs and inForm® component TIFFs - Curio Seeker® - DBiT-seq - MCMICRO (output data) diff --git a/src/spatialdata_io/readers/_utils/_qptiff.py b/src/spatialdata_io/readers/_utils/_qptiff.py index e6acf2dc..253e7674 100644 --- a/src/spatialdata_io/readers/_utils/_qptiff.py +++ b/src/spatialdata_io/readers/_utils/_qptiff.py @@ -33,6 +33,8 @@ _FULL_RESOLUTION = "FullResolution" _SCAN_SUFFIX = re.compile(r"_Scan\d+$") +# element names may only hold these characters; inForm names its files e.g. ``_[13860,52919]_component_data.tif`` +_INVALID_REGION_CHARS = re.compile(r"[^0-9A-Za-z_.-]") _XML_DECLARATION = re.compile(r"^\s*<\?xml[^>]*\?>") # micrometres per unit of the TIFF ResolutionUnit; XPosition/YPosition are expressed in the same unit _UM_PER_RESOLUTION_UNIT = {tifffile.RESUNIT.CENTIMETER: 1e4, tifffile.RESUNIT.INCH: 25400.0} @@ -292,9 +294,12 @@ def qptiff_regions_with_duplicates(files: Sequence[Path]) -> tuple[dict[str, Pat # an 8-bit file is only a duplicate if the scan is read from an extended-range file eight_bit = {key: file for key, file in eight_bit.items() if _is_extended_range(scans[key])} bases = [base for _, base in scans] - stripped = [_SCAN_SUFFIX.sub("", base) or base for base in bases] + stripped = [_INVALID_REGION_CHARS.sub("_", _SCAN_SUFFIX.sub("", base) or base) for base in bases] # stripping `_Scan` must not merge two scans into one region - regions = [base if stripped.count(region) > 1 else region for base, region in zip(bases, stripped, strict=True)] + regions = [ + _INVALID_REGION_CHARS.sub("_", base) if stripped.count(region) > 1 else region + for base, region in zip(bases, stripped, strict=True) + ] if len(set(regions)) < len(regions): duplicated = sorted({region for region in regions if regions.count(region) > 1}) found = [str(file) for file, region in zip(scans.values(), regions, strict=True) if region in duplicated] diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index f7ff14e6..a1348a43 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -56,7 +56,7 @@ def phenocycler( labels_models_kwargs: Mapping[str, Any] = MappingProxyType({}), cells_table_kwargs: Mapping[str, Any] = MappingProxyType({}), ) -> SpatialData: - """Read *Akoya PhenoCycler-Fusion* 1.x/2.x data. + """Read *Akoya PhenoCycler-Fusion* 1.x/2.x data, and PerkinElmer/Akoya Vectra and PhenoImager QPTIFFs. This function reads the following files: @@ -67,13 +67,19 @@ def phenocycler( - ``*{vx.CHANNELS_CSV_SUFFIX!r}``: optional channel list next to a scan or, in the HuBMAP layout, in ``lab_processed/images/``. + The same QPTIFF container is written by Vectra Polaris and PhenoImager HT (also unmixed, uint16) scans, whose + channels are read like the Fusion's, by brightfield (H&E) scans, read as the three channels ``r``, ``g`` and + ``b``, and by inForm component data (float32, usually a ``.tif`` file, which is read if ``path`` is that single + file). The image keeps the dtype of the file. These files have the generation ``"qpi_other"`` in the metadata. + Sidecar files are matched to a scan by file-name prefix and stored as metadata only: the channel names always come from the per-page XML of the qptiff. A sidecar whose number of markers differs from the number of channels raises a warning. Acquisition files (``*.raw.qptiff``, ``*.qptiff.raw``, ``*.qptiff.intermediate`` and anything in a ``raw/images/phenocycler/`` folder below ``path``) are ignored. File suffixes are matched ignoring case. The region name is the file name without the ``{vx.ER_QPTIFF_SUFFIX!r}``/``{vx.QPTIFF_SUFFIX!r}`` suffix and - without a trailing ``_Scan``, unless that makes two regions share a name. Each region is stored as the image + without a trailing ``_Scan``, unless that makes two regions share a name; characters other than letters, digits, + ``_``, ``.`` and ``-`` are replaced by ``_``. Each region is stored as the image ``{vx.IMAGE_SUFFIX}``. A single region is stored in the ``"global"`` coordinate system. Several regions of the same sample space (scanned on the same slide, with stage positions and equal pixel sizes) are placed at their stage positions in ``"global"``; otherwise the regions are unrelated and each is stored in its own @@ -106,7 +112,8 @@ def phenocycler( Parameters ---------- path - Path to a ``{vx.QPTIFF_SUFFIX!r}`` file, or to a directory that is searched recursively for them. + Path to a ``{vx.QPTIFF_SUFFIX!r}`` file (or an inForm ``.tif`` QPI file), or to a directory that is searched + recursively for ``{vx.QPTIFF_SUFFIX!r}`` files. regions Regions to read. If `None`, all regions are read. channel_names diff --git a/tests/_akoya_data.py b/tests/_akoya_data.py index 1bf4f1b8..c2cf6ac5 100644 --- a/tests/_akoya_data.py +++ b/tests/_akoya_data.py @@ -131,7 +131,12 @@ def write_qptiff( groups (the last one stripped), an RGB overview and an RGB label. """ rng = np.random.default_rng(0) - high = np.iinfo(dtype).max + + def random(size: tuple[int, ...]) -> np.ndarray: + if np.dtype(dtype).kind == "f": # unmixed component data + return (rng.random(size) * 1000).astype(dtype) + return rng.integers(0, np.iinfo(dtype).max, size, dtype=dtype) + n_pages = 1 if rgb else len(biomarkers) names = ( names if names is not None else [b if b is not None else f"Channel {i + 1}" for i, b in enumerate(biomarkers)] @@ -173,7 +178,7 @@ def rgb_page(side: tuple[int, int] = (64, 64)) -> np.ndarray: for code, value in zip((286, 287), stage_position_cm, strict=True): extratags.append((code, 5, 1, (round(value * 1e6), 1_000_000), False)) - data = rng.integers(0, high, (shape_c if rgb else (n_pages, *shape)), dtype=dtype) + data = random(shape_c if rgb else (n_pages, *shape)) with tifffile.TiffWriter(path, bigtiff=bigtiff) as tw: for i in range(n_pages): page = data if rgb else data[i] @@ -189,7 +194,7 @@ def rgb_page(side: tuple[int, int] = (64, 64)) -> np.ndarray: side = (shape[0] // 2**level, shape[1] // 2**level) is_last = level == levels - 1 for i in range(n_pages): - page = rng.integers(0, high, (*side, 3) if rgb else side, dtype=dtype) + page = random((*side, 3) if rgb else side) write_page( tw, page, diff --git a/tests/test_akoya_real_data.py b/tests/test_akoya_real_data.py index f39f7e62..4c3d1e16 100644 --- a/tests/test_akoya_real_data.py +++ b/tests/test_akoya_real_data.py @@ -41,6 +41,7 @@ class Scan: first_names: tuple[str, ...] pixel_size_um: float | None shape: tuple[int, int, int] | None = None + dtype: str | None = None _SCANS = [ @@ -133,6 +134,76 @@ class Scan: None, (2, 18440, 25816), ), + # Vectra / PhenoImager (generation `qpi_other`): Opal, unmixed uint16, JPEG, brightfield RGB and inForm float32 + Scan( + "zenodo_15265351/P074_Scan1.qptiff", + "phenocycler", + "P074", + "qpi_other", + 8, + ("DAPI", "Opal 570", "Opal 690", "Opal 480"), + 0.4984, + (8, 17280, 17280), + "uint8", + ), + Scan( + "ome_vectra_qptiff/LuCa-7color_Scan1.qptiff", + "phenocycler", + "LuCa-7color", + "qpi_other", + 5, + ("DAPI", "FITC", "CY3", "Texas Red", "CY5"), + 0.498, + (5, 34560, 24960), + "uint8", + ), + Scan( + "zenodo_22107401/Panel33C_Scan2.unmixed.qptiff", + "phenocycler", + "Panel33C_Scan2.unmixed", + "qpi_other", + 8, + ("CD21", "CD23", "CD20", "CD4", "CK"), + 0.4965, + (8, 36000, 24960), + "uint16", + ), + # a third-party (KFBIO) scanner writing JPEG-compressed QPI + Scan( + "zenodo_16569043/2025-07-15_13_13_34.qptiff", + "phenocycler", + "2025-07-15_13_13_34", + "qpi_other", + 3, + ("DAPI", "520", "620"), + 0.25, + (3, 71472, 61440), + "uint8", + ), + # brightfield: one RGB page read as the channels r, g, b + Scan( + "ome_vectra_qptiff/HandEcompressed_Scan1.qptiff", + "phenocycler", + "HandEcompressed", + "qpi_other", + 3, + ("r", "g", "b"), + 0.4989, + (3, 26640, 30720), + "uint8", + ), + # inForm component data: a single `.tif` file with float32 pages + Scan( + "ome_vectra_qptiff/LuCa-7color_[13860,52919]_1x1component_data.tif", + "phenocycler", + "LuCa-7color__13860_52919__1x1component_data", + "qpi_other", + 8, + ("PDL1 (Opal 520)",), + 0.498, + (8, 1400, 1868), + "float32", + ), ] @@ -150,12 +221,15 @@ def test_real_scan(scan: Scan) -> None: assert data.shape[0] == scan.n_channels if scan.shape is not None: assert data.shape == scan.shape + if scan.dtype is not None: + assert data.dtype == scan.dtype names = get_channel_names(image) assert tuple(names[: len(scan.first_names)]) == scan.first_names assert len(set(names)) == len(names) metadata = sdata.attrs["akoya"]["regions"][scan.region] assert metadata["generation"] == scan.generation - assert len(metadata["channels"]) == scan.n_channels + # a brightfield scan has one RGB page, which is read as three channels + assert len(metadata["channels"]) == (1 if names == ["r", "g", "b"] else scan.n_channels) if scan.pixel_size_um is None: assert metadata["pixel_size_um"] is None else: diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 83085390..6a7c515a 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -40,6 +40,64 @@ def test_read_single_file(tmp_path: Path) -> None: assert sdata.attrs["spatialdata_io_reader"] == "phenocycler" +def test_vectra_unmixed_float32_component_qptiff(tmp_path: Path) -> None: + path = tmp_path / "Slide_Scan1.unmixed.qptiff" + names = ["PDL1 (Opal 520)", "DAPI"] + data = write_qptiff( + path, + biomarkers=[None, None], + names=names, + dtype="float32", + software="VectraPolaris 1.0", + resolution_px_per_cm=2e4, + ) + + sdata = phenocycler(path, image_models_kwargs=_FAST) + + image = _level0(sdata, "Slide_Scan1.unmixed_image") + assert image.dtype == np.float32 + np.testing.assert_array_equal(image.values, data) + assert get_channel_names(sdata.images["Slide_Scan1.unmixed_image"]) == names + metadata = sdata.attrs["akoya"]["regions"]["Slide_Scan1.unmixed"] + assert metadata["generation"] == "qpi_other" + assert metadata["pixel_size_um"] == pytest.approx(0.5) + + +def test_vectra_brightfield_rgb_qptiff(tmp_path: Path) -> None: + path = tmp_path / "HandE_Scan1.qptiff" + data = write_qptiff(path, biomarkers=[None], names=["BF"], rgb=True, software="VectraPolaris 1.0") + + sdata = phenocycler(path, image_models_kwargs=_FAST) + + assert get_channel_names(sdata.images["HandE_image"]) == ["r", "g", "b"] + np.testing.assert_array_equal(_level0(sdata, "HandE_image").values, np.moveaxis(data, -1, 0)) + assert sdata.attrs["akoya"]["regions"]["HandE"]["generation"] == "qpi_other" + + +def test_inform_component_tif_with_special_characters_in_name(tmp_path: Path) -> None: + # inForm names its component files `_[x,y]_component_data.tif`; the brackets and comma are not valid + # in an element name + path = tmp_path / "LuCa_[13860,52919]_1x1component_data.tif" + data = write_qptiff( + path, biomarkers=[None, None], names=["A (Opal 520)", "B"], dtype="float32", software="inForm", levels=1 + ) + + sdata = phenocycler(path, image_models_kwargs=_FAST) + + assert list(sdata.images) == ["LuCa__13860_52919__1x1component_data_image"] + image = _level0(sdata, "LuCa__13860_52919__1x1component_data_image") + assert image.dtype == np.float32 + np.testing.assert_array_equal(image.values, data) + + +def test_non_qpi_tif_raises(tmp_path: Path) -> None: + path = tmp_path / "plain.tif" + tifffile.imwrite(path, np.zeros((8, 8), dtype=np.uint8)) + + with pytest.raises(ValueError, match="not a PerkinElmer/Akoya QPTIFF"): + phenocycler(path) + + def test_read_directory_multiple_scans_and_subset(tmp_path: Path) -> None: write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) write_qptiff(tmp_path / "B_Scan1.qptiff", biomarkers=["DAPI", "CD8"]) From ad02cb9f67c202e984d8bd1d0229db78c8e04ae6 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 23:38:45 +0200 Subject: [PATCH 035/107] feat: read real QuPath, inForm and mcquant exports Validated on public exports (Dryad hqbzkh1v1, Zenodo 15514605, phenoptr, auto-cell, MCMICRO exemplar-001): - QuPath: match columns ignoring case/punctuation (R `Object.ID`, `Centroid.X.um`), `__Mean` markers, `X`/`Y` centroids in the unit of the other measurements (or `coordinates_unit`), bare marker names with a warning or `markers`. - QuPath GeoJSON cells without a table row are skipped with a warning. - inForm: strip column names; infer the unit of `Cell X Position` from the other columns; read the nuclei page of `*_binary_seg_maps.tif` by default and accept its page names as `mask_channel`. - Warn when most cells of a region lie outside its image. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 + .../readers/_utils/_cell_tables.py | 307 ++++++++++++++---- src/spatialdata_io/readers/codex.py | 9 +- src/spatialdata_io/readers/phenocycler.py | 9 +- tests/readers/test_cell_tables.py | 255 ++++++++++++++- tests/test_akoya_real_data.py | 127 +++++++- 6 files changed, 633 insertions(+), 82 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7071850d..0987a24d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,14 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. `"qpi_other"`): Opal and unmixed (uint16) scans, brightfield RGB scans (channels `r`, `g`, `b`) and inForm component data (float32). A single `.tif` file is accepted if it is a QPI file. Characters of a file name that are not valid in an element name (e.g. the `[x,y]` of inForm tiles) are replaced by `_` in the region id. +- The Akoya cell-table adapters read real QuPath, inForm and mcquant exports without column mapping: R-exported + QuPath names (`Object.ID`, `Centroid.X.um`, `__Mean`), QuPath `X`/`Y` centroids (in the unit + of the other measurements, or `cells_table_kwargs["coordinates_unit"]`) with bare marker names (read with a + warning, or chosen with `cells_table_kwargs["markers"]`), and inForm tables whose `Cell X Position` names no unit + (read in the unit of the other columns, e.g. microns) or whose column names have trailing spaces. An inForm + `*_binary_seg_maps.tif` given as `labels` reads its nuclei page (the `Cell ID`s) by default, and `mask_channel` + accepts its page names (`"Tissue"`, `"Nucleus"`, `"Cytoplasm"`, `"Membrane"`). QuPath GeoJSON cells without a + table row are skipped with a warning, and a region whose cells lie mostly outside its image raises a warning. ### Changed diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index a10ed67b..4c14a555 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -85,9 +85,14 @@ _META_COLUMNS = frozenset( {"cell_id", "region", "tile_num", "x", "y", "z", "x_tile", "y_tile", "size", "tsne_x", "tsne_y", "homogeneity"} ) -# QuPath measurement columns, e.g. `CD3: Cell: Mean`; the per-marker intensity is taken from the first compartment -_QUPATH_MARKER = re.compile(r"(.+): (Cell|Nucleus|Cytoplasm|Membrane): Mean") +# QuPath measurement columns, e.g. `CD3: Cell: Mean`, also in the R `make.names` form `CD3..Cell..Mean` and as +# `CD3_Cell_Mean`; the per-marker intensity is taken from the first compartment +_QUPATH_MARKER = re.compile(r"(.+?)(?:: |\.\.|_)(Cell|Nucleus|Cytoplasm|Membrane)(?:: |\.\.|_)Mean") _QUPATH_COMPARTMENTS = ("Cell", "Nucleus", "Cytoplasm", "Membrane") +# other QuPath measurements, e.g. `Nucleus: Area µm^2` or `Nucleus/Cell area ratio`, are not markers +_QUPATH_MEASUREMENT = re.compile(r"(?:Cell|Nucleus|Cytoplasm|Membrane)\b|.*: ") +# the unit of a QuPath measurement, e.g. `Cell: Area µm^2` +_QUPATH_UNIT = re.compile(r" (µm|um|px)(?:\^2)?$") _QUPATH_IMAGE = "Image" _QUPATH_ID = "Object ID" # the QuPath object types of cells; other features of a GeoJSON export (e.g. annotations) are not cells @@ -96,6 +101,14 @@ _INFORM_POSITION = re.compile(r"Cell X Position(?: \((.*)\))?") _INFORM_MARKER = re.compile(r"Entire Cell (.+) Mean(?: \(.*\))?") _INFORM_AREA = re.compile(r"Entire Cell Area \((.*)\)") +# the unit of an inForm measurement, e.g. `Tissue Category Area (square microns)` +_INFORM_UNIT = re.compile(r"\((?:square )?(microns|pixels)\)$") +# the pages of an inForm `*_binary_seg_maps.tif`: the tissue class map, then one per compartment +_INFORM_TISSUE_MAP = "" +_INFORM_COMPARTMENT = re.compile(r"\s*(\w+)\s*") +_INFORM_TISSUE_PAGE = "Tissue" +# the default mask of an inForm segmentation map: the nuclei carry the `Cell ID` of every cell +_INFORM_DEFAULT_MASK = "Nucleus" _INFORM_REGION = "Sample Name" _INFORM_ID = "Cell ID" # HuBMAP SPRM outputs: `-cell_channel_mean.csv` with the centroids in `-cell_centers.csv` @@ -278,8 +291,8 @@ def detect_table_format(path: Path) -> str: Returns ------- - ``"codex"`` (FCS, or ``x``/``y`` with ``cell_id`` or ``cycCCC_chHHH`` columns), ``"qupath"`` (``Centroid X µm`` - or ``Object ID``), ``"mcquant"`` (``CellID`` and ``X_centroid``), ``"inform"`` (``Cell X Position``, optionally + ``"codex"`` (FCS, or ``x``/``y`` with ``cell_id`` or ``cycCCC_chHHH`` columns), ``"qupath"`` (``Centroid X µm``, + ``Centroid X px`` or ``Object ID``, ignoring case and punctuation as in R's ``Centroid.X.um``), ``"mcquant"`` (``CellID`` and ``X_centroid``), ``"inform"`` (``Cell X Position``, optionally followed by its unit in parentheses), ``"sprm"`` (first column ``ID``), else ``"generic"``. """ @@ -292,7 +305,7 @@ def detect_table_format(path: Path) -> str: CodexKeys.INSTANCE_KEY.value in names or any(_MARKER_COLUMN.fullmatch(name) for name in names) ): return "codex" - if {"Centroid X µm", "Centroid X px", _QUPATH_ID} & set(columns): + if {"centroidxum", "centroidxpx", _column_key(_QUPATH_ID)} & {_column_key(column) for column in columns}: return "qupath" if {"CellID", "X_centroid"} <= set(columns): return "mcquant" @@ -322,14 +335,21 @@ def read_cell_table( ``"sprm"``, ``"inform"`` or ``"generic"``: - ``"codex"``: see :func:`read_codex_table`. - - ``"qupath"``: a QuPath measurement export. Centroids ``Centroid X µm``/``Centroid Y µm`` (divided by - ``pixel_size_um``) or ``Centroid X px``/``Centroid Y px``; instance ids from a numeric ``Object ID``, - else the row number (from 1); the region from the ``Image`` column. The markers are the - ``: : Mean`` columns, one per marker from the first of the ``Cell``, - ``Nucleus``, ``Cytoplasm`` and ``Membrane`` compartments (in ``var["compartment"]``). The radius - comes from ``Cell: Area µm^2`` (or ``px^2``). The cell outlines are read from a QuPath GeoJSON - export, see :func:`read_qupath_polygons`: ``cells_table_kwargs["geojson"]``, else - ``
.geojson`` or ``
_cells.geojson`` next to the table, if it exists. + - ``"qupath"``: a QuPath measurement export. Column names are matched ignoring case, spaces and + punctuation, so R's ``make.names`` forms (``Centroid.X.um``, ``Object.ID``) are read too. Centroids + ``Centroid X µm``/``Centroid Y µm`` (divided by ``pixel_size_um``) or ``Centroid X px``/``Centroid Y + px``, else ``X``/``Y`` in ``coordinates_unit`` or, without it, in the unit of the other measurements + (e.g. ``Cell: Area µm^2``; QuPath writes all of them in one unit), raising if that is not unique. + Instance ids from a numeric ``Object ID``, else the row number (from 1); the region from the + ``Image`` column. The markers are ``markers``, else the ``: : Mean`` columns + (also ``....Mean`` and ``__Mean``), one per marker from + the first of the ``Cell``, ``Nucleus``, ``Cytoplasm`` and ``Membrane`` compartments (in + ``var["compartment"]``), else, with a `UserWarning`, the numeric columns that are not QuPath + measurements (e.g. ``CD44`` of a detection export). The radius comes from ``Cell: Area µm^2`` (or + ``px^2``) in the unit of the centroids. A table split into centroids and measurements (two files) + must be joined on the object id first. The cell outlines are read from a QuPath GeoJSON export, see + :func:`read_qupath_polygons`: ``cells_table_kwargs["geojson"]``, else ``
.geojson`` or + ``
_cells.geojson`` next to the table, if it exists. - ``"mcquant"``: an MCMICRO ``mcquant`` table. Instance ids ``CellID``, pixel centroids ``X_centroid``/``Y_centroid``, the markers are the columns between ``CellID`` and ``X_centroid``, the radius comes from ``Area``. @@ -338,12 +358,15 @@ def read_cell_table( one marker per remaining column. Pixel centroids come from ``-cell_centers.csv`` (``ID,x,y``) next to the table, which must hold every cell; without it a `UserWarning` is raised and the table has no ``obsm["spatial"]`` (the centroids are never computed from the mask), so its region needs a mask. - - ``"inform"``: a PerkinElmer/Akoya inForm ``*_cell_seg_data.txt`` (tab-separated; validated on synthetic - data only). Instance ids ``Cell ID``, centroids ``Cell X Position``/``Cell Y Position`` in the unit - named in the header, ``(microns)`` (divided by ``pixel_size_um``) or ``(pixels)``; pixels if the - header names none. The region comes from ``Sample Name``. The markers are the - ``Entire Cell Mean (...)`` columns; the radius comes from ``Entire Cell Area (...)`` if its - unit equals that of the positions. + - ``"inform"``: a PerkinElmer/Akoya inForm ``*_cell_seg_data.txt`` (tab-separated; spaces around the + column names are removed). Instance ids ``Cell ID``, centroids ``Cell X Position``/``Cell Y + Position`` in the unit named in their header, ``(microns)`` (divided by ``pixel_size_um``) or + ``(pixels)``; if it names none, in ``coordinates_unit`` or, without it, in the unit of the other + columns (e.g. ``Tissue Category Area (square microns)``; inForm writes all of them in one unit), + raising if they name both, pixels if they name none. The region comes from ``Sample Name``. The + markers are the ``Entire Cell Mean (...)`` columns; the radius comes from + ``Entire Cell Area (...)`` if its unit equals that of the positions. Positions in microns may be + relative to the slide rather than to the field image, see :func:`build_spatialdata`. - ``"generic"``: columns given by ``cells_table_kwargs``. pixel_size_um @@ -356,8 +379,9 @@ def read_cell_table( Optional keys: ``region_column`` (the column holding the region of each cell; QuPath default ``Image``), and for ``"generic"`` tables ``x`` and ``y`` (required centroid columns), ``instance_id`` (else the row number, from 1), ``markers`` (a list of columns or a regular expression matching the full column name; - default: all other numeric columns) and ``coordinates_unit`` (``"px"``, default, or ``"um"``); for - ``"qupath"`` tables ``geojson`` (the GeoJSON file of the cell outlines) and ``geojson_units`` (the unit of + default: all other numeric columns; also for ``"qupath"`` tables) and ``coordinates_unit`` (``"px"``, + default, or ``"um"``; for ``"qupath"`` and ``"inform"`` tables the unit of centroid columns whose name + has none); for ``"qupath"`` tables ``geojson`` (the GeoJSON file of the cell outlines) and ``geojson_units`` (the unit of its coordinates, ``"px"`` of the full-resolution image, default and what QuPath writes, or ``"um"``). The key ``mask_channel`` is used by :func:`read_labels`. @@ -380,7 +404,8 @@ def read_cell_table( _, sep = _read_header(path) df = pd.read_csv(path, sep=sep, engine="pyarrow") - df.columns = pd.Index([str(column) for column in df.columns]) + # inForm writes some column names with trailing spaces + df.columns = pd.Index([str(column).strip() for column in df.columns]) reader = { "qupath": _qupath_roles, "mcquant": _mcquant_roles, @@ -439,6 +464,17 @@ def _row_pixel_sizes( return values.map(sizes_by_value).to_numpy(dtype=np.float64) +def _column_key(column: str) -> str: + """``column`` without case, punctuation and spaces, ``µ`` as ``u``: QuPath's ``Centroid X µm`` is R's ``Centroid.X.um``.""" + return re.sub(r"[^0-9a-z]", "", column.replace("µ", "u").replace("μ", "u").casefold()) + + +def _object_id_column(columns: Sequence[str]) -> str | None: + """The QuPath ``Object ID`` column (also R's ``Object.ID``), or `None`.""" + key = _column_key(_QUPATH_ID) + return next((column for column in columns if _column_key(column) == key), None) + + def _find_geojson(table_path: Path) -> Path | None: """The GeoJSON next to a QuPath table: ``.geojson``, else ``_cells.geojson``. @@ -458,10 +494,11 @@ def read_qupath_polygons( ) -> np.ndarray: """Read the cell outlines of a QuPath GeoJSON export and match them to the cells of its measurement table. - A feature belongs to the cell whose ``Object ID`` is the feature ``id`` (the object's UUID, QuPath >= 0.4); - features of other objects (e.g. annotations) are skipped. If the table has no ``Object ID`` column or the - features have no ids, the cell and detection features (``objectType``) are matched to the table rows in order, - and their number must equal the number of rows. + A feature belongs to the cell whose ``Object ID`` (or R's ``Object.ID``) is the feature ``id`` (the object's + UUID, QuPath >= 0.4); features of other objects (e.g. annotations) are skipped, and so are cell features without + a table row (e.g. cells filtered out of the table), with a `UserWarning`. If the table has no ``Object ID`` + column or the features have no ids, the cell and detection features (``objectType``) are matched to the table + rows in order, and their number must equal the number of rows. Every matched polygon must contain its cell's centroid or lie within 1 px of it; if more than 5% do not, the polygons belong to other cells (or ``units`` is wrong) and a `ValueError` is raised. @@ -492,21 +529,30 @@ def read_qupath_polygons( ids, geometries, kinds = _read_geojson(geojson) polygons = np.full(len(table.obs), None, dtype=object) - if _QUPATH_ID in table.obs.columns and ids is not None: - object_ids = pd.Index(table.obs[_QUPATH_ID].astype(str)) + id_column = _object_id_column(list(table.obs.columns)) + if id_column is not None and ids is not None: + object_ids = pd.Index(table.obs[id_column].astype(str)) if not object_ids.is_unique: - raise ValueError(f"The '{_QUPATH_ID}' column of {table_path} is not unique.") + raise ValueError(f"The '{id_column}' column of {table_path} is not unique.") rows = object_ids.get_indexer(pd.Index(ids).astype(str)) found = rows >= 0 if not found.any(): raise ValueError( - f"{geojson} does not hold the cells of {table_path}: none of the feature ids is an '{_QUPATH_ID}' " + f"{geojson} does not hold the cells of {table_path}: none of the feature ids is an '{id_column}' " "of the table." ) if not pd.Index(rows[found]).is_unique: raise ValueError(f"{geojson} has several features with the same id.") polygons[rows[found]] = geometries[found] logger.info("Matched %d of the %d features of %s to cells by id.", found.sum(), len(geometries), geojson) + is_cell = np.ones(len(ids), dtype=bool) if kinds is None else np.isin(kinds, list(_QUPATH_CELL_TYPES)) + if unmatched := int(np.count_nonzero(is_cell & ~found)): + warnings.warn( + f"{unmatched} of the {is_cell.sum()} cell features of {geojson} have no row in {table_path} (e.g. " + "cells filtered out of the table); they are ignored.", + UserWarning, + stacklevel=5, + ) if found.sum() < len(polygons): warnings.warn( f"{len(polygons) - found.sum()} of the {len(polygons)} cells of {table_path} have no polygon in " @@ -590,39 +636,105 @@ def _read_geojson(path: Path) -> tuple[np.ndarray | None, np.ndarray, np.ndarray def _qupath_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: - unit = next((unit for unit in ("µm", "px") if f"Centroid X {unit}" in df.columns), None) + columns = {_column_key(column): column for column in df.columns} + unit = next((unit for unit in ("um", "px") if f"centroidx{unit}" in columns), None) + x_key, y_key = ("x", "y") if unit is None else (f"centroidx{unit}", f"centroidy{unit}") + x, y = columns.get(x_key), columns.get(y_key) + if x is None or y is None: + raise ValueError( + f"{path} has no centroid columns ('Centroid X µm' and 'Centroid Y µm', 'Centroid X px' and " + f"'Centroid Y px', or 'X' and 'Y'); found {list(df.columns)}. If the export splits the centroids and " + "the measurements into two files, join them on the object id first." + ) if unit is None: - raise ValueError(f"{path} has no 'Centroid X µm' or 'Centroid X px' column; found {list(df.columns)}.") + unit = _qupath_xy_unit(path, list(df.columns), kwargs) + object_id = _object_id_column(list(df.columns)) + instance_column = object_id if object_id is not None and pd.api.types.is_integer_dtype(df[object_id]) else None + region_column = kwargs.get("region_column", _QUPATH_IMAGE if _QUPATH_IMAGE in df.columns else None) + area = columns.get(f"cellarea{unit}2") + roles = {x, y, object_id, region_column, area} + compartments: dict[str, dict[str, str]] = {} for column in df.columns: if (match := _QUPATH_MARKER.fullmatch(column)) is not None: compartments.setdefault(match.group(1), {})[match.group(2)] = column - chosen = {marker: next(c for c in _QUPATH_COMPARTMENTS if c in found) for marker, found in compartments.items()} - markers = [compartments[marker][compartment] for marker, compartment in chosen.items()] - if not markers: + if kwargs.get("markers") is not None: + markers = _select_markers(df, kwargs["markers"], roles) + var = pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(markers))) + elif compartments: + chosen = {marker: next(c for c in _QUPATH_COMPARTMENTS if c in found) for marker, found in compartments.items()} + markers = [compartments[marker][compartment] for marker, compartment in chosen.items()] + var = pd.DataFrame( + {"source_column": markers, "compartment": list(chosen.values())}, + index=pd.Index(dedupe_channel_names(list(chosen))), + ) + else: + # e.g. a detection export with the measurements named after the markers only + candidates = [column for column in df.columns if not _QUPATH_MEASUREMENT.match(column)] + markers = _select_markers(df[candidates], None, roles) + var = pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(markers))) + if markers: + shown = f"{markers[:10]}{' ...' if len(markers) > 10 else ''}" + message = f"reading the {len(markers)} other numeric columns {shown} as the markers" + else: + message = "its table has no marker intensities" warnings.warn( - f"{path} has no ': : Mean' columns, so its table has no marker intensities.", + f"{path} has no ': : Mean' columns; {message}. Set " + "`cells_table_kwargs['markers']` to choose the marker columns.", UserWarning, stacklevel=5, ) - var = pd.DataFrame( - {"source_column": markers, "compartment": list(chosen.values())}, - index=pd.Index(dedupe_channel_names(list(chosen))), - ) - object_id = df.get(_QUPATH_ID) - area = f"Cell: Area {unit}^2" return { "markers": markers, "var": var, - "x": f"Centroid X {unit}", - "y": f"Centroid Y {unit}", - "in_um": unit == "µm", - "instance_column": _QUPATH_ID if object_id is not None and pd.api.types.is_integer_dtype(object_id) else None, - "region_column": kwargs.get("region_column", _QUPATH_IMAGE if _QUPATH_IMAGE in df.columns else None), - "area_column": area if area in df.columns else None, + "x": x, + "y": y, + "in_um": unit == "um", + "instance_column": instance_column, + "region_column": region_column, + "area_column": area, } +def _qupath_xy_unit(path: Path, columns: list[str], kwargs: Mapping[str, Any]) -> str: + """The unit of the ``X``/``Y`` centroids of a QuPath table: ``coordinates_unit``, else that of its measurements. + + QuPath writes all measurements of an image in one unit, µm if the image is calibrated, else pixels. + """ + if kwargs.get("coordinates_unit") is not None: + return _coordinates_unit(kwargs) + units = {"px" if m.group(1) == "px" else "um" for column in columns if (m := _QUPATH_UNIT.search(column))} + if len(units) != 1: + found = f"in {sorted(units)}" if units else "in no unit" + raise ValueError( + f"The centroid columns 'X' and 'Y' of {path} name no unit, and its other measurements are {found}; set " + "`cells_table_kwargs['coordinates_unit']` to 'px' or 'um'." + ) + unit = units.pop() + logger.info("Reading the centroids 'X' and 'Y' of %s in %s, the unit of its other measurements.", path, unit) + return unit + + +def _coordinates_unit(kwargs: Mapping[str, Any]) -> str: + """``cells_table_kwargs['coordinates_unit']``: ``"px"`` (the default) or ``"um"``.""" + unit = kwargs.get("coordinates_unit", "px") + if unit not in ("px", "um"): + raise ValueError(f"`cells_table_kwargs['coordinates_unit']` must be 'px' or 'um', got {unit!r}.") + return str(unit) + + +def _select_markers(df: pd.DataFrame, selection: str | Sequence[str] | None, exclude: set[str | None]) -> list[str]: + """The marker columns: ``selection`` (columns, or a regular expression matching the full column name). + + Without ``selection``, all numeric columns. Columns in ``exclude`` are only selected if listed in ``selection``. + """ + if selection is None: + return [c for c in df.columns if c not in exclude and pd.api.types.is_numeric_dtype(df[c])] + if isinstance(selection, str): + return [c for c in df.columns if c not in exclude and re.fullmatch(selection, c)] + return [str(marker) for marker in selection] + + def _mcquant_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: columns = list(df.columns) if "CellID" not in columns or "X_centroid" not in columns: @@ -692,7 +804,7 @@ def _inform_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> di if not positions or y is None: raise ValueError(f"{path} has no 'Cell X Position' and 'Cell Y Position' columns; found {list(df.columns)}.") x = positions[0].group(0) - unit = (positions[0].group(1) or "pixels").casefold() + unit = (positions[0].group(1) or _inform_position_unit(path, list(df.columns), kwargs)).casefold() if unit not in ("microns", "pixels"): raise ValueError(f"The unit '{unit}' of the column '{x}' of {path} must be 'microns' or 'pixels'.") markers, names = [], [] @@ -722,23 +834,34 @@ def _inform_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> di } +def _inform_position_unit(path: Path, columns: list[str], kwargs: Mapping[str, Any]) -> str: + """The unit of ``Cell X Position`` without one in its name: ``coordinates_unit``, else that of the other columns. + + inForm writes all columns in one unit, but names it only for some, e.g. ``Tissue Category Area (pixels)`` or + ``Distance from Process Region Edge (microns)``; a table that names none is in pixels. + """ + if kwargs.get("coordinates_unit") is not None: + return {"px": "pixels", "um": "microns"}[_coordinates_unit(kwargs)] + units = {m.group(1) for column in columns if (m := _INFORM_UNIT.search(column)) is not None} + if len(units) > 1: + raise ValueError( + f"The columns 'Cell X Position' and 'Cell Y Position' of {path} name no unit, and its other columns are " + f"in {sorted(units)}; set `cells_table_kwargs['coordinates_unit']` to 'px' or 'um'." + ) + unit = units.pop() if units else "pixels" + logger.info("Reading the cell positions of %s in %s, the unit of its other columns.", path, unit) + return unit + + def _generic_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: if "x" not in kwargs or "y" not in kwargs: raise ValueError( f"Reading {path} as a 'generic' cell table needs the centroid columns as `cells_table_kwargs` 'x' and " f"'y'; found the columns {list(df.columns)}." ) - unit = kwargs.get("coordinates_unit", "px") - if unit not in ("px", "um"): - raise ValueError(f"`cells_table_kwargs['coordinates_unit']` must be 'px' or 'um', got {unit!r}.") + unit = _coordinates_unit(kwargs) roles = {kwargs["x"], kwargs["y"], kwargs.get("instance_id"), kwargs.get("region_column")} - selection = kwargs.get("markers") - if selection is None: - markers = [c for c in df.columns if c not in roles and pd.api.types.is_numeric_dtype(df[c])] - elif isinstance(selection, str): - markers = [c for c in df.columns if c not in roles and re.fullmatch(selection, c)] - else: - markers = [str(marker) for marker in selection] + markers = _select_markers(df, kwargs.get("markers"), roles) return { "markers": markers, "var": pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(markers))), @@ -986,8 +1109,10 @@ def read_labels( transformations The transformations of the region's image, by coordinate system. mask_channel - The mask to read from a file with several: its index or OME channel name (e.g. ``"cells"``). If `None`, - the first one. + The mask to read from a file with several: its index or channel name, the OME channel name (e.g. + ``"cells"``) or, in an inForm ``*_binary_seg_maps.tif``, ``"Tissue"`` (the tissue class map) or the + compartment (``"Nucleus"``, ``"Cytoplasm"``, ``"Membrane"``). If `None`, the first one; for an inForm + segmentation map ``"Nucleus"``, whose labels are the ``Cell ID`` of every cell. Returns ------- @@ -999,11 +1124,19 @@ def read_labels( raise ValueError(f"{path} has dtype {image.dtype}; a segmentation mask must have an integer dtype.") n_masks = image.shape[0] + names = _mask_channel_names(path, n_masks) if isinstance(mask_channel, str): - names = _ome_channel_names(path) if mask_channel not in names: - raise ValueError(f"{path} has no mask channel {mask_channel!r}; its OME channel names are {names}.") + raise ValueError(f"{path} has no mask channel {mask_channel!r}; its channel names are {names}.") index = names.index(mask_channel) + elif mask_channel is None and _INFORM_TISSUE_PAGE in names and _INFORM_DEFAULT_MASK in names: + index = names.index(_INFORM_DEFAULT_MASK) + logger.info( + "%s is an inForm segmentation map %s; reading %r. Set `mask_channel` to choose another.", + path, + names, + _INFORM_DEFAULT_MASK, + ) else: index = 0 if mask_channel is None else mask_channel if mask_channel is None and n_masks > 1: @@ -1022,11 +1155,25 @@ def _labels_models_kwargs(labels_models_kwargs: Mapping[str, Any]) -> tuple[dict return labels_models_kwargs, spatial_chunks(labels_models_kwargs["chunks"], defaults["chunks"]) -def _ome_channel_names(path: Path) -> list[str]: +def _mask_channel_names(path: Path, n_masks: int) -> list[str]: + """The OME channel names of a mask file, else the page names of an inForm segmentation map, else ``[]``.""" with tifffile.TiffFile(path) as tf: - if not tf.ome_metadata: + if tf.ome_metadata: + return _ome_channel_names(tf.ome_metadata) + if len(tf.pages) != n_masks: return [] - images = tifffile.xml2dict(tf.ome_metadata)["OME"]["Image"] + descriptions = [str(tf.pages[i].description) for i in range(n_masks)] # type: ignore[union-attr] + names = [ + _INFORM_TISSUE_PAGE + if _INFORM_TISSUE_MAP in description + else (match.group(1) if (match := _INFORM_COMPARTMENT.search(description)) is not None else None) + for description in descriptions + ] + return [str(name) for name in names] if all(name is not None for name in names) else [] + + +def _ome_channel_names(ome_metadata: str) -> list[str]: + images = tifffile.xml2dict(ome_metadata)["OME"]["Image"] pixels = (images[0] if isinstance(images, list) else images)["Pixels"] channels = pixels.get("Channel", []) return [str(channel.get("Name", "")) for channel in (channels if isinstance(channels, list) else [channels])] @@ -1336,7 +1483,9 @@ def build_spatialdata( the coordinate systems of :func:`region_transformations`; regions with cells or a mask but no image are unrelated to the scans. A mask downsampled from its region image by an integer factor is scaled onto the image (see :func:`read_region_labels`), and the centroids and radii in mask pixels of its cells (``mask_pixels``, see - :class:`CellTable`) are scaled by the same factor, so that the table and circles are in image pixels. + :class:`CellTable`) are scaled by the same factor, so that the table and circles are in image pixels. A + `UserWarning` is raised for a region if more than 5% of its cells lie outside its image: their coordinates are + in another unit or relative to another origin (e.g. inForm positions in microns relative to the slide). Parameters ---------- @@ -1390,6 +1539,7 @@ def build_spatialdata( mask_files, transformations, labels_models_kwargs, cells_table_kwargs, image_shapes ) cell_tables = _to_image_pixels(cell_tables, factors) + _check_cells_in_images(cell_tables, image_shapes) polygons, geojson_files = polygon_labels(cell_tables, labels, image_shapes, transformations, labels_models_kwargs) labels |= polygons tables, shapes = build_cells(cell_tables, {}, labels, transformations) if cell_tables else ({}, {}) @@ -1412,6 +1562,29 @@ def build_spatialdata( return sdata +def _check_cells_in_images(tables: Sequence[CellTable], image_shapes: Mapping[str, tuple[int, int]]) -> None: + """Warn for each region with an image if more than 5% of its cells' centroids lie outside the image.""" + for table in tables: + if table.centroids_px is None: + continue + for region, rows in _region_rows(table).items(): + if region not in image_shapes: + continue + centroids = table.centroids_px[rows] + centroids = centroids[np.isfinite(centroids).all(axis=1)] + height, width = image_shapes[region] + outside = int(np.count_nonzero(((centroids < 0) | (centroids > [width, height])).any(axis=1))) + if outside > _MAX_MISPLACED_FRACTION * len(centroids): + warnings.warn( + f"{outside} of the {len(centroids)} cells of region '{region}' lie outside its image ({width} x " + f"{height} px): the table's coordinates are in another unit (see " + "`cells_table_kwargs['coordinates_unit']`) or relative to another origin (e.g. inForm positions " + "in microns, which may be relative to the slide).", + UserWarning, + stacklevel=3, + ) + + def _region_rows(table: CellTable) -> dict[str, np.ndarray]: """The row indices of each region of ``table``, in order of first appearance.""" codes, regions = pd.factorize(table.region) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 23b7f5f0..53a6410d 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -191,9 +191,12 @@ def codex( Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`. cells_table_kwargs Options for reading ``cells_table`` and ``labels``: ``region_column``; for ``"generic"`` tables ``x``, - ``y``, ``instance_id``, ``markers`` (list or regular expression) and ``coordinates_unit`` (``"px"`` or - ``"um"``); ``mask_channel`` (index or OME channel name of the mask in a multi-channel ``labels`` file); - for QuPath tables ``geojson`` (the GeoJSON export of the cell outlines, a path or a mapping from region id + ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath tables) and + ``coordinates_unit`` (``"px"`` or ``"um"``; for QuPath ``X``/``Y`` and inForm ``Cell X Position`` columns + without a unit, which are otherwise read in the unit of the table's other columns); ``mask_channel`` + (index or channel name of the mask in a multi-channel ``labels`` file: an OME channel name, or for an + inForm ``*_binary_seg_maps.tif`` ``"Tissue"``, ``"Nucleus"`` (the default there, its labels are the + ``Cell ID`` of every cell), ``"Cytoplasm"`` or ``"Membrane"``); for QuPath tables ``geojson`` (the GeoJSON export of the cell outlines, a path or a mapping from region id to path like ``cells_table``; default ``
.geojson`` or ``
_cells.geojson`` next to the table) and ``geojson_units`` (``"px"`` of the full-resolution image, default and what QuPath writes, or ``"um"``). diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index a1348a43..846253e4 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -150,9 +150,12 @@ def phenocycler( Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`. cells_table_kwargs Options for reading ``cells_table`` and ``labels``: ``region_column``; for ``"generic"`` tables ``x``, - ``y``, ``instance_id``, ``markers`` (list or regular expression) and ``coordinates_unit`` (``"px"`` or - ``"um"``); ``mask_channel`` (index or OME channel name of the mask in a multi-channel ``labels`` file); - for QuPath tables ``geojson`` (the GeoJSON export of the cell outlines, a path or a mapping from region id + ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath tables) and + ``coordinates_unit`` (``"px"`` or ``"um"``; for QuPath ``X``/``Y`` and inForm ``Cell X Position`` columns + without a unit, which are otherwise read in the unit of the table's other columns); ``mask_channel`` + (index or channel name of the mask in a multi-channel ``labels`` file: an OME channel name, or for an + inForm ``*_binary_seg_maps.tif`` ``"Tissue"``, ``"Nucleus"`` (the default there, its labels are the + ``Cell ID`` of every cell), ``"Cytoplasm"`` or ``"Membrane"``); for QuPath tables ``geojson`` (the GeoJSON export of the cell outlines, a path or a mapping from region id to path like ``cells_table``; default ``
.geojson`` or ``
_cells.geojson`` next to the table) and ``geojson_units`` (``"px"`` of the full-resolution image, default and what QuPath writes, or ``"um"``). diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index 5689df72..d5346b00 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -1,11 +1,13 @@ from __future__ import annotations +import json import warnings -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import numpy as np import pandas as pd import pytest +import shapely import tifffile from spatialdata import get_extent, read_zarr from spatialdata.models import Labels2DModel, get_axes_names @@ -190,15 +192,15 @@ def _write_qupath(path: Path, image: str = "S_Scan1.qptiff") -> None: path.write_text("\n".join([_QUPATH_HEADER, *rows]) + "\n", encoding="utf-8") -def _write_mcquant(path: Path, n: int = 5) -> pd.DataFrame: +def _write_mcquant(path: Path, n: int = 5, extent: tuple[float, float] = (60, 40)) -> pd.DataFrame: rng = np.random.default_rng(1) cells = pd.DataFrame( { "CellID": np.arange(1, n + 1), "DAPI": rng.uniform(0, 100, n), "CD3": rng.uniform(0, 100, n), - "X_centroid": rng.uniform(0, 60, n), - "Y_centroid": rng.uniform(0, 40, n), + "X_centroid": rng.uniform(0, extent[0], n), + "Y_centroid": rng.uniform(0, extent[1], n), "Area": rng.uniform(10, 50, n), "Eccentricity": rng.uniform(0, 1, n), } @@ -452,18 +454,254 @@ def test_inform_adapter_pixels(tmp_path: Path) -> None: def test_inform_without_unit_is_pixels_and_renames_cell_id(tmp_path: Path) -> None: path = tmp_path / "Scan1_cell_seg_data.txt" - _write_inform(path, None) + _write_inform(path, None, area_unit="pixels") table = read_cell_table(path, "inform", pixel_size_um=None, default_region=None, cells_table_kwargs={}) assert table.centroids_px is not None np.testing.assert_allclose(table.centroids_px, [[100, 50], [200, 80]]) + assert table.radius_px is not None # mixed units (px positions, µm areas) cannot give a radius + _write_inform(path, "pixels") + table = read_cell_table(path, "inform", pixel_size_um=None, default_region=None, cells_table_kwargs={}) assert table.radius_px is None elements, _ = build_cells([table], {}, {}, {}) assert "source_Cell ID" in elements["table"].obs.columns or "source_Cell_ID" in elements["table"].obs.columns +# the header of the inForm micron variant (github.com/lilab-stanford/auto-cell): no unit after `Cell X Position`, +# trailing spaces after the marker columns, CRLF line ends +_INFORM_MICRON_HEADER = [ + "Sample Name", + "Tissue Category", + "Cell ID", + "Total Cells", + "Tissue Category Area (square microns)", + "Cell Density (per square mm)", + "Cell X Position", + "Cell Y Position", + "Distance from Process Region Edge (microns)", + "Nucleus PD1 (PPD520) Mean ", + "Entire Cell Area (square microns)", + "Entire Cell PD1 (PPD520) Mean ", + "Entire Cell PPDDAPI Mean ", + "Lab ID", + "Slide ID", +] + + +def test_inform_micron_variant_without_position_unit(tmp_path: Path) -> None: + path = tmp_path / "sample_cell_seg_data.txt" + rows = [ + ['"TMA_Core[1,1,A]_[4768,30518].im3"', "STROMA", "1", "", "", "", "100", "50", "#N/A", "0", "314.159", "1.5"], + ['"TMA_Core[1,1,A]_[4768,30518].im3"', "TUMOR", "2", "", "", "", "200", "80", "#N/A", "0", "78.54", "3.5"], + ] + rows = [[*row, "2.5" if i == 0 else "4.5", "", ""] for i, row in enumerate(rows)] + path.write_text("\r\n".join("\t".join(row) for row in [_INFORM_MICRON_HEADER, *rows]), encoding="utf-8") + + assert detect_table_format(path) == "inform" + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) + + # the other columns are in microns, so the positions are too + assert table.centroids_px is not None + np.testing.assert_allclose(table.centroids_px, [[200, 100], [400, 160]]) + assert list(table.var.index) == ["PD1 (PPD520)", "PPDDAPI"] + np.testing.assert_allclose(table.X, [[1.5, 2.5], [3.5, 4.5]]) + assert list(table.region) == ["TMA_Core[1,1,A]_[4768,30518].im3"] * 2 + assert table.radius_px is not None + np.testing.assert_allclose(table.radius_px, np.sqrt(np.array([314.159, 78.54]) / 0.25 / np.pi)) + assert "Nucleus PD1 (PPD520) Mean" in table.obs.columns + + +def test_inform_position_unit_from_other_columns(tmp_path: Path) -> None: + # the phenoptr sample (inForm 2.3) names the unit of its areas and distances only, here pixels + path = tmp_path / "S_cell_seg_data.txt" + header = ["Sample Name", "Cell ID", "Tissue Category Area (pixels)", "Cell X Position", "Cell Y Position"] + path.write_text("\t".join(header) + "\nS.im3\t1\t5\t10\t20\n", encoding="utf-8") + + table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) + assert table.centroids_px is not None + np.testing.assert_allclose(table.centroids_px, [[10, 20]]) + + # columns in both units leave the unit of the positions open + header[2] = "Distance from Process Region Edge (microns)" + path.write_text("\t".join([*header, "Entire Cell Area (pixels)"]) + "\nS.im3\t1\t5\t10\t20\t3\n", encoding="utf-8") + with pytest.raises(ValueError, match="coordinates_unit"): + read_cell_table(path, "inform", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) + table = read_cell_table( + path, "inform", pixel_size_um=0.5, default_region=None, cells_table_kwargs={"coordinates_unit": "um"} + ) + assert table.centroids_px is not None + np.testing.assert_allclose(table.centroids_px, [[20, 40]]) + + +# R-exported QuPath tables (dryad doi:10.5061/dryad.hqbzkh1v1): `make.names` turns `Centroid X µm` into +# `Centroid.X.um`; the coordinates and the expression are split into two files +_R_COORDS_HEADER = "Object.ID,Centroid.X.um,Centroid.Y.um,core" + + +def _square(x: float, y: float, size: float = 8) -> dict[str, Any]: + return {"type": "Polygon", "coordinates": [[[x, y], [x + size, y], [x + size, y + size], [x, y + size], [x, y]]]} + + +def test_qupath_r_exported_coordinates_with_geojson(tmp_path: Path) -> None: + path = tmp_path / "C-7_coords.csv" + # µm centroids at 0.5 µm/px: the centres of the 8 px squares at (0, 0) and (20, 20) + path.write_text(f"{_R_COORDS_HEADER}\n0000aa8d-e87f,2,2,C-7\n0001e8f3-07bc,12,12,C-7\n", encoding="utf-8") + annotation = {"objectType": "annotation", "isLocked": True} + features = [ + {"type": "Feature", "id": "c28880d8", "geometry": _square(0, 0, 2270), "properties": annotation}, + *( + { + "type": "Feature", + "id": object_id, + "geometry": _square(x, x), + "nucleusGeometry": _square(x + 2, x + 2, 4), + "properties": {"objectType": "cell"}, + } + # in another order than the table; the last cell was filtered out of the table + for object_id, x in [("0001e8f3-07bc", 20), ("0000aa8d-e87f", 0), ("ffff0000-0000", 40)] + ), + ] + geojson = tmp_path / "C-7.geojson" + geojson.write_text(json.dumps(features), encoding="utf-8") + + assert detect_table_format(path) == "qupath" + with pytest.warns(UserWarning, match=r"1 of the 3 cell features of .*C-7.geojson have no row"): + table = read_cell_table( + path, "auto", pixel_size_um=0.5, default_region="C-7", cells_table_kwargs={"geojson": geojson} + ) + + assert table.centroids_px is not None + np.testing.assert_allclose(table.centroids_px, [[4, 4], [24, 24]]) + assert table.polygons_px is not None + assert table.polygons_px[0].equals(shapely.box(0, 0, 8, 8)) + assert table.polygons_px[1].equals(shapely.box(20, 20, 28, 28)) + assert list(table.instance_id) == [1, 2] + assert list(table.obs["Object.ID"]) == ["0000aa8d-e87f", "0001e8f3-07bc"] + + +def test_qupath_r_exported_marker_columns(tmp_path: Path) -> None: + # the expression file joined to the coordinates on `Object.ID`: `__Mean`, and the + # `make.names` form of `CD3: Cell: Mean` + path = tmp_path / "C-7.csv" + header = f"{_R_COORDS_HEADER},DAPI_Nucleus_Mean,HLA.A_Cytoplasm_Mean,Pan.Cytokeratin_Cytoplasm_Mean,CD3..Cell..Mean" + path.write_text(f"{header}\nab,2,2,C-7,1,2,3,4\n", encoding="utf-8") + + table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region="C-7", cells_table_kwargs={}) + + assert list(table.var.index) == ["DAPI", "HLA.A", "Pan.Cytokeratin", "CD3"] + assert list(table.var["compartment"]) == ["Nucleus", "Cytoplasm", "Cytoplasm", "Cell"] + np.testing.assert_allclose(table.X, [[1, 2, 3, 4]]) + + +def test_qupath_expression_without_centroids_raises(tmp_path: Path) -> None: + path = tmp_path / "C-7_expression.csv" + path.write_text("Object.ID,DAPI_Nucleus_Mean,Ki67_Nucleus_Mean\nab,1,2\n", encoding="utf-8") + + assert detect_table_format(path) == "qupath" + with pytest.raises(ValueError, match="no centroid columns"): + read_cell_table(path, "auto", pixel_size_um=0.5, default_region="C-7", cells_table_kwargs={}) + + +# a QuPath detection export with `X`/`Y` centroids and bare marker names (zenodo.org/records/15514605) +_QUPATH_XY_HEADER = ( + "Image,Object ID,Parent,X,Y,Nucleus: Area µm^2,Nucleus: Length µm,Cell: Area µm^2,Nucleus/Cell area ratio," + "DAPI: Nucleus: Median,CD44,HLA-DR-AF647,CD45RO" +) + + +def test_qupath_xy_centroids_and_bare_markers(tmp_path: Path) -> None: + path = tmp_path / "measurement_TA4369.csv" + rows = [ + "STardist,74fcc0e7-d660,A-1,10,5,7.7,11.5,3.14159,0.13,20.5,1.5,34.2,4.3", + "STardist,9b2e5c11-0a3f,A-2,20,15,8.1,12.0,12.56637,0.2,21,2.5,35.2,5.3", + ] + path.write_text("\r\n".join([_QUPATH_XY_HEADER, *rows]) + "\r\n", encoding="utf-8") + + assert detect_table_format(path) == "qupath" + with pytest.warns(UserWarning, match=r"reading the 3 other numeric columns \['CD44', 'HLA-DR-AF647', 'CD45RO'\]"): + table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) + + # the other measurements are in µm, so the centroids are too + assert table.centroids_px is not None + np.testing.assert_allclose(table.centroids_px, [[20, 10], [40, 30]]) + assert list(table.var.index) == ["CD44", "HLA-DR-AF647", "CD45RO"] + np.testing.assert_allclose(table.X, [[1.5, 34.2, 4.3], [2.5, 35.2, 5.3]], rtol=1e-6) + assert {"Parent", "DAPI: Nucleus: Median", "Nucleus/Cell area ratio", "Object ID"} <= set(table.obs.columns) + assert list(table.region) == ["STardist"] * 2 + assert table.radius_px is not None + np.testing.assert_allclose(table.radius_px, [2, 4], rtol=1e-5) + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + table = read_cell_table( + path, + "qupath", + pixel_size_um=0.5, + default_region=None, + cells_table_kwargs={"region_column": "Parent", "markers": ["CD44", "CD45RO"]}, + ) + assert list(table.region) == ["A-1", "A-2"] + assert list(table.var.index) == ["CD44", "CD45RO"] + + +def test_qupath_xy_centroids_need_a_unit(tmp_path: Path) -> None: + path = tmp_path / "cells.csv" + path.write_text("Object ID,X,Y,CD3: Cell: Mean\nab,10,5,1\n", encoding="utf-8") + + with pytest.raises(ValueError, match="coordinates_unit"): + read_cell_table(path, "auto", pixel_size_um=0.5, default_region="S", cells_table_kwargs={}) + table = read_cell_table( + path, "auto", pixel_size_um=0.5, default_region="S", cells_table_kwargs={"coordinates_unit": "px"} + ) + assert table.centroids_px is not None + np.testing.assert_allclose(table.centroids_px, [[10, 5]]) + + +def test_cells_outside_the_image_warn(tmp_path: Path) -> None: + # inForm positions in microns relative to the slide, not to the field of the image + write_qptiff(tmp_path / "Scan1.qptiff", biomarkers=["DAPI"], shape=(64, 80)) + _write_inform(tmp_path / "Scan1_cell_seg_data.txt", "microns") + + with pytest.warns(UserWarning, match=r"2 of the 2 cells of region 'Scan1' lie outside its image \(80 x 64 px\)"): + phenocycler( + tmp_path, cells_table=tmp_path / "Scan1_cell_seg_data.txt", image_models_kwargs={"scale_factors": [2]} + ) + + +def _write_inform_seg_maps(path: Path, shape: tuple[int, int] = (16, 24)) -> np.ndarray: + # inForm `*_binary_seg_maps.tif`: the tissue class map, then one page per compartment, each with its XML + pages = np.stack([np.full(shape, i, dtype=np.uint16) for i in range(4)]) + descriptions = [ + '\n\n 1\n', + *( + '\n\n 1\n' + f" {compartment}\n" + for compartment in ("Nucleus", "Cytoplasm", "Membrane") + ), + ] + with tifffile.TiffWriter(path) as tw: + for page, description in zip(pages, descriptions, strict=True): + tw.write(page, description=description, metadata=None) + return pages + + +@pytest.mark.parametrize(("mask_channel", "expected"), [(None, 1), ("Cytoplasm", 2), (0, 0)]) +def test_read_labels_inform_seg_maps(tmp_path: Path, mask_channel: int | str | None, expected: int) -> None: + path = tmp_path / "S_binary_seg_maps.tif" + pages = _write_inform_seg_maps(path) + + labels = read_labels(path, {"scale_factors": None}, {"global": Identity()}, mask_channel=mask_channel) + + # by default the nuclei, which hold the `Cell ID` of every cell, not the tissue class map + np.testing.assert_array_equal(labels.values, pages[expected]) + with pytest.raises(ValueError, match=r"'Tissue', 'Nucleus', 'Cytoplasm', 'Membrane'"): + read_labels(path, {}, {"global": Identity()}, mask_channel="cells") + + def test_unknown_format_raises(tmp_path: Path) -> None: path = tmp_path / "cells.csv" path.write_text("ID,DAPI\n1,2\n") @@ -552,7 +790,8 @@ def test_mask_integer_downsample_autoscaled(tmp_path: Path, mask_shape: tuple[in mask = np.zeros(mask_shape, dtype=np.uint32) mask[2:6, 2:6], mask[10:14, 8:12] = 1, 2 tifffile.imwrite(tmp_path / "B_mask.tif", mask) - cells = _write_mcquant(tmp_path / "B.csv", n=2) + # the centroids are in mask pixels, inside the mask + cells = _write_mcquant(tmp_path / "B.csv", n=2, extent=(mask_shape[1], mask_shape[0])) with warnings.catch_warnings(): warnings.simplefilter("error", UserWarning) @@ -598,8 +837,8 @@ def test_mask_integer_downsample_autoscaled(tmp_path: Path, mask_shape: tuple[in def test_mask_downsample_keeps_image_pixel_centroids(tmp_path: Path) -> None: # inForm centroids are in image pixels, so a downsampled mask does not scale them - write_qptiff(tmp_path / "Scan1.qptiff", biomarkers=["DAPI"], shape=(64, 80)) - tifffile.imwrite(tmp_path / "mask.tif", np.zeros((32, 40), dtype=np.uint32)) + write_qptiff(tmp_path / "Scan1.qptiff", biomarkers=["DAPI"], shape=(128, 256)) + tifffile.imwrite(tmp_path / "mask.tif", np.zeros((64, 128), dtype=np.uint32)) _write_inform(tmp_path / "Scan1_cell_seg_data.txt", "pixels", area_unit="pixels") sdata = phenocycler( diff --git a/tests/test_akoya_real_data.py b/tests/test_akoya_real_data.py index 4c3d1e16..aa8ffbc5 100644 --- a/tests/test_akoya_real_data.py +++ b/tests/test_akoya_real_data.py @@ -1,7 +1,8 @@ """Tests of the Akoya readers on real public datasets. They run only when the environment variable ``AKOYA_TEST_DATA`` points to a directory holding the files below (paths -relative to it). The files are public (Zenodo, Figshare, BioImage Archive, GitHub) but too large to download in CI. +relative to it). The files are public (Zenodo, Figshare, BioImage Archive, Dryad, GitHub, the MCMICRO bucket) but too large to download +in CI. """ from __future__ import annotations @@ -11,10 +12,13 @@ from pathlib import Path import dask.array as da +import numpy as np +import pandas as pd import pytest from spatialdata.models import get_channel_names from spatialdata_io import codex, phenocycler +from spatialdata_io.readers._utils._cell_tables import detect_table_format, read_cell_table _ROOT = os.environ.get("AKOYA_TEST_DATA") @@ -252,3 +256,124 @@ def test_real_codex_processor_csv() -> None: assert list(table.obs["region"].cat.categories) == ["reg001_cells"] assert table.obsm["spatial"].shape == (500, 2) assert len(sdata.shapes["reg001_cells"]) == 500 + + +# public cell-table exports (see the docstrings of `read_cell_table` and `read_labels` for the variants) +_EXPORTS = "exports" +_PHENOPTR = f"{_EXPORTS}/github_phenoptr/repo/inst/extdata/sample/Set4_1-6plex_[16142,55840]" +_DRYAD = f"{_EXPORTS}/dryad_hqbzkh1v1" +# the OME `PhysicalSizeX` of the Dryad images, which no Akoya reader reads +_DRYAD_PIXEL_SIZE_UM = 0.5072844 + + +def _export(relative: str) -> Path: + path = Path(_ROOT) / relative # type: ignore[arg-type] + if not path.exists(): + pytest.skip(f"{path} is missing") + return path + + +def test_real_inform_table_with_seg_maps_and_component_data() -> None: + # inForm 2.3 (github.com/akoyabio/phenoptr): positions in pixels, the `Cell ID`s are the labels of page 1 (nuclei) + # of the segmentation map; page 0 is the tissue class map + image = _export(f"{_PHENOPTR}_component_data.tif") + + sdata = phenocycler( + image, + cells_table=_export(f"{_PHENOPTR}_cell_seg_data.txt"), + labels=_export(f"{_PHENOPTR}_binary_seg_maps.tif"), + ) + + region = "Set4_1-6plex__16142_55840__component_data" + table = sdata.tables["table"] + assert table.shape == (6072, 8) + assert list(table.var_names[:2]) == ["PDL1 (Opal 520)", "CD8 (Opal 540)"] + assert list(table.obs["region"].cat.categories) == [f"{region}_labels"] + labels = sdata.labels[f"{region}_labels"]["scale0"]["image"] + assert labels.shape == (1400, 1868) + assert set(table.obs["cell_id"]) == set(np.unique(labels.values)) - {0} + np.testing.assert_allclose(table.obsm["spatial"][0], [515, 13]) + + +def test_real_inform_micron_variant() -> None: + # github.com/lilab-stanford/auto-cell: positions in microns without a unit in their name, trailing spaces + path = _export(f"{_EXPORTS}/github_auto-cell/repo/cell_annotation/sample_cell_seg_data.txt") + + assert detect_table_format(path) == "inform" + table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) + + assert len(table.obs) == 4 + assert list(table.var.index) == [ + "PD1 (PPD520)", + "FOXP3 (PPD540)", + "CD20 (PPD620)", + "CD3 (PPD650)", + "PANCK (PPD690)", + "PPDDAPI", + "Autofluorescence", + ] + assert table.centroids_px is not None + np.testing.assert_allclose(table.centroids_px[0], [4230.4 / 0.5, 29684.5 / 0.5]) + assert table.radius_px is not None + + +def test_real_qupath_r_exported_coordinates_with_geojson(tmp_path: Path) -> None: + # doi:10.5061/dryad.hqbzkh1v1: R-exported QuPath centroids (`Centroid.X.um`) and measurements in two files, and + # the GeoJSON of all StarDist cells (UUID ids, one `annotation`); 116 cells were filtered out of the tables + coords = _export(f"{_DRYAD}/C-7_coords.csv") + geojson = _export(f"{_DRYAD}/C-7.geojson") + expression = _export(f"{_DRYAD}/C-7_expression.csv") + kwargs = {"geojson": geojson} + + assert detect_table_format(coords) == "qupath" + with pytest.warns(UserWarning, match="116 of the 9531 cell features"): + table = read_cell_table( + coords, "auto", pixel_size_um=_DRYAD_PIXEL_SIZE_UM, default_region="C-7", cells_table_kwargs=kwargs + ) + assert table.polygons_px is not None + assert pd.notna(table.polygons_px).all() + assert len(table.obs) == 9415 + + # joined on the object id, the measurements are the markers, from the `Cell` compartment (`DAPI_Cell_Mean`); the + # expression file alone has no centroids in QuPath's form (its `Y_coord` is negated) + joined = tmp_path / "C-7.csv" + pd.read_csv(coords).merge(pd.read_csv(expression), on="Object.ID", validate="1:1").to_csv(joined, index=False) + with pytest.warns(UserWarning, match="116 of the 9531 cell features"): + table = read_cell_table( + joined, "auto", pixel_size_um=_DRYAD_PIXEL_SIZE_UM, default_region="C-7", cells_table_kwargs=kwargs + ) + assert table.X.shape == (9415, 30) + assert {"DAPI", "HLA.A", "Pan.Cytokeratin"} <= set(table.var.index) + assert set(table.var["compartment"]) == {"Cell"} + assert table.var.loc["DAPI", "source_column"] == "DAPI_Cell_Mean" + + +def test_real_qupath_detection_xy_and_bare_markers() -> None: + # zenodo.org/records/15514605: `X`/`Y` centroids in µm, markers named after the marker only, `Parent` = TMA core + path = _export(f"{_EXPORTS}/zenodo_15514605/measurement_TA4369.csv") + + assert detect_table_format(path) == "qupath" + with pytest.warns(UserWarning, match="reading the 48 other numeric columns"): + table = read_cell_table( + path, "auto", pixel_size_um=0.5, default_region=None, cells_table_kwargs={"region_column": "Parent"} + ) + + assert table.X.shape == (348314, 48) + assert list(table.var.index[:3]) == ["CD44", "CD68", "CD31"] + assert table.region.nunique() == 59 + assert table.centroids_px is not None + np.testing.assert_allclose(table.centroids_px[0], [2620.8 / 0.5, 661.3 / 0.5]) + + +def test_real_mcquant_table_with_ome_tiff() -> None: + # MCMICRO exemplar-001: mcquant table in pixels with the OME-TIFF of its first cycle, read as a hyperstack + image = _export(f"{_EXPORTS}/s3_mcmicro_exemplar001/exemplar-001-cycle6.ome.tif") + cells = _export(f"{_EXPORTS}/s3_mcmicro_exemplar001/unmicst-exemplar-001_cell.csv") + + with pytest.warns(UserWarning, match="cells are represented as circles"): + sdata = codex(image, cells_table=cells) + + table = sdata.tables["table"] + assert table.shape == (11170, 12) + assert list(table.var_names[:4]) == ["DNA_6", "ELANE", "CD57", "CD45"] + assert len(sdata.shapes["exemplar-001-cycle6.ome_cells"]) == 11170 From 7c7591895236126d150bac39de33e1024ce3956f Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Fri, 16 Oct 2026 23:56:25 +0200 Subject: [PATCH 036/107] fix: tighten QuPath detection and inForm defaults - Match QuPath columns only by QuPath's own names or their R `make.names` form, so `object_id` / HALO `Object Id` tables stay generic; `auto` with `x`/`y` kwargs reads a generic table. - Warn when QuPath `X`/`Y` units are inferred. - Name inForm seg-map pages per page by XML root; default to `Nucleus` whenever present. - Own threshold and stacklevel for the out-of-image warning; real-data tests assert it is not raised. - CHANGELOG: list behaviour changes under "Changed"; reflow docstrings. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 16 ++- .../readers/_utils/_cell_tables.py | 115 +++++++++++------- src/spatialdata_io/readers/codex.py | 15 +-- src/spatialdata_io/readers/phenocycler.py | 15 +-- tests/readers/test_cell_tables.py | 67 +++++++++- tests/test_akoya_real_data.py | 34 ++++-- 6 files changed, 182 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0987a24d..174491b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,15 +42,19 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. an element name (e.g. the `[x,y]` of inForm tiles) are replaced by `_` in the region id. - The Akoya cell-table adapters read real QuPath, inForm and mcquant exports without column mapping: R-exported QuPath names (`Object.ID`, `Centroid.X.um`, `__Mean`), QuPath `X`/`Y` centroids (in the unit - of the other measurements, or `cells_table_kwargs["coordinates_unit"]`) with bare marker names (read with a - warning, or chosen with `cells_table_kwargs["markers"]`), and inForm tables whose `Cell X Position` names no unit - (read in the unit of the other columns, e.g. microns) or whose column names have trailing spaces. An inForm - `*_binary_seg_maps.tif` given as `labels` reads its nuclei page (the `Cell ID`s) by default, and `mask_channel` - accepts its page names (`"Tissue"`, `"Nucleus"`, `"Cytoplasm"`, `"Membrane"`). QuPath GeoJSON cells without a - table row are skipped with a warning, and a region whose cells lie mostly outside its image raises a warning. + of the other measurements with a warning, or `cells_table_kwargs["coordinates_unit"]`) with bare marker names + (read with a warning, or chosen with `cells_table_kwargs["markers"]`), and inForm column names with trailing + spaces. `mask_channel` accepts the page names of an inForm `*_binary_seg_maps.tif` (`"Tissue"`, `"Nucleus"`, + `"Cytoplasm"`, `"Membrane"`). QuPath GeoJSON cells without a table row are skipped with a warning, and a region + whose cells lie mostly outside its image raises a warning. ### Changed +- inForm cell tables whose `Cell X Position` names no unit are read in the unit of their other columns (e.g. + microns, divided by the pixel size), no longer always in pixels; `cells_table_kwargs["coordinates_unit"]` sets it. +- An inForm `*_binary_seg_maps.tif` given as `labels` reads its `Nucleus` page (whose labels are the `Cell ID`s) by + default, not its first page (the tissue class map). +- With `x` and `y` in `cells_table_kwargs`, `cells_table_format="auto"` reads a generic table. - Adopted the current `cookiecutter-scverse` template: `hatch`-managed test environments, `uv`-based CI and documentation builds, `mypy` type checking of `src` and `tests`, `biome`/`pyproject-fmt`/`zizmor` pre-commit hooks, and Dependabot updates. diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 4c14a555..b50b18ca 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -95,6 +95,8 @@ _QUPATH_UNIT = re.compile(r" (µm|um|px)(?:\^2)?$") _QUPATH_IMAGE = "Image" _QUPATH_ID = "Object ID" +# QuPath's centroid columns by unit; R's `make.names` writes `Centroid X µm` as `Centroid.X.um` +_QUPATH_CENTROIDS = {"um": ("Centroid X µm", "Centroid Y µm"), "px": ("Centroid X px", "Centroid Y px")} # the QuPath object types of cells; other features of a GeoJSON export (e.g. annotations) are not cells _QUPATH_CELL_TYPES = frozenset({"cell", "detection", "PathCellObject", "PathDetectionObject"}) # inForm `*_cell_seg_data.txt` columns: `Cell X Position (microns)`, `Entire Cell Mean ()`, ... @@ -103,10 +105,12 @@ _INFORM_AREA = re.compile(r"Entire Cell Area \((.*)\)") # the unit of an inForm measurement, e.g. `Tissue Category Area (square microns)` _INFORM_UNIT = re.compile(r"\((?:square )?(microns|pixels)\)$") -# the pages of an inForm `*_binary_seg_maps.tif`: the tissue class map, then one per compartment -_INFORM_TISSUE_MAP = "" +# the pages of an inForm `*_binary_seg_maps.tif`, named by the root tag of their XML description: the tissue class +# map `` (named `Tissue`), and a `` per compartment (named after the compartment) +_INFORM_XML_ROOT = re.compile(r"<([A-Za-z][\w-]*)") +_INFORM_SEGMENTATION = "SegmentationImage" _INFORM_COMPARTMENT = re.compile(r"\s*(\w+)\s*") -_INFORM_TISSUE_PAGE = "Tissue" +_INFORM_PAGE_NAMES = {"TissueClassMap": "Tissue"} # the default mask of an inForm segmentation map: the nuclei carry the `Cell ID` of every cell _INFORM_DEFAULT_MASK = "Nucleus" _INFORM_REGION = "Sample Name" @@ -118,6 +122,8 @@ _CENTROID_TOLERANCE_PX = 1.0 # the largest fraction of matched polygons that may be far from their centroids before the match is rejected _MAX_MISPLACED_FRACTION = 0.05 +# the largest fraction of a region's cells that may lie outside its image without a warning +_MAX_OUTSIDE_FRACTION = 0.05 @dataclass @@ -292,9 +298,9 @@ def detect_table_format(path: Path) -> str: Returns ------- ``"codex"`` (FCS, or ``x``/``y`` with ``cell_id`` or ``cycCCC_chHHH`` columns), ``"qupath"`` (``Centroid X µm``, - ``Centroid X px`` or ``Object ID``, ignoring case and punctuation as in R's ``Centroid.X.um``), ``"mcquant"`` (``CellID`` and ``X_centroid``), ``"inform"`` (``Cell X Position``, optionally - followed by its unit in parentheses), - ``"sprm"`` (first column ``ID``), else ``"generic"``. + ``Centroid X px`` or ``Object ID``, also in R's ``make.names`` form, e.g. ``Centroid.X.um``), ``"mcquant"`` + (``CellID`` and ``X_centroid``), ``"inform"`` (``Cell X Position``, optionally followed by its unit in + parentheses), ``"sprm"`` (first column ``ID``), else ``"generic"``. """ if path.suffix.lower() == CodexKeys.FCS_FILE.value: return "codex" @@ -305,7 +311,8 @@ def detect_table_format(path: Path) -> str: CodexKeys.INSTANCE_KEY.value in names or any(_MARKER_COLUMN.fullmatch(name) for name in names) ): return "codex" - if {"centroidxum", "centroidxpx", _column_key(_QUPATH_ID)} & {_column_key(column) for column in columns}: + qupath = {_QUPATH_ID, *(name for names in _QUPATH_CENTROIDS.values() for name in names)} + if qupath & set(_qupath_names(columns)): return "qupath" if {"CellID", "X_centroid"} <= set(columns): return "mcquant" @@ -331,15 +338,16 @@ def read_cell_table( path The cell table. fmt - One of ``"auto"`` (see :func:`detect_table_format`), ``"codex"``, ``"qupath"``, ``"mcquant"``, + One of ``"auto"`` (see :func:`detect_table_format`; ``"generic"`` if ``cells_table_kwargs`` has ``x`` and + ``y``), ``"codex"``, ``"qupath"``, ``"mcquant"``, ``"sprm"``, ``"inform"`` or ``"generic"``: - ``"codex"``: see :func:`read_codex_table`. - - ``"qupath"``: a QuPath measurement export. Column names are matched ignoring case, spaces and - punctuation, so R's ``make.names`` forms (``Centroid.X.um``, ``Object.ID``) are read too. Centroids - ``Centroid X µm``/``Centroid Y µm`` (divided by ``pixel_size_um``) or ``Centroid X px``/``Centroid Y - px``, else ``X``/``Y`` in ``coordinates_unit`` or, without it, in the unit of the other measurements - (e.g. ``Cell: Area µm^2``; QuPath writes all of them in one unit), raising if that is not unique. + - ``"qupath"``: a QuPath measurement export, also with R's ``make.names`` column names + (``Centroid.X.um``, ``Object.ID``). Centroids ``Centroid X µm``/``Centroid Y µm`` (divided by + ``pixel_size_um``) or ``Centroid X px``/``Centroid Y px``, else ``X``/``Y`` in ``coordinates_unit`` + or, without it, in the unit of the other measurements (e.g. ``Cell: Area µm^2``) with a + `UserWarning`, raising if that is not unique. Instance ids from a numeric ``Object ID``, else the row number (from 1); the region from the ``Image`` column. The markers are ``markers``, else the ``: : Mean`` columns (also ``....Mean`` and ``__Mean``), one per marker from @@ -381,9 +389,9 @@ def read_cell_table( number, from 1), ``markers`` (a list of columns or a regular expression matching the full column name; default: all other numeric columns; also for ``"qupath"`` tables) and ``coordinates_unit`` (``"px"``, default, or ``"um"``; for ``"qupath"`` and ``"inform"`` tables the unit of centroid columns whose name - has none); for ``"qupath"`` tables ``geojson`` (the GeoJSON file of the cell outlines) and ``geojson_units`` (the unit of - its coordinates, ``"px"`` of the full-resolution image, default and what QuPath writes, or ``"um"``). The - key ``mask_channel`` is used by :func:`read_labels`. + has none); for ``"qupath"`` tables ``geojson`` (the GeoJSON file of the cell outlines) and + ``geojson_units`` (the unit of its coordinates, ``"px"`` of the full-resolution image, default and what + QuPath writes, or ``"um"``). The key ``mask_channel`` is used by :func:`read_labels`. Returns ------- @@ -393,7 +401,8 @@ def read_cell_table( if unknown: raise ValueError(f"`cells_table_kwargs` has unknown key(s) {unknown}; expected some of {CELLS_TABLE_KWARGS}.") if fmt == "auto": - fmt = detect_table_format(path) + # centroid columns given by the user make a generic table + fmt = "generic" if {"x", "y"} <= cells_table_kwargs.keys() else detect_table_format(path) logger.info("Reading %s as a %r cell table.", path, fmt) if fmt != "qupath" and cells_table_kwargs.get("geojson") is not None: raise ValueError(f"`cells_table_kwargs['geojson']` is only read with QuPath cell tables; {path} is {fmt!r}.") @@ -464,15 +473,18 @@ def _row_pixel_sizes( return values.map(sizes_by_value).to_numpy(dtype=np.float64) -def _column_key(column: str) -> str: - """``column`` without case, punctuation and spaces, ``µ`` as ``u``: QuPath's ``Centroid X µm`` is R's ``Centroid.X.um``.""" - return re.sub(r"[^0-9a-z]", "", column.replace("µ", "u").replace("μ", "u").casefold()) +def _qupath_names(columns: Sequence[str]) -> dict[str, str]: + """The columns by their QuPath name: as written, or undoing R's ``make.names`` (``Centroid.X.um``).""" + names = {} + for column in columns: + name = column.replace(".", " ") + names[name.removesuffix(" um") + " µm" if name.endswith(" um") else name] = column + return names | {column: column for column in columns} def _object_id_column(columns: Sequence[str]) -> str | None: """The QuPath ``Object ID`` column (also R's ``Object.ID``), or `None`.""" - key = _column_key(_QUPATH_ID) - return next((column for column in columns if _column_key(column) == key), None) + return _qupath_names(columns).get(_QUPATH_ID) def _find_geojson(table_path: Path) -> Path | None: @@ -636,10 +648,10 @@ def _read_geojson(path: Path) -> tuple[np.ndarray | None, np.ndarray, np.ndarray def _qupath_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: - columns = {_column_key(column): column for column in df.columns} - unit = next((unit for unit in ("um", "px") if f"centroidx{unit}" in columns), None) - x_key, y_key = ("x", "y") if unit is None else (f"centroidx{unit}", f"centroidy{unit}") - x, y = columns.get(x_key), columns.get(y_key) + columns = _qupath_names(list(df.columns)) + unit = next((unit for unit, (x_name, _) in _QUPATH_CENTROIDS.items() if x_name in columns), None) + x_name, y_name = ("X", "Y") if unit is None else _QUPATH_CENTROIDS[unit] + x, y = columns.get(x_name), columns.get(y_name) if x is None or y is None: raise ValueError( f"{path} has no centroid columns ('Centroid X µm' and 'Centroid Y µm', 'Centroid X px' and " @@ -651,7 +663,7 @@ def _qupath_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> di object_id = _object_id_column(list(df.columns)) instance_column = object_id if object_id is not None and pd.api.types.is_integer_dtype(df[object_id]) else None region_column = kwargs.get("region_column", _QUPATH_IMAGE if _QUPATH_IMAGE in df.columns else None) - area = columns.get(f"cellarea{unit}2") + area = columns.get(f"Cell: Area {'µm' if unit == 'um' else 'px'}^2") roles = {x, y, object_id, region_column, area} compartments: dict[str, dict[str, str]] = {} @@ -711,7 +723,12 @@ def _qupath_xy_unit(path: Path, columns: list[str], kwargs: Mapping[str, Any]) - "`cells_table_kwargs['coordinates_unit']` to 'px' or 'um'." ) unit = units.pop() - logger.info("Reading the centroids 'X' and 'Y' of %s in %s, the unit of its other measurements.", path, unit) + warnings.warn( + f"The centroid columns 'X' and 'Y' of {path} name no unit; reading them in {unit}, the unit of its other " + "measurements. Set `cells_table_kwargs['coordinates_unit']` to 'px' or 'um' to choose it.", + UserWarning, + stacklevel=6, + ) return unit @@ -1124,12 +1141,12 @@ def read_labels( raise ValueError(f"{path} has dtype {image.dtype}; a segmentation mask must have an integer dtype.") n_masks = image.shape[0] - names = _mask_channel_names(path, n_masks) + names, inform = _mask_channel_names(path, n_masks) if isinstance(mask_channel, str): if mask_channel not in names: raise ValueError(f"{path} has no mask channel {mask_channel!r}; its channel names are {names}.") index = names.index(mask_channel) - elif mask_channel is None and _INFORM_TISSUE_PAGE in names and _INFORM_DEFAULT_MASK in names: + elif mask_channel is None and inform and _INFORM_DEFAULT_MASK in names: index = names.index(_INFORM_DEFAULT_MASK) logger.info( "%s is an inForm segmentation map %s; reading %r. Set `mask_channel` to choose another.", @@ -1155,21 +1172,31 @@ def _labels_models_kwargs(labels_models_kwargs: Mapping[str, Any]) -> tuple[dict return labels_models_kwargs, spatial_chunks(labels_models_kwargs["chunks"], defaults["chunks"]) -def _mask_channel_names(path: Path, n_masks: int) -> list[str]: - """The OME channel names of a mask file, else the page names of an inForm segmentation map, else ``[]``.""" +def _mask_channel_names(path: Path, n_masks: int) -> tuple[list[str], bool]: + """The channel names of a mask file and whether it is an inForm segmentation map. + + The names are the OME channel names, else for an inForm segmentation map (a file with a ```` + page) the page names, see :func:`_inform_page_name`, else ``[]``. + """ with tifffile.TiffFile(path) as tf: if tf.ome_metadata: - return _ome_channel_names(tf.ome_metadata) + return _ome_channel_names(tf.ome_metadata), False if len(tf.pages) != n_masks: - return [] - descriptions = [str(tf.pages[i].description) for i in range(n_masks)] # type: ignore[union-attr] - names = [ - _INFORM_TISSUE_PAGE - if _INFORM_TISSUE_MAP in description - else (match.group(1) if (match := _INFORM_COMPARTMENT.search(description)) is not None else None) - for description in descriptions - ] - return [str(name) for name in names] if all(name is not None for name in names) else [] + return [], False + descriptions = [str(tf.pages[i].description or "") for i in range(n_masks)] # type: ignore[union-attr] + roots = [match.group(1) if (match := _INFORM_XML_ROOT.search(d)) else None for d in descriptions] + if _INFORM_SEGMENTATION not in roots: + return [], False + return [_inform_page_name(d, root, i) for i, (d, root) in enumerate(zip(descriptions, roots, strict=True))], True + + +def _inform_page_name(description: str, root: str | None, index: int) -> str: + """The compartment of a ```` page, else its XML root tag (``Tissue`` for the tissue map).""" + if (compartment := _INFORM_COMPARTMENT.search(description)) is not None: + return compartment.group(1) + if root is not None: + return _INFORM_PAGE_NAMES.get(root, root) + return f"page {index}" def _ome_channel_names(ome_metadata: str) -> list[str]: @@ -1574,14 +1601,14 @@ def _check_cells_in_images(tables: Sequence[CellTable], image_shapes: Mapping[st centroids = centroids[np.isfinite(centroids).all(axis=1)] height, width = image_shapes[region] outside = int(np.count_nonzero(((centroids < 0) | (centroids > [width, height])).any(axis=1))) - if outside > _MAX_MISPLACED_FRACTION * len(centroids): + if outside > _MAX_OUTSIDE_FRACTION * len(centroids): warnings.warn( f"{outside} of the {len(centroids)} cells of region '{region}' lie outside its image ({width} x " f"{height} px): the table's coordinates are in another unit (see " "`cells_table_kwargs['coordinates_unit']`) or relative to another origin (e.g. inForm positions " "in microns, which may be relative to the slide).", UserWarning, - stacklevel=3, + stacklevel=4, ) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 53a6410d..95a421b1 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -193,13 +193,14 @@ def codex( Options for reading ``cells_table`` and ``labels``: ``region_column``; for ``"generic"`` tables ``x``, ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath tables) and ``coordinates_unit`` (``"px"`` or ``"um"``; for QuPath ``X``/``Y`` and inForm ``Cell X Position`` columns - without a unit, which are otherwise read in the unit of the table's other columns); ``mask_channel`` - (index or channel name of the mask in a multi-channel ``labels`` file: an OME channel name, or for an - inForm ``*_binary_seg_maps.tif`` ``"Tissue"``, ``"Nucleus"`` (the default there, its labels are the - ``Cell ID`` of every cell), ``"Cytoplasm"`` or ``"Membrane"``); for QuPath tables ``geojson`` (the GeoJSON export of the cell outlines, a path or a mapping from region id - to path like ``cells_table``; default ``
.geojson`` or ``
_cells.geojson`` next to - the table) and ``geojson_units`` (``"px"`` of the full-resolution image, default and what QuPath writes, - or ``"um"``). + without a unit, which are otherwise read in the unit of the table's other columns); with ``x`` and ``y``, + ``cells_table_format="auto"`` reads a generic table. ``mask_channel`` (index or channel name of the mask in + a multi-channel ``labels`` file: an OME channel name, or for an inForm ``*_binary_seg_maps.tif`` + ``"Tissue"``, ``"Nucleus"`` (the default there, its labels are the ``Cell ID`` of every cell), + ``"Cytoplasm"`` or ``"Membrane"``); for QuPath tables ``geojson`` (the GeoJSON export of the cell + outlines, a path or a mapping from region id to path like ``cells_table``; default ``
.geojson`` + or ``
_cells.geojson`` next to the table) and ``geojson_units`` (``"px"`` of the + full-resolution image, default and what QuPath writes, or ``"um"``). Returns ------- diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 846253e4..8bd8d24b 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -152,13 +152,14 @@ def phenocycler( Options for reading ``cells_table`` and ``labels``: ``region_column``; for ``"generic"`` tables ``x``, ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath tables) and ``coordinates_unit`` (``"px"`` or ``"um"``; for QuPath ``X``/``Y`` and inForm ``Cell X Position`` columns - without a unit, which are otherwise read in the unit of the table's other columns); ``mask_channel`` - (index or channel name of the mask in a multi-channel ``labels`` file: an OME channel name, or for an - inForm ``*_binary_seg_maps.tif`` ``"Tissue"``, ``"Nucleus"`` (the default there, its labels are the - ``Cell ID`` of every cell), ``"Cytoplasm"`` or ``"Membrane"``); for QuPath tables ``geojson`` (the GeoJSON export of the cell outlines, a path or a mapping from region id - to path like ``cells_table``; default ``
.geojson`` or ``
_cells.geojson`` next to - the table) and ``geojson_units`` (``"px"`` of the full-resolution image, default and what QuPath writes, - or ``"um"``). + without a unit, which are otherwise read in the unit of the table's other columns); with ``x`` and ``y``, + ``cells_table_format="auto"`` reads a generic table. ``mask_channel`` (index or channel name of the mask in + a multi-channel ``labels`` file: an OME channel name, or for an inForm ``*_binary_seg_maps.tif`` + ``"Tissue"``, ``"Nucleus"`` (the default there, its labels are the ``Cell ID`` of every cell), + ``"Cytoplasm"`` or ``"Membrane"``); for QuPath tables ``geojson`` (the GeoJSON export of the cell + outlines, a path or a mapping from region id to path like ``cells_table``; default ``
.geojson`` + or ``
_cells.geojson`` next to the table) and ``geojson_units`` (``"px"`` of the + full-resolution image, default and what QuPath writes, or ``"um"``). Returns ------- diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index d5346b00..c346bcda 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -622,7 +622,10 @@ def test_qupath_xy_centroids_and_bare_markers(tmp_path: Path) -> None: path.write_text("\r\n".join([_QUPATH_XY_HEADER, *rows]) + "\r\n", encoding="utf-8") assert detect_table_format(path) == "qupath" - with pytest.warns(UserWarning, match=r"reading the 3 other numeric columns \['CD44', 'HLA-DR-AF647', 'CD45RO'\]"): + with ( + pytest.warns(UserWarning, match=r"reading the 3 other numeric columns \['CD44', 'HLA-DR-AF647', 'CD45RO'\]"), + pytest.warns(UserWarning, match=r"'X' and 'Y' .* name no unit; reading them in um.*coordinates_unit"), + ): table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) # the other measurements are in µm, so the centroids are too @@ -642,7 +645,7 @@ def test_qupath_xy_centroids_and_bare_markers(tmp_path: Path) -> None: "qupath", pixel_size_um=0.5, default_region=None, - cells_table_kwargs={"region_column": "Parent", "markers": ["CD44", "CD45RO"]}, + cells_table_kwargs={"region_column": "Parent", "markers": ["CD44", "CD45RO"], "coordinates_unit": "um"}, ) assert list(table.region) == ["A-1", "A-2"] assert list(table.var.index) == ["CD44", "CD45RO"] @@ -672,17 +675,19 @@ def test_cells_outside_the_image_warn(tmp_path: Path) -> None: ) -def _write_inform_seg_maps(path: Path, shape: tuple[int, int] = (16, 24)) -> np.ndarray: +def _write_inform_seg_maps( + path: Path, shape: tuple[int, int] = (16, 24), first: str = "TissueClassMap", compartments: int = 3 +) -> np.ndarray: # inForm `*_binary_seg_maps.tif`: the tissue class map, then one page per compartment, each with its XML - pages = np.stack([np.full(shape, i, dtype=np.uint16) for i in range(4)]) descriptions = [ - '\n\n 1\n', + f'\n<{first}>\n 1\n', *( '\n\n 1\n' f" {compartment}\n" - for compartment in ("Nucleus", "Cytoplasm", "Membrane") + for compartment in ("Nucleus", "Cytoplasm", "Membrane")[:compartments] ), ] + pages = np.stack([np.full(shape, i, dtype=np.uint16) for i in range(len(descriptions))]) with tifffile.TiffWriter(path) as tw: for page, description in zip(pages, descriptions, strict=True): tw.write(page, description=description, metadata=None) @@ -702,6 +707,56 @@ def test_read_labels_inform_seg_maps(tmp_path: Path, mask_channel: int | str | N read_labels(path, {}, {"global": Identity()}, mask_channel="cells") +def test_read_labels_inform_seg_maps_other_pages(tmp_path: Path) -> None: + # pages are named by their XML root tag; the nuclei are the default without a tissue class map too + path = tmp_path / "S_binary_seg_maps.tif" + pages = _write_inform_seg_maps(path, first="ProcessRegionMap", compartments=2) + + labels = read_labels(path, {"scale_factors": None}, {"global": Identity()}) + + np.testing.assert_array_equal(labels.values, pages[1]) + with pytest.raises(ValueError, match=r"\['ProcessRegionMap', 'Nucleus', 'Cytoplasm'\]"): + read_labels(path, {}, {"global": Identity()}, mask_channel="Membrane") + + +@pytest.mark.parametrize( + ("header", "x", "y"), [("object_id,x,y,CD3", "x", "y"), ("Object Id,XMin,XMax,cx,cy,CD3", "cx", "cy")] +) +def test_object_id_lookalikes_stay_generic(tmp_path: Path, header: str, x: str, y: str) -> None: + # only QuPath's own `Object ID` (or R's `Object.ID`) makes a QuPath table, e.g. not HALO's `Object Id` + path = tmp_path / "cells.csv" + row = "7,10,5,2" if header.startswith("object_id") else "7,0,20,10,5,2" + path.write_text(f"{header}\n{row}\n", encoding="utf-8") + + assert detect_table_format(path) == "generic" + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + table = read_cell_table( + path, "auto", pixel_size_um=0.5, default_region="S", cells_table_kwargs={"x": x, "y": y} + ) + + assert table.centroids_px is not None + np.testing.assert_allclose(table.centroids_px, [[10, 5]]) + assert "CD3" in table.var.index + + +def test_auto_with_centroid_kwargs_is_generic(tmp_path: Path) -> None: + # user-given centroid columns win over the detected format + path = tmp_path / "measurements.tsv" + _write_qupath(path) + + table = read_cell_table( + path, + "auto", + pixel_size_um=0.5, + default_region=None, + cells_table_kwargs={"x": "Centroid X µm", "y": "Centroid Y µm", "region_column": "Image", "markers": "CD.*"}, + ) + + assert table.centroids_px is not None + np.testing.assert_allclose(table.centroids_px, [[10, 5], [20, 15]]) + + def test_unknown_format_raises(tmp_path: Path) -> None: path = tmp_path / "cells.csv" path.write_text("ID,DAPI\n1,2\n") diff --git a/tests/test_akoya_real_data.py b/tests/test_akoya_real_data.py index aa8ffbc5..3fd895e2 100644 --- a/tests/test_akoya_real_data.py +++ b/tests/test_akoya_real_data.py @@ -1,15 +1,17 @@ """Tests of the Akoya readers on real public datasets. They run only when the environment variable ``AKOYA_TEST_DATA`` points to a directory holding the files below (paths -relative to it). The files are public (Zenodo, Figshare, BioImage Archive, Dryad, GitHub, the MCMICRO bucket) but too large to download -in CI. +relative to it). The files are public (Zenodo, Figshare, BioImage Archive, Dryad, GitHub, the MCMICRO bucket) but +too large to download in CI. """ from __future__ import annotations import os +import warnings from dataclasses import dataclass from pathlib import Path +from typing import TYPE_CHECKING import dask.array as da import numpy as np @@ -20,6 +22,9 @@ from spatialdata_io import codex, phenocycler from spatialdata_io.readers._utils._cell_tables import detect_table_format, read_cell_table +if TYPE_CHECKING: + from collections.abc import Iterable + _ROOT = os.environ.get("AKOYA_TEST_DATA") pytestmark = [ @@ -273,16 +278,23 @@ def _export(relative: str) -> Path: return path +def _assert_cells_inside_images(caught: Iterable[warnings.WarningMessage]) -> None: + assert not [w for w in caught if "outside its image" in str(w.message)] + + def test_real_inform_table_with_seg_maps_and_component_data() -> None: # inForm 2.3 (github.com/akoyabio/phenoptr): positions in pixels, the `Cell ID`s are the labels of page 1 (nuclei) # of the segmentation map; page 0 is the tissue class map image = _export(f"{_PHENOPTR}_component_data.tif") - sdata = phenocycler( - image, - cells_table=_export(f"{_PHENOPTR}_cell_seg_data.txt"), - labels=_export(f"{_PHENOPTR}_binary_seg_maps.tif"), - ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + sdata = phenocycler( + image, + cells_table=_export(f"{_PHENOPTR}_cell_seg_data.txt"), + labels=_export(f"{_PHENOPTR}_binary_seg_maps.tif"), + ) + _assert_cells_inside_images(caught) region = "Set4_1-6plex__16142_55840__component_data" table = sdata.tables["table"] @@ -334,8 +346,9 @@ def test_real_qupath_r_exported_coordinates_with_geojson(tmp_path: Path) -> None assert pd.notna(table.polygons_px).all() assert len(table.obs) == 9415 - # joined on the object id, the measurements are the markers, from the `Cell` compartment (`DAPI_Cell_Mean`); the - # expression file alone has no centroids in QuPath's form (its `Y_coord` is negated) + # joined on the object id, the measurements are the markers: 30 markers in the `Cell` compartment (also 27 + # `Cytoplasm`, 27 `Membrane`, 3 `Nucleus`), the first one taken; the expression file alone has no centroids in + # QuPath's form (its `X_coord`/`Y_coord` are µm, `Y_coord` negated) joined = tmp_path / "C-7.csv" pd.read_csv(coords).merge(pd.read_csv(expression), on="Object.ID", validate="1:1").to_csv(joined, index=False) with pytest.warns(UserWarning, match="116 of the 9531 cell features"): @@ -370,8 +383,9 @@ def test_real_mcquant_table_with_ome_tiff() -> None: image = _export(f"{_EXPORTS}/s3_mcmicro_exemplar001/exemplar-001-cycle6.ome.tif") cells = _export(f"{_EXPORTS}/s3_mcmicro_exemplar001/unmicst-exemplar-001_cell.csv") - with pytest.warns(UserWarning, match="cells are represented as circles"): + with pytest.warns(UserWarning, match="cells are represented as circles") as caught: sdata = codex(image, cells_table=cells) + _assert_cells_inside_images(caught) table = sdata.tables["table"] assert table.shape == (11170, 12) From 9330eb5efddff86383731ed735d8a38f8763c373 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Sat, 17 Oct 2026 00:04:14 +0200 Subject: [PATCH 037/107] fix: match R-exported QuPath measurement names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Undo `make.names` for measurement names (`Cell..Area.µm.2` is `Cell: Area µm^2`), so R-form tables keep their cell radius and give the unit of `X`/`Y` centroids. Co-Authored-By: Claude Opus 5.5 --- .../readers/_utils/_cell_tables.py | 12 +++++++---- tests/readers/test_cell_tables.py | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index b50b18ca..a1949cf6 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -474,11 +474,15 @@ def _row_pixel_sizes( def _qupath_names(columns: Sequence[str]) -> dict[str, str]: - """The columns by their QuPath name: as written, or undoing R's ``make.names`` (``Centroid.X.um``).""" + """The columns by their QuPath name: as written, or undoing R's ``make.names``. + + ``make.names`` replaces ``: ``, `` `` and ``^`` by dots: ``Centroid.X.um`` is ``Centroid X µm`` and + ``Cell..Area.µm.2`` is ``Cell: Area µm^2``. + """ names = {} for column in columns: - name = column.replace(".", " ") - names[name.removesuffix(" um") + " µm" if name.endswith(" um") else name] = column + name = re.sub(r"\.2$", "^2", column.replace("..", ": ")).replace(".", " ") + names[re.sub(r" um(?=\^2$|$)", " µm", name)] = column return names | {column: column for column in columns} @@ -659,7 +663,7 @@ def _qupath_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> di "the measurements into two files, join them on the object id first." ) if unit is None: - unit = _qupath_xy_unit(path, list(df.columns), kwargs) + unit = _qupath_xy_unit(path, list(columns), kwargs) object_id = _object_id_column(list(df.columns)) instance_column = object_id if object_id is not None and pd.api.types.is_integer_dtype(df[object_id]) else None region_column = kwargs.get("region_column", _QUPATH_IMAGE if _QUPATH_IMAGE in df.columns else None) diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index c346bcda..05bd3731 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -597,6 +597,26 @@ def test_qupath_r_exported_marker_columns(tmp_path: Path) -> None: np.testing.assert_allclose(table.X, [[1, 2, 3, 4]]) +@pytest.mark.parametrize("area", ["Cell..Area.µm.2", "Cell..Area.um.2"]) +def test_qupath_r_exported_area(tmp_path: Path, area: str) -> None: + # `make.names` of `Cell: Area µm^2`; areas of pi and 4 pi µm^2 at 0.5 µm/px are circles of radius 2 and 4 px + path = tmp_path / "cells.csv" + path.write_text(f"{_R_COORDS_HEADER},{area}\nab,2,2,C-7,3.14159\ncd,12,12,C-7,12.56637\n", encoding="utf-8") + + table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region="C-7", cells_table_kwargs={}) + + assert table.radius_px is not None + np.testing.assert_allclose(table.radius_px, [2, 4], rtol=1e-5) + + # the unit of R-form measurements gives that of `X`/`Y` too + path.write_text(f"Object.ID,X,Y,{area}\nab,2,2,3.14159\n", encoding="utf-8") + with pytest.warns(UserWarning, match="reading them in um"): + table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region="C-7", cells_table_kwargs={}) + assert table.centroids_px is not None + np.testing.assert_allclose(table.centroids_px, [[4, 4]]) + assert table.radius_px is not None + + def test_qupath_expression_without_centroids_raises(tmp_path: Path) -> None: path = tmp_path / "C-7_expression.csv" path.write_text("Object.ID,DAPI_Nucleus_Mean,Ki67_Nucleus_Mean\nab,1,2\n", encoding="utf-8") From e8d6a04155af888d4c4e8975b404d99e5bf22233 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Sat, 17 Oct 2026 00:25:18 +0200 Subject: [PATCH 038/107] fix: match raw region names against sanitised region ids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Region ids are now made valid element names with spatialdata's sanitize_name, and cell-table region values are matched both raw and sanitised, so a QuPath `Image` column holding `A_[1,2].qptiff` assigns its cells (and µm pixel sizes) to region `A__1_2_` instead of dropping them or raising. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 ++-- .../readers/_utils/_cell_tables.py | 13 +++++++--- src/spatialdata_io/readers/_utils/_qptiff.py | 8 +++--- src/spatialdata_io/readers/phenocycler.py | 9 ++++--- tests/test_phenocycler.py | 26 +++++++++++++++++++ 5 files changed, 47 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 174491b6..67a52457 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,8 +38,9 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. mask pixels, by the same factor; the source columns in `obs` are kept as read. Other shape mismatches still warn. - `phenocycler()` officially supports the PerkinElmer/Akoya QPTIFFs of Vectra Polaris and PhenoImager HT (generation `"qpi_other"`): Opal and unmixed (uint16) scans, brightfield RGB scans (channels `r`, `g`, `b`) and inForm component - data (float32). A single `.tif` file is accepted if it is a QPI file. Characters of a file name that are not valid in - an element name (e.g. the `[x,y]` of inForm tiles) are replaced by `_` in the region id. + data (float32). A single `.tif` file is accepted if it is a QPI file. Region ids are made valid element names with + `spatialdata.sanitize_name` (e.g. the `[x,y]` of inForm tiles: `A_[1,2]` becomes `A__1_2_`); the region column of a + cell table may hold the raw file names. - The Akoya cell-table adapters read real QuPath, inForm and mcquant exports without column mapping: R-exported QuPath names (`Object.ID`, `Centroid.X.um`, `__Mean`), QuPath `X`/`Y` centroids (in the unit of the other measurements with a warning, or `cells_table_kwargs["coordinates_unit"]`) with bare marker names diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index a1949cf6..9c1b317a 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -1104,10 +1104,15 @@ def _assign_regions( def _match_region(value: str, regions: Sequence[str]) -> str | None: - """The region named by ``value``: equal to it, else the longest one it starts with before a non-alphanumeric.""" - if value in regions: - return value - prefixes = [r for r in regions if value.startswith(r) and not value[len(r)].isalnum()] + """The region named by ``value``: equal to it, else the longest one it starts with before a non-alphanumeric. + + ``value`` is also matched after :func:`spatialdata.sanitize_name`, as the region ids are, so that e.g. the + QuPath ``Image`` value ``A_[1,2].qptiff`` names the region ``A__1_2_``. + """ + candidates = list(dict.fromkeys((value, sanitize_name(value)))) + if exact := next((candidate for candidate in candidates if candidate in regions), None): + return exact + prefixes = [r for r in regions for c in candidates if c.startswith(r) and not c[len(r)].isalnum()] return max(prefixes, key=len, default=None) diff --git a/src/spatialdata_io/readers/_utils/_qptiff.py b/src/spatialdata_io/readers/_utils/_qptiff.py index 253e7674..46c88a16 100644 --- a/src/spatialdata_io/readers/_utils/_qptiff.py +++ b/src/spatialdata_io/readers/_utils/_qptiff.py @@ -12,6 +12,7 @@ import dask.array as da import tifffile +from spatialdata import sanitize_name from spatialdata_io._constants._constants import PhenocyclerKeys from spatialdata_io.readers._utils._akoya import lazy_tiff_array @@ -33,8 +34,6 @@ _FULL_RESOLUTION = "FullResolution" _SCAN_SUFFIX = re.compile(r"_Scan\d+$") -# element names may only hold these characters; inForm names its files e.g. ``_[13860,52919]_component_data.tif`` -_INVALID_REGION_CHARS = re.compile(r"[^0-9A-Za-z_.-]") _XML_DECLARATION = re.compile(r"^\s*<\?xml[^>]*\?>") # micrometres per unit of the TIFF ResolutionUnit; XPosition/YPosition are expressed in the same unit _UM_PER_RESOLUTION_UNIT = {tifffile.RESUNIT.CENTIMETER: 1e4, tifffile.RESUNIT.INCH: 25400.0} @@ -294,10 +293,11 @@ def qptiff_regions_with_duplicates(files: Sequence[Path]) -> tuple[dict[str, Pat # an 8-bit file is only a duplicate if the scan is read from an extended-range file eight_bit = {key: file for key, file in eight_bit.items() if _is_extended_range(scans[key])} bases = [base for _, base in scans] - stripped = [_INVALID_REGION_CHARS.sub("_", _SCAN_SUFFIX.sub("", base) or base) for base in bases] + # region ids must be valid element names; inForm names its files e.g. `_[13860,52919]_component_data.tif` + stripped = [sanitize_name(_SCAN_SUFFIX.sub("", base) or base) for base in bases] # stripping `_Scan` must not merge two scans into one region regions = [ - _INVALID_REGION_CHARS.sub("_", base) if stripped.count(region) > 1 else region + sanitize_name(base) if stripped.count(region) > 1 else region for base, region in zip(bases, stripped, strict=True) ] if len(set(regions)) < len(regions): diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 8bd8d24b..989e1e82 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -78,8 +78,9 @@ def phenocycler( in a ``raw/images/phenocycler/`` folder below ``path``) are ignored. File suffixes are matched ignoring case. The region name is the file name without the ``{vx.ER_QPTIFF_SUFFIX!r}``/``{vx.QPTIFF_SUFFIX!r}`` suffix and - without a trailing ``_Scan``, unless that makes two regions share a name; characters other than letters, digits, - ``_``, ``.`` and ``-`` are replaced by ``_``. Each region is stored as the image + without a trailing ``_Scan``, unless that makes two regions share a name, made a valid element name by + :func:`spatialdata.sanitize_name` (characters other than letters, digits, ``_``, ``.`` and ``-`` are replaced by + ``_``, e.g. ``A_[1,2]`` becomes ``A__1_2_``). Each region is stored as the image ``{vx.IMAGE_SUFFIX}``. A single region is stored in the ``"global"`` coordinate system. Several regions of the same sample space (scanned on the same slide, with stage positions and equal pixel sizes) are placed at their stage positions in ``"global"``; otherwise the regions are unrelated and each is stored in its own @@ -122,8 +123,8 @@ def phenocycler( cells_table A cell table: a path, or a mapping from region id to path, e.g. ``{{"A": "A_cells.csv"}}``. A single path for several regions needs a region column (``Image`` for QuPath, ``region`` for CODEX tables, else - ``cells_table_kwargs["region_column"]``); its values are matched to the region ids, e.g. - ``A_Scan1.qptiff`` to ``A``. + ``cells_table_kwargs["region_column"]``); its values are matched to the region ids, also after + :func:`spatialdata.sanitize_name`, e.g. ``A_Scan1.qptiff`` to ``A`` and ``A_[1,2].qptiff`` to ``A__1_2_``. cells_table_format The format of ``cells_table``: ``"auto"`` (detected from the header), ``"codex"`` (CODEX Processor), ``"qupath"`` (QuPath measurements; µm centroids are converted with the region's pixel size), ``"mcquant"`` diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 6a7c515a..31eff6a6 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -641,6 +641,32 @@ def test_single_qupath_table_many_regions(tmp_path: Path) -> None: assert read_zarr(tmp_path / "x.zarr").tables["table"].n_obs == 2 +@pytest.mark.parametrize("unit", ["px", "µm"]) +def test_single_qupath_table_matches_raw_names_of_sanitised_regions(tmp_path: Path, unit: str) -> None: + # the region ids are sanitised (`A_[1,2]` -> `A__1_2_`), the table's `Image` column holds the raw file names + for name in ("A_[1,2]", "B [3,4]"): + write_qptiff(tmp_path / f"{name}.qptiff", biomarkers=["DAPI"]) + header = f"Image\tObject ID\tCentroid X {unit}\tCentroid Y {unit}\tCD3: Cell: Mean" + rows = ["A_[1,2].qptiff\tu1\t5\t5\t1", "B [3,4].qptiff\tu2\t10\t10\t2"] + (tmp_path / "cells.tsv").write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") + + with pytest.warns(UserWarning, match="circles"): + sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) + + table = sdata.tables["table"] + assert list(table.obs["region"]) == ["A__1_2__cells", "B__3_4__cells"] + scale = 1 / 0.5069 if unit == "µm" else 1 + np.testing.assert_allclose(np.asarray(table.obsm["spatial"]), np.array([[5, 5], [10, 10]]) * scale) + + +def test_regions_colliding_after_sanitising_raise(tmp_path: Path) -> None: + write_qptiff(tmp_path / "A [1].qptiff", biomarkers=["DAPI"]) + write_qptiff(tmp_path / "A_[1].qptiff", biomarkers=["DAPI"]) + + with pytest.raises(ValueError, match=r"same region\(s\) \['A__1_'\]"): + phenocycler(tmp_path, image_models_kwargs=_FAST) + + def test_single_qupath_table_regions_with_different_pixel_sizes(tmp_path: Path) -> None: # scans of the same instrument differ in the 4th decimal; each cell is converted with its own scan's pixel size sizes = {"A": 0.5069, "B": 0.5079} From 60973fb3530e2437111b5a7e03741c3a1e9960ef Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Sat, 17 Oct 2026 00:28:50 +0200 Subject: [PATCH 039/107] fix: sanitise user-given region ids and mapping keys `regions=["Slide 1"]` and `cells_table={"Slide 1": ...}` now select the region `Slide_1`, since region ids are sanitised with sanitize_name. Keys that collide after sanitising raise a ValueError. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- src/spatialdata_io/readers/_utils/_akoya.py | 9 ++++-- .../readers/_utils/_cell_tables.py | 13 +++++++-- src/spatialdata_io/readers/phenocycler.py | 4 ++- tests/test_phenocycler.py | 29 +++++++++++++++++++ 5 files changed, 49 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67a52457..7099bb6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,7 +40,7 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. `"qpi_other"`): Opal and unmixed (uint16) scans, brightfield RGB scans (channels `r`, `g`, `b`) and inForm component data (float32). A single `.tif` file is accepted if it is a QPI file. Region ids are made valid element names with `spatialdata.sanitize_name` (e.g. the `[x,y]` of inForm tiles: `A_[1,2]` becomes `A__1_2_`); the region column of a - cell table may hold the raw file names. + cell table, `regions` and the keys of the `cells_table`/`labels` mappings may hold the raw names. - The Akoya cell-table adapters read real QuPath, inForm and mcquant exports without column mapping: R-exported QuPath names (`Object.ID`, `Centroid.X.um`, `__Mean`), QuPath `X`/`Y` centroids (in the unit of the other measurements with a warning, or `cells_table_kwargs["coordinates_unit"]`) with bare marker names diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index d9885a06..a0d9f248 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -17,6 +17,7 @@ import zarr from dask.base import tokenize from dask_image.imread import imread +from spatialdata import sanitize_name from spatialdata._logging import logger from spatialdata.models import Image2DModel from spatialdata.transformations import BaseTransformation, Identity, Translation @@ -176,17 +177,19 @@ def scans_to_attrs(scans: Sequence[AkoyaScan], channel_names: Mapping[str, list[ def normalize_regions(regions: str | Sequence[str] | None) -> list[str] | None: """Validate the ``regions`` argument of a reader: a region id or a sequence of them, without duplicates. - Raises a `TypeError` for a mapping and a `ValueError` for an empty sequence. + The region ids are sanitised with :func:`spatialdata.sanitize_name`, like those of the datasets, so raw names + (e.g. ``"Slide 1"`` for the region ``Slide_1``) select their region. Raises a `TypeError` for a mapping and a + `ValueError` for an empty sequence. """ if regions is None: return None if isinstance(regions, str): - return [regions] + return [sanitize_name(regions)] if isinstance(regions, Mapping): raise TypeError(f"`regions` must be a sequence of region ids, got {type(regions).__name__}.") if not regions: raise ValueError("`regions` is empty; pass `None` to read all regions.") - return list(dict.fromkeys(regions)) + return list(dict.fromkeys(sanitize_name(str(region)) for region in regions)) def select_regions(available: Sequence[str], regions: str | Sequence[str] | None) -> list[str]: diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 9c1b317a..62a0a9f8 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -996,7 +996,8 @@ def resolve_per_region( Parameters ---------- value - A path, a mapping from region id to path, or `None`. + A path, a mapping from region id to path, or `None`. The keys are sanitised with + :func:`spatialdata.sanitize_name`, like the region ids. regions The regions read. available @@ -1010,10 +1011,16 @@ def resolve_per_region( if value is None: return {} if isinstance(value, Mapping): - unknown = sorted(set(value) - set(regions) - set(available)) + by_region = {sanitize_name(str(key)): file for key, file in value.items()} + if len(by_region) < len(value): + raise ValueError( + f"Several of the keys {list(value)} name the same region after `spatialdata.sanitize_name`; pass " + "each region once." + ) + unknown = sorted(set(by_region) - set(regions) - set(available)) if unknown: raise ValueError(f"Unknown region(s) {unknown}. Available regions: {list(available or regions)}.") - files = {region: Path(value[region]) for region in regions if region in value} + files = {region: Path(by_region[region]) for region in regions if region in by_region} else: files = {regions[0] if len(regions) == 1 else "*": Path(value)} for file in files.values(): diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 989e1e82..ae2e7dd2 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -116,7 +116,9 @@ def phenocycler( Path to a ``{vx.QPTIFF_SUFFIX!r}`` file (or an inForm ``.tif`` QPI file), or to a directory that is searched recursively for ``{vx.QPTIFF_SUFFIX!r}`` files. regions - Regions to read. If `None`, all regions are read. + Regions to read. If `None`, all regions are read. Region ids, like the keys of the ``cells_table``, + ``labels`` and ``cells_table_kwargs["geojson"]`` mappings, may also be given as raw names, which are + sanitised like the region ids (e.g. ``"Slide 1"`` selects the region ``Slide_1``). channel_names Channel names to use instead of the ones stored in the qptiff; must have one entry per channel of every region read. diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 31eff6a6..3d68b669 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -500,6 +500,35 @@ def test_mapping_keys_of_unselected_regions_are_ignored(tmp_path: Path) -> None: assert sdata.attrs["akoya"]["cell_tables"] == {"A": str(tmp_path / "A.csv")} +def test_raw_region_names_select_sanitised_regions(tmp_path: Path) -> None: + write_qptiff(tmp_path / "Slide 1.qptiff", biomarkers=["DAPI", "CD3"]) + write_qptiff(tmp_path / "Slide 2.qptiff", biomarkers=["DAPI", "CD3"]) + _write_mcquant(tmp_path / "cells.csv") + + with pytest.warns(UserWarning, match="circles"): + sdata = phenocycler( + tmp_path, + regions=["Slide 1"], + cells_table={"Slide 1": tmp_path / "cells.csv"}, + image_models_kwargs=_FAST, + ) + + assert list(sdata.images) == ["Slide_1_image"] + assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["Slide_1_cells"] + + +def test_mapping_keys_colliding_after_sanitising_raise(tmp_path: Path) -> None: + write_qptiff(tmp_path / "Slide_1.qptiff", biomarkers=["DAPI", "CD3"]) + _write_mcquant(tmp_path / "cells.csv") + + with pytest.raises(ValueError, match="same region"): + phenocycler( + tmp_path, + cells_table={"Slide 1": tmp_path / "cells.csv", "Slide_1": tmp_path / "cells.csv"}, + image_models_kwargs=_FAST, + ) + + def test_mask_shape_mismatch_warns(tmp_path: Path) -> None: write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) _write_mask(tmp_path / "mask.tif", shape=(200, 300)) From d3e9bc7f74936163d6ad17f89e43c1c281b09bc0 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Sat, 17 Oct 2026 00:33:14 +0200 Subject: [PATCH 040/107] fix: read generic pixel tables in image pixels by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generic tables are no longer assumed to be in mask pixels, which scaled image-pixel centroids (e.g. a QuPath export mapped with x/y) when the mask is downsampled. `cells_table_kwargs["coordinates_space"]="mask"` opts in to the mcquant/SPRM scaling; invalid values, "mask" with µm centroids and use with non-generic tables raise a ValueError. The CHANGELOG folds the adapter defaults of the new readers into "Added" and drops the duplicated codex parameter bullet. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 25 +++++------- .../readers/_utils/_cell_tables.py | 31 ++++++++++++-- src/spatialdata_io/readers/codex.py | 16 ++++---- src/spatialdata_io/readers/phenocycler.py | 16 ++++---- tests/readers/test_cell_tables.py | 40 +++++++++++++++++++ 5 files changed, 96 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7099bb6f..1ab59452 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,8 +34,9 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. of the `.er` region. By default only the extended-range file is read. - The Akoya readers (`phenocycler()`, `codex()`) scale a segmentation mask downsampled from its region image by an integer factor (exactly, or rounded up) onto the image (a `Scale` before the region's transformation), and scale - the centroids (`obsm["spatial"]`) and circle radii of `mcquant`, SPRM and generic pixel tables, taken to be in - mask pixels, by the same factor; the source columns in `obs` are kept as read. Other shape mismatches still warn. + the centroids (`obsm["spatial"]`) and circle radii of `mcquant` and SPRM tables, which are in mask pixels, by the + same factor; the source columns in `obs` are kept as read. Generic pixel tables are in image pixels unless + `cells_table_kwargs["coordinates_space"]="mask"`. Other shape mismatches warn. - `phenocycler()` officially supports the PerkinElmer/Akoya QPTIFFs of Vectra Polaris and PhenoImager HT (generation `"qpi_other"`): Opal and unmixed (uint16) scans, brightfield RGB scans (channels `r`, `g`, `b`) and inForm component data (float32). A single `.tif` file is accepted if it is a QPI file. Region ids are made valid element names with @@ -45,17 +46,15 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. QuPath names (`Object.ID`, `Centroid.X.um`, `__Mean`), QuPath `X`/`Y` centroids (in the unit of the other measurements with a warning, or `cells_table_kwargs["coordinates_unit"]`) with bare marker names (read with a warning, or chosen with `cells_table_kwargs["markers"]`), and inForm column names with trailing - spaces. `mask_channel` accepts the page names of an inForm `*_binary_seg_maps.tif` (`"Tissue"`, `"Nucleus"`, - `"Cytoplasm"`, `"Membrane"`). QuPath GeoJSON cells without a table row are skipped with a warning, and a region - whose cells lie mostly outside its image raises a warning. + spaces; an inForm `Cell X Position` that names no unit is read in the unit of the table's other columns (e.g. + microns, divided by the pixel size), or in `cells_table_kwargs["coordinates_unit"]`. `mask_channel` accepts the + page names of an inForm `*_binary_seg_maps.tif` (`"Tissue"`, `"Nucleus"`, `"Cytoplasm"`, `"Membrane"`); such a + file reads its `Nucleus` page (whose labels are the `Cell ID`s) by default. With `x` and `y` in + `cells_table_kwargs`, `cells_table_format="auto"` reads a generic table. QuPath GeoJSON cells without a table row + are skipped with a warning, and a region whose cells lie mostly outside its image raises a warning. ### Changed -- inForm cell tables whose `Cell X Position` names no unit are read in the unit of their other columns (e.g. - microns, divided by the pixel size), no longer always in pixels; `cells_table_kwargs["coordinates_unit"]` sets it. -- An inForm `*_binary_seg_maps.tif` given as `labels` reads its `Nucleus` page (whose labels are the `Cell ID`s) by - default, not its first page (the tissue class map). -- With `x` and `y` in `cells_table_kwargs`, `cells_table_format="auto"` reads a generic table. - Adopted the current `cookiecutter-scverse` template: `hatch`-managed test environments, `uv`-based CI and documentation builds, `mypy` type checking of `src` and `tests`, `biome`/`pyproject-fmt`/`zizmor` pre-commit hooks, and Dependabot updates. @@ -68,13 +67,11 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. (previously `cycCCC_chHHH`); the image channels are named too. - The positional signature `codex(path, fcs, imread_kwargs)` is unchanged; all new parameters (`regions`, `channel_names`, `cells_table`, `cells_table_format`, `labels` and the `*_models_kwargs` / `cells_table_kwargs`) - are keyword-only. An empty `regions` raises a `ValueError`. `imread_kwargs` is only used by the - `dask_image.imread` fallback for TIFFs that `tifffile` cannot read lazily. + are keyword-only, and the `codex` CLI command exposes them. An empty `regions` raises a `ValueError`. + `imread_kwargs` is only used by the `dask_image.imread` fallback for TIFFs that `tifffile` cannot read lazily. - A single region is stored in `"global"`. Several regions are placed at their stage positions in `"global"` only when all of them share a sample space (same slide id, stage positions and pixel size); otherwise each region, including multi-region reads of unrelated regions, gets its own coordinate system named after the region. - - New parameters `channel_names`, `cells_table`, `cells_table_format`, `labels`, `image_models_kwargs`, - `labels_models_kwargs` and `cells_table_kwargs`; the `codex` CLI command exposes them. - `channelNames.txt` and `experiment.json` are found in any case (e.g. SPACEc's `channelnames.txt`); CSV files that are not CODEX Processor cell tables are ignored, also when their name holds a region id. - A CIM `.qptiff` without `regNNN` in its name is named as `phenocycler()` names it (without the suffix and a diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 62a0a9f8..01f705c3 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -74,6 +74,7 @@ "region_column", "markers", "coordinates_unit", + "coordinates_space", "mask_channel", "geojson", "geojson_units", @@ -150,8 +151,9 @@ class CellTable: The radius of a circle with the cell's area, or `None` if the table has no area. mask_pixels Whether ``centroids_px`` and ``radius_px`` are in pixels of the segmentation mask (pixel coordinates of - mask-based pipelines: mcquant, SPRM, ``"generic"`` in ``"px"``) rather than of the image; they are scaled - to image pixels when the mask is a downsample of the image, see :func:`build_spatialdata`. + mask-based pipelines: mcquant, SPRM, ``"generic"`` with ``coordinates_space="mask"``) rather than of the + image; they are scaled to image pixels when the mask is a downsample of the image, see + :func:`build_spatialdata`. row_instance_ids Whether the instance ids are row numbers because the table has no instance id column. polygons_px @@ -389,7 +391,9 @@ def read_cell_table( number, from 1), ``markers`` (a list of columns or a regular expression matching the full column name; default: all other numeric columns; also for ``"qupath"`` tables) and ``coordinates_unit`` (``"px"``, default, or ``"um"``; for ``"qupath"`` and ``"inform"`` tables the unit of centroid columns whose name - has none); for ``"qupath"`` tables ``geojson`` (the GeoJSON file of the cell outlines) and + has none) and ``coordinates_space`` (the pixels of ``"px"`` centroids: ``"image"``, default, those of the + full-resolution region image, or ``"mask"``, those of the region's mask, scaled to image pixels like the + ``"mcquant"`` and ``"sprm"`` centroids if the mask is downsampled); for ``"qupath"`` tables ``geojson`` (the GeoJSON file of the cell outlines) and ``geojson_units`` (the unit of its coordinates, ``"px"`` of the full-resolution image, default and what QuPath writes, or ``"um"``). The key ``mask_channel`` is used by :func:`read_labels`. @@ -406,6 +410,10 @@ def read_cell_table( logger.info("Reading %s as a %r cell table.", path, fmt) if fmt != "qupath" and cells_table_kwargs.get("geojson") is not None: raise ValueError(f"`cells_table_kwargs['geojson']` is only read with QuPath cell tables; {path} is {fmt!r}.") + if fmt != "generic" and cells_table_kwargs.get("coordinates_space") is not None: + raise ValueError( + f"`cells_table_kwargs['coordinates_space']` is only used with generic cell tables; {path} is {fmt!r}." + ) if fmt == "codex": return read_codex_table(path, region=default_region) if fmt not in TABLE_FORMATS: @@ -423,7 +431,9 @@ def read_cell_table( "generic": _generic_roles, }[fmt] roles = reader(path, df, cells_table_kwargs) - mask_pixels = fmt in {"mcquant", "sprm"} or (fmt == "generic" and not roles["in_um"]) + mask_pixels = fmt in {"mcquant", "sprm"} + if fmt == "generic": + mask_pixels = _coordinates_space(path, roles["in_um"], cells_table_kwargs) == "mask" units = cells_table_kwargs.get("geojson_units", "px") row_pixel_sizes = None if roles["in_um"] or (fmt == "qupath" and units == "um"): @@ -744,6 +754,19 @@ def _coordinates_unit(kwargs: Mapping[str, Any]) -> str: return str(unit) +def _coordinates_space(path: Path, in_um: bool, kwargs: Mapping[str, Any]) -> str: + """``cells_table_kwargs['coordinates_space']`` of a generic table: ``"image"`` (the default) or ``"mask"``.""" + space = kwargs.get("coordinates_space", "image") + if space not in ("image", "mask"): + raise ValueError(f"`cells_table_kwargs['coordinates_space']` must be 'image' or 'mask', got {space!r}.") + if space == "mask" and in_um: + raise ValueError( + f"`cells_table_kwargs['coordinates_space']` is 'mask', but the centroids of {path} are in µm; mask " + "pixels need `cells_table_kwargs['coordinates_unit']` 'px'." + ) + return str(space) + + def _select_markers(df: pd.DataFrame, selection: str | Sequence[str] | None, exclude: set[str | None]) -> list[str]: """The marker columns: ``selection`` (columns, or a regular expression matching the full column name). diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 95a421b1..ab4e6226 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -180,20 +180,22 @@ def codex( with the transformation of the region's image; the cells of the region annotate it instead of circles, and their instance ids must be the mask values. A mask whose shape is the region image's divided by an integer factor ``f`` (exactly, or rounded up) is scaled onto the image (``Scale([f, f])`` before the region's - transformation); any other shape mismatch raises a warning. The pixel centroids of ``"mcquant"``, - ``"sprm"`` and ``"generic"`` (``"px"``) tables are taken to be in mask pixels, so their centroids - (``obsm["spatial"]``) and circle radii are scaled by ``f`` to image pixels; the source columns in ``obs`` - are kept as read (mask pixels). To avoid this for a generic table in image pixels, use - ``coordinates_unit="um"`` or a full-resolution mask. + transformation); any other shape mismatch raises a warning. The pixel centroids of ``"mcquant"`` and + ``"sprm"`` tables are taken to be in mask pixels, so their centroids (``obsm["spatial"]``) and circle radii + are scaled by ``f`` to image pixels; the source columns in ``obs`` are kept as read (mask pixels). The pixel + centroids of other tables are in image pixels and are not scaled; + ``cells_table_kwargs["coordinates_space"]="mask"`` scales those of a ``"generic"`` table too. image_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. labels_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`. cells_table_kwargs Options for reading ``cells_table`` and ``labels``: ``region_column``; for ``"generic"`` tables ``x``, - ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath tables) and + ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath tables), ``coordinates_unit`` (``"px"`` or ``"um"``; for QuPath ``X``/``Y`` and inForm ``Cell X Position`` columns - without a unit, which are otherwise read in the unit of the table's other columns); with ``x`` and ``y``, + without a unit, which are otherwise read in the unit of the table's other columns) and + ``coordinates_space`` (``"image"``, default, or ``"mask"``: whether the pixel centroids of a generic table + are in pixels of the region image or of its mask, see ``labels``); with ``x`` and ``y``, ``cells_table_format="auto"`` reads a generic table. ``mask_channel`` (index or channel name of the mask in a multi-channel ``labels`` file: an OME channel name, or for an inForm ``*_binary_seg_maps.tif`` ``"Tissue"``, ``"Nucleus"`` (the default there, its labels are the ``Cell ID`` of every cell), diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index ae2e7dd2..247b9cd2 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -136,11 +136,11 @@ def phenocycler( A segmentation mask (TIFF/OME-TIFF with integer instance ids) per region: a path for a single region, or a mapping from region id to path. A mask whose shape is the region image's divided by an integer factor ``f`` (exactly, or rounded up) is scaled onto the image (``Scale([f, f])`` before the region's transformation); - any other shape mismatch raises a warning. The pixel centroids of ``"mcquant"``, ``"sprm"`` and - ``"generic"`` (``"px"``) tables are taken to be in mask pixels, so their centroids (``obsm["spatial"]``) and - circle radii are scaled by ``f`` to image pixels; the source columns in ``obs`` are kept as read (mask - pixels). To avoid this for a generic table in image pixels, use ``coordinates_unit="um"`` or a - full-resolution mask. + any other shape mismatch raises a warning. The pixel centroids of ``"mcquant"`` and ``"sprm"`` tables + are taken to be in mask pixels, so their centroids (``obsm["spatial"]``) and circle radii are scaled by + ``f`` to image pixels; the source columns in ``obs`` are kept as read (mask pixels). The pixel centroids of + other tables are in image pixels and are not scaled; ``cells_table_kwargs["coordinates_space"]="mask"`` + scales those of a ``"generic"`` table too. read_all_bit_depths If `True`, a scan that exists as both ``X{vx.QPTIFF_SUFFIX}`` (8-bit) and ``X{vx.ER_QPTIFF_SUFFIX}`` (extended range, e.g. uint16) is read from both files: the extended-range file as region ``X`` and the @@ -153,9 +153,11 @@ def phenocycler( Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`. cells_table_kwargs Options for reading ``cells_table`` and ``labels``: ``region_column``; for ``"generic"`` tables ``x``, - ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath tables) and + ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath tables), ``coordinates_unit`` (``"px"`` or ``"um"``; for QuPath ``X``/``Y`` and inForm ``Cell X Position`` columns - without a unit, which are otherwise read in the unit of the table's other columns); with ``x`` and ``y``, + without a unit, which are otherwise read in the unit of the table's other columns) and + ``coordinates_space`` (``"image"``, default, or ``"mask"``: whether the pixel centroids of a generic table + are in pixels of the region image or of its mask, see ``labels``); with ``x`` and ``y``, ``cells_table_format="auto"`` reads a generic table. ``mask_channel`` (index or channel name of the mask in a multi-channel ``labels`` file: an OME channel name, or for an inForm ``*_binary_seg_maps.tif`` ``"Tissue"``, ``"Nucleus"`` (the default there, its labels are the ``Cell ID`` of every cell), diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index 05bd3731..c062ecac 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -927,6 +927,46 @@ def test_mask_downsample_keeps_image_pixel_centroids(tmp_path: Path) -> None: np.testing.assert_allclose(sdata.tables["table"].obsm["spatial"], [[100, 50], [200, 80]]) +@pytest.mark.parametrize(("space", "factor"), [(None, 1), ("image", 1), ("mask", 2)]) +def test_generic_px_table_coordinates_space(tmp_path: Path, space: str | None, factor: int) -> None: + # generic pixel centroids are in image pixels unless `coordinates_space="mask"` + write_qptiff(tmp_path / "S.qptiff", biomarkers=["DAPI"], shape=(64, 80)) + mask = np.zeros((32, 40), dtype=np.uint32) + mask[2:4, 2:4], mask[5:7, 5:7] = 1, 2 + tifffile.imwrite(tmp_path / "mask.tif", mask) + (tmp_path / "cells.csv").write_text("label,cx,cy,CD3\n1,6,6,1\n2,12,12,2\n") + kwargs = {"x": "cx", "y": "cy", "instance_id": "label"} + if space is not None: + kwargs["coordinates_space"] = space + + sdata = phenocycler( + tmp_path, + cells_table=tmp_path / "cells.csv", + labels=tmp_path / "mask.tif", + image_models_kwargs={"scale_factors": [2]}, + cells_table_kwargs=kwargs, + ) + + np.testing.assert_allclose(sdata.tables["table"].obsm["spatial"], np.array([[6, 6], [12, 12]]) * factor) + + +@pytest.mark.parametrize( + ("fmt", "kwargs", "match"), + [ + ("generic", {"coordinates_space": "labels"}, "must be 'image' or 'mask'"), + ("generic", {"coordinates_space": "mask", "coordinates_unit": "um"}, "in µm"), + ("mcquant", {"coordinates_space": "image"}, "only used with generic"), + ], +) +def test_bad_coordinates_space_raises(tmp_path: Path, fmt: str, kwargs: dict[str, str], match: str) -> None: + path = tmp_path / "cells.csv" + path.write_text("CellID,CD3,X_centroid,Y_centroid\n1,1,6,6\n") + kwargs = {"x": "X_centroid", "y": "Y_centroid", **kwargs} if fmt == "generic" else kwargs + + with pytest.raises(ValueError, match=match): + read_cell_table(path, fmt, pixel_size_um=0.5, default_region="S", cells_table_kwargs=kwargs) + + def test_mask_non_integer_mismatch_warns(tmp_path: Path) -> None: write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"], shape=(64, 80)) tifffile.imwrite(tmp_path / "mask.tif", np.zeros((48, 60), dtype=np.uint32)) From 290d57ad53f6a5b96bbdc1272a7fddfd8b430b46 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Sat, 17 Oct 2026 00:37:55 +0200 Subject: [PATCH 041/107] fix: keep QuPath morphology out of bare-marker fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without `: : Mean` columns, QuPath columns with a unit (`Area µm^2`, `Max diameter µm`, also R's `Max.diameter.um`) and shape measures (`Circularity`, `Solidity`, `Eccentricity`, `Detection probability`) are no longer read as markers; they stay in obs. Co-Authored-By: Claude Opus 5.5 --- .../readers/_utils/_cell_tables.py | 15 ++++++++-- tests/readers/test_cell_tables.py | 29 +++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 01f705c3..6ce54e75 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -94,6 +94,8 @@ _QUPATH_MEASUREMENT = re.compile(r"(?:Cell|Nucleus|Cytoplasm|Membrane)\b|.*: ") # the unit of a QuPath measurement, e.g. `Cell: Area µm^2` _QUPATH_UNIT = re.compile(r" (µm|um|px)(?:\^2)?$") +# QuPath shape and detection measurements without a unit, e.g. of a detection export without compartments +_QUPATH_UNITLESS_MEASUREMENTS = frozenset({"Circularity", "Detection probability", "Eccentricity", "Solidity"}) _QUPATH_IMAGE = "Image" _QUPATH_ID = "Object ID" # QuPath's centroid columns by unit; R's `make.names` writes `Centroid X µm` as `Centroid.X.um` @@ -355,7 +357,8 @@ def read_cell_table( (also ``....Mean`` and ``__Mean``), one per marker from the first of the ``Cell``, ``Nucleus``, ``Cytoplasm`` and ``Membrane`` compartments (in ``var["compartment"]``), else, with a `UserWarning`, the numeric columns that are not QuPath - measurements (e.g. ``CD44`` of a detection export). The radius comes from ``Cell: Area µm^2`` (or + measurements (e.g. ``CD44`` of a detection export; not morphology such as ``Area µm^2``, + ``Max diameter µm`` or ``Circularity``). The radius comes from ``Cell: Area µm^2`` (or ``px^2``) in the unit of the centroids. A table split into centroids and measurements (two files) must be joined on the object id first. The cell outlines are read from a QuPath GeoJSON export, see :func:`read_qupath_polygons`: ``cells_table_kwargs["geojson"]``, else ``
.geojson`` or @@ -695,8 +698,9 @@ def _qupath_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> di index=pd.Index(dedupe_channel_names(list(chosen))), ) else: - # e.g. a detection export with the measurements named after the markers only - candidates = [column for column in df.columns if not _QUPATH_MEASUREMENT.match(column)] + # e.g. a detection export with the measurements named after the markers only; its morphology (e.g. + # `Area µm^2`, `Max diameter µm`, `Circularity`) is not a marker + candidates = [c for c in df.columns if not _QUPATH_MEASUREMENT.match(c) and not _is_qupath_morphology(c)] markers = _select_markers(df[candidates], None, roles) var = pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(markers))) if markers: @@ -722,6 +726,11 @@ def _qupath_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> di } +def _is_qupath_morphology(column: str) -> bool: + """Whether ``column`` (also in R's ``make.names`` form) is a QuPath measurement with a unit or a shape measure.""" + return any(_QUPATH_UNIT.search(name) or name in _QUPATH_UNITLESS_MEASUREMENTS for name in _qupath_names([column])) + + def _qupath_xy_unit(path: Path, columns: list[str], kwargs: Mapping[str, Any]) -> str: """The unit of the ``X``/``Y`` centroids of a QuPath table: ``coordinates_unit``, else that of its measurements. diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index c062ecac..9544f7b7 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -671,6 +671,35 @@ def test_qupath_xy_centroids_and_bare_markers(tmp_path: Path) -> None: assert list(table.var.index) == ["CD44", "CD45RO"] +@pytest.mark.parametrize("r_names", [False, True]) +def test_qupath_bare_markers_exclude_morphology(tmp_path: Path, r_names: bool) -> None: + path = tmp_path / "detections.csv" + header = [ + "Image", + "Object ID", + "Centroid X µm", + "Centroid Y µm", + "Area µm^2", + "Perimeter µm", + "Max diameter µm", + "Circularity", + "Solidity", + "Eccentricity", + "Detection probability", + "CD44", + "CD45RO", + ] + if r_names: + header = [name.replace(" ", ".").replace("µm^2", "um.2").replace("µm", "um") for name in header] + path.write_text(",".join(header) + "\nS,u1,1,2,3,4,5,0.9,0.8,0.5,0.99,1.5,4.3\n", encoding="utf-8") + + with pytest.warns(UserWarning, match=r"reading the 2 other numeric columns \['CD44', 'CD45RO'\]"): + table = read_cell_table(path, "qupath", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) + + assert list(table.var.index) == ["CD44", "CD45RO"] + assert {header[4], header[7], header[10]} <= set(table.obs.columns) + + def test_qupath_xy_centroids_need_a_unit(tmp_path: Path) -> None: path = tmp_path / "cells.csv" path.write_text("Object ID,X,Y,CD3: Cell: Mean\nab,10,5,1\n", encoding="utf-8") From 9b2720caef1665acf7f7b05848ab19e04f9d8d86 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Sat, 17 Oct 2026 00:40:44 +0200 Subject: [PATCH 042/107] test: cover read_all_bit_depths with masks and tables Lock that a downsampled mask is scaled onto the extended-range region, the 8-bit image shares its transformations, the table annotates only that region, and `labels={"S_8bit": ...}` raises. Warn when the 8-bit and extended-range scans differ in shape. Tighten two existing tests. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- src/spatialdata_io/readers/phenocycler.py | 16 ++++++-- tests/readers/test_cell_tables.py | 4 +- tests/test_phenocycler.py | 47 ++++++++++++++++++++++- 4 files changed, 62 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ab59452..078a6b74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. regions converts µm coordinates with each region's own pixel size. - `phenocycler(read_all_bit_depths=True)` (CLI `--read-all-bit-depths`) reads both the 8-bit `.qptiff` and the extended-range `.er.qptiff` of a scan: the 8-bit one becomes the image `_8bit_image` with the transformations - of the `.er` region. By default only the extended-range file is read. + of the `.er` region (a warning is raised if their shapes differ). By default only the extended-range file is read. - The Akoya readers (`phenocycler()`, `codex()`) scale a segmentation mask downsampled from its region image by an integer factor (exactly, or rounded up) onto the image (a `Scale` before the region's transformation), and scale the centroids (`obsm["spatial"]`) and circle radii of `mcquant` and SPRM tables, which are in mask pixels, by the diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 247b9cd2..8712158f 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -145,8 +145,8 @@ def phenocycler( If `True`, a scan that exists as both ``X{vx.QPTIFF_SUFFIX}`` (8-bit) and ``X{vx.ER_QPTIFF_SUFFIX}`` (extended range, e.g. uint16) is read from both files: the extended-range file as region ``X`` and the 8-bit file as the image ``X_8bit{vx.IMAGE_SUFFIX}`` (region ``X_8bit`` in ``sdata.attrs[{vx.ATTRS_KEY!r}]``). - Both are the same physical scan, so the 8-bit image has exactly the transformations of region ``X``. Cell - tables and masks annotate region ``X`` only. If `False`, only the extended-range file is read. + Both are the same physical scan, so the 8-bit image has exactly the transformations of region ``X`` (a + warning is raised if their shapes differ). Cell tables and masks annotate region ``X`` only. If `False`, only the extended-range file is read. image_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. labels_models_kwargs @@ -201,9 +201,19 @@ def phenocycler( pixel_sizes={scan.region: scan.pixel_size_um for scan in scans}, cells_table_kwargs=cells_table_kwargs, ) + images = {scan.region: read_qptiff_image(scan.path) for scan in [*scans, *duplicates.values()]} + for region, duplicate in duplicates.items(): + if images[region].shape[-2:] != images[duplicate.region].shape[-2:]: + warnings.warn( + f"The 8-bit scan {duplicate.path} has shape {images[duplicate.region].shape[-2:]} (y, x), the " + f"extended-range scan {qptiffs[region]} {images[region].shape[-2:]}; they may not be the same scan, " + "but the 8-bit image is stored with its transformations.", + UserWarning, + stacklevel=2, + ) sdata = build_spatialdata( scans, - {scan.region: read_qptiff_image(scan.path) for scan in [*scans, *duplicates.values()]}, + images, names, cell_tables, mask_files, diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index 9544f7b7..21b528c9 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -952,7 +952,9 @@ def test_mask_downsample_keeps_image_pixel_centroids(tmp_path: Path) -> None: image_models_kwargs={"scale_factors": [2]}, ) - assert isinstance(get_transformation(sdata.labels[next(iter(sdata.labels))], "global"), Sequence) + labels_transformation = get_transformation(sdata.labels[next(iter(sdata.labels))], "global") + assert isinstance(labels_transformation, Sequence) + assert labels_transformation.transformations[0] == Scale([2, 2], axes=("y", "x")) np.testing.assert_allclose(sdata.tables["table"].obsm["spatial"], [[100, 50], [200, 80]]) diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 3d68b669..63624ea4 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -14,7 +14,7 @@ from click.testing import CliRunner from spatialdata import read_zarr from spatialdata.models import get_channel_names -from spatialdata.transformations import Identity, Translation, get_transformation +from spatialdata.transformations import Identity, Scale, Sequence, Translation, get_transformation from spatialdata_io import phenocycler from spatialdata_io.__main__ import phenocycler_wrapper @@ -194,6 +194,49 @@ def test_read_all_bit_depths_shares_stage_placement(tmp_path: Path) -> None: assert get_transformation(sdata["B_8bit_image"], get_all=True) == get_transformation(sdata["B_image"], get_all=True) +def test_read_all_bit_depths_with_labels_and_cells_table(tmp_path: Path) -> None: + for suffix, dtype in ((".qptiff", "uint8"), (".er.qptiff", "uint16")): + write_qptiff( + tmp_path / f"S{suffix}", biomarkers=["DAPI"], shape=(64, 80), dtype=dtype, stage_position_cm=(1, 2) + ) + mask = np.zeros((32, 40), dtype=np.uint32) + mask[2:4, 2:4], mask[5:7, 5:7] = 1, 2 + tifffile.imwrite(tmp_path / "mask.tif", mask) + (tmp_path / "cells.csv").write_text("CellID,CD3,X_centroid,Y_centroid,Area\n1,1,3,3,4\n2,2,6,6,4\n") + + sdata = phenocycler( + tmp_path, + cells_table=tmp_path / "cells.csv", + labels=tmp_path / "mask.tif", + read_all_bit_depths=True, + image_models_kwargs=_FAST, + ) + + image_transformations = get_transformation(sdata["S_image"], get_all=True) + assert get_transformation(sdata["S_8bit_image"], get_all=True) == image_transformations + # the mask is scaled onto the extended-range region only + assert list(sdata.labels) == ["S_labels"] + labels_transformation = get_transformation(sdata["S_labels"], "global") + assert isinstance(labels_transformation, Sequence) + assert labels_transformation.transformations == [Scale([2, 2], axes=("y", "x")), image_transformations["global"]] + table = sdata.tables["table"] + assert table.uns["spatialdata_attrs"]["region"] == ["S_labels"] + np.testing.assert_allclose(table.obsm["spatial"], [[6, 6], [12, 12]]) + + with pytest.raises(ValueError, match=r"Unknown region\(s\) \['S_8bit'\]"): + phenocycler(tmp_path, labels={"S_8bit": tmp_path / "mask.tif"}, read_all_bit_depths=True) + + +def test_read_all_bit_depths_shape_mismatch_warns(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S.qptiff", biomarkers=["DAPI"], shape=(64, 80), dtype="uint8") + write_qptiff(tmp_path / "S.er.qptiff", biomarkers=["DAPI"], shape=(64, 96), dtype="uint16") + + with pytest.warns(UserWarning, match=r"8-bit scan .*\(64, 80\).*\(64, 96\)"): + sdata = phenocycler(tmp_path, read_all_bit_depths=True, image_models_kwargs=_FAST) + + assert sorted(sdata.images) == ["S_8bit_image", "S_image"] + + def test_read_all_bit_depths_without_pair_reads_one(tmp_path: Path) -> None: write_qptiff(tmp_path / "S.qptiff", biomarkers=["DAPI"], dtype="uint8") @@ -822,5 +865,5 @@ def fail(*args: object, **kwargs: object) -> None: def test_phenocycler_keyword_only(tmp_path: Path) -> None: write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI"]) - with pytest.raises(TypeError): + with pytest.raises(TypeError, match="positional argument"): phenocycler(tmp_path, ["A"]) # type: ignore[misc] From ae9b3017a914fc622f8e1382e6f8b039ab56d2d0 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 02:50:50 +0200 Subject: [PATCH 043/107] fix: honour TIFF byte order in memmap reads --- src/spatialdata_io/readers/_utils/_akoya.py | 4 +++- tests/readers/test_akoya_utils.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index a0d9f248..fcb426c0 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -232,7 +232,9 @@ def read_tiff( """ with tifffile.TiffFile(path) as tf: series = tf.series[0] - axes, offset, shape, dtype = series.axes, series.dataoffset, series.shape, series.dtype + axes, offset, shape = series.axes, series.dataoffset, series.shape + # tifffile reports the dtype in native byte order; a memory map needs the file's (big-endian for ImageJ) + dtype = series.dtype.newbyteorder(tf.byteorder) metadata = tf.imagej_metadata or {} labels = list(metadata["Labels"]) if isinstance(metadata.get("Labels"), list | tuple) else None read_by_tifffile = True diff --git a/tests/readers/test_akoya_utils.py b/tests/readers/test_akoya_utils.py index 52943635..99e0e7d8 100644 --- a/tests/readers/test_akoya_utils.py +++ b/tests/readers/test_akoya_utils.py @@ -181,3 +181,18 @@ def test_build_image_rgb_and_multiplex(): assert list(element.coords["c"].values) == ["r", "g", "b"] assert element.shape == (3, 16, 16) assert get_transformation(element, get_all=True) == {"reg1": trans} + + +@pytest.mark.parametrize("imagej", [False, True]) +@pytest.mark.parametrize( + "data", [np.array([[[1, 8], [15, 0]]], dtype=np.uint16), np.array([[[1, 2], [300, 0]]], np.uint16)] +) +def test_read_tiff_big_endian(tmp_path: Path, imagej: bool, data: np.ndarray) -> None: + # ImageJ/Fiji write big-endian tiffs by default; the memory-mapped path must use the file's byte order + path = tmp_path / "x.tif" + tifffile.imwrite(path, data, byteorder=">", imagej=imagej) + + image, _, _, _ = read_tiff(path, {}, (16, 16)) + + assert image.dtype == np.dtype("uint16") + np.testing.assert_array_equal(image.compute(), data) From 1696ca4f6c7308581dc45007bf90c08bbd90bc68 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:03:07 +0200 Subject: [PATCH 044/107] fix: hold no file descriptors in lazy TIFF chunks --- src/spatialdata_io/readers/_utils/_akoya.py | 79 ++++++++++++--------- tests/readers/test_akoya_utils.py | 20 ++++++ 2 files changed, 65 insertions(+), 34 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index fcb426c0..e013089a 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -2,6 +2,7 @@ from __future__ import annotations +import io import json import math import threading @@ -274,7 +275,8 @@ def __init__(self, path: Path, offset: int, shape: tuple[int, ...], dtype: np.dt def __getitem__(self, key: Any) -> np.ndarray: data = np.memmap(self.path, dtype=self._file_dtype, mode="r", offset=self.offset, shape=self.shape) - return np.asarray(data[key], dtype=self.dtype) + # a copy, as a view would keep the memory map and its file descriptor open as long as the chunk lives + return np.array(data[key], dtype=self.dtype) def lazy_tiff_array(path: Path, *, series: int = 0, page: int | None = None) -> da.Array: @@ -305,8 +307,9 @@ def lazy_tiff_array(path: Path, *, series: int = 0, page: int | None = None) -> class _TiffZarrSource: """Picklable array-like view of a tiff series (level 0) or page, read chunk-wise through tifffile's zarr store. - Neither the ``TiffFile`` nor its zarr store are pickled: they are opened on the first read in each process and - closed with the object, so dask graphs built on it work across processes and leave no file handle unclosed. + Neither the ``TiffFile`` nor its zarr store are pickled, but opened again in each process that reads chunks, so + dask graphs built on it work across processes. The file is read through a :class:`_TransientFile`, so the parsed + tiff stays cached without holding a file descriptor. """ def __init__(self, path: Path, *, series: int = 0, page: int | None = None) -> None: @@ -314,56 +317,64 @@ def __init__(self, path: Path, *, series: int = 0, page: int | None = None) -> N self.series = series self.page = page self._lock = threading.Lock() - self._tiff: tifffile.TiffFile | None = None self._array: zarr.Array | None = None - try: - array = self._open() - self.shape = tuple(array.shape) - self.chunks = tuple(array.chunks) - self.dtype = array.dtype - finally: - # the metadata is all a lazy read needs; the file is opened again once chunks are read - self.close() + array = self._open() + self.shape = tuple(array.shape) + self.chunks = tuple(array.chunks) + self.dtype = array.dtype self.ndim = len(self.shape) token = tokenize(str(path), path.stat().st_mtime_ns, series, page, self.shape, str(self.dtype)) self.name = f"akoya-tiff-zarr-{token}" def _open(self) -> zarr.Array: with self._lock: - if self._array is not None: - return self._array - tiff = tifffile.TiffFile(self.path) - try: + if self._array is None: + tiff = tifffile.TiffFile(io.BufferedReader(_TransientFile(self.path))) if self.page is not None: store = tiff.pages[self.page].aszarr() else: store = tiff.series[self.series].aszarr(level=0) - array = zarr.open_array(store, mode="r") - except BaseException: - tiff.close() - raise - self._tiff, self._array = tiff, array - return array + self._array = zarr.open_array(store, mode="r") + return self._array def __getitem__(self, key: Any) -> np.ndarray: return np.asarray(self._open()[key]) - def close(self) -> None: - """Close the file; it is opened again by the next read.""" - with self._lock: - if self._tiff is not None: - self._tiff.close() - self._tiff, self._array = None, None - def __getstate__(self) -> dict[str, Any]: - return {key: value for key, value in self.__dict__.items() if key not in ("_lock", "_tiff", "_array")} + return {key: value for key, value in self.__dict__.items() if key not in ("_lock", "_array")} def __setstate__(self, state: dict[str, Any]) -> None: - self.__dict__.update(state, _lock=threading.Lock(), _tiff=None, _array=None) + self.__dict__.update(state, _lock=threading.Lock(), _array=None) + + +class _TransientFile(io.RawIOBase): + """Read-only binary file that opens ``path`` for every read, so it holds no file descriptor in between.""" + + def __init__(self, path: Path) -> None: + super().__init__() + self.name = str(path) + self._size = path.stat().st_size + self._position = 0 + + def readable(self) -> bool: + return True + + def seekable(self) -> bool: + return True + + def tell(self) -> int: + return self._position + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + self._position = (0, self._position, self._size)[whence] + offset + return self._position - def __del__(self) -> None: - if getattr(self, "_tiff", None) is not None: - self.close() + def readinto(self, buffer: Any) -> int: + with open(self.name, "rb") as f: + f.seek(self._position) + n = f.readinto(buffer) + self._position += n + return n def spatial_chunks(chunks: Any, default: Sequence[int]) -> tuple[int, int]: diff --git a/tests/readers/test_akoya_utils.py b/tests/readers/test_akoya_utils.py index 99e0e7d8..ca3e1e3f 100644 --- a/tests/readers/test_akoya_utils.py +++ b/tests/readers/test_akoya_utils.py @@ -1,9 +1,11 @@ from __future__ import annotations import json +import os import pickle from pathlib import Path +import dask import dask.array as da import numpy as np import pytest @@ -196,3 +198,21 @@ def test_read_tiff_big_endian(tmp_path: Path, imagej: bool, data: np.ndarray) -> assert image.dtype == np.dtype("uint16") np.testing.assert_array_equal(image.compute(), data) + + +@pytest.mark.skipif(not os.path.isdir("/dev/fd"), reason="needs /dev/fd to count open files") +@pytest.mark.parametrize("compression", [None, "zlib"]) +def test_read_tiff_does_not_keep_files_open(tmp_path: Path, compression: str | None) -> None: + # computed chunks and arrays kept alive must not hold a file descriptor each (EMFILE on large reads) + data = np.arange(64 * 96, dtype=np.uint16).reshape(64, 96) + images = [] + for i in range(20): + tifffile.imwrite(tmp_path / f"{i}.tif", data, compression=compression, tile=(16, 16) if compression else None) + images.append(read_tiff(tmp_path / f"{i}.tif", {}, (16, 16))[0]) + n_open = len(os.listdir("/dev/fd")) + + # the chunks as computed, before dask assembles them (e.g. while ``compute`` or ``sdata.write`` runs) + chunks = dask.compute(*[chunk for image in images for chunk in image.to_delayed().ravel()]) + + assert len(os.listdir("/dev/fd")) - n_open <= 2 + np.testing.assert_array_equal(chunks[-1][0], data[48:, 80:]) From 5df2ebe06326806a692aaf739876a670518f6b35 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:04:03 +0200 Subject: [PATCH 045/107] build: require tifffile>=2026.5.2 and zarr>=3.2 --- CHANGELOG.md | 1 + pyproject.toml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 078a6b74..602de912 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,7 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. - Adopted the current `cookiecutter-scverse` template: `hatch`-managed test environments, `uv`-based CI and documentation builds, `mypy` type checking of `src` and `tests`, `biome`/`pyproject-fmt`/`zizmor` pre-commit hooks, and Dependabot updates. +- Requires `tifffile>=2026.5.2` and `zarr>=3.2`, the first versions whose zarr store the lazy Akoya TIFF reads work with. - `codex()` was rewritten to read CODEX Processor, CODEX Uploader and CODEX Instrument Manager outputs. This changes its output and signature: - Element names are now `{region}_image`, `{region}_cells` (circles), `{region}_labels`, `{region}_polygons` and diff --git a/pyproject.toml b/pyproject.toml index 9c7135e1..f2a7ad86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,8 +40,9 @@ dependencies = [ # for debug logging (referenced from the issue template) "session-info2", "spatialdata>=0.7.3a0", - "tifffile>=2023.8.12", + "tifffile>=2026.5.2", "xmltodict", + "zarr>=3.2", ] # https://docs.pypi.org/project_metadata/#project-urls urls.Documentation = "https://spatialdata-io.readthedocs.io/" From 3b86fb819d91f58ac798801076bd0d1ce355f969 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:07:58 +0200 Subject: [PATCH 046/107] perf: group TIFF strips and tiles into larger dask chunks --- src/spatialdata_io/readers/_utils/_akoya.py | 26 ++++++++++++++------- tests/readers/test_akoya_utils.py | 18 ++++++++++++++ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index e013089a..8ee4270d 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -245,7 +245,7 @@ def read_tiff( image = da.from_array(source, chunks=tuple(tiles[i] for i in range(source.ndim)), name=source.name) else: try: - image = lazy_tiff_array(path, series=0) + image = lazy_tiff_array(path, series=0, chunks=chunks) except ValueError as e: # e.g. a compression tifffile cannot decode chunk-wise logger.info("Reading %s with dask_image.imread because tifffile cannot read it lazily: %s", path, e) image = imread(str(path), **imread_kwargs) @@ -279,8 +279,10 @@ def __getitem__(self, key: Any) -> np.ndarray: return np.array(data[key], dtype=self.dtype) -def lazy_tiff_array(path: Path, *, series: int = 0, page: int | None = None) -> da.Array: - """Lazily read the level 0 of a tiff series, or one page, with the native tile chunks of the file. +def lazy_tiff_array( + path: Path, *, series: int = 0, page: int | None = None, chunks: tuple[int, int] = (4096, 4096) +) -> da.Array: + """Lazily read the level 0 of a tiff series, or one page, in chunks of whole native tiles or strips. The array can be pickled (e.g. for ``dask.distributed`` or process-based schedulers): its chunks are read by a :class:`_TiffZarrSource`, which opens the file in each process that reads them. @@ -293,14 +295,22 @@ def lazy_tiff_array(path: Path, *, series: int = 0, page: int | None = None) -> The index of the series to read, if ``page`` is `None`. page The index of the page to read instead of a series. + chunks + The ``(y, x)`` chunk size to aim for: the native tiles or strips are grouped into chunks of about as many + pixels (the default is that of the raster models), so the graph does not get one task per strip. Returns ------- The array, with the shape of the series or page. """ source = _TiffZarrSource(path, series=series, page=page) + y, x = source.axes.index("Y"), source.axes.index("X") + grouped = list(source.chunks) + native_y, native_x = grouped[y], grouped[x] + grouped[x] = min(source.shape[x], max(native_x, chunks[1] // native_x * native_x)) + grouped[y] = min(source.shape[y], max(native_y, chunks[0] * chunks[1] // grouped[x] // native_y * native_y)) return da.from_array( - source, chunks=source.chunks, name=source.name, meta=np.empty((0,) * source.ndim, dtype=source.dtype) + source, chunks=tuple(grouped), name=source.name, meta=np.empty((0,) * source.ndim, dtype=source.dtype) ) @@ -330,11 +340,9 @@ def _open(self) -> zarr.Array: with self._lock: if self._array is None: tiff = tifffile.TiffFile(io.BufferedReader(_TransientFile(self.path))) - if self.page is not None: - store = tiff.pages[self.page].aszarr() - else: - store = tiff.series[self.series].aszarr(level=0) - self._array = zarr.open_array(store, mode="r") + data = tiff.pages[self.page] if self.page is not None else tiff.series[self.series] + self.axes = data.axes + self._array = zarr.open_array(data.aszarr(level=0), mode="r") return self._array def __getitem__(self, key: Any) -> np.ndarray: diff --git a/tests/readers/test_akoya_utils.py b/tests/readers/test_akoya_utils.py index ca3e1e3f..2424dd04 100644 --- a/tests/readers/test_akoya_utils.py +++ b/tests/readers/test_akoya_utils.py @@ -4,6 +4,7 @@ import os import pickle from pathlib import Path +from typing import Any import dask import dask.array as da @@ -216,3 +217,20 @@ def test_read_tiff_does_not_keep_files_open(tmp_path: Path, compression: str | N assert len(os.listdir("/dev/fd")) - n_open <= 2 np.testing.assert_array_equal(chunks[-1][0], data[48:, 80:]) + + +@pytest.mark.parametrize( + ("layout", "chunksize"), + [({"rowsperstrip": 2}, (1, 16, 64)), ({"tile": (16, 16)}, (1, 32, 32)), ({"tile": (64, 64)}, (1, 64, 64))], + ids=["strips", "small_tiles", "large_tiles"], +) +def test_read_tiff_groups_native_chunks(tmp_path: Path, layout: dict[str, Any], chunksize: tuple[int, ...]) -> None: + # one dask chunk per strip or tile makes huge graphs; chunks group whole strips/tiles to about `chunks` pixels + path = tmp_path / "x.tif" + data = np.arange(256 * 64, dtype=np.uint16).reshape(256, 64) + tifffile.imwrite(path, data, compression="zlib", **layout) + + image, _, _, _ = read_tiff(path, {}, (32, 32)) + + assert image.chunksize == chunksize + np.testing.assert_array_equal(image.compute()[0], data) From 8003a5b3acfa610bf9aaf487fb7e1c982b06bce5 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:09:30 +0200 Subject: [PATCH 047/107] fix: include chunks in lazy TIFF dask names --- src/spatialdata_io/readers/_utils/_akoya.py | 10 +++++++--- tests/readers/test_akoya_utils.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index 8ee4270d..00073428 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -242,7 +242,8 @@ def read_tiff( if offset is not None: source = _ContiguousTiff(path, offset, shape, dtype) tiles = dict.fromkeys(range(source.ndim), 1) | {axes.index("Y"): chunks[0], axes.index("X"): chunks[1]} - image = da.from_array(source, chunks=tuple(tiles[i] for i in range(source.ndim)), name=source.name) + tile_chunks = tuple(tiles[i] for i in range(source.ndim)) + image = da.from_array(source, chunks=tile_chunks, name=f"{source.name}-{tokenize(tile_chunks)}") else: try: image = lazy_tiff_array(path, series=0, chunks=chunks) @@ -261,7 +262,7 @@ class _ContiguousTiff: """Array-like view of uncompressed contiguous tiff data, memory-mapped anew for every chunk. Handing an ``np.memmap`` to :func:`dask.array.from_array` would read the whole file (dask hashes and slices - numpy arrays eagerly), so dask only gets this lightweight object and a name derived from the file. + numpy arrays eagerly), so dask only gets this lightweight object and a name derived from the file and chunks. """ def __init__(self, path: Path, offset: int, shape: tuple[int, ...], dtype: np.dtype[Any]) -> None: @@ -310,7 +311,10 @@ def lazy_tiff_array( grouped[x] = min(source.shape[x], max(native_x, chunks[1] // native_x * native_x)) grouped[y] = min(source.shape[y], max(native_y, chunks[0] * chunks[1] // grouped[x] // native_y * native_y)) return da.from_array( - source, chunks=tuple(grouped), name=source.name, meta=np.empty((0,) * source.ndim, dtype=source.dtype) + source, + chunks=tuple(grouped), + name=f"{source.name}-{tokenize(grouped)}", + meta=np.empty((0,) * source.ndim, dtype=source.dtype), ) diff --git a/tests/readers/test_akoya_utils.py b/tests/readers/test_akoya_utils.py index 2424dd04..384cabfe 100644 --- a/tests/readers/test_akoya_utils.py +++ b/tests/readers/test_akoya_utils.py @@ -234,3 +234,17 @@ def test_read_tiff_groups_native_chunks(tmp_path: Path, layout: dict[str, Any], assert image.chunksize == chunksize np.testing.assert_array_equal(image.compute()[0], data) + + +@pytest.mark.parametrize("compression", [None, "zlib"]) +def test_read_tiff_names_depend_on_chunks(tmp_path: Path, compression: str | None) -> None: + # arrays of one file with different chunks must not share dask keys, or computing both mixes their chunks + path = tmp_path / "x.tif" + data = np.arange(64 * 64, dtype=np.uint16).reshape(64, 64) + tifffile.imwrite(path, data, compression=compression, tile=(16, 16) if compression else None) + + small, _, _, _ = read_tiff(path, {}, (16, 16)) + large, _, _, _ = read_tiff(path, {}, (32, 32)) + + assert small.name != large.name + np.testing.assert_array_equal(da.stack([small.rechunk(large.chunks), large]).compute()[:, 0], [data, data]) From 49423ee14fb1614ea76affa4f1e38ab5a8a1a376 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:12:16 +0200 Subject: [PATCH 048/107] refactor: drop the dead dask_image fallback of read_tiff With tifffile>=2026.5.2 and zarr>=3.2 building the zarr store no longer raises; decode errors surface at compute, where dask_image (pims -> tifffile) fails the same way. codex() now warns about imread_kwargs once, also for qptiff regions. --- CHANGELOG.md | 2 +- src/spatialdata_io/__main__.py | 2 +- src/spatialdata_io/readers/_utils/_akoya.py | 22 +++-------- .../readers/_utils/_cell_tables.py | 2 +- src/spatialdata_io/readers/codex.py | 39 ++++++------------- tests/readers/test_akoya_utils.py | 13 +++---- tests/test_codex.py | 11 ++++-- 7 files changed, 34 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 602de912..0814e294 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,7 +69,7 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. - The positional signature `codex(path, fcs, imread_kwargs)` is unchanged; all new parameters (`regions`, `channel_names`, `cells_table`, `cells_table_format`, `labels` and the `*_models_kwargs` / `cells_table_kwargs`) are keyword-only, and the `codex` CLI command exposes them. An empty `regions` raises a `ValueError`. - `imread_kwargs` is only used by the `dask_image.imread` fallback for TIFFs that `tifffile` cannot read lazily. + `imread_kwargs` is ignored, with a warning: every image is read lazily with `tifffile`. - A single region is stored in `"global"`. Several regions are placed at their stage positions in `"global"` only when all of them share a sample space (same slide id, stage positions and pixel size); otherwise each region, including multi-region reads of unrelated regions, gets its own coordinate system named after the region. diff --git a/src/spatialdata_io/__main__.py b/src/spatialdata_io/__main__.py index c9fa6dbb..42b85038 100644 --- a/src/spatialdata_io/__main__.py +++ b/src/spatialdata_io/__main__.py @@ -97,7 +97,7 @@ def _parse_path_or_mapping_param(value: str | None, param_name: str) -> str | di "--imread-kwargs", type=str, default="{}", - help="JSON string of keyword arguments passed to imread. [default: {}]", + help="Ignored, with a warning; kept for backward compatibility. [default: {}]", ) @click.option( "--image-models-kwargs", diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index 00073428..f67cf950 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -17,7 +17,6 @@ import tifffile import zarr from dask.base import tokenize -from dask_image.imread import imread from spatialdata import sanitize_name from spatialdata._logging import logger from spatialdata.models import Image2DModel @@ -220,16 +219,13 @@ def read_json_object(path: Path) -> dict[str, Any] | None: return content -def read_tiff( - path: Path, imread_kwargs: Mapping[str, Any], chunks: tuple[int, int] -) -> tuple[da.Array, tuple[int, ...], list[str] | None, bool]: +def read_tiff(path: Path, chunks: tuple[int, int]) -> tuple[da.Array, tuple[int, ...], list[str] | None]: """Lazily read a tiff as ``(c, y, x)``, all non-spatial axes flattened into ``c``. Uncompressed contiguous data (e.g. ImageJ hyperstacks and per-marker images, often a single strip per plane) is memory-mapped and split into ``chunks``, so a chunk never needs a whole plane in memory; other layouts keep - their native tiles via zarr. Returns the image, the shape of the flattened axes, the ImageJ ``Labels`` (if - any) and whether tifffile, rather than the :func:`dask_image.imread.imread` fallback, read the file. The - array opens the file only when its chunks are read and can be pickled, e.g. for ``dask.distributed``. + their native tiles via zarr. Returns the image, the shape of the flattened axes and the ImageJ ``Labels`` (if + any). The array opens the file only when its chunks are read and can be pickled, e.g. for ``dask.distributed``. """ with tifffile.TiffFile(path) as tf: series = tf.series[0] @@ -238,24 +234,16 @@ def read_tiff( dtype = series.dtype.newbyteorder(tf.byteorder) metadata = tf.imagej_metadata or {} labels = list(metadata["Labels"]) if isinstance(metadata.get("Labels"), list | tuple) else None - read_by_tifffile = True if offset is not None: source = _ContiguousTiff(path, offset, shape, dtype) tiles = dict.fromkeys(range(source.ndim), 1) | {axes.index("Y"): chunks[0], axes.index("X"): chunks[1]} tile_chunks = tuple(tiles[i] for i in range(source.ndim)) image = da.from_array(source, chunks=tile_chunks, name=f"{source.name}-{tokenize(tile_chunks)}") else: - try: - image = lazy_tiff_array(path, series=0, chunks=chunks) - except ValueError as e: # e.g. a compression tifffile cannot decode chunk-wise - logger.info("Reading %s with dask_image.imread because tifffile cannot read it lazily: %s", path, e) - image = imread(str(path), **imread_kwargs) - axes = "Q" * (image.ndim - 2) + "YX" - labels = None - read_by_tifffile = False + image = lazy_tiff_array(path, series=0, chunks=chunks) image = da.moveaxis(image, [axes.index("Y"), axes.index("X")], [-2, -1]) leading_shape = tuple(image.shape[:-2]) - return image.reshape((math.prod(leading_shape), *image.shape[-2:])), leading_shape, labels, read_by_tifffile + return image.reshape((math.prod(leading_shape), *image.shape[-2:])), leading_shape, labels class _ContiguousTiff: diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 6ce54e75..c5f7a856 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -1184,7 +1184,7 @@ def read_labels( The labels element. """ labels_models_kwargs, chunks = _labels_models_kwargs(labels_models_kwargs) - image, _, _, _ = read_tiff(path, {}, chunks) + image, _, _ = read_tiff(path, chunks) if not np.issubdtype(image.dtype, np.integer): raise ValueError(f"{path} has dtype {image.dtype}; a segmentation mask must have an integer dtype.") diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index ab4e6226..614acef9 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -158,8 +158,7 @@ def codex( Whether to prefer ``{vx.FCS_FILE!r}`` cell tables over ``{vx.FCS_FILE_CSV!r}`` ones when both exist. With `False`, a region that only has a ``{vx.FCS_FILE!r}`` table is read from it, with a warning. imread_kwargs - Keyword arguments passed to :func:`dask_image.imread.imread`, which is only used for ``{vx.IMAGE_TIF}(f)`` - files that ``tifffile`` cannot read lazily (e.g. unsupported compression). + Ignored, with a warning: every image is read lazily with ``tifffile``. Kept for backward compatibility. regions Regions to read, e.g. ``["reg001"]``, including only their cells. If `None`, all regions are read. channel_names @@ -208,6 +207,10 @@ def codex( ------- :class:`spatialdata.SpatialData` """ + if imread_kwargs: + warnings.warn( + "`imread_kwargs` is ignored: every image is read lazily with tifffile.", UserWarning, stacklevel=2 + ) regions = normalize_regions(regions) channel_names = [channel_names] if isinstance(channel_names, str) else channel_names path = Path(path) @@ -230,7 +233,7 @@ def codex( arrays[region] = read_qptiff_image(files[0]) continue arrays[region], channels = _read_legacy_image( - files, layout.channel_names_txt, imread_kwargs, image_models_kwargs=image_models_kwargs + files, layout.channel_names_txt, image_models_kwargs=image_models_kwargs ) scans.append( AkoyaScan( @@ -356,7 +359,6 @@ def _discover_legacy(path: Path, fcs: bool) -> _LegacyLayout: def _read_legacy_image( files: list[Path], channel_names_txt: Path | None, - imread_kwargs: Mapping[str, Any], image_models_kwargs: Mapping[str, Any] = MappingProxyType({}), ) -> tuple[da.Array, list[AkoyaChannel]]: """Lazily read the image of one legacy CODEX region. @@ -367,8 +369,6 @@ def _read_legacy_image( The per-marker images of the region, or its single hyperstack, as returned by :func:`_discover_legacy`. channel_names_txt Optional ``channelNames.txt`` naming the hyperstack channels when the file has no ImageJ ``Labels``. - imread_kwargs - Keyword arguments passed to :func:`dask_image.imread.imread` for files :mod:`tifffile` cannot read lazily. image_models_kwargs Keyword arguments for :class:`spatialdata.models.Image2DModel`; their ``chunks`` set the spatial chunks of uncompressed files, which are memory-mapped. @@ -380,14 +380,11 @@ def _read_legacy_image( chunks = _spatial_chunks(image_models_kwargs) markers = [(match, file) for file in files if (match := _PER_MARKER.search(file.name)) is not None] if len(markers) == len(files): - image, channels, read_by_tifffile = _read_per_marker(markers, imread_kwargs, chunks) - _warn_unused_imread_kwargs(imread_kwargs, read_by_tifffile) - return image, channels + return _read_per_marker(markers, chunks) if len(files) != 1: raise ValueError(f"Expected per-marker images or a single hyperstack, got {[str(f) for f in files]}.") - image, leading_shape, labels, read_by_tifffile = read_tiff(files[0], imread_kwargs, chunks) - _warn_unused_imread_kwargs(imread_kwargs, read_by_tifffile) + image, leading_shape, labels = read_tiff(files[0], chunks) n_planes = image.shape[0] # the last non-spatial axis holds the channels of a cycle; ImageJ axis labels are unreliable (cycles as Z) n_channels = leading_shape[-1] if leading_shape else 1 @@ -411,15 +408,13 @@ def _read_legacy_image( def _read_per_marker( - markers: list[tuple[re.Match[str], Path]], imread_kwargs: Mapping[str, Any], chunks: tuple[int, int] -) -> tuple[da.Array, list[AkoyaChannel], bool]: + markers: list[tuple[re.Match[str], Path]], chunks: tuple[int, int] +) -> tuple[da.Array, list[AkoyaChannel]]: parsed = sorted((int(match.group(2)), int(match.group(3)), match.group(4), file) for match, file in markers) planes = [] - read_by_tifffile = False for _, _, _, file in parsed: - plane, _, _, lazy = read_tiff(file, imread_kwargs, chunks) - read_by_tifffile |= lazy + plane, _, _ = read_tiff(file, chunks) if plane.shape[0] != 1: raise ValueError(f"{file} holds {plane.shape[0]} planes; expected a single 2-D marker image.") planes.append(plane[0]) @@ -430,7 +425,7 @@ def _read_per_marker( AkoyaChannel(name=marker, cycle=cycle, channel=channel, extra={"file": file.name}) for cycle, channel, marker, file in parsed ] - return da.stack(planes), channels, read_by_tifffile + return da.stack(planes), channels def _spatial_chunks(image_models_kwargs: Mapping[str, Any]) -> tuple[int, int]: @@ -440,16 +435,6 @@ def _spatial_chunks(image_models_kwargs: Mapping[str, Any]) -> tuple[int, int]: return spatial_chunks(chunks, default["chunks"][-2:]) -def _warn_unused_imread_kwargs(imread_kwargs: Mapping[str, Any], read_by_tifffile: bool) -> None: - if imread_kwargs and read_by_tifffile: - warnings.warn( - "`imread_kwargs` is ignored for files read lazily with tifffile; it is only used by the " - "dask_image.imread fallback for files tifffile cannot read lazily.", - UserWarning, - stacklevel=3, - ) - - def _walk(root: Path) -> list[Path]: """All files below ``root`` in sorted order, skipping raw acquisition and intermediate processing folders.""" files: list[Path] = [] diff --git a/tests/readers/test_akoya_utils.py b/tests/readers/test_akoya_utils.py index 384cabfe..fbf12235 100644 --- a/tests/readers/test_akoya_utils.py +++ b/tests/readers/test_akoya_utils.py @@ -162,9 +162,8 @@ def test_read_tiff_is_picklable(tmp_path: Path, compression: str | None) -> None data = np.arange(2 * 64 * 96, dtype=np.uint16).reshape(2, 64, 96) tifffile.imwrite(path, data, compression=compression, tile=(32, 32) if compression else None) - image, _, _, read_by_tifffile = read_tiff(path, {}, (16, 16)) + image, _, _ = read_tiff(path, (16, 16)) - assert read_by_tifffile restored = pickle.loads(pickle.dumps(image)) np.testing.assert_array_equal(restored[0, :8, :8].compute(), data[0, :8, :8]) np.testing.assert_array_equal(restored.compute(), data) @@ -195,7 +194,7 @@ def test_read_tiff_big_endian(tmp_path: Path, imagej: bool, data: np.ndarray) -> path = tmp_path / "x.tif" tifffile.imwrite(path, data, byteorder=">", imagej=imagej) - image, _, _, _ = read_tiff(path, {}, (16, 16)) + image, _, _ = read_tiff(path, (16, 16)) assert image.dtype == np.dtype("uint16") np.testing.assert_array_equal(image.compute(), data) @@ -209,7 +208,7 @@ def test_read_tiff_does_not_keep_files_open(tmp_path: Path, compression: str | N images = [] for i in range(20): tifffile.imwrite(tmp_path / f"{i}.tif", data, compression=compression, tile=(16, 16) if compression else None) - images.append(read_tiff(tmp_path / f"{i}.tif", {}, (16, 16))[0]) + images.append(read_tiff(tmp_path / f"{i}.tif", (16, 16))[0]) n_open = len(os.listdir("/dev/fd")) # the chunks as computed, before dask assembles them (e.g. while ``compute`` or ``sdata.write`` runs) @@ -230,7 +229,7 @@ def test_read_tiff_groups_native_chunks(tmp_path: Path, layout: dict[str, Any], data = np.arange(256 * 64, dtype=np.uint16).reshape(256, 64) tifffile.imwrite(path, data, compression="zlib", **layout) - image, _, _, _ = read_tiff(path, {}, (32, 32)) + image, _, _ = read_tiff(path, (32, 32)) assert image.chunksize == chunksize np.testing.assert_array_equal(image.compute()[0], data) @@ -243,8 +242,8 @@ def test_read_tiff_names_depend_on_chunks(tmp_path: Path, compression: str | Non data = np.arange(64 * 64, dtype=np.uint16).reshape(64, 64) tifffile.imwrite(path, data, compression=compression, tile=(16, 16) if compression else None) - small, _, _, _ = read_tiff(path, {}, (16, 16)) - large, _, _, _ = read_tiff(path, {}, (32, 32)) + small, _, _ = read_tiff(path, (16, 16)) + large, _, _ = read_tiff(path, (32, 32)) assert small.name != large.name np.testing.assert_array_equal(da.stack([small.rechunk(large.chunks), large]).compute()[:, 0], [data, data]) diff --git a/tests/test_codex.py b/tests/test_codex.py index fe7d8312..55e434cb 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -309,7 +309,7 @@ def test_read_legacy_image_tiles_uncompressed_planes(tmp_path: Path, imagej: boo tifffile.imwrite(hyperstack, stack[None], imagej=True, metadata={"axes": "TCYX"}) for source in (files, [hyperstack]): - image, _ = _read_legacy_image(source, None, {}, image_models_kwargs={"chunks": (1, 32, 64)}) + image, _ = _read_legacy_image(source, None, image_models_kwargs={"chunks": (1, 32, 64)}) assert max(image.chunks[-2]) <= 32 assert max(image.chunks[-1]) <= 64 np.testing.assert_array_equal(image.compute(), stack) @@ -361,8 +361,13 @@ def test_codex_region_named_non_cell_csv_is_ignored(tmp_path: Path, with_table: assert list(sdata.images) == ["reg001_image"] -def test_codex_imread_kwargs_warns_when_unused(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path, layout="per_marker") +@pytest.mark.parametrize("qptiff", [False, True]) +def test_codex_imread_kwargs_warns_when_unused(tmp_path: Path, qptiff: bool) -> None: + # every image is read lazily with tifffile, so `imread_kwargs` of the old dask_image reader are ignored + if qptiff: + write_qptiff(tmp_path / "Tonsil_Scan1.qptiff", biomarkers=["DAPI"]) + else: + write_legacy_processor_tree(tmp_path, layout="per_marker") with pytest.warns(UserWarning, match="imread_kwargs"): codex(tmp_path, imread_kwargs={"nframes": 1}, image_models_kwargs=_FAST) From 80a097891ea9cb80a3f8d3b2574712c3854a0cdb Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:13:12 +0200 Subject: [PATCH 049/107] fix: ignore blank ImageJ slice labels as channel names --- src/spatialdata_io/readers/_utils/_akoya.py | 6 +++++- tests/test_codex.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index f67cf950..7bc14dea 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -225,7 +225,8 @@ def read_tiff(path: Path, chunks: tuple[int, int]) -> tuple[da.Array, tuple[int, Uncompressed contiguous data (e.g. ImageJ hyperstacks and per-marker images, often a single strip per plane) is memory-mapped and split into ``chunks``, so a chunk never needs a whole plane in memory; other layouts keep their native tiles via zarr. Returns the image, the shape of the flattened axes and the ImageJ ``Labels`` (if - any). The array opens the file only when its chunks are read and can be pickled, e.g. for ``dask.distributed``. + every plane has a non-blank one). The array opens the file only when its chunks are read and can be pickled, + e.g. for ``dask.distributed``. """ with tifffile.TiffFile(path) as tf: series = tf.series[0] @@ -234,6 +235,9 @@ def read_tiff(path: Path, chunks: tuple[int, int]) -> tuple[da.Array, tuple[int, dtype = series.dtype.newbyteorder(tf.byteorder) metadata = tf.imagej_metadata or {} labels = list(metadata["Labels"]) if isinstance(metadata.get("Labels"), list | tuple) else None + # ImageJ writes an empty label for every slice without one, so partial labels do not name the channels + if labels is not None and not all(isinstance(label, str) and label.strip() for label in labels): + labels = None if offset is not None: source = _ContiguousTiff(path, offset, shape, dtype) tiles = dict.fromkeys(range(source.ndim), 1) | {axes.index("Y"): chunks[0], axes.index("X"): chunks[1]} diff --git a/tests/test_codex.py b/tests/test_codex.py index 55e434cb..c57f837a 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -100,6 +100,22 @@ def test_codex_hyperstack_imagej_labels_win(tmp_path: Path) -> None: assert get_channel_names(sdata.images["reg001_image"]) == labels +@pytest.mark.parametrize("labels", [["", "", "", ""], ["cyc1.tif", "", "", ""]], ids=["blank", "partly_blank"]) +def test_codex_hyperstack_blank_imagej_labels_ignored(tmp_path: Path, labels: list[str]) -> None: + # ImageJ writes an empty label for every unlabelled slice; such labels must not replace channelNames.txt + tifffile.imwrite( + tmp_path / "reg001_montage.tif", + np.zeros((4, 32, 48), dtype=np.uint16), + imagej=True, + metadata={"axes": "CYX", "Labels": labels}, + ) + (tmp_path / "channelNames.txt").write_text("DAPI\nCD3\nDAPI2\nCD8\n") + + sdata = codex(tmp_path, image_models_kwargs=_FAST) + + assert get_channel_names(sdata.images["reg001_image"]) == ["DAPI", "CD3", "DAPI2", "CD8"] + + def test_codex_cim_qptiff(tmp_path: Path) -> None: path = tmp_path / "experiment_reg001.qptiff" data = write_qptiff( From 2d74dca80aabc260071a21a2f6628113d6796c8e Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:13:46 +0200 Subject: [PATCH 050/107] fix: reject a mapping or set as channel_names --- src/spatialdata_io/readers/_utils/_akoya.py | 4 +++- tests/readers/test_akoya_utils.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index 7bc14dea..5208a30e 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -8,7 +8,7 @@ import threading import warnings from collections import Counter -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence, Set from dataclasses import asdict from typing import TYPE_CHECKING, Any @@ -60,6 +60,8 @@ def apply_channel_names(scan: AkoyaScan, channel_names: Sequence[str] | None) -> """Return the channel names of ``scan``: the user override if given, else the de-duplicated metadata names.""" if channel_names is None: return dedupe_channel_names([channel.name for channel in scan.channels]) + if isinstance(channel_names, Mapping | Set): + raise TypeError(f"`channel_names` must be a sequence of names, got {type(channel_names).__name__}.") if len(channel_names) != len(scan.channels): raise ValueError( f"`channel_names` has {len(channel_names)} entries, but region '{scan.region}' ({scan.path}) has " diff --git a/tests/readers/test_akoya_utils.py b/tests/readers/test_akoya_utils.py index fbf12235..833e7c17 100644 --- a/tests/readers/test_akoya_utils.py +++ b/tests/readers/test_akoya_utils.py @@ -60,6 +60,13 @@ def test_apply_channel_names_override_and_default(): assert apply_channel_names(scan, ["a", "b", "c"]) == ["a", "b", "c"] +@pytest.mark.parametrize("names", [{"a": 0, "b": 1, "c": 2}, {"a", "b", "c"}], ids=["mapping", "set"]) +def test_apply_channel_names_unordered_raises(names: object): + # the keys of a mapping or the iteration order of a set would silently become the channel names + with pytest.raises(TypeError, match="sequence"): + apply_channel_names(_scan(), names) # type: ignore[arg-type] + + def test_apply_channel_names_length_mismatch(): with pytest.raises(ValueError, match="expected 3"): apply_channel_names(_scan(), ["a", "b"]) From 6705d1175f429534a1a9e8e782f2fbe3db1fa8d0 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:17:24 +0200 Subject: [PATCH 051/107] test: make the laziness tests fail on eager pixel reads --- tests/_akoya_data.py | 17 +++++++++++++++++ tests/readers/test_cell_tables.py | 18 +++++++++--------- tests/test_codex.py | 16 ++++++++++++++-- tests/test_phenocycler.py | 15 +++++++-------- 4 files changed, 47 insertions(+), 19 deletions(-) diff --git a/tests/_akoya_data.py b/tests/_akoya_data.py index c2cf6ac5..73a5d99b 100644 --- a/tests/_akoya_data.py +++ b/tests/_akoya_data.py @@ -9,9 +9,13 @@ import pandas as pd import tifffile +from spatialdata_io.readers._utils import _akoya + if TYPE_CHECKING: from pathlib import Path + import pytest + _XML_DECLARATION = '' @@ -321,3 +325,16 @@ def write_legacy_processor_tree( if csv and not (layout == "old_flat" and fcs): cells.to_csv(table_dir / f"{table_stem}.csv", index=False) return {"markers": markers, "images": images, "cells": cells} + + +def forbid_pixel_reads(monkeypatch: pytest.MonkeyPatch) -> None: + """Make every read of pixel data fail: the chunk reads of the lazy tiff arrays and tifffile's own decoding.""" + + def fail(*args: object, **kwargs: object) -> None: + raise AssertionError("pixel data read eagerly") + + monkeypatch.setattr(_akoya._TiffZarrSource, "__getitem__", fail) + monkeypatch.setattr(_akoya._ContiguousTiff, "__getitem__", fail) + monkeypatch.setattr(tifffile.TiffPage, "decode", property(lambda self: fail)) + monkeypatch.setattr(tifffile.TiffPage, "asarray", fail) + monkeypatch.setattr(tifffile.TiffFile, "asarray", fail) diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index 21b528c9..1a7e2501 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -12,7 +12,7 @@ from spatialdata import get_extent, read_zarr from spatialdata.models import Labels2DModel, get_axes_names from spatialdata.transformations import Identity, Scale, Sequence, Translation, get_transformation -from tests._akoya_data import write_fcs, write_qptiff +from tests._akoya_data import forbid_pixel_reads, write_fcs, write_qptiff from spatialdata_io import phenocycler from spatialdata_io.readers._utils._akoya import dedupe_channel_names @@ -866,18 +866,18 @@ def test_read_labels_errors(tmp_path: Path) -> None: read_labels(path, {}, {"global": Identity()}, mask_channel=5) -def test_read_labels_pyramidal_ome_is_lazy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +@pytest.mark.parametrize("compression", [None, "zlib"]) +def test_read_labels_pyramidal_ome_is_lazy(tmp_path: Path, compression: str | None) -> None: mask = np.repeat(np.arange(64, dtype=np.uint16), 48).reshape(64, 48) path = tmp_path / "mask.ome.tif" + tile = (16, 16) if compression else None with tifffile.TiffWriter(path) as tw: - tw.write(mask, subifds=1, tile=(16, 16), compression="zlib", metadata={"axes": "YX"}) - tw.write(mask[::2, ::2], subfiletype=1, tile=(16, 16), compression="zlib") + tw.write(mask, subifds=1, tile=tile, compression=compression, metadata={"axes": "YX"}) + tw.write(mask[::2, ::2], subfiletype=1, tile=tile, compression=compression) - def fail(*args: object, **kwargs: object) -> None: - raise AssertionError("full page decode during read") - - monkeypatch.setattr(tifffile.TiffPage, "asarray", fail) - labels = read_labels(path, {"scale_factors": None}, {"global": Identity()}) + with pytest.MonkeyPatch.context() as monkeypatch: + forbid_pixel_reads(monkeypatch) + labels = read_labels(path, {"scale_factors": None}, {"global": Identity()}) assert labels.shape == (64, 48) np.testing.assert_array_equal(labels.values, mask) diff --git a/tests/test_codex.py b/tests/test_codex.py index c57f837a..3e163eaa 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -4,7 +4,7 @@ import json import shutil -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal import numpy as np import pandas as pd @@ -17,7 +17,7 @@ from spatialdata_io import codex, phenocycler from spatialdata_io.readers._utils._akoya import dedupe_channel_names from spatialdata_io.readers.codex import _discover_legacy, _read_legacy_image -from tests._akoya_data import cim_experiment_v4_xml, write_legacy_processor_tree, write_qptiff +from tests._akoya_data import cim_experiment_v4_xml, forbid_pixel_reads, write_legacy_processor_tree, write_qptiff if TYPE_CHECKING: from pathlib import Path @@ -377,6 +377,18 @@ def test_codex_region_named_non_cell_csv_is_ignored(tmp_path: Path, with_table: assert list(sdata.images) == ["reg001_image"] +@pytest.mark.parametrize("layout", ["per_marker", "hyperstack"]) +def test_codex_is_lazy(tmp_path: Path, layout: Literal["per_marker", "hyperstack"]) -> None: + data = write_legacy_processor_tree(tmp_path, layout=layout) + + with pytest.MonkeyPatch.context() as monkeypatch: + forbid_pixel_reads(monkeypatch) + sdata = codex(tmp_path, image_models_kwargs=_FAST) + + expected = data["images"]["reg001"] + np.testing.assert_array_equal(_level0(sdata, "reg001_image").values, expected.reshape(-1, *expected.shape[-2:])) + + @pytest.mark.parametrize("qptiff", [False, True]) def test_codex_imread_kwargs_warns_when_unused(tmp_path: Path, qptiff: bool) -> None: # every image is read lazily with tifffile, so `imread_kwargs` of the old dask_image reader are ignored diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 63624ea4..76126169 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -18,7 +18,7 @@ from spatialdata_io import phenocycler from spatialdata_io.__main__ import phenocycler_wrapper -from tests._akoya_data import write_qptiff +from tests._akoya_data import forbid_pixel_reads, write_qptiff _FAST = {"scale_factors": [2]} @@ -850,17 +850,16 @@ def test_write_roundtrip(tmp_path: Path) -> None: assert read.attrs["spatialdata_io_reader"] == "phenocycler" -def test_is_lazy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_is_lazy(tmp_path: Path) -> None: path = tmp_path / "S_Scan1.qptiff" - write_qptiff(path, biomarkers=["DAPI", "CD3"]) - - def fail(*args: object, **kwargs: object) -> None: - raise AssertionError("full page decode during read") + written = write_qptiff(path, biomarkers=["DAPI", "CD3"]) - monkeypatch.setattr(tifffile.TiffPage, "asarray", fail) - sdata = phenocycler(path, image_models_kwargs=_FAST) + with pytest.MonkeyPatch.context() as monkeypatch: + forbid_pixel_reads(monkeypatch) + sdata = phenocycler(path, image_models_kwargs=_FAST) assert isinstance(_level0(sdata, "S_image").data, da.Array) + np.testing.assert_array_equal(_level0(sdata, "S_image").values, written) def test_phenocycler_keyword_only(tmp_path: Path) -> None: From 9604d2290eb06136b842c751afcde59e80ffcfe2 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:19:16 +0200 Subject: [PATCH 052/107] fix: de-duplicate channel_names overrides without suffix collisions --- src/spatialdata_io/readers/_utils/_akoya.py | 19 +++++++++++++++---- tests/readers/test_akoya_utils.py | 8 ++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index 5208a30e..e224e2ff 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -47,17 +47,28 @@ def dedupe_channel_names(names: Sequence[str]) -> list[str]: - """Make channel names unique as ``name``, ``name (2)``, ``name (3)``; the first occurrence is unchanged.""" + """Make channel names unique as ``name``, ``name (2)``, ``name (3)``; the first occurrence is unchanged. + + A suffix is skipped if it would repeat a name of the list, e.g. ``["A", "A", "A (2)"]`` -> ``A``, ``A (3)``, + ``A (2)``. + """ + taken = set(names) seen: Counter[str] = Counter() deduped = [] for name in names: seen[name] += 1 - deduped.append(name if seen[name] == 1 else f"{name} ({seen[name]})") + if seen[name] == 1: + deduped.append(name) + continue + while (unique := f"{name} ({seen[name]})") in taken: + seen[name] += 1 + taken.add(unique) + deduped.append(unique) return deduped def apply_channel_names(scan: AkoyaScan, channel_names: Sequence[str] | None) -> list[str]: - """Return the channel names of ``scan``: the user override if given, else the de-duplicated metadata names.""" + """Return the de-duplicated channel names of ``scan``: the user override if given, else the metadata names.""" if channel_names is None: return dedupe_channel_names([channel.name for channel in scan.channels]) if isinstance(channel_names, Mapping | Set): @@ -67,7 +78,7 @@ def apply_channel_names(scan: AkoyaScan, channel_names: Sequence[str] | None) -> f"`channel_names` has {len(channel_names)} entries, but region '{scan.region}' ({scan.path}) has " f"{len(scan.channels)} channels: expected {len(scan.channels)}." ) - return list(channel_names) + return dedupe_channel_names(list(channel_names)) def region_transformations( diff --git a/tests/readers/test_akoya_utils.py b/tests/readers/test_akoya_utils.py index 833e7c17..22fb6817 100644 --- a/tests/readers/test_akoya_utils.py +++ b/tests/readers/test_akoya_utils.py @@ -54,10 +54,18 @@ def test_dedupe_channel_names(): assert dedupe_channel_names(["DAPI", "CD3", "DAPI", "DAPI"]) == ["DAPI", "CD3", "DAPI (2)", "DAPI (3)"] +def test_dedupe_channel_names_never_collides(): + # a generated suffix must not repeat a name that is already in the list + assert dedupe_channel_names(["DAPI", "DAPI", "DAPI (2)"]) == ["DAPI", "DAPI (3)", "DAPI (2)"] + assert dedupe_channel_names(["A (2)", "A", "A"]) == ["A (2)", "A", "A (3)"] + + def test_apply_channel_names_override_and_default(): scan = _scan(names=("DAPI", "DAPI", "CD3")) assert apply_channel_names(scan, None) == ["DAPI", "DAPI (2)", "CD3"] assert apply_channel_names(scan, ["a", "b", "c"]) == ["a", "b", "c"] + # an override is de-duplicated like the metadata names, so channels stay selectable by name + assert apply_channel_names(scan, ["a", "b", "a"]) == ["a", "b", "a (2)"] @pytest.mark.parametrize("names", [{"a": 0, "b": 1, "c": 2}, {"a", "b", "c"}], ids=["mapping", "set"]) From 50ecef05ecc53496b32e1a49c78139fd549cd55d Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:19:46 +0200 Subject: [PATCH 053/107] fix: accept array-like regions --- src/spatialdata_io/readers/_utils/_akoya.py | 5 +++-- tests/readers/test_akoya_utils.py | 9 +++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index e224e2ff..75d3ffbb 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -200,9 +200,10 @@ def normalize_regions(regions: str | Sequence[str] | None) -> list[str] | None: return [sanitize_name(regions)] if isinstance(regions, Mapping): raise TypeError(f"`regions` must be a sequence of region ids, got {type(regions).__name__}.") - if not regions: + sanitized = list(dict.fromkeys(sanitize_name(str(region)) for region in regions)) + if not sanitized: raise ValueError("`regions` is empty; pass `None` to read all regions.") - return list(dict.fromkeys(sanitize_name(str(region)) for region in regions)) + return sanitized def select_regions(available: Sequence[str], regions: str | Sequence[str] | None) -> list[str]: diff --git a/tests/readers/test_akoya_utils.py b/tests/readers/test_akoya_utils.py index 22fb6817..79cdf8c5 100644 --- a/tests/readers/test_akoya_utils.py +++ b/tests/readers/test_akoya_utils.py @@ -9,6 +9,7 @@ import dask import dask.array as da import numpy as np +import pandas as pd import pytest import tifffile from spatialdata.transformations import Identity, Translation, get_transformation @@ -147,6 +148,14 @@ def test_select_regions(): assert select_regions(["a", "b"], ["b"]) == ["b"] +def test_select_regions_array_like(): + # e.g. `sdata.tables["table"].obs["region"].unique()` or a `pd.Index` + assert select_regions(["a", "b"], np.array(["b"])) == ["b"] # type: ignore[arg-type] + assert select_regions(["a", "b"], pd.Index(["a", "b"])) == ["a", "b"] # type: ignore[arg-type] + with pytest.raises(ValueError, match="empty"): + select_regions(["a", "b"], np.array([], dtype=str)) # type: ignore[arg-type] + + def test_select_regions_deduplicates(): assert select_regions(["a", "b"], ["b", "a", "b"]) == ["b", "a"] assert select_regions(["a", "b"], "b") == ["b"] From 6898500a0519ddbfed90000332df2459b85b3a18 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:20:54 +0200 Subject: [PATCH 054/107] fix: keep non-ASCII letters when matching marker names --- src/spatialdata_io/readers/_utils/_qptiff.py | 4 ++-- tests/readers/test_qptiff.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_qptiff.py b/src/spatialdata_io/readers/_utils/_qptiff.py index 46c88a16..57335820 100644 --- a/src/spatialdata_io/readers/_utils/_qptiff.py +++ b/src/spatialdata_io/readers/_utils/_qptiff.py @@ -464,8 +464,8 @@ def _is_blank(marker: str) -> bool: def _name_key(name: str) -> str: - # CIM sanitises page Biomarker names (``F4/80`` -> ``F4-80``), so compare alphanumerics only - return re.sub(r"[^0-9a-z]", "", name.casefold()) + # CIM sanitises page Biomarker names (``F4/80`` -> ``F4-80``), so compare alphanumerics only (in any script) + return "".join(char for char in name.casefold() if char.isalnum()) def _enrich(channels: list[AkoyaChannel], records: list[tuple[str, dict[str, Any]]]) -> None: diff --git a/tests/readers/test_qptiff.py b/tests/readers/test_qptiff.py index 12c450ba..808433a9 100644 --- a/tests/readers/test_qptiff.py +++ b/tests/readers/test_qptiff.py @@ -179,6 +179,18 @@ def test_enrichment_not_positional_when_orders_differ(tmp_path: Path) -> None: assert [c.channel for c in scan.channels] == [1, 2, 4, 3] +def test_enrichment_keeps_non_ascii_names_apart(tmp_path: Path) -> None: + # names that differ only in non-ASCII letters must not be paired with each other's records; the ASCII page + # descriptions hold them as character references + p = tmp_path / "x.qptiff" + wells = [[("DAPI", "DAPI"), ("TCRαβ", "ATTO 550"), ("TCRγδ", "CY5")]] + biomarkers: list[str | None] = ["DAPI", "TCRγδ", "TCRαβ"] + write_qptiff(p, biomarkers=biomarkers, scan_profile=fusion_scan_profile_json(wells)) + scan = read_qptiff_metadata(p) + assert [c.name for c in scan.channels] == ["DAPI", "TCRγδ", "TCRαβ"] + assert [c.channel for c in scan.channels] == [1, 3, 2] + + def test_profile_record_without_marker_skipped(tmp_path: Path) -> None: p = tmp_path / "x.qptiff" profile = json.loads(fusion_scan_profile_json([[("DAPI", "DAPI"), ("CD3", "ATTO 550")]])) From 2418371e3f2bdf1949c2e3d290ea58760fa9ea31 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:21:46 +0200 Subject: [PATCH 055/107] test: cover 8-bit duplicates under a stage translation --- tests/test_phenocycler.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 76126169..0d989b0e 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -193,6 +193,17 @@ def test_read_all_bit_depths_shares_stage_placement(tmp_path: Path) -> None: assert sorted(sdata.images) == ["B_8bit_image", "B_image"] assert get_transformation(sdata["B_8bit_image"], get_all=True) == get_transformation(sdata["B_image"], get_all=True) + # with both regions placed by their stage positions, the 8-bit image gets the translation of its region + sdata = phenocycler(tmp_path, read_all_bit_depths=True, image_models_kwargs=_FAST) + + assert sorted(sdata.images) == ["A_8bit_image", "A_image", "B_8bit_image", "B_image"] + translation = get_transformation(sdata["B_image"], "global") + assert isinstance(translation, Translation) + assert translation.translation[0] > 0 + for region in ("A", "B"): + expected = get_transformation(sdata[f"{region}_image"], get_all=True) + assert get_transformation(sdata[f"{region}_8bit_image"], get_all=True) == expected + def test_read_all_bit_depths_with_labels_and_cells_table(tmp_path: Path) -> None: for suffix, dtype in ((".qptiff", "uint8"), (".er.qptiff", "uint16")): From a9a5c1d9d791d4f42c54034371b9a657d2d2ac07 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:01:15 +0200 Subject: [PATCH 056/107] fix(codex): discover only CODEX FCS tables and raise on ambiguous tables An .fcs file now counts as a cell table only if its $PnN channels make it a CODEX table (read from the TEXT segment); unparseable or foreign FCS files are skipped. Compensated beats uncompensated before the file format, which only chooses between the .fcs and .csv of the same table. Any other tie for one region raises a ValueError naming the candidates instead of picking the shallowest or lexicographically first path (segm-1 vs segm-10). --- CHANGELOG.md | 5 +- .../readers/_utils/_cell_tables.py | 25 ++++++-- src/spatialdata_io/readers/codex.py | 60 ++++++++++++------- tests/test_codex.py | 42 ++++++++++++- 4 files changed, 101 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0814e294..4719109c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,8 +73,9 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. - A single region is stored in `"global"`. Several regions are placed at their stage positions in `"global"` only when all of them share a sample space (same slide id, stage positions and pixel size); otherwise each region, including multi-region reads of unrelated regions, gets its own coordinate system named after the region. - - `channelNames.txt` and `experiment.json` are found in any case (e.g. SPACEc's `channelnames.txt`); CSV files - that are not CODEX Processor cell tables are ignored, also when their name holds a region id. + - `channelNames.txt` and `experiment.json` are found in any case (e.g. SPACEc's `channelnames.txt`); CSV and FCS + files that are not CODEX Processor cell tables are ignored, also when their name holds a region id. Compensated + tables are preferred; other tables of the same region (e.g. of two segmentation runs) raise a `ValueError`. - A CIM `.qptiff` without `regNNN` in its name is named as `phenocycler()` names it (without the suffix and a trailing `_Scan`), and its `.er.qptiff` is preferred. diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index c5f7a856..ea4b1757 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any import anndata as ad +import flowio # type: ignore[import-untyped] # readfcs' parser, for the TEXT segment only import geopandas as gpd import numpy as np import pandas as pd @@ -301,14 +302,28 @@ def detect_table_format(path: Path) -> str: Returns ------- - ``"codex"`` (FCS, or ``x``/``y`` with ``cell_id`` or ``cycCCC_chHHH`` columns), ``"qupath"`` (``Centroid X µm``, - ``Centroid X px`` or ``Object ID``, also in R's ``make.names`` form, e.g. ``Centroid.X.um``), ``"mcquant"`` - (``CellID`` and ``X_centroid``), ``"inform"`` (``Cell X Position``, optionally followed by its unit in - parentheses), ``"sprm"`` (first column ``ID``), else ``"generic"``. + ``"codex"`` (``x``/``y`` with ``cell_id`` or ``cycCCC_chHHH`` columns, also as the channels of an FCS file), + ``"qupath"`` (``Centroid X µm``, ``Centroid X px`` or ``Object ID``, also in R's ``make.names`` form, e.g. + ``Centroid.X.um``), ``"mcquant"`` (``CellID`` and ``X_centroid``), ``"inform"`` (``Cell X Position``, + optionally followed by its unit in parentheses), ``"sprm"`` (first column ``ID``), else ``"generic"``. + + Raises + ------ + ValueError + If ``path`` is an FCS file that cannot be parsed or is not a CODEX table (its ``$PnN`` channel names). """ if path.suffix.lower() == CodexKeys.FCS_FILE.value: + try: + columns = list(flowio.read_multiple_data_sets(str(path), only_text=True)[0].pnn_labels) + except (ValueError, flowio.exceptions.FlowIOException) as e: + raise ValueError(f"{path} is not a readable FCS file: {e}") from e + if _detect_header_format(columns) != "codex": + raise ValueError(f"{path} is not a CODEX cell table: its channels are {columns}.") return "codex" - columns, _ = _read_header(path) + return _detect_header_format(_read_header(path)[0]) + + +def _detect_header_format(columns: list[str]) -> str: # the name part of the CODEX Processor header form `cyc001_ch001:DAPI1` names = {column.partition(":")[0].casefold() for column in columns} if {"x", "y"} <= names and ( diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 614acef9..2024aac2 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -111,13 +111,14 @@ def codex( - ``{vx.EXPERIMENT_FILE!r}``: optional; the pixel size is taken from its ``xyResolution`` (nm). Like ``{vx.CHANNEL_NAMES_FILE!r}``, it is found in ``path`` or up to three folders above it, ignoring the case of the file name (e.g. ``channelnames.txt``). - - ``*{vx.FCS_FILE!r}`` / ``*{vx.FCS_FILE_CSV!r}``: cell tables of the CODEX Processor, one per region (or - one for all regions), e.g. ``segm/segm-1/fcs/compensated/regNNN_compensated.fcs``. Compensated tables - are preferred over uncompensated ones. The ``cycCCC_chHHH`` marker columns are stored in ``X`` (dense - float32) under their marker names, taken from the FCS ``$PnS`` keywords or the CSV header - ``cycCCC_chHHH:``; all other columns go to ``obs``, the ``x``/``y`` pixel centroids to - ``obsm["spatial"]``. Other CSV files (without ``x``/``y`` and ``cell_id`` or ``cycCCC_chHHH`` columns, - e.g. antibody lists or HuBMAP SPRM outputs) are ignored; pass them as ``cells_table`` to read them. + - ``*{vx.FCS_FILE!r}`` / ``*{vx.FCS_FILE_CSV!r}``: cell tables of the CODEX Processor, one per region (or one + for all regions), e.g. ``segm/segm-1/fcs/compensated/regNNN_compensated.fcs``. Compensated tables are + preferred over uncompensated ones; other tables of the same region (e.g. of a second segmentation run) raise + an error. The ``cycCCC_chHHH`` marker columns are stored in ``X`` (dense float32) under their marker names, + taken from the FCS ``$PnS`` keywords or the CSV header ``cycCCC_chHHH:``; all other columns go to + ``obs``, the ``x``/``y`` pixel centroids to ``obsm["spatial"]``. Other CSV files (without ``x``/``y`` and + ``cell_id`` or ``cycCCC_chHHH`` columns, e.g. antibody lists or HuBMAP SPRM outputs) are ignored; pass them as + ``cells_table`` to read them. FCS files of other instruments are ignored too. Raw acquisition and intermediate folders (``cycCCC_regNNN/``, ``tiles/`` and, next to ``stitched/``, ``bestFocus/``) are ignored. The region id ``regNNN`` is taken from the file names; a ``{vx.QPTIFF!r}`` without @@ -521,31 +522,44 @@ def _discover_tables(files: Sequence[Path], fcs: bool) -> dict[str, Path]: preferred = CodexKeys.FCS_FILE.value if fcs else CodexKeys.FCS_FILE_CSV.value candidates: dict[str, list[Path]] = {} for file in files: - if file.suffix.lower() not in _TABLE_SUFFIXES: - continue - # a CSV may be any sidecar, also with a region in its name (e.g. `reg001_antibodies.csv` or a HuBMAP SPRM - # `reg001_...-cell_channel_mean.csv`); only CODEX Processor tables are discovered - if detect_table_format(file) != "codex": - continue - candidates.setdefault(_region_id(file.name) or "*", []).append(file) + if file.suffix.lower() in _TABLE_SUFFIXES and _is_codex_table(file): + candidates.setdefault(_region_id(file.name) or "*", []).append(file) - def rank(file: Path) -> tuple[bool, bool]: - # the preferred format first, then compensated over uncompensated - return file.suffix.lower() != preferred, any("uncompensated" in part.casefold() for part in file.parts) + def uncompensated(file: Path) -> bool: + return any("uncompensated" in part.casefold() for part in file.parts) tables = {} for region, paths in candidates.items(): - best = min(rank(path) for path in paths) - paths = [path for path in paths if rank(path) == best] - if region == "*" and len(paths) > 1: + best = min(map(uncompensated, paths)) + # the same table may be written as both .fcs and .csv: the preferred format is read + same_table: dict[Path, list[Path]] = {} + for path in sorted(paths): + if uncompensated(path) == best: + same_table.setdefault(path.with_suffix(""), []).append(path) + if len(same_table) > 1: + where = "without a region in their file name" if region == "*" else f"of region '{region}'" raise ValueError( - f"Found several cell tables without a region in their file name: {[str(p) for p in paths]}; " - "the reader cannot choose between them. Pass the path of a directory that contains only one." + f"Found several cell tables {where}: {[str(p) for group in same_table.values() for p in group]}; the reader cannot " + "choose between them. Pass the path of a directory that contains only one, or pass the table as " + "`cells_table`." ) - tables[region] = min(paths, key=lambda path: (len(path.parts), str(path))) + (formats,) = same_table.values() + tables[region] = min(formats, key=lambda path: path.suffix.lower() != preferred) return tables +def _is_codex_table(file: Path) -> bool: + """Whether ``file`` is a CODEX Processor table. + + A CSV may be any sidecar, also with a region in its name (e.g. ``reg001_antibodies.csv`` or a HuBMAP SPRM + ``reg001_...-cell_channel_mean.csv``), an FCS file the export of any flow cytometer. + """ + try: + return detect_table_format(file) == "codex" + except ValueError: + return False + + def _find_sidecar(path: Path, files: Sequence[Path], name: str) -> Path | None: """The shallowest file called ``name`` in ``files``, else in the nearest ancestor of ``path`` holding one. diff --git a/tests/test_codex.py b/tests/test_codex.py index 3e163eaa..b8ac99ce 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -17,7 +17,13 @@ from spatialdata_io import codex, phenocycler from spatialdata_io.readers._utils._akoya import dedupe_channel_names from spatialdata_io.readers.codex import _discover_legacy, _read_legacy_image -from tests._akoya_data import cim_experiment_v4_xml, forbid_pixel_reads, write_legacy_processor_tree, write_qptiff +from tests._akoya_data import ( + cim_experiment_v4_xml, + forbid_pixel_reads, + write_fcs, + write_legacy_processor_tree, + write_qptiff, +) if TYPE_CHECKING: from pathlib import Path @@ -214,6 +220,40 @@ def test_discover_legacy_tables_prefer_requested_format(tmp_path: Path) -> None: } +def test_discover_legacy_tables_ranking(tmp_path: Path) -> None: + cells = pd.DataFrame({"cell_id": [1.0, 2.0], "x": [1.0, 2.0], "y": [3.0, 4.0], "cyc001_ch001": [5.0, 6.0]}) + tifffile.imwrite(tmp_path / "Kidney.tif", np.zeros((2, 16, 16), dtype=np.uint16)) + cells.to_csv(tmp_path / "Kidney.csv", index=False) + # an unrelated flow cytometry tube and a file that is no FCS at all do not replace the CODEX table + (tmp_path / "flow").mkdir() + write_fcs(tmp_path / "flow" / "tube1.fcs", pd.DataFrame({"FSC-A": [1.0], "SSC-A": [2.0]})) + (tmp_path / "flow" / "broken.fcs").write_bytes(b"not an fcs file") + assert _discover_legacy(tmp_path, fcs=True).tables == {"*": tmp_path / "Kidney.csv"} + + # a CODEX gate export next to the table: the reader cannot tell which one holds the cells + (tmp_path / "gates").mkdir() + write_fcs(tmp_path / "gates" / "CD3pos.fcs", cells.iloc[:1]) + with pytest.raises(ValueError, match=r"CD3pos\.fcs.*Kidney\.csv|Kidney\.csv.*CD3pos\.fcs"): + _discover_legacy(tmp_path, fcs=True) + + # compensated beats uncompensated before the file format + fcs_dir = tmp_path / "exp" / "segm" / "segm-1" / "fcs" + for kind in ("compensated", "uncompensated"): + (fcs_dir / kind).mkdir(parents=True) + cells.to_csv(fcs_dir / "compensated" / "reg001_compensated.csv", index=False) + write_fcs(fcs_dir / "uncompensated" / "reg001_uncompensated.fcs", cells) + assert _discover_legacy(tmp_path / "exp", fcs=True).tables == { + "reg001": fcs_dir / "compensated" / "reg001_compensated.csv" + } + + # two segmentation runs of one region are not resolved by their path + segm_10 = tmp_path / "exp" / "segm" / "segm-10" / "fcs" / "compensated" + segm_10.mkdir(parents=True) + cells.to_csv(segm_10 / "reg001_compensated.csv", index=False) + with pytest.raises(ValueError, match=r"segm-1/.*segm-10/|segm-10/.*segm-1/"): + _discover_legacy(tmp_path / "exp", fcs=True) + + def test_codex_user_cells_table_replaces_discovered(tmp_path: Path) -> None: write_legacy_processor_tree(tmp_path, regions=2, layout="per_marker", n_cells=50) pd.DataFrame( From be8993f066469bca9640579aacbea16d17f4acdc Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:04:42 +0200 Subject: [PATCH 057/107] fix(codex): let explicit cells_table and labels override discovery A given cells_table disables table discovery (so ambiguous discovered tables no longer raise), and the labels masks are excluded from image discovery (HuBMAP reg001_mask.ome.tiff next to reg001_expr.ome.tiff). The explicit branch is chosen by cells_table, not by whether it resolved to tables for the selected regions, so a table of an unselected region no longer brings the discovered tables back. Regions that only appear in the region column of a region-less table are now available to regions=. --- CHANGELOG.md | 1 + src/spatialdata_io/readers/codex.py | 105 +++++++++++++++------------- tests/test_codex.py | 40 +++++++++++ 3 files changed, 99 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4719109c..b681e0bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,7 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. - `channelNames.txt` and `experiment.json` are found in any case (e.g. SPACEc's `channelnames.txt`); CSV and FCS files that are not CODEX Processor cell tables are ignored, also when their name holds a region id. Compensated tables are preferred; other tables of the same region (e.g. of two segmentation runs) raise a `ValueError`. + A `cells_table` disables table discovery, and the `labels` masks are not read as images. - A CIM `.qptiff` without `regNNN` in its name is named as `phenocycler()` names it (without the suffix and a trailing `_Scan`), and its `.er.qptiff` is preferred. diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 2024aac2..82fd6f36 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -3,7 +3,7 @@ import os import re import warnings -from collections.abc import Mapping +from collections.abc import Collection, Iterable, Mapping from dataclasses import dataclass from pathlib import Path from types import MappingProxyType @@ -176,12 +176,12 @@ def codex( (columns given by ``cells_table_kwargs``). labels A segmentation mask (TIFF/OME-TIFF with integer instance ids) per region: a path for a single region, or a - mapping from region id to path. Each mask is stored lazily as the labels ``_labels`` - with the transformation of the region's image; the cells of the region annotate it instead of circles, and - their instance ids must be the mask values. A mask whose shape is the region image's divided by an integer - factor ``f`` (exactly, or rounded up) is scaled onto the image (``Scale([f, f])`` before the region's - transformation); any other shape mismatch raises a warning. The pixel centroids of ``"mcquant"`` and - ``"sprm"`` tables are taken to be in mask pixels, so their centroids (``obsm["spatial"]``) and circle radii + mapping from region id to path; a mask inside ``path`` is not read as an image. Each mask is stored lazily as + the labels ``_labels`` with the transformation of the region's image; the cells of the region annotate + it instead of circles, and their instance ids must be the mask values. A mask whose shape is the region image's + divided by an integer factor ``f`` (exactly, or rounded up) is scaled onto the image (``Scale([f, f])`` before + the region's transformation); any other shape mismatch raises a warning. The pixel centroids of ``"mcquant"`` + and ``"sprm"`` tables are taken to be in mask pixels, so their centroids (``obsm["spatial"]``) and circle radii are scaled by ``f`` to image pixels; the source columns in ``obs`` are kept as read (mask pixels). The pixel centroids of other tables are in image pixels and are not scaled; ``cells_table_kwargs["coordinates_space"]="mask"`` scales those of a ``"generic"`` table too. @@ -215,11 +215,17 @@ def codex( regions = normalize_regions(regions) channel_names = [channel_names] if isinstance(channel_names, str) else channel_names path = Path(path) - layout = _discover_legacy(path, fcs) - # regions that only have a discovered table exist only while the discovered tables are read - table_regions = layout.tables.keys() - {"*"} if cells_table is None else set() - available = sorted(layout.images.keys() | table_regions) + # explicit beats discovered: a given table disables table discovery, and a given mask is no image + masks = labels.values() if isinstance(labels, Mapping) else [] if labels is None else [labels] + layout = _discover_legacy( + path, fcs, find_tables=cells_table is None, masks={Path(mask).resolve() for mask in masks} + ) + # the regions of a table without a region in its file name are known only once it is read + regionless = _read_regionless_table(layout.tables["*"], layout) if "*" in layout.tables else None + table_regions = set(regionless.region.unique()) if regionless is not None else set() + available = sorted(layout.images.keys() | layout.tables.keys() - {"*"} | table_regions) selected = select_regions(available, regions) + _warn_fcs_fallback([file for key, file in layout.tables.items() if key == "*" or key in selected], fcs) table_files = resolve_per_region(cells_table, selected, available) mask_files = resolve_per_region(labels, selected, available) experiment = read_json_object(layout.experiment_json) if layout.experiment_json is not None else None @@ -257,7 +263,7 @@ def codex( ) names = {scan.region: apply_channel_names(scan, channel_names) for scan in scans} - if table_files: + if cells_table is not None: cell_tables = read_region_tables( table_files, cells_table_format, @@ -268,10 +274,15 @@ def codex( ) table_paths = {key: str(file) for key, file in table_files.items()} else: - discovered = {key: file for key, file in layout.tables.items() if key == "*" or key in selected} - read = _read_tables(discovered, layout, selected if regions is not None else None, fcs) + named = {key: read_codex_table(layout.tables[key], region=key) for key in selected if key in layout.tables} + read = {"*": regionless, **named} if regionless is not None else named + if regions is not None: + read = { + key: table.take(np.flatnonzero(table.region.isin(selected).to_numpy())) for key, table in read.items() + } + read = {key: table for key, table in read.items() if len(table.region)} cell_tables = list(read.values()) - table_paths = {key: str(discovered[key]) for key in read} + table_paths = {key: str(layout.tables[key]) for key in read} sdata = build_spatialdata( scans, arrays, @@ -286,45 +297,39 @@ def codex( return _set_reader_metadata(sdata, "codex") -def _read_tables( - files: Mapping[str, Path], layout: _LegacyLayout, regions: Sequence[str] | None, fcs: bool -) -> dict[str, CellTable]: - """Read the cell tables of ``files`` (by region, ``"*"`` for one without a region), keeping only ``regions``. - - Returns the tables that have cells of ``regions``. A table without a region in its file name that cannot be - read, or whose cells cannot be assigned to a region, is skipped with a warning. - """ - fallback = [str(file) for file in files.values() if file.suffix.lower() == CodexKeys.FCS_FILE.value] +def _warn_fcs_fallback(files: Iterable[Path], fcs: bool) -> None: + fallback = [str(file) for file in files if file.suffix.lower() == CodexKeys.FCS_FILE.value] if not fcs and fallback: warnings.warn( f"`fcs=False`, but no CSV cell table was found for {fallback}; reading the FCS table(s) instead.", UserWarning, stacklevel=3, ) - only_image = next(iter(layout.images)) if len(layout.images) == 1 else None - tables = {} - for key, file in files.items(): - if key != "*": - table = read_codex_table(file, region=key) - else: - # a table without a region in its file name may be any CSV: skip it rather than fail the whole read - try: - table = read_codex_table(file, region=only_image) - except ValueError as e: - warnings.warn(f"Ignoring the cell table {file}: {e}", UserWarning, stacklevel=3) - continue - # next to a single image named after its file (e.g. `Kidney.tif`), its cells belong to that image - values = table.region.unique() - if only_image is not None and len(values) == 1 and values[0] not in layout.images: - table.region = pd.Series(only_image, index=table.region.index, name=table.region.name) - if regions is not None: - table = table.take(np.flatnonzero(table.region.isin(regions).to_numpy())) - if len(table.region): - tables[key] = table - return tables -def _discover_legacy(path: Path, fcs: bool) -> _LegacyLayout: +def _read_regionless_table(file: Path, layout: _LegacyLayout) -> CellTable | None: + """Read the discovered cell table ``file`` that has no region in its file name. + + A table without a region that cannot be read, or whose cells cannot be assigned to a region, is skipped with a + warning (`None` is returned). + """ + only_image = next(iter(layout.images)) if len(layout.images) == 1 else None + # a table without a region in its file name may be any CSV: skip it rather than fail the whole read + try: + table = read_codex_table(file, region=only_image) + except ValueError as e: + warnings.warn(f"Ignoring the cell table {file}: {e}", UserWarning, stacklevel=3) + return None + # next to a single image named after its file (e.g. `Kidney.tif`), its cells belong to that image + values = table.region.unique() + if only_image is not None and len(values) == 1 and values[0] not in layout.images: + table.region = pd.Series(only_image, index=table.region.index, name=table.region.name) + return table + + +def _discover_legacy( + path: Path, fcs: bool, *, find_tables: bool = True, masks: Collection[Path] = frozenset() +) -> _LegacyLayout: """Find the images, cell tables and sidecar files of a CODEX dataset. Parameters @@ -333,6 +338,10 @@ def _discover_legacy(path: Path, fcs: bool) -> _LegacyLayout: A CODEX output directory, searched recursively in sorted order, or a single image or table file. fcs Whether to prefer ``.fcs`` over ``.csv`` tables of the same region. + find_tables + Whether to discover cell tables. + masks + The resolved paths of the segmentation masks, which are not images. Returns ------- @@ -341,12 +350,14 @@ def _discover_legacy(path: Path, fcs: bool) -> _LegacyLayout: if not path.exists(): raise FileNotFoundError(f"{path} does not exist.") files = [path] if path.is_file() else _walk(path) + if masks: + files = [file for file in files if file.resolve() not in masks] images = _discover_images(files) if not images: # the original (2019) layout: one stitched `.tif` next to its table, without a region in the file name top_level = files if path.is_file() else [file for file in files if file.parent == path] images = {file.stem: [file] for file in top_level if _is_tiff(file)} - tables = _discover_tables(files, fcs) + tables = _discover_tables(files, fcs) if find_tables else {} if not images and not tables: raise FileNotFoundError(f"No CODEX images or cell tables found in {path}. Expected: {'; '.join(_PATTERNS)}.") return _LegacyLayout( diff --git a/tests/test_codex.py b/tests/test_codex.py index b8ac99ce..a5e3b76c 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -280,6 +280,46 @@ def test_codex_user_cells_table_replaces_discovered(tmp_path: Path) -> None: assert sdata.attrs["akoya"]["cell_tables"] == {"reg002": str(tmp_path / "mine.csv")} +def test_codex_explicit_inputs_disable_discovery(tmp_path: Path) -> None: + # HuBMAP-like: the mask has the region id of its image, and two region-less tables are discoverable + for folder in ("expr", "mask"): + (tmp_path / folder).mkdir() + tifffile.imwrite(tmp_path / "expr" / "reg001_expr.ome.tiff", np.zeros((2, 16, 16), dtype=np.uint16)) + mask = tmp_path / "mask" / "reg001_mask.ome.tiff" + tifffile.imwrite(mask, np.arange(256, dtype=np.uint16).reshape(16, 16)) + header = "cell_id,x,y,cyc001_ch001\n" + (tmp_path / "cells_all.csv").write_text(header + "1,1.0,1.0,1.0\n2,2.0,2.0,2.0\n") + (tmp_path / "cells_filtered.csv").write_text(header + "2,2.0,2.0,2.0\n") + + for labels in (mask, {"reg001": mask}): + sdata = codex( + tmp_path, + cells_table=tmp_path / "cells_filtered.csv", + labels=labels, + image_models_kwargs=_FAST, + labels_models_kwargs=_FAST, + ) + assert list(sdata.images) == ["reg001_image"] + assert list(sdata.labels) == ["reg001_labels"] + assert sdata.tables["table"].n_obs == 1 + + # an explicit table of an unselected region does not bring the discovered tables back + write_legacy_processor_tree(tmp_path / "tree", regions=2, cycles=1, channels=1, n_cells=5) + sdata = codex(tmp_path / "tree", regions=["reg001"], cells_table={"reg002": tmp_path / "cells_all.csv"}) + assert not sdata.tables + + +def test_codex_regions_of_regionless_table_are_available(tmp_path: Path) -> None: + data = write_legacy_processor_tree(tmp_path, regions=1, fcs=False, csv=False, n_cells=5) + pd.concat([data["cells"], data["cells"].assign(region=2.0)]).to_csv(tmp_path / "cells.csv", index=False) + + with pytest.warns(UserWarning, match="circles"): + sdata = codex(tmp_path, regions=["reg002"], image_models_kwargs=_FAST) + + assert not sdata.images + assert list(sdata.shapes) == ["reg002_cells"] + + def test_codex_labels_with_discovered_tables(tmp_path: Path) -> None: write_legacy_processor_tree(tmp_path, regions=1, layout="per_marker", n_cells=5) tifffile.imwrite(tmp_path / "mask.tif", np.zeros((128, 160), dtype=np.uint16)) From bd280a423e45f2273b2df4b61b65c4327d046d54 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:05:35 +0200 Subject: [PATCH 058/107] fix(codex): raise when a discovered CODEX table cannot be read A region-less table that discovery has already identified as a CODEX table (by its header) was dropped with a warning when reading it failed, so codex() and the CLI wrote image-only output, or an empty SpatialData for a bare .fcs without a region column. Such errors now propagate. --- src/spatialdata_io/readers/codex.py | 15 +++------------ tests/test_codex.py | 21 ++++++++++++++------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 82fd6f36..6ff7ad35 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -307,19 +307,10 @@ def _warn_fcs_fallback(files: Iterable[Path], fcs: bool) -> None: ) -def _read_regionless_table(file: Path, layout: _LegacyLayout) -> CellTable | None: - """Read the discovered cell table ``file`` that has no region in its file name. - - A table without a region that cannot be read, or whose cells cannot be assigned to a region, is skipped with a - warning (`None` is returned). - """ +def _read_regionless_table(file: Path, layout: _LegacyLayout) -> CellTable: + """Read the discovered cell table ``file`` that has no region in its file name.""" only_image = next(iter(layout.images)) if len(layout.images) == 1 else None - # a table without a region in its file name may be any CSV: skip it rather than fail the whole read - try: - table = read_codex_table(file, region=only_image) - except ValueError as e: - warnings.warn(f"Ignoring the cell table {file}: {e}", UserWarning, stacklevel=3) - return None + table = read_codex_table(file, region=only_image) # next to a single image named after its file (e.g. `Kidney.tif`), its cells belong to that image values = table.region.unique() if only_image is not None and len(values) == 1 and values[0] not in layout.images: diff --git a/tests/test_codex.py b/tests/test_codex.py index a5e3b76c..9aa66038 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -596,19 +596,26 @@ def test_codex_single_fcs_file(tmp_path: Path) -> None: assert sdata.tables["table"].n_obs == 50 -def test_codex_unreadable_regionless_table_is_skipped(tmp_path: Path) -> None: +def test_codex_unreadable_regionless_table_raises(tmp_path: Path) -> None: tifffile.imwrite(tmp_path / "reg001_montage.tif", np.zeros((2, 16, 16), dtype=np.uint16)) - # a region-less table that is not assigned to a region: there are two images and no `region` column + # a region-less table that cannot be assigned to a region: there are two images and no `region` column tifffile.imwrite(tmp_path / "reg002_montage.tif", np.zeros((2, 16, 16), dtype=np.uint16)) (tmp_path / "cells.csv").write_text("cell_id,x,y,cyc001_ch001\n1,1.0,2.0,3.0\n") (tmp_path / "clusters.csv").write_text("cellid,cluster\n1,2\n") - with pytest.warns(UserWarning, match="Ignoring the cell table"): - sdata = codex(tmp_path, image_models_kwargs=_FAST) + with pytest.raises(ValueError, match="cannot be assigned to a region"): + codex(tmp_path, image_models_kwargs=_FAST) - assert sorted(sdata.images) == ["reg001_image", "reg002_image"] - assert not sdata.tables - assert sdata.attrs["akoya"]["cell_tables"] == {} + # a CODEX table next to a single image that cannot be read + (tmp_path / "reg002_montage.tif").unlink() + (tmp_path / "cells.csv").write_text("cell_id,x,y,cyc001_ch001\n1,1.0,2.0,3.0\n1,4.0,5.0,6.0\n") + with pytest.raises(ValueError, match="duplicate instance id"): + codex(tmp_path, image_models_kwargs=_FAST) + + # a single table file without a region column and no image + write_fcs(tmp_path / "Kidney.fcs", pd.DataFrame({"cell_id": [1.0], "x": [1.0], "y": [2.0], "cyc001_ch001": [3.0]})) + with pytest.raises(ValueError, match="cannot be assigned to a region"): + codex(tmp_path / "Kidney.fcs") def test_codex_regionless_table_belongs_to_single_stem_image(tmp_path: Path) -> None: From 803b9ea22ceca547e2d8f6f90a49e422b3968266 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:07:15 +0200 Subject: [PATCH 059/107] fix(codex): take regNNN only when not preceded by a letter The unanchored reg(\d+) search turned Treg2, Preg12 or coreg1 into region ids, so a stray analysis tif replaced the 2019 single-.tif fallback and Preg12_Scan1.qptiff got different region ids in codex and phenocycler. The fallback's region ids now go through sanitize_name like all others, so regions=['Kidney sample'] selects Kidney_sample. --- CHANGELOG.md | 5 +++-- src/spatialdata_io/readers/codex.py | 21 ++++++++++++--------- tests/test_codex.py | 23 +++++++++++++++++++++++ 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b681e0bb..9337609f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,8 +77,9 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. files that are not CODEX Processor cell tables are ignored, also when their name holds a region id. Compensated tables are preferred; other tables of the same region (e.g. of two segmentation runs) raise a `ValueError`. A `cells_table` disables table discovery, and the `labels` masks are not read as images. - - A CIM `.qptiff` without `regNNN` in its name is named as `phenocycler()` names it (without the suffix and a - trailing `_Scan`), and its `.er.qptiff` is preferred. + - Region ids `regNNN` are taken from `reg` and digits not preceded by a letter (`exp_reg001`, not `Treg2`). A CIM + `.qptiff` without one is named as `phenocycler()` names it (without the suffix and a trailing `_Scan`), and its + `.er.qptiff` is preferred. Top-level `.tif` files of the original layout are named after their sanitised stems. ### Fixed diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 6ff7ad35..32e99c63 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -12,7 +12,7 @@ import dask.array as da import numpy as np import pandas as pd -from spatialdata import SpatialData +from spatialdata import SpatialData, sanitize_name from spatialdata_io._constants._constants import CodexKeys from spatialdata_io._docs import inject_docs @@ -46,7 +46,8 @@ __all__ = ["codex"] -_REGION = re.compile(r"reg(\d+)", re.IGNORECASE) +# `reg001`, `exp_reg001` or `1_reg001`, but not `Treg2` or `coreg1` +_REGION = re.compile(r"(?`` (unless that makes two regions share a name), and the - extended-range ``.er.qptiff`` of a scan is preferred. If a directory contains no such files, its top-level ``{vx.IMAGE_TIF}(f)`` files - are read as hyperstacks named after their files. Each region is stored as the image ``_image`` and its + Raw acquisition and intermediate folders (``cycCCC_regNNN/``, ``tiles/`` and, next to ``stitched/``, ``bestFocus/``) + are ignored. The region id ``regNNN`` is taken from the file names (``reg`` and digits not preceded by a letter, + e.g. ``exp_reg001`` but not ``Treg2``); a ``{vx.QPTIFF!r}`` without one is named as + :func:`spatialdata_io.phenocycler` names it: after its file, without the ``.er.qptiff`` or ``{vx.QPTIFF!r}`` suffix + and a trailing ``_Scan`` (unless that makes two regions share a name), and the extended-range ``.er.qptiff`` of a + scan is preferred. If a directory contains no such files, its top-level ``{vx.IMAGE_TIF}(f)`` files are read as + hyperstacks named after their files, made valid element names by :func:`spatialdata.sanitize_name` (e.g. + ``Kidney sample.tif`` becomes the region ``Kidney_sample``). Each region is stored as the image ``_image`` and its metadata in ``sdata.attrs["akoya"]``. A single region is stored in the ``"global"`` coordinate system. Several CIM scans of the same sample space @@ -347,7 +350,7 @@ def _discover_legacy( if not images: # the original (2019) layout: one stitched `.tif` next to its table, without a region in the file name top_level = files if path.is_file() else [file for file in files if file.parent == path] - images = {file.stem: [file] for file in top_level if _is_tiff(file)} + images = {sanitize_name(file.stem): [file] for file in top_level if _is_tiff(file)} tables = _discover_tables(files, fcs) if find_tables else {} if not images and not tables: raise FileNotFoundError(f"No CODEX images or cell tables found in {path}. Expected: {'; '.join(_PATTERNS)}.") diff --git a/tests/test_codex.py b/tests/test_codex.py index 9aa66038..d9938221 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -421,6 +421,29 @@ def test_codex_single_tif_without_region(tmp_path: Path) -> None: assert _level0(sdata, "Kidney_image").shape == (3, 32, 48) +def test_codex_region_id_needs_reg_without_a_letter_before(tmp_path: Path) -> None: + for name in ("reg001_montage.tif", "exp_reg002.tif", "1_reg003.tif", "Treg2.tif", "Preg12.tif", "coreg1.tif"): + tifffile.imwrite(tmp_path / name, np.zeros((1, 4, 4), dtype=np.uint8)) + assert sorted(_discover_legacy(tmp_path, fcs=True).images) == ["reg001", "reg002", "reg003"] + + +def test_codex_flat_layout_ignores_non_region_tifs(tmp_path: Path) -> None: + tifffile.imwrite(tmp_path / "Kidney sample.tif", np.zeros((2, 16, 16), dtype=np.uint16)) + (tmp_path / "analysis").mkdir() + tifffile.imwrite(tmp_path / "analysis" / "Treg2_density.tif", np.zeros((16, 16), dtype=np.uint16)) + + for regions in (None, ["Kidney sample"], ["Kidney_sample"]): + sdata = codex(tmp_path, regions=regions, image_models_kwargs=_FAST) + assert list(sdata.images) == ["Kidney_sample_image"] + + +def test_codex_and_phenocycler_name_a_non_region_qptiff_alike(tmp_path: Path) -> None: + write_qptiff(tmp_path / "Preg12_Scan1.qptiff", biomarkers=["DAPI"], dtype="uint8") + + for reader in (codex, phenocycler): + assert list(reader(tmp_path, image_models_kwargs=_FAST).images) == ["Preg12_image"] + + def test_discover_legacy_regionless_table_ignores_sidecar_csv(tmp_path: Path) -> None: write_legacy_processor_tree(tmp_path / "exp", layout="per_marker", fcs=False, csv=False) (tmp_path / "antibody_list.csv").write_text("marker,clone\nCD3,UCHT1\n") From dbe59cc0d86dd1ac4cb65d194427dcb69a96841f Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:09:28 +0200 Subject: [PATCH 060/107] fix(akoya): skip hidden files and follow symlinked folders in discovery codex skipped hidden folders but not hidden files, and phenocycler's rglob skipped neither, so macOS AppleDouble ._* files and __MACOSX/ or .Trash-*/ copies crashed the read or added regions. Neither followed symlinked folders, so a symlinked stitched/ or scan folder was silently missed. Both readers now share walk_visible, which follows symlinks once per real folder. --- CHANGELOG.md | 2 ++ src/spatialdata_io/readers/_utils/_utils.py | 20 +++++++++++- src/spatialdata_io/readers/codex.py | 34 +++++++++------------ src/spatialdata_io/readers/phenocycler.py | 25 +++++++++------ tests/test_codex.py | 23 ++++++++++++++ tests/test_phenocycler.py | 18 +++++++++++ 6 files changed, 92 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9337609f..58248aac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,8 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. - Region ids `regNNN` are taken from `reg` and digits not preceded by a letter (`exp_reg001`, not `Treg2`). A CIM `.qptiff` without one is named as `phenocycler()` names it (without the suffix and a trailing `_Scan`), and its `.er.qptiff` is preferred. Top-level `.tif` files of the original layout are named after their sanitised stems. + - Hidden files and folders (e.g. macOS `._*` files, `__MACOSX/`, `.Trash-*/`) are ignored and symlinked folders + are followed, by `phenocycler()` too. ### Fixed diff --git a/src/spatialdata_io/readers/_utils/_utils.py b/src/spatialdata_io/readers/_utils/_utils.py index 428ac900..49b29057 100644 --- a/src/spatialdata_io/readers/_utils/_utils.py +++ b/src/spatialdata_io/readers/_utils/_utils.py @@ -10,7 +10,7 @@ from spatialdata._logging import logger if TYPE_CHECKING: - from collections.abc import Mapping + from collections.abc import Iterator, Mapping from anndata import AnnData from ome_types.model import Pixels @@ -118,6 +118,24 @@ def _set_reader_metadata(sdata: SpatialData, reader: str) -> SpatialData: return sdata +def walk_visible(root: Path) -> Iterator[tuple[Path, list[str], list[str]]]: + """Like :func:`os.walk` of ``root`` in sorted order, without hidden files and folders, following symlinks. + + Hidden are names starting with ``.`` (also macOS AppleDouble ``._*`` files and ``.Trash-*`` folders) and + ``__MACOSX`` folders. A folder reached a second time through a symlink (e.g. a link to an ancestor) is skipped. + The directory names can be pruned in place, as with :func:`os.walk`. + """ + visited = set() + for dirpath, dirnames, filenames in os.walk(root, followlinks=True): + real = os.path.realpath(dirpath) + if real in visited: + dirnames.clear() + continue + visited.add(real) + dirnames[:] = sorted(name for name in dirnames if not name.startswith(".") and name != "__MACOSX") + yield Path(dirpath), dirnames, sorted(name for name in filenames if not name.startswith(".")) + + def parse_physical_size(path: Path | None = None, ome_pixels: Pixels | None = None) -> float: """Parse physical size from OME-TIFF to micrometer.""" from ome_types import from_tiff diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 32e99c63..2e254cc5 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import re import warnings from collections.abc import Collection, Iterable, Mapping @@ -39,7 +38,11 @@ read_qptiff_image, read_qptiff_metadata, ) -from spatialdata_io.readers._utils._utils import _initialize_raster_models_kwargs, _set_reader_metadata +from spatialdata_io.readers._utils._utils import ( + _initialize_raster_models_kwargs, + _set_reader_metadata, + walk_visible, +) if TYPE_CHECKING: from collections.abc import Sequence @@ -122,14 +125,15 @@ def codex( ``cells_table`` to read them. FCS files of other instruments are ignored too. Raw acquisition and intermediate folders (``cycCCC_regNNN/``, ``tiles/`` and, next to ``stitched/``, ``bestFocus/``) - are ignored. The region id ``regNNN`` is taken from the file names (``reg`` and digits not preceded by a letter, - e.g. ``exp_reg001`` but not ``Treg2``); a ``{vx.QPTIFF!r}`` without one is named as + are ignored, as are hidden files and folders (names starting with ``.``, e.g. macOS ``._*`` files, and + ``__MACOSX/``); symlinked folders are followed. The region id ``regNNN`` is taken from the file names (``reg`` and + digits not preceded by a letter, e.g. ``exp_reg001`` but not ``Treg2``); a ``{vx.QPTIFF!r}`` without one is named as :func:`spatialdata_io.phenocycler` names it: after its file, without the ``.er.qptiff`` or ``{vx.QPTIFF!r}`` suffix and a trailing ``_Scan`` (unless that makes two regions share a name), and the extended-range ``.er.qptiff`` of a scan is preferred. If a directory contains no such files, its top-level ``{vx.IMAGE_TIF}(f)`` files are read as hyperstacks named after their files, made valid element names by :func:`spatialdata.sanitize_name` (e.g. - ``Kidney sample.tif`` becomes the region ``Kidney_sample``). Each region is stored as the image ``_image`` and its - metadata in ``sdata.attrs["akoya"]``. + ``Kidney sample.tif`` becomes the region ``Kidney_sample``). Each region is stored as the image ``_image`` + and its metadata in ``sdata.attrs["akoya"]``. A single region is stored in the ``"global"`` coordinate system. Several CIM scans of the same sample space (same slide, with stage positions and equal pixel sizes) are placed at their stage positions in ``"global"``; @@ -442,27 +446,19 @@ def _spatial_chunks(image_models_kwargs: Mapping[str, Any]) -> tuple[int, int]: def _walk(root: Path) -> list[Path]: - """All files below ``root`` in sorted order, skipping raw acquisition and intermediate processing folders.""" + """All visible files below ``root`` in sorted order, skipping raw acquisition and intermediate processing folders.""" files: list[Path] = [] - for dirpath, dirnames, filenames in os.walk(root): - directory = Path(dirpath) + for directory, dirnames, filenames in walk_visible(root): depth = len(directory.relative_to(root).parts) has_stitched = any(name.casefold() == "stitched" for name in dirnames) - dirnames[:] = sorted( - name for name in dirnames if depth < _MAX_DEPTH and not _is_ignored_dir(name, has_stitched) - ) - files.extend(directory / name for name in sorted(filenames)) + dirnames[:] = [name for name in dirnames if depth < _MAX_DEPTH and not _is_ignored_dir(name, has_stitched)] + files.extend(directory / name for name in filenames) return files def _is_ignored_dir(name: str, has_stitched: bool) -> bool: folded = name.casefold() - return ( - folded == "tiles" - or (folded == "bestfocus" and has_stitched) - or _RAW_CYCLE_DIR.fullmatch(name) is not None - or name.startswith(".") - ) + return folded == "tiles" or (folded == "bestfocus" and has_stitched) or _RAW_CYCLE_DIR.fullmatch(name) is not None def _discover_images(files: Sequence[Path]) -> dict[str, list[Path]]: diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 8712158f..059c3613 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -27,7 +27,7 @@ read_qptiff_image, read_qptiff_metadata, ) -from spatialdata_io.readers._utils._utils import _set_reader_metadata +from spatialdata_io.readers._utils._utils import _set_reader_metadata, walk_visible if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -72,10 +72,12 @@ def phenocycler( ``b``, and by inForm component data (float32, usually a ``.tif`` file, which is read if ``path`` is that single file). The image keeps the dtype of the file. These files have the generation ``"qpi_other"`` in the metadata. - Sidecar files are matched to a scan by file-name prefix and stored as metadata only: the channel names always - come from the per-page XML of the qptiff. A sidecar whose number of markers differs from the number of channels - raises a warning. Acquisition files (``*.raw.qptiff``, ``*.qptiff.raw``, ``*.qptiff.intermediate`` and anything - in a ``raw/images/phenocycler/`` folder below ``path``) are ignored. File suffixes are matched ignoring case. + Sidecar files are matched to a scan by file-name prefix and stored as metadata only: the channel names always come + from the per-page XML of the qptiff. A sidecar whose number of markers differs from the number of channels raises a + warning. Acquisition files (``*.raw.qptiff``, ``*.qptiff.raw``, ``*.qptiff.intermediate`` and anything in a + ``raw/images/phenocycler/`` folder below ``path``) are ignored, as are hidden files and folders (names starting with + ``.``, e.g. macOS ``._*`` files, and ``__MACOSX/``); symlinked folders are followed. File suffixes are matched + ignoring case. The region name is the file name without the ``{vx.ER_QPTIFF_SUFFIX!r}``/``{vx.QPTIFF_SUFFIX!r}`` suffix and without a trailing ``_Scan``, unless that makes two regions share a name, made a valid element name by @@ -236,11 +238,14 @@ def _discover_qptiffs(path: Path) -> tuple[dict[str, Path], dict[str, Path]]: raise FileNotFoundError(f"{path} does not exist.") if path.is_file(): return qptiff_regions_with_duplicates([path]) - files = [ - qptiff - for qptiff in sorted(path.rglob(f"*{PhenocyclerKeys.QPTIFF_SUFFIX}", case_sensitive=False)) - if qptiff.is_file() and not _is_acquisition_file(qptiff.relative_to(path)) - ] + files = sorted( + directory / name + for directory, _, names in walk_visible(path) + for name in names + if name.lower().endswith(PhenocyclerKeys.QPTIFF_SUFFIX) + and (directory / name).is_file() + and not _is_acquisition_file((directory / name).relative_to(path)) + ) if not files: raise FileNotFoundError( f"No .qptiff files found in {path} (searched recursively for '*.qptiff', excluding acquisition files)." diff --git a/tests/test_codex.py b/tests/test_codex.py index d9938221..8573b149 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -190,6 +190,29 @@ def test_codex_ignores_raw_tiles(tmp_path: Path) -> None: assert sorted(codex(tmp_path, image_models_kwargs=_FAST).images) == expected +def test_codex_skips_hidden_files_and_follows_symlinks(tmp_path: Path) -> None: + root = tmp_path / "exp" + write_legacy_processor_tree(root, regions=2, layout="per_marker", fcs=True, csv=False, n_cells=5) + expected = _discover_legacy(root, fcs=True) + stitched = root / "processed_2020-01-01" / "stitched" + # macOS AppleDouble files and copies in __MACOSX/ and .Trash-1000/ + for file in [*stitched.rglob("*.tif"), *root.rglob("*.fcs")]: + (file.parent / f"._{file.name}").write_bytes(b"AppleDouble") + for folder in ("__MACOSX", ".Trash-1000"): + shutil.copytree(stitched, root / folder / "stitched") + # the stitched images moved to another volume, and a link back up the tree + shutil.move(stitched, tmp_path / "scratch") + stitched.symlink_to(tmp_path / "scratch", target_is_directory=True) + (root / "loop").symlink_to(root, target_is_directory=True) + + layout = _discover_legacy(root, fcs=True) + + assert layout.tables == expected.tables + assert {region: [file.name for file in files] for region, files in layout.images.items()} == { + region: [file.name for file in files] for region, files in expected.images.items() + } + + def test_codex_regions_channel_names_and_pixel_size(tmp_path: Path) -> None: write_legacy_processor_tree(tmp_path, regions=2, cycles=1, channels=2, layout="per_marker") (tmp_path / "experiment.json").write_text(json.dumps({"xyResolution": 377.5, "numChannels": 2})) diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 0d989b0e..b09e9184 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -289,6 +289,24 @@ def test_discovery_ignores_suffix_case(tmp_path: Path) -> None: assert sorted(sdata.images) == ["A_image", "B_image"] +def test_discovery_skips_hidden_files_and_follows_symlinks(tmp_path: Path) -> None: + root = tmp_path / "ds" + for folder in (".Trash-1000/files", "__MACOSX"): + (root / folder).mkdir(parents=True) + write_qptiff(root / folder / "A_Scan1.qptiff", biomarkers=["DAPI"]) + write_qptiff(root / "A_Scan1.qptiff", biomarkers=["DAPI"]) + (root / "._A_Scan1.qptiff").write_bytes(b"AppleDouble") + # a scan folder on another volume, and a link back up the tree + (tmp_path / "scratch").mkdir() + write_qptiff(tmp_path / "scratch" / "B_Scan1.qptiff", biomarkers=["DAPI"]) + (root / "scans").symlink_to(tmp_path / "scratch", target_is_directory=True) + (root / "loop").symlink_to(root, target_is_directory=True) + + sdata = phenocycler(root, image_models_kwargs=_FAST) + + assert sorted(sdata.images) == ["A_image", "B_image"] + + def test_dataset_below_an_acquisition_like_folder_is_read(tmp_path: Path) -> None: # only the folders below `path` are checked for the HuBMAP acquisition folder `raw/images/phenocycler/` root = tmp_path / "raw" / "images" / "phenocycler" / "project" From fdf70931190184c6e960d94a3f6dd2eda7e3dc26 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:10:56 +0200 Subject: [PATCH 061/107] fix(akoya): find sidecars above relative paths and decode channelNames.txt codex('.') and phenocycler('.') never looked above the working directory for channelNames.txt/experiment.json or the HuBMAP lab_processed/ sidecars, because the ancestors of an unresolved relative path are empty. channelNames.txt was read with the locale encoding: a UTF-8 BOM ended up in the first name and UTF-16 or cp1252 files crashed every read. It is now decoded as UTF-16 (with a BOM), UTF-8 (BOM stripped), else cp1252. --- src/spatialdata_io/readers/codex.py | 15 +++++++++++++-- src/spatialdata_io/readers/phenocycler.py | 1 + tests/test_codex.py | 21 ++++++++++++++++++--- tests/test_phenocycler.py | 8 ++++++-- 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 2e254cc5..7a2feef6 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -1,5 +1,6 @@ from __future__ import annotations +import codecs import re import warnings from collections.abc import Collection, Iterable, Mapping @@ -572,7 +573,8 @@ def _find_sidecar(path: Path, files: Sequence[Path], name: str) -> Path | None: ) if matches: return matches[0] - start = path.parent if path.is_file() else path + # absolute, so that the ancestors of e.g. `codex(".")` are searched too + start = (path.parent if path.is_file() else path).absolute() for directory in [start, *start.parents][: _SIDECAR_ANCESTORS + 1]: try: found = sorted(file for file in directory.iterdir() if file.name.casefold() == folded and file.is_file()) @@ -584,7 +586,16 @@ def _find_sidecar(path: Path, files: Sequence[Path], name: str) -> Path | None: def _read_channel_names_txt(path: Path) -> list[str]: - return [line.strip() for line in path.read_text().splitlines() if line.strip()] + data = path.read_bytes() + # written on Windows: UTF-16 with a byte order mark, UTF-8 (with or without one) or the ANSI code page + if data.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)): + text = data.decode("utf-16") + else: + try: + text = data.decode("utf-8-sig") + except UnicodeDecodeError: + text = data.decode("cp1252", errors="replace") + return [line.strip() for line in text.splitlines() if line.strip()] def _pixel_size_um(experiment: Mapping[str, Any] | None) -> float | None: diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 059c3613..d93f7436 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -302,6 +302,7 @@ def _read_sidecars(scan: AkoyaScan) -> None: def _find_sidecar(qptiff: Path, base: str, suffix: str) -> Path | None: """First file ending in ``suffix`` whose name starts with ``base``, next to the scan or in HuBMAP processed dir.""" pattern = re.compile(rf"{re.escape(base)}(?![0-9A-Za-z]).*{re.escape(suffix)}$") + qptiff = qptiff.absolute() # also look above the working directory, e.g. for `phenocycler(".")` directories = [qptiff.parent] # HuBMAP keeps `raw/images/.qptiff` and `lab_processed/images/.qptiff.channels.csv` for ancestor in qptiff.parents: diff --git a/tests/test_codex.py b/tests/test_codex.py index 8573b149..91ec3c59 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -64,14 +64,19 @@ def test_codex_hyperstack_channel_names_txt(tmp_path: Path) -> None: np.testing.assert_array_equal(image.values, data["images"]["reg001"].reshape(8, *image.shape[1:])) -@pytest.mark.parametrize("from_subdirectory", [False, True]) -def test_codex_sidecar_names_are_case_insensitive(tmp_path: Path, from_subdirectory: bool) -> None: +@pytest.mark.parametrize("from_subdirectory", [False, True, "relative"]) +def test_codex_sidecar_names_are_case_insensitive( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, from_subdirectory: bool | str +) -> None: # SPACEc and the HuBMAP codex-pipeline write `channelnames.txt`; older exports `Experiment.json` data = write_legacy_processor_tree(tmp_path, cycles=2, channels=4, layout="hyperstack") (tmp_path / "channelNames.txt").rename(tmp_path / "channelnames.txt") (tmp_path / "experiment.json").unlink() (tmp_path / "Experiment.json").write_text(json.dumps({"xyResolution": 377.5})) - source = tmp_path / "processed_2020-01-01" / "stitched" if from_subdirectory else tmp_path + source: Path | str = tmp_path / "processed_2020-01-01" / "stitched" if from_subdirectory else tmp_path + if from_subdirectory == "relative": + monkeypatch.chdir(source) + source = "." sdata = codex(source, image_models_kwargs=_FAST) @@ -79,6 +84,16 @@ def test_codex_sidecar_names_are_case_insensitive(tmp_path: Path, from_subdirect assert sdata.attrs["akoya"]["regions"]["reg001"]["pixel_size_um"] == pytest.approx(0.3775) +@pytest.mark.parametrize("encoding", ["utf-8-sig", "utf-16", "cp1252"]) +def test_codex_channel_names_txt_encodings(tmp_path: Path, encoding: str) -> None: + write_legacy_processor_tree(tmp_path, cycles=1, channels=2, layout="hyperstack", fcs=False, csv=False) + (tmp_path / "channelNames.txt").write_text("DAPI\nIFNµ\n", encoding=encoding) + + sdata = codex(tmp_path, image_models_kwargs=_FAST) + + assert get_channel_names(sdata.images["reg001_image"]) == ["DAPI", "IFNµ"] + + def test_codex_hyperstack_bad_channel_names_warns(tmp_path: Path) -> None: data = write_legacy_processor_tree(tmp_path, cycles=2, channels=4, layout="hyperstack") (tmp_path / "channelNames.txt").write_text("\n".join(data["markers"][:7]) + "\n") diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index b09e9184..72979968 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -318,7 +318,8 @@ def test_dataset_below_an_acquisition_like_folder_is_read(tmp_path: Path) -> Non assert list(sdata.images) == ["S_image"] -def test_sidecars_hubmap_layout(tmp_path: Path) -> None: +@pytest.mark.parametrize("relative", [False, True]) +def test_sidecars_hubmap_layout(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, relative: bool) -> None: raw_images = tmp_path / "raw" / "images" processed = tmp_path / "lab_processed" / "images" raw_images.mkdir(parents=True) @@ -332,10 +333,13 @@ def test_sidecars_hubmap_layout(tmp_path: Path) -> None: (raw_images / "S_Scan1.xpd").write_text(json.dumps(xpd)) (processed / "S_Scan1.qptiff.channels.csv").write_text("Channel,Marker\n1,DAPI\n2,CD3\n") + if relative: + monkeypatch.chdir(raw_images) + # DAPI repeats per cycle and blanks do not count as channels, so the sidecars match the two pages with warnings.catch_warnings(): warnings.simplefilter("error", UserWarning) - sdata = phenocycler(tmp_path, image_models_kwargs=_FAST) + sdata = phenocycler("." if relative else tmp_path, image_models_kwargs=_FAST) raw = sdata.attrs["akoya"]["regions"]["S"]["raw"] assert raw["xpd"] == xpd From e3a4972bab380cf1e183aa9bd65577ae921e3813 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:12:01 +0200 Subject: [PATCH 062/107] fix(codex): key a discovered single-region table by its region in attrs A discovered table without a region in its file name was stored under the internal '*' key of attrs['akoya']['cell_tables'] even when all its cells belong to one region, while the same file passed as cells_table (or read by phenocycler) is keyed by that region. '*' now only marks a table of several regions, as documented. --- src/spatialdata_io/readers/codex.py | 10 ++++++++-- src/spatialdata_io/readers/phenocycler.py | 3 ++- tests/test_codex.py | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 7a2feef6..386d2c35 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -153,7 +153,8 @@ def codex( The outlines are always kept as the shapes ``_polygons`` in pixels, indexed by the same instance ids, also when a mask is given for the region (the mask is used for the labels, with a warning) or the region has no image (its cells are then circles). The paths of the tables, masks and GeoJSON files read are stored in - ``sdata.attrs["akoya"]["cell_tables"]``, ``["labels"]`` and ``["geojson"]``. + ``sdata.attrs["akoya"]["cell_tables"]``, ``["labels"]`` and ``["geojson"]``, by region id (``"*"`` for a file of + several regions). .. seealso:: @@ -290,7 +291,12 @@ def codex( } read = {key: table for key, table in read.items() if len(table.region)} cell_tables = list(read.values()) - table_paths = {key: str(layout.tables[key]) for key in read} + table_paths = {} + for key, table in read.items(): + # by region id, like a given table; `"*"` only for a region-less table of several regions + values = table.region.unique() + region = values[0] if key == "*" and len(values) == 1 and values[0] not in read else key + table_paths[region] = str(layout.tables[key]) sdata = build_spatialdata( scans, arrays, diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index d93f7436..09cdd042 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -106,7 +106,8 @@ def phenocycler( the higher instance id wins; the table annotates the labels. The outlines are always kept as the shapes ``{vx.POLYGONS_SUFFIX}`` in pixels, indexed by the same instance ids, also when a mask is given for the region (the mask is used for the labels, with a warning). The paths of the tables, masks and GeoJSON files are - stored in ``sdata.attrs[{vx.ATTRS_KEY!r}]["cell_tables"]``, ``["labels"]`` and ``["geojson"]``. + stored in ``sdata.attrs[{vx.ATTRS_KEY!r}]["cell_tables"]``, ``["labels"]`` and ``["geojson"]``, by region id + (``"*"`` for a file of several regions). .. seealso:: diff --git a/tests/test_codex.py b/tests/test_codex.py index 91ec3c59..94911336 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -688,7 +688,7 @@ def test_codex_regionless_table_belongs_to_single_stem_image(tmp_path: Path) -> sdata = codex(tmp_path, regions=regions, image_models_kwargs=_FAST) assert list(sdata.tables["table"].obs["region"].cat.categories) == ["Kidney_cells"] assert list(sdata.shapes) == ["Kidney_cells"] - assert sdata.attrs["akoya"]["cell_tables"] == {"*": str(tmp_path / "cells.csv")} + assert sdata.attrs["akoya"]["cell_tables"] == {"Kidney": str(tmp_path / "cells.csv")} def test_codex_duplicate_cell_ids_raise(tmp_path: Path) -> None: From 4bb3e24ddcbfdb209ef29d43522538bbca154a1f Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:15:00 +0200 Subject: [PATCH 063/107] fix(cli): repeat --regions/--channel-names and reject non-object JSON kwargs --regions and --channel-names were split on commas, so region or channel names containing one (A_[1,2], HLA-A,B,C) could not be passed; they are now repeatable options like every other list option of the CLI. A JSON value that is not an object (e.g. null) for a *-kwargs option raised a traceback in the reader instead of a usage error. The options shared by the codex and phenocycler commands are declared once. --- CHANGELOG.md | 3 +- src/spatialdata_io/__main__.py | 162 +++++++++++++-------------------- tests/test_phenocycler.py | 29 ++++++ 3 files changed, 96 insertions(+), 98 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58248aac..70fc5091 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,7 +68,8 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. (previously `cycCCC_chHHH`); the image channels are named too. - The positional signature `codex(path, fcs, imread_kwargs)` is unchanged; all new parameters (`regions`, `channel_names`, `cells_table`, `cells_table_format`, `labels` and the `*_models_kwargs` / `cells_table_kwargs`) - are keyword-only, and the `codex` CLI command exposes them. An empty `regions` raises a `ValueError`. + are keyword-only, and the `codex` CLI command exposes them (`--regions` and `--channel-names` are repeated per + value, as in the `phenocycler` command). An empty `regions` raises a `ValueError`. `imread_kwargs` is ignored, with a warning: every image is read lazily with `tifffile`. - A single region is stored in `"global"`. Several regions are placed at their stage positions in `"global"` only when all of them share a sample space (same slide id, stage positions and pixel size); otherwise each region, diff --git a/src/spatialdata_io/__main__.py b/src/spatialdata_io/__main__.py index 42b85038..6939415c 100644 --- a/src/spatialdata_io/__main__.py +++ b/src/spatialdata_io/__main__.py @@ -40,10 +40,12 @@ def _input_output_click_options(func: Callable[..., None]) -> Callable[..., None def _parse_json_param(value: str, param_name: str) -> dict[str, Any]: try: - result: dict[str, Any] = json.loads(value) - return result + result = json.loads(value) except json.JSONDecodeError as e: raise click.BadParameter(f"Invalid JSON for {param_name!r}: {e}") from e + if not isinstance(result, dict): + raise click.BadParameter(f"{param_name!r} must be a JSON object, got {value!r}.") + return result def _parse_path_or_mapping_param(value: str | None, param_name: str) -> str | dict[str, Any] | None: @@ -51,11 +53,61 @@ def _parse_path_or_mapping_param(value: str | None, param_name: str) -> str | di if value is None or not value.lstrip().startswith("{"): return value mapping = _parse_json_param(value, param_name) - if not isinstance(mapping, dict) or not all(isinstance(path, str) for path in mapping.values()): + if not all(isinstance(path, str) for path in mapping.values()): raise click.BadParameter(f"{param_name!r} must be a path or a JSON object mapping regions to paths.") return mapping +def _akoya_click_options(func: Callable[..., None]) -> Callable[..., None]: + """Decorator adding the options shared by the Akoya readers (``codex``, ``phenocycler``).""" + options = [ + click.option("--output", "-o", type=click.Path(), help="Path to the output.zarr file.", required=True), + click.option("--regions", multiple=True, help="A region to read (repeat for several); all if omitted."), + click.option( + "--channel-names", + multiple=True, + help="A channel name overriding the discovered ones (repeat for every channel).", + ), + click.option( + "--cells-table", + type=str, + default=None, + help='Path to a cell table, or a JSON mapping from region to path, e.g. \'{"A": "a.csv"}\'. ' + "[default: None]", + ), + click.option( + "--cells-table-format", type=str, default="auto", help="Format of the cell table. [default: auto]" + ), + click.option( + "--labels", + type=str, + default=None, + help="Path to a segmentation mask, or a JSON mapping from region to path. [default: None]", + ), + click.option( + "--image-models-kwargs", + type=str, + default="{}", + help="JSON string of keyword arguments passed to Image2DModel. [default: {}]", + ), + click.option( + "--labels-models-kwargs", + type=str, + default="{}", + help="JSON string of keyword arguments passed to Labels2DModel. [default: {}]", + ), + click.option( + "--cells-table-kwargs", + type=str, + default="{}", + help="JSON string of keyword arguments for reading the cell table. [default: {}]", + ), + ] + for option in reversed(options): + func = option(func) + return func + + @cli.command(name="codex") @click.option( "--input", @@ -64,65 +116,25 @@ def _parse_path_or_mapping_param(value: str | None, param_name: str) -> str | di help="Path to a CODEX output directory or to a single .qptiff/.tif image.", required=True, ) -@click.option("--output", "-o", type=click.Path(), help="Path to the output.zarr file.", required=True) +@_akoya_click_options @click.option( "--fcs", type=bool, default=True, help="Whether to prefer .fcs over .csv cell tables when both exist. [default: True]", ) -@click.option( - "--regions", type=str, default=None, help="Comma-separated regions to read; all if omitted. [default: None]" -) -@click.option( - "--channel-names", - type=str, - default=None, - help="Comma-separated channel names overriding the discovered ones. [default: None]", -) -@click.option( - "--cells-table", - type=str, - default=None, - help='Path to a cell table, or a JSON mapping from region to path, e.g. \'{"A": "a.csv"}\'. [default: None]', -) -@click.option("--cells-table-format", type=str, default="auto", help="Format of the cell table. [default: auto]") -@click.option( - "--labels", - type=str, - default=None, - help="Path to a segmentation mask, or a JSON mapping from region to path. [default: None]", -) @click.option( "--imread-kwargs", type=str, default="{}", help="Ignored, with a warning; kept for backward compatibility. [default: {}]", ) -@click.option( - "--image-models-kwargs", - type=str, - default="{}", - help="JSON string of keyword arguments passed to Image2DModel. [default: {}]", -) -@click.option( - "--labels-models-kwargs", - type=str, - default="{}", - help="JSON string of keyword arguments passed to Labels2DModel. [default: {}]", -) -@click.option( - "--cells-table-kwargs", - type=str, - default="{}", - help="JSON string of keyword arguments for reading the cell table. [default: {}]", -) def codex_wrapper( input: str, output: str, fcs: bool = True, - regions: str | None = None, - channel_names: str | None = None, + regions: tuple[str, ...] = (), + channel_names: tuple[str, ...] = (), cells_table: str | None = None, cells_table_format: str = "auto", labels: str | None = None, @@ -137,8 +149,8 @@ def codex_wrapper( sdata = codex( input, fcs=fcs, - regions=_parse_list_param(regions), - channel_names=_parse_list_param(channel_names), + regions=list(regions) or None, + channel_names=list(channel_names) or None, cells_table=_parse_path_or_mapping_param(cells_table, "cells_table"), cells_table_format=cells_table_format, labels=_parse_path_or_mapping_param(labels, "labels"), @@ -434,10 +446,6 @@ def merscope_wrapper( sdata.write(output) -def _parse_list_param(value: str | None) -> list[str] | None: - return None if value is None else [item.strip() for item in value.split(",")] - - @cli.command(name="phenocycler") @click.option( "--input", @@ -446,58 +454,18 @@ def _parse_list_param(value: str | None) -> list[str] | None: help="Path to a .qptiff file or to a directory containing .qptiff files.", required=True, ) -@click.option("--output", "-o", type=click.Path(), help="Path to the output.zarr file.", required=True) -@click.option( - "--regions", type=str, default=None, help="Comma-separated regions to read; all if omitted. [default: None]" -) -@click.option( - "--channel-names", - type=str, - default=None, - help="Comma-separated channel names overriding the ones in the qptiff. [default: None]", -) -@click.option( - "--cells-table", - type=str, - default=None, - help='Path to a cell table, or a JSON mapping from region to path, e.g. \'{"A": "a.csv"}\'. [default: None]', -) -@click.option("--cells-table-format", type=str, default="auto", help="Format of the cell table. [default: auto]") -@click.option( - "--labels", - type=str, - default=None, - help="Path to a segmentation mask, or a JSON mapping from region to path. [default: None]", -) +@_akoya_click_options @click.option( "--read-all-bit-depths/--no-read-all-bit-depths", default=False, help="Read both the 8-bit .qptiff and the extended-range .er.qptiff of a scan (the 8-bit one as region " "'_8bit'). [default: no-read-all-bit-depths]", ) -@click.option( - "--image-models-kwargs", - type=str, - default="{}", - help="JSON string of keyword arguments passed to Image2DModel. [default: {}]", -) -@click.option( - "--labels-models-kwargs", - type=str, - default="{}", - help="JSON string of keyword arguments passed to Labels2DModel. [default: {}]", -) -@click.option( - "--cells-table-kwargs", - type=str, - default="{}", - help="JSON string of keyword arguments for reading the cell table. [default: {}]", -) def phenocycler_wrapper( input: str, output: str, - regions: str | None = None, - channel_names: str | None = None, + regions: tuple[str, ...] = (), + channel_names: tuple[str, ...] = (), cells_table: str | None = None, cells_table_format: str = "auto", labels: str | None = None, @@ -511,8 +479,8 @@ def phenocycler_wrapper( sdata = phenocycler( input, - regions=_parse_list_param(regions), - channel_names=_parse_list_param(channel_names), + regions=list(regions) or None, + channel_names=list(channel_names) or None, cells_table=_parse_path_or_mapping_param(cells_table, "cells_table"), cells_table_format=cells_table_format, labels=_parse_path_or_mapping_param(labels, "labels"), diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 72979968..355cab03 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -860,6 +860,35 @@ def test_cli_bad_mapping_is_bad_parameter(runner: CliRunner, tmp_path: Path, val assert "cells_table" in result.output +def test_cli_regions_and_channel_names_may_hold_commas(runner: CliRunner, tmp_path: Path) -> None: + write_qptiff(tmp_path / "A_[1,2].qptiff", biomarkers=["DAPI", "HLA-A"]) + write_qptiff(tmp_path / "B.qptiff", biomarkers=["DAPI", "CD3"]) + args = ["--input", str(tmp_path), "--image-models-kwargs", json.dumps(_FAST)] + names = ["--channel-names", "DAPI", "--channel-names", "HLA-A,B,C"] + + result = runner.invoke(phenocycler_wrapper, [*args, "-o", str(tmp_path / "a.zarr"), "--regions", "A_[1,2]", *names]) + assert result.exit_code == 0, result.output + sdata = read_zarr(tmp_path / "a.zarr") + assert list(sdata.images) == ["A__1_2__image"] + assert get_channel_names(sdata.images["A__1_2__image"]) == ["DAPI", "HLA-A,B,C"] + + result = runner.invoke( + phenocycler_wrapper, [*args, "-o", str(tmp_path / "b.zarr"), "--regions", "A__1_2_", "--regions", "B"] + ) + assert result.exit_code == 0, result.output + assert sorted(read_zarr(tmp_path / "b.zarr").images) == ["A__1_2__image", "B_image"] + + +@pytest.mark.parametrize("option", ["--image-models-kwargs", "--cells-table-kwargs"]) +def test_cli_kwargs_must_be_json_objects(runner: CliRunner, tmp_path: Path, option: str) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + + result = runner.invoke(phenocycler_wrapper, ["-i", str(tmp_path), "-o", str(tmp_path / "o.zarr"), option, "null"]) + + assert result.exit_code == 2 + assert "JSON object" in result.output + + def test_str_regions_and_channel_names(tmp_path: Path) -> None: write_qptiff(tmp_path / "AB_Scan1.qptiff", biomarkers=["DAPI"]) write_qptiff(tmp_path / "C_Scan1.qptiff", biomarkers=["DAPI"]) From b6a7ae48dfb3a1dfa2bc7e822d6352510c0df958 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:24:09 +0200 Subject: [PATCH 064/107] build: declare flowio, used directly to detect FCS tables --- pyproject.toml | 2 ++ src/spatialdata_io/readers/_utils/_cell_tables.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f2a7ad86..143e60c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "anndata", "click", "dask-image", + "flowio", "h5py", "imagecodecs", "joblib", @@ -157,6 +158,7 @@ exclude = "^tests/data/" [[tool.mypy.overrides]] module = [ "dask_image.*", + "flowio.*", "h5py.*", "multiscale_spatial_image.*", "pyarrow.*", diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index ea4b1757..f0c9f0bb 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Any import anndata as ad -import flowio # type: ignore[import-untyped] # readfcs' parser, for the TEXT segment only +import flowio # readfcs' parser, for the TEXT segment only import geopandas as gpd import numpy as np import pandas as pd From ae4c9eb057c15d928ff09e9a71bad94dbfc71131 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:28:24 +0200 Subject: [PATCH 065/107] fix(codex): leave the obs index of CODEX tables unnamed so single-region reads can be written --- .../readers/_utils/_cell_tables.py | 3 ++- tests/test_codex.py | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index f0c9f0bb..d3fec20a 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -260,7 +260,8 @@ def read_codex_table(path: Path, region: str | None = None) -> CellTable: else: # label images use 0 for the background, so instance ids start at 1 instance_id = pd.Series(np.arange(1, len(obs) + 1, dtype=np.int64)) - instance_id.index = pd.Index(instance_id.astype(str)) + # unnamed, like the index of the other formats: the table element gets an `instance_key` column of that name + instance_id.index = pd.Index(instance_id.astype(str).to_numpy()) obs.index = instance_id.index instance_id.name = instance_key diff --git a/tests/test_codex.py b/tests/test_codex.py index 94911336..98d23c08 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -743,3 +743,29 @@ def test_codex_old_positional_call(tmp_path: Path) -> None: with pytest.warns(UserWarning, match="imread_kwargs"): codex(tmp_path, True, {"nframes": 1}) + + +@pytest.mark.parametrize("case", ["one_region", "kidney_2019", "regions_subset", "phenocycler_codex_csv"]) +def test_single_region_codex_table_write_roundtrip(tmp_path: Path, case: str) -> None: + data = tmp_path / "data" + data.mkdir() + if case == "kidney_2019": + tifffile.imwrite(data / "Kidney.tif", np.zeros((2, 16, 16), dtype=np.uint16)) + (data / "Kidney.csv").write_text("cell_id,x,y,size,cyc001_ch001\n1,1.0,2.0,3.0,4.0\n2,5.0,6.0,7.0,8.0\n") + elif case == "phenocycler_codex_csv": + write_qptiff(data / "S_Scan1.qptiff", biomarkers=["DAPI"]) + (data / "cells.csv").write_text("cell_id,x,y,size,DAPI\n1,1.0,2.0,3.0,4.0\n2,5.0,6.0,7.0,8.0\n") + else: + write_legacy_processor_tree(data, regions=1 if case == "one_region" else 2, cycles=1, channels=1, n_cells=5) + + with pytest.warns(UserWarning, match="circles"): + if case == "phenocycler_codex_csv": + sdata = phenocycler(data, cells_table=data / "cells.csv", image_models_kwargs=_FAST) + else: + sdata = codex(data, regions=["reg001"] if case == "regions_subset" else None, image_models_kwargs=_FAST) + sdata.write(tmp_path / "x.zarr") + + table = read_zarr(tmp_path / "x.zarr").tables["table"] + assert table.obs.index.name is None + assert table.obs["cell_id"].dtype == np.int64 + np.testing.assert_array_equal(table.obs["cell_id"], sdata.tables["table"].obs["cell_id"]) From c0eb2b765e9ccd4f01729a925a66d29022367064 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:33:27 +0200 Subject: [PATCH 066/107] fix(akoya): read CODEX tables through the shared roles path cells_table_kwargs (region_column, markers, instance_id) now apply to CODEX tables, the delimiter is sniffed like in detection, and discovered tables match their region values against the dataset's regions like given ones. Drops read_codex_table. --- .../readers/_utils/_cell_tables.py | 185 ++++++++---------- src/spatialdata_io/readers/codex.py | 33 ++-- src/spatialdata_io/readers/phenocycler.py | 2 +- tests/readers/test_cell_tables.py | 56 ++++-- tests/test_codex.py | 27 ++- tests/test_phenocycler.py | 18 ++ 6 files changed, 187 insertions(+), 134 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index d3fec20a..5f795618 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -58,7 +58,6 @@ "polygon_shapes", "raster_shape", "read_cell_table", - "read_codex_table", "read_labels", "read_qupath_polygons", "read_region_labels", @@ -195,104 +194,6 @@ def take(self, rows: np.ndarray) -> CellTable: ) -def read_codex_table(path: Path, region: str | None = None) -> CellTable: - """Read a CODEX Processor ``.fcs`` or ``.csv`` cell table. - - Marker columns are the ``cycCCC_chHHH`` columns; their names come from the FCS ``$PnS`` keywords or from the - display part of the Processor CSV header ``cycCCC_chHHH:``. A table without such columns has one - marker column per numeric column that is not a known CODEX Processor column (``x``, ``size``, ...). - - Parameters - ---------- - path - The ``.fcs`` or ``.csv`` file. - region - The region id of all cells if the table has no ``region`` column. A numeric ``region`` column ``r`` is - mapped to the region id ``regRRR``. - - Returns - ------- - The cell table. - """ - if path.suffix.lower() == CodexKeys.FCS_FILE.value: - fcs = readfcs.ReadFCS(path) - df = fcs.data # positional columns, named by `$PnN` - names = list(fcs.channels["PnN"]) - labels = list(fcs.channels["PnS"]) if "PnS" in fcs.channels.columns else [""] * len(names) - else: - df = pd.read_csv(path, engine="pyarrow") - names = [str(column) for column in df.columns] - labels = [""] * len(names) - sources = names - # the Processor CSV header form `cyc001_ch001:DAPI1` (also `cell_id:cell_id`) - split = [name.partition(":") for name in names] - df.columns = pd.Index([name for name, _, _ in split]) - displays = [label.strip() or display or name for (name, _, display), label in zip(split, labels, strict=True)] - - marker_matches = [_MARKER_COLUMN.fullmatch(column) for column in df.columns] - if any(marker_matches): - is_marker = [match is not None for match in marker_matches] - else: - is_marker = [ - column.casefold() not in _META_COLUMNS and pd.api.types.is_numeric_dtype(df[column]) - for column in df.columns - ] - markers = [i for i, marker in enumerate(is_marker) if marker] - var = pd.DataFrame( - {"source_column": [sources[i] for i in markers]}, - index=pd.Index(dedupe_channel_names([displays[i] for i in markers])), - ) - if any(marker_matches): - var["cycle"] = [int(marker_matches[i].group(1)) for i in markers] # type: ignore[union-attr] - var["channel"] = [int(marker_matches[i].group(2)) for i in markers] # type: ignore[union-attr] - X = df.iloc[:, markers].to_numpy(dtype=np.float32) - obs = df.iloc[:, [i for i, marker in enumerate(is_marker) if not marker]] - - columns = {column.casefold(): column for column in obs.columns} - if "x" not in columns or "y" not in columns: - raise ValueError(f"{path} has no 'x' and 'y' centroid columns; found {list(obs.columns)}.") - centroids = obs[[columns["x"], columns["y"]]].to_numpy(dtype=np.float64) - radius = np.sqrt(obs[columns["size"]].to_numpy(dtype=np.float64) / np.pi) if "size" in columns else None - - instance_key = CodexKeys.INSTANCE_KEY.value - if instance_key in columns: - instance_id = obs[columns[instance_key]].astype(np.int64) - else: - # label images use 0 for the background, so instance ids start at 1 - instance_id = pd.Series(np.arange(1, len(obs) + 1, dtype=np.int64)) - # unnamed, like the index of the other formats: the table element gets an `instance_key` column of that name - instance_id.index = pd.Index(instance_id.astype(str).to_numpy()) - obs.index = instance_id.index - instance_id.name = instance_key - - region_key = CodexKeys.REGION_KEY.value - if region_key in columns: - regions = obs[columns[region_key]] - if pd.api.types.is_numeric_dtype(regions): - ids = {value: f"reg{int(value):03d}" for value in regions.unique()} - regions = regions.map(ids) - region_ids = regions.astype(str) - elif region is not None: - region_ids = pd.Series(region, index=obs.index) - else: - raise ValueError( - f"{path} has no '{region_key}' column and no region id in its file name, so its cells cannot be " - "assigned to a region; a table for several regions needs a region column." - ) - region_ids.name = region_key - _check_unique_instance_ids(path, region_ids, instance_id) - return CellTable( - obs=_drop_key_columns(obs, columns.get(instance_key), columns.get(region_key)), - X=X, - var=var, - region=region_ids, - instance_id=instance_id, - centroids_px=centroids, - radius_px=radius, - row_instance_ids=instance_key not in columns, - ) - - def detect_table_format(path: Path) -> str: """Detect the format of a cell table from its file type and header. @@ -362,7 +263,12 @@ def read_cell_table( ``y``), ``"codex"``, ``"qupath"``, ``"mcquant"``, ``"sprm"``, ``"inform"`` or ``"generic"``: - - ``"codex"``: see :func:`read_codex_table`. + - ``"codex"``: a CODEX Processor ``.fcs`` or ``.csv`` table. The markers are ``markers``, else the + ``cycCCC_chHHH`` columns, named by the FCS ``$PnS`` keywords or by the display part of the Processor + CSV header ``cycCCC_chHHH:`` (in ``var`` with their ``cycle`` and ``channel``), else the + numeric columns that are not CODEX Processor columns (``x``, ``size``, ...). Centroids ``x``/``y`` in + pixels, instance ids ``cell_id``, the region from ``region`` (a numeric region ``r`` is the region id + ``regRRR``), the radius from ``size``. - ``"qupath"``: a QuPath measurement export, also with R's ``make.names`` column names (``Centroid.X.um``, ``Object.ID``). Centroids ``Centroid X µm``/``Centroid Y µm`` (divided by ``pixel_size_um``) or ``Centroid X px``/``Centroid Y px``, else ``X``/``Y`` in ``coordinates_unit`` @@ -408,7 +314,7 @@ def read_cell_table( Optional keys: ``region_column`` (the column holding the region of each cell; QuPath default ``Image``), and for ``"generic"`` tables ``x`` and ``y`` (required centroid columns), ``instance_id`` (else the row number, from 1), ``markers`` (a list of columns or a regular expression matching the full column name; - default: all other numeric columns; also for ``"qupath"`` tables) and ``coordinates_unit`` (``"px"``, + default: all other numeric columns; also for ``"qupath"`` and ``"codex"`` tables) and ``coordinates_unit`` (``"px"``, default, or ``"um"``; for ``"qupath"`` and ``"inform"`` tables the unit of centroid columns whose name has none) and ``coordinates_space`` (the pixels of ``"px"`` centroids: ``"image"``, default, those of the full-resolution region image, or ``"mask"``, those of the region's mask, scaled to image pixels like the @@ -433,16 +339,18 @@ def read_cell_table( raise ValueError( f"`cells_table_kwargs['coordinates_space']` is only used with generic cell tables; {path} is {fmt!r}." ) - if fmt == "codex": - return read_codex_table(path, region=default_region) if fmt not in TABLE_FORMATS: raise ValueError(f"Unknown cell table format {fmt!r}; expected 'auto' or one of {TABLE_FORMATS}.") - _, sep = _read_header(path) - df = pd.read_csv(path, sep=sep, engine="pyarrow") - # inForm writes some column names with trailing spaces - df.columns = pd.Index([str(column).strip() for column in df.columns]) + if path.suffix.lower() == CodexKeys.FCS_FILE.value: + df = _read_fcs(path) + else: + _, sep = _read_header(path) + df = pd.read_csv(path, sep=sep, engine="pyarrow") + # inForm writes some column names with trailing spaces + df.columns = pd.Index([str(column).strip() for column in df.columns]) reader = { + "codex": _codex_roles, "qupath": _qupath_roles, "mcquant": _mcquant_roles, "sprm": _sprm_roles, @@ -680,6 +588,69 @@ def _read_geojson(path: Path) -> tuple[np.ndarray | None, np.ndarray, np.ndarray ) +def _read_fcs(path: Path) -> pd.DataFrame: + """The events of an FCS file, with the columns named ``$PnN:$PnS`` like the CODEX Processor CSV header.""" + fcs = readfcs.ReadFCS(path) + df = fcs.data # positional columns, named by `$PnN` + labels = fcs.channels["PnS"] if "PnS" in fcs.channels.columns else [""] * df.shape[1] + names = fcs.channels["PnN"] + df.columns = pd.Index( + [f"{name}:{label.strip()}" if label.strip() else name for name, label in zip(names, labels, strict=True)] + ) + return df + + +def _codex_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: + """The roles of a CODEX table; renames the columns of ``df`` to the name part of ``cycCCC_chHHH:``.""" + split = [str(column).partition(":") for column in df.columns] + # the source of an FCS column is its `$PnN` + fcs = path.suffix.lower() == CodexKeys.FCS_FILE.value + sources = [name if fcs else str(column) for column, (name, _, _) in zip(df.columns, split, strict=True)] + names = [name for name, _, _ in split] + displays = [display.strip() or name for name, _, display in split] + df.columns = pd.Index(names) + columns = {column.casefold(): column for column in names} + x, y = kwargs.get("x", columns.get("x")), kwargs.get("y", columns.get("y")) + if x is None or y is None: + raise ValueError(f"{path} has no 'x' and 'y' centroid columns; found {names}.") + instance_column = kwargs.get("instance_id", columns.get(CodexKeys.INSTANCE_KEY.value)) + region_column = kwargs.get("region_column", columns.get(CodexKeys.REGION_KEY.value)) + if region_column in df.columns and pd.api.types.is_numeric_dtype(df[region_column]): + values = df[region_column] + df[region_column] = values.map({value: f"reg{int(value):03d}" for value in values.dropna().unique()}) + area = columns.get("size") + + matches = [_MARKER_COLUMN.fullmatch(name) for name in names] + if kwargs.get("markers") is not None: + selected = set(_select_markers(df, kwargs["markers"], {x, y, instance_column, region_column, area})) + is_marker = [name in selected for name in names] + elif any(matches): + is_marker = [match is not None for match in matches] + else: + is_marker = [ + name.casefold() not in _META_COLUMNS and pd.api.types.is_numeric_dtype(df.iloc[:, i]) + for i, name in enumerate(names) + ] + positions = [i for i, marker in enumerate(is_marker) if marker] + var = pd.DataFrame( + {"source_column": [sources[i] for i in positions]}, + index=pd.Index(dedupe_channel_names([displays[i] for i in positions])), + ) + if positions and all(matches[i] for i in positions): + var["cycle"] = [int(matches[i].group(1)) for i in positions] # type: ignore[union-attr] + var["channel"] = [int(matches[i].group(2)) for i in positions] # type: ignore[union-attr] + return { + "markers": [names[i] for i in positions], + "var": var, + "x": x, + "y": y, + "in_um": False, + "instance_column": instance_column, + "region_column": region_column, + "area_column": area, + } + + def _qupath_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: columns = _qupath_names(list(df.columns)) unit = next((unit for unit, (x_name, _) in _QUPATH_CENTROIDS.items() if x_name in columns), None) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 386d2c35..3daed2ec 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -26,9 +26,10 @@ ) from spatialdata_io.readers._utils._cell_tables import ( CellTable, + _match_region, build_spatialdata, detect_table_format, - read_codex_table, + read_cell_table, read_region_tables, resolve_per_region, ) @@ -200,7 +201,7 @@ def codex( Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`. cells_table_kwargs Options for reading ``cells_table`` and ``labels``: ``region_column``; for ``"generic"`` tables ``x``, - ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath tables), + ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath and CODEX tables), ``coordinates_unit`` (``"px"`` or ``"um"``; for QuPath ``X``/``Y`` and inForm ``Cell X Position`` columns without a unit, which are otherwise read in the unit of the table's other columns) and ``coordinates_space`` (``"image"``, default, or ``"mask"``: whether the pixel centroids of a generic table @@ -283,13 +284,15 @@ def codex( ) table_paths = {key: str(file) for key, file in table_files.items()} else: - named = {key: read_codex_table(layout.tables[key], region=key) for key in selected if key in layout.tables} - read = {"*": regionless, **named} if regionless is not None else named - if regions is not None: - read = { - key: table.take(np.flatnonzero(table.region.isin(selected).to_numpy())) for key, table in read.items() - } - read = {key: table for key, table in read.items() if len(table.region)} + named_files = {key: layout.tables[key] for key in selected if key in layout.tables} + named = read_region_tables( + named_files, "codex", available=available, selected=selected, pixel_sizes={}, cells_table_kwargs={} + ) + read = dict(zip(named_files, named, strict=True)) + if regionless is not None: + regionless = regionless.take(np.flatnonzero(regionless.region.isin(selected).to_numpy())) + if len(regionless.region): + read = {"*": regionless, **read} cell_tables = list(read.values()) table_paths = {} for key, table in read.items(): @@ -324,11 +327,17 @@ def _warn_fcs_fallback(files: Iterable[Path], fcs: bool) -> None: def _read_regionless_table(file: Path, layout: _LegacyLayout) -> CellTable: """Read the discovered cell table ``file`` that has no region in its file name.""" only_image = next(iter(layout.images)) if len(layout.images) == 1 else None - table = read_codex_table(file, region=only_image) - # next to a single image named after its file (e.g. `Kidney.tif`), its cells belong to that image + table = read_cell_table(file, "codex", pixel_size_um=None, default_region=only_image, cells_table_kwargs={}) values = table.region.unique() - if only_image is not None and len(values) == 1 and values[0] not in layout.images: + if only_image is not None and len(values) == 1 and _match_region(values[0], [only_image]) is None: + # next to a single image named after its file (e.g. `Kidney.tif`), its cells belong to that image table.region = pd.Series(only_image, index=table.region.index, name=table.region.name) + else: + # values that name no region of the dataset are regions without an image + regions = list(layout.images.keys() | layout.tables.keys() - {"*"}) + table.region = table.region.map( + {value: _match_region(value, regions) or sanitize_name(value) for value in values} + ) return table diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 09cdd042..50917dca 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -156,7 +156,7 @@ def phenocycler( Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`. cells_table_kwargs Options for reading ``cells_table`` and ``labels``: ``region_column``; for ``"generic"`` tables ``x``, - ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath tables), + ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath and CODEX tables), ``coordinates_unit`` (``"px"`` or ``"um"``; for QuPath ``X``/``Y`` and inForm ``Cell X Position`` columns without a unit, which are otherwise read in the unit of the table's other columns) and ``coordinates_space`` (``"image"``, default, or ``"mask"``: whether the pixel centroids of a generic table diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index 1a7e2501..1ead114d 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -20,7 +20,6 @@ build_cells, detect_table_format, read_cell_table, - read_codex_table, read_labels, resolve_per_region, ) @@ -28,6 +27,8 @@ if TYPE_CHECKING: from pathlib import Path + from spatialdata_io.readers._utils._cell_tables import CellTable + def _cells(n: int = 5, region: int = 1, markers: tuple[str, ...] = ("cyc001_ch001", "cyc001_ch002")) -> pd.DataFrame: rng = np.random.default_rng(0) @@ -43,12 +44,16 @@ def _cells(n: int = 5, region: int = 1, markers: tuple[str, ...] = ("cyc001_ch00 ) +def _read_codex(path: Path, region: str | None = None) -> CellTable: + return read_cell_table(path, "codex", pixel_size_um=None, default_region=region, cells_table_kwargs={}) + + def test_read_codex_fcs_marker_names_from_pns(tmp_path: Path) -> None: cells = _cells(markers=("cyc001_ch001", "cyc001_ch002", "cyc002_ch001")) path = tmp_path / "reg001_compensated.fcs" write_fcs(path, cells, {"cyc001_ch001": "DAPI", "cyc001_ch002": "CD3", "cyc002_ch001": "DAPI"}) - table = read_codex_table(path) + table = _read_codex(path) assert list(table.var.index) == ["DAPI", "CD3", "DAPI (2)"] assert list(table.var["cycle"]) == [1, 1, 2] @@ -70,7 +75,7 @@ def test_read_codex_fcs_without_pns_keeps_column_names(tmp_path: Path) -> None: path = tmp_path / "reg001.fcs" write_fcs(path, _cells()) - assert list(read_codex_table(path).var.index) == ["cyc001_ch001", "cyc001_ch002"] + assert list(_read_codex(path).var.index) == ["cyc001_ch001", "cyc001_ch002"] def test_read_codex_csv_colon_header(tmp_path: Path) -> None: @@ -81,7 +86,7 @@ def test_read_codex_csv_colon_header(tmp_path: Path) -> None: "2,2,11,21,31,200.5,8\n" ) - table = read_codex_table(path) + table = _read_codex(path) assert list(table.var.index) == ["DAPI1", "CD3"] assert list(table.var["source_column"]) == ["cyc001_ch001:DAPI1", "cyc001_ch002:CD3"] @@ -96,7 +101,7 @@ def test_read_codex_csv_marker_name_columns(tmp_path: Path) -> None: path = tmp_path / "cells.csv" path.write_text("cell_id,x,y,tile_num,CD3,CD8,label\n1,1.0,2.0,1,5.0,6.0,T\n") - table = read_codex_table(path, region="Kidney") + table = _read_codex(path, region="Kidney") assert list(table.var.index) == ["CD3", "CD8"] assert "cycle" not in table.var.columns @@ -109,17 +114,44 @@ def test_read_codex_table_errors(tmp_path: Path) -> None: no_xy = tmp_path / "a.csv" no_xy.write_text("cell_id,cyc001_ch001\n1,2.0\n") with pytest.raises(ValueError, match="'x' and 'y'"): - read_codex_table(no_xy) + _read_codex(no_xy) no_region = tmp_path / "b.csv" no_region.write_text("cell_id,x,y,cyc001_ch001\n1,1.0,2.0,2.0\n") with pytest.raises(ValueError, match="region"): - read_codex_table(no_region) + _read_codex(no_region) + + +@pytest.mark.parametrize("sep", ["\t", ", "]) +def test_read_codex_table_delimiters_like_detection(tmp_path: Path, sep: str) -> None: + path = tmp_path / "cells.tsv" + header = ["cell_id", "region", "x", "y", "size", "cyc001_ch001:DAPI"] + path.write_text("\n".join(sep.join(row) for row in [header, ["1", "2", "1.0", "2.0", "3.0", "4.0"]]) + "\n") + + assert detect_table_format(path) == "codex" + table = read_cell_table(path, "auto", pixel_size_um=None, default_region=None, cells_table_kwargs={}) + + assert list(table.var.index) == ["DAPI"] + assert list(table.region) == ["reg002"] + np.testing.assert_array_equal(table.centroids_px, [[1, 2]]) + + +def test_read_codex_table_cells_table_kwargs(tmp_path: Path) -> None: + path = tmp_path / "cells.csv" + path.write_text("cell_id,label,Sample,x,y,CD3,CD8\n1,7,A,1.0,2.0,3.0,4.0\n2,8,B,5.0,6.0,7.0,8.0\n") + kwargs = {"region_column": "Sample", "instance_id": "label", "markers": ["CD8"]} + + table = read_cell_table(path, "codex", pixel_size_um=None, default_region="A", cells_table_kwargs=kwargs) + + assert list(table.region) == ["A", "B"] + assert list(table.instance_id) == [7, 8] + assert list(table.var.index) == ["CD8"] + assert "CD3" in table.obs.columns def test_build_cells_single_table_and_circles(tmp_path: Path) -> None: write_fcs(tmp_path / "reg001.fcs", _cells(n=3, region=1)) write_fcs(tmp_path / "reg002.fcs", _cells(n=4, region=2)) - tables = [read_codex_table(tmp_path / f"reg00{r}.fcs") for r in (1, 2)] + tables = [_read_codex(tmp_path / f"reg00{r}.fcs") for r in (1, 2)] translation = Translation([10, 0], axes=("x", "y")) with pytest.warns(UserWarning, match=r"No segmentation mask found for region.*circles") as record: @@ -145,7 +177,7 @@ def test_build_cells_single_table_and_circles(tmp_path: Path) -> None: def test_build_cells_different_panels_split_tables(tmp_path: Path) -> None: write_fcs(tmp_path / "reg001.fcs", _cells(region=1, markers=("cyc001_ch001",))) write_fcs(tmp_path / "reg002.fcs", _cells(region=2, markers=("cyc001_ch001", "cyc001_ch002"))) - tables = [read_codex_table(tmp_path / f"reg00{r}.fcs") for r in (1, 2)] + tables = [_read_codex(tmp_path / f"reg00{r}.fcs") for r in (1, 2)] with pytest.warns(UserWarning, match="circles"): anndatas, _ = build_cells(tables, {}, {}, {}) @@ -158,13 +190,13 @@ def test_build_cells_different_panels_split_tables(tmp_path: Path) -> None: def test_dedupe_matches_akoya_convention(tmp_path: Path) -> None: write_fcs(tmp_path / "reg001.fcs", _cells(), {"cyc001_ch001": "X", "cyc001_ch002": "X"}) - assert list(read_codex_table(tmp_path / "reg001.fcs").var.index) == dedupe_channel_names(["X", "X"]) + assert list(_read_codex(tmp_path / "reg001.fcs").var.index) == dedupe_channel_names(["X", "X"]) def test_read_codex_fcs_blank_pns_falls_back_to_pnn(tmp_path: Path) -> None: write_fcs(tmp_path / "reg001.fcs", _cells(), {"cyc001_ch001": " ", "cyc001_ch002": "CD3"}) - assert list(read_codex_table(tmp_path / "reg001.fcs").var.index) == ["cyc001_ch001", "CD3"] + assert list(_read_codex(tmp_path / "reg001.fcs").var.index) == ["cyc001_ch001", "CD3"] @pytest.mark.parametrize("different_panels", [False, True]) @@ -172,7 +204,7 @@ def test_build_cells_overlapping_regions_raise(tmp_path: Path, different_panels: write_fcs(tmp_path / "a.fcs", _cells(region=1)) markers = ("cyc001_ch001",) if different_panels else ("cyc001_ch001", "cyc001_ch002") write_fcs(tmp_path / "b.fcs", _cells(region=1, markers=markers)) - tables = [read_codex_table(tmp_path / name) for name in ("a.fcs", "b.fcs")] + tables = [_read_codex(tmp_path / name) for name in ("a.fcs", "b.fcs")] with pytest.raises(ValueError, match="Several cell tables"): build_cells(tables, {}, {}, {}) diff --git a/tests/test_codex.py b/tests/test_codex.py index 98d23c08..bd24a1a6 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -664,7 +664,7 @@ def test_codex_unreadable_regionless_table_raises(tmp_path: Path) -> None: (tmp_path / "cells.csv").write_text("cell_id,x,y,cyc001_ch001\n1,1.0,2.0,3.0\n") (tmp_path / "clusters.csv").write_text("cellid,cluster\n1,2\n") - with pytest.raises(ValueError, match="cannot be assigned to a region"): + with pytest.raises(ValueError, match="cannot be assigned to regions"): codex(tmp_path, image_models_kwargs=_FAST) # a CODEX table next to a single image that cannot be read @@ -675,7 +675,7 @@ def test_codex_unreadable_regionless_table_raises(tmp_path: Path) -> None: # a single table file without a region column and no image write_fcs(tmp_path / "Kidney.fcs", pd.DataFrame({"cell_id": [1.0], "x": [1.0], "y": [2.0], "cyc001_ch001": [3.0]})) - with pytest.raises(ValueError, match="cannot be assigned to a region"): + with pytest.raises(ValueError, match="cannot be assigned to regions"): codex(tmp_path / "Kidney.fcs") @@ -769,3 +769,26 @@ def test_single_region_codex_table_write_roundtrip(tmp_path: Path, case: str) -> assert table.obs.index.name is None assert table.obs["cell_id"].dtype == np.int64 np.testing.assert_array_equal(table.obs["cell_id"], sdata.tables["table"].obs["cell_id"]) + + +def test_codex_discovered_tables_assign_regions_like_given_ones(tmp_path: Path) -> None: + for name in ("Tonsil 1", "Tonsil 2"): + tifffile.imwrite(tmp_path / f"{name}.tif", np.zeros((2, 16, 16), dtype=np.uint16)) + (tmp_path / "cells.csv").write_text( + "cell_id,region,x,y,size,cyc001_ch001\n1,Tonsil 1,1.0,2.0,3.0,4.0\n1,Tonsil 2,3.0,4.0,3.0,5.0\n" + ) + + with pytest.warns(UserWarning, match="circles"): + sdata = codex(tmp_path, image_models_kwargs=_FAST) + + assert sorted(sdata.shapes) == ["Tonsil_1_cells", "Tonsil_2_cells"] + assert sorted(sdata.coordinate_systems) == ["Tonsil_1", "Tonsil_2"] + + +def test_codex_discovered_table_of_another_region_raises(tmp_path: Path) -> None: + data = write_legacy_processor_tree(tmp_path, regions=2, cycles=1, channels=1, fcs=False, n_cells=3) + # the table of reg002 whose region column names reg001 + data["cells"].assign(region=1.0).to_csv(next(tmp_path.rglob("reg002_compensated.csv")), index=False) + + with pytest.raises(ValueError, match="has no cells of region 'reg002'"): + codex(tmp_path, image_models_kwargs=_FAST) diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 355cab03..09cb88b5 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -548,6 +548,24 @@ def test_phenocycler_cells_mapping_per_region(tmp_path: Path) -> None: assert sorted(sdata.labels) == ["A_labels", "B_labels"] +def test_codex_format_table_uses_region_column(tmp_path: Path) -> None: + for region in ("A", "B"): + write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI"]) + (tmp_path / "cells.csv").write_text("cell_id,Sample,x,y,DAPI\n1,A,1.0,2.0,3.0\n2,B,1.0,2.0,3.0\n3,B,4,5,6\n") + + with pytest.warns(UserWarning, match="circles"): + sdata = phenocycler( + tmp_path, + regions=["A"], + cells_table=tmp_path / "cells.csv", + cells_table_kwargs={"region_column": "Sample"}, + image_models_kwargs=_FAST, + ) + + assert list(sdata.tables["table"].obs["cell_id"]) == [1] + assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["A_cells"] + + def test_mapping_with_unknown_region_raises(tmp_path: Path) -> None: write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI"]) _write_mcquant(tmp_path / "A.csv", markers=("DAPI",)) From 09c020ab960f22b8fdd477d48a4182fffa154aa0 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:36:34 +0200 Subject: [PATCH 067/107] fix(akoya): recognise QuPath built-in, 0.6 and detection intensity columns Also exclude empty text columns and R-named measurements from the bare-marker fallback. --- .../readers/_utils/_cell_tables.py | 69 ++++++++++++++----- tests/readers/test_cell_tables.py | 48 +++++++++++++ 2 files changed, 99 insertions(+), 18 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 5f795618..cf7c0357 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -86,9 +86,16 @@ _META_COLUMNS = frozenset( {"cell_id", "region", "tile_num", "x", "y", "z", "x_tile", "y_tile", "size", "tsne_x", "tsne_y", "homogeneity"} ) -# QuPath measurement columns, e.g. `CD3: Cell: Mean`, also in the R `make.names` form `CD3..Cell..Mean` and as -# `CD3_Cell_Mean`; the per-marker intensity is taken from the first compartment -_QUPATH_MARKER = re.compile(r"(.+?)(?:: |\.\.|_)(Cell|Nucleus|Cytoplasm|Membrane)(?:: |\.\.|_)Mean") +# QuPath mean intensities: `CD3: Cell: Mean` (also `CD3..Cell..Mean` of R's `make.names` and `CD3_Cell_Mean`), +# `Cell: CD3: Mean` (QuPath >= 0.6) and `Cell: CD3 mean` (built-in cell detection); the per-marker intensity is +# taken from the first compartment. Detections without compartments have `CD3: Mean`. +_QUPATH_MARKER = re.compile( + r"(?P.+?)(?:: |\.\.|_)(?PCell|Nucleus|Cytoplasm|Membrane)(?:: |\.\.|_)Mean" +) +_QUPATH_COMPARTMENT_MARKER = re.compile( + r"(?PCell|Nucleus|Cytoplasm|Membrane): (?P.+?)(?:: M| m)ean" +) +_QUPATH_DETECTION_MARKER = re.compile(r"(?P.+?): Mean") _QUPATH_COMPARTMENTS = ("Cell", "Nucleus", "Cytoplasm", "Membrane") # other QuPath measurements, e.g. `Nucleus: Area µm^2` or `Nucleus/Cell area ratio`, are not markers _QUPATH_MEASUREMENT = re.compile(r"(?:Cell|Nucleus|Cytoplasm|Membrane)\b|.*: ") @@ -276,10 +283,12 @@ def read_cell_table( `UserWarning`, raising if that is not unique. Instance ids from a numeric ``Object ID``, else the row number (from 1); the region from the ``Image`` column. The markers are ``markers``, else the ``: : Mean`` columns - (also ``....Mean`` and ``__Mean``), one per marker from - the first of the ``Cell``, ``Nucleus``, ``Cytoplasm`` and ``Membrane`` compartments (in - ``var["compartment"]``), else, with a `UserWarning`, the numeric columns that are not QuPath - measurements (e.g. ``CD44`` of a detection export; not morphology such as ``Area µm^2``, + (also ``....Mean`` and ``__Mean``), ``: + : Mean`` (QuPath >= 0.6) and ``: mean`` (built-in cell detection) + columns, one per marker from the first of the ``Cell``, ``Nucleus``, ``Cytoplasm`` and ``Membrane`` + compartments (in ``var["compartment"]``), else the ``: Mean`` columns of detections + without compartments, else, with a `UserWarning`, the numeric columns that are not QuPath + measurements or empty (e.g. ``CD44`` of a detection export; not morphology such as ``Area µm^2``, ``Max diameter µm`` or ``Circularity``). The radius comes from ``Cell: Area µm^2`` (or ``px^2``) in the unit of the centroids. A table split into centroids and measurements (two files) must be joined on the object id first. The cell outlines are read from a QuPath GeoJSON export, see @@ -670,24 +679,36 @@ def _qupath_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> di area = columns.get(f"Cell: Area {'µm' if unit == 'um' else 'px'}^2") roles = {x, y, object_id, region_column, area} - compartments: dict[str, dict[str, str]] = {} + intensities: dict[str, dict[str | None, str]] = {} for column in df.columns: - if (match := _QUPATH_MARKER.fullmatch(column)) is not None: - compartments.setdefault(match.group(1), {})[match.group(2)] = column + if (found := _qupath_intensity(column)) is not None: + intensities.setdefault(found[0], {}).setdefault(found[1], column) + # `: Mean` columns are the markers only of detections without compartments + with_compartments = {marker: found for marker, found in intensities.items() if set(found) - {None}} + compartments = with_compartments or intensities if kwargs.get("markers") is not None: markers = _select_markers(df, kwargs["markers"], roles) var = pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(markers))) elif compartments: - chosen = {marker: next(c for c in _QUPATH_COMPARTMENTS if c in found) for marker, found in compartments.items()} + chosen = { + marker: next(c for c in (*_QUPATH_COMPARTMENTS, None) if c in found) + for marker, found in compartments.items() + } markers = [compartments[marker][compartment] for marker, compartment in chosen.items()] - var = pd.DataFrame( - {"source_column": markers, "compartment": list(chosen.values())}, - index=pd.Index(dedupe_channel_names(list(chosen))), - ) + var = pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(list(chosen)))) + if with_compartments: + var["compartment"] = list(chosen.values()) else: # e.g. a detection export with the measurements named after the markers only; its morphology (e.g. - # `Area µm^2`, `Max diameter µm`, `Circularity`) is not a marker - candidates = [c for c in df.columns if not _QUPATH_MEASUREMENT.match(c) and not _is_qupath_morphology(c)] + # `Area µm^2`, `Max diameter µm`, `Circularity`), other measurements (e.g. `DAPI: Median`) and empty text + # columns (e.g. `Name`, read as all-NaN numbers) are not markers + candidates = [ + c + for c in df.columns + if not any(_QUPATH_MEASUREMENT.match(name) for name in _qupath_names([c])) + and not _is_qupath_morphology(c) + and df[c].notna().any() + ] markers = _select_markers(df[candidates], None, roles) var = pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(markers))) if markers: @@ -696,7 +717,7 @@ def _qupath_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> di else: message = "its table has no marker intensities" warnings.warn( - f"{path} has no ': : Mean' columns; {message}. Set " + f"{path} has no QuPath mean intensity columns (e.g. ': : Mean'); {message}. Set " "`cells_table_kwargs['markers']` to choose the marker columns.", UserWarning, stacklevel=5, @@ -713,6 +734,18 @@ def _qupath_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> di } +def _qupath_intensity(column: str) -> tuple[str, str | None] | None: + """The marker and compartment (`None` without one) of a QuPath mean intensity column, also in R's form.""" + for name in (column, *_qupath_names([column])): + for pattern in (_QUPATH_MARKER, _QUPATH_COMPARTMENT_MARKER): + if (match := pattern.fullmatch(name)) is not None: + return match["marker"], match["compartment"] + for name in (column, *_qupath_names([column])): + if (match := _QUPATH_DETECTION_MARKER.fullmatch(name)) is not None: + return match["marker"], None + return None + + def _is_qupath_morphology(column: str) -> bool: """Whether ``column`` (also in R's ``make.names`` form) is a QuPath measurement with a unit or a shape measure.""" return any(_QUPATH_UNIT.search(name) or name in _QUPATH_UNITLESS_MEASUREMENTS for name in _qupath_names([column])) diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index 1ead114d..33cdd84b 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -732,6 +732,54 @@ def test_qupath_bare_markers_exclude_morphology(tmp_path: Path, r_names: bool) - assert {header[4], header[7], header[10]} <= set(table.obs.columns) +@pytest.mark.parametrize( + ("intensities", "compartments"), + [ + # built-in cell detection (all versions) + (["Nucleus: DAPI mean", "Nucleus: DAPI std dev", "Cytoplasm: CD3 mean", "Cell: CD3 mean"], ["Nucleus", "Cell"]), + # cells of QuPath >= 0.6 + ( + ["Nucleus: DAPI: Mean", "Nucleus: DAPI: Median", "Cell: CD3: Mean", "Nucleus: CD3: Mean"], + ["Nucleus", "Cell"], + ), + # the same in R's `make.names` form + (["Nucleus..DAPI.mean", "Cytoplasm..CD3.mean", "Cell..CD3.mean"], ["Nucleus", "Cell"]), + # detections without compartments + (["DAPI: Mean", "DAPI: Median", "DAPI: Std.Dev.", "CD3: Mean", "CD3: Max"], None), + ], +) +def test_qupath_intensity_column_forms(tmp_path: Path, intensities: list[str], compartments: list[str] | None) -> None: + path = tmp_path / "cells.csv" + header = ["Image", "Object ID", "Name", "Classification", "Centroid X µm", "Centroid Y µm", *intensities] + rows = [f"S,u{i},,,{i},{i},{','.join(str(i + j) for j in range(len(intensities)))}" for i in range(2)] + path.write_text("\n".join([",".join(header), *rows]) + "\n", encoding="utf-8") + + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) + table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) + + assert list(table.var.index) == ["DAPI", "CD3"] + if compartments is None: + assert "compartment" not in table.var.columns + else: + assert list(table.var["compartment"]) == compartments + assert {"Name", "Classification"} <= set(table.obs.columns) + + +@pytest.mark.parametrize("r_names", [False, True]) +def test_qupath_bare_markers_exclude_empty_text_and_measurements(tmp_path: Path, r_names: bool) -> None: + path = tmp_path / "cells.csv" + header = ["Image", "Object ID", "Name", "Classification", "Centroid X µm", "Centroid Y µm", "DAPI: Median", "CD44"] + if r_names: + header = [name.replace(": ", "..").replace(" ", ".").replace("µm", "um") for name in header] + path.write_text(",".join(header) + "\nS,u1,,,1,2,3,4\nS,u2,,,1,2,3,4\n", encoding="utf-8") + + with pytest.warns(UserWarning, match=r"reading the 1 other numeric columns \['CD44'\]"): + table = read_cell_table(path, "qupath", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) + + assert list(table.var.index) == ["CD44"] + + def test_qupath_xy_centroids_need_a_unit(tmp_path: Path) -> None: path = tmp_path / "cells.csv" path.write_text("Object ID,X,Y,CD3: Cell: Mean\nab,10,5,1\n", encoding="utf-8") From 1d80d0ecc10c43269a4274a9541f1bfb8bb7fe65 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:44:15 +0200 Subject: [PATCH 068/107] fix(akoya): match region values to region ids and image file names only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A value no longer names a region it merely starts with (Tonsil_2_Scan1 is not Tonsil), a table for one region takes the cells of unmatched values only if its region column holds a single value, and region values are matched before µm conversion and the duplicate-id check, so each row converts with its own region's pixel size. --- .../readers/_utils/_cell_tables.py | 121 +++++++++++------- src/spatialdata_io/readers/codex.py | 29 ++--- src/spatialdata_io/readers/phenocycler.py | 7 +- tests/test_phenocycler.py | 85 +++++++++++- 4 files changed, 177 insertions(+), 65 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index cf7c0357..d6352400 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -104,6 +104,8 @@ # QuPath shape and detection measurements without a unit, e.g. of a detection export without compartments _QUPATH_UNITLESS_MEASUREMENTS = frozenset({"Circularity", "Detection probability", "Eccentricity", "Solidity"}) _QUPATH_IMAGE = "Image" +# the extensions of image files, also named in the region column of a table (e.g. QuPath's `Image`) +_IMAGE_SUFFIX = re.compile(r"(?:\.er)?\.(?:qptiff|ome\.tiff?|tiff?)$", re.IGNORECASE) _QUPATH_ID = "Object ID" # QuPath's centroid columns by unit; R's `make.names` writes `Centroid X µm` as `Centroid.X.um` _QUPATH_CENTROIDS = {"um": ("Centroid X µm", "Centroid Y µm"), "px": ("Centroid X px", "Centroid Y px")} @@ -258,6 +260,7 @@ def read_cell_table( pixel_size_um: float | Mapping[str, float | None] | None, default_region: str | None, cells_table_kwargs: Mapping[str, Any], + regions: Mapping[str, Sequence[Path]] | None = None, ) -> CellTable: """Read a cell table of any supported format. @@ -315,10 +318,16 @@ def read_cell_table( pixel_size_um The pixel size of the region image, to convert µm centroids to pixels; for a table of several regions, the - pixel size of each region, by region id: every cell is converted with the pixel size of the region its - region column names (see :func:`read_region_tables`), cells of other regions get NaN coordinates. + pixel size of each region, by region id: every cell is converted with the pixel size of its region, cells + of other regions get NaN coordinates. default_region The region id of all cells if the table has no region column. + regions + The region ids of the dataset with the image files of each, to which the values of the region column are + matched, see :func:`_match_region`; values that name none of them are kept. If the table has + ``default_region`` and its region column holds a single value that names no region, that value names + ``default_region`` (e.g. the source image of a table for one region). If `None`, the values are the + region ids. cells_table_kwargs Optional keys: ``region_column`` (the column holding the region of each cell; QuPath default ``Image``), and for ``"generic"`` tables ``x`` and ``y`` (required centroid columns), ``instance_id`` (else the row @@ -333,7 +342,7 @@ def read_cell_table( Returns ------- - The cell table. Its regions are the values of the region column, or ``default_region``. + The cell table. Its regions are the (matched) values of the region column, or ``default_region``. """ unknown = sorted(set(cells_table_kwargs) - set(CELLS_TABLE_KWARGS)) if unknown: @@ -370,10 +379,11 @@ def read_cell_table( mask_pixels = fmt in {"mcquant", "sprm"} if fmt == "generic": mask_pixels = _coordinates_space(path, roles["in_um"], cells_table_kwargs) == "mask" + region = _region_values(df, roles["region_column"], default_region, regions) units = cells_table_kwargs.get("geojson_units", "px") row_pixel_sizes = None if roles["in_um"] or (fmt == "qupath" and units == "um"): - row_pixel_sizes = _row_pixel_sizes(path, df, roles["region_column"], pixel_size_um) + row_pixel_sizes = _row_pixel_sizes(path, region, pixel_size_um) if roles.pop("in_um"): if row_pixel_sizes is None: raise ValueError( @@ -381,7 +391,7 @@ def read_cell_table( "pixel size is missing)." ) roles["px_per_unit"] = 1 / row_pixel_sizes - table = _from_frame(path, df, default_region=default_region, **roles) + table = _from_frame(path, df, region=region, **roles) table.mask_pixels = mask_pixels if fmt == "sprm": table.centroids_px = _read_sprm_centers(path, table) @@ -394,29 +404,46 @@ def read_cell_table( return table +def _region_values( + df: pd.DataFrame, + region_column: str | None, + default_region: str | None, + regions: Mapping[str, Sequence[Path]] | None, +) -> pd.Series | None: + """The region of every row of ``df``, see :func:`read_cell_table`; `None` if it is unknown.""" + if region_column is None or region_column not in df.columns: + return None if default_region is None else pd.Series(default_region, index=df.index) + values = df[region_column].astype(str) + if regions is None: + return values + matched = {value: _match_region(value, regions) for value in values.unique()} + if default_region is not None and len(matched) == 1 and None in matched.values(): + # the region column of a table for one region names its source image only + return pd.Series(default_region, index=df.index) + return values.map({value: value if region is None else region for value, region in matched.items()}) + + def _row_pixel_sizes( - path: Path, df: pd.DataFrame, region_column: str | None, pixel_size_um: float | Mapping[str, float | None] | None + path: Path, region: pd.Series | None, pixel_size_um: float | Mapping[str, float | None] | None ) -> float | np.ndarray | None: - """The pixel size of every row of ``df``, or one for all rows; `None` if it is unknown. + """The pixel size of every row, by its ``region``, or one for all rows; `None` if it is unknown. - For pixel sizes by region id, each row gets that of the region its region column names, NaN if it names none of - them (its cells are not read). Raises if a region named by the table has no pixel size. + For pixel sizes by region id, rows of other regions get NaN (their cells are not read). Raises if a region of + the rows has no pixel size. """ if not isinstance(pixel_size_um, Mapping): return pixel_size_um - if region_column is None or region_column not in df.columns: + if region is None: sizes = set(pixel_size_um.values()) return sizes.pop() if len(sizes) == 1 else None - values = df[region_column].astype(str) - regions = {value: _match_region(value, list(pixel_size_um)) for value in values.unique()} - unknown = sorted({region for region in regions.values() if region is not None and pixel_size_um[region] is None}) + unknown = sorted(r for r in region.unique() if r in pixel_size_um and pixel_size_um[r] is None) if unknown: raise ValueError( f"{path} has coordinates in µm, but the pixel size of region(s) {unknown} is unknown, so they cannot be " "converted to pixels." ) - sizes_by_value = {value: np.nan if region is None else pixel_size_um[region] for value, region in regions.items()} - return values.map(sizes_by_value).to_numpy(dtype=np.float64) + sizes_by_region = {r: pixel_size_um.get(r, np.nan) for r in region.unique()} + return region.map(sizes_by_region).to_numpy(dtype=np.float64) def _qupath_names(columns: Sequence[str]) -> dict[str, str]: @@ -958,10 +985,13 @@ def _from_frame( instance_column: str | None, region_column: str | None, area_column: str | None, - default_region: str | None, + region: pd.Series | None, px_per_unit: float | np.ndarray = 1.0, ) -> CellTable: - """Build a `CellTable` from a data frame and the roles of its columns; ``px_per_unit`` may be given per row.""" + """Build a `CellTable` from a data frame, the roles of its columns and the region of every row. + + ``px_per_unit`` may be given per row. + """ roles = [x, y, instance_column, region_column, area_column, *markers] missing = [column for column in roles if column is not None and column not in df.columns] if missing: @@ -976,17 +1006,13 @@ def _from_frame( instance_id.index = index instance_id.name = CodexKeys.INSTANCE_KEY.value - if region_column is not None: - region = pd.Series(df[region_column].astype(str).to_numpy(), index=index) - elif default_region is not None: - region = pd.Series(default_region, index=index) - else: + if region is None: raise ValueError( f"{path} has no region column, so its cells cannot be assigned to regions: a table for several regions " "needs a region column (`cells_table_kwargs['region_column']`). Alternatively, pass one table per " "region as a mapping {region: path}." ) - region.name = CodexKeys.REGION_KEY.value + region = pd.Series(region.to_numpy(), index=index, name=CodexKeys.REGION_KEY.value) _check_unique_instance_ids(path, region, instance_id) obs = _drop_key_columns(df.drop(columns=markers), instance_column, region_column) @@ -1085,18 +1111,17 @@ def read_region_tables( files: Mapping[str, Path], fmt: str, *, - available: Sequence[str], + available: Mapping[str, Sequence[Path]], selected: Sequence[str], pixel_sizes: Mapping[str, float | None], cells_table_kwargs: Mapping[str, Any], ) -> list[CellTable]: """Read user-supplied cell tables, as resolved by :func:`resolve_per_region`, and assign their cells to regions. - A region column value names the region whose id it equals, else the longest region id it starts with followed - by a non-alphanumeric character (``A_Scan1.qptiff`` -> ``A``). A table given for one region holds the cells of - that region; if its region column matches no region, it holds only cells of that region. Cells of a table for - all regions (``"*"``) whose region matches no available region are dropped with a warning; cells of available - regions that are not read are dropped silently. + The values of a region column are matched to the regions, see :func:`_match_region`. A table given for one + region holds the cells of that region; if its region column holds a single value that names no region, it + holds only cells of that region. Cells whose region names no available region are dropped with a warning; + cells of available regions that are not read are dropped silently. Parameters ---------- @@ -1105,7 +1130,7 @@ def read_region_tables( fmt The table format, see :func:`read_cell_table`. available - All region ids of the dataset. + All region ids of the dataset, with the image files of each. selected The region ids read. pixel_sizes @@ -1118,43 +1143,39 @@ def read_region_tables( The cell tables, one per file. """ geojson = cells_table_kwargs.get("geojson") - geojson_files = resolve_per_region(geojson, selected, available) if geojson is not None else {} + geojson_files = resolve_per_region(geojson, selected, list(available)) if geojson is not None else {} if unknown := sorted(geojson_files.keys() - files.keys()): raise ValueError( f"`cells_table_kwargs['geojson']` is given for {unknown}, but the cell tables are given for " f"{list(files)}; pass the GeoJSON files in the same form as the tables." ) tables = [] + # the µm coordinates of each cell are converted with the pixel size of its region + sizes = {region: pixel_sizes.get(region) for region in selected} for key, path in files.items(): - # a table for several regions converts the µm coordinates of each cell with the pixel size of its region - size = {region: pixel_sizes.get(region) for region in selected} if key == "*" else pixel_sizes.get(key) table = read_cell_table( path, fmt, - pixel_size_um=size, + pixel_size_um=sizes, default_region=None if key == "*" else key, cells_table_kwargs={**cells_table_kwargs, "geojson": geojson_files.get(key)}, + regions=available, ) tables.append(_assign_regions(table, path, key, available, selected)) return tables def _assign_regions( - table: CellTable, path: Path, key: str, available: Sequence[str], selected: Sequence[str] + table: CellTable, path: Path, key: str, available: Mapping[str, Sequence[Path]], selected: Sequence[str] ) -> CellTable: values = list(table.region.unique()) - matched = {value: _match_region(value, available) for value in values} - if key != "*" and all(region is None for region in matched.values()): - # the region column of a table for one region names its source image only - matched = dict.fromkeys(values, key) - unmatched = sorted(value for value, region in matched.items() if region is None) + unmatched = sorted(value for value in values if value not in available) if unmatched: warnings.warn( f"Ignoring the cells of {unmatched} in {path}: they match none of the regions {list(available)}.", UserWarning, stacklevel=4, ) - table.region = table.region.map(matched) keep = table.region.isin(selected if key == "*" else [key]).to_numpy() if not keep.any(): target = f"region '{key}'" if key != "*" else f"the regions read {list(selected)}" @@ -1162,17 +1183,23 @@ def _assign_regions( return table.take(np.flatnonzero(keep)) -def _match_region(value: str, regions: Sequence[str]) -> str | None: - """The region named by ``value``: equal to it, else the longest one it starts with before a non-alphanumeric. +def _match_region(value: str, regions: Mapping[str, Sequence[Path]]) -> str | None: + """The region named by ``value``: its id, else the region of an image file with that name or stem, else `None`. - ``value`` is also matched after :func:`spatialdata.sanitize_name`, as the region ids are, so that e.g. the - QuPath ``Image`` value ``A_[1,2].qptiff`` names the region ``A__1_2_``. + ``value`` is also matched after :func:`spatialdata.sanitize_name`, as the region ids are, so that e.g. ``A_[1,2]`` + names the region ``A__1_2_``. A file name may be followed by a non-alphanumeric character and more, e.g. QuPath's + ``A_Scan1.qptiff - resolution #1``; a name that merely starts with another region's name (``A_2_Scan1.qptiff``) + does not name it. """ candidates = list(dict.fromkeys((value, sanitize_name(value)))) if exact := next((candidate for candidate in candidates if candidate in regions), None): return exact - prefixes = [r for r in regions for c in candidates if c.startswith(r) and not c[len(r)].isalnum()] - return max(prefixes, key=len, default=None) + stem = _IMAGE_SUFFIX.sub("", value) + for region, files in regions.items(): + for name in (file.name for file in files): + if stem == _IMAGE_SUFFIX.sub("", name) or (value.startswith(name) and not value[len(name)].isalnum()): + return region + return None def read_labels( diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 3daed2ec..d4e9ac74 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -11,7 +11,6 @@ import dask.array as da import numpy as np -import pandas as pd from spatialdata import SpatialData, sanitize_name from spatialdata_io._constants._constants import CodexKeys @@ -26,7 +25,6 @@ ) from spatialdata_io.readers._utils._cell_tables import ( CellTable, - _match_region, build_spatialdata, detect_table_format, read_cell_table, @@ -277,7 +275,7 @@ def codex( cell_tables = read_region_tables( table_files, cells_table_format, - available=list(layout.images), + available=layout.images, selected=selected, pixel_sizes={scan.region: scan.pixel_size_um for scan in scans}, cells_table_kwargs=cells_table_kwargs, @@ -286,7 +284,12 @@ def codex( else: named_files = {key: layout.tables[key] for key in selected if key in layout.tables} named = read_region_tables( - named_files, "codex", available=available, selected=selected, pixel_sizes={}, cells_table_kwargs={} + named_files, + "codex", + available={region: layout.images.get(region, []) for region in available}, + selected=selected, + pixel_sizes={}, + cells_table_kwargs={}, ) read = dict(zip(named_files, named, strict=True)) if regionless is not None: @@ -327,17 +330,13 @@ def _warn_fcs_fallback(files: Iterable[Path], fcs: bool) -> None: def _read_regionless_table(file: Path, layout: _LegacyLayout) -> CellTable: """Read the discovered cell table ``file`` that has no region in its file name.""" only_image = next(iter(layout.images)) if len(layout.images) == 1 else None - table = read_cell_table(file, "codex", pixel_size_um=None, default_region=only_image, cells_table_kwargs={}) - values = table.region.unique() - if only_image is not None and len(values) == 1 and _match_region(values[0], [only_image]) is None: - # next to a single image named after its file (e.g. `Kidney.tif`), its cells belong to that image - table.region = pd.Series(only_image, index=table.region.index, name=table.region.name) - else: - # values that name no region of the dataset are regions without an image - regions = list(layout.images.keys() | layout.tables.keys() - {"*"}) - table.region = table.region.map( - {value: _match_region(value, regions) or sanitize_name(value) for value in values} - ) + regions = {region: layout.images.get(region, []) for region in layout.images.keys() | layout.tables.keys() - {"*"}} + # next to a single image named after its file (e.g. `Kidney.tif`), a single region value names that image + table = read_cell_table( + file, "codex", pixel_size_um=None, default_region=only_image, cells_table_kwargs={}, regions=regions + ) + # values that name no region of the dataset are regions without an image + table.region = table.region.map({value: sanitize_name(value) for value in table.region.unique()}) return table diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 50917dca..98e7e941 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -129,7 +129,8 @@ def phenocycler( A cell table: a path, or a mapping from region id to path, e.g. ``{{"A": "A_cells.csv"}}``. A single path for several regions needs a region column (``Image`` for QuPath, ``region`` for CODEX tables, else ``cells_table_kwargs["region_column"]``); its values are matched to the region ids, also after - :func:`spatialdata.sanitize_name`, e.g. ``A_Scan1.qptiff`` to ``A`` and ``A_[1,2].qptiff`` to ``A__1_2_``. + :func:`spatialdata.sanitize_name` (``A_[1,2]`` to ``A__1_2_``), or to the names of the region's files + (``A_Scan1.qptiff`` or ``A_Scan1`` to ``A``). cells_table_format The format of ``cells_table``: ``"auto"`` (detected from the header), ``"codex"`` (CODEX Processor), ``"qupath"`` (QuPath measurements; µm centroids are converted with the region's pixel size), ``"mcquant"`` @@ -199,7 +200,9 @@ def phenocycler( cell_tables = read_region_tables( table_files, cells_table_format, - available=list(qptiffs), + available={ + region: [file, *([eight_bit[region]] if region in eight_bit else [])] for region, file in qptiffs.items() + }, selected=selected, pixel_sizes={scan.region: scan.pixel_size_um for scan in scans}, cells_table_kwargs=cells_table_kwargs, diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 09cb88b5..617806de 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -16,7 +16,7 @@ from spatialdata.models import get_channel_names from spatialdata.transformations import Identity, Scale, Sequence, Translation, get_transformation -from spatialdata_io import phenocycler +from spatialdata_io import codex, phenocycler from spatialdata_io.__main__ import phenocycler_wrapper from tests._akoya_data import forbid_pixel_reads, write_qptiff @@ -946,3 +946,86 @@ def test_phenocycler_keyword_only(tmp_path: Path) -> None: write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI"]) with pytest.raises(TypeError, match="positional argument"): phenocycler(tmp_path, ["A"]) # type: ignore[misc] + + +def _write_qupath_px(path: Path, images: dict[str, int]) -> None: + rows = ["Image,Object ID,Centroid X px,Centroid Y px,CD3: Cell: Mean"] + rows += [f"{image},{i},{i},{i},1" for image, n in images.items() for i in range(1, n + 1)] + path.write_text("\n".join(rows) + "\n", encoding="utf-8") + + +def test_region_values_match_image_names_not_prefixes(tmp_path: Path) -> None: + write_qptiff(tmp_path / "Tonsil_Scan1.qptiff", biomarkers=["DAPI"]) + images = {"Tonsil_Scan1.qptiff - resolution #1": 3, "Tonsil_2_Scan1.qptiff": 2, "Spleen_Scan1.qptiff": 2} + _write_qupath_px(tmp_path / "cells.csv", images) + + with pytest.warns(UserWarning, match=r"Ignoring the cells of \['Spleen_Scan1.qptiff', 'Tonsil_2_Scan1.qptiff'\]"): + sdata = phenocycler( + tmp_path / "Tonsil_Scan1.qptiff", cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST + ) + + assert list(sdata.tables["table"].obs["cell_id"]) == [1, 2, 3] + + +@pytest.mark.parametrize("regions", [None, ["A"]]) +def test_region_values_naming_no_image_are_not_assigned_to_the_selected_one( + tmp_path: Path, regions: list[str] | None +) -> None: + for region in ("A", "B"): + write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI"]) + _write_qupath_px(tmp_path / "cells.csv", {"Patient 7 tonsil": 2, "Patient 9 lymph node": 2}) + + with pytest.warns(UserWarning, match="Ignoring the cells"), pytest.raises(ValueError, match="has no cells of"): + phenocycler(tmp_path, regions=regions, cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) + + +def test_duplicate_instance_ids_are_checked_per_assigned_region(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + # two names of the same image + _write_qupath_px(tmp_path / "cells.csv", {"S": 1, "S_Scan1.qptiff": 1}) + + with pytest.raises(ValueError, match=r"cells\.csv has 1 duplicate instance id.*region 'S'"): + phenocycler(tmp_path, cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) + + +def test_codex_region_values_match_qptiff_file_names(tmp_path: Path) -> None: + for region in ("reg001", "reg002"): + write_qptiff(tmp_path / f"exp_{region}.qptiff", biomarkers=["DAPI"], software="Codex Instrument Manager 1.30") + _write_qupath_px(tmp_path / "cells.csv", {"exp_reg001.qptiff": 1, "exp_reg002.qptiff": 2}) + + with pytest.warns(UserWarning, match="circles"): + sdata = codex(tmp_path, regions=["reg001"], cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) + + assert list(sdata.tables["table"].obs["cell_id"]) == [1] + + +@pytest.mark.parametrize("regions", [["A"], ["B"]]) +def test_selected_region_of_qupath_table_uses_each_rows_pixel_size(tmp_path: Path, regions: list[str]) -> None: + rows, features = ["Image,Object ID,Centroid X µm,Centroid Y µm,CD3: Cell: Mean"], [] + for region, pixel_size in (("A", 0.5), ("B", 0.25)): + write_qptiff( + tmp_path / f"{region}_Scan1.qptiff", + biomarkers=["DAPI"], + resolution_px_per_cm=1e4 / pixel_size, + slide_id=region, + ) + for i in range(4): + x, y = 20 + 25 * i, 30 + 15 * i + rows.append(f"{region}_Scan1.qptiff,{region}{i},{x * pixel_size},{y * pixel_size},{i}") + ring = [[x - 3, y - 3], [x + 3, y - 3], [x + 3, y + 3], [x - 3, y + 3], [x - 3, y - 3]] + features.append( + {"type": "Feature", "id": f"{region}{i}", "geometry": {"type": "Polygon", "coordinates": [ring]}} + ) + (tmp_path / "cells.csv").write_text("\n".join(rows) + "\n", encoding="utf-8") + (tmp_path / "cells.geojson").write_text(json.dumps({"type": "FeatureCollection", "features": features})) + + sdata = phenocycler( + tmp_path, + regions=regions, + cells_table=tmp_path / "cells.csv", + image_models_kwargs=_FAST, + labels_models_kwargs=_FAST, + ) + + assert list(sdata.labels) == [f"{regions[0]}_labels"] + np.testing.assert_allclose(np.asarray(sdata.tables["table"].obsm["spatial"])[:, 0], [20, 45, 70, 95]) From c9cc2c56e30e88855217ec9a005458e8ff7d2557 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:45:30 +0200 Subject: [PATCH 069/107] fix(akoya): make repeated cell-table column names unique like marker names pyarrow keeps repeated header names, so selecting a repeated marker by label built an X wider than var (mcquant, SPRM, generic) or pushed it into obs (CODEX). --- .../readers/_utils/_cell_tables.py | 9 ++++---- tests/readers/test_cell_tables.py | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index d6352400..9f5c3cdf 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -365,8 +365,9 @@ def read_cell_table( else: _, sep = _read_header(path) df = pd.read_csv(path, sep=sep, engine="pyarrow") - # inForm writes some column names with trailing spaces - df.columns = pd.Index([str(column).strip() for column in df.columns]) + # inForm writes some column names with trailing spaces; pyarrow keeps repeated names, which are made unique + # like the marker names + df.columns = pd.Index(dedupe_channel_names([str(column).strip() for column in df.columns])) reader = { "codex": _codex_roles, "qupath": _qupath_roles, @@ -642,8 +643,8 @@ def _codex_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dic # the source of an FCS column is its `$PnN` fcs = path.suffix.lower() == CodexKeys.FCS_FILE.value sources = [name if fcs else str(column) for column, (name, _, _) in zip(df.columns, split, strict=True)] - names = [name for name, _, _ in split] - displays = [display.strip() or name for name, _, display in split] + names = dedupe_channel_names([name for name, _, _ in split]) + displays = [display.strip() or name for name, (_, _, display) in zip(names, split, strict=True)] df.columns = pd.Index(names) columns = {column.casefold(): column for column in names} x, y = kwargs.get("x", columns.get("x")), kwargs.get("y", columns.get("y")) diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index 33cdd84b..bae1fb1d 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -148,6 +148,29 @@ def test_read_codex_table_cells_table_kwargs(tmp_path: Path) -> None: assert "CD3" in table.obs.columns +@pytest.mark.parametrize( + ("fmt", "header", "markers"), + [ + ("mcquant", "CellID,DNA,CD3,DNA,X_centroid,Y_centroid,Area", ["DNA", "CD3", "DNA (2)"]), + ("sprm", "ID,DAPI,Blank,CD3,Blank", ["DAPI", "Blank", "CD3", "Blank (2)"]), + ("codex", "cell_id,x,y,size,DAPI,CD3,DAPI", ["DAPI", "CD3", "DAPI (2)"]), + ("generic", "cx,cy,DAPI,CD3,DAPI", ["DAPI", "CD3", "DAPI (2)"]), + ], +) +def test_repeated_column_names_are_deduplicated(tmp_path: Path, fmt: str, header: str, markers: list[str]) -> None: + path = tmp_path / "cells.csv" + n = header.count(",") + 1 + path.write_text(f"{header}\n{','.join(str(i + 1) for i in range(n))}\n", encoding="utf-8") + kwargs = {"x": "cx", "y": "cy"} if fmt == "generic" else {} + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) # SPRM without centroids + table = read_cell_table(path, fmt, pixel_size_um=None, default_region="S", cells_table_kwargs=kwargs) + + assert list(table.var.index) == markers + assert table.X.shape == (1, len(markers)) + + def test_build_cells_single_table_and_circles(tmp_path: Path) -> None: write_fcs(tmp_path / "reg001.fcs", _cells(n=3, region=1)) write_fcs(tmp_path / "reg002.fcs", _cells(n=4, region=2)) From c599f4aea3910319802a040cb98778a1339c34a8 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:47:23 +0200 Subject: [PATCH 070/107] fix(akoya): raise for missing or non-integer instance ids They were cast to int64, so an empty id became 0 (the background) and 1.5 became 1. --- .../readers/_utils/_cell_tables.py | 17 ++++++++++++++++- tests/readers/test_cell_tables.py | 9 +++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 9f5c3cdf..6a6bb8cd 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -999,7 +999,7 @@ def _from_frame( raise ValueError(f"{path} has no column(s) {missing}; found {list(df.columns)}.") if instance_column is not None: - instance_id = pd.Series(df[instance_column].to_numpy(dtype=np.int64)) + instance_id = pd.Series(_instance_ids(path, df[instance_column])) else: # label images use 0 for the background, so instance ids start at 1 instance_id = pd.Series(np.arange(1, len(df) + 1, dtype=np.int64)) @@ -1035,6 +1035,21 @@ def _from_frame( ) +def _instance_ids(path: Path, column: pd.Series) -> np.ndarray: + """The integer instance ids in ``column``; raises for missing or non-integer ids, which no label can have.""" + if pd.api.types.is_integer_dtype(column): + return column.to_numpy(dtype=np.int64) + ids = pd.to_numeric(column, errors="coerce").to_numpy(dtype=np.float64) + invalid = ~(np.isfinite(ids) & (ids == np.round(ids))) + if invalid.any(): + raise ValueError( + f"{path} has {invalid.sum()} cell(s) whose instance id in '{column.name}' is missing or not an integer " + f"(e.g. {column[invalid].tolist()[:5]}); every cell needs an integer instance id, the value of its " + "label. Choose another instance id column, or remove these rows." + ) + return ids.astype(np.int64) + + def _check_unique_instance_ids(path: Path, region: pd.Series, instance_id: pd.Series) -> None: """Raise if two cells of the same region have the same instance id: they could not be told apart.""" regions, ids = region.to_numpy(), instance_id.to_numpy() diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index bae1fb1d..c3da8257 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -171,6 +171,15 @@ def test_repeated_column_names_are_deduplicated(tmp_path: Path, fmt: str, header assert table.X.shape == (1, len(markers)) +@pytest.mark.parametrize("cell_id", ["", "1.5", "a"]) +def test_missing_or_fractional_instance_ids_raise(tmp_path: Path, cell_id: str) -> None: + path = tmp_path / "cells.csv" + path.write_text(f"CellID,CD3,X_centroid,Y_centroid\n1,1,1,1\n{cell_id},1,2,2\n", encoding="utf-8") + + with pytest.raises(ValueError, match=r"1 cell\(s\) whose instance id in 'CellID' is missing or not an integer"): + read_cell_table(path, "mcquant", pixel_size_um=None, default_region="S", cells_table_kwargs={}) + + def test_build_cells_single_table_and_circles(tmp_path: Path) -> None: write_fcs(tmp_path / "reg001.fcs", _cells(n=3, region=1)) write_fcs(tmp_path / "reg002.fcs", _cells(n=4, region=2)) From ef36f092a20b71020de3f1e962841a531ac00e08 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:48:19 +0200 Subject: [PATCH 071/107] fix(akoya): ignore cells with an empty region value with a warning Under pandas 3 the empty value stayed NaN and crashed region matching. --- src/spatialdata_io/readers/_utils/_cell_tables.py | 7 +++++++ tests/test_phenocycler.py | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 6a6bb8cd..b798230e 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -380,6 +380,13 @@ def read_cell_table( mask_pixels = fmt in {"mcquant", "sprm"} if fmt == "generic": mask_pixels = _coordinates_space(path, roles["in_um"], cells_table_kwargs) == "mask" + if roles["region_column"] in df.columns and (empty := df[roles["region_column"]].isna().to_numpy()).any(): + warnings.warn( + f"Ignoring the {empty.sum()} cell(s) of {path} without a region in '{roles['region_column']}'.", + UserWarning, + stacklevel=4, + ) + df = df.loc[~empty].reset_index(drop=True) region = _region_values(df, roles["region_column"], default_region, regions) units = cells_table_kwargs.get("geojson_units", "px") row_pixel_sizes = None diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 617806de..60599dad 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -1029,3 +1029,14 @@ def test_selected_region_of_qupath_table_uses_each_rows_pixel_size(tmp_path: Pat assert list(sdata.labels) == [f"{regions[0]}_labels"] np.testing.assert_allclose(np.asarray(sdata.tables["table"].obsm["spatial"])[:, 0], [20, 45, 70, 95]) + + +def test_cells_without_region_are_ignored(tmp_path: Path) -> None: + for region in ("A", "B"): + write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI"]) + _write_qupath_px(tmp_path / "cells.csv", {"A_Scan1.qptiff": 1, "": 1, "B_Scan1.qptiff": 1}) + + with pytest.warns(UserWarning, match=r"Ignoring the 1 cell\(s\) of .*cells\.csv without a region in 'Image'"): + sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) + + assert list(sdata.tables["table"].obs["region"]) == ["A_cells", "B_cells"] From 254c5c04421ab16c5c6665d4836e33618ac68baf Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:49:17 +0200 Subject: [PATCH 072/107] fix(akoya): keep per-region tables apart when their marker metadata differs Tables with the same marker names but e.g. other QuPath compartments were concatenated under the first table's var. --- .../readers/_utils/_cell_tables.py | 5 +++-- tests/test_phenocycler.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index b798230e..9d0a4f2a 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -1549,7 +1549,7 @@ def build_cells( Returns ------- - The table elements, ``{"table": ...}`` if all tables have the same markers, else ``{"_table": ...}`` + The table elements, ``{"table": ...}`` if all tables have the same ``var``, else ``{"_table": ...}`` per region, and the circle shapes by element name. The circles have the cells' area (radius 1 px without one). """ _check_disjoint_regions(tables) @@ -1568,7 +1568,8 @@ def build_cells( UserWarning, stacklevel=3, ) - pieces = _split_by_region(tables) if len({tuple(table.var.index) for table in tables}) > 1 else None + # tables with other markers or other marker metadata (e.g. QuPath compartments) are not concatenated + pieces = _split_by_region(tables) if any(not table.var.equals(tables[0].var) for table in tables) else None def element(region: str) -> str: if region in region_elements: diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 60599dad..8e692585 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -1040,3 +1040,20 @@ def test_cells_without_region_are_ignored(tmp_path: Path) -> None: sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) assert list(sdata.tables["table"].obs["region"]) == ["A_cells", "B_cells"] + + +def test_tables_with_different_marker_metadata_are_not_merged(tmp_path: Path) -> None: + for region in ("A", "B"): + write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI"]) + header = "Object ID\tCentroid X px\tCentroid Y px\tCD3: Nucleus: Mean" + (tmp_path / "a.tsv").write_text(f"{header}\tCD3: Cell: Mean\na1\t10\t10\t1\t100\n", encoding="utf-8") + (tmp_path / "b.tsv").write_text(f"{header}\nb1\t10\t10\t2\n", encoding="utf-8") + + with pytest.warns(UserWarning, match="circles"): + sdata = phenocycler( + tmp_path, cells_table={"A": tmp_path / "a.tsv", "B": tmp_path / "b.tsv"}, image_models_kwargs=_FAST + ) + + assert sorted(sdata.tables) == ["A_table", "B_table"] + assert list(sdata.tables["A_table"].var["compartment"]) == ["Cell"] + assert list(sdata.tables["B_table"].var["compartment"]) == ["Nucleus"] From d0bda0370b73a75a664afe5fa990a42d6bb5fdf3 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:51:02 +0200 Subject: [PATCH 073/107] fix(akoya): place mask-pixel centroids at the centre of their image pixels A centroid c of a mask downsampled by f moves to f * c + (f - 1) / 2, not f * c, which put it ~f/2 px off its label. --- CHANGELOG.md | 4 ++-- src/spatialdata_io/readers/_utils/_cell_tables.py | 12 +++++++++--- src/spatialdata_io/readers/codex.py | 3 ++- src/spatialdata_io/readers/phenocycler.py | 3 ++- tests/readers/test_cell_tables.py | 9 ++++++--- tests/test_phenocycler.py | 3 ++- 6 files changed, 23 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70fc5091..3b2967fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,8 +34,8 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. of the `.er` region (a warning is raised if their shapes differ). By default only the extended-range file is read. - The Akoya readers (`phenocycler()`, `codex()`) scale a segmentation mask downsampled from its region image by an integer factor (exactly, or rounded up) onto the image (a `Scale` before the region's transformation), and scale - the centroids (`obsm["spatial"]`) and circle radii of `mcquant` and SPRM tables, which are in mask pixels, by the - same factor; the source columns in `obs` are kept as read. Generic pixel tables are in image pixels unless + the centroids (`obsm["spatial"]`, to the centre of their mask pixel) and circle radii of `mcquant` and SPRM + tables, which are in mask pixels, by the same factor; the source columns in `obs` are kept as read. Generic pixel tables are in image pixels unless `cells_table_kwargs["coordinates_space"]="mask"`. Other shape mismatches warn. - `phenocycler()` officially supports the PerkinElmer/Akoya QPTIFFs of Vectra Polaris and PhenoImager HT (generation `"qpi_other"`): Opal and unmixed (uint16) scans, brightfield RGB scans (channels `r`, `g`, `b`) and inForm component diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 9d0a4f2a..f954b651 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -1400,14 +1400,19 @@ def mask_downsample_factor(mask_shape: tuple[int, int], image_shape: tuple[int, def _to_image_pixels(tables: Sequence[CellTable], factors: Mapping[str, int]) -> list[CellTable]: - """``tables`` with mask-pixel centroids and radii scaled to image pixels by their region's mask factor.""" + """``tables`` with mask-pixel centroids and radii scaled to image pixels by their region's mask factor. + + A mask pixel covers ``f x f`` image pixels, so a centroid ``c`` moves to the centre of those, ``f * c + (f - 1) / 2``. + """ scaled = [] for table in tables: if table.mask_pixels and factors: row_factors = table.region.map(factors).fillna(1).to_numpy(dtype=np.float64) table = replace( table, - centroids_px=None if table.centroids_px is None else table.centroids_px * row_factors[:, None], + centroids_px=None + if table.centroids_px is None + else table.centroids_px * row_factors[:, None] + (row_factors[:, None] - 1) / 2, radius_px=None if table.radius_px is None else table.radius_px * row_factors, ) scaled.append(table) @@ -1629,7 +1634,8 @@ def build_spatialdata( the coordinate systems of :func:`region_transformations`; regions with cells or a mask but no image are unrelated to the scans. A mask downsampled from its region image by an integer factor is scaled onto the image (see :func:`read_region_labels`), and the centroids and radii in mask pixels of its cells (``mask_pixels``, see - :class:`CellTable`) are scaled by the same factor, so that the table and circles are in image pixels. A + :class:`CellTable`) are scaled by the same factor (see :func:`_to_image_pixels`), so that the table and circles + are in image pixels. A `UserWarning` is raised for a region if more than 5% of its cells lie outside its image: their coordinates are in another unit or relative to another origin (e.g. inForm positions in microns relative to the slide). diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index d4e9ac74..277d5f20 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -190,7 +190,8 @@ def codex( divided by an integer factor ``f`` (exactly, or rounded up) is scaled onto the image (``Scale([f, f])`` before the region's transformation); any other shape mismatch raises a warning. The pixel centroids of ``"mcquant"`` and ``"sprm"`` tables are taken to be in mask pixels, so their centroids (``obsm["spatial"]``) and circle radii - are scaled by ``f`` to image pixels; the source columns in ``obs`` are kept as read (mask pixels). The pixel + are scaled by ``f`` to image pixels (a centroid ``c`` to ``f * c + (f - 1) / 2``, the centre of its mask + pixel); the source columns in ``obs`` are kept as read (mask pixels). The pixel centroids of other tables are in image pixels and are not scaled; ``cells_table_kwargs["coordinates_space"]="mask"`` scales those of a ``"generic"`` table too. image_models_kwargs diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 98e7e941..37e21e8a 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -142,7 +142,8 @@ def phenocycler( (exactly, or rounded up) is scaled onto the image (``Scale([f, f])`` before the region's transformation); any other shape mismatch raises a warning. The pixel centroids of ``"mcquant"`` and ``"sprm"`` tables are taken to be in mask pixels, so their centroids (``obsm["spatial"]``) and circle radii are scaled by - ``f`` to image pixels; the source columns in ``obs`` are kept as read (mask pixels). The pixel centroids of + ``f`` to image pixels (a centroid ``c`` to ``f * c + (f - 1) / 2``, the centre of its mask pixel); the + source columns in ``obs`` are kept as read (mask pixels). The pixel centroids of other tables are in image pixels and are not scaled; ``cells_table_kwargs["coordinates_space"]="mask"`` scales those of a ``"generic"`` table too. read_all_bit_depths diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index c3da8257..00807f34 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -1036,9 +1036,11 @@ def test_mask_integer_downsample_autoscaled(tmp_path: Path, mask_shape: tuple[in assert labels_extent[axis][1] == pytest.approx(expected_end) if image_size % mask_size == 0: assert labels_extent[axis][1] == pytest.approx(image_extent[axis][1]) - # mcquant centroids of the mask are in mask pixels, the table's are in image pixels; obs keeps them as read + # mcquant centroids of the mask are in mask pixels, the table's are in image pixels, at the centre of the + # factor x factor image pixels of their mask pixel; obs keeps them as read table = sdata.tables["table"] - np.testing.assert_allclose(table.obsm["spatial"], cells[["X_centroid", "Y_centroid"]].to_numpy() * factor) + expected = cells[["X_centroid", "Y_centroid"]].to_numpy() * factor + (factor - 1) / 2 + np.testing.assert_allclose(table.obsm["spatial"], expected) np.testing.assert_allclose(table.obs["X_centroid"], cells["X_centroid"]) # writing may reorder the axes of the translation, so compare the composed affine maps @@ -1090,7 +1092,8 @@ def test_generic_px_table_coordinates_space(tmp_path: Path, space: str | None, f cells_table_kwargs=kwargs, ) - np.testing.assert_allclose(sdata.tables["table"].obsm["spatial"], np.array([[6, 6], [12, 12]]) * factor) + expected = np.array([[6, 6], [12, 12]]) * factor + (factor - 1) / 2 + np.testing.assert_allclose(sdata.tables["table"].obsm["spatial"], expected) @pytest.mark.parametrize( diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 8e692585..bd5fb990 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -232,7 +232,8 @@ def test_read_all_bit_depths_with_labels_and_cells_table(tmp_path: Path) -> None assert labels_transformation.transformations == [Scale([2, 2], axes=("y", "x")), image_transformations["global"]] table = sdata.tables["table"] assert table.uns["spatialdata_attrs"]["region"] == ["S_labels"] - np.testing.assert_allclose(table.obsm["spatial"], [[6, 6], [12, 12]]) + # the centre of the 2 x 2 image pixels of each mask pixel + np.testing.assert_allclose(table.obsm["spatial"], [[6.5, 6.5], [12.5, 12.5]]) with pytest.raises(ValueError, match=r"Unknown region\(s\) \['S_8bit'\]"): phenocycler(tmp_path, labels={"S_8bit": tmp_path / "mask.tif"}, read_all_bit_depths=True) From bfff78c9dbc8f3622632beb929007ece79c64829 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:52:32 +0200 Subject: [PATCH 074/107] fix(akoya): give cells without a positive area unit circles An area of 0 made ShapesModel reject the whole read, a missing one gave a NaN radius. --- src/spatialdata_io/readers/_utils/_cell_tables.py | 10 ++++++++-- tests/test_phenocycler.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index f954b651..85f8c330 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -1555,7 +1555,8 @@ def build_cells( Returns ------- The table elements, ``{"table": ...}`` if all tables have the same ``var``, else ``{"_table": ...}`` - per region, and the circle shapes by element name. The circles have the cells' area (radius 1 px without one). + per region, and the circle shapes by element name. The circles have the cells' area (radius 1 px without a + positive one). """ _check_disjoint_regions(tables) row_ids = sorted( @@ -1594,7 +1595,12 @@ def element(region: str) -> str: "placed; pass a mask as `labels` or a table with centroids." ) circle_regions.append(region) - radius = 1.0 if table.radius_px is None else table.radius_px[rows] + radius = np.ones(len(rows)) if table.radius_px is None else table.radius_px[rows] + if (no_area := ~(radius > 0)).any(): + logger.info( + "%d cells of region '%s' have no positive area; their circles get radius 1.", no_area.sum(), region + ) + radius = np.where(no_area, 1.0, radius) shapes[element(region)] = ShapesModel.parse( table.centroids_px[rows], geometry=0, diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index bd5fb990..1c1a572e 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -1058,3 +1058,13 @@ def test_tables_with_different_marker_metadata_are_not_merged(tmp_path: Path) -> assert sorted(sdata.tables) == ["A_table", "B_table"] assert list(sdata.tables["A_table"].var["compartment"]) == ["Cell"] assert list(sdata.tables["B_table"].var["compartment"]) == ["Nucleus"] + + +def test_cells_without_a_positive_area_get_unit_circles(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + (tmp_path / "cells.csv").write_text("CellID,CD3,X_centroid,Y_centroid,Area\n1,1,1,1,0\n2,1,2,2,\n3,1,3,3,3.14159\n") + + with pytest.warns(UserWarning, match="circles"): + sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) + + np.testing.assert_allclose(sdata.shapes["S_cells"]["radius"], [1, 1, 1], rtol=1e-5) From 1ffdffc550d944b57c70993f007bb6e364623d0b Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:53:23 +0200 Subject: [PATCH 075/107] fix(akoya): do not detect tables with x/y centroids as SPRM A generic 'ID,x,y,...' table was read as SPRM means with its centroids as markers. --- src/spatialdata_io/readers/_utils/_cell_tables.py | 6 ++++-- tests/readers/test_cell_tables.py | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 85f8c330..d749641c 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -216,7 +216,8 @@ def detect_table_format(path: Path) -> str: ``"codex"`` (``x``/``y`` with ``cell_id`` or ``cycCCC_chHHH`` columns, also as the channels of an FCS file), ``"qupath"`` (``Centroid X µm``, ``Centroid X px`` or ``Object ID``, also in R's ``make.names`` form, e.g. ``Centroid.X.um``), ``"mcquant"`` (``CellID`` and ``X_centroid``), ``"inform"`` (``Cell X Position``, - optionally followed by its unit in parentheses), ``"sprm"`` (first column ``ID``), else ``"generic"``. + optionally followed by its unit in parentheses), ``"sprm"`` (first column ``ID``, no ``x`` or ``y``), else + ``"generic"``. Raises ------ @@ -248,7 +249,8 @@ def _detect_header_format(columns: list[str]) -> str: return "mcquant" if any(_INFORM_POSITION.fullmatch(column) for column in columns): return "inform" - if columns[:1] == ["ID"]: + # SPRM means have no centroids, which are in `*-cell_centers.csv` + if columns[:1] == ["ID"] and not {"x", "y"} & names: return "sprm" return "generic" diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index 00807f34..65f542b7 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -280,6 +280,8 @@ def _write_mcquant(path: Path, n: int = 5, extent: tuple[float, float] = (60, 40 (_QUPATH_HEADER + "\n", "qupath"), ("CellID,DAPI,CD3,X_centroid,Y_centroid,Area\n", "mcquant"), ("ID,DAPI,CD3\n", "sprm"), + # SPRM means have no centroids + ("ID,x,y,CD3\n", "generic"), ("Path\tSample Name\tCell ID\tCell X Position\tCell Y Position\n", "inform"), ("label,cx,cy,CD3\n", "generic"), ], From 9ece64726de4433014bf3a7e47003ce65db1b5ca Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:54:29 +0200 Subject: [PATCH 076/107] fix(akoya): check cells_table_format and cells_table_kwargs keys without a table A typo such as mask_chanel was silently ignored unless cells_table was given. --- .../readers/_utils/_cell_tables.py | 16 +++++++++++----- src/spatialdata_io/readers/codex.py | 2 ++ src/spatialdata_io/readers/phenocycler.py | 8 +++++++- tests/test_phenocycler.py | 10 ++++++++++ 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index d749641c..af84d2e5 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -52,6 +52,7 @@ "CellTable", "build_cells", "build_spatialdata", + "check_cells_table_options", "detect_table_format", "mask_downsample_factor", "polygon_labels", @@ -346,9 +347,7 @@ def read_cell_table( ------- The cell table. Its regions are the (matched) values of the region column, or ``default_region``. """ - unknown = sorted(set(cells_table_kwargs) - set(CELLS_TABLE_KWARGS)) - if unknown: - raise ValueError(f"`cells_table_kwargs` has unknown key(s) {unknown}; expected some of {CELLS_TABLE_KWARGS}.") + check_cells_table_options(fmt, cells_table_kwargs) if fmt == "auto": # centroid columns given by the user make a generic table fmt = "generic" if {"x", "y"} <= cells_table_kwargs.keys() else detect_table_format(path) @@ -359,8 +358,6 @@ def read_cell_table( raise ValueError( f"`cells_table_kwargs['coordinates_space']` is only used with generic cell tables; {path} is {fmt!r}." ) - if fmt not in TABLE_FORMATS: - raise ValueError(f"Unknown cell table format {fmt!r}; expected 'auto' or one of {TABLE_FORMATS}.") if path.suffix.lower() == CodexKeys.FCS_FILE.value: df = _read_fcs(path) @@ -433,6 +430,15 @@ def _region_values( return values.map({value: value if region is None else region for value, region in matched.items()}) +def check_cells_table_options(fmt: str, cells_table_kwargs: Mapping[str, Any]) -> None: + """Raise for an unknown cell table format or key of ``cells_table_kwargs``, also when no table is read.""" + if fmt != "auto" and fmt not in TABLE_FORMATS: + raise ValueError(f"Unknown cell table format {fmt!r}; expected 'auto' or one of {TABLE_FORMATS}.") + unknown = sorted(set(cells_table_kwargs) - set(CELLS_TABLE_KWARGS)) + if unknown: + raise ValueError(f"`cells_table_kwargs` has unknown key(s) {unknown}; expected some of {CELLS_TABLE_KWARGS}.") + + def _row_pixel_sizes( path: Path, region: pd.Series | None, pixel_size_um: float | Mapping[str, float | None] | None ) -> float | np.ndarray | None: diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 277d5f20..47b48ad8 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -26,6 +26,7 @@ from spatialdata_io.readers._utils._cell_tables import ( CellTable, build_spatialdata, + check_cells_table_options, detect_table_format, read_cell_table, read_region_tables, @@ -221,6 +222,7 @@ def codex( warnings.warn( "`imread_kwargs` is ignored: every image is read lazily with tifffile.", UserWarning, stacklevel=2 ) + check_cells_table_options(cells_table_format, cells_table_kwargs) regions = normalize_regions(regions) channel_names = [channel_names] if isinstance(channel_names, str) else channel_names path = Path(path) diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 37e21e8a..cc6b9e51 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -18,7 +18,12 @@ read_json_object, select_regions, ) -from spatialdata_io.readers._utils._cell_tables import build_spatialdata, read_region_tables, resolve_per_region +from spatialdata_io.readers._utils._cell_tables import ( + build_spatialdata, + check_cells_table_options, + read_region_tables, + resolve_per_region, +) from spatialdata_io.readers._utils._qptiff import ( _fusion_profile_records, _is_blank, @@ -175,6 +180,7 @@ def phenocycler( ------- :class:`spatialdata.SpatialData` """ + check_cells_table_options(cells_table_format, cells_table_kwargs) regions = normalize_regions(regions) channel_names = [channel_names] if isinstance(channel_names, str) else channel_names path = Path(path) diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 1c1a572e..b8442544 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -1068,3 +1068,13 @@ def test_cells_without_a_positive_area_get_unit_circles(tmp_path: Path) -> None: sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) np.testing.assert_allclose(sdata.shapes["S_cells"]["radius"], [1, 1, 1], rtol=1e-5) + + +@pytest.mark.parametrize("reader", [codex, phenocycler]) +def test_cells_table_options_are_checked_without_a_cells_table(tmp_path: Path, reader) -> None: # type: ignore[no-untyped-def] + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + + with pytest.raises(ValueError, match=r"unknown key\(s\) \['mask_chanel'\]"): + reader(tmp_path, cells_table_kwargs={"mask_chanel": 1}) + with pytest.raises(ValueError, match="Unknown cell table format 'excel'"): + reader(tmp_path, cells_table_format="excel") From e7493bc21e1246e6e7c4a8e5a7f79e14bb6d9396 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:58:02 +0200 Subject: [PATCH 077/107] fix(akoya): give regions without cell polygons circles instead of empty labels --- .../readers/_utils/_cell_tables.py | 7 +++++-- tests/readers/test_rasterize.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index af84d2e5..1ae0d202 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -1438,7 +1438,7 @@ def polygon_labels( Where cells overlap, the one with the higher instance id wins. Regions with a mask in ``labels`` keep it, with a warning; regions without an image are skipped, their cells become circles. In both cases the polygons are - still kept as shapes, see :func:`polygon_shapes`. + still kept as shapes, see :func:`polygon_shapes`. Regions none of whose cells has a polygon get no labels. Parameters ---------- @@ -1465,6 +1465,10 @@ def polygon_labels( if table.polygons_px is None: continue for region, rows in _region_rows(table).items(): + rows = rows[pd.notna(table.polygons_px[rows])] + # regions without polygons get circles, like tables without a GeoJSON + if len(rows) == 0: + continue if region in labels: row_ids = " Its cells' instance ids are row numbers and may not match the mask's values." warnings.warn( @@ -1478,7 +1482,6 @@ def polygon_labels( if region not in image_shapes: logger.info("Region '%s' has no image, so its cell polygons are not rasterised.", region) continue - rows = rows[pd.notna(table.polygons_px[rows])] data = rasterize_polygons_lazy( table.polygons_px[rows], table.instance_id.to_numpy()[rows], image_shapes[region], chunks ) diff --git a/tests/readers/test_rasterize.py b/tests/readers/test_rasterize.py index 5ad442ec..75dd3599 100644 --- a/tests/readers/test_rasterize.py +++ b/tests/readers/test_rasterize.py @@ -347,3 +347,21 @@ def test_qupath_geojson_partial_and_duplicate_ids(tmp_path: Path) -> None: _write_geojson(tmp_path / "cells.geojson", [_cell("a", 0, 0), _cell("b", 20, 20), _cell("a", 40, 40)]) with pytest.raises(ValueError, match="same id"): phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) + + +def test_qupath_geojson_region_without_polygons_gets_circles(tmp_path: Path) -> None: + for region in "AB": + write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI"]) + header = "Image\tObject ID\tCentroid X px\tCentroid Y px\tCD3: Cell: Mean" + rows = ["A_Scan1.qptiff\ta\t4\t4\t1", "B_Scan1.qptiff\tb\t24\t24\t2"] + (tmp_path / "cells.tsv").write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") + # QuPath exports the GeoJSON of one image: B's cells have no polygon + _write_geojson(tmp_path / "cells.geojson", [_cell("a", 0, 0)]) + + with pytest.warns(UserWarning, match="have no polygon"): + sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) + + assert list(sdata.labels) == ["A_labels"] + assert set(sdata.shapes) == {"A_polygons", "B_cells"} + assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["A_labels", "B_cells"] + assert sdata.attrs["akoya"]["geojson"] == {"A": str(tmp_path / "cells.geojson")} From 5c8cd7dccbb52cbb632422ec43d4b54ee8224720 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 03:58:51 +0200 Subject: [PATCH 078/107] fix(akoya): record the GeoJSON of regions whose polygons are not rasterised --- src/spatialdata_io/readers/_utils/_cell_tables.py | 4 ++-- tests/readers/test_rasterize.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 1ae0d202..de197421 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -1457,7 +1457,7 @@ def polygon_labels( Returns ------- - The labels elements by region id, and the GeoJSON file of each. + The labels elements by region id, and the GeoJSON file of each region with polygons. """ labels_models_kwargs, chunks = _labels_models_kwargs(labels_models_kwargs) elements, sources = {}, {} @@ -1469,6 +1469,7 @@ def polygon_labels( # regions without polygons get circles, like tables without a GeoJSON if len(rows) == 0: continue + sources[region] = str(table.geojson) if region in labels: row_ids = " Its cells' instance ids are row numbers and may not match the mask's values." warnings.warn( @@ -1491,7 +1492,6 @@ def polygon_labels( transformations=_transformations_of(transformations, region), **labels_models_kwargs, ) - sources[region] = str(table.geojson) return elements, sources diff --git a/tests/readers/test_rasterize.py b/tests/readers/test_rasterize.py index 75dd3599..63b4d6bc 100644 --- a/tests/readers/test_rasterize.py +++ b/tests/readers/test_rasterize.py @@ -263,6 +263,7 @@ def test_qupath_geojson_mask_wins(tmp_path: Path) -> None: np.testing.assert_array_equal(sdata.labels["S_labels"]["scale0"]["image"].values, mask) assert "S_polygons" in sdata.shapes + assert sdata.attrs["akoya"]["geojson"] == {"S": str(tmp_path / "cells_cells.geojson")} def test_qupath_geojson_errors(tmp_path: Path) -> None: From 0c223991b1e63a2834a402fdb9d8eee9feb066f4 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 04:01:13 +0200 Subject: [PATCH 079/107] fix(akoya): ignore a discovered GeoJSON without the table's cells and accept geojson=False --- CHANGELOG.md | 4 +- .../readers/_utils/_cell_tables.py | 48 ++++++++++++++----- src/spatialdata_io/readers/codex.py | 3 +- src/spatialdata_io/readers/phenocycler.py | 3 +- tests/readers/test_rasterize.py | 40 ++++++++++++++++ 5 files changed, 84 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b2967fe..92c4d73c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,7 +51,9 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. page names of an inForm `*_binary_seg_maps.tif` (`"Tissue"`, `"Nucleus"`, `"Cytoplasm"`, `"Membrane"`); such a file reads its `Nucleus` page (whose labels are the `Cell ID`s) by default. With `x` and `y` in `cells_table_kwargs`, `cells_table_format="auto"` reads a generic table. QuPath GeoJSON cells without a table row - are skipped with a warning, and a region whose cells lie mostly outside its image raises a warning. + are skipped with a warning; a GeoJSON found next to a QuPath table that holds none of its cells (e.g. an + annotation export) is ignored with a warning, and `cells_table_kwargs["geojson"]=False` reads no outlines. A region + whose cells lie mostly outside its image raises a warning. ### Changed diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index de197421..72df2227 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -298,8 +298,9 @@ def read_cell_table( ``Max diameter µm`` or ``Circularity``). The radius comes from ``Cell: Area µm^2`` (or ``px^2``) in the unit of the centroids. A table split into centroids and measurements (two files) must be joined on the object id first. The cell outlines are read from a QuPath GeoJSON export, see - :func:`read_qupath_polygons`: ``cells_table_kwargs["geojson"]``, else ``
.geojson`` or - ``
_cells.geojson`` next to the table, if it exists. + :func:`read_qupath_polygons`: ``cells_table_kwargs["geojson"]`` (``False`` for none), else + ``
.geojson`` or ``
_cells.geojson`` next to the table, if it exists and holds + any of the table's cells. - ``"mcquant"``: an MCMICRO ``mcquant`` table. Instance ids ``CellID``, pixel centroids ``X_centroid``/``Y_centroid``, the markers are the columns between ``CellID`` and ``X_centroid``, the radius comes from ``Area``. @@ -352,7 +353,7 @@ def read_cell_table( # centroid columns given by the user make a generic table fmt = "generic" if {"x", "y"} <= cells_table_kwargs.keys() else detect_table_format(path) logger.info("Reading %s as a %r cell table.", path, fmt) - if fmt != "qupath" and cells_table_kwargs.get("geojson") is not None: + if fmt != "qupath" and cells_table_kwargs.get("geojson") not in (None, False): raise ValueError(f"`cells_table_kwargs['geojson']` is only read with QuPath cell tables; {path} is {fmt!r}.") if fmt != "generic" and cells_table_kwargs.get("coordinates_space") is not None: raise ValueError( @@ -403,11 +404,13 @@ def read_cell_table( if fmt == "sprm": table.centroids_px = _read_sprm_centers(path, table) if fmt == "qupath": - geojson = cells_table_kwargs.get("geojson") - geojson = _find_geojson(path) if geojson is None else Path(geojson) + option = cells_table_kwargs.get("geojson") + geojson = _find_geojson(path) if option is None else None if option is False else Path(option) if geojson is not None: - table.polygons_px = read_qupath_polygons(geojson, table, path, units=units, pixel_size_um=row_pixel_sizes) - table.geojson = geojson + table.polygons_px = read_qupath_polygons( + geojson, table, path, units=units, pixel_size_um=row_pixel_sizes, discovered=option is None + ) + table.geojson = None if table.polygons_px is None else geojson return table @@ -496,7 +499,8 @@ def read_qupath_polygons( *, units: str = "px", pixel_size_um: float | np.ndarray | None = None, -) -> np.ndarray: + discovered: bool = False, +) -> np.ndarray | None: """Read the cell outlines of a QuPath GeoJSON export and match them to the cells of its measurement table. A feature belongs to the cell whose ``Object ID`` (or R's ``Object.ID``) is the feature ``id`` (the object's @@ -520,10 +524,14 @@ def read_qupath_polygons( The unit of the coordinates: ``"px"`` (of the full-resolution image, what QuPath writes) or ``"um"``. pixel_size_um The pixel size, to convert ``"um"`` coordinates, or the pixel size of every row of ``table``. + discovered + Whether ``geojson`` was found next to the table rather than given: if it holds none of the table's cells + (e.g. QuPath's export of an annotation), it is ignored with a `UserWarning` instead of raising. Returns ------- - The polygon of every row of ``table`` in pixels, `None` for rows without one. + The polygon of every row of ``table`` in pixels, `None` for rows without one; `None` for an ignored + ``geojson``. """ if units not in ("px", "um"): raise ValueError(f"`cells_table_kwargs['geojson_units']` must be 'px' or 'um', got {units!r}.") @@ -541,6 +549,9 @@ def read_qupath_polygons( raise ValueError(f"The '{id_column}' column of {table_path} is not unique.") rows = object_ids.get_indexer(pd.Index(ids).astype(str)) found = rows >= 0 + if not found.any() and discovered: + _ignore_geojson(geojson, table_path) + return None if not found.any(): raise ValueError( f"{geojson} does not hold the cells of {table_path}: none of the feature ids is an '{id_column}' " @@ -567,6 +578,9 @@ def read_qupath_polygons( ) else: cells = geometries if kinds is None else geometries[np.isin(kinds, list(_QUPATH_CELL_TYPES))] + if len(cells) == 0 and discovered: + _ignore_geojson(geojson, table_path) + return None if len(cells) != len(polygons): raise ValueError( f"{geojson} has {len(cells)} cell polygons without ids, but {table_path} has {len(polygons)} rows; " @@ -580,6 +594,15 @@ def read_qupath_polygons( return polygons +def _ignore_geojson(geojson: Path, table_path: Path) -> None: + warnings.warn( + f"Ignoring {geojson}: it holds none of the cells of {table_path}. Pass the cell outlines as " + "`cells_table_kwargs['geojson']`.", + UserWarning, + stacklevel=6, + ) + + def _scale_polygons(polygons: np.ndarray, factors: np.ndarray) -> None: """Scale every polygon by its factor in place; polygons with a NaN factor (cells that are not read) are dropped.""" polygons[np.isnan(factors)] = None @@ -1174,7 +1197,7 @@ def read_region_tables( The cell tables, one per file. """ geojson = cells_table_kwargs.get("geojson") - geojson_files = resolve_per_region(geojson, selected, list(available)) if geojson is not None else {} + geojson_files = {} if geojson is False else resolve_per_region(geojson, selected, list(available)) if unknown := sorted(geojson_files.keys() - files.keys()): raise ValueError( f"`cells_table_kwargs['geojson']` is given for {unknown}, but the cell tables are given for " @@ -1189,7 +1212,10 @@ def read_region_tables( fmt, pixel_size_um=sizes, default_region=None if key == "*" else key, - cells_table_kwargs={**cells_table_kwargs, "geojson": geojson_files.get(key)}, + cells_table_kwargs={ + **cells_table_kwargs, + "geojson": False if geojson is False else geojson_files.get(key), + }, regions=available, ) tables.append(_assign_regions(table, path, key, available, selected)) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 47b48ad8..84e4f90c 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -211,7 +211,8 @@ def codex( ``"Tissue"``, ``"Nucleus"`` (the default there, its labels are the ``Cell ID`` of every cell), ``"Cytoplasm"`` or ``"Membrane"``); for QuPath tables ``geojson`` (the GeoJSON export of the cell outlines, a path or a mapping from region id to path like ``cells_table``; default ``
.geojson`` - or ``
_cells.geojson`` next to the table) and ``geojson_units`` (``"px"`` of the + or ``
_cells.geojson`` next to the table, ignored with a warning if it holds none of the + table's cells; ``False`` reads none) and ``geojson_units`` (``"px"`` of the full-resolution image, default and what QuPath writes, or ``"um"``). Returns diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index cc6b9e51..fc7ea032 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -173,7 +173,8 @@ def phenocycler( ``"Tissue"``, ``"Nucleus"`` (the default there, its labels are the ``Cell ID`` of every cell), ``"Cytoplasm"`` or ``"Membrane"``); for QuPath tables ``geojson`` (the GeoJSON export of the cell outlines, a path or a mapping from region id to path like ``cells_table``; default ``
.geojson`` - or ``
_cells.geojson`` next to the table) and ``geojson_units`` (``"px"`` of the + or ``
_cells.geojson`` next to the table, ignored with a warning if it holds none of the + table's cells; ``False`` reads none) and ``geojson_units`` (``"px"`` of the full-resolution image, default and what QuPath writes, or ``"um"``). Returns diff --git a/tests/readers/test_rasterize.py b/tests/readers/test_rasterize.py index 63b4d6bc..e2a24643 100644 --- a/tests/readers/test_rasterize.py +++ b/tests/readers/test_rasterize.py @@ -366,3 +366,43 @@ def test_qupath_geojson_region_without_polygons_gets_circles(tmp_path: Path) -> assert set(sdata.shapes) == {"A_polygons", "B_cells"} assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["A_labels", "B_cells"] assert sdata.attrs["akoya"]["geojson"] == {"A": str(tmp_path / "cells.geojson")} + + +@pytest.mark.parametrize("object_ids", [True, False]) +def test_qupath_discovered_annotation_geojson_is_ignored(tmp_path: Path, object_ids: bool) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + if object_ids: + _write_qupath_cells(tmp_path / "cells.tsv", ["a", "b"]) + else: + _write_order_table(tmp_path / "cells.tsv") + # QuPath's "Export objects as GeoJSON" of the tissue annotation, named after the image like the table + _write_geojson(tmp_path / "cells.geojson", [_cell("tissue", 0, 0, "annotation")]) + + with pytest.warns(UserWarning, match="Ignoring .*cells.geojson.* none of the cells"): + sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) + + assert list(sdata.shapes) == ["S_cells"] + assert not sdata.labels + assert "geojson" not in sdata.attrs["akoya"] + # an explicit GeoJSON must hold the cells + with pytest.raises(ValueError, match="cell"): + phenocycler( + tmp_path, + cells_table=tmp_path / "cells.tsv", + image_models_kwargs=_FAST, + cells_table_kwargs={"geojson": tmp_path / "cells.geojson"}, + ) + + +def test_qupath_geojson_false_reads_no_outlines(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) + _write_qupath_cells(tmp_path / "cells.tsv", ["a", "b"]) + _write_geojson(tmp_path / "cells.geojson", [_cell("a", 0, 0), _cell("b", 20, 20)]) + + sdata = phenocycler( + tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST, cells_table_kwargs={"geojson": False} + ) + + assert list(sdata.shapes) == ["S_cells"] + assert not sdata.labels + assert "geojson" not in sdata.attrs["akoya"] From 6e5614951ba0afd68ca7b41ff66726ba097fc54d Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 04:10:55 +0200 Subject: [PATCH 080/107] perf(akoya): rasterise polygons with one GEOS call per batch and ship each chunk only its polygons --- .../readers/_utils/_rasterize.py | 117 +++++++++++------- tests/readers/test_rasterize.py | 27 +++- 2 files changed, 93 insertions(+), 51 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_rasterize.py b/src/spatialdata_io/readers/_utils/_rasterize.py index e40cdefd..a32e8734 100644 --- a/src/spatialdata_io/readers/_utils/_rasterize.py +++ b/src/spatialdata_io/readers/_utils/_rasterize.py @@ -2,13 +2,15 @@ from __future__ import annotations +import hashlib +from itertools import product from typing import TYPE_CHECKING, Any import dask.array as da import numpy as np import shapely +from dask.array.core import normalize_chunks from dask.base import tokenize -from skimage.draw import polygon as draw_polygon if TYPE_CHECKING: from collections.abc import Sequence @@ -16,6 +18,9 @@ __all__ = ["rasterize_polygons_lazy"] _MAX_ID = np.iinfo(np.uint32).max +# the polygon pixels tested at once, bounding the memory of a block's temporaries (~25 MB) +_BATCH_PIXELS = 1 << 18 +_TOKEN_BATCH = 1 << 16 def rasterize_polygons_lazy( @@ -27,10 +32,11 @@ def rasterize_polygons_lazy( ) -> da.Array: """Rasterise polygons into a lazy ``uint32`` labels array, one chunk at a time. - A pixel ``(row, col)`` gets the id of a polygon that contains its centre ``(col + 0.5, row + 0.5)``: pixel - ``(0, 0)`` covers ``[0, 1) x [0, 1)``, as in QuPath. Holes are not part of a polygon. Where polygons overlap, - the one with the higher id wins, independently of their order. Only a spatial index of the polygons is built - here; each chunk draws just the polygons that intersect it when it is computed. + A pixel ``(row, col)`` gets the id of a polygon that contains its centre ``(col + 0.5, row + 0.5)`` or has it on + its boundary: pixel ``(0, 0)`` covers ``[0, 1) x [0, 1)``, as in QuPath. Holes are not part of a polygon. Where + polygons overlap or touch, the one with the higher id wins, independently of their order, so cells that share an + edge leave no gap. Only the polygons that may + intersect each chunk are found here; a chunk draws them when it is computed. Parameters ---------- @@ -54,51 +60,70 @@ def rasterize_polygons_lazy( labels = np.asarray(ids, dtype=np.int64) if len(labels) and (labels.min() < 1 or labels.max() > _MAX_ID): raise ValueError(f"Polygon ids must be between 1 and {_MAX_ID}; got {labels.min()} to {labels.max()}.") - # drawing in ascending id order lets the higher id win overlaps; the stable sort keeps ties in input order - order = np.argsort(labels, kind="stable") - geometries = np.asarray(polygons, dtype=object)[order] + geometries = np.asarray(polygons, dtype=object) if scale != 1.0: geometries = shapely.transform(geometries, lambda xy: xy * scale) - labels = labels[order].astype(np.uint32) - template = da.empty(shape, chunks=chunks, dtype=np.uint32) - # dask would otherwise tokenize the geometries one by one; their vectorised WKB is much faster to hash - wkb = np.frombuffer(b"".join(shapely.to_wkb(geometries)), dtype=np.uint8) - token = tokenize(wkb, labels, template.shape, template.chunks) - return da.map_blocks( - _draw_block, - template, - dtype=np.uint32, - name=f"rasterize-polygons-{token}", - tree=shapely.STRtree(geometries), - labels=labels, - meta=np.empty((0, 0), dtype=np.uint32), - ) + labels = labels.astype(np.uint32) + sizes = normalize_chunks(chunks, shape, dtype=np.uint32) + name = f"rasterize-polygons-{_token(geometries, labels, sizes)}" + # every task holds just the polygons that may intersect its chunk, so the polygons are shipped once in all + starts = [np.cumsum((0, *axis[:-1])) for axis in sizes] + blocks = list(product(*(range(len(axis)) for axis in sizes))) + boxes = [ + shapely.box(starts[1][j], starts[0][i], starts[1][j] + sizes[1][j], starts[0][i] + sizes[0][i]) + for i, j in blocks + ] + block_of, polygon = shapely.STRtree(geometries).query(boxes) + counts = np.bincount(block_of, minlength=len(blocks)) + by_block = np.split(polygon[np.argsort(block_of, kind="stable")], np.cumsum(counts)[:-1]) + graph = { + (name, i, j): ( + _draw_block, + geometries[rows], + labels[rows], + (int(starts[0][i]), int(starts[1][j])), + (sizes[0][i], sizes[1][j]), + ) + for (i, j), rows in zip(blocks, by_block, strict=True) + } + return da.Array(graph, name, sizes, meta=np.empty((0, 0), dtype=np.uint32)) + + +def _token(geometries: np.ndarray, labels: np.ndarray, *args: Any) -> str: + """A deterministic token of the polygons, hashing their WKB in batches to bound the memory.""" + wkb = hashlib.blake2b(digest_size=16) + for start in range(0, len(geometries), _TOKEN_BATCH): + wkb.update(b"".join(shapely.to_wkb(geometries[start : start + _TOKEN_BATCH]))) + return tokenize(wkb.hexdigest(), labels, *args) def _draw_block( - block: np.ndarray, tree: shapely.STRtree, labels: np.ndarray, block_info: dict[Any, Any] | None = None + geometries: np.ndarray, labels: np.ndarray, origin: tuple[int, int], shape: tuple[int, int] ) -> np.ndarray: - (y0, y1), (x0, x1) = block_info[0]["array-location"] # type: ignore[index] - out = np.zeros(block.shape, dtype=np.uint32) - # the tree indices follow the ascending ids - for i in np.sort(tree.query(shapely.box(x0, y0, x1, y1))): - for part in shapely.get_parts(tree.geometries[i]): - if isinstance(part, shapely.Polygon) and not part.is_empty: - _draw_polygon(out, part, labels[i], y0, x0) + """The labels of the block of ``shape`` at ``origin``, the highest id of the polygons covering each pixel centre.""" + out = np.zeros(shape, dtype=np.uint32) + if not len(geometries): + return out + # private copies in block coordinates, prepared without racing other threads over the shared polygons + geometries = shapely.transform(geometries, lambda xy: xy - origin[::-1]) + shapely.prepare(geometries) + # the pixel window of each polygon in the block; pixel centres are at +0.5 + minx, miny, maxx, maxy = shapely.bounds(geometries).T + r0, r1 = np.clip([np.floor(miny - 0.5), np.ceil(maxy - 0.5) + 1], 0, shape[0]).astype(np.int64) + c0, c1 = np.clip([np.floor(minx - 0.5), np.ceil(maxx - 0.5) + 1], 0, shape[1]).astype(np.int64) + width = c1 - c0 + sizes = np.maximum(r1 - r0, 0) * np.maximum(width, 0) + firsts = np.cumsum(sizes) - sizes + flat = out.reshape(-1) + # all candidate pixels of a batch of polygons are tested in one GEOS call, which releases the GIL + start = 0 + while start < len(geometries): + stop = max(int(np.searchsorted(firsts, firsts[start] + _BATCH_PIXELS)), start + 1) + owner = np.repeat(np.arange(start, stop), sizes[start:stop]) + offset = np.arange(len(owner)) - (firsts[owner] - firsts[start]) + rows = r0[owner] + offset // width[owner] + cols = c0[owner] + offset % width[owner] + inside = shapely.intersects_xy(geometries[owner], cols + 0.5, rows + 0.5) + np.maximum.at(flat, rows[inside] * shape[1] + cols[inside], labels[owner[inside]]) + start = stop return out - - -def _draw_polygon(out: np.ndarray, polygon: shapely.Polygon, value: int, y0: int, x0: int) -> None: - """Set the pixels of ``out`` (the block at ``(y0, x0)``) whose centres are inside ``polygon`` to ``value``.""" - minx, miny, maxx, maxy = polygon.bounds - # the pixel window of the polygon in the block; pixel centres are at +0.5 - r0, c0 = max(int(np.floor(miny - 0.5)) - y0, 0), max(int(np.floor(minx - 0.5)) - x0, 0) - r1, c1 = min(int(np.ceil(maxy - 0.5)) + 1 - y0, out.shape[0]), min(int(np.ceil(maxx - 0.5)) + 1 - x0, out.shape[1]) - if r0 >= r1 or c0 >= c1: - return - inside = np.zeros((r1 - r0, c1 - c0), dtype=bool) - # holes are cleared from this polygon only, so cells lying in a hole keep their pixels - for ring, fill in [(polygon.exterior, True), *((ring, False) for ring in polygon.interiors)]: - xy = shapely.get_coordinates(ring) - inside[draw_polygon(xy[:, 1] - 0.5 - (y0 + r0), xy[:, 0] - 0.5 - (x0 + c0), shape=inside.shape)] = fill - out[r0:r1, c0:c1][inside] = value diff --git a/tests/readers/test_rasterize.py b/tests/readers/test_rasterize.py index e2a24643..002d738b 100644 --- a/tests/readers/test_rasterize.py +++ b/tests/readers/test_rasterize.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import pickle import warnings from typing import TYPE_CHECKING, Any @@ -67,6 +68,11 @@ def test_rasterize_pixel_convention_and_scale() -> None: # coordinates in µm with 0.5 µm pixels half = shapely.box(0.5, 1, 1.5, 2.5) np.testing.assert_array_equal(rasterize_polygons_lazy([half], [7], shape=(8, 8), scale=2).compute(), expected) + # pixel centres on the shared diagonal go to the higher id + halves = [shapely.Polygon([(0, 0), (4, 0), (0, 4)]), shapely.Polygon([(4, 0), (4, 4), (0, 4)])] + labels = rasterize_polygons_lazy(halves, [1, 2], shape=(4, 4)).compute() + rows, cols = np.indices((4, 4)) + np.testing.assert_array_equal(labels, np.where(rows + cols >= 3, 2, 1)) def test_rasterize_overlap_higher_id_wins() -> None: @@ -107,11 +113,12 @@ def test_rasterize_errors() -> None: def test_rasterize_is_lazy(monkeypatch: pytest.MonkeyPatch) -> None: calls = [] - def counting_polygon(*args: Any, **kwargs: Any) -> tuple[np.ndarray, np.ndarray]: - calls.append(1) - return draw_polygon(*args, **kwargs) + def counting_draw_block(geometries: np.ndarray, *args: Any) -> np.ndarray: + calls.append(len(geometries)) + return draw_block(geometries, *args) - monkeypatch.setattr(_rasterize, "draw_polygon", counting_polygon) + draw_block = _rasterize._draw_block + monkeypatch.setattr(_rasterize, "_draw_block", counting_draw_block) polygons = [shapely.box(i, i, i + 3, i + 3) for i in range(0, 60, 6)] labels = rasterize_polygons_lazy(polygons, list(range(1, 11)), shape=(64, 64), chunks=(16, 16)) @@ -119,10 +126,20 @@ def counting_polygon(*args: Any, **kwargs: Any) -> tuple[np.ndarray, np.ndarray] assert not calls labels.blocks[0, 0].compute() # only the polygons of the chunk are drawn - assert len(calls) == 3 + assert calls == [3] assert labels.compute().max() == 10 +def test_rasterize_tasks_hold_only_their_polygons() -> None: + polygons = [shapely.box(x, y, x + 3, y + 3) for x in range(0, 64, 8) for y in range(0, 64, 8)] + + labels = rasterize_polygons_lazy(polygons, list(range(1, 65)), shape=(64, 64), chunks=(16, 16)) + + # with dask.distributed, every task is shipped on its own + largest = max(len(pickle.dumps(task)) for task in dict(labels.__dask_graph__()).values()) + assert largest < len(pickle.dumps(polygons)) / 4 + + def _write_qupath_cells(path: Path, object_ids: list[str]) -> None: # centroids in the 8 px squares `_cell(..., 20 * i, 20 * i)` header = "Image\tObject ID\tClass\tCentroid X px\tCentroid Y px\tCD3: Cell: Mean" From 195858adc6c47b67b12f2ef9eb9e8b16c41d856b Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 04:12:19 +0200 Subject: [PATCH 081/107] perf(akoya): read only the ids and object types of GeoJSON features --- .../readers/_utils/_cell_tables.py | 3 ++- tests/readers/test_rasterize.py | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 72df2227..8c229db2 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -647,7 +647,8 @@ def _read_geojson(path: Path) -> tuple[np.ndarray | None, np.ndarray, np.ndarray ids = [f.get("id") for f in features] types = [(f.get("properties") or {}).get("objectType") for f in features] else: - frame = gpd.read_file(path) + # QuPath's other properties (measurements, nucleus outlines) would take several times the memory + frame = gpd.read_file(path, columns=["id", "objectType"]) geometries = frame.geometry.to_numpy() ids = list(frame["id"]) if "id" in frame.columns else [None] * len(frame) types = list(frame["objectType"]) if "objectType" in frame.columns else [None] * len(frame) diff --git a/tests/readers/test_rasterize.py b/tests/readers/test_rasterize.py index 002d738b..d06e614b 100644 --- a/tests/readers/test_rasterize.py +++ b/tests/readers/test_rasterize.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any import dask.array as da +import geopandas as gpd import numpy as np import pytest import shapely @@ -17,6 +18,7 @@ from spatialdata_io import phenocycler from spatialdata_io.readers._utils import _rasterize +from spatialdata_io.readers._utils._cell_tables import _read_geojson from spatialdata_io.readers._utils._rasterize import rasterize_polygons_lazy if TYPE_CHECKING: @@ -423,3 +425,27 @@ def test_qupath_geojson_false_reads_no_outlines(tmp_path: Path) -> None: assert list(sdata.shapes) == ["S_cells"] assert not sdata.labels assert "geojson" not in sdata.attrs["akoya"] + + +def test_read_geojson_reads_only_ids_and_object_types(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + frames = [] + + def spy(*args: Any, **kwargs: Any) -> gpd.GeoDataFrame: + frames.append(read_file(*args, **kwargs)) + return frames[-1] + + read_file = gpd.read_file + monkeypatch.setattr(gpd, "read_file", spy) + feature = _cell("a", 0, 0) + # QuPath also writes the nucleus outline and every measurement + feature["properties"] |= {"nucleusGeometry": feature["geometry"], "measurements": {"CD3: Cell: Mean": 1.0}} + _write_geojson(tmp_path / "cells.geojson", [feature]) + + ids, geometries, kinds = _read_geojson(tmp_path / "cells.geojson") + + assert ids is not None + assert kinds is not None + assert list(ids) == ["a"] + assert list(kinds) == ["cell"] + assert geometries[0].equals(shapely.box(0, 0, 8, 8)) + assert set(frames[0].columns) == {"id", "objectType", "geometry"} From 00b4e6a7ef41d3f9432b5009fd7ac6c5eff16252 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 04:13:49 +0200 Subject: [PATCH 082/107] perf(akoya): reduce GeoJSON array features while parsing them --- .../readers/_utils/_cell_tables.py | 22 +++++++++++++------ tests/readers/test_rasterize.py | 15 +++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 8c229db2..a8644c35 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -639,13 +639,13 @@ def _read_geojson(path: Path) -> tuple[np.ndarray | None, np.ndarray, np.ndarray with path.open("rb") as f: is_array = f.read(1024).lstrip(b"\xef\xbb\xbf \t\r\n").startswith(b"[") if is_array: - # QuPath's default export without the FeatureCollection option, which GDAL does not read - features = json.loads(path.read_text(encoding="utf-8-sig")) - geometries = np.array( - [shapely.geometry.shape(f["geometry"]) if f.get("geometry") else None for f in features], dtype=object - ) - ids = [f.get("id") for f in features] - types = [(f.get("properties") or {}).get("objectType") for f in features] + # QuPath's default export without the FeatureCollection option, which GDAL does not read; each feature is + # reduced as soon as it is parsed, so that its other properties are not all held at once + with path.open("rb") as f: + features = json.load(f, object_hook=_reduce_feature) + ids = [feature[0] for feature in features] + geometries = np.array([feature[1] for feature in features], dtype=object) + types = [feature[2] for feature in features] else: # QuPath's other properties (measurements, nucleus outlines) would take several times the memory frame = gpd.read_file(path, columns=["id", "objectType"]) @@ -664,6 +664,14 @@ def _read_geojson(path: Path) -> tuple[np.ndarray | None, np.ndarray, np.ndarray ) +def _reduce_feature(obj: dict[str, Any]) -> Any: + """The id, geometry and QuPath object type of a parsed GeoJSON feature; other JSON objects unchanged.""" + if "geometry" not in obj: + return obj + geometry = shapely.geometry.shape(obj["geometry"]) if obj.get("geometry") else None + return obj.get("id"), geometry, (obj.get("properties") or {}).get("objectType") + + def _read_fcs(path: Path) -> pd.DataFrame: """The events of an FCS file, with the columns named ``$PnN:$PnS`` like the CODEX Processor CSV header.""" fcs = readfcs.ReadFCS(path) diff --git a/tests/readers/test_rasterize.py b/tests/readers/test_rasterize.py index d06e614b..cc967e05 100644 --- a/tests/readers/test_rasterize.py +++ b/tests/readers/test_rasterize.py @@ -449,3 +449,18 @@ def spy(*args: Any, **kwargs: Any) -> gpd.GeoDataFrame: assert list(kinds) == ["cell"] assert geometries[0].equals(shapely.box(0, 0, 8, 8)) assert set(frames[0].columns) == {"id", "objectType", "geometry"} + + +def test_read_geojson_array_with_bom_and_properties(tmp_path: Path) -> None: + feature = _cell("a", 0, 0) + feature["properties"] |= {"nucleusGeometry": feature["geometry"], "measurements": {"CD3: Cell: Mean": 1.0}} + path = tmp_path / "cells.json" + path.write_bytes(b"\xef\xbb\xbf" + json.dumps([feature, _cell(None, 20, 20, "annotation")]).encode()) + + ids, geometries, kinds = _read_geojson(path) + + assert ids is not None + assert kinds is not None + assert list(ids) == ["a", None] + assert list(kinds) == ["cell", "annotation"] + assert geometries[1].equals(shapely.box(20, 20, 28, 28)) From db9e756e11c0c69e7573cfe32cc1a12422175e83 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 04:14:37 +0200 Subject: [PATCH 083/107] test(phenocycler): cover the *.raw.qptiff acquisition file rule outside the acquisition folder --- tests/test_phenocycler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index b8442544..fd1ca378 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -275,6 +275,7 @@ def test_ignores_acquisition_files(tmp_path: Path) -> None: write_qptiff(acquisition / "S.raw.qptiff", biomarkers=["DAPI"]) write_qptiff(acquisition / "S_Scan1_Cycle1.qptiff", biomarkers=["DAPI"]) write_qptiff(images / "S_Scan1.qptiff.intermediate", biomarkers=["DAPI"]) + write_qptiff(images / "S_Scan1.raw.qptiff", biomarkers=["DAPI"]) sdata = phenocycler(tmp_path, image_models_kwargs=_FAST) From c90b384444cf89e725717ec5798eb04a51e65b2e Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 04:15:30 +0200 Subject: [PATCH 084/107] fix(phenocycler): read the 8-bit file of an .er.qptiff path with read_all_bit_depths --- src/spatialdata_io/readers/phenocycler.py | 17 ++++++++++++----- tests/test_phenocycler.py | 12 ++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index fc7ea032..ea9b76a5 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -27,6 +27,7 @@ from spatialdata_io.readers._utils._qptiff import ( _fusion_profile_records, _is_blank, + _is_extended_range, qptiff_base_name, qptiff_regions_with_duplicates, read_qptiff_image, @@ -153,10 +154,11 @@ def phenocycler( scales those of a ``"generic"`` table too. read_all_bit_depths If `True`, a scan that exists as both ``X{vx.QPTIFF_SUFFIX}`` (8-bit) and ``X{vx.ER_QPTIFF_SUFFIX}`` - (extended range, e.g. uint16) is read from both files: the extended-range file as region ``X`` and the - 8-bit file as the image ``X_8bit{vx.IMAGE_SUFFIX}`` (region ``X_8bit`` in ``sdata.attrs[{vx.ATTRS_KEY!r}]``). - Both are the same physical scan, so the 8-bit image has exactly the transformations of region ``X`` (a - warning is raised if their shapes differ). Cell tables and masks annotate region ``X`` only. If `False`, only the extended-range file is read. + (extended range, e.g. uint16) is read from both files, also if ``path`` is the extended-range file: the + extended-range file as region ``X`` and the 8-bit file as the image ``X_8bit{vx.IMAGE_SUFFIX}`` (region + ``X_8bit`` in ``sdata.attrs[{vx.ATTRS_KEY!r}]``). Both are the same physical scan, so the 8-bit image has + exactly the transformations of region ``X`` (a warning is raised if their shapes differ). Cell tables and + masks annotate region ``X`` only. If `False`, only the extended-range file is read. image_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. labels_models_kwargs @@ -249,7 +251,12 @@ def _discover_qptiffs(path: Path) -> tuple[dict[str, Path], dict[str, Path]]: if not path.exists(): raise FileNotFoundError(f"{path} does not exist.") if path.is_file(): - return qptiff_regions_with_duplicates([path]) + files = [path] + if _is_extended_range(path): + # its 8-bit file, read with `read_all_bit_depths` + base = qptiff_base_name(path) + files.append(path.with_name(base + path.name[len(base) + len(".er") :])) + return qptiff_regions_with_duplicates([file for file in files if file.is_file()]) files = sorted( directory / name for directory, _, names in walk_visible(path) diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index fd1ca378..8783977c 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -183,6 +183,18 @@ def test_read_all_bit_depths(tmp_path: Path) -> None: assert sdata.attrs["akoya"]["regions"]["S_8bit"]["path"].endswith("S_Scan1.qptiff") +def test_read_all_bit_depths_of_an_extended_range_file(tmp_path: Path) -> None: + write_qptiff(tmp_path / "S_Scan1.QPTIFF", biomarkers=["DAPI"], dtype="uint8") + write_qptiff(tmp_path / "S_Scan1.ER.QPTIFF", biomarkers=["DAPI"], dtype="uint16") + + sdata = phenocycler(tmp_path / "S_Scan1.ER.QPTIFF", read_all_bit_depths=True, image_models_kwargs=_FAST) + + assert sorted(sdata.images) == ["S_8bit_image", "S_image"] + assert _level0(sdata, "S_8bit_image").dtype == np.uint8 + # the 8-bit file alone is read as it is + assert list(phenocycler(tmp_path / "S_Scan1.QPTIFF", read_all_bit_depths=True).images) == ["S_image"] + + def test_read_all_bit_depths_shares_stage_placement(tmp_path: Path) -> None: for name, pos in (("A", (0.0, 0.0)), ("B", (0.1, 0.0))): write_qptiff(tmp_path / f"{name}.qptiff", biomarkers=["DAPI"], dtype="uint8", stage_position_cm=pos) From 1dbdde0f48119a27ebeb21da8abc5b764ca9a8ba Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 04:17:48 +0200 Subject: [PATCH 085/107] perf(akoya): take the full-resolution pages of read_qptiff_image from the scan metadata --- src/spatialdata_io/readers/_utils/_qptiff.py | 34 +++++++----------- src/spatialdata_io/readers/codex.py | 2 +- src/spatialdata_io/readers/phenocycler.py | 2 +- tests/readers/test_qptiff.py | 36 +++++++++++++++----- 4 files changed, 42 insertions(+), 32 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_qptiff.py b/src/spatialdata_io/readers/_utils/_qptiff.py index 57335820..3e384424 100644 --- a/src/spatialdata_io/readers/_utils/_qptiff.py +++ b/src/spatialdata_io/readers/_utils/_qptiff.py @@ -64,7 +64,8 @@ class AkoyaScan: ``generation`` is one of ``"fusion1"``, ``"fusion2"``, ``"cim_qptiff"``, ``"qpi_other"`` or ``"legacy_processor"``. ``pixel_size_um`` is the calibrated pixel size, ``nominal_pixel_size_um`` the one the acquisition software reports. ``stage_position_um`` is the ``(x, y)`` stage position of the upper-left - corner. ``raw`` keeps the unparsed source metadata (JSON-serialisable). + corner. ``raw`` keeps the unparsed source metadata (JSON-serialisable). ``pages`` are the indices of the + full-resolution pages of a QPTIFF, the image's channels. """ path: Path @@ -78,6 +79,7 @@ class AkoyaScan: channels: list[AkoyaChannel] rgb: bool raw: dict[str, Any] + pages: list[int] = field(default_factory=list) def read_qptiff_metadata(path: Path, region: str | None = None) -> AkoyaScan: @@ -140,10 +142,11 @@ def read_qptiff_metadata(path: Path, region: str | None = None) -> AkoyaScan: "scan_profile": scan_profile, "image_types": dict(Counter(_text(xml, "ImageType") or "unknown" for xml in page_xml)), }, + pages=full_res, ) -def read_qptiff_image(path: Path) -> da.Array: +def read_qptiff_image(scan: AkoyaScan) -> da.Array: """Lazily read the full-resolution image of a PerkinElmer/Akoya QPTIFF. The array keeps the native tile chunks of the file, opens the file only when its chunks are read and can be @@ -151,34 +154,23 @@ def read_qptiff_image(path: Path) -> da.Array: Parameters ---------- - path - Path to the ``.qptiff`` file. + scan + The metadata of the ``.qptiff`` file, see :func:`read_qptiff_metadata`. Returns ------- ``(c, y, x)`` array, or ``(y, x, 3)`` for a brightfield RGB scan. """ - with _open_qpi(path) as tf: - full_res = [ - i - for i in range(len(tf.pages)) - if _text(_parse_description(_page(tf, i).description), "ImageType") == _FULL_RESOLUTION - ] - if not full_res: - raise ValueError(f"{path} has no page with {_FULL_RESOLUTION}.") - first = _page(tf, full_res[0]) - page_shape = first.shape - if len(full_res) == 1 and first.samplesperpixel == 3: - expected_shape: tuple[int, ...] = page_shape - else: - expected_shape = (len(full_res), *page_shape) + with _open_qpi(scan.path) as tf: + page_shape = _page(tf, scan.pages[0]).shape + expected_shape = page_shape if scan.rgb else (len(scan.pages), *page_shape) - image = lazy_tiff_array(path, series=0) + image = lazy_tiff_array(scan.path, series=0) if image.shape == expected_shape: return image # tifffile groups pages into series by shape heuristics; fall back to the pages the XML says are full-res - pages = [lazy_tiff_array(path, page=i) for i in full_res] - return pages[0] if len(expected_shape) == len(page_shape) else da.stack(pages) + pages = [lazy_tiff_array(scan.path, page=i) for i in scan.pages] + return pages[0] if scan.rgb else da.stack(pages) def qpi_pixel_size_um(page: tifffile.TiffPage) -> float | None: diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 84e4f90c..a368dfb6 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -249,7 +249,7 @@ def codex( files = layout.images[region] if _is_qptiff(files[0]): scans.append(read_qptiff_metadata(files[0], region=region)) - arrays[region] = read_qptiff_image(files[0]) + arrays[region] = read_qptiff_image(scans[-1]) continue arrays[region], channels = _read_legacy_image( files, layout.channel_names_txt, image_models_kwargs=image_models_kwargs diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index ea9b76a5..d2fa30dd 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -217,7 +217,7 @@ def phenocycler( pixel_sizes={scan.region: scan.pixel_size_um for scan in scans}, cells_table_kwargs=cells_table_kwargs, ) - images = {scan.region: read_qptiff_image(scan.path) for scan in [*scans, *duplicates.values()]} + images = {scan.region: read_qptiff_image(scan) for scan in [*scans, *duplicates.values()]} for region, duplicate in duplicates.items(): if images[region].shape[-2:] != images[duplicate.region].shape[-2:]: warnings.warn( diff --git a/tests/readers/test_qptiff.py b/tests/readers/test_qptiff.py index 808433a9..d50443dd 100644 --- a/tests/readers/test_qptiff.py +++ b/tests/readers/test_qptiff.py @@ -3,6 +3,7 @@ import pickle import warnings from pathlib import Path +from typing import Any import dask.array as da import numpy as np @@ -92,7 +93,7 @@ def test_cim_without_label_page_counts_pages_by_imagetype(tmp_path: Path) -> Non label_page=False, resolution_px_per_cm=1e4 / 0.37739, ) - image = read_qptiff_image(p) + image = read_qptiff_image(read_qptiff_metadata(p)) assert image.shape == (5, 256, 320) assert np.array_equal(image.compute(), written) scan = read_qptiff_metadata(p) @@ -205,7 +206,7 @@ def test_profile_record_without_marker_skipped(tmp_path: Path) -> None: def test_image_is_lazy_and_correct(tmp_path: Path, dtype: str, name: str) -> None: p = tmp_path / name written = write_qptiff(p, biomarkers=["DAPI", "CD3"], dtype=dtype) - arr = read_qptiff_image(p) + arr = read_qptiff_image(read_qptiff_metadata(p)) assert isinstance(arr, da.Array) assert arr.chunksize != arr.shape assert arr.dtype == np.dtype(dtype) @@ -218,7 +219,7 @@ def test_image_is_picklable_and_closes_its_file(tmp_path: Path) -> None: written = write_qptiff(p, biomarkers=["DAPI", "CD3"]) with warnings.catch_warnings(record=True) as record: warnings.simplefilter("always", ResourceWarning) - arr = read_qptiff_image(p) + arr = read_qptiff_image(read_qptiff_metadata(p)) restored = pickle.loads(pickle.dumps(arr)) np.testing.assert_array_equal(restored[0, :8, :8].compute(), written[0, :8, :8]) np.testing.assert_array_equal(arr.compute(), written) @@ -237,17 +238,36 @@ def wrong_series(path: Path, *, series: int = 0, page: int | None = None) -> da. return real(path, series=series, page=page) if page is not None else da.zeros((1, 1, 1)) monkeypatch.setattr(_qptiff, "lazy_tiff_array", wrong_series) - arr = read_qptiff_image(p) + arr = read_qptiff_image(read_qptiff_metadata(p)) assert arr.shape == written.shape np.testing.assert_array_equal(pickle.loads(pickle.dumps(arr))[1, :8, :8].compute(), written[1, :8, :8]) +def test_image_does_not_parse_the_page_xml_again(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + p = tmp_path / "x.qptiff" + written = write_qptiff(p, biomarkers=["DAPI", "CD3"]) + scan = read_qptiff_metadata(p) + parsed = [] + parse = _qptiff._parse_description + + def counting_parse(description: str) -> Any: + parsed.append(description) + return parse(description) + + monkeypatch.setattr(_qptiff, "_parse_description", counting_parse) + + image = read_qptiff_image(scan) + + assert not parsed + np.testing.assert_array_equal(image.compute(), written) + + def test_three_channel_not_rgb(tmp_path: Path) -> None: p = tmp_path / "x.qptiff" written = write_qptiff(p, biomarkers=["DAPI", "CD3", "CD8"]) scan = read_qptiff_metadata(p) assert scan.rgb is False - image = read_qptiff_image(p) + image = read_qptiff_image(read_qptiff_metadata(p)) assert image.ndim == 3 assert image.shape == (3, 256, 320) assert np.array_equal(image.compute(), written) @@ -258,7 +278,7 @@ def test_brightfield_rgb(tmp_path: Path) -> None: written = write_qptiff(p, biomarkers=[None], names=["BF"], rgb=True, software="VectraPolaris 1.0") scan = read_qptiff_metadata(p) assert scan.rgb is True - image = read_qptiff_image(p) + image = read_qptiff_image(read_qptiff_metadata(p)) assert image.shape == (256, 320, 3) assert np.array_equal(image.compute(), written) @@ -268,8 +288,6 @@ def test_not_qpi_raises(tmp_path: Path) -> None: tifffile.imwrite(p, np.zeros((16, 16), dtype=np.uint8)) with pytest.raises(ValueError, match="not a PerkinElmer"): read_qptiff_metadata(p) - with pytest.raises(ValueError, match="not a PerkinElmer"): - read_qptiff_image(p) def test_missing_file_raises(tmp_path: Path) -> None: @@ -284,4 +302,4 @@ def test_classic_tiff_and_stripped_level(tmp_path: Path) -> None: warnings.simplefilter("error") scan = read_qptiff_metadata(p) assert len(scan.channels) == 2 - assert np.array_equal(read_qptiff_image(p).compute(), written) + assert np.array_equal(read_qptiff_image(read_qptiff_metadata(p)).compute(), written) From 53722a3671aeae500fc6c9743ef97bb26ef83365 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 04:18:29 +0200 Subject: [PATCH 086/107] refactor(akoya): drop the unused scale of rasterize_polygons_lazy --- src/spatialdata_io/readers/_utils/_rasterize.py | 7 +------ tests/readers/test_rasterize.py | 5 +---- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_rasterize.py b/src/spatialdata_io/readers/_utils/_rasterize.py index a32e8734..386f6b35 100644 --- a/src/spatialdata_io/readers/_utils/_rasterize.py +++ b/src/spatialdata_io/readers/_utils/_rasterize.py @@ -28,7 +28,6 @@ def rasterize_polygons_lazy( ids: Sequence[int], shape: tuple[int, int], chunks: tuple[int, int] = (4096, 4096), - scale: float = 1.0, ) -> da.Array: """Rasterise polygons into a lazy ``uint32`` labels array, one chunk at a time. @@ -41,15 +40,13 @@ def rasterize_polygons_lazy( Parameters ---------- polygons - The polygons, in pixel units after multiplying by ``scale``. + The polygons, in pixels. ids The label of each polygon, between 1 and ``2**32 - 1``. shape The ``(y, x)`` shape of the labels. chunks The ``(y, x)`` chunk size. - scale - The number of pixels per polygon coordinate unit, e.g. ``1 / pixel_size_um`` for coordinates in µm. Returns ------- @@ -61,8 +58,6 @@ def rasterize_polygons_lazy( if len(labels) and (labels.min() < 1 or labels.max() > _MAX_ID): raise ValueError(f"Polygon ids must be between 1 and {_MAX_ID}; got {labels.min()} to {labels.max()}.") geometries = np.asarray(polygons, dtype=object) - if scale != 1.0: - geometries = shapely.transform(geometries, lambda xy: xy * scale) labels = labels.astype(np.uint32) sizes = normalize_chunks(chunks, shape, dtype=np.uint32) name = f"rasterize-polygons-{_token(geometries, labels, sizes)}" diff --git a/tests/readers/test_rasterize.py b/tests/readers/test_rasterize.py index cc967e05..e981893e 100644 --- a/tests/readers/test_rasterize.py +++ b/tests/readers/test_rasterize.py @@ -61,15 +61,12 @@ def test_rasterize_matches_eager_reference() -> None: assert result[29, 38] == 0 # in the hole -def test_rasterize_pixel_convention_and_scale() -> None: +def test_rasterize_pixel_convention() -> None: square = shapely.box(1, 2, 3, 5) # covers the pixels x 1..2, y 2..4 expected = np.zeros((8, 8), dtype=np.uint32) expected[2:5, 1:3] = 7 np.testing.assert_array_equal(rasterize_polygons_lazy([square], [7], shape=(8, 8)).compute(), expected) - # coordinates in µm with 0.5 µm pixels - half = shapely.box(0.5, 1, 1.5, 2.5) - np.testing.assert_array_equal(rasterize_polygons_lazy([half], [7], shape=(8, 8), scale=2).compute(), expected) # pixel centres on the shared diagonal go to the higher id halves = [shapely.Polygon([(0, 0), (4, 0), (0, 4)]), shapely.Polygon([(4, 0), (4, 4), (0, 4)])] labels = rasterize_polygons_lazy(halves, [1, 2], shape=(4, 4)).compute() From cf72ec8e55bc918ffbef37017f0af2be2f57bc2c Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 13:53:07 +0200 Subject: [PATCH 087/107] fix(akoya): read semicolon-separated cell tables with decimal commas --- CHANGELOG.md | 1 + .../readers/_utils/_cell_tables.py | 42 +++++++++++++++---- tests/readers/test_cell_tables.py | 23 ++++++++++ 3 files changed, 58 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92c4d73c..46874b42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. are skipped with a warning; a GeoJSON found next to a QuPath table that holds none of its cells (e.g. an annotation export) is ignored with a warning, and `cells_table_kwargs["geojson"]=False` reads no outlines. A region whose cells lie mostly outside its image raises a warning. +- The Akoya cell tables may be semicolon-separated (European locales); their numbers may have decimal commas. ### Changed diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index a8644c35..d6f1e781 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -210,7 +210,9 @@ def detect_table_format(path: Path) -> str: Parameters ---------- path - The ``.fcs``, ``.csv``, ``.tsv`` or ``.txt`` (tab-separated) cell table. + The ``.fcs``, ``.csv``, ``.tsv`` or ``.txt`` cell table. A text table is tab-separated if its header has a + tab, else semicolon-separated (with decimal commas or points) if it has more semicolons than commas, else + comma-separated. Returns ------- @@ -363,11 +365,7 @@ def read_cell_table( if path.suffix.lower() == CodexKeys.FCS_FILE.value: df = _read_fcs(path) else: - _, sep = _read_header(path) - df = pd.read_csv(path, sep=sep, engine="pyarrow") - # inForm writes some column names with trailing spaces; pyarrow keeps repeated names, which are made unique - # like the marker names - df.columns = pd.Index(dedupe_channel_names([str(column).strip() for column in df.columns])) + df = _read_text_table(path) reader = { "codex": _codex_roles, "qupath": _qupath_roles, @@ -1120,13 +1118,41 @@ def _drop_key_columns(obs: pd.DataFrame, instance_column: str | None, region_col def _read_header(path: Path) -> tuple[list[str], str]: - """The column names of a delimited text table and its delimiter (tab if the header has one, else comma).""" + """The column names of a delimited text table and its delimiter. + + The delimiter is a tab if the header has one, else a semicolon if it has more semicolons than commas, else a comma. + """ with path.open(encoding="utf-8-sig", errors="replace") as f: header = f.readline().rstrip("\r\n") - sep = "\t" if "\t" in header else "," + sep = "\t" if "\t" in header else ";" if header.count(";") > header.count(",") else "," return [column.strip().strip('"') for column in header.split(sep)], sep +def _read_text_table(path: Path) -> pd.DataFrame: + """Read a delimited text table with the delimiter of :func:`_read_header`, with unique column names. + + Semicolon-separated tables (European locales) are read with decimal commas; their columns with decimal points, + which that leaves as text, are converted to numbers too. + """ + names, sep = _read_header(path) + if sep == ";": + # the C engine keeps the header's repeated names apart only by renaming them, so the names come from the header + df = pd.read_csv(path, sep=sep, decimal=",", engine="c", header=None, skiprows=1) + df.columns = pd.Index(names) + for i in range(df.shape[1]): + if not pd.api.types.is_numeric_dtype(df.iloc[:, i]): + try: + df.isetitem(i, pd.to_numeric(df.iloc[:, i]).to_numpy()) + except (ValueError, TypeError): + pass + else: + df = pd.read_csv(path, sep=sep, engine="pyarrow") + # inForm writes some column names with trailing spaces; pyarrow keeps repeated names, which are made unique like + # the marker names + df.columns = pd.Index(dedupe_channel_names([str(column).strip() for column in df.columns])) + return df + + def resolve_per_region( value: str | Path | Mapping[str, str | Path] | None, regions: Sequence[str], diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index 65f542b7..daf1aa13 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -135,6 +135,29 @@ def test_read_codex_table_delimiters_like_detection(tmp_path: Path, sep: str) -> np.testing.assert_array_equal(table.centroids_px, [[1, 2]]) +@pytest.mark.parametrize("decimal", [",", "."]) +@pytest.mark.parametrize( + ("header", "row", "fmt"), + [ + ("cell_id;region;x;y;size;cyc001_ch001:DAPI", "1;2;1{d}5;2{d}0;3{d}0;4{d}25", "codex"), + ("CellID;DAPI;X_centroid;Y_centroid;Area", "1;4{d}25;1{d}5;2{d}0;3{d}0", "mcquant"), + ("Image;Object ID;Centroid X px;Centroid Y px;DAPI: Cell: Mean", "reg002;a;1{d}5;2{d}0;4{d}25", "qupath"), + ], +) +def test_semicolon_tables_read_comma_decimals(tmp_path: Path, header: str, row: str, fmt: str, decimal: str) -> None: + path = tmp_path / "cells.csv" + path.write_text(f"{header}\n{row.format(d=decimal)}\n", encoding="utf-8") + + assert detect_table_format(path) == fmt + table = read_cell_table(path, "auto", pixel_size_um=None, default_region="reg002", cells_table_kwargs={}) + + assert list(table.var.index) == ["DAPI"] + assert list(table.region) == ["reg002"] + assert table.centroids_px is not None + np.testing.assert_allclose(table.centroids_px, [[1.5, 2]]) + np.testing.assert_allclose(table.X, [[4.25]]) + + def test_read_codex_table_cells_table_kwargs(tmp_path: Path) -> None: path = tmp_path / "cells.csv" path.write_text("cell_id,label,Sample,x,y,CD3,CD8\n1,7,A,1.0,2.0,3.0,4.0\n2,8,B,5.0,6.0,7.0,8.0\n") From b63a06b723f030c79fc563e53ea652791416eaef Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 13:55:56 +0200 Subject: [PATCH 088/107] fix(akoya): warn when CODEX table markers are taken from its numeric columns --- src/spatialdata_io/readers/_utils/_cell_tables.py | 12 ++++++++++-- tests/readers/test_cell_tables.py | 9 +++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index d6f1e781..5d1d8209 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -280,8 +280,9 @@ def read_cell_table( - ``"codex"``: a CODEX Processor ``.fcs`` or ``.csv`` table. The markers are ``markers``, else the ``cycCCC_chHHH`` columns, named by the FCS ``$PnS`` keywords or by the display part of the Processor - CSV header ``cycCCC_chHHH:`` (in ``var`` with their ``cycle`` and ``channel``), else the - numeric columns that are not CODEX Processor columns (``x``, ``size``, ...). Centroids ``x``/``y`` in + CSV header ``cycCCC_chHHH:`` (in ``var`` with their ``cycle`` and ``channel``), else, with a + `UserWarning`, the numeric columns that are not CODEX Processor columns (``x``, ``size``, ...). + Centroids ``x``/``y`` in pixels, instance ids ``cell_id``, the region from ``region`` (a numeric region ``r`` is the region id ``regRRR``), the radius from ``size``. - ``"qupath"``: a QuPath measurement export, also with R's ``make.names`` column names @@ -713,6 +714,13 @@ def _codex_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dic name.casefold() not in _META_COLUMNS and pd.api.types.is_numeric_dtype(df.iloc[:, i]) for i, name in enumerate(names) ] + taken = [name for name, marker in zip(names, is_marker, strict=True) if marker] + warnings.warn( + f"{path} has no 'cycCCC_chHHH' columns; reading the {len(taken)} numeric columns {taken} that are not " + "CODEX Processor columns as the markers. Set `cells_table_kwargs['markers']` to choose the marker columns.", + UserWarning, + stacklevel=5, + ) positions = [i for i, marker in enumerate(is_marker) if marker] var = pd.DataFrame( {"source_column": [sources[i] for i in positions]}, diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index daf1aa13..b7ded99a 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -86,7 +86,9 @@ def test_read_codex_csv_colon_header(tmp_path: Path) -> None: "2,2,11,21,31,200.5,8\n" ) - table = _read_codex(path) + with warnings.catch_warnings(): + warnings.simplefilter("error") # the markers are the cycCCC_chHHH columns, no heuristic + table = _read_codex(path) assert list(table.var.index) == ["DAPI1", "CD3"] assert list(table.var["source_column"]) == ["cyc001_ch001:DAPI1", "cyc001_ch002:CD3"] @@ -101,7 +103,10 @@ def test_read_codex_csv_marker_name_columns(tmp_path: Path) -> None: path = tmp_path / "cells.csv" path.write_text("cell_id,x,y,tile_num,CD3,CD8,label\n1,1.0,2.0,1,5.0,6.0,T\n") - table = _read_codex(path, region="Kidney") + with pytest.warns( + UserWarning, match=r"no 'cycCCC_chHHH' columns.*\['CD3', 'CD8'\].*cells_table_kwargs\['markers'\]" + ): + table = _read_codex(path, region="Kidney") assert list(table.var.index) == ["CD3", "CD8"] assert "cycle" not in table.var.columns From 004b7e4f67ac744aa625d40c485c08d92eef92f9 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 13:58:09 +0200 Subject: [PATCH 089/107] fix(akoya): name qptiffs with a regNNN in both readers by that region id --- src/spatialdata_io/readers/_utils/_akoya.py | 11 +++++++ src/spatialdata_io/readers/_utils/_qptiff.py | 8 ++++-- src/spatialdata_io/readers/codex.py | 30 +++++++------------- src/spatialdata_io/readers/phenocycler.py | 4 ++- tests/test_codex.py | 21 ++++++++++++++ 5 files changed, 50 insertions(+), 24 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index 75d3ffbb..32ee7ccf 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -5,6 +5,7 @@ import io import json import math +import re import threading import warnings from collections import Counter @@ -39,12 +40,22 @@ "normalize_regions", "read_json_object", "read_tiff", + "region_id", "region_transformations", "scans_to_attrs", "select_regions", "spatial_chunks", ] +# `reg001`, `exp_reg001` or `1_reg001`, but not `Treg2` or `coreg1` +_REGION = re.compile(r"(? str | None: + """The CODEX region id ``regRRR`` in a file name: ``reg`` and digits, not preceded by a letter; else `None`.""" + match = _REGION.search(name) + return f"reg{int(match.group(1)):03d}" if match else None + def dedupe_channel_names(names: Sequence[str]) -> list[str]: """Make channel names unique as ``name``, ``name (2)``, ``name (3)``; the first occurrence is unchanged. diff --git a/src/spatialdata_io/readers/_utils/_qptiff.py b/src/spatialdata_io/readers/_utils/_qptiff.py index 3e384424..e2c2a2bd 100644 --- a/src/spatialdata_io/readers/_utils/_qptiff.py +++ b/src/spatialdata_io/readers/_utils/_qptiff.py @@ -15,7 +15,7 @@ from spatialdata import sanitize_name from spatialdata_io._constants._constants import PhenocyclerKeys -from spatialdata_io.readers._utils._akoya import lazy_tiff_array +from spatialdata_io.readers._utils._akoya import lazy_tiff_array, region_id if TYPE_CHECKING: from collections.abc import Sequence @@ -248,7 +248,9 @@ def qptiff_regions(files: Sequence[Path]) -> dict[str, Path]: """The region id of every qptiff scan among ``files``, the same for both Akoya readers. Files of the same scan (folder and base name) are read once, preferring the extended-range ``.er.qptiff``. - The region id is the base name without a trailing ``_Scan``, unless that makes two scans share a region id. + The region id is the CODEX region id ``regRRR`` of a base name with ``reg`` and digits not preceded by a letter + (``exp_reg001`` to ``reg001``), else the base name without a trailing ``_Scan``, unless that makes two scans + share a region id. Raises if two scans have the same region id. Parameters ---------- @@ -289,7 +291,7 @@ def qptiff_regions_with_duplicates(files: Sequence[Path]) -> tuple[dict[str, Pat stripped = [sanitize_name(_SCAN_SUFFIX.sub("", base) or base) for base in bases] # stripping `_Scan` must not merge two scans into one region regions = [ - sanitize_name(base) if stripped.count(region) > 1 else region + region_id(base) or (sanitize_name(base) if stripped.count(region) > 1 else region) for base, region in zip(bases, stripped, strict=True) ] if len(set(regions)) < len(regions): diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index a368dfb6..d75da673 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -20,6 +20,7 @@ normalize_regions, read_json_object, read_tiff, + region_id, select_regions, spatial_chunks, ) @@ -50,8 +51,6 @@ __all__ = ["codex"] -# `reg001`, `exp_reg001` or `1_reg001`, but not `Treg2` or `coreg1` -_REGION = re.compile(r"(?`` (unless that makes two regions share a name), and the extended-range ``.er.qptiff`` of a - scan is preferred. If a directory contains no such files, its top-level ``{vx.IMAGE_TIF}(f)`` files are read as + scan is preferred, also for one with a region id (``exp_reg001.er.qptiff`` and ``exp_reg001.qptiff`` are region + ``reg001``). Several scans with the same region id raise an error. If a directory contains no such files, its top-level ``{vx.IMAGE_TIF}(f)`` files are read as hyperstacks named after their files, made valid element names by :func:`spatialdata.sanitize_name` (e.g. ``Kidney sample.tif`` becomes the region ``Kidney_sample``). Each region is stored as the image ``_image`` and its metadata in ``sdata.attrs["akoya"]``. @@ -483,15 +483,11 @@ def _is_ignored_dir(name: str, has_stitched: bool) -> bool: def _discover_images(files: Sequence[Path]) -> dict[str, list[Path]]: per_marker: dict[str, dict[tuple[int, int], Path]] = {} stacks: dict[str, list[Path]] = {} - qptiffs: dict[str, list[Path]] = {} - unnamed_qptiffs = [] + all_qptiffs = [] for file in files: - region = _region_id(file.name) + region = region_id(file.name) if _is_qptiff(file): - if region is None: - unnamed_qptiffs.append(file) - else: - qptiffs.setdefault(region, []).append(file) + all_qptiffs.append(file) elif _is_tiff(file) and region is not None: if (match := _PER_MARKER.search(file.name)) is None: stacks.setdefault(region, []).append(file) @@ -505,9 +501,8 @@ def _discover_images(files: Sequence[Path]) -> dict[str, list[Path]]: ) markers[key] = file - # named like `phenocycler()` names them, e.g. `Tonsil_Scan1.er.qptiff` -> `Tonsil` - for region, file in qptiff_regions(unnamed_qptiffs).items(): - qptiffs.setdefault(region, []).append(file) + # named like `phenocycler()` names them, e.g. `exp_reg001.qptiff` -> `reg001`, `Tonsil_Scan1.er.qptiff` -> `Tonsil` + qptiffs = qptiff_regions(all_qptiffs) images: dict[str, list[Path]] = {} # the Processor's stitched outputs come first: cell tables refer to their pixel space @@ -517,7 +512,7 @@ def _discover_images(files: Sequence[Path]) -> dict[str, list[Path]]: elif region in stacks: images[region] = [_single(region, _preferred_stacks(stacks[region]))] else: - images[region] = [_single(region, qptiffs[region])] + images[region] = [qptiffs[region]] return images @@ -543,7 +538,7 @@ def _discover_tables(files: Sequence[Path], fcs: bool) -> dict[str, Path]: candidates: dict[str, list[Path]] = {} for file in files: if file.suffix.lower() in _TABLE_SUFFIXES and _is_codex_table(file): - candidates.setdefault(_region_id(file.name) or "*", []).append(file) + candidates.setdefault(region_id(file.name) or "*", []).append(file) def uncompensated(file: Path) -> bool: return any("uncompensated" in part.casefold() for part in file.parts) @@ -625,11 +620,6 @@ def _pixel_size_um(experiment: Mapping[str, Any] | None) -> float | None: return None -def _region_id(name: str) -> str | None: - match = _REGION.search(name) - return f"reg{int(match.group(1)):03d}" if match else None - - def _is_qptiff(path: Path) -> bool: return path.suffix.lower() == CodexKeys.QPTIFF.value diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index d2fa30dd..bed9d5ea 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -85,7 +85,9 @@ def phenocycler( ``.``, e.g. macOS ``._*`` files, and ``__MACOSX/``); symlinked folders are followed. File suffixes are matched ignoring case. - The region name is the file name without the ``{vx.ER_QPTIFF_SUFFIX!r}``/``{vx.QPTIFF_SUFFIX!r}`` suffix and + The region name is the CODEX region id ``regNNN`` of a file name with ``reg`` and digits not preceded by a letter + (``exp_reg001.qptiff`` is ``reg001``, as in :func:`spatialdata_io.codex`; several scans with the same region id + raise an error), else the file name without the ``{vx.ER_QPTIFF_SUFFIX!r}``/``{vx.QPTIFF_SUFFIX!r}`` suffix and without a trailing ``_Scan``, unless that makes two regions share a name, made a valid element name by :func:`spatialdata.sanitize_name` (characters other than letters, digits, ``_``, ``.`` and ``-`` are replaced by ``_``, e.g. ``A_[1,2]`` becomes ``A__1_2_``). Each region is stored as the image diff --git a/tests/test_codex.py b/tests/test_codex.py index bd24a1a6..eeddd445 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -475,6 +475,27 @@ def test_codex_flat_layout_ignores_non_region_tifs(tmp_path: Path) -> None: assert list(sdata.images) == ["Kidney_sample_image"] +def test_codex_and_phenocycler_take_the_region_id_of_a_qptiff(tmp_path: Path) -> None: + write_qptiff(tmp_path / "exp_reg001.qptiff", biomarkers=["DAPI"], dtype="uint8") + write_qptiff(tmp_path / "exp_reg001.er.qptiff", biomarkers=["DAPI"], dtype="uint16") + write_qptiff(tmp_path / "Tonsil_Scan1.qptiff", biomarkers=["DAPI"], dtype="uint8") + + for reader in (codex, phenocycler): + sdata = reader(tmp_path, image_models_kwargs=_FAST) + assert sorted(sdata.images) == ["Tonsil_image", "reg001_image"] + assert sdata.attrs["akoya"]["regions"]["reg001"]["path"].endswith("exp_reg001.er.qptiff") + assert list(reader(tmp_path / "exp_reg001.qptiff", image_models_kwargs=_FAST).images) == ["reg001_image"] + + +def test_codex_and_phenocycler_raise_for_qptiffs_of_one_region_id(tmp_path: Path) -> None: + write_qptiff(tmp_path / "exp_reg001_Scan1.qptiff", biomarkers=["DAPI"]) + write_qptiff(tmp_path / "other_reg1.qptiff", biomarkers=["DAPI"]) + + for reader in (codex, phenocycler): + with pytest.raises(ValueError, match=r"Several qptiff files map to the same region\(s\) \['reg001'\]"): + reader(tmp_path, image_models_kwargs=_FAST) + + def test_codex_and_phenocycler_name_a_non_region_qptiff_alike(tmp_path: Path) -> None: write_qptiff(tmp_path / "Preg12_Scan1.qptiff", biomarkers=["DAPI"], dtype="uint8") From fd703be4e90f8fd7cf9df584ddd65ad7283ab01d Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 14:00:37 +0200 Subject: [PATCH 090/107] feat(codex): read a region's _mask/_labels TIFF as its mask, not an image --- CHANGELOG.md | 3 +- src/spatialdata_io/readers/codex.py | 49 ++++++++++++++++++++++++++--- tests/test_codex.py | 44 ++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46874b42..12020f40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,7 +80,8 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. - `channelNames.txt` and `experiment.json` are found in any case (e.g. SPACEc's `channelnames.txt`); CSV and FCS files that are not CODEX Processor cell tables are ignored, also when their name holds a region id. Compensated tables are preferred; other tables of the same region (e.g. of two segmentation runs) raise a `ValueError`. - A `cells_table` disables table discovery, and the `labels` masks are not read as images. + A `cells_table` disables table discovery, and the `labels` masks are not read as images. Without `labels`, a + `.tif(f)` ending in `_mask` or `_labels` and named after a region (e.g. `reg001_mask.ome.tiff`) is its mask. - Region ids `regNNN` are taken from `reg` and digits not preceded by a letter (`exp_reg001`, not `Treg2`). A CIM `.qptiff` without one is named as `phenocycler()` names it (without the suffix and a trailing `_Scan`), and its `.er.qptiff` is preferred. Top-level `.tif` files of the original layout are named after their sanitised stems. diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index d75da673..f30fc3b8 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -57,6 +57,9 @@ # raw Keyence acquisition folders, e.g. `cyc001_reg001/1_00001_Z001_CH1.tif` _RAW_CYCLE_DIR = re.compile(r"cyc\d+_reg\d+", re.IGNORECASE) _TIFF_SUFFIXES = (".tif", ".tiff") +# the segmentation mask of a region, e.g. HuBMAP's `reg001_mask.ome.tiff` next to `reg001_expr.ome.tiff` +_MASK = re.compile(r"[_-](?:mask|labels)(?:\.ome)?\.tiff?$", re.IGNORECASE) +_TIFF_NAME = re.compile(r"(?:\.ome)?\.tiff?$", re.IGNORECASE) _TABLE_SUFFIXES = (CodexKeys.FCS_FILE.value, CodexKeys.FCS_FILE_CSV.value) # deep enough for `/processed_/segm/segm-1/fcs/compensated/
` _MAX_DEPTH = 8 @@ -76,11 +79,12 @@ class _LegacyLayout: ``images`` maps each region to its per-marker images (sorted by cycle and channel), to its single hyperstack or to its single ``.qptiff``. ``tables`` maps each region, or ``"*"`` for a table without a region in its - file name, to its ``.fcs``/``.csv`` cell table. + file name, to its ``.fcs``/``.csv`` cell table. ``masks`` maps regions to their segmentation mask. """ images: dict[str, list[Path]] tables: dict[str, Path] + masks: dict[str, Path] channel_names_txt: Path | None experiment_json: Path | None @@ -185,7 +189,10 @@ def codex( (columns given by ``cells_table_kwargs``). labels A segmentation mask (TIFF/OME-TIFF with integer instance ids) per region: a path for a single region, or a - mapping from region id to path; a mask inside ``path`` is not read as an image. Each mask is stored lazily as + mapping from region id to path; a mask inside ``path`` is not read as an image. If `None`, a TIFF named after + a region image or with its region id, ending in ``_mask`` or ``_labels`` (e.g. ``reg001_mask.ome.tiff`` or + ``Kidney_labels.tif`` next to ``Kidney.tif``), is the mask of that region; such files are never read as images. + Each mask is stored lazily as the labels ``_labels`` with the transformation of the region's image; the cells of the region annotate it instead of circles, and their instance ids must be the mask values. A mask whose shape is the region image's divided by an integer factor ``f`` (exactly, or rounded up) is scaled onto the image (``Scale([f, f])`` before @@ -239,7 +246,10 @@ def codex( selected = select_regions(available, regions) _warn_fcs_fallback([file for key, file in layout.tables.items() if key == "*" or key in selected], fcs) table_files = resolve_per_region(cells_table, selected, available) - mask_files = resolve_per_region(labels, selected, available) + if labels is None: + mask_files = {region: file for region, file in layout.masks.items() if region in selected} + else: + mask_files = resolve_per_region(labels, selected, available) experiment = read_json_object(layout.experiment_json) if layout.experiment_json is not None else None names_txt = _read_channel_names_txt(layout.channel_names_txt) if layout.channel_names_txt is not None else None @@ -369,10 +379,12 @@ def _discover_legacy( files = [path] if path.is_file() else _walk(path) if masks: files = [file for file in files if file.resolve() not in masks] - images = _discover_images(files) + # masks named after their region are never images + image_files = [file for file in files if not (_is_tiff(file) and _MASK.search(file.name))] + images = _discover_images(image_files) if not images: # the original (2019) layout: one stitched `.tif` next to its table, without a region in the file name - top_level = files if path.is_file() else [file for file in files if file.parent == path] + top_level = image_files if path.is_file() else [file for file in image_files if file.parent == path] images = {sanitize_name(file.stem): [file] for file in top_level if _is_tiff(file)} tables = _discover_tables(files, fcs) if find_tables else {} if not images and not tables: @@ -380,6 +392,7 @@ def _discover_legacy( return _LegacyLayout( images=dict(sorted(images.items())), tables=dict(sorted(tables.items())), + masks=_discover_masks([file for file in files if file not in image_files], images), channel_names_txt=_find_sidecar(path, files, CodexKeys.CHANNEL_NAMES_FILE.value), experiment_json=_find_sidecar(path, files, CodexKeys.EXPERIMENT_FILE.value), ) @@ -516,6 +529,32 @@ def _discover_images(files: Sequence[Path]) -> dict[str, list[Path]]: return images +def _discover_masks(files: Sequence[Path], images: Mapping[str, Sequence[Path]]) -> dict[str, Path]: + """The masks among ``files`` (``_mask.tif`` or ``_labels``, also ``.ome.tiff``) of the regions of ``images``. + + A mask belongs to the region in its file name (``regNNN``), else to the region of the image it is named after. + """ + by_image = {_TIFF_NAME.sub("", file.name): region for region, paths in images.items() for file in paths} + masks: dict[str, list[Path]] = {} + for file in files: + region = region_id(file.name) or by_image.get(_MASK.sub("", file.name)) + if region is None or region not in images: + warnings.warn( + f"Ignoring the mask {file}: it is named after no region image. Pass it as `labels` to read it.", + UserWarning, + stacklevel=4, + ) + continue + masks.setdefault(region, []).append(file) + for region, paths in masks.items(): + if len(paths) > 1: + raise ValueError( + f"Found several masks for region '{region}': {[str(p) for p in paths]}. Pass the mask to read as " + "`labels`." + ) + return {region: paths[0] for region, paths in sorted(masks.items())} + + def _preferred_stacks(files: list[Path]) -> list[Path]: for pattern in (_MONTAGE, _TILE_STACK): matching = [file for file in files if pattern.search(file.name)] diff --git a/tests/test_codex.py b/tests/test_codex.py index eeddd445..39ca9b0f 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -347,6 +347,50 @@ def test_codex_explicit_inputs_disable_discovery(tmp_path: Path) -> None: assert not sdata.tables +@pytest.mark.parametrize( + ("image", "mask", "region"), + [ + ("expr/reg001_expr.ome.tiff", "mask/reg001_mask.ome.tiff", "reg001"), + ("Kidney sample.tif", "Kidney sample_labels.tif", "Kidney_sample"), + ], +) +def test_codex_discovers_the_mask_of_a_region(tmp_path: Path, image: str, mask: str, region: str) -> None: + for name in (image, mask): + (tmp_path / name).parent.mkdir(exist_ok=True) + tifffile.imwrite(tmp_path / image, np.zeros((2, 16, 16), dtype=np.uint16)) + tifffile.imwrite(tmp_path / mask, np.arange(256, dtype=np.uint16).reshape(16, 16)) + (tmp_path / "cells.csv").write_text("cell_id,x,y,cyc001_ch001\n1,0.0,0.0,1.0\n2,1.0,0.0,2.0\n") + + sdata = codex(tmp_path, image_models_kwargs=_FAST, labels_models_kwargs=_FAST) + + assert list(sdata.images) == [f"{region}_image"] + assert list(sdata.labels) == [f"{region}_labels"] + assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == [f"{region}_labels"] + assert sdata.attrs["akoya"]["labels"] == {region: str(tmp_path / mask)} + + # an explicit mask wins + other = tmp_path / "other.tif" + tifffile.imwrite(other, np.ones((16, 16), dtype=np.uint16)) + sdata = codex(tmp_path, labels=other, image_models_kwargs=_FAST, labels_models_kwargs=_FAST) + assert list(sdata.images) == [f"{region}_image"] + assert sdata.attrs["akoya"]["labels"] == {region: str(other)} + + +def test_codex_masks_of_no_region_or_one_region_twice(tmp_path: Path) -> None: + tifffile.imwrite(tmp_path / "reg001.tif", np.zeros((2, 16, 16), dtype=np.uint16)) + tifffile.imwrite(tmp_path / "Other_mask.tif", np.zeros((16, 16), dtype=np.uint16)) + + with pytest.warns(UserWarning, match="Ignoring the mask .*Other_mask.tif"): + sdata = codex(tmp_path, image_models_kwargs=_FAST) + assert list(sdata.images) == ["reg001_image"] + assert not sdata.labels + + for name in ("reg001_mask.tif", "reg001_labels.tif"): + tifffile.imwrite(tmp_path / name, np.zeros((16, 16), dtype=np.uint16)) + with pytest.raises(ValueError, match="several masks for region 'reg001'"): + codex(tmp_path, image_models_kwargs=_FAST) + + def test_codex_regions_of_regionless_table_are_available(tmp_path: Path) -> None: data = write_legacy_processor_tree(tmp_path, regions=1, fcs=False, csv=False, n_cells=5) pd.concat([data["cells"], data["cells"].assign(region=2.0)]).to_csv(tmp_path / "cells.csv", index=False) From 73fdf2c6c0f10aed00c13bf4796f60937578c375 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 14:00:42 +0200 Subject: [PATCH 091/107] docs(changelog): note the regNNN ids of qptiffs in both readers --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12020f40..8963bf6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,9 +82,10 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. tables are preferred; other tables of the same region (e.g. of two segmentation runs) raise a `ValueError`. A `cells_table` disables table discovery, and the `labels` masks are not read as images. Without `labels`, a `.tif(f)` ending in `_mask` or `_labels` and named after a region (e.g. `reg001_mask.ome.tiff`) is its mask. - - Region ids `regNNN` are taken from `reg` and digits not preceded by a letter (`exp_reg001`, not `Treg2`). A CIM - `.qptiff` without one is named as `phenocycler()` names it (without the suffix and a trailing `_Scan`), and its - `.er.qptiff` is preferred. Top-level `.tif` files of the original layout are named after their sanitised stems. + - Region ids `regNNN` are taken from `reg` and digits not preceded by a letter (`exp_reg001`, not `Treg2`), also + by `phenocycler()` for a `.qptiff`. A CIM `.qptiff` without one is named as `phenocycler()` names it (without the + suffix and a trailing `_Scan`), and its `.er.qptiff` is preferred. Top-level `.tif` files of the original + layout are named after their sanitised stems. - Hidden files and folders (e.g. macOS `._*` files, `__MACOSX/`, `.Trash-*/`) are ignored and symlinked folders are followed, by `phenocycler()` too. From 21ef910ff7b7bb4d8be4e7f7cb3c3ae68614b3c8 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 14:44:22 +0200 Subject: [PATCH 092/107] fix(codex): apply cells_table_kwargs to discovered cell tables --- CHANGELOG.md | 3 ++- src/spatialdata_io/readers/codex.py | 19 +++++++++++++------ tests/test_codex.py | 11 +++++++++++ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8963bf6c..e99a72f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,7 +80,8 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. - `channelNames.txt` and `experiment.json` are found in any case (e.g. SPACEc's `channelnames.txt`); CSV and FCS files that are not CODEX Processor cell tables are ignored, also when their name holds a region id. Compensated tables are preferred; other tables of the same region (e.g. of two segmentation runs) raise a `ValueError`. - A `cells_table` disables table discovery, and the `labels` masks are not read as images. Without `labels`, a + A `cells_table` disables table discovery, and the `labels` masks are not read as images. `cells_table_kwargs` + (e.g. `markers`, `region_column`) apply to the discovered tables too. Without `labels`, a `.tif(f)` ending in `_mask` or `_labels` and named after a region (e.g. `reg001_mask.ome.tiff`) is its mask. - Region ids `regNNN` are taken from `reg` and digits not preceded by a letter (`exp_reg001`, not `Treg2`), also by `phenocycler()` for a `.qptiff`. A CIM `.qptiff` without one is named as `phenocycler()` names it (without the diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index f30fc3b8..e0f1d3b5 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -207,8 +207,8 @@ def codex( labels_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`. cells_table_kwargs - Options for reading ``cells_table`` and ``labels``: ``region_column``; for ``"generic"`` tables ``x``, - ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath and CODEX tables), + Options for reading ``cells_table`` (or the discovered cell tables) and ``labels``: ``region_column``; for + ``"generic"`` tables ``x``, ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath and CODEX tables), ``coordinates_unit`` (``"px"`` or ``"um"``; for QuPath ``X``/``Y`` and inForm ``Cell X Position`` columns without a unit, which are otherwise read in the unit of the table's other columns) and ``coordinates_space`` (``"image"``, default, or ``"mask"``: whether the pixel centroids of a generic table @@ -240,7 +240,9 @@ def codex( path, fcs, find_tables=cells_table is None, masks={Path(mask).resolve() for mask in masks} ) # the regions of a table without a region in its file name are known only once it is read - regionless = _read_regionless_table(layout.tables["*"], layout) if "*" in layout.tables else None + regionless = ( + _read_regionless_table(layout.tables["*"], layout, cells_table_kwargs) if "*" in layout.tables else None + ) table_regions = set(regionless.region.unique()) if regionless is not None else set() available = sorted(layout.images.keys() | layout.tables.keys() - {"*"} | table_regions) selected = select_regions(available, regions) @@ -303,7 +305,7 @@ def codex( available={region: layout.images.get(region, []) for region in available}, selected=selected, pixel_sizes={}, - cells_table_kwargs={}, + cells_table_kwargs=cells_table_kwargs, ) read = dict(zip(named_files, named, strict=True)) if regionless is not None: @@ -341,13 +343,18 @@ def _warn_fcs_fallback(files: Iterable[Path], fcs: bool) -> None: ) -def _read_regionless_table(file: Path, layout: _LegacyLayout) -> CellTable: +def _read_regionless_table(file: Path, layout: _LegacyLayout, cells_table_kwargs: Mapping[str, Any]) -> CellTable: """Read the discovered cell table ``file`` that has no region in its file name.""" only_image = next(iter(layout.images)) if len(layout.images) == 1 else None regions = {region: layout.images.get(region, []) for region in layout.images.keys() | layout.tables.keys() - {"*"}} # next to a single image named after its file (e.g. `Kidney.tif`), a single region value names that image table = read_cell_table( - file, "codex", pixel_size_um=None, default_region=only_image, cells_table_kwargs={}, regions=regions + file, + "codex", + pixel_size_um=None, + default_region=only_image, + cells_table_kwargs=cells_table_kwargs, + regions=regions, ) # values that name no region of the dataset are regions without an image table.region = table.region.map({value: sanitize_name(value) for value in table.region.unique()}) diff --git a/tests/test_codex.py b/tests/test_codex.py index 39ca9b0f..871ee181 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -857,3 +857,14 @@ def test_codex_discovered_table_of_another_region_raises(tmp_path: Path) -> None with pytest.raises(ValueError, match="has no cells of region 'reg002'"): codex(tmp_path, image_models_kwargs=_FAST) + + +def test_codex_cells_table_kwargs_apply_to_discovered_tables(tmp_path: Path) -> None: + tifffile.imwrite(tmp_path / "reg001_montage.tif", np.zeros((2, 16, 16), dtype=np.uint16)) + (tmp_path / "reg001_cells.csv").write_text("cell_id,x,y,CD3,CD4\n1,1.0,2.0,3.0,4.0\n2,3.0,4.0,5.0,6.0\n") + + with pytest.warns(UserWarning, match="circles") as record: + sdata = codex(tmp_path, cells_table_kwargs={"markers": ["CD3"]}, image_models_kwargs=_FAST) + + assert list(sdata.tables["table"].var_names) == ["CD3"] + assert not [warning for warning in record if "cycCCC_chHHH" in str(warning.message)] From 56c8f8b665ed341fcaf525b1a6a1147381d1863e Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 14:45:32 +0200 Subject: [PATCH 093/107] test(codex): a region-less discovered table reads with cells_table_kwargs['region_column'] --- tests/test_codex.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_codex.py b/tests/test_codex.py index 871ee181..1f2ab0f9 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -868,3 +868,17 @@ def test_codex_cells_table_kwargs_apply_to_discovered_tables(tmp_path: Path) -> assert list(sdata.tables["table"].var_names) == ["CD3"] assert not [warning for warning in record if "cycCCC_chHHH" in str(warning.message)] + + +def test_codex_region_column_of_discovered_regionless_table(tmp_path: Path) -> None: + for region in ("reg001", "reg002"): + tifffile.imwrite(tmp_path / f"{region}_montage.tif", np.zeros((2, 16, 16), dtype=np.uint16)) + (tmp_path / "cells.csv").write_text("cell_id,Sample,x,y,cyc001_ch001\n1,reg001,1.0,2.0,3.0\n1,reg002,3.0,4.0,5.0\n") + + with pytest.raises(ValueError, match=r"cannot be assigned to regions.*cells_table_kwargs\['region_column'\]"): + codex(tmp_path, image_models_kwargs=_FAST) + + with pytest.warns(UserWarning, match="circles"): + sdata = codex(tmp_path, cells_table_kwargs={"region_column": "Sample"}, image_models_kwargs=_FAST) + assert sorted(sdata.shapes) == ["reg001_cells", "reg002_cells"] + assert sdata.tables["table"].n_obs == 2 From 2fa14dd0655ac621e9334688c90eb96d30907d83 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 14:47:29 +0200 Subject: [PATCH 094/107] fix(codex): skip mask discovery when labels are given; split masks from images in linear time --- src/spatialdata_io/readers/codex.py | 21 +++++++++++++++++---- tests/test_codex.py | 20 ++++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index e0f1d3b5..6d194f63 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -237,7 +237,11 @@ def codex( # explicit beats discovered: a given table disables table discovery, and a given mask is no image masks = labels.values() if isinstance(labels, Mapping) else [] if labels is None else [labels] layout = _discover_legacy( - path, fcs, find_tables=cells_table is None, masks={Path(mask).resolve() for mask in masks} + path, + fcs, + find_tables=cells_table is None, + find_masks=labels is None, + masks={Path(mask).resolve() for mask in masks}, ) # the regions of a table without a region in its file name are known only once it is read regionless = ( @@ -362,7 +366,12 @@ def _read_regionless_table(file: Path, layout: _LegacyLayout, cells_table_kwargs def _discover_legacy( - path: Path, fcs: bool, *, find_tables: bool = True, masks: Collection[Path] = frozenset() + path: Path, + fcs: bool, + *, + find_tables: bool = True, + find_masks: bool = True, + masks: Collection[Path] = frozenset(), ) -> _LegacyLayout: """Find the images, cell tables and sidecar files of a CODEX dataset. @@ -374,6 +383,8 @@ def _discover_legacy( Whether to prefer ``.fcs`` over ``.csv`` tables of the same region. find_tables Whether to discover cell tables. + find_masks + Whether to discover the segmentation masks of the regions. masks The resolved paths of the segmentation masks, which are not images. @@ -387,7 +398,9 @@ def _discover_legacy( if masks: files = [file for file in files if file.resolve() not in masks] # masks named after their region are never images - image_files = [file for file in files if not (_is_tiff(file) and _MASK.search(file.name))] + named_masks = [_is_tiff(file) and _MASK.search(file.name) is not None for file in files] + image_files = [file for file, is_mask in zip(files, named_masks, strict=True) if not is_mask] + mask_files = [file for file, is_mask in zip(files, named_masks, strict=True) if is_mask] images = _discover_images(image_files) if not images: # the original (2019) layout: one stitched `.tif` next to its table, without a region in the file name @@ -399,7 +412,7 @@ def _discover_legacy( return _LegacyLayout( images=dict(sorted(images.items())), tables=dict(sorted(tables.items())), - masks=_discover_masks([file for file in files if file not in image_files], images), + masks=_discover_masks(mask_files, images) if find_masks else {}, channel_names_txt=_find_sidecar(path, files, CodexKeys.CHANNEL_NAMES_FILE.value), experiment_json=_find_sidecar(path, files, CodexKeys.EXPERIMENT_FILE.value), ) diff --git a/tests/test_codex.py b/tests/test_codex.py index 1f2ab0f9..2e079798 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -4,6 +4,7 @@ import json import shutil +import warnings from typing import TYPE_CHECKING, Literal import numpy as np @@ -882,3 +883,22 @@ def test_codex_region_column_of_discovered_regionless_table(tmp_path: Path) -> N sdata = codex(tmp_path, cells_table_kwargs={"region_column": "Sample"}, image_models_kwargs=_FAST) assert sorted(sdata.shapes) == ["reg001_cells", "reg002_cells"] assert sdata.tables["table"].n_obs == 2 + + +def test_codex_labels_disable_mask_discovery(tmp_path: Path) -> None: + for region in ("reg001", "reg002"): + tifffile.imwrite(tmp_path / f"{region}_montage.tif", np.zeros((2, 16, 16), dtype=np.uint16)) + for name in ("reg002_mask.tif", "reg002_labels.tif", "Other_mask.tif", "mask.tif"): + tifffile.imwrite(tmp_path / name, np.ones((16, 16), dtype=np.uint16)) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + sdata = codex( + tmp_path, + labels={"reg001": tmp_path / "mask.tif"}, + image_models_kwargs=_FAST, + labels_models_kwargs=_FAST, + ) + + assert sorted(sdata.images) == ["reg001_image", "reg002_image"] + assert list(sdata.labels) == ["reg001_labels"] From 2a81ae560b90fabb1e7b7c15de92f04ec352d889 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 14:49:30 +0200 Subject: [PATCH 095/107] fix(codex): match the masks of qptiff regions by region id and image name --- src/spatialdata_io/readers/_utils/_akoya.py | 3 +++ src/spatialdata_io/readers/_utils/_cell_tables.py | 7 +++---- src/spatialdata_io/readers/codex.py | 14 +++++++++----- tests/test_codex.py | 12 ++++++++++++ 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index 32ee7ccf..e201e7b3 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -33,6 +33,7 @@ from spatialdata_io.readers._utils._qptiff import AkoyaScan __all__ = [ + "IMAGE_EXTENSION", "apply_channel_names", "build_image_element", "dedupe_channel_names", @@ -47,6 +48,8 @@ "spatial_chunks", ] +# the extensions of image files, also named in the region column of a table (e.g. QuPath's `Image`) +IMAGE_EXTENSION = re.compile(r"(?:\.er)?\.(?:qptiff|ome\.tiff?|tiff?)$", re.IGNORECASE) # `reg001`, `exp_reg001` or `1_reg001`, but not `Treg2` or `coreg1` _REGION = re.compile(r"(? str | No candidates = list(dict.fromkeys((value, sanitize_name(value)))) if exact := next((candidate for candidate in candidates if candidate in regions), None): return exact - stem = _IMAGE_SUFFIX.sub("", value) + stem = IMAGE_EXTENSION.sub("", value) for region, files in regions.items(): for name in (file.name for file in files): - if stem == _IMAGE_SUFFIX.sub("", name) or (value.startswith(name) and not value[len(name)].isalnum()): + if stem == IMAGE_EXTENSION.sub("", name) or (value.startswith(name) and not value[len(name)].isalnum()): return region return None diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 6d194f63..6f19bf0a 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -16,6 +16,7 @@ from spatialdata_io._constants._constants import CodexKeys from spatialdata_io._docs import inject_docs from spatialdata_io.readers._utils._akoya import ( + IMAGE_EXTENSION, apply_channel_names, normalize_regions, read_json_object, @@ -59,7 +60,6 @@ _TIFF_SUFFIXES = (".tif", ".tiff") # the segmentation mask of a region, e.g. HuBMAP's `reg001_mask.ome.tiff` next to `reg001_expr.ome.tiff` _MASK = re.compile(r"[_-](?:mask|labels)(?:\.ome)?\.tiff?$", re.IGNORECASE) -_TIFF_NAME = re.compile(r"(?:\.ome)?\.tiff?$", re.IGNORECASE) _TABLE_SUFFIXES = (CodexKeys.FCS_FILE.value, CodexKeys.FCS_FILE_CSV.value) # deep enough for `/processed_/segm/segm-1/fcs/compensated/
` _MAX_DEPTH = 8 @@ -190,8 +190,9 @@ def codex( labels A segmentation mask (TIFF/OME-TIFF with integer instance ids) per region: a path for a single region, or a mapping from region id to path; a mask inside ``path`` is not read as an image. If `None`, a TIFF named after - a region image or with its region id, ending in ``_mask`` or ``_labels`` (e.g. ``reg001_mask.ome.tiff`` or - ``Kidney_labels.tif`` next to ``Kidney.tif``), is the mask of that region; such files are never read as images. + a region image or with its region id, ending in ``_mask`` or ``_labels`` (e.g. ``reg001_mask.ome.tiff``, + ``Kidney_labels.tif`` next to ``Kidney.tif``, or ``Tonsil_mask.tif`` or ``Tonsil_Scan1_mask.tif`` next to + ``Tonsil_Scan1.qptiff``), is the mask of that region; such files are never read as images. Each mask is stored lazily as the labels ``_labels`` with the transformation of the region's image; the cells of the region annotate it instead of circles, and their instance ids must be the mask values. A mask whose shape is the region image's @@ -552,9 +553,12 @@ def _discover_images(files: Sequence[Path]) -> dict[str, list[Path]]: def _discover_masks(files: Sequence[Path], images: Mapping[str, Sequence[Path]]) -> dict[str, Path]: """The masks among ``files`` (``_mask.tif`` or ``_labels``, also ``.ome.tiff``) of the regions of ``images``. - A mask belongs to the region in its file name (``regNNN``), else to the region of the image it is named after. + A mask belongs to the region in its file name (``regNNN``), else to the region it is named after: the region id + (e.g. ``Tonsil_mask.tif``) or the file name of its image (``Tonsil_Scan1_mask.tif`` of ``Tonsil_Scan1.qptiff``). """ - by_image = {_TIFF_NAME.sub("", file.name): region for region, paths in images.items() for file in paths} + by_image = {region: region for region in images} | { + IMAGE_EXTENSION.sub("", file.name): region for region, paths in images.items() for file in paths + } masks: dict[str, list[Path]] = {} for file in files: region = region_id(file.name) or by_image.get(_MASK.sub("", file.name)) diff --git a/tests/test_codex.py b/tests/test_codex.py index 2e079798..aeaed1e5 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -902,3 +902,15 @@ def test_codex_labels_disable_mask_discovery(tmp_path: Path) -> None: assert sorted(sdata.images) == ["reg001_image", "reg002_image"] assert list(sdata.labels) == ["reg001_labels"] + + +@pytest.mark.parametrize("mask", ["Tonsil_mask.tif", "Tonsil_Scan1_mask.tif"]) +def test_codex_discovers_the_mask_of_a_qptiff_region(tmp_path: Path, mask: str) -> None: + write_qptiff(tmp_path / "Tonsil_Scan1.er.qptiff", biomarkers=["DAPI"], shape=(16, 16)) + tifffile.imwrite(tmp_path / mask, np.ones((16, 16), dtype=np.uint16)) + + sdata = codex(tmp_path, image_models_kwargs=_FAST, labels_models_kwargs=_FAST) + + assert list(sdata.images) == ["Tonsil_image"] + assert list(sdata.labels) == ["Tonsil_labels"] + assert sdata.attrs["akoya"]["labels"] == {"Tonsil": str(tmp_path / mask)} From 52ae10a6375f9a8dd7eaa41e6b740a4b149eaa76 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 14:53:40 +0200 Subject: [PATCH 096/107] fix(akoya): concatenate tables whose markers differ only in their source columns --- CHANGELOG.md | 3 ++- .../readers/_utils/_cell_tables.py | 20 +++++++++++++++---- tests/test_codex.py | 20 +++++++++++++++++++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e99a72f2..c5de7a60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,7 +81,8 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. files that are not CODEX Processor cell tables are ignored, also when their name holds a region id. Compensated tables are preferred; other tables of the same region (e.g. of two segmentation runs) raise a `ValueError`. A `cells_table` disables table discovery, and the `labels` masks are not read as images. `cells_table_kwargs` - (e.g. `markers`, `region_column`) apply to the discovered tables too. Without `labels`, a + (e.g. `markers`, `region_column`) apply to the discovered tables too. The CSV and FCS tables of one marker panel (e.g. with + `fcs=False` and a region with only an FCS table) share one `table`, without `var["source_column"]`. Without `labels`, a `.tif(f)` ending in `_mask` or `_labels` and named after a region (e.g. `reg001_mask.ome.tiff`) is its mask. - Region ids `regNNN` are taken from `reg` and digits not preceded by a letter (`exp_reg001`, not `Treg2`), also by `phenocycler()` for a `.qptiff`. A CIM `.qptiff` without one is named as `phenocycler()` names it (without the diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index f0187d9f..051884c2 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -1633,7 +1633,8 @@ def build_cells( Returns ------- - The table elements, ``{"table": ...}`` if all tables have the same ``var``, else ``{"_table": ...}`` + The table elements, ``{"table": ...}`` if all tables have the same ``var`` apart from its ``source_column`` + (dropped if it differs), else ``{"_table": ...}`` per region, and the circle shapes by element name. The circles have the cells' area (radius 1 px without a positive one). """ @@ -1653,8 +1654,10 @@ def build_cells( UserWarning, stacklevel=3, ) - # tables with other markers or other marker metadata (e.g. QuPath compartments) are not concatenated - pieces = _split_by_region(tables) if any(not table.var.equals(tables[0].var) for table in tables) else None + # tables with other markers or other marker metadata (e.g. QuPath compartments) are not concatenated; the source + # columns only record where a marker was read from (e.g. `cyc001_ch001:DAPI1` of a CSV, `cyc001_ch001` of an FCS) + panels = [_panel(table.var) for table in tables] + pieces = _split_by_region(tables) if any(not panel.equals(panels[0]) for panel in panels) else None def element(region: str) -> str: if region in region_elements: @@ -1837,6 +1840,11 @@ def _check_disjoint_regions(tables: Sequence[CellTable]) -> None: seen |= regions +def _panel(var: pd.DataFrame) -> pd.DataFrame: + """The marker metadata of ``var`` without the provenance column ``source_column``.""" + return var.drop(columns="source_column", errors="ignore") + + def _split_by_region(tables: Sequence[CellTable]) -> dict[str, CellTable]: return {region: table.take(rows) for table in tables for region, rows in _region_rows(table).items()} @@ -1860,7 +1868,11 @@ def _to_anndata(tables: Sequence[CellTable], element: Callable[[str], str]) -> a if not obs.index.is_unique: obs.index = pd.Index(regions.to_numpy(dtype=object) + "_" + obs.index.to_numpy(dtype=object)) X = np.concatenate([table.X for table in tables]) if len(tables) > 1 else tables[0].X - adata = ad.AnnData(X=X, obs=obs, var=tables[0].var.copy()) + var = tables[0].var.copy() + if any(not table.var.equals(tables[0].var) for table in tables): + # the same markers read from columns of other names + var = _panel(var) + adata = ad.AnnData(X=X, obs=obs, var=var) centroids = [table.centroids_px for table in tables if table.centroids_px is not None] if len(centroids) == len(tables): adata.obsm[CodexKeys.SPATIAL_KEY.value] = np.concatenate(centroids) diff --git a/tests/test_codex.py b/tests/test_codex.py index aeaed1e5..7ad9e9c2 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -914,3 +914,23 @@ def test_codex_discovers_the_mask_of_a_qptiff_region(tmp_path: Path, mask: str) assert list(sdata.images) == ["Tonsil_image"] assert list(sdata.labels) == ["Tonsil_labels"] assert sdata.attrs["akoya"]["labels"] == {"Tonsil": str(tmp_path / mask)} + + +def test_codex_csv_and_fcs_tables_of_one_panel_share_a_table(tmp_path: Path) -> None: + data = write_legacy_processor_tree(tmp_path, regions=2, layout="per_marker", fcs=True, csv=True, n_cells=5) + next(tmp_path.rglob("reg002_compensated.csv")).unlink() + # the CODEX Processor names the markers in the CSV header, e.g. `cyc001_ch001:DAPI` + csv = next(tmp_path.rglob("reg001_compensated.csv")) + cells = pd.read_csv(csv) + names = dict(zip(cells.filter(regex="^cyc").columns, data["markers"], strict=True)) + cells.rename(columns={column: f"{column}:{marker}" for column, marker in names.items()}).to_csv(csv, index=False) + + with pytest.warns(UserWarning, match="CSV"): + sdata = codex(tmp_path, fcs=False, image_models_kwargs=_FAST) + + assert list(sdata.tables) == ["table"] + table = sdata.tables["table"] + assert list(table.obs["region"].cat.categories) == ["reg001_cells", "reg002_cells"] + assert list(table.var_names) == dedupe_channel_names(data["markers"]) + # the markers were read from columns of other names + assert "source_column" not in table.var From f4e79cdb0b4c421cb145828c4aaa04b40a8e1709 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 14:55:40 +0200 Subject: [PATCH 097/107] fix(akoya): bound the pixels rasterised at once for large polygons; draw only polygonal parts --- .../readers/_utils/_rasterize.py | 35 ++++++++++++++----- tests/readers/test_rasterize.py | 34 ++++++++++++++++++ 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_rasterize.py b/src/spatialdata_io/readers/_utils/_rasterize.py index 386f6b35..6432ea31 100644 --- a/src/spatialdata_io/readers/_utils/_rasterize.py +++ b/src/spatialdata_io/readers/_utils/_rasterize.py @@ -18,13 +18,15 @@ __all__ = ["rasterize_polygons_lazy"] _MAX_ID = np.iinfo(np.uint32).max -# the polygon pixels tested at once, bounding the memory of a block's temporaries (~25 MB) +_POLYGONAL = (shapely.GeometryType.POLYGON, shapely.GeometryType.MULTIPOLYGON) +# the polygon pixels tested at once (at least one row of a polygon), bounding the memory of a block's temporaries +# (~25 MB) _BATCH_PIXELS = 1 << 18 _TOKEN_BATCH = 1 << 16 def rasterize_polygons_lazy( - polygons: Sequence[shapely.Polygon | shapely.MultiPolygon], + polygons: Sequence[shapely.Geometry], ids: Sequence[int], shape: tuple[int, int], chunks: tuple[int, int] = (4096, 4096), @@ -40,7 +42,7 @@ def rasterize_polygons_lazy( Parameters ---------- polygons - The polygons, in pixels. + The polygons, in pixels; of other geometries (e.g. a ``GeometryCollection``) only the polygons are drawn. ids The label of each polygon, between 1 and ``2**32 - 1``. shape @@ -57,7 +59,7 @@ def rasterize_polygons_lazy( labels = np.asarray(ids, dtype=np.int64) if len(labels) and (labels.min() < 1 or labels.max() > _MAX_ID): raise ValueError(f"Polygon ids must be between 1 and {_MAX_ID}; got {labels.min()} to {labels.max()}.") - geometries = np.asarray(polygons, dtype=object) + geometries = _polygonal(np.array(polygons, dtype=object)) labels = labels.astype(np.uint32) sizes = normalize_chunks(chunks, shape, dtype=np.uint32) name = f"rasterize-polygons-{_token(geometries, labels, sizes)}" @@ -84,6 +86,14 @@ def rasterize_polygons_lazy( return da.Array(graph, name, sizes, meta=np.empty((0, 0), dtype=np.uint32)) +def _polygonal(geometries: np.ndarray) -> np.ndarray: + """``geometries`` with only the polygons of collections; lines and points cover no pixels.""" + for i in np.flatnonzero(~np.isin(shapely.get_type_id(geometries), _POLYGONAL)): + parts = shapely.get_parts(shapely.get_parts(geometries[i])) + geometries[i] = shapely.multipolygons(parts[shapely.get_type_id(parts) == shapely.GeometryType.POLYGON]) + return geometries + + def _token(geometries: np.ndarray, labels: np.ndarray, *args: Any) -> str: """A deterministic token of the polygons, hashing their WKB in batches to bound the memory.""" wkb = hashlib.blake2b(digest_size=16) @@ -106,14 +116,23 @@ def _draw_block( minx, miny, maxx, maxy = shapely.bounds(geometries).T r0, r1 = np.clip([np.floor(miny - 0.5), np.ceil(maxy - 0.5) + 1], 0, shape[0]).astype(np.int64) c0, c1 = np.clip([np.floor(minx - 0.5), np.ceil(maxx - 0.5) + 1], 0, shape[1]).astype(np.int64) - width = c1 - c0 - sizes = np.maximum(r1 - r0, 0) * np.maximum(width, 0) - firsts = np.cumsum(sizes) - sizes + # a polygon larger than a batch is tested in bands of rows + width = np.maximum(c1 - c0, 0) + band = np.maximum(_BATCH_PIXELS // np.maximum(width, 1), 1) + n_bands = np.maximum(-(-(r1 - r0) // band), 1) + of = np.repeat(np.arange(len(geometries)), n_bands) + nth = np.arange(len(of)) - np.repeat(np.cumsum(n_bands) - n_bands, n_bands) + r0 = r0[of] + nth * band[of] + r1, c0, width = np.minimum(r0 + band[of], r1[of]), c0[of], width[of] + geometries, labels = geometries[of], labels[of] + sizes = np.maximum(r1 - r0, 0) * width + ends = np.cumsum(sizes) + firsts = ends - sizes flat = out.reshape(-1) # all candidate pixels of a batch of polygons are tested in one GEOS call, which releases the GIL start = 0 while start < len(geometries): - stop = max(int(np.searchsorted(firsts, firsts[start] + _BATCH_PIXELS)), start + 1) + stop = max(int(np.searchsorted(ends, firsts[start] + _BATCH_PIXELS, side="right")), start + 1) owner = np.repeat(np.arange(start, stop), sizes[start:stop]) offset = np.arange(len(owner)) - (firsts[owner] - firsts[start]) rows = r0[owner] + offset // width[owner] diff --git a/tests/readers/test_rasterize.py b/tests/readers/test_rasterize.py index e981893e..f7c5205b 100644 --- a/tests/readers/test_rasterize.py +++ b/tests/readers/test_rasterize.py @@ -102,6 +102,40 @@ def test_rasterize_multipolygon_and_empty() -> None: assert not rasterize_polygons_lazy([], [], shape=(5, 6)).compute().any() +def test_rasterize_bounds_the_pixels_of_a_large_polygon(monkeypatch: pytest.MonkeyPatch) -> None: + tested = [] + + def counting_intersects_xy(geometries: np.ndarray, x: np.ndarray, y: np.ndarray) -> np.ndarray: + tested.append(len(x)) + return intersects_xy(geometries, x, y) + + intersects_xy = shapely.intersects_xy + polygons = [shapely.Polygon([(-1.3, -2.2), (40.5, 3.1), (37.2, 41.7), (2.4, 30.3)]), shapely.box(3, 3, 9, 9)] + expected = rasterize_polygons_lazy(polygons, [1, 2], shape=(32, 32), chunks=(32, 32)).compute() + monkeypatch.setattr(_rasterize, "_BATCH_PIXELS", 100) + monkeypatch.setattr(_rasterize.shapely, "intersects_xy", counting_intersects_xy) + + # the first polygon covers the whole chunk: its pixels are tested in bands of rows + labels = rasterize_polygons_lazy(polygons, [1, 2], shape=(32, 32), chunks=(32, 32)).compute() + + np.testing.assert_array_equal(labels, expected) + assert max(tested) <= 100 + assert sum(tested) == 32 * 32 + 8 * 8 + + +def test_rasterize_draws_only_polygonal_parts() -> None: + collection = shapely.GeometryCollection( + [shapely.box(0, 0, 2, 2), shapely.LineString([(4, 0.5), (7, 0.5)]), shapely.Point(6.5, 6.5)] + ) + line = shapely.LineString([(0.5, 5), (0.5, 7)]) + + labels = rasterize_polygons_lazy([collection, line], [1, 2], shape=(8, 8), chunks=(4, 4)).compute() + + expected = np.zeros((8, 8), dtype=np.uint32) + expected[:2, :2] = 1 + np.testing.assert_array_equal(labels, expected) + + def test_rasterize_errors() -> None: with pytest.raises(ValueError, match="2 polygons but 1 ids"): rasterize_polygons_lazy([shapely.box(0, 0, 1, 1)] * 2, [1], shape=(4, 4)) From 3d5ddfadf8e235657d136f2ebb7de287a74e3dc5 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 14:57:23 +0200 Subject: [PATCH 098/107] fix(akoya): keep the file row numbers as instance ids when cells without a region are dropped --- .../readers/_utils/_cell_tables.py | 17 +++++++++++------ tests/readers/test_cell_tables.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 051884c2..22ebd4ff 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -289,7 +289,7 @@ def read_cell_table( ``pixel_size_um``) or ``Centroid X px``/``Centroid Y px``, else ``X``/``Y`` in ``coordinates_unit`` or, without it, in the unit of the other measurements (e.g. ``Cell: Area µm^2``) with a `UserWarning`, raising if that is not unique. - Instance ids from a numeric ``Object ID``, else the row number (from 1); the region from the + Instance ids from a numeric ``Object ID``, else the row number in the file (from 1); the region from the ``Image`` column. The markers are ``markers``, else the ``: : Mean`` columns (also ``....Mean`` and ``__Mean``), ``: : Mean`` (QuPath >= 0.6) and ``: mean`` (built-in cell detection) @@ -337,7 +337,7 @@ def read_cell_table( cells_table_kwargs Optional keys: ``region_column`` (the column holding the region of each cell; QuPath default ``Image``), and for ``"generic"`` tables ``x`` and ``y`` (required centroid columns), ``instance_id`` (else the row - number, from 1), ``markers`` (a list of columns or a regular expression matching the full column name; + number in the file, from 1), ``markers`` (a list of columns or a regular expression matching the full column name; default: all other numeric columns; also for ``"qupath"`` and ``"codex"`` tables) and ``coordinates_unit`` (``"px"``, default, or ``"um"``; for ``"qupath"`` and ``"inform"`` tables the unit of centroid columns whose name has none) and ``coordinates_space`` (the pixels of ``"px"`` centroids: ``"image"``, default, those of the @@ -378,12 +378,14 @@ def read_cell_table( mask_pixels = fmt in {"mcquant", "sprm"} if fmt == "generic": mask_pixels = _coordinates_space(path, roles["in_um"], cells_table_kwargs) == "mask" + file_rows = None if roles["region_column"] in df.columns and (empty := df[roles["region_column"]].isna().to_numpy()).any(): warnings.warn( f"Ignoring the {empty.sum()} cell(s) of {path} without a region in '{roles['region_column']}'.", UserWarning, stacklevel=4, ) + file_rows = np.flatnonzero(~empty) df = df.loc[~empty].reset_index(drop=True) region = _region_values(df, roles["region_column"], default_region, regions) units = cells_table_kwargs.get("geojson_units", "px") @@ -397,7 +399,7 @@ def read_cell_table( "pixel size is missing)." ) roles["px_per_unit"] = 1 / row_pixel_sizes - table = _from_frame(path, df, region=region, **roles) + table = _from_frame(path, df, region=region, file_rows=file_rows, **roles) table.mask_pixels = mask_pixels if fmt == "sprm": table.centroids_px = _read_sprm_centers(path, table) @@ -1040,10 +1042,12 @@ def _from_frame( area_column: str | None, region: pd.Series | None, px_per_unit: float | np.ndarray = 1.0, + file_rows: np.ndarray | None = None, ) -> CellTable: """Build a `CellTable` from a data frame, the roles of its columns and the region of every row. - ``px_per_unit`` may be given per row. + ``px_per_unit`` may be given per row. ``file_rows`` are the rows of the file that ``df`` holds (from 0), if it + holds not all of them. """ roles = [x, y, instance_column, region_column, area_column, *markers] missing = [column for column in roles if column is not None and column not in df.columns] @@ -1053,8 +1057,9 @@ def _from_frame( if instance_column is not None: instance_id = pd.Series(_instance_ids(path, df[instance_column])) else: - # label images use 0 for the background, so instance ids start at 1 - instance_id = pd.Series(np.arange(1, len(df) + 1, dtype=np.int64)) + # the row numbers in the file; label images use 0 for the background, so instance ids start at 1 + rows = np.arange(len(df)) if file_rows is None else file_rows + instance_id = pd.Series(rows.astype(np.int64) + 1) index = pd.Index(instance_id.astype(str)) instance_id.index = index instance_id.name = CodexKeys.INSTANCE_KEY.value diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index b7ded99a..56b22662 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -415,6 +415,20 @@ def test_generic_kwargs(tmp_path: Path) -> None: assert list(listed.instance_id) == [1, 2] +def test_row_number_ids_count_the_cells_without_a_region(tmp_path: Path) -> None: + path = tmp_path / "cells.csv" + path.write_text("cx,cy,sample,CD3\n1,2,A,1\n3,4,A,2\n5,6,,3\n7,8,B,4\n9,10,B,5\n") + kwargs = {"x": "cx", "y": "cy", "region_column": "sample"} + + with pytest.warns(UserWarning, match=r"Ignoring the 1 cell\(s\)"): + table = read_cell_table(path, "generic", pixel_size_um=None, default_region=None, cells_table_kwargs=kwargs) + + # the instance ids are the rows of the file + assert list(table.instance_id) == [1, 2, 4, 5] + assert list(table.region) == ["A", "A", "B", "B"] + assert table.row_instance_ids + + def test_generic_bad_kwargs_raise(tmp_path: Path) -> None: path = tmp_path / "cells.csv" path.write_text("cx,cy,CD3\n1,2,3\n") From ff81b196e16bd51b0389adfd9e65c53193cf409e Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 14:59:15 +0200 Subject: [PATCH 099/107] feat(akoya): select CODEX markers by name, CSV header or cycCCC_chHHH column --- CHANGELOG.md | 4 ++- .../readers/_utils/_cell_tables.py | 26 +++++++++++++---- src/spatialdata_io/readers/codex.py | 3 +- src/spatialdata_io/readers/phenocycler.py | 3 +- tests/readers/test_cell_tables.py | 29 +++++++++++++++++++ 5 files changed, 56 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5de7a60..219969cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,7 +82,9 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. tables are preferred; other tables of the same region (e.g. of two segmentation runs) raise a `ValueError`. A `cells_table` disables table discovery, and the `labels` masks are not read as images. `cells_table_kwargs` (e.g. `markers`, `region_column`) apply to the discovered tables too. The CSV and FCS tables of one marker panel (e.g. with - `fcs=False` and a region with only an FCS table) share one `table`, without `var["source_column"]`. Without `labels`, a + `fcs=False` and a region with only an FCS table) share one `table`, without `var["source_column"]`. + `cells_table_kwargs["markers"]` names a CODEX marker by its name (`DAPI1`), CSV header (`cyc001_ch001:DAPI1`) + or column (`cyc001_ch001`); a listed marker that the table does not have raises a `ValueError`. Without `labels`, a `.tif(f)` ending in `_mask` or `_labels` and named after a region (e.g. `reg001_mask.ome.tiff`) is its mask. - Region ids `regNNN` are taken from `reg` and digits not preceded by a letter (`exp_reg001`, not `Treg2`), also by `phenocycler()` for a `.qptiff`. A CIM `.qptiff` without one is named as `phenocycler()` names it (without the diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index 22ebd4ff..a5ef1e1a 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -337,8 +337,10 @@ def read_cell_table( cells_table_kwargs Optional keys: ``region_column`` (the column holding the region of each cell; QuPath default ``Image``), and for ``"generic"`` tables ``x`` and ``y`` (required centroid columns), ``instance_id`` (else the row - number in the file, from 1), ``markers`` (a list of columns or a regular expression matching the full column name; - default: all other numeric columns; also for ``"qupath"`` and ``"codex"`` tables) and ``coordinates_unit`` (``"px"``, + number in the file, from 1), ``markers`` (a list of columns or a regular expression matching the full + column name; default: all other numeric columns; also for ``"qupath"`` and ``"codex"`` tables, a CODEX + marker named by its name ``DAPI1``, its CSV header ``cyc001_ch001:DAPI1`` or its column ``cyc001_ch001``) + and ``coordinates_unit`` (``"px"``, default, or ``"um"``; for ``"qupath"`` and ``"inform"`` tables the unit of centroid columns whose name has none) and ``coordinates_space`` (the pixels of ``"px"`` centroids: ``"image"``, default, those of the full-resolution region image, or ``"mask"``, those of the region's mask, scaled to image pixels like the @@ -686,7 +688,8 @@ def _read_fcs(path: Path) -> pd.DataFrame: def _codex_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: """The roles of a CODEX table; renames the columns of ``df`` to the name part of ``cycCCC_chHHH:``.""" - split = [str(column).partition(":") for column in df.columns] + headers = [str(column) for column in df.columns] + split = [header.partition(":") for header in headers] # the source of an FCS column is its `$PnN` fcs = path.suffix.lower() == CodexKeys.FCS_FILE.value sources = [name if fcs else str(column) for column, (name, _, _) in zip(df.columns, split, strict=True)] @@ -705,9 +708,20 @@ def _codex_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dic area = columns.get("size") matches = [_MARKER_COLUMN.fullmatch(name) for name in names] - if kwargs.get("markers") is not None: - selected = set(_select_markers(df, kwargs["markers"], {x, y, instance_column, region_column, area})) - is_marker = [name in selected for name in names] + if (selection := kwargs.get("markers")) is not None: + # a marker is named by its column (`cyc001_ch001`), its CSV header (`cyc001_ch001:DAPI1`) or its name (`DAPI1`) + forms = [{*form} for form in zip(names, headers, displays, dedupe_channel_names(displays), strict=True)] + if isinstance(selection, str): + exclude = {x, y, instance_column, region_column, area} + is_marker = [ + name not in exclude and any(re.fullmatch(selection, form) for form in aliases) + for name, aliases in zip(names, forms, strict=True) + ] + else: + wanted = {str(marker) for marker in selection} + if missing := wanted.difference(*forms): + raise ValueError(f"{path} has no marker column(s) {sorted(missing)}; found {headers}.") + is_marker = [bool(aliases & wanted) for aliases in forms] elif any(matches): is_marker = [match is not None for match in matches] else: diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index 6f19bf0a..fa66e529 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -209,7 +209,8 @@ def codex( Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`. cells_table_kwargs Options for reading ``cells_table`` (or the discovered cell tables) and ``labels``: ``region_column``; for - ``"generic"`` tables ``x``, ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath and CODEX tables), + ``"generic"`` tables ``x``, ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath + and CODEX tables, whose markers are named by their name, ``cycCCC_chHHH:`` header or ``cycCCC_chHHH``), ``coordinates_unit`` (``"px"`` or ``"um"``; for QuPath ``X``/``Y`` and inForm ``Cell X Position`` columns without a unit, which are otherwise read in the unit of the table's other columns) and ``coordinates_space`` (``"image"``, default, or ``"mask"``: whether the pixel centroids of a generic table diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index bed9d5ea..3a447a0c 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -167,7 +167,8 @@ def phenocycler( Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`. cells_table_kwargs Options for reading ``cells_table`` and ``labels``: ``region_column``; for ``"generic"`` tables ``x``, - ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath and CODEX tables), + ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath and CODEX tables, whose + markers are named by their name, ``cycCCC_chHHH:`` header or ``cycCCC_chHHH``), ``coordinates_unit`` (``"px"`` or ``"um"``; for QuPath ``X``/``Y`` and inForm ``Cell X Position`` columns without a unit, which are otherwise read in the unit of the table's other columns) and ``coordinates_space`` (``"image"``, default, or ``"mask"``: whether the pixel centroids of a generic table diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index 56b22662..c6a6bb41 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -176,6 +176,35 @@ def test_read_codex_table_cells_table_kwargs(tmp_path: Path) -> None: assert "CD3" in table.obs.columns +@pytest.mark.parametrize( + ("markers", "expected"), + [ + (["DAPI1", "CD3"], ["DAPI1", "CD3"]), + (["cyc001_ch001:DAPI1"], ["DAPI1"]), + (["cyc001_ch001", "cyc002_ch001"], ["DAPI1", "DAPI"]), + (["DAPI (2)"], ["DAPI"]), + ("DAPI.*", ["DAPI1", "DAPI", "DAPI (2)"]), + ("cyc00\\d_ch001", ["DAPI1", "DAPI"]), + ], +) +def test_read_codex_table_markers_by_name_header_or_column( + tmp_path: Path, markers: str | list[str], expected: list[str] +) -> None: + path = tmp_path / "cells.csv" + path.write_text( + "cell_id,x,y,cyc001_ch001:DAPI1,cyc001_ch002:CD3,cyc002_ch001:DAPI,cyc002_ch002:DAPI\n1,1.0,2.0,3.0,4.0,5.0,6.0\n" + ) + + table = read_cell_table( + path, "codex", pixel_size_um=None, default_region="A", cells_table_kwargs={"markers": markers} + ) + + assert list(table.var.index) == expected + + with pytest.raises(ValueError, match=r"no marker column\(s\) \['CD99'\]"): + read_cell_table(path, "codex", pixel_size_um=None, default_region="A", cells_table_kwargs={"markers": ["CD99"]}) + + @pytest.mark.parametrize( ("fmt", "header", "markers"), [ From 7c78ffa9cc7135e7a401aa8eb3f074273d4488a9 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 15:00:55 +0200 Subject: [PATCH 100/107] fix(akoya): require flowio>=1.4; skip FCS files flowio fails to parse in any way --- CHANGELOG.md | 1 + pyproject.toml | 2 +- src/spatialdata_io/readers/_utils/_cell_tables.py | 3 ++- tests/test_codex.py | 15 +++++++++++++++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 219969cb..4a6d30cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,7 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. documentation builds, `mypy` type checking of `src` and `tests`, `biome`/`pyproject-fmt`/`zizmor` pre-commit hooks, and Dependabot updates. - Requires `tifffile>=2026.5.2` and `zarr>=3.2`, the first versions whose zarr store the lazy Akoya TIFF reads work with. +- Requires `flowio>=1.4`, the first version with `FlowData.pnn_labels`, with which `codex()` detects CODEX FCS tables. - `codex()` was rewritten to read CODEX Processor, CODEX Uploader and CODEX Instrument Manager outputs. This changes its output and signature: - Element names are now `{region}_image`, `{region}_cells` (circles), `{region}_labels`, `{region}_polygons` and diff --git a/pyproject.toml b/pyproject.toml index 143e60c8..2c666895 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "anndata", "click", "dask-image", - "flowio", + "flowio>=1.4", "h5py", "imagecodecs", "joblib", diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index a5ef1e1a..ea198861 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -229,7 +229,8 @@ def detect_table_format(path: Path) -> str: if path.suffix.lower() == CodexKeys.FCS_FILE.value: try: columns = list(flowio.read_multiple_data_sets(str(path), only_text=True)[0].pnn_labels) - except (ValueError, flowio.exceptions.FlowIOException) as e: + # flowio raises these, not only its own exceptions, for malformed files (e.g. truncated, without `$DATATYPE`) + except (ValueError, KeyError, IndexError, EOFError, OSError, flowio.exceptions.FlowIOException) as e: raise ValueError(f"{path} is not a readable FCS file: {e}") from e if _detect_header_format(columns) != "codex": raise ValueError(f"{path} is not a CODEX cell table: its channels are {columns}.") diff --git a/tests/test_codex.py b/tests/test_codex.py index 7ad9e9c2..d70d1395 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -934,3 +934,18 @@ def test_codex_csv_and_fcs_tables_of_one_panel_share_a_table(tmp_path: Path) -> assert list(table.var_names) == dedupe_channel_names(data["markers"]) # the markers were read from columns of other names assert "source_column" not in table.var + + +@pytest.mark.parametrize("corrupt", ["truncated", "no_datatype"]) +def test_codex_ignores_malformed_fcs_files(tmp_path: Path, corrupt: str) -> None: + tifffile.imwrite(tmp_path / "reg001_montage.tif", np.zeros((2, 16, 16), dtype=np.uint16)) + fcs = tmp_path / "reg001.fcs" + write_fcs(fcs, pd.DataFrame({"cell_id": [1.0], "x": [1.0], "y": [2.0], "cyc001_ch001": [3.0]})) + content = fcs.read_bytes() + # flowio raises an EOFError and a KeyError for these + fcs.write_bytes(content[:100] if corrupt == "truncated" else content.replace(b"$DATATYPE", b"$DATATYPX")) + + sdata = codex(tmp_path, image_models_kwargs=_FAST) + + assert list(sdata.images) == ["reg001_image"] + assert not sdata.tables From 38d3cef7a4a988b1e0e882ad678d3cd0a58dc48e Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 15:20:37 +0200 Subject: [PATCH 101/107] fix(akoya): take cell-table column names from the header, not from pandas --- .../readers/_utils/_cell_tables.py | 11 ++++--- tests/readers/test_cell_tables.py | 30 ++++++++++++++++++- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/spatialdata_io/readers/_utils/_cell_tables.py b/src/spatialdata_io/readers/_utils/_cell_tables.py index ea198861..a0d9cfee 100644 --- a/src/spatialdata_io/readers/_utils/_cell_tables.py +++ b/src/spatialdata_io/readers/_utils/_cell_tables.py @@ -2,6 +2,7 @@ from __future__ import annotations +import csv import json import re import warnings @@ -1152,7 +1153,7 @@ def _read_header(path: Path) -> tuple[list[str], str]: with path.open(encoding="utf-8-sig", errors="replace") as f: header = f.readline().rstrip("\r\n") sep = "\t" if "\t" in header else ";" if header.count(";") > header.count(",") else "," - return [column.strip().strip('"') for column in header.split(sep)], sep + return [column.strip() for column in next(csv.reader([header], delimiter=sep))], sep def _read_text_table(path: Path) -> pd.DataFrame: @@ -1163,9 +1164,7 @@ def _read_text_table(path: Path) -> pd.DataFrame: """ names, sep = _read_header(path) if sep == ";": - # the C engine keeps the header's repeated names apart only by renaming them, so the names come from the header df = pd.read_csv(path, sep=sep, decimal=",", engine="c", header=None, skiprows=1) - df.columns = pd.Index(names) for i in range(df.shape[1]): if not pd.api.types.is_numeric_dtype(df.iloc[:, i]): try: @@ -1174,9 +1173,9 @@ def _read_text_table(path: Path) -> pd.DataFrame: pass else: df = pd.read_csv(path, sep=sep, engine="pyarrow") - # inForm writes some column names with trailing spaces; pyarrow keeps repeated names, which are made unique like - # the marker names - df.columns = pd.Index(dedupe_channel_names([str(column).strip() for column in df.columns])) + # pandas renames repeated names (DAPI.1; with the pyarrow engine since 3.1), so the names come from the header and + # are made unique like the marker names + df.columns = pd.Index(dedupe_channel_names(names)) return df diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index c6a6bb41..376a848e 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -214,7 +214,24 @@ def test_read_codex_table_markers_by_name_header_or_column( ("generic", "cx,cy,DAPI,CD3,DAPI", ["DAPI", "CD3", "DAPI (2)"]), ], ) -def test_repeated_column_names_are_deduplicated(tmp_path: Path, fmt: str, header: str, markers: list[str]) -> None: +@pytest.mark.parametrize("mangled", [False, True]) +def test_repeated_column_names_are_deduplicated( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fmt: str, header: str, markers: list[str], mangled: bool +) -> None: + if mangled: # pandas >= 3.1 renames repeated names (DAPI.1) with the pyarrow engine too + read_csv = pd.read_csv + + def mangling_read_csv(*args: Any, **kwargs: Any) -> pd.DataFrame: + df = read_csv(*args, **kwargs) + seen: dict[str, int] = {} + columns = [] + for column in df.columns: + columns.append(f"{column}.{seen[column]}" if column in seen else column) + seen[column] = seen.get(column, 0) + 1 + df.columns = pd.Index(columns) + return df + + monkeypatch.setattr(pd, "read_csv", mangling_read_csv) path = tmp_path / "cells.csv" n = header.count(",") + 1 path.write_text(f"{header}\n{','.join(str(i + 1) for i in range(n))}\n", encoding="utf-8") @@ -228,6 +245,17 @@ def test_repeated_column_names_are_deduplicated(tmp_path: Path, fmt: str, header assert table.X.shape == (1, len(markers)) +def test_quoted_column_names_may_hold_the_delimiter(tmp_path: Path) -> None: + path = tmp_path / "cells.csv" + path.write_text('cx,cy,"CD3, mean",DAPI\n1,2,3,4\n', encoding="utf-8") + + table = read_cell_table( + path, "generic", pixel_size_um=None, default_region="S", cells_table_kwargs={"x": "cx", "y": "cy"} + ) + + assert list(table.var.index) == ["CD3, mean", "DAPI"] + + @pytest.mark.parametrize("cell_id", ["", "1.5", "a"]) def test_missing_or_fractional_instance_ids_raise(tmp_path: Path, cell_id: str) -> None: path = tmp_path / "cells.csv" From 4931cc5dd91c29df16f0b614504c9c7e664aee5e Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 19:41:17 +0200 Subject: [PATCH 102/107] fix(codex): strip .ome from the region id of an OME-TIFF named after its region --- src/spatialdata_io/readers/codex.py | 6 +++--- tests/test_codex.py | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index fa66e529..bb081857 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -136,8 +136,8 @@ def codex( and a trailing ``_Scan`` (unless that makes two regions share a name), and the extended-range ``.er.qptiff`` of a scan is preferred, also for one with a region id (``exp_reg001.er.qptiff`` and ``exp_reg001.qptiff`` are region ``reg001``). Several scans with the same region id raise an error. If a directory contains no such files, its top-level ``{vx.IMAGE_TIF}(f)`` files are read as - hyperstacks named after their files, made valid element names by :func:`spatialdata.sanitize_name` (e.g. - ``Kidney sample.tif`` becomes the region ``Kidney_sample``). Each region is stored as the image ``_image`` + hyperstacks named after their files without the extension (``.ome.tif`` included), made valid element names by + :func:`spatialdata.sanitize_name` (e.g. ``Kidney sample.ome.tif`` becomes the region ``Kidney_sample``). Each region is stored as the image ``_image`` and its metadata in ``sdata.attrs["akoya"]``. A single region is stored in the ``"global"`` coordinate system. Several CIM scans of the same sample space @@ -407,7 +407,7 @@ def _discover_legacy( if not images: # the original (2019) layout: one stitched `.tif` next to its table, without a region in the file name top_level = image_files if path.is_file() else [file for file in image_files if file.parent == path] - images = {sanitize_name(file.stem): [file] for file in top_level if _is_tiff(file)} + images = {sanitize_name(IMAGE_EXTENSION.sub("", file.name)): [file] for file in top_level if _is_tiff(file)} tables = _discover_tables(files, fcs) if find_tables else {} if not images and not tables: raise FileNotFoundError(f"No CODEX images or cell tables found in {path}. Expected: {'; '.join(_PATTERNS)}.") diff --git a/tests/test_codex.py b/tests/test_codex.py index d70d1395..6c58120f 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -745,8 +745,10 @@ def test_codex_unreadable_regionless_table_raises(tmp_path: Path) -> None: codex(tmp_path / "Kidney.fcs") -def test_codex_regionless_table_belongs_to_single_stem_image(tmp_path: Path) -> None: - tifffile.imwrite(tmp_path / "Kidney.tif", np.zeros((2, 16, 16), dtype=np.uint16)) +@pytest.mark.parametrize("file_name", ["Kidney.tif", "Kidney.ome.tif", "Kidney.OME.TIFF"]) +def test_codex_regionless_table_belongs_to_single_stem_image(tmp_path: Path, file_name: str) -> None: + # the region is the file name without its image extension, `.ome` included + tifffile.imwrite(tmp_path / file_name, np.zeros((2, 16, 16), dtype=np.uint16)) (tmp_path / "cells.csv").write_text("cell_id,region,x,y,size,cyc001_ch001\n1,1,1.0,2.0,3.0,4.0\n") for regions in (None, ["Kidney"]): From 3a61f1735d58688c470be2ce3fd17f82fd49b849 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 19:41:17 +0200 Subject: [PATCH 103/107] test(akoya): check pixels, cell alignment and round trip on the real datasets --- tests/test_akoya_real_data.py | 119 +++++++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 2 deletions(-) diff --git a/tests/test_akoya_real_data.py b/tests/test_akoya_real_data.py index 3fd895e2..402f6e33 100644 --- a/tests/test_akoya_real_data.py +++ b/tests/test_akoya_real_data.py @@ -17,7 +17,11 @@ import numpy as np import pandas as pd import pytest +import tifffile +from scipy.stats import spearmanr +from spatialdata import read_zarr from spatialdata.models import get_channel_names +from spatialdata.transformations import get_transformation from spatialdata_io import codex, phenocycler from spatialdata_io.readers._utils._cell_tables import detect_table_format, read_cell_table @@ -244,6 +248,60 @@ def test_real_scan(scan: Scan) -> None: else: assert metadata["pixel_size_um"] == pytest.approx(scan.pixel_size_um, abs=1e-3) assert sdata.attrs["spatialdata_io_reader"] == scan.reader + if path.is_file(): + _assert_tile_matches_tifffile(path, data, channel=scan.n_channels - 1) + + +def _assert_tile_matches_tifffile(path: Path, data: da.Array, channel: int) -> None: + """A full-resolution tile of ``channel`` equals tifffile's decoding of the whole page (not through zarr).""" + with tifffile.TiffFile(path) as tf: + full = [page for page in tf.pages if page.shape[:2] == data.shape[1:]] + # brightfield: one RGB page holds the three channels + page = full[0] if full[0].ndim == 3 else full[channel] + y, x = data.shape[1] // 2, data.shape[2] // 2 + expected = page.asarray()[y : y + 256, x : x + 256] + if expected.ndim == 3: + expected = expected[..., channel] + np.testing.assert_array_equal(data[channel, y : y + 256, x : x + 256].compute(), expected) + + +def test_real_same_qptiff_through_both_readers() -> None: + path = Path(_ROOT) / "figshare_25123538/AML2_183_Scan1.qptiff" # type: ignore[arg-type] + if not path.exists(): + pytest.skip(f"{path} is missing") + + a, b = phenocycler(path), codex(path) + + image_a, image_b = a.images["AML2_183_image"], b.images["AML2_183_image"] + assert get_channel_names(image_a) == get_channel_names(image_b) + assert list(image_a) == list(image_b) + coarsest = list(image_a)[-1] + np.testing.assert_array_equal(image_a[coarsest]["image"].values, image_b[coarsest]["image"].values) + assert get_transformation(image_a, get_all=True) == get_transformation(image_b, get_all=True) + assert a.attrs["akoya"]["regions"] == b.attrs["akoya"]["regions"] + + +def _image_at(sdata, region: str, channel: str, xy: np.ndarray) -> np.ndarray: + """The full-resolution values of ``channel`` of the image of ``region`` at the pixel positions ``xy``.""" + image = sdata.images[f"{region}_image"]["scale0"]["image"] + plane = image.sel(c=channel).values + x = np.clip(xy[:, 0].astype(int), 0, plane.shape[1] - 1) + y = np.clip(xy[:, 1].astype(int), 0, plane.shape[0] - 1) + return plane[y, x] + + +def _assert_cells_on_their_signal(sdata, region: str, element: str, pairs: dict[str, str], rho: float) -> None: + """The table's per-cell means correlate with the image at the cell centroids, and drop away from them.""" + table = sdata.tables["table"] + cells = table[table.obs["region"] == element] + xy = cells.obsm["spatial"] + for var, channel in pairs.items(): + values = np.asarray(cells[:, var].X).ravel() + at_cells = spearmanr(values, _image_at(sdata, region, channel, xy)).statistic + shifted = spearmanr(values, _image_at(sdata, region, channel, xy + 25)).statistic + assert at_cells > rho, (var, at_cells) + # clustered markers stay correlated nearby, but less so + assert shifted < at_cells - 0.1, (var, at_cells, shifted) def test_real_codex_processor_csv() -> None: @@ -305,6 +363,15 @@ def test_real_inform_table_with_seg_maps_and_component_data() -> None: assert labels.shape == (1400, 1868) assert set(table.obs["cell_id"]) == set(np.unique(labels.values)) - {0} np.testing.assert_allclose(table.obsm["spatial"][0], [515, 13]) + # the table's centroids are those of its labels, and its DAPI is the image's DAPI there + mask = labels.values + ys, xs = np.nonzero(mask) + counts = np.bincount(mask[ys, xs]) + ids = table.obs["cell_id"].to_numpy() + centroids = np.c_[np.bincount(mask[ys, xs], xs)[ids], np.bincount(mask[ys, xs], ys)[ids]] / counts[ids, None] + # inForm rounds the positions to whole pixels + assert np.median(np.linalg.norm(centroids + 0.5 - table.obsm["spatial"], axis=1)) < 2 + _assert_cells_on_their_signal(sdata, region, f"{region}_labels", {"DAPI": "DAPI"}, rho=0.5) def test_real_inform_micron_variant() -> None: @@ -383,11 +450,59 @@ def test_real_mcquant_table_with_ome_tiff() -> None: image = _export(f"{_EXPORTS}/s3_mcmicro_exemplar001/exemplar-001-cycle6.ome.tif") cells = _export(f"{_EXPORTS}/s3_mcmicro_exemplar001/unmicst-exemplar-001_cell.csv") + # the OME-XML has no channel names: those of cycle 6 in `markers.csv` with pytest.warns(UserWarning, match="cells are represented as circles") as caught: - sdata = codex(image, cells_table=cells) + sdata = codex(image, cells_table=cells, channel_names=["DNA_6", "ELANE", "CD57", "CD45"]) _assert_cells_inside_images(caught) table = sdata.tables["table"] assert table.shape == (11170, 12) assert list(table.var_names[:4]) == ["DNA_6", "ELANE", "CD57", "CD45"] - assert len(sdata.shapes["exemplar-001-cycle6.ome_cells"]) == 11170 + assert len(sdata.shapes["exemplar-001-cycle6_cells"]) == 11170 + pairs = {"DNA_6": "DNA_6", "CD57": "CD57"} + _assert_cells_on_their_signal(sdata, "exemplar-001-cycle6", "exemplar-001-cycle6_cells", pairs, rho=0.8) + + +def test_real_qupath_cores_with_geojson_end_to_end(tmp_path: Path) -> None: + # two TMA cores of doi:10.5061/dryad.hqbzkh1v1 in one read: OME-TIFFs, QuPath centroids + measurements, StarDist + # GeoJSON; the pixel size comes from `experiment.json` (the OME `PhysicalSizeX`) + cores = ("C-7", "N-12") + for core in cores: + (tmp_path / f"{core}.ome.tif").symlink_to(_export(f"{_DRYAD}/{core}.ome.tif")) + coords = pd.read_csv(_export(f"{_DRYAD}/{core}_coords.csv")) + expression = pd.read_csv(_export(f"{_DRYAD}/{core}_expression.csv")) + coords.merge(expression, on="Object.ID", validate="1:1").to_csv(tmp_path / f"{core}.csv", index=False) + (tmp_path / "channelnames.txt").symlink_to(_export(f"{_DRYAD}/channelnames.txt")) + (tmp_path / "experiment.json").write_text(f'{{"xyResolution": {_DRYAD_PIXEL_SIZE_UM * 1000}}}') + + with pytest.warns(UserWarning, match="cell features"): + sdata = codex( + tmp_path, + cells_table={core: tmp_path / f"{core}.csv" for core in cores}, + cells_table_kwargs={"geojson": {core: _export(f"{_DRYAD}/{core}.geojson") for core in cores}}, + ) + + assert sorted(sdata.coordinate_systems) == list(cores) + table = sdata.tables["table"] + assert list(table.obs["region"].cat.categories) == [f"{core}_labels" for core in cores] + for core, n_cells in zip(cores, (9415, 14242), strict=True): + cells = table[table.obs["region"] == f"{core}_labels"] + assert cells.n_obs == n_cells + assert sdata.shapes[f"{core}_polygons"].geometry.is_valid.all() + mask = sdata.labels[f"{core}_labels"]["scale0"]["image"].values + assert set(np.unique(mask)) - {0} == set(cells.obs["cell_id"]) + # the rasterised cell under each centroid is that cell + x, y = cells.obsm["spatial"].astype(int).T + assert (mask[y, x] == cells.obs["cell_id"].to_numpy()).mean() > 0.99 + _assert_cells_on_their_signal(sdata, core, f"{core}_labels", {"DAPI": "DAPI"}, rho=0.6) + + sdata.write(tmp_path / "dryad.zarr") + read = read_zarr(tmp_path / "dryad.zarr") + assert read.attrs["akoya"] == sdata.attrs["akoya"] + for name in ("C-7_labels", "N-12_polygons", "N-12_image"): + assert get_transformation(read[name], get_all=True) == get_transformation(sdata[name], get_all=True) + np.testing.assert_array_equal(read.tables["table"].X, table.X) + for name in ("C-7_labels", "N-12_image"): + element = read.labels.get(name) or read.images[name] + expected = sdata.labels.get(name) or sdata.images[name] + np.testing.assert_array_equal(element["scale0"]["image"].values, expected["scale0"]["image"].values) From fa7cc011fb58d20de0792b72d4e3892c26b144b0 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 21:05:03 +0200 Subject: [PATCH 104/107] refactor(akoya): read CODEX Processor cell tables only; phenocycler reads images and labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the QuPath, MCMICRO mcquant, HuBMAP SPRM, inForm and generic table adapters with their format detection, cells_table_format and cells_table_kwargs (also in the CLI), the µm/mask-pixel coordinate conversions and mask channel selection. phenocycler() loses cells_table. --- CHANGELOG.md | 49 +- src/spatialdata_io/__main__.py | 33 +- src/spatialdata_io/_constants/_constants.py | 1 - src/spatialdata_io/readers/_utils/_akoya.py | 2 +- .../readers/_utils/_cell_tables.py | 1453 ++--------------- src/spatialdata_io/readers/codex.py | 110 +- src/spatialdata_io/readers/phenocycler.py | 95 +- tests/readers/test_cell_tables.py | 923 +---------- tests/test_akoya_real_data.py | 223 +-- tests/test_codex.py | 82 +- tests/test_phenocycler.py | 473 +----- 11 files changed, 230 insertions(+), 3214 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a6d30cd..23028555 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,36 +25,19 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. - `phenocycler()` reader (and `phenocycler` CLI command) for Akoya PhenoCycler-Fusion 1.x/2.x `.qptiff` / `.er.qptiff` scans, including PhenoImager (Opal) mode: lazy level-0 image with a recomputed pyramid, channel names from the per-page XML, structured and raw scan metadata in `sdata.attrs["akoya"]`, optional `.xpd` / - `*channels.csv` sidecars, and optional cell tables (QuPath with GeoJSON outlines, MCMICRO `mcquant`, HuBMAP SPRM, - inForm, CODEX Processor or generic CSV) with segmentation masks. The lazy images and masks can be pickled (e.g. for - `dask.distributed`): they open their file in each process on the first chunk read. A cell table for several - regions converts µm coordinates with each region's own pixel size. + `*channels.csv` sidecars, and optional segmentation masks (`labels`). The lazy images and masks can be pickled (e.g. + for `dask.distributed`): they open their file in each process on the first chunk read. - `phenocycler(read_all_bit_depths=True)` (CLI `--read-all-bit-depths`) reads both the 8-bit `.qptiff` and the extended-range `.er.qptiff` of a scan: the 8-bit one becomes the image `_8bit_image` with the transformations of the `.er` region (a warning is raised if their shapes differ). By default only the extended-range file is read. - The Akoya readers (`phenocycler()`, `codex()`) scale a segmentation mask downsampled from its region image by an - integer factor (exactly, or rounded up) onto the image (a `Scale` before the region's transformation), and scale - the centroids (`obsm["spatial"]`, to the centre of their mask pixel) and circle radii of `mcquant` and SPRM - tables, which are in mask pixels, by the same factor; the source columns in `obs` are kept as read. Generic pixel tables are in image pixels unless - `cells_table_kwargs["coordinates_space"]="mask"`. Other shape mismatches warn. + integer factor (exactly, or rounded up) onto the image (a `Scale` before the region's transformation); the cell + centroids stay in image pixels. Other shape mismatches warn. - `phenocycler()` officially supports the PerkinElmer/Akoya QPTIFFs of Vectra Polaris and PhenoImager HT (generation `"qpi_other"`): Opal and unmixed (uint16) scans, brightfield RGB scans (channels `r`, `g`, `b`) and inForm component data (float32). A single `.tif` file is accepted if it is a QPI file. Region ids are made valid element names with - `spatialdata.sanitize_name` (e.g. the `[x,y]` of inForm tiles: `A_[1,2]` becomes `A__1_2_`); the region column of a - cell table, `regions` and the keys of the `cells_table`/`labels` mappings may hold the raw names. -- The Akoya cell-table adapters read real QuPath, inForm and mcquant exports without column mapping: R-exported - QuPath names (`Object.ID`, `Centroid.X.um`, `__Mean`), QuPath `X`/`Y` centroids (in the unit - of the other measurements with a warning, or `cells_table_kwargs["coordinates_unit"]`) with bare marker names - (read with a warning, or chosen with `cells_table_kwargs["markers"]`), and inForm column names with trailing - spaces; an inForm `Cell X Position` that names no unit is read in the unit of the table's other columns (e.g. - microns, divided by the pixel size), or in `cells_table_kwargs["coordinates_unit"]`. `mask_channel` accepts the - page names of an inForm `*_binary_seg_maps.tif` (`"Tissue"`, `"Nucleus"`, `"Cytoplasm"`, `"Membrane"`); such a - file reads its `Nucleus` page (whose labels are the `Cell ID`s) by default. With `x` and `y` in - `cells_table_kwargs`, `cells_table_format="auto"` reads a generic table. QuPath GeoJSON cells without a table row - are skipped with a warning; a GeoJSON found next to a QuPath table that holds none of its cells (e.g. an - annotation export) is ignored with a warning, and `cells_table_kwargs["geojson"]=False` reads no outlines. A region - whose cells lie mostly outside its image raises a warning. -- The Akoya cell tables may be semicolon-separated (European locales); their numbers may have decimal commas. + `spatialdata.sanitize_name` (e.g. the `[x,y]` of inForm tiles: `A_[1,2]` becomes `A__1_2_`); `regions` and the + keys of the `labels` mapping may hold the raw names. ### Changed @@ -65,15 +48,14 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. - Requires `flowio>=1.4`, the first version with `FlowData.pnn_labels`, with which `codex()` detects CODEX FCS tables. - `codex()` was rewritten to read CODEX Processor, CODEX Uploader and CODEX Instrument Manager outputs. This changes its output and signature: - - Element names are now `{region}_image`, `{region}_cells` (circles), `{region}_labels`, `{region}_polygons` and - `table` (previously `images` and `{region}` for the shapes). + - Element names are now `{region}_image`, `{region}_cells` (circles), `{region}_labels` and `table` (previously + `images` and `{region}` for the shapes). - Every region is read (previously only the first file found); `regions=[...]` selects a subset. - Marker names in `table.var_names` come from the FCS `$PnS` keywords or the CSV header `cycCCC_chHHH:` (previously `cycCCC_chHHH`); the image channels are named too. - The positional signature `codex(path, fcs, imread_kwargs)` is unchanged; all new parameters (`regions`, - `channel_names`, `cells_table`, `cells_table_format`, `labels` and the `*_models_kwargs` / `cells_table_kwargs`) - are keyword-only, and the `codex` CLI command exposes them (`--regions` and `--channel-names` are repeated per - value, as in the `phenocycler` command). An empty `regions` raises a `ValueError`. + `channel_names`, `cells_table`, `labels` and the `*_models_kwargs`) are keyword-only, and the `codex` CLI command + exposes them (`--regions` and `--channel-names` are repeated per value, as in the `phenocycler` command). An empty `regions` raises a `ValueError`. `imread_kwargs` is ignored, with a warning: every image is read lazily with `tifffile`. - A single region is stored in `"global"`. Several regions are placed at their stage positions in `"global"` only when all of them share a sample space (same slide id, stage positions and pixel size); otherwise each region, @@ -81,12 +63,11 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. - `channelNames.txt` and `experiment.json` are found in any case (e.g. SPACEc's `channelnames.txt`); CSV and FCS files that are not CODEX Processor cell tables are ignored, also when their name holds a region id. Compensated tables are preferred; other tables of the same region (e.g. of two segmentation runs) raise a `ValueError`. - A `cells_table` disables table discovery, and the `labels` masks are not read as images. `cells_table_kwargs` - (e.g. `markers`, `region_column`) apply to the discovered tables too. The CSV and FCS tables of one marker panel (e.g. with - `fcs=False` and a region with only an FCS table) share one `table`, without `var["source_column"]`. - `cells_table_kwargs["markers"]` names a CODEX marker by its name (`DAPI1`), CSV header (`cyc001_ch001:DAPI1`) - or column (`cyc001_ch001`); a listed marker that the table does not have raises a `ValueError`. Without `labels`, a - `.tif(f)` ending in `_mask` or `_labels` and named after a region (e.g. `reg001_mask.ome.tiff`) is its mask. + A `cells_table` disables table discovery, and the `labels` masks are not read as images. The CSV and FCS tables + of one marker panel (e.g. with `fcs=False` and a region with only an FCS table) share one `table`, without + `var["source_column"]`. A table without `cycCCC_chHHH` columns reads its other numeric columns as the markers, + with a warning listing them. Without `labels`, a `.tif(f)` ending in `_mask` or `_labels` and named after a region + (e.g. `reg001_mask.ome.tiff`) is its mask. - Region ids `regNNN` are taken from `reg` and digits not preceded by a letter (`exp_reg001`, not `Treg2`), also by `phenocycler()` for a `.qptiff`. A CIM `.qptiff` without one is named as `phenocycler()` names it (without the suffix and a trailing `_Scan`), and its `.er.qptiff` is preferred. Top-level `.tif` files of the original diff --git a/src/spatialdata_io/__main__.py b/src/spatialdata_io/__main__.py index 6939415c..c871cb25 100644 --- a/src/spatialdata_io/__main__.py +++ b/src/spatialdata_io/__main__.py @@ -68,16 +68,6 @@ def _akoya_click_options(func: Callable[..., None]) -> Callable[..., None]: multiple=True, help="A channel name overriding the discovered ones (repeat for every channel).", ), - click.option( - "--cells-table", - type=str, - default=None, - help='Path to a cell table, or a JSON mapping from region to path, e.g. \'{"A": "a.csv"}\'. ' - "[default: None]", - ), - click.option( - "--cells-table-format", type=str, default="auto", help="Format of the cell table. [default: auto]" - ), click.option( "--labels", type=str, @@ -96,12 +86,6 @@ def _akoya_click_options(func: Callable[..., None]) -> Callable[..., None]: default="{}", help="JSON string of keyword arguments passed to Labels2DModel. [default: {}]", ), - click.option( - "--cells-table-kwargs", - type=str, - default="{}", - help="JSON string of keyword arguments for reading the cell table. [default: {}]", - ), ] for option in reversed(options): func = option(func) @@ -117,6 +101,13 @@ def _akoya_click_options(func: Callable[..., None]) -> Callable[..., None]: required=True, ) @_akoya_click_options +@click.option( + "--cells-table", + type=str, + default=None, + help='Path to a CODEX cell table, or a JSON mapping from region to path, e.g. \'{"reg001": "a.csv"}\'. ' + "[default: None]", +) @click.option( "--fcs", type=bool, @@ -136,12 +127,10 @@ def codex_wrapper( regions: tuple[str, ...] = (), channel_names: tuple[str, ...] = (), cells_table: str | None = None, - cells_table_format: str = "auto", labels: str | None = None, imread_kwargs: str = "{}", image_models_kwargs: str = "{}", labels_models_kwargs: str = "{}", - cells_table_kwargs: str = "{}", ) -> None: """Codex conversion to SpatialData.""" from spatialdata_io.readers.codex import codex @@ -152,12 +141,10 @@ def codex_wrapper( regions=list(regions) or None, channel_names=list(channel_names) or None, cells_table=_parse_path_or_mapping_param(cells_table, "cells_table"), - cells_table_format=cells_table_format, labels=_parse_path_or_mapping_param(labels, "labels"), imread_kwargs=_parse_json_param(imread_kwargs, "imread_kwargs"), image_models_kwargs=_parse_json_param(image_models_kwargs, "image_models_kwargs"), labels_models_kwargs=_parse_json_param(labels_models_kwargs, "labels_models_kwargs"), - cells_table_kwargs=_parse_json_param(cells_table_kwargs, "cells_table_kwargs"), ) sdata.write(output) @@ -466,13 +453,10 @@ def phenocycler_wrapper( output: str, regions: tuple[str, ...] = (), channel_names: tuple[str, ...] = (), - cells_table: str | None = None, - cells_table_format: str = "auto", labels: str | None = None, read_all_bit_depths: bool = False, image_models_kwargs: str = "{}", labels_models_kwargs: str = "{}", - cells_table_kwargs: str = "{}", ) -> None: """PhenoCycler-Fusion conversion to SpatialData.""" from spatialdata_io.readers.phenocycler import phenocycler @@ -481,13 +465,10 @@ def phenocycler_wrapper( input, regions=list(regions) or None, channel_names=list(channel_names) or None, - cells_table=_parse_path_or_mapping_param(cells_table, "cells_table"), - cells_table_format=cells_table_format, labels=_parse_path_or_mapping_param(labels, "labels"), read_all_bit_depths=read_all_bit_depths, image_models_kwargs=_parse_json_param(image_models_kwargs, "image_models_kwargs"), labels_models_kwargs=_parse_json_param(labels_models_kwargs, "labels_models_kwargs"), - cells_table_kwargs=_parse_json_param(cells_table_kwargs, "cells_table_kwargs"), ) sdata.write(output) diff --git a/src/spatialdata_io/_constants/_constants.py b/src/spatialdata_io/_constants/_constants.py index c2ffda3f..cfbbd08a 100644 --- a/src/spatialdata_io/_constants/_constants.py +++ b/src/spatialdata_io/_constants/_constants.py @@ -34,7 +34,6 @@ class PhenocyclerKeys(ModeEnum): IMAGE_SUFFIX = "_image" LABELS_SUFFIX = "_labels" CELLS_SUFFIX = "_cells" - POLYGONS_SUFFIX = "_polygons" TABLE_KEY = "table" # metadata REGION_KEY = "region" diff --git a/src/spatialdata_io/readers/_utils/_akoya.py b/src/spatialdata_io/readers/_utils/_akoya.py index e201e7b3..760861f7 100644 --- a/src/spatialdata_io/readers/_utils/_akoya.py +++ b/src/spatialdata_io/readers/_utils/_akoya.py @@ -48,7 +48,7 @@ "spatial_chunks", ] -# the extensions of image files, also named in the region column of a table (e.g. QuPath's `Image`) +# the extensions of image files, removed to name a region after its file IMAGE_EXTENSION = re.compile(r"(?:\.er)?\.(?:qptiff|ome\.tiff?|tiff?)$", re.IGNORECASE) # `reg001`, `exp_reg001` or `1_reg001`, but not `Treg2` or `coreg1` _REGION = re.compile(r"(?= 0.6) and `Cell: CD3 mean` (built-in cell detection); the per-marker intensity is -# taken from the first compartment. Detections without compartments have `CD3: Mean`. -_QUPATH_MARKER = re.compile( - r"(?P.+?)(?:: |\.\.|_)(?PCell|Nucleus|Cytoplasm|Membrane)(?:: |\.\.|_)Mean" -) -_QUPATH_COMPARTMENT_MARKER = re.compile( - r"(?PCell|Nucleus|Cytoplasm|Membrane): (?P.+?)(?:: M| m)ean" -) -_QUPATH_DETECTION_MARKER = re.compile(r"(?P.+?): Mean") -_QUPATH_COMPARTMENTS = ("Cell", "Nucleus", "Cytoplasm", "Membrane") -# other QuPath measurements, e.g. `Nucleus: Area µm^2` or `Nucleus/Cell area ratio`, are not markers -_QUPATH_MEASUREMENT = re.compile(r"(?:Cell|Nucleus|Cytoplasm|Membrane)\b|.*: ") -# the unit of a QuPath measurement, e.g. `Cell: Area µm^2` -_QUPATH_UNIT = re.compile(r" (µm|um|px)(?:\^2)?$") -# QuPath shape and detection measurements without a unit, e.g. of a detection export without compartments -_QUPATH_UNITLESS_MEASUREMENTS = frozenset({"Circularity", "Detection probability", "Eccentricity", "Solidity"}) -_QUPATH_IMAGE = "Image" -_QUPATH_ID = "Object ID" -# QuPath's centroid columns by unit; R's `make.names` writes `Centroid X µm` as `Centroid.X.um` -_QUPATH_CENTROIDS = {"um": ("Centroid X µm", "Centroid Y µm"), "px": ("Centroid X px", "Centroid Y px")} -# the QuPath object types of cells; other features of a GeoJSON export (e.g. annotations) are not cells -_QUPATH_CELL_TYPES = frozenset({"cell", "detection", "PathCellObject", "PathDetectionObject"}) -# inForm `*_cell_seg_data.txt` columns: `Cell X Position (microns)`, `Entire Cell Mean ()`, ... -_INFORM_POSITION = re.compile(r"Cell X Position(?: \((.*)\))?") -_INFORM_MARKER = re.compile(r"Entire Cell (.+) Mean(?: \(.*\))?") -_INFORM_AREA = re.compile(r"Entire Cell Area \((.*)\)") -# the unit of an inForm measurement, e.g. `Tissue Category Area (square microns)` -_INFORM_UNIT = re.compile(r"\((?:square )?(microns|pixels)\)$") -# the pages of an inForm `*_binary_seg_maps.tif`, named by the root tag of their XML description: the tissue class -# map `` (named `Tissue`), and a `` per compartment (named after the compartment) -_INFORM_XML_ROOT = re.compile(r"<([A-Za-z][\w-]*)") -_INFORM_SEGMENTATION = "SegmentationImage" -_INFORM_COMPARTMENT = re.compile(r"\s*(\w+)\s*") -_INFORM_PAGE_NAMES = {"TissueClassMap": "Tissue"} -# the default mask of an inForm segmentation map: the nuclei carry the `Cell ID` of every cell -_INFORM_DEFAULT_MASK = "Nucleus" -_INFORM_REGION = "Sample Name" -_INFORM_ID = "Cell ID" -# HuBMAP SPRM outputs: `-cell_channel_mean.csv` with the centroids in `-cell_centers.csv` -_SPRM_MEANS_SUFFIX = "-cell_channel_mean.csv" -_SPRM_CENTERS_SUFFIX = "-cell_centers.csv" -# a matched polygon must lie within this distance (px) of its cell's centroid, else the match is considered wrong -_CENTROID_TOLERANCE_PX = 1.0 -# the largest fraction of matched polygons that may be far from their centroids before the match is rejected -_MAX_MISPLACED_FRACTION = 0.05 -# the largest fraction of a region's cells that may lie outside its image without a warning -_MAX_OUTSIDE_FRACTION = 0.05 @dataclass class CellTable: - """A cell table in reader-independent form. + """A CODEX cell table. Attributes ---------- @@ -157,22 +83,11 @@ class CellTable: instance_id The instance id of every cell (int64). centroids_px - The ``(x, y)`` centroids in level-0 pixels of the region image, or `None` if the table has none (SPRM without - ``*-cell_centers.csv``). + The ``(x, y)`` centroids in level-0 pixels of the region image. radius_px The radius of a circle with the cell's area, or `None` if the table has no area. - mask_pixels - Whether ``centroids_px`` and ``radius_px`` are in pixels of the segmentation mask (pixel coordinates of - mask-based pipelines: mcquant, SPRM, ``"generic"`` with ``coordinates_space="mask"``) rather than of the - image; they are scaled to image pixels when the mask is a downsample of the image, see - :func:`build_spatialdata`. row_instance_ids Whether the instance ids are row numbers because the table has no instance id column. - polygons_px - The outline of every cell in level-0 pixels (`None` for a cell without one), or `None` if the table has no - polygons. - geojson - The file of ``polygons_px``. """ obs: pd.DataFrame @@ -180,12 +95,9 @@ class CellTable: var: pd.DataFrame region: pd.Series instance_id: pd.Series - centroids_px: np.ndarray | None + centroids_px: np.ndarray radius_px: np.ndarray | None row_instance_ids: bool = False - polygons_px: np.ndarray | None = None - geojson: Path | None = None - mask_pixels: bool = False def take(self, rows: np.ndarray) -> CellTable: """The cells at the positional indices ``rows``.""" @@ -195,536 +107,78 @@ def take(self, rows: np.ndarray) -> CellTable: var=self.var, region=self.region.iloc[rows], instance_id=self.instance_id.iloc[rows], - centroids_px=None if self.centroids_px is None else self.centroids_px[rows], + centroids_px=self.centroids_px[rows], radius_px=None if self.radius_px is None else self.radius_px[rows], row_instance_ids=self.row_instance_ids, - polygons_px=None if self.polygons_px is None else self.polygons_px[rows], - geojson=self.geojson, - mask_pixels=self.mask_pixels, ) -def detect_table_format(path: Path) -> str: - """Detect the format of a cell table from its file type and header. - - Parameters - ---------- - path - The ``.fcs``, ``.csv``, ``.tsv`` or ``.txt`` cell table. A text table is tab-separated if its header has a - tab, else semicolon-separated (with decimal commas or points) if it has more semicolons than commas, else - comma-separated. +def is_codex_table(path: Path) -> bool: + """Whether ``path`` is a CODEX Processor ``.fcs`` or ``.csv`` cell table. - Returns - ------- - ``"codex"`` (``x``/``y`` with ``cell_id`` or ``cycCCC_chHHH`` columns, also as the channels of an FCS file), - ``"qupath"`` (``Centroid X µm``, ``Centroid X px`` or ``Object ID``, also in R's ``make.names`` form, e.g. - ``Centroid.X.um``), ``"mcquant"`` (``CellID`` and ``X_centroid``), ``"inform"`` (``Cell X Position``, - optionally followed by its unit in parentheses), ``"sprm"`` (first column ``ID``, no ``x`` or ``y``), else - ``"generic"``. - - Raises - ------ - ValueError - If ``path`` is an FCS file that cannot be parsed or is not a CODEX table (its ``$PnN`` channel names). + Its columns (the ``$PnN`` channel names of an FCS file) must include ``x`` and ``y`` and ``cell_id`` or a + ``cycCCC_chHHH`` column. Other CSV files (e.g. antibody lists) and FCS files of other instruments, also + unreadable ones, are not. """ if path.suffix.lower() == CodexKeys.FCS_FILE.value: try: columns = list(flowio.read_multiple_data_sets(str(path), only_text=True)[0].pnn_labels) # flowio raises these, not only its own exceptions, for malformed files (e.g. truncated, without `$DATATYPE`) - except (ValueError, KeyError, IndexError, EOFError, OSError, flowio.exceptions.FlowIOException) as e: - raise ValueError(f"{path} is not a readable FCS file: {e}") from e - if _detect_header_format(columns) != "codex": - raise ValueError(f"{path} is not a CODEX cell table: its channels are {columns}.") - return "codex" - return _detect_header_format(_read_header(path)[0]) - - -def _detect_header_format(columns: list[str]) -> str: + except (ValueError, KeyError, IndexError, EOFError, OSError, flowio.exceptions.FlowIOException): + return False + else: + columns = _read_header(path) # the name part of the CODEX Processor header form `cyc001_ch001:DAPI1` names = {column.partition(":")[0].casefold() for column in columns} - if {"x", "y"} <= names and ( + return {"x", "y"} <= names and ( CodexKeys.INSTANCE_KEY.value in names or any(_MARKER_COLUMN.fullmatch(name) for name in names) - ): - return "codex" - qupath = {_QUPATH_ID, *(name for names in _QUPATH_CENTROIDS.values() for name in names)} - if qupath & set(_qupath_names(columns)): - return "qupath" - if {"CellID", "X_centroid"} <= set(columns): - return "mcquant" - if any(_INFORM_POSITION.fullmatch(column) for column in columns): - return "inform" - # SPRM means have no centroids, which are in `*-cell_centers.csv` - if columns[:1] == ["ID"] and not {"x", "y"} & names: - return "sprm" - return "generic" - - -def read_cell_table( - path: Path, - fmt: str, - *, - pixel_size_um: float | Mapping[str, float | None] | None, - default_region: str | None, - cells_table_kwargs: Mapping[str, Any], - regions: Mapping[str, Sequence[Path]] | None = None, -) -> CellTable: - """Read a cell table of any supported format. + ) + + +def read_cell_table(path: Path, *, default_region: str | None, regions: Collection[str] | None = None) -> CellTable: + """Read a CODEX Processor ``.fcs`` or ``.csv`` cell table. + + The markers are the ``cycCCC_chHHH`` columns, named by the FCS ``$PnS`` keywords or by the display part of the + Processor CSV header ``cycCCC_chHHH:`` (in ``var`` with their ``cycle`` and ``channel``), else, with a + `UserWarning`, the numeric columns that are not CODEX Processor columns (``x``, ``size``, ...). Centroids + ``x``/``y`` in pixels, instance ids ``cell_id`` (else the row number in the file, from 1), the region from + ``region`` (a numeric region ``r`` is the region id ``regRRR``), the radius from ``size``. Parameters ---------- path The cell table. - fmt - One of ``"auto"`` (see :func:`detect_table_format`; ``"generic"`` if ``cells_table_kwargs`` has ``x`` and - ``y``), ``"codex"``, ``"qupath"``, ``"mcquant"``, - ``"sprm"``, ``"inform"`` or ``"generic"``: - - - ``"codex"``: a CODEX Processor ``.fcs`` or ``.csv`` table. The markers are ``markers``, else the - ``cycCCC_chHHH`` columns, named by the FCS ``$PnS`` keywords or by the display part of the Processor - CSV header ``cycCCC_chHHH:`` (in ``var`` with their ``cycle`` and ``channel``), else, with a - `UserWarning`, the numeric columns that are not CODEX Processor columns (``x``, ``size``, ...). - Centroids ``x``/``y`` in - pixels, instance ids ``cell_id``, the region from ``region`` (a numeric region ``r`` is the region id - ``regRRR``), the radius from ``size``. - - ``"qupath"``: a QuPath measurement export, also with R's ``make.names`` column names - (``Centroid.X.um``, ``Object.ID``). Centroids ``Centroid X µm``/``Centroid Y µm`` (divided by - ``pixel_size_um``) or ``Centroid X px``/``Centroid Y px``, else ``X``/``Y`` in ``coordinates_unit`` - or, without it, in the unit of the other measurements (e.g. ``Cell: Area µm^2``) with a - `UserWarning`, raising if that is not unique. - Instance ids from a numeric ``Object ID``, else the row number in the file (from 1); the region from the - ``Image`` column. The markers are ``markers``, else the ``: : Mean`` columns - (also ``....Mean`` and ``__Mean``), ``: - : Mean`` (QuPath >= 0.6) and ``: mean`` (built-in cell detection) - columns, one per marker from the first of the ``Cell``, ``Nucleus``, ``Cytoplasm`` and ``Membrane`` - compartments (in ``var["compartment"]``), else the ``: Mean`` columns of detections - without compartments, else, with a `UserWarning`, the numeric columns that are not QuPath - measurements or empty (e.g. ``CD44`` of a detection export; not morphology such as ``Area µm^2``, - ``Max diameter µm`` or ``Circularity``). The radius comes from ``Cell: Area µm^2`` (or - ``px^2``) in the unit of the centroids. A table split into centroids and measurements (two files) - must be joined on the object id first. The cell outlines are read from a QuPath GeoJSON export, see - :func:`read_qupath_polygons`: ``cells_table_kwargs["geojson"]`` (``False`` for none), else - ``
.geojson`` or ``
_cells.geojson`` next to the table, if it exists and holds - any of the table's cells. - - ``"mcquant"``: an MCMICRO ``mcquant`` table. Instance ids ``CellID``, pixel centroids - ``X_centroid``/``Y_centroid``, the markers are the columns between ``CellID`` and ``X_centroid``, the - radius comes from ``Area``. - - ``"sprm"``: a HuBMAP SPRM ``-cell_channel_mean.csv`` (validated on synthetic data only). Instance - ids ``ID`` (the values of the mask, ``pipeline_output/mask/*.ome.tiff`` channel 0, passed as ``labels``), - one marker per remaining column. Pixel centroids come from ``-cell_centers.csv`` (``ID,x,y``) - next to the table, which must hold every cell; without it a `UserWarning` is raised and the table has - no ``obsm["spatial"]`` (the centroids are never computed from the mask), so its region needs a mask. - - ``"inform"``: a PerkinElmer/Akoya inForm ``*_cell_seg_data.txt`` (tab-separated; spaces around the - column names are removed). Instance ids ``Cell ID``, centroids ``Cell X Position``/``Cell Y - Position`` in the unit named in their header, ``(microns)`` (divided by ``pixel_size_um``) or - ``(pixels)``; if it names none, in ``coordinates_unit`` or, without it, in the unit of the other - columns (e.g. ``Tissue Category Area (square microns)``; inForm writes all of them in one unit), - raising if they name both, pixels if they name none. The region comes from ``Sample Name``. The - markers are the ``Entire Cell Mean (...)`` columns; the radius comes from - ``Entire Cell Area (...)`` if its unit equals that of the positions. Positions in microns may be - relative to the slide rather than to the field image, see :func:`build_spatialdata`. - - ``"generic"``: columns given by ``cells_table_kwargs``. - - pixel_size_um - The pixel size of the region image, to convert µm centroids to pixels; for a table of several regions, the - pixel size of each region, by region id: every cell is converted with the pixel size of its region, cells - of other regions get NaN coordinates. default_region The region id of all cells if the table has no region column. regions - The region ids of the dataset with the image files of each, to which the values of the region column are - matched, see :func:`_match_region`; values that name none of them are kept. If the table has + The region ids of the dataset, to which the values of the region column are matched, also after + :func:`spatialdata.sanitize_name`; values that name none of them are kept. If the table has ``default_region`` and its region column holds a single value that names no region, that value names - ``default_region`` (e.g. the source image of a table for one region). If `None`, the values are the - region ids. - cells_table_kwargs - Optional keys: ``region_column`` (the column holding the region of each cell; QuPath default ``Image``), - and for ``"generic"`` tables ``x`` and ``y`` (required centroid columns), ``instance_id`` (else the row - number in the file, from 1), ``markers`` (a list of columns or a regular expression matching the full - column name; default: all other numeric columns; also for ``"qupath"`` and ``"codex"`` tables, a CODEX - marker named by its name ``DAPI1``, its CSV header ``cyc001_ch001:DAPI1`` or its column ``cyc001_ch001``) - and ``coordinates_unit`` (``"px"``, - default, or ``"um"``; for ``"qupath"`` and ``"inform"`` tables the unit of centroid columns whose name - has none) and ``coordinates_space`` (the pixels of ``"px"`` centroids: ``"image"``, default, those of the - full-resolution region image, or ``"mask"``, those of the region's mask, scaled to image pixels like the - ``"mcquant"`` and ``"sprm"`` centroids if the mask is downsampled); for ``"qupath"`` tables ``geojson`` (the GeoJSON file of the cell outlines) and - ``geojson_units`` (the unit of its coordinates, ``"px"`` of the full-resolution image, default and what - QuPath writes, or ``"um"``). The key ``mask_channel`` is used by :func:`read_labels`. + ``default_region``. If `None`, the values are the region ids. Returns ------- - The cell table. Its regions are the (matched) values of the region column, or ``default_region``. + The cell table. """ - check_cells_table_options(fmt, cells_table_kwargs) - if fmt == "auto": - # centroid columns given by the user make a generic table - fmt = "generic" if {"x", "y"} <= cells_table_kwargs.keys() else detect_table_format(path) - logger.info("Reading %s as a %r cell table.", path, fmt) - if fmt != "qupath" and cells_table_kwargs.get("geojson") not in (None, False): - raise ValueError(f"`cells_table_kwargs['geojson']` is only read with QuPath cell tables; {path} is {fmt!r}.") - if fmt != "generic" and cells_table_kwargs.get("coordinates_space") is not None: - raise ValueError( - f"`cells_table_kwargs['coordinates_space']` is only used with generic cell tables; {path} is {fmt!r}." - ) - - if path.suffix.lower() == CodexKeys.FCS_FILE.value: - df = _read_fcs(path) - else: - df = _read_text_table(path) - reader = { - "codex": _codex_roles, - "qupath": _qupath_roles, - "mcquant": _mcquant_roles, - "sprm": _sprm_roles, - "inform": _inform_roles, - "generic": _generic_roles, - }[fmt] - roles = reader(path, df, cells_table_kwargs) - mask_pixels = fmt in {"mcquant", "sprm"} - if fmt == "generic": - mask_pixels = _coordinates_space(path, roles["in_um"], cells_table_kwargs) == "mask" - file_rows = None - if roles["region_column"] in df.columns and (empty := df[roles["region_column"]].isna().to_numpy()).any(): - warnings.warn( - f"Ignoring the {empty.sum()} cell(s) of {path} without a region in '{roles['region_column']}'.", - UserWarning, - stacklevel=4, - ) - file_rows = np.flatnonzero(~empty) - df = df.loc[~empty].reset_index(drop=True) - region = _region_values(df, roles["region_column"], default_region, regions) - units = cells_table_kwargs.get("geojson_units", "px") - row_pixel_sizes = None - if roles["in_um"] or (fmt == "qupath" and units == "um"): - row_pixel_sizes = _row_pixel_sizes(path, region, pixel_size_um) - if roles.pop("in_um"): - if row_pixel_sizes is None: - raise ValueError( - f"{path} has centroids in µm, but no pixel size is known to convert them to pixels (the region's " - "pixel size is missing)." - ) - roles["px_per_unit"] = 1 / row_pixel_sizes - table = _from_frame(path, df, region=region, file_rows=file_rows, **roles) - table.mask_pixels = mask_pixels - if fmt == "sprm": - table.centroids_px = _read_sprm_centers(path, table) - if fmt == "qupath": - option = cells_table_kwargs.get("geojson") - geojson = _find_geojson(path) if option is None else None if option is False else Path(option) - if geojson is not None: - table.polygons_px = read_qupath_polygons( - geojson, table, path, units=units, pixel_size_um=row_pixel_sizes, discovered=option is None - ) - table.geojson = None if table.polygons_px is None else geojson - return table - - -def _region_values( - df: pd.DataFrame, - region_column: str | None, - default_region: str | None, - regions: Mapping[str, Sequence[Path]] | None, -) -> pd.Series | None: - """The region of every row of ``df``, see :func:`read_cell_table`; `None` if it is unknown.""" - if region_column is None or region_column not in df.columns: - return None if default_region is None else pd.Series(default_region, index=df.index) - values = df[region_column].astype(str) - if regions is None: - return values - matched = {value: _match_region(value, regions) for value in values.unique()} - if default_region is not None and len(matched) == 1 and None in matched.values(): - # the region column of a table for one region names its source image only - return pd.Series(default_region, index=df.index) - return values.map({value: value if region is None else region for value, region in matched.items()}) - - -def check_cells_table_options(fmt: str, cells_table_kwargs: Mapping[str, Any]) -> None: - """Raise for an unknown cell table format or key of ``cells_table_kwargs``, also when no table is read.""" - if fmt != "auto" and fmt not in TABLE_FORMATS: - raise ValueError(f"Unknown cell table format {fmt!r}; expected 'auto' or one of {TABLE_FORMATS}.") - unknown = sorted(set(cells_table_kwargs) - set(CELLS_TABLE_KWARGS)) - if unknown: - raise ValueError(f"`cells_table_kwargs` has unknown key(s) {unknown}; expected some of {CELLS_TABLE_KWARGS}.") - - -def _row_pixel_sizes( - path: Path, region: pd.Series | None, pixel_size_um: float | Mapping[str, float | None] | None -) -> float | np.ndarray | None: - """The pixel size of every row, by its ``region``, or one for all rows; `None` if it is unknown. - - For pixel sizes by region id, rows of other regions get NaN (their cells are not read). Raises if a region of - the rows has no pixel size. - """ - if not isinstance(pixel_size_um, Mapping): - return pixel_size_um - if region is None: - sizes = set(pixel_size_um.values()) - return sizes.pop() if len(sizes) == 1 else None - unknown = sorted(r for r in region.unique() if r in pixel_size_um and pixel_size_um[r] is None) - if unknown: - raise ValueError( - f"{path} has coordinates in µm, but the pixel size of region(s) {unknown} is unknown, so they cannot be " - "converted to pixels." - ) - sizes_by_region = {r: pixel_size_um.get(r, np.nan) for r in region.unique()} - return region.map(sizes_by_region).to_numpy(dtype=np.float64) - - -def _qupath_names(columns: Sequence[str]) -> dict[str, str]: - """The columns by their QuPath name: as written, or undoing R's ``make.names``. - - ``make.names`` replaces ``: ``, `` `` and ``^`` by dots: ``Centroid.X.um`` is ``Centroid X µm`` and - ``Cell..Area.µm.2`` is ``Cell: Area µm^2``. - """ - names = {} - for column in columns: - name = re.sub(r"\.2$", "^2", column.replace("..", ": ")).replace(".", " ") - names[re.sub(r" um(?=\^2$|$)", " µm", name)] = column - return names | {column: column for column in columns} - - -def _object_id_column(columns: Sequence[str]) -> str | None: - """The QuPath ``Object ID`` column (also R's ``Object.ID``), or `None`.""" - return _qupath_names(columns).get(_QUPATH_ID) - - -def _find_geojson(table_path: Path) -> Path | None: - """The GeoJSON next to a QuPath table: ``.geojson``, else ``_cells.geojson``. - - Other GeoJSON files, e.g. HuBMAP's ``*tissue-boundary.geojson`` tissue outline, are not cells. - """ - candidates = (table_path.with_name(f"{table_path.stem}{suffix}.geojson") for suffix in ("", "_cells")) - return next((candidate for candidate in candidates if candidate.is_file()), None) - - -def read_qupath_polygons( - geojson: Path, - table: CellTable, - table_path: Path, - *, - units: str = "px", - pixel_size_um: float | np.ndarray | None = None, - discovered: bool = False, -) -> np.ndarray | None: - """Read the cell outlines of a QuPath GeoJSON export and match them to the cells of its measurement table. - - A feature belongs to the cell whose ``Object ID`` (or R's ``Object.ID``) is the feature ``id`` (the object's - UUID, QuPath >= 0.4); features of other objects (e.g. annotations) are skipped, and so are cell features without - a table row (e.g. cells filtered out of the table), with a `UserWarning`. If the table has no ``Object ID`` - column or the features have no ids, the cell and detection features (``objectType``) are matched to the table - rows in order, and their number must equal the number of rows. - - Every matched polygon must contain its cell's centroid or lie within 1 px of it; if more than 5% do not, the - polygons belong to other cells (or ``units`` is wrong) and a `ValueError` is raised. - - Parameters - ---------- - geojson - A GeoJSON ``FeatureCollection`` or JSON array of features (QuPath *Export as GeoJSON*). - table - The cell table read from ``table_path``. - table_path - The QuPath measurement table, for error messages. - units - The unit of the coordinates: ``"px"`` (of the full-resolution image, what QuPath writes) or ``"um"``. - pixel_size_um - The pixel size, to convert ``"um"`` coordinates, or the pixel size of every row of ``table``. - discovered - Whether ``geojson`` was found next to the table rather than given: if it holds none of the table's cells - (e.g. QuPath's export of an annotation), it is ignored with a `UserWarning` instead of raising. - - Returns - ------- - The polygon of every row of ``table`` in pixels, `None` for rows without one; `None` for an ignored - ``geojson``. - """ - if units not in ("px", "um"): - raise ValueError(f"`cells_table_kwargs['geojson_units']` must be 'px' or 'um', got {units!r}.") - if units == "um" and pixel_size_um is None: - raise ValueError(f"{geojson} has coordinates in µm, but no pixel size is known to convert them to pixels.") - if not geojson.is_file(): - raise FileNotFoundError(f"{geojson} does not exist or is not a file.") - ids, geometries, kinds = _read_geojson(geojson) - - polygons = np.full(len(table.obs), None, dtype=object) - id_column = _object_id_column(list(table.obs.columns)) - if id_column is not None and ids is not None: - object_ids = pd.Index(table.obs[id_column].astype(str)) - if not object_ids.is_unique: - raise ValueError(f"The '{id_column}' column of {table_path} is not unique.") - rows = object_ids.get_indexer(pd.Index(ids).astype(str)) - found = rows >= 0 - if not found.any() and discovered: - _ignore_geojson(geojson, table_path) - return None - if not found.any(): - raise ValueError( - f"{geojson} does not hold the cells of {table_path}: none of the feature ids is an '{id_column}' " - "of the table." - ) - if not pd.Index(rows[found]).is_unique: - raise ValueError(f"{geojson} has several features with the same id.") - polygons[rows[found]] = geometries[found] - logger.info("Matched %d of the %d features of %s to cells by id.", found.sum(), len(geometries), geojson) - is_cell = np.ones(len(ids), dtype=bool) if kinds is None else np.isin(kinds, list(_QUPATH_CELL_TYPES)) - if unmatched := int(np.count_nonzero(is_cell & ~found)): - warnings.warn( - f"{unmatched} of the {is_cell.sum()} cell features of {geojson} have no row in {table_path} (e.g. " - "cells filtered out of the table); they are ignored.", - UserWarning, - stacklevel=5, - ) - if found.sum() < len(polygons): - warnings.warn( - f"{len(polygons) - found.sum()} of the {len(polygons)} cells of {table_path} have no polygon in " - f"{geojson}; they are in the table but not in the labels and polygons.", - UserWarning, - stacklevel=5, - ) - else: - cells = geometries if kinds is None else geometries[np.isin(kinds, list(_QUPATH_CELL_TYPES))] - if len(cells) == 0 and discovered: - _ignore_geojson(geojson, table_path) - return None - if len(cells) != len(polygons): - raise ValueError( - f"{geojson} has {len(cells)} cell polygons without ids, but {table_path} has {len(polygons)} rows; " - f"without ids ('{_QUPATH_ID}'), polygons are matched to cells in order." - ) - polygons[:] = cells - if units == "um": - _scale_polygons(polygons, 1 / np.broadcast_to(np.asarray(pixel_size_um, dtype=np.float64), polygons.shape)) - if table.centroids_px is not None: - _check_centroids(polygons, table.centroids_px, geojson, table_path) - return polygons - - -def _ignore_geojson(geojson: Path, table_path: Path) -> None: - warnings.warn( - f"Ignoring {geojson}: it holds none of the cells of {table_path}. Pass the cell outlines as " - "`cells_table_kwargs['geojson']`.", - UserWarning, - stacklevel=6, - ) - - -def _scale_polygons(polygons: np.ndarray, factors: np.ndarray) -> None: - """Scale every polygon by its factor in place; polygons with a NaN factor (cells that are not read) are dropped.""" - polygons[np.isnan(factors)] = None - for factor in np.unique(factors[~np.isnan(factors)]): - rows = np.flatnonzero((factors == factor) & pd.notna(polygons)) - polygons[rows] = shapely.transform(polygons[rows], partial(np.multiply, factor)) - - -def _check_centroids(polygons: np.ndarray, centroids_px: np.ndarray, geojson: Path, table_path: Path) -> None: - """Raise if too many polygons are far from the centroids of the cells they were matched to.""" - # cells of regions that are not read have no pixel coordinates - matched = pd.notna(polygons) & np.isfinite(centroids_px).all(axis=1) - if not matched.any(): - return - distance = shapely.distance(polygons[matched], shapely.points(centroids_px[matched])) - misplaced = int(np.count_nonzero(~(distance <= _CENTROID_TOLERANCE_PX))) - if misplaced > _MAX_MISPLACED_FRACTION * matched.sum(): - raise ValueError( - f"{misplaced} of the {matched.sum()} polygons of {geojson} are more than {_CENTROID_TOLERANCE_PX:g} px " - f"from the centroid of the cell of {table_path} they were matched to: the polygons are not those of " - "these cells, are in another order, or `cells_table_kwargs['geojson_units']` is wrong." - ) - if misplaced: - logger.info( - "%d polygons of %s are more than %g px from their cell's centroid.", - misplaced, - geojson, - _CENTROID_TOLERANCE_PX, - ) - - -def _read_geojson(path: Path) -> tuple[np.ndarray | None, np.ndarray, np.ndarray | None]: - """The ids (`None` if there are none), geometries and QuPath object types (`None` if unknown) of the features.""" - with path.open("rb") as f: - is_array = f.read(1024).lstrip(b"\xef\xbb\xbf \t\r\n").startswith(b"[") - if is_array: - # QuPath's default export without the FeatureCollection option, which GDAL does not read; each feature is - # reduced as soon as it is parsed, so that its other properties are not all held at once - with path.open("rb") as f: - features = json.load(f, object_hook=_reduce_feature) - ids = [feature[0] for feature in features] - geometries = np.array([feature[1] for feature in features], dtype=object) - types = [feature[2] for feature in features] - else: - # QuPath's other properties (measurements, nucleus outlines) would take several times the memory - frame = gpd.read_file(path, columns=["id", "objectType"]) - geometries = frame.geometry.to_numpy() - ids = list(frame["id"]) if "id" in frame.columns else [None] * len(frame) - types = list(frame["objectType"]) if "objectType" in frame.columns else [None] * len(frame) - if all(kind is None for kind in types) and any(i in _QUPATH_CELL_TYPES for i in ids): - # QuPath < 0.4 names the object type in the feature id, e.g. `PathCellObject` - types, ids = ids, [None] * len(ids) - has_ids = any(i is not None and not pd.isna(i) for i in ids) - has_types = any(kind is not None for kind in types) - return ( - np.asarray(ids, dtype=object) if has_ids else None, - geometries, - np.asarray(types, dtype=object) if has_types else None, - ) - - -def _reduce_feature(obj: dict[str, Any]) -> Any: - """The id, geometry and QuPath object type of a parsed GeoJSON feature; other JSON objects unchanged.""" - if "geometry" not in obj: - return obj - geometry = shapely.geometry.shape(obj["geometry"]) if obj.get("geometry") else None - return obj.get("id"), geometry, (obj.get("properties") or {}).get("objectType") - - -def _read_fcs(path: Path) -> pd.DataFrame: - """The events of an FCS file, with the columns named ``$PnN:$PnS`` like the CODEX Processor CSV header.""" - fcs = readfcs.ReadFCS(path) - df = fcs.data # positional columns, named by `$PnN` - labels = fcs.channels["PnS"] if "PnS" in fcs.channels.columns else [""] * df.shape[1] - names = fcs.channels["PnN"] - df.columns = pd.Index( - [f"{name}:{label.strip()}" if label.strip() else name for name, label in zip(names, labels, strict=True)] - ) - return df - - -def _codex_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: - """The roles of a CODEX table; renames the columns of ``df`` to the name part of ``cycCCC_chHHH:``.""" + df = _read_fcs(path) if path.suffix.lower() == CodexKeys.FCS_FILE.value else _read_text_table(path) headers = [str(column) for column in df.columns] split = [header.partition(":") for header in headers] # the source of an FCS column is its `$PnN` fcs = path.suffix.lower() == CodexKeys.FCS_FILE.value - sources = [name if fcs else str(column) for column, (name, _, _) in zip(df.columns, split, strict=True)] + sources = [name if fcs else header for header, (name, _, _) in zip(headers, split, strict=True)] names = dedupe_channel_names([name for name, _, _ in split]) displays = [display.strip() or name for name, (_, _, display) in zip(names, split, strict=True)] df.columns = pd.Index(names) columns = {column.casefold(): column for column in names} - x, y = kwargs.get("x", columns.get("x")), kwargs.get("y", columns.get("y")) + x, y = columns.get("x"), columns.get("y") if x is None or y is None: raise ValueError(f"{path} has no 'x' and 'y' centroid columns; found {names}.") - instance_column = kwargs.get("instance_id", columns.get(CodexKeys.INSTANCE_KEY.value)) - region_column = kwargs.get("region_column", columns.get(CodexKeys.REGION_KEY.value)) - if region_column in df.columns and pd.api.types.is_numeric_dtype(df[region_column]): - values = df[region_column] - df[region_column] = values.map({value: f"reg{int(value):03d}" for value in values.dropna().unique()}) - area = columns.get("size") + instance_column = columns.get(CodexKeys.INSTANCE_KEY.value) + region_column = columns.get(CodexKeys.REGION_KEY.value) + area_column = columns.get("size") matches = [_MARKER_COLUMN.fullmatch(name) for name in names] - if (selection := kwargs.get("markers")) is not None: - # a marker is named by its column (`cyc001_ch001`), its CSV header (`cyc001_ch001:DAPI1`) or its name (`DAPI1`) - forms = [{*form} for form in zip(names, headers, displays, dedupe_channel_names(displays), strict=True)] - if isinstance(selection, str): - exclude = {x, y, instance_column, region_column, area} - is_marker = [ - name not in exclude and any(re.fullmatch(selection, form) for form in aliases) - for name, aliases in zip(names, forms, strict=True) - ] - else: - wanted = {str(marker) for marker in selection} - if missing := wanted.difference(*forms): - raise ValueError(f"{path} has no marker column(s) {sorted(missing)}; found {headers}.") - is_marker = [bool(aliases & wanted) for aliases in forms] - elif any(matches): + if any(matches): is_marker = [match is not None for match in matches] else: is_marker = [ @@ -734,11 +188,12 @@ def _codex_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dic taken = [name for name, marker in zip(names, is_marker, strict=True) if marker] warnings.warn( f"{path} has no 'cycCCC_chHHH' columns; reading the {len(taken)} numeric columns {taken} that are not " - "CODEX Processor columns as the markers. Set `cells_table_kwargs['markers']` to choose the marker columns.", + "CODEX Processor columns as the markers.", UserWarning, - stacklevel=5, + stacklevel=4, ) positions = [i for i, marker in enumerate(is_marker) if marker] + markers = [names[i] for i in positions] var = pd.DataFrame( {"source_column": [sources[i] for i in positions]}, index=pd.Index(dedupe_channel_names([displays[i] for i in positions])), @@ -746,368 +201,77 @@ def _codex_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dic if positions and all(matches[i] for i in positions): var["cycle"] = [int(matches[i].group(1)) for i in positions] # type: ignore[union-attr] var["channel"] = [int(matches[i].group(2)) for i in positions] # type: ignore[union-attr] - return { - "markers": [names[i] for i in positions], - "var": var, - "x": x, - "y": y, - "in_um": False, - "instance_column": instance_column, - "region_column": region_column, - "area_column": area, - } - - -def _qupath_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: - columns = _qupath_names(list(df.columns)) - unit = next((unit for unit, (x_name, _) in _QUPATH_CENTROIDS.items() if x_name in columns), None) - x_name, y_name = ("X", "Y") if unit is None else _QUPATH_CENTROIDS[unit] - x, y = columns.get(x_name), columns.get(y_name) - if x is None or y is None: - raise ValueError( - f"{path} has no centroid columns ('Centroid X µm' and 'Centroid Y µm', 'Centroid X px' and " - f"'Centroid Y px', or 'X' and 'Y'); found {list(df.columns)}. If the export splits the centroids and " - "the measurements into two files, join them on the object id first." - ) - if unit is None: - unit = _qupath_xy_unit(path, list(columns), kwargs) - object_id = _object_id_column(list(df.columns)) - instance_column = object_id if object_id is not None and pd.api.types.is_integer_dtype(df[object_id]) else None - region_column = kwargs.get("region_column", _QUPATH_IMAGE if _QUPATH_IMAGE in df.columns else None) - area = columns.get(f"Cell: Area {'µm' if unit == 'um' else 'px'}^2") - roles = {x, y, object_id, region_column, area} - - intensities: dict[str, dict[str | None, str]] = {} - for column in df.columns: - if (found := _qupath_intensity(column)) is not None: - intensities.setdefault(found[0], {}).setdefault(found[1], column) - # `: Mean` columns are the markers only of detections without compartments - with_compartments = {marker: found for marker, found in intensities.items() if set(found) - {None}} - compartments = with_compartments or intensities - if kwargs.get("markers") is not None: - markers = _select_markers(df, kwargs["markers"], roles) - var = pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(markers))) - elif compartments: - chosen = { - marker: next(c for c in (*_QUPATH_COMPARTMENTS, None) if c in found) - for marker, found in compartments.items() - } - markers = [compartments[marker][compartment] for marker, compartment in chosen.items()] - var = pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(list(chosen)))) - if with_compartments: - var["compartment"] = list(chosen.values()) - else: - # e.g. a detection export with the measurements named after the markers only; its morphology (e.g. - # `Area µm^2`, `Max diameter µm`, `Circularity`), other measurements (e.g. `DAPI: Median`) and empty text - # columns (e.g. `Name`, read as all-NaN numbers) are not markers - candidates = [ - c - for c in df.columns - if not any(_QUPATH_MEASUREMENT.match(name) for name in _qupath_names([c])) - and not _is_qupath_morphology(c) - and df[c].notna().any() - ] - markers = _select_markers(df[candidates], None, roles) - var = pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(markers))) - if markers: - shown = f"{markers[:10]}{' ...' if len(markers) > 10 else ''}" - message = f"reading the {len(markers)} other numeric columns {shown} as the markers" - else: - message = "its table has no marker intensities" - warnings.warn( - f"{path} has no QuPath mean intensity columns (e.g. ': : Mean'); {message}. Set " - "`cells_table_kwargs['markers']` to choose the marker columns.", - UserWarning, - stacklevel=5, - ) - return { - "markers": markers, - "var": var, - "x": x, - "y": y, - "in_um": unit == "um", - "instance_column": instance_column, - "region_column": region_column, - "area_column": area, - } - - -def _qupath_intensity(column: str) -> tuple[str, str | None] | None: - """The marker and compartment (`None` without one) of a QuPath mean intensity column, also in R's form.""" - for name in (column, *_qupath_names([column])): - for pattern in (_QUPATH_MARKER, _QUPATH_COMPARTMENT_MARKER): - if (match := pattern.fullmatch(name)) is not None: - return match["marker"], match["compartment"] - for name in (column, *_qupath_names([column])): - if (match := _QUPATH_DETECTION_MARKER.fullmatch(name)) is not None: - return match["marker"], None - return None - - -def _is_qupath_morphology(column: str) -> bool: - """Whether ``column`` (also in R's ``make.names`` form) is a QuPath measurement with a unit or a shape measure.""" - return any(_QUPATH_UNIT.search(name) or name in _QUPATH_UNITLESS_MEASUREMENTS for name in _qupath_names([column])) - - -def _qupath_xy_unit(path: Path, columns: list[str], kwargs: Mapping[str, Any]) -> str: - """The unit of the ``X``/``Y`` centroids of a QuPath table: ``coordinates_unit``, else that of its measurements. - - QuPath writes all measurements of an image in one unit, µm if the image is calibrated, else pixels. - """ - if kwargs.get("coordinates_unit") is not None: - return _coordinates_unit(kwargs) - units = {"px" if m.group(1) == "px" else "um" for column in columns if (m := _QUPATH_UNIT.search(column))} - if len(units) != 1: - found = f"in {sorted(units)}" if units else "in no unit" - raise ValueError( - f"The centroid columns 'X' and 'Y' of {path} name no unit, and its other measurements are {found}; set " - "`cells_table_kwargs['coordinates_unit']` to 'px' or 'um'." - ) - unit = units.pop() - warnings.warn( - f"The centroid columns 'X' and 'Y' of {path} name no unit; reading them in {unit}, the unit of its other " - "measurements. Set `cells_table_kwargs['coordinates_unit']` to 'px' or 'um' to choose it.", - UserWarning, - stacklevel=6, - ) - return unit - - -def _coordinates_unit(kwargs: Mapping[str, Any]) -> str: - """``cells_table_kwargs['coordinates_unit']``: ``"px"`` (the default) or ``"um"``.""" - unit = kwargs.get("coordinates_unit", "px") - if unit not in ("px", "um"): - raise ValueError(f"`cells_table_kwargs['coordinates_unit']` must be 'px' or 'um', got {unit!r}.") - return str(unit) - - -def _coordinates_space(path: Path, in_um: bool, kwargs: Mapping[str, Any]) -> str: - """``cells_table_kwargs['coordinates_space']`` of a generic table: ``"image"`` (the default) or ``"mask"``.""" - space = kwargs.get("coordinates_space", "image") - if space not in ("image", "mask"): - raise ValueError(f"`cells_table_kwargs['coordinates_space']` must be 'image' or 'mask', got {space!r}.") - if space == "mask" and in_um: - raise ValueError( - f"`cells_table_kwargs['coordinates_space']` is 'mask', but the centroids of {path} are in µm; mask " - "pixels need `cells_table_kwargs['coordinates_unit']` 'px'." - ) - return str(space) - - -def _select_markers(df: pd.DataFrame, selection: str | Sequence[str] | None, exclude: set[str | None]) -> list[str]: - """The marker columns: ``selection`` (columns, or a regular expression matching the full column name). - - Without ``selection``, all numeric columns. Columns in ``exclude`` are only selected if listed in ``selection``. - """ - if selection is None: - return [c for c in df.columns if c not in exclude and pd.api.types.is_numeric_dtype(df[c])] - if isinstance(selection, str): - return [c for c in df.columns if c not in exclude and re.fullmatch(selection, c)] - return [str(marker) for marker in selection] - - -def _mcquant_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: - columns = list(df.columns) - if "CellID" not in columns or "X_centroid" not in columns: - raise ValueError(f"{path} has no 'CellID' and 'X_centroid' columns; found {columns}.") - markers = columns[columns.index("CellID") + 1 : columns.index("X_centroid")] - return { - "markers": markers, - "var": pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(markers))), - "x": "X_centroid", - "y": "Y_centroid", - "in_um": False, - "instance_column": "CellID", - "region_column": kwargs.get("region_column"), - "area_column": "Area" if "Area" in columns else None, - } - - -def _sprm_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: - if "ID" not in df.columns: - raise ValueError(f"{path} has no 'ID' column; found {list(df.columns)}.") - if not pd.api.types.is_integer_dtype(df["ID"]): - raise ValueError(f"The 'ID' column of {path} must hold integer cell ids, got dtype {df['ID'].dtype}.") - region_column = kwargs.get("region_column") - markers = [c for c in df.columns if c not in ("ID", region_column)] - return { - "markers": markers, - "var": pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(markers))), - "x": None, - "y": None, - "in_um": False, - "instance_column": "ID", - "region_column": region_column, - "area_column": None, - } - - -def _read_sprm_centers(path: Path, table: CellTable) -> np.ndarray | None: - """The ``(x, y)`` pixel centroids of the cells of an SPRM table from its ``*-cell_centers.csv``, if it exists.""" - name = path.name - centers = path.with_name( - f"{name.removesuffix(_SPRM_MEANS_SUFFIX)}{_SPRM_CENTERS_SUFFIX}" - if name.endswith(_SPRM_MEANS_SUFFIX) - else f"{path.stem}{_SPRM_CENTERS_SUFFIX}" - ) - if not centers.is_file(): - warnings.warn( - f"No {centers.name} next to {path}, so the cells of this SPRM table have no centroids and the table has " - "no 'spatial' coordinates; the cells need a segmentation mask (`labels`).", - UserWarning, - stacklevel=4, - ) - return None - frame = pd.read_csv(centers) - if not {"ID", "x", "y"} <= set(frame.columns): - raise ValueError(f"{centers} must have the columns 'ID', 'x' and 'y'; found {list(frame.columns)}.") - if not frame["ID"].is_unique: - raise ValueError(f"The 'ID' column of {centers} is not unique.") - rows = pd.Index(frame["ID"]).get_indexer(pd.Index(table.instance_id)) - if (rows < 0).any(): - raise ValueError(f"{centers} has no centroid for {int((rows < 0).sum())} of the {len(rows)} cells of {path}.") - return frame[["x", "y"]].to_numpy(dtype=np.float64)[rows] - - -def _inform_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: - positions = [m for column in df.columns if (m := _INFORM_POSITION.fullmatch(column)) is not None] - y = next((c for c in df.columns if c.startswith("Cell Y Position")), None) - if not positions or y is None: - raise ValueError(f"{path} has no 'Cell X Position' and 'Cell Y Position' columns; found {list(df.columns)}.") - x = positions[0].group(0) - unit = (positions[0].group(1) or _inform_position_unit(path, list(df.columns), kwargs)).casefold() - if unit not in ("microns", "pixels"): - raise ValueError(f"The unit '{unit}' of the column '{x}' of {path} must be 'microns' or 'pixels'.") - markers, names = [], [] - for column in df.columns: - if (match := _INFORM_MARKER.fullmatch(column)) is not None: - markers.append(column) - names.append(match.group(1)) - if not markers: - warnings.warn( - f"{path} has no 'Entire Cell Mean' columns, so its table has no marker intensities.", - UserWarning, - stacklevel=5, - ) - area = next((c for c in df.columns if (m := _INFORM_AREA.fullmatch(c)) is not None and m.group(1)), None) - if area is not None: - area_unit = "microns" if "micron" in area else "pixels" - area = area if area_unit == unit else None - return { - "markers": markers, - "var": pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(names))), - "x": x, - "y": y, - "in_um": unit == "microns", - "instance_column": _INFORM_ID if _INFORM_ID in df.columns else None, - "region_column": kwargs.get("region_column", _INFORM_REGION if _INFORM_REGION in df.columns else None), - "area_column": area, - } - - -def _inform_position_unit(path: Path, columns: list[str], kwargs: Mapping[str, Any]) -> str: - """The unit of ``Cell X Position`` without one in its name: ``coordinates_unit``, else that of the other columns. - - inForm writes all columns in one unit, but names it only for some, e.g. ``Tissue Category Area (pixels)`` or - ``Distance from Process Region Edge (microns)``; a table that names none is in pixels. - """ - if kwargs.get("coordinates_unit") is not None: - return {"px": "pixels", "um": "microns"}[_coordinates_unit(kwargs)] - units = {m.group(1) for column in columns if (m := _INFORM_UNIT.search(column)) is not None} - if len(units) > 1: - raise ValueError( - f"The columns 'Cell X Position' and 'Cell Y Position' of {path} name no unit, and its other columns are " - f"in {sorted(units)}; set `cells_table_kwargs['coordinates_unit']` to 'px' or 'um'." - ) - unit = units.pop() if units else "pixels" - logger.info("Reading the cell positions of %s in %s, the unit of its other columns.", path, unit) - return unit - - -def _generic_roles(path: Path, df: pd.DataFrame, kwargs: Mapping[str, Any]) -> dict[str, Any]: - if "x" not in kwargs or "y" not in kwargs: - raise ValueError( - f"Reading {path} as a 'generic' cell table needs the centroid columns as `cells_table_kwargs` 'x' and " - f"'y'; found the columns {list(df.columns)}." - ) - unit = _coordinates_unit(kwargs) - roles = {kwargs["x"], kwargs["y"], kwargs.get("instance_id"), kwargs.get("region_column")} - markers = _select_markers(df, kwargs.get("markers"), roles) - return { - "markers": markers, - "var": pd.DataFrame({"source_column": markers}, index=pd.Index(dedupe_channel_names(markers))), - "x": kwargs["x"], - "y": kwargs["y"], - "in_um": unit == "um", - "instance_column": kwargs.get("instance_id"), - "region_column": kwargs.get("region_column"), - "area_column": None, - } - - -def _from_frame( - path: Path, - df: pd.DataFrame, - *, - markers: list[str], - var: pd.DataFrame, - x: str | None, - y: str | None, - instance_column: str | None, - region_column: str | None, - area_column: str | None, - region: pd.Series | None, - px_per_unit: float | np.ndarray = 1.0, - file_rows: np.ndarray | None = None, -) -> CellTable: - """Build a `CellTable` from a data frame, the roles of its columns and the region of every row. - - ``px_per_unit`` may be given per row. ``file_rows`` are the rows of the file that ``df`` holds (from 0), if it - holds not all of them. - """ - roles = [x, y, instance_column, region_column, area_column, *markers] - missing = [column for column in roles if column is not None and column not in df.columns] - if missing: - raise ValueError(f"{path} has no column(s) {missing}; found {list(df.columns)}.") if instance_column is not None: instance_id = pd.Series(_instance_ids(path, df[instance_column])) else: # the row numbers in the file; label images use 0 for the background, so instance ids start at 1 - rows = np.arange(len(df)) if file_rows is None else file_rows - instance_id = pd.Series(rows.astype(np.int64) + 1) + instance_id = pd.Series(np.arange(1, len(df) + 1, dtype=np.int64)) index = pd.Index(instance_id.astype(str)) instance_id.index = index instance_id.name = CodexKeys.INSTANCE_KEY.value + region = _region_values(df, region_column, default_region, regions) if region is None: raise ValueError( f"{path} has no region column, so its cells cannot be assigned to regions: a table for several regions " - "needs a region column (`cells_table_kwargs['region_column']`). Alternatively, pass one table per " - "region as a mapping {region: path}." + "needs a 'region' column. Alternatively, pass one table per region as a mapping {region: path}." ) region = pd.Series(region.to_numpy(), index=index, name=CodexKeys.REGION_KEY.value) _check_unique_instance_ids(path, region, instance_id) - obs = _drop_key_columns(df.drop(columns=markers), instance_column, region_column) + # the instance ids and regions are stored under the keys of the table element + obs = df.drop(columns=[*markers, *(c for c in (instance_column, region_column) if c is not None)]) obs.index = index - radius = None - if area_column is not None: - radius = np.sqrt(df[area_column].to_numpy(dtype=np.float64) * px_per_unit**2 / np.pi) + radius = None if area_column is None else np.sqrt(df[area_column].to_numpy(dtype=np.float64) / np.pi) return CellTable( obs=obs, X=df[markers].to_numpy(dtype=np.float32), var=var, region=region, instance_id=instance_id, - centroids_px=None - if x is None or y is None - else df[[x, y]].to_numpy(dtype=np.float64) * np.reshape(px_per_unit, (-1, 1)), + centroids_px=df[[x, y]].to_numpy(dtype=np.float64), radius_px=radius, row_instance_ids=instance_column is None, ) +def _region_values( + df: pd.DataFrame, region_column: str | None, default_region: str | None, regions: Collection[str] | None +) -> pd.Series | None: + """The region of every row of ``df``, see :func:`read_cell_table`; `None` if it is unknown.""" + if region_column is None: + return None if default_region is None else pd.Series(default_region, index=df.index) + values = df[region_column] + if pd.api.types.is_numeric_dtype(values): + values = values.map({value: f"reg{int(value):03d}" for value in values.dropna().unique()}) + values = values.astype(str) + if regions is None: + return values + matched = {value: _match_region(value, regions) for value in values.unique()} + if default_region is not None and len(matched) == 1 and None in matched.values(): + # the region column of a table for one region names another region (e.g. `1` of an image `Kidney.tif`) + return pd.Series(default_region, index=df.index) + return values.map({value: value if region is None else region for value, region in matched.items()}) + + +def _match_region(value: str, regions: Collection[str]) -> str | None: + """The region named by ``value``, also after :func:`spatialdata.sanitize_name` (``A [1]`` names ``A__1_``).""" + return next((candidate for candidate in (value, sanitize_name(value)) if candidate in regions), None) + + +def _read_fcs(path: Path) -> pd.DataFrame: + """The events of an FCS file, with the columns named ``$PnN:$PnS`` like the CODEX Processor CSV header.""" + fcs = readfcs.ReadFCS(path) + df = fcs.data # positional columns, named by `$PnN` + labels = fcs.channels["PnS"] if "PnS" in fcs.channels.columns else [""] * df.shape[1] + names = fcs.channels["PnN"] + df.columns = pd.Index( + [f"{name}:{label.strip()}" if label.strip() else name for name, label in zip(names, labels, strict=True)] + ) + return df + + def _instance_ids(path: Path, column: pd.Series) -> np.ndarray: """The integer instance ids in ``column``; raises for missing or non-integer ids, which no label can have.""" if pd.api.types.is_integer_dtype(column): @@ -1117,8 +281,7 @@ def _instance_ids(path: Path, column: pd.Series) -> np.ndarray: if invalid.any(): raise ValueError( f"{path} has {invalid.sum()} cell(s) whose instance id in '{column.name}' is missing or not an integer " - f"(e.g. {column[invalid].tolist()[:5]}); every cell needs an integer instance id, the value of its " - "label. Choose another instance id column, or remove these rows." + f"(e.g. {column[invalid].tolist()[:5]}); every cell needs an integer instance id, the value of its label." ) return ids.astype(np.int64) @@ -1132,50 +295,23 @@ def _check_unique_instance_ids(path: Path, region: pd.Series, instance_id: pd.Se found = sorted(set(ids[duplicated & (regions == first)].tolist())) raise ValueError( f"{path} has {len(found)} duplicate instance id(s) {found[:5]} in region '{first}'; every cell of a " - "region needs its own instance id. Choose another instance id column, or remove the duplicate rows." + "region needs its own instance id." ) -def _drop_key_columns(obs: pd.DataFrame, instance_column: str | None, region_column: str | None) -> pd.DataFrame: - """Drop the instance id and region columns if they are named like the keys whose values replace them. - - Any other column named like a key keeps its values as ``source_`` in the table element. - """ - consumed = {CodexKeys.INSTANCE_KEY.value: instance_column, CodexKeys.REGION_KEY.value: region_column} - return obs.drop(columns=[key for key, column in consumed.items() if column == key]) - - -def _read_header(path: Path) -> tuple[list[str], str]: - """The column names of a delimited text table and its delimiter. - - The delimiter is a tab if the header has one, else a semicolon if it has more semicolons than commas, else a comma. - """ +def _read_header(path: Path) -> list[str]: + """The column names of a comma-separated table.""" with path.open(encoding="utf-8-sig", errors="replace") as f: header = f.readline().rstrip("\r\n") - sep = "\t" if "\t" in header else ";" if header.count(";") > header.count(",") else "," - return [column.strip() for column in next(csv.reader([header], delimiter=sep))], sep + return [column.strip() for column in next(csv.reader([header]))] def _read_text_table(path: Path) -> pd.DataFrame: - """Read a delimited text table with the delimiter of :func:`_read_header`, with unique column names. - - Semicolon-separated tables (European locales) are read with decimal commas; their columns with decimal points, - which that leaves as text, are converted to numbers too. - """ - names, sep = _read_header(path) - if sep == ";": - df = pd.read_csv(path, sep=sep, decimal=",", engine="c", header=None, skiprows=1) - for i in range(df.shape[1]): - if not pd.api.types.is_numeric_dtype(df.iloc[:, i]): - try: - df.isetitem(i, pd.to_numeric(df.iloc[:, i]).to_numpy()) - except (ValueError, TypeError): - pass - else: - df = pd.read_csv(path, sep=sep, engine="pyarrow") + """Read a comma-separated table with unique column names.""" + df = pd.read_csv(path, engine="pyarrow") # pandas renames repeated names (DAPI.1; with the pyarrow engine since 3.1), so the names come from the header and # are made unique like the marker names - df.columns = pd.Index(dedupe_channel_names(names)) + df.columns = pd.Index(dedupe_channel_names(_read_header(path))) return df @@ -1223,74 +359,42 @@ def resolve_per_region( def read_region_tables( - files: Mapping[str, Path], - fmt: str, - *, - available: Mapping[str, Sequence[Path]], - selected: Sequence[str], - pixel_sizes: Mapping[str, float | None], - cells_table_kwargs: Mapping[str, Any], + files: Mapping[str, Path], *, available: Collection[str], selected: Sequence[str] ) -> list[CellTable]: - """Read user-supplied cell tables, as resolved by :func:`resolve_per_region`, and assign their cells to regions. + """Read cell tables, as resolved by :func:`resolve_per_region`, and assign their cells to regions. - The values of a region column are matched to the regions, see :func:`_match_region`. A table given for one - region holds the cells of that region; if its region column holds a single value that names no region, it - holds only cells of that region. Cells whose region names no available region are dropped with a warning; - cells of available regions that are not read are dropped silently. + A table given for one region holds the cells of that region; if its region column holds a single value that + names no region, it holds only cells of that region. Cells whose region names no available region are dropped + with a warning; cells of available regions that are not read are dropped silently. Parameters ---------- files The table of each region, or ``{"*": path}`` for one table for all regions. - fmt - The table format, see :func:`read_cell_table`. available - All region ids of the dataset, with the image files of each. + All region ids of the dataset. selected The region ids read. - pixel_sizes - The pixel size in µm of each region with an image. - cells_table_kwargs - See :func:`read_cell_table`; ``geojson`` may also be a mapping from region id to path, like ``files``. Returns ------- The cell tables, one per file. """ - geojson = cells_table_kwargs.get("geojson") - geojson_files = {} if geojson is False else resolve_per_region(geojson, selected, list(available)) - if unknown := sorted(geojson_files.keys() - files.keys()): - raise ValueError( - f"`cells_table_kwargs['geojson']` is given for {unknown}, but the cell tables are given for " - f"{list(files)}; pass the GeoJSON files in the same form as the tables." - ) tables = [] - # the µm coordinates of each cell are converted with the pixel size of its region - sizes = {region: pixel_sizes.get(region) for region in selected} for key, path in files.items(): - table = read_cell_table( - path, - fmt, - pixel_size_um=sizes, - default_region=None if key == "*" else key, - cells_table_kwargs={ - **cells_table_kwargs, - "geojson": False if geojson is False else geojson_files.get(key), - }, - regions=available, - ) + table = read_cell_table(path, default_region=None if key == "*" else key, regions=available) tables.append(_assign_regions(table, path, key, available, selected)) return tables def _assign_regions( - table: CellTable, path: Path, key: str, available: Mapping[str, Sequence[Path]], selected: Sequence[str] + table: CellTable, path: Path, key: str, available: Collection[str], selected: Sequence[str] ) -> CellTable: values = list(table.region.unique()) unmatched = sorted(value for value in values if value not in available) if unmatched: warnings.warn( - f"Ignoring the cells of {unmatched} in {path}: they match none of the regions {list(available)}.", + f"Ignoring the cells of {unmatched} in {path}: they match none of the regions {sorted(available)}.", UserWarning, stacklevel=4, ) @@ -1301,131 +405,40 @@ def _assign_regions( return table.take(np.flatnonzero(keep)) -def _match_region(value: str, regions: Mapping[str, Sequence[Path]]) -> str | None: - """The region named by ``value``: its id, else the region of an image file with that name or stem, else `None`. - - ``value`` is also matched after :func:`spatialdata.sanitize_name`, as the region ids are, so that e.g. ``A_[1,2]`` - names the region ``A__1_2_``. A file name may be followed by a non-alphanumeric character and more, e.g. QuPath's - ``A_Scan1.qptiff - resolution #1``; a name that merely starts with another region's name (``A_2_Scan1.qptiff``) - does not name it. - """ - candidates = list(dict.fromkeys((value, sanitize_name(value)))) - if exact := next((candidate for candidate in candidates if candidate in regions), None): - return exact - stem = IMAGE_EXTENSION.sub("", value) - for region, files in regions.items(): - for name in (file.name for file in files): - if stem == IMAGE_EXTENSION.sub("", name) or (value.startswith(name) and not value[len(name)].isalnum()): - return region - return None - - def read_labels( - path: Path, - labels_models_kwargs: Mapping[str, Any], - transformations: Mapping[str, BaseTransformation], - *, - mask_channel: int | str | None = None, + path: Path, labels_models_kwargs: Mapping[str, Any], transformations: Mapping[str, BaseTransformation] ) -> DataArray | DataTree: """Lazily read a segmentation mask as a ``Labels2DModel``. Parameters ---------- path - A TIFF or OME-TIFF with integer instance ids (0 = background). + A TIFF or OME-TIFF with integer instance ids (0 = background). Of a file with several planes, the first is + read. labels_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`; their ``chunks`` also set the spatial chunks of uncompressed files, which are memory-mapped. transformations The transformations of the region's image, by coordinate system. - mask_channel - The mask to read from a file with several: its index or channel name, the OME channel name (e.g. - ``"cells"``) or, in an inForm ``*_binary_seg_maps.tif``, ``"Tissue"`` (the tissue class map) or the - compartment (``"Nucleus"``, ``"Cytoplasm"``, ``"Membrane"``). If `None`, the first one; for an inForm - segmentation map ``"Nucleus"``, whose labels are the ``Cell ID`` of every cell. Returns ------- The labels element. """ - labels_models_kwargs, chunks = _labels_models_kwargs(labels_models_kwargs) - image, _, _ = read_tiff(path, chunks) - if not np.issubdtype(image.dtype, np.integer): - raise ValueError(f"{path} has dtype {image.dtype}; a segmentation mask must have an integer dtype.") - - n_masks = image.shape[0] - names, inform = _mask_channel_names(path, n_masks) - if isinstance(mask_channel, str): - if mask_channel not in names: - raise ValueError(f"{path} has no mask channel {mask_channel!r}; its channel names are {names}.") - index = names.index(mask_channel) - elif mask_channel is None and inform and _INFORM_DEFAULT_MASK in names: - index = names.index(_INFORM_DEFAULT_MASK) - logger.info( - "%s is an inForm segmentation map %s; reading %r. Set `mask_channel` to choose another.", - path, - names, - _INFORM_DEFAULT_MASK, - ) - else: - index = 0 if mask_channel is None else mask_channel - if mask_channel is None and n_masks > 1: - logger.info("%s holds %d masks; reading the first. Set `mask_channel` to choose another.", path, n_masks) - if not 0 <= index < n_masks: - raise ValueError(f"{path} holds {n_masks} mask channel(s); `mask_channel` {mask_channel!r} is out of range.") - return Labels2DModel.parse( - image[index], dims=("y", "x"), transformations=dict(transformations), **labels_models_kwargs - ) - - -def _labels_models_kwargs(labels_models_kwargs: Mapping[str, Any]) -> tuple[dict[str, Any], tuple[int, int]]: - """``labels_models_kwargs`` with the reader defaults, and their ``(y, x)`` chunk size.""" _, labels_models_kwargs = _initialize_raster_models_kwargs({}, labels_models_kwargs) _, defaults = _initialize_raster_models_kwargs({}, {}) - return labels_models_kwargs, spatial_chunks(labels_models_kwargs["chunks"], defaults["chunks"]) - - -def _mask_channel_names(path: Path, n_masks: int) -> tuple[list[str], bool]: - """The channel names of a mask file and whether it is an inForm segmentation map. - - The names are the OME channel names, else for an inForm segmentation map (a file with a ```` - page) the page names, see :func:`_inform_page_name`, else ``[]``. - """ - with tifffile.TiffFile(path) as tf: - if tf.ome_metadata: - return _ome_channel_names(tf.ome_metadata), False - if len(tf.pages) != n_masks: - return [], False - descriptions = [str(tf.pages[i].description or "") for i in range(n_masks)] # type: ignore[union-attr] - roots = [match.group(1) if (match := _INFORM_XML_ROOT.search(d)) else None for d in descriptions] - if _INFORM_SEGMENTATION not in roots: - return [], False - return [_inform_page_name(d, root, i) for i, (d, root) in enumerate(zip(descriptions, roots, strict=True))], True - - -def _inform_page_name(description: str, root: str | None, index: int) -> str: - """The compartment of a ```` page, else its XML root tag (``Tissue`` for the tissue map).""" - if (compartment := _INFORM_COMPARTMENT.search(description)) is not None: - return compartment.group(1) - if root is not None: - return _INFORM_PAGE_NAMES.get(root, root) - return f"page {index}" - - -def _ome_channel_names(ome_metadata: str) -> list[str]: - images = tifffile.xml2dict(ome_metadata)["OME"]["Image"] - pixels = (images[0] if isinstance(images, list) else images)["Pixels"] - channels = pixels.get("Channel", []) - return [str(channel.get("Name", "")) for channel in (channels if isinstance(channels, list) else [channels])] + image, _, _ = read_tiff(path, spatial_chunks(labels_models_kwargs["chunks"], defaults["chunks"])) + if not np.issubdtype(image.dtype, np.integer): + raise ValueError(f"{path} has dtype {image.dtype}; a segmentation mask must have an integer dtype.") + return Labels2DModel.parse(image[0], dims=("y", "x"), transformations=dict(transformations), **labels_models_kwargs) def read_region_labels( files: Mapping[str, Path], transformations: Mapping[str, Mapping[str, BaseTransformation]], labels_models_kwargs: Mapping[str, Any], - cells_table_kwargs: Mapping[str, Any], image_shapes: Mapping[str, tuple[int, int]], -) -> tuple[dict[str, DataArray | DataTree], dict[str, int]]: +) -> dict[str, DataArray | DataTree]: """Read the mask of each region, as resolved by :func:`resolve_per_region`, with the region's transformations. A mask whose ``(y, x)`` shape differs from ``image_shapes`` of its region is scaled onto the image if it is @@ -1435,19 +448,17 @@ def read_region_labels( Returns ------- - The labels elements by region id, and the downsampling factor of the scaled masks by region id. + The labels elements by region id. """ if "*" in files: raise ValueError( f"A single labels mask {files['*']} was given for several regions; a mask belongs to one region image, " "so pass a mapping {region: path}." ) - labels, factors = {}, {} + labels = {} for region, path in files.items(): region_transformations = _transformations_of(transformations, region) - labels[region] = read_labels( - path, labels_models_kwargs, region_transformations, mask_channel=cells_table_kwargs.get("mask_channel") - ) + labels[region] = read_labels(path, labels_models_kwargs, region_transformations) shape, image_shape = raster_shape(labels[region]), image_shapes.get(region) if image_shape is None or shape == tuple(image_shape): continue @@ -1477,8 +488,7 @@ def read_region_labels( }, set_all=True, ) - factors[region] = factor - return labels, factors + return labels def mask_downsample_factor(mask_shape: tuple[int, int], image_shape: tuple[int, int]) -> int | None: @@ -1494,128 +504,6 @@ def mask_downsample_factor(mask_shape: tuple[int, int], image_shape: tuple[int, return None -def _to_image_pixels(tables: Sequence[CellTable], factors: Mapping[str, int]) -> list[CellTable]: - """``tables`` with mask-pixel centroids and radii scaled to image pixels by their region's mask factor. - - A mask pixel covers ``f x f`` image pixels, so a centroid ``c`` moves to the centre of those, ``f * c + (f - 1) / 2``. - """ - scaled = [] - for table in tables: - if table.mask_pixels and factors: - row_factors = table.region.map(factors).fillna(1).to_numpy(dtype=np.float64) - table = replace( - table, - centroids_px=None - if table.centroids_px is None - else table.centroids_px * row_factors[:, None] + (row_factors[:, None] - 1) / 2, - radius_px=None if table.radius_px is None else table.radius_px * row_factors, - ) - scaled.append(table) - return scaled - - -def polygon_labels( - tables: Sequence[CellTable], - labels: Mapping[str, DataArray | DataTree], - image_shapes: Mapping[str, tuple[int, int]], - transformations: Mapping[str, Mapping[str, BaseTransformation]], - labels_models_kwargs: Mapping[str, Any], -) -> tuple[dict[str, DataArray | DataTree], dict[str, str]]: - """Lazily rasterise the cell polygons of each region into labels whose values are the cells' instance ids. - - Where cells overlap, the one with the higher instance id wins. Regions with a mask in ``labels`` keep it, with - a warning; regions without an image are skipped, their cells become circles. In both cases the polygons are - still kept as shapes, see :func:`polygon_shapes`. Regions none of whose cells has a polygon get no labels. - - Parameters - ---------- - tables - The cell tables; those with ``polygons_px`` are rasterised. - labels - The labels read from masks, by region id. - image_shapes - The level-0 ``(y, x)`` shape of each region's image; the labels get the same shape. - transformations - The transformations of each region's image, see :func:`region_transformations`; missing regions are in - ``"global"``. - labels_models_kwargs - Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`; their ``chunks`` are the chunks - that are rasterised one at a time. - - Returns - ------- - The labels elements by region id, and the GeoJSON file of each region with polygons. - """ - labels_models_kwargs, chunks = _labels_models_kwargs(labels_models_kwargs) - elements, sources = {}, {} - for table in tables: - if table.polygons_px is None: - continue - for region, rows in _region_rows(table).items(): - rows = rows[pd.notna(table.polygons_px[rows])] - # regions without polygons get circles, like tables without a GeoJSON - if len(rows) == 0: - continue - sources[region] = str(table.geojson) - if region in labels: - row_ids = " Its cells' instance ids are row numbers and may not match the mask's values." - warnings.warn( - f"Region '{region}' has both a segmentation mask and the cell polygons {table.geojson}; the mask " - f"is used for the labels and the polygons are kept as '{region}{PhenocyclerKeys.POLYGONS_SUFFIX}'." - f"{row_ids if table.row_instance_ids else ''}", - UserWarning, - stacklevel=3, - ) - continue - if region not in image_shapes: - logger.info("Region '%s' has no image, so its cell polygons are not rasterised.", region) - continue - data = rasterize_polygons_lazy( - table.polygons_px[rows], table.instance_id.to_numpy()[rows], image_shapes[region], chunks - ) - elements[region] = Labels2DModel.parse( - data, - dims=("y", "x"), - transformations=_transformations_of(transformations, region), - **labels_models_kwargs, - ) - return elements, sources - - -def polygon_shapes( - tables: Sequence[CellTable], transformations: Mapping[str, Mapping[str, BaseTransformation]] -) -> dict[str, GeoDataFrame]: - """The cell polygons of each region as shapes indexed by the cells' instance ids, the ids of their labels. - - Cells without a polygon are left out. - - Parameters - ---------- - tables - The cell tables; those with ``polygons_px`` are kept. - transformations - The transformations of each region, see :func:`region_transformations`; missing regions are in - ``"global"``. - - Returns - ------- - The shapes elements by region id. - """ - shapes = {} - for table in tables: - if table.polygons_px is None: - continue - for region, rows in _region_rows(table).items(): - rows = rows[pd.notna(table.polygons_px[rows])] - if len(rows) == 0: - continue - geometry = gpd.GeoDataFrame( - geometry=list(table.polygons_px[rows]), index=pd.Index(table.instance_id.to_numpy()[rows]) - ) - shapes[region] = ShapesModel.parse(geometry, transformations=_transformations_of(transformations, region)) - return shapes - - def _transformations_of( transformations: Mapping[str, Mapping[str, BaseTransformation]], region: str ) -> dict[str, BaseTransformation]: @@ -1631,7 +519,6 @@ def raster_shape(element: DataArray | DataTree) -> tuple[int, int]: def build_cells( tables: Sequence[CellTable], - region_elements: Mapping[str, str], labels: Mapping[str, DataArray | DataTree], transformations: Mapping[str, Mapping[str, BaseTransformation]], ) -> tuple[dict[str, ad.AnnData], dict[str, GeoDataFrame]]: @@ -1641,11 +528,9 @@ def build_cells( ---------- tables The cell tables, each covering one or more regions. - region_elements - The element each region's cells annotate. Regions missing from it annotate ``_labels`` if they are - in ``labels``, else the circles ``_cells`` built here. labels - The labels elements by region id; their regions get no circles. + The labels elements by region id; their regions' cells annotate ``_labels`` instead of the circles + ``_cells`` built here. transformations The transformations of each region, see :func:`region_transformations`; missing regions get ``Identity`` to ``"global"``. @@ -1653,18 +538,12 @@ def build_cells( Returns ------- The table elements, ``{"table": ...}`` if all tables have the same ``var`` apart from its ``source_column`` - (dropped if it differs), else ``{"_table": ...}`` - per region, and the circle shapes by element name. The circles have the cells' area (radius 1 px without a - positive one). + (dropped if it differs), else ``{"_table": ...}`` per region, and the circle shapes by element name. The + circles have the cells' area (radius 1 px without a positive one). """ _check_disjoint_regions(tables) row_ids = sorted( - region - for table in tables - # labels rasterised from a table's polygons have its instance ids by construction - if table.row_instance_ids and table.polygons_px is None - for region in table.region.unique() - if region in labels + region for table in tables if table.row_instance_ids for region in table.region.unique() if region in labels ) if row_ids: warnings.warn( @@ -1673,29 +552,20 @@ def build_cells( UserWarning, stacklevel=3, ) - # tables with other markers or other marker metadata (e.g. QuPath compartments) are not concatenated; the source - # columns only record where a marker was read from (e.g. `cyc001_ch001:DAPI1` of a CSV, `cyc001_ch001` of an FCS) + # tables with other markers are not concatenated; the source columns only record where a marker was read from + # (e.g. `cyc001_ch001:DAPI1` of a CSV, `cyc001_ch001` of an FCS) panels = [_panel(table.var) for table in tables] pieces = _split_by_region(tables) if any(not panel.equals(panels[0]) for panel in panels) else None def element(region: str) -> str: - if region in region_elements: - return region_elements[region] suffix = PhenocyclerKeys.LABELS_SUFFIX if region in labels else PhenocyclerKeys.CELLS_SUFFIX return f"{region}{suffix}" shapes = {} - circle_regions = [] for table in tables: for region, rows in _region_rows(table).items(): if region in labels: continue - if table.centroids_px is None: - raise ValueError( - f"The cells of region '{region}' have no centroids and no segmentation mask, so they cannot be " - "placed; pass a mask as `labels` or a table with centroids." - ) - circle_regions.append(region) radius = np.ones(len(rows)) if table.radius_px is None else table.radius_px[rows] if (no_area := ~(radius > 0)).any(): logger.info( @@ -1709,9 +579,10 @@ def element(region: str) -> str: index=pd.Index(table.instance_id.to_numpy()[rows]), transformations=_transformations_of(transformations, region), ) - if circle_regions: + if shapes: + circle_regions = sorted(name.removesuffix(PhenocyclerKeys.CELLS_SUFFIX) for name in shapes) warnings.warn( - f"No segmentation mask found for region(s) {sorted(circle_regions)}; cells are represented as circles.", + f"No segmentation mask found for region(s) {circle_regions}; cells are represented as circles.", UserWarning, stacklevel=3, ) @@ -1731,20 +602,14 @@ def build_spatialdata( table_paths: Mapping[str, str] | None, image_models_kwargs: Mapping[str, Any], labels_models_kwargs: Mapping[str, Any], - cells_table_kwargs: Mapping[str, Any], duplicate_scans: Mapping[str, AkoyaScan] | None = None, ) -> SpatialData: """Assemble the regions of an Akoya read into a `SpatialData` with their metadata in ``attrs["akoya"]``. - Each region gets the image ``_image``, the labels ``_labels`` from its mask or its cell polygons - (see :func:`polygon_labels`), the polygons ``_polygons`` and the cells (see :func:`build_cells`), all in - the coordinate systems of :func:`region_transformations`; regions with cells or a mask but no image are - unrelated to the scans. A mask downsampled from its region image by an integer factor is scaled onto the image - (see :func:`read_region_labels`), and the centroids and radii in mask pixels of its cells (``mask_pixels``, see - :class:`CellTable`) are scaled by the same factor (see :func:`_to_image_pixels`), so that the table and circles - are in image pixels. A - `UserWarning` is raised for a region if more than 5% of its cells lie outside its image: their coordinates are - in another unit or relative to another origin (e.g. inForm positions in microns relative to the slide). + Each region gets the image ``_image``, the labels ``_labels`` from its mask (see + :func:`read_region_labels`) and the cells (see :func:`build_cells`), all in the coordinate systems of + :func:`region_transformations`; regions with cells or a mask but no image are unrelated to the scans. The cell + centroids are in image pixels, also when the mask is downsampled. Parameters ---------- @@ -1764,8 +629,6 @@ def build_spatialdata( Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. labels_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`. - cells_table_kwargs - See :func:`read_labels` (``mask_channel``). duplicate_scans Further scans of the same physical scan as a region (e.g. its 8-bit file), by that region's id. Each is stored as its own image ``_image`` with exactly the transformations of the region; their @@ -1794,16 +657,8 @@ def build_spatialdata( image_shapes = { scan.region: raster_shape(image_elements[f"{scan.region}{PhenocyclerKeys.IMAGE_SUFFIX}"]) for scan in scans } - labels, factors = read_region_labels( - mask_files, transformations, labels_models_kwargs, cells_table_kwargs, image_shapes - ) - cell_tables = _to_image_pixels(cell_tables, factors) - _check_cells_in_images(cell_tables, image_shapes) - polygons, geojson_files = polygon_labels(cell_tables, labels, image_shapes, transformations, labels_models_kwargs) - labels |= polygons - tables, shapes = build_cells(cell_tables, {}, labels, transformations) if cell_tables else ({}, {}) - for region, element in polygon_shapes(cell_tables, transformations).items(): - shapes[f"{region}{PhenocyclerKeys.POLYGONS_SUFFIX}"] = element + labels = read_region_labels(mask_files, transformations, labels_models_kwargs, image_shapes) + tables, shapes = build_cells(cell_tables, labels, transformations) if cell_tables else ({}, {}) sdata = SpatialData( images=image_elements, labels={f"{region}{PhenocyclerKeys.LABELS_SUFFIX}": element for region, element in labels.items()}, @@ -1815,35 +670,10 @@ def build_spatialdata( attrs["cell_tables"] = dict(table_paths) if mask_files: attrs["labels"] = {region: str(file) for region, file in mask_files.items()} - if geojson_files: - attrs["geojson"] = geojson_files sdata.attrs[PhenocyclerKeys.ATTRS_KEY.value] = attrs return sdata -def _check_cells_in_images(tables: Sequence[CellTable], image_shapes: Mapping[str, tuple[int, int]]) -> None: - """Warn for each region with an image if more than 5% of its cells' centroids lie outside the image.""" - for table in tables: - if table.centroids_px is None: - continue - for region, rows in _region_rows(table).items(): - if region not in image_shapes: - continue - centroids = table.centroids_px[rows] - centroids = centroids[np.isfinite(centroids).all(axis=1)] - height, width = image_shapes[region] - outside = int(np.count_nonzero(((centroids < 0) | (centroids > [width, height])).any(axis=1))) - if outside > _MAX_OUTSIDE_FRACTION * len(centroids): - warnings.warn( - f"{outside} of the {len(centroids)} cells of region '{region}' lie outside its image ({width} x " - f"{height} px): the table's coordinates are in another unit (see " - "`cells_table_kwargs['coordinates_unit']`) or relative to another origin (e.g. inForm positions " - "in microns, which may be relative to the slide).", - UserWarning, - stacklevel=4, - ) - - def _region_rows(table: CellTable) -> dict[str, np.ndarray]: """The row indices of each region of ``table``, in order of first appearance.""" codes, regions = pd.factorize(table.region) @@ -1872,17 +702,6 @@ def _to_anndata(tables: Sequence[CellTable], element: Callable[[str], str]) -> a """Concatenate cell tables with the same markers into one table annotating the elements of their regions.""" region_key, instance_key = CodexKeys.REGION_KEY.value, CodexKeys.INSTANCE_KEY.value obs = pd.concat([table.obs for table in tables]) if len(tables) > 1 else tables[0].obs.copy() - # the region and instance keys are set below; the adapters dropped the columns they were read from, and other - # columns that would clash with them (also case-insensitively, as spatialdata requires, e.g. `Cell ID` -> - # `Cell_ID` vs `cell_id`) keep their values under `source_` - reserved = {region_key, instance_key} - obs = obs.rename( - columns={ - column: f"source_{column}" - for column in obs.columns - if sanitize_name(str(column), is_dataframe_column=True).casefold() in reserved - } - ) regions = pd.concat([table.region for table in tables]) if len(tables) > 1 else tables[0].region if not obs.index.is_unique: obs.index = pd.Index(regions.to_numpy(dtype=object) + "_" + obs.index.to_numpy(dtype=object)) @@ -1892,14 +711,8 @@ def _to_anndata(tables: Sequence[CellTable], element: Callable[[str], str]) -> a # the same markers read from columns of other names var = _panel(var) adata = ad.AnnData(X=X, obs=obs, var=var) - centroids = [table.centroids_px for table in tables if table.centroids_px is not None] - if len(centroids) == len(tables): - adata.obsm[CodexKeys.SPATIAL_KEY.value] = np.concatenate(centroids) - elif centroids: - warnings.warn( - "Some cell tables have no centroids, so the table has no 'spatial' coordinates.", UserWarning, stacklevel=4 - ) - # e.g. QuPath's `Centroid X µm` -> `Centroid_X_µm`: spatialdata only writes alphanumeric names and `_.-` + adata.obsm[CodexKeys.SPATIAL_KEY.value] = np.concatenate([table.centroids_px for table in tables]) + # spatialdata only writes alphanumeric column names and `_.-` if any(sanitize_name(str(column), is_dataframe_column=True) != column for column in adata.obs.columns): sanitize_table(adata, inplace=True) elements = regions.map({region: element(region) for region in regions.unique()}).to_numpy() diff --git a/src/spatialdata_io/readers/codex.py b/src/spatialdata_io/readers/codex.py index bb081857..3280e033 100644 --- a/src/spatialdata_io/readers/codex.py +++ b/src/spatialdata_io/readers/codex.py @@ -28,8 +28,7 @@ from spatialdata_io.readers._utils._cell_tables import ( CellTable, build_spatialdata, - check_cells_table_options, - detect_table_format, + is_codex_table, read_cell_table, read_region_tables, resolve_per_region, @@ -98,11 +97,9 @@ def codex( regions: Sequence[str] | None = None, channel_names: Sequence[str] | None = None, cells_table: str | Path | Mapping[str, str | Path] | None = None, - cells_table_format: str = "auto", labels: str | Path | Mapping[str, str | Path] | None = None, image_models_kwargs: Mapping[str, Any] = MappingProxyType({}), labels_models_kwargs: Mapping[str, Any] = MappingProxyType({}), - cells_table_kwargs: Mapping[str, Any] = MappingProxyType({}), ) -> SpatialData: """Read *CODEX* (CODEX Processor, CODEX Uploader and CODEX Instrument Manager) data. @@ -124,9 +121,10 @@ def codex( preferred over uncompensated ones; other tables of the same region (e.g. of a second segmentation run) raise an error. The ``cycCCC_chHHH`` marker columns are stored in ``X`` (dense float32) under their marker names, taken from the FCS ``$PnS`` keywords or the CSV header ``cycCCC_chHHH:``; all other columns go to - ``obs``, the ``x``/``y`` pixel centroids to ``obsm["spatial"]``. Other CSV files (without ``x``/``y`` and - ``cell_id`` or ``cycCCC_chHHH`` columns, e.g. antibody lists or HuBMAP SPRM outputs) are ignored; pass them as - ``cells_table`` to read them. FCS files of other instruments are ignored too. + ``obs``, the ``x``/``y`` pixel centroids to ``obsm["spatial"]``. A table without ``cycCCC_chHHH`` columns + reads its numeric columns that are not CODEX Processor columns (``x``, ``size``, ...) as the markers, with a + warning listing them. Other CSV files (without ``x``/``y`` and ``cell_id`` or ``cycCCC_chHHH`` columns, + e.g. antibody lists) and FCS files of other instruments are ignored. Raw acquisition and intermediate folders (``cycCCC_regNNN/``, ``tiles/`` and, next to ``stitched/``, ``bestFocus/``) are ignored, as are hidden files and folders (names starting with ``.``, e.g. macOS ``._*`` files, and @@ -149,16 +147,10 @@ def codex( The cells of each region are stored as circles ``_cells`` with the cells' area (``size`` column), annotated by one table ``"table"`` (``obs["region"]`` names the annotated element, ``obs["cell_id"]`` is the instance key), or by one table ``_table`` per region if the regions have different markers. Columns of - the cell table that clash with ``region`` or ``cell_id`` are kept as ``source_``, and column names that - spatialdata cannot write (e.g. QuPath's ``Centroid X µm``) are sanitised (``Centroid_X_µm``). A region with a - segmentation mask in ``labels`` gets the labels ``_labels`` instead of circles, and its cells annotate - them; so does a region with the cell outlines of a QuPath GeoJSON export (see ``cells_table_kwargs``), - rasterised lazily one chunk at a time with the cells' instance ids (the higher id wins where cells overlap). - The outlines are always kept as the shapes ``_polygons`` in pixels, indexed by the same instance ids, - also when a mask is given for the region (the mask is used for the labels, with a warning) or the region has no - image (its cells are then circles). The paths of the tables, masks and GeoJSON files read are stored in - ``sdata.attrs["akoya"]["cell_tables"]``, ``["labels"]`` and ``["geojson"]``, by region id (``"*"`` for a file of - several regions). + the cell table that spatialdata cannot write are sanitised. A region with a segmentation mask (see ``labels``) + gets the labels ``_labels`` instead of circles, and its cells annotate them. The paths of the tables and + masks read are stored in ``sdata.attrs["akoya"]["cell_tables"]`` and ``["labels"]``, by region id (``"*"`` for a + file of several regions). .. seealso:: @@ -178,15 +170,9 @@ def codex( channel_names Channel names to use instead of the discovered ones; must have one entry per channel of every region read. cells_table - A cell table to read instead of the discovered ones (explicit beats discovered): a path, or a mapping from - region id to path, e.g. ``{{"reg001": "reg001_cells.csv"}}``. A single path for several regions needs a - region column (``region`` for CODEX tables, ``Image`` for QuPath, else - ``cells_table_kwargs["region_column"]``). See ``cells_table_format`` for the formats. - cells_table_format - The format of ``cells_table``: ``"auto"`` (detected from the header), ``"codex"``, ``"qupath"`` (QuPath - measurements; µm centroids are converted with the region's pixel size), ``"mcquant"`` (MCMICRO), ``"sprm"`` - (HuBMAP SPRM ``*-cell_channel_mean.csv``), ``"inform"`` (inForm ``*_cell_seg_data.txt``) or ``"generic"`` - (columns given by ``cells_table_kwargs``). + A CODEX cell table to read instead of the discovered ones (explicit beats discovered): a path, or a mapping + from region id to path, e.g. ``{{"reg001": "reg001_cells.csv"}}``. A single path for several regions needs a + ``region`` column. labels A segmentation mask (TIFF/OME-TIFF with integer instance ids) per region: a path for a single region, or a mapping from region id to path; a mask inside ``path`` is not read as an image. If `None`, a TIFF named after @@ -197,32 +183,12 @@ def codex( the labels ``_labels`` with the transformation of the region's image; the cells of the region annotate it instead of circles, and their instance ids must be the mask values. A mask whose shape is the region image's divided by an integer factor ``f`` (exactly, or rounded up) is scaled onto the image (``Scale([f, f])`` before - the region's transformation); any other shape mismatch raises a warning. The pixel centroids of ``"mcquant"`` - and ``"sprm"`` tables are taken to be in mask pixels, so their centroids (``obsm["spatial"]``) and circle radii - are scaled by ``f`` to image pixels (a centroid ``c`` to ``f * c + (f - 1) / 2``, the centre of its mask - pixel); the source columns in ``obs`` are kept as read (mask pixels). The pixel - centroids of other tables are in image pixels and are not scaled; - ``cells_table_kwargs["coordinates_space"]="mask"`` scales those of a ``"generic"`` table too. + the region's transformation); the cell centroids stay in image pixels. Any other shape mismatch raises a + warning. Of a mask file with several planes, the first is read. image_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. labels_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`. - cells_table_kwargs - Options for reading ``cells_table`` (or the discovered cell tables) and ``labels``: ``region_column``; for - ``"generic"`` tables ``x``, ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath - and CODEX tables, whose markers are named by their name, ``cycCCC_chHHH:`` header or ``cycCCC_chHHH``), - ``coordinates_unit`` (``"px"`` or ``"um"``; for QuPath ``X``/``Y`` and inForm ``Cell X Position`` columns - without a unit, which are otherwise read in the unit of the table's other columns) and - ``coordinates_space`` (``"image"``, default, or ``"mask"``: whether the pixel centroids of a generic table - are in pixels of the region image or of its mask, see ``labels``); with ``x`` and ``y``, - ``cells_table_format="auto"`` reads a generic table. ``mask_channel`` (index or channel name of the mask in - a multi-channel ``labels`` file: an OME channel name, or for an inForm ``*_binary_seg_maps.tif`` - ``"Tissue"``, ``"Nucleus"`` (the default there, its labels are the ``Cell ID`` of every cell), - ``"Cytoplasm"`` or ``"Membrane"``); for QuPath tables ``geojson`` (the GeoJSON export of the cell - outlines, a path or a mapping from region id to path like ``cells_table``; default ``
.geojson`` - or ``
_cells.geojson`` next to the table, ignored with a warning if it holds none of the - table's cells; ``False`` reads none) and ``geojson_units`` (``"px"`` of the - full-resolution image, default and what QuPath writes, or ``"um"``). Returns ------- @@ -232,7 +198,6 @@ def codex( warnings.warn( "`imread_kwargs` is ignored: every image is read lazily with tifffile.", UserWarning, stacklevel=2 ) - check_cells_table_options(cells_table_format, cells_table_kwargs) regions = normalize_regions(regions) channel_names = [channel_names] if isinstance(channel_names, str) else channel_names path = Path(path) @@ -246,9 +211,7 @@ def codex( masks={Path(mask).resolve() for mask in masks}, ) # the regions of a table without a region in its file name are known only once it is read - regionless = ( - _read_regionless_table(layout.tables["*"], layout, cells_table_kwargs) if "*" in layout.tables else None - ) + regionless = _read_regionless_table(layout.tables["*"], layout) if "*" in layout.tables else None table_regions = set(regionless.region.unique()) if regionless is not None else set() available = sorted(layout.images.keys() | layout.tables.keys() - {"*"} | table_regions) selected = select_regions(available, regions) @@ -294,25 +257,11 @@ def codex( names = {scan.region: apply_channel_names(scan, channel_names) for scan in scans} if cells_table is not None: - cell_tables = read_region_tables( - table_files, - cells_table_format, - available=layout.images, - selected=selected, - pixel_sizes={scan.region: scan.pixel_size_um for scan in scans}, - cells_table_kwargs=cells_table_kwargs, - ) + cell_tables = read_region_tables(table_files, available=list(layout.images), selected=selected) table_paths = {key: str(file) for key, file in table_files.items()} else: named_files = {key: layout.tables[key] for key in selected if key in layout.tables} - named = read_region_tables( - named_files, - "codex", - available={region: layout.images.get(region, []) for region in available}, - selected=selected, - pixel_sizes={}, - cells_table_kwargs=cells_table_kwargs, - ) + named = read_region_tables(named_files, available=available, selected=selected) read = dict(zip(named_files, named, strict=True)) if regionless is not None: regionless = regionless.take(np.flatnonzero(regionless.region.isin(selected).to_numpy())) @@ -334,7 +283,6 @@ def codex( table_paths=table_paths, image_models_kwargs=image_models_kwargs, labels_models_kwargs=labels_models_kwargs, - cells_table_kwargs=cells_table_kwargs, ) return _set_reader_metadata(sdata, "codex") @@ -349,18 +297,12 @@ def _warn_fcs_fallback(files: Iterable[Path], fcs: bool) -> None: ) -def _read_regionless_table(file: Path, layout: _LegacyLayout, cells_table_kwargs: Mapping[str, Any]) -> CellTable: +def _read_regionless_table(file: Path, layout: _LegacyLayout) -> CellTable: """Read the discovered cell table ``file`` that has no region in its file name.""" only_image = next(iter(layout.images)) if len(layout.images) == 1 else None - regions = {region: layout.images.get(region, []) for region in layout.images.keys() | layout.tables.keys() - {"*"}} # next to a single image named after its file (e.g. `Kidney.tif`), a single region value names that image table = read_cell_table( - file, - "codex", - pixel_size_um=None, - default_region=only_image, - cells_table_kwargs=cells_table_kwargs, - regions=regions, + file, default_region=only_image, regions=layout.images.keys() | layout.tables.keys() - {"*"} ) # values that name no region of the dataset are regions without an image table.region = table.region.map({value: sanitize_name(value) for value in table.region.unique()}) @@ -601,7 +543,7 @@ def _discover_tables(files: Sequence[Path], fcs: bool) -> dict[str, Path]: preferred = CodexKeys.FCS_FILE.value if fcs else CodexKeys.FCS_FILE_CSV.value candidates: dict[str, list[Path]] = {} for file in files: - if file.suffix.lower() in _TABLE_SUFFIXES and _is_codex_table(file): + if file.suffix.lower() in _TABLE_SUFFIXES and is_codex_table(file): candidates.setdefault(region_id(file.name) or "*", []).append(file) def uncompensated(file: Path) -> bool: @@ -627,18 +569,6 @@ def uncompensated(file: Path) -> bool: return tables -def _is_codex_table(file: Path) -> bool: - """Whether ``file`` is a CODEX Processor table. - - A CSV may be any sidecar, also with a region in its name (e.g. ``reg001_antibodies.csv`` or a HuBMAP SPRM - ``reg001_...-cell_channel_mean.csv``), an FCS file the export of any flow cytometer. - """ - try: - return detect_table_format(file) == "codex" - except ValueError: - return False - - def _find_sidecar(path: Path, files: Sequence[Path], name: str) -> Path | None: """The shallowest file called ``name`` in ``files``, else in the nearest ancestor of ``path`` holding one. diff --git a/src/spatialdata_io/readers/phenocycler.py b/src/spatialdata_io/readers/phenocycler.py index 3a447a0c..fdae290a 100644 --- a/src/spatialdata_io/readers/phenocycler.py +++ b/src/spatialdata_io/readers/phenocycler.py @@ -18,12 +18,7 @@ read_json_object, select_regions, ) -from spatialdata_io.readers._utils._cell_tables import ( - build_spatialdata, - check_cells_table_options, - read_region_tables, - resolve_per_region, -) +from spatialdata_io.readers._utils._cell_tables import build_spatialdata, resolve_per_region from spatialdata_io.readers._utils._qptiff import ( _fusion_profile_records, _is_blank, @@ -54,13 +49,10 @@ def phenocycler( *, regions: Sequence[str] | None = None, channel_names: Sequence[str] | None = None, - cells_table: str | Path | Mapping[str, str | Path] | None = None, - cells_table_format: str = "auto", labels: str | Path | Mapping[str, str | Path] | None = None, read_all_bit_depths: bool = False, image_models_kwargs: Mapping[str, Any] = MappingProxyType({}), labels_models_kwargs: Mapping[str, Any] = MappingProxyType({}), - cells_table_kwargs: Mapping[str, Any] = MappingProxyType({}), ) -> SpatialData: """Read *Akoya PhenoCycler-Fusion* 1.x/2.x data, and PerkinElmer/Akoya Vectra and PhenoImager QPTIFFs. @@ -94,28 +86,12 @@ def phenocycler( ``{vx.IMAGE_SUFFIX}``. A single region is stored in the ``"global"`` coordinate system. Several regions of the same sample space (scanned on the same slide, with stage positions and equal pixel sizes) are placed at their stage positions in ``"global"``; otherwise the regions are unrelated and each is stored in its own - coordinate system, named after the region. A region with cells or a mask but no scan gets its own coordinate - system too. All elements of a region share its coordinate system. The per-region metadata is stored in - ``sdata.attrs[{vx.ATTRS_KEY!r}]``. + coordinate system, named after the region. All elements of a region share its coordinate system. The + per-region metadata is stored in ``sdata.attrs[{vx.ATTRS_KEY!r}]``. A segmentation mask given in ``labels`` is stored lazily as the labels ``{vx.LABELS_SUFFIX}`` with the - transformation of the region's image. The cells of ``cells_table`` are stored in one table - ``{vx.TABLE_KEY!r}`` (marker intensities dense float32 in ``X``, pixel centroids in ``obsm["spatial"]``, - ``obs[{vx.REGION_KEY!r}]`` the annotated element, ``obs[{vx.INSTANCE_KEY!r}]`` the instance key), or in one - table ``_table`` per region if the regions have different markers. Columns of the cell table that clash - with ``{vx.REGION_KEY!r}`` or ``{vx.INSTANCE_KEY!r}`` are kept as ``source_``, and column names that - spatialdata cannot write (e.g. QuPath's ``Centroid X µm``) are sanitised (``Centroid_X_µm``). Cells annotate - the labels of their region, whose values must be their instance ids; without a mask they are stored as circles - ``{vx.CELLS_SUFFIX}``, with a warning. - - The cell outlines of a QuPath GeoJSON export (see ``cells_table_kwargs``) are rasterised lazily, one chunk at a - time, into the labels of a region without a mask; a feature's ``id`` is matched to the table's ``Object ID`` - (else features and rows are matched in order), the labels hold the cells' instance ids, and where cells overlap - the higher instance id wins; the table annotates the labels. The outlines are always kept as the shapes - ``{vx.POLYGONS_SUFFIX}`` in pixels, indexed by the same instance ids, also when a mask is given for the - region (the mask is used for the labels, with a warning). The paths of the tables, masks and GeoJSON files are - stored in ``sdata.attrs[{vx.ATTRS_KEY!r}]["cell_tables"]``, ``["labels"]`` and ``["geojson"]``, by region id - (``"*"`` for a file of several regions). + transformation of the region's image; the mask paths are stored in ``sdata.attrs[{vx.ATTRS_KEY!r}]["labels"]``, + by region id. .. seealso:: @@ -127,72 +103,38 @@ def phenocycler( Path to a ``{vx.QPTIFF_SUFFIX!r}`` file (or an inForm ``.tif`` QPI file), or to a directory that is searched recursively for ``{vx.QPTIFF_SUFFIX!r}`` files. regions - Regions to read. If `None`, all regions are read. Region ids, like the keys of the ``cells_table``, - ``labels`` and ``cells_table_kwargs["geojson"]`` mappings, may also be given as raw names, which are - sanitised like the region ids (e.g. ``"Slide 1"`` selects the region ``Slide_1``). + Regions to read. If `None`, all regions are read. Region ids, like the keys of the ``labels`` mapping, may + also be given as raw names, which are sanitised like the region ids (e.g. ``"Slide 1"`` selects the region + ``Slide_1``). channel_names Channel names to use instead of the ones stored in the qptiff; must have one entry per channel of every region read. - cells_table - A cell table: a path, or a mapping from region id to path, e.g. ``{{"A": "A_cells.csv"}}``. A single path - for several regions needs a region column (``Image`` for QuPath, ``region`` for CODEX tables, else - ``cells_table_kwargs["region_column"]``); its values are matched to the region ids, also after - :func:`spatialdata.sanitize_name` (``A_[1,2]`` to ``A__1_2_``), or to the names of the region's files - (``A_Scan1.qptiff`` or ``A_Scan1`` to ``A``). - cells_table_format - The format of ``cells_table``: ``"auto"`` (detected from the header), ``"codex"`` (CODEX Processor), - ``"qupath"`` (QuPath measurements; µm centroids are converted with the region's pixel size), ``"mcquant"`` - (MCMICRO), ``"sprm"`` (HuBMAP SPRM ``*-cell_channel_mean.csv``), ``"inform"`` (inForm - ``*_cell_seg_data.txt``) or ``"generic"`` (columns given by ``cells_table_kwargs``). labels A segmentation mask (TIFF/OME-TIFF with integer instance ids) per region: a path for a single region, or a mapping from region id to path. A mask whose shape is the region image's divided by an integer factor ``f`` (exactly, or rounded up) is scaled onto the image (``Scale([f, f])`` before the region's transformation); - any other shape mismatch raises a warning. The pixel centroids of ``"mcquant"`` and ``"sprm"`` tables - are taken to be in mask pixels, so their centroids (``obsm["spatial"]``) and circle radii are scaled by - ``f`` to image pixels (a centroid ``c`` to ``f * c + (f - 1) / 2``, the centre of its mask pixel); the - source columns in ``obs`` are kept as read (mask pixels). The pixel centroids of - other tables are in image pixels and are not scaled; ``cells_table_kwargs["coordinates_space"]="mask"`` - scales those of a ``"generic"`` table too. + any other shape mismatch raises a warning. Of a mask file with several planes, the first is read. read_all_bit_depths If `True`, a scan that exists as both ``X{vx.QPTIFF_SUFFIX}`` (8-bit) and ``X{vx.ER_QPTIFF_SUFFIX}`` (extended range, e.g. uint16) is read from both files, also if ``path`` is the extended-range file: the extended-range file as region ``X`` and the 8-bit file as the image ``X_8bit{vx.IMAGE_SUFFIX}`` (region ``X_8bit`` in ``sdata.attrs[{vx.ATTRS_KEY!r}]``). Both are the same physical scan, so the 8-bit image has - exactly the transformations of region ``X`` (a warning is raised if their shapes differ). Cell tables and - masks annotate region ``X`` only. If `False`, only the extended-range file is read. + exactly the transformations of region ``X`` (a warning is raised if their shapes differ). Masks belong to + region ``X`` only. If `False`, only the extended-range file is read. image_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Image2DModel`. labels_models_kwargs Keyword arguments passed to :class:`spatialdata.models.Labels2DModel`. - cells_table_kwargs - Options for reading ``cells_table`` and ``labels``: ``region_column``; for ``"generic"`` tables ``x``, - ``y``, ``instance_id``, ``markers`` (list or regular expression; also for QuPath and CODEX tables, whose - markers are named by their name, ``cycCCC_chHHH:`` header or ``cycCCC_chHHH``), - ``coordinates_unit`` (``"px"`` or ``"um"``; for QuPath ``X``/``Y`` and inForm ``Cell X Position`` columns - without a unit, which are otherwise read in the unit of the table's other columns) and - ``coordinates_space`` (``"image"``, default, or ``"mask"``: whether the pixel centroids of a generic table - are in pixels of the region image or of its mask, see ``labels``); with ``x`` and ``y``, - ``cells_table_format="auto"`` reads a generic table. ``mask_channel`` (index or channel name of the mask in - a multi-channel ``labels`` file: an OME channel name, or for an inForm ``*_binary_seg_maps.tif`` - ``"Tissue"``, ``"Nucleus"`` (the default there, its labels are the ``Cell ID`` of every cell), - ``"Cytoplasm"`` or ``"Membrane"``); for QuPath tables ``geojson`` (the GeoJSON export of the cell - outlines, a path or a mapping from region id to path like ``cells_table``; default ``
.geojson`` - or ``
_cells.geojson`` next to the table, ignored with a warning if it holds none of the - table's cells; ``False`` reads none) and ``geojson_units`` (``"px"`` of the - full-resolution image, default and what QuPath writes, or ``"um"``). Returns ------- :class:`spatialdata.SpatialData` """ - check_cells_table_options(cells_table_format, cells_table_kwargs) regions = normalize_regions(regions) channel_names = [channel_names] if isinstance(channel_names, str) else channel_names path = Path(path) qptiffs, eight_bit = _discover_qptiffs(path) selected = select_regions(list(qptiffs), regions) - table_files = resolve_per_region(cells_table, selected, list(qptiffs)) mask_files = resolve_per_region(labels, selected, list(qptiffs)) scans = [read_qptiff_metadata(qptiffs[region], region=region) for region in selected] @@ -210,16 +152,6 @@ def phenocycler( for scan in [*scans, *duplicates.values()]: names[scan.region] = apply_channel_names(scan, channel_names) _read_sidecars(scan) - cell_tables = read_region_tables( - table_files, - cells_table_format, - available={ - region: [file, *([eight_bit[region]] if region in eight_bit else [])] for region, file in qptiffs.items() - }, - selected=selected, - pixel_sizes={scan.region: scan.pixel_size_um for scan in scans}, - cells_table_kwargs=cells_table_kwargs, - ) images = {scan.region: read_qptiff_image(scan) for scan in [*scans, *duplicates.values()]} for region, duplicate in duplicates.items(): if images[region].shape[-2:] != images[duplicate.region].shape[-2:]: @@ -234,12 +166,11 @@ def phenocycler( scans, images, names, - cell_tables, + [], mask_files, - table_paths={key: str(file) for key, file in table_files.items()} if table_files else None, + table_paths=None, image_models_kwargs=image_models_kwargs, labels_models_kwargs=labels_models_kwargs, - cells_table_kwargs=cells_table_kwargs, duplicate_scans=duplicates, ) return _set_reader_metadata(sdata, "phenocycler") diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index 376a848e..3e57b0dd 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -1,16 +1,14 @@ from __future__ import annotations -import json import warnings from typing import TYPE_CHECKING, Any import numpy as np import pandas as pd import pytest -import shapely import tifffile from spatialdata import get_extent, read_zarr -from spatialdata.models import Labels2DModel, get_axes_names +from spatialdata.models import get_axes_names from spatialdata.transformations import Identity, Scale, Sequence, Translation, get_transformation from tests._akoya_data import forbid_pixel_reads, write_fcs, write_qptiff @@ -18,7 +16,6 @@ from spatialdata_io.readers._utils._akoya import dedupe_channel_names from spatialdata_io.readers._utils._cell_tables import ( build_cells, - detect_table_format, read_cell_table, read_labels, resolve_per_region, @@ -45,7 +42,7 @@ def _cells(n: int = 5, region: int = 1, markers: tuple[str, ...] = ("cyc001_ch00 def _read_codex(path: Path, region: str | None = None) -> CellTable: - return read_cell_table(path, "codex", pixel_size_um=None, default_region=region, cells_table_kwargs={}) + return read_cell_table(path, default_region=region) def test_read_codex_fcs_marker_names_from_pns(tmp_path: Path) -> None: @@ -103,9 +100,7 @@ def test_read_codex_csv_marker_name_columns(tmp_path: Path) -> None: path = tmp_path / "cells.csv" path.write_text("cell_id,x,y,tile_num,CD3,CD8,label\n1,1.0,2.0,1,5.0,6.0,T\n") - with pytest.warns( - UserWarning, match=r"no 'cycCCC_chHHH' columns.*\['CD3', 'CD8'\].*cells_table_kwargs\['markers'\]" - ): + with pytest.warns(UserWarning, match=r"no 'cycCCC_chHHH' columns.*\['CD3', 'CD8'\]"): table = _read_codex(path, region="Kidney") assert list(table.var.index) == ["CD3", "CD8"] @@ -126,98 +121,8 @@ def test_read_codex_table_errors(tmp_path: Path) -> None: _read_codex(no_region) -@pytest.mark.parametrize("sep", ["\t", ", "]) -def test_read_codex_table_delimiters_like_detection(tmp_path: Path, sep: str) -> None: - path = tmp_path / "cells.tsv" - header = ["cell_id", "region", "x", "y", "size", "cyc001_ch001:DAPI"] - path.write_text("\n".join(sep.join(row) for row in [header, ["1", "2", "1.0", "2.0", "3.0", "4.0"]]) + "\n") - - assert detect_table_format(path) == "codex" - table = read_cell_table(path, "auto", pixel_size_um=None, default_region=None, cells_table_kwargs={}) - - assert list(table.var.index) == ["DAPI"] - assert list(table.region) == ["reg002"] - np.testing.assert_array_equal(table.centroids_px, [[1, 2]]) - - -@pytest.mark.parametrize("decimal", [",", "."]) -@pytest.mark.parametrize( - ("header", "row", "fmt"), - [ - ("cell_id;region;x;y;size;cyc001_ch001:DAPI", "1;2;1{d}5;2{d}0;3{d}0;4{d}25", "codex"), - ("CellID;DAPI;X_centroid;Y_centroid;Area", "1;4{d}25;1{d}5;2{d}0;3{d}0", "mcquant"), - ("Image;Object ID;Centroid X px;Centroid Y px;DAPI: Cell: Mean", "reg002;a;1{d}5;2{d}0;4{d}25", "qupath"), - ], -) -def test_semicolon_tables_read_comma_decimals(tmp_path: Path, header: str, row: str, fmt: str, decimal: str) -> None: - path = tmp_path / "cells.csv" - path.write_text(f"{header}\n{row.format(d=decimal)}\n", encoding="utf-8") - - assert detect_table_format(path) == fmt - table = read_cell_table(path, "auto", pixel_size_um=None, default_region="reg002", cells_table_kwargs={}) - - assert list(table.var.index) == ["DAPI"] - assert list(table.region) == ["reg002"] - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, [[1.5, 2]]) - np.testing.assert_allclose(table.X, [[4.25]]) - - -def test_read_codex_table_cells_table_kwargs(tmp_path: Path) -> None: - path = tmp_path / "cells.csv" - path.write_text("cell_id,label,Sample,x,y,CD3,CD8\n1,7,A,1.0,2.0,3.0,4.0\n2,8,B,5.0,6.0,7.0,8.0\n") - kwargs = {"region_column": "Sample", "instance_id": "label", "markers": ["CD8"]} - - table = read_cell_table(path, "codex", pixel_size_um=None, default_region="A", cells_table_kwargs=kwargs) - - assert list(table.region) == ["A", "B"] - assert list(table.instance_id) == [7, 8] - assert list(table.var.index) == ["CD8"] - assert "CD3" in table.obs.columns - - -@pytest.mark.parametrize( - ("markers", "expected"), - [ - (["DAPI1", "CD3"], ["DAPI1", "CD3"]), - (["cyc001_ch001:DAPI1"], ["DAPI1"]), - (["cyc001_ch001", "cyc002_ch001"], ["DAPI1", "DAPI"]), - (["DAPI (2)"], ["DAPI"]), - ("DAPI.*", ["DAPI1", "DAPI", "DAPI (2)"]), - ("cyc00\\d_ch001", ["DAPI1", "DAPI"]), - ], -) -def test_read_codex_table_markers_by_name_header_or_column( - tmp_path: Path, markers: str | list[str], expected: list[str] -) -> None: - path = tmp_path / "cells.csv" - path.write_text( - "cell_id,x,y,cyc001_ch001:DAPI1,cyc001_ch002:CD3,cyc002_ch001:DAPI,cyc002_ch002:DAPI\n1,1.0,2.0,3.0,4.0,5.0,6.0\n" - ) - - table = read_cell_table( - path, "codex", pixel_size_um=None, default_region="A", cells_table_kwargs={"markers": markers} - ) - - assert list(table.var.index) == expected - - with pytest.raises(ValueError, match=r"no marker column\(s\) \['CD99'\]"): - read_cell_table(path, "codex", pixel_size_um=None, default_region="A", cells_table_kwargs={"markers": ["CD99"]}) - - -@pytest.mark.parametrize( - ("fmt", "header", "markers"), - [ - ("mcquant", "CellID,DNA,CD3,DNA,X_centroid,Y_centroid,Area", ["DNA", "CD3", "DNA (2)"]), - ("sprm", "ID,DAPI,Blank,CD3,Blank", ["DAPI", "Blank", "CD3", "Blank (2)"]), - ("codex", "cell_id,x,y,size,DAPI,CD3,DAPI", ["DAPI", "CD3", "DAPI (2)"]), - ("generic", "cx,cy,DAPI,CD3,DAPI", ["DAPI", "CD3", "DAPI (2)"]), - ], -) @pytest.mark.parametrize("mangled", [False, True]) -def test_repeated_column_names_are_deduplicated( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fmt: str, header: str, markers: list[str], mangled: bool -) -> None: +def test_repeated_column_names_are_deduplicated(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mangled: bool) -> None: if mangled: # pandas >= 3.1 renames repeated names (DAPI.1) with the pyarrow engine too read_csv = pd.read_csv @@ -233,36 +138,22 @@ def mangling_read_csv(*args: Any, **kwargs: Any) -> pd.DataFrame: monkeypatch.setattr(pd, "read_csv", mangling_read_csv) path = tmp_path / "cells.csv" - n = header.count(",") + 1 - path.write_text(f"{header}\n{','.join(str(i + 1) for i in range(n))}\n", encoding="utf-8") - kwargs = {"x": "cx", "y": "cy"} if fmt == "generic" else {} + path.write_text("cell_id,x,y,size,DAPI,CD3,DAPI\n1,2,3,4,5,6,7\n", encoding="utf-8") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", UserWarning) # SPRM without centroids - table = read_cell_table(path, fmt, pixel_size_um=None, default_region="S", cells_table_kwargs=kwargs) - - assert list(table.var.index) == markers - assert table.X.shape == (1, len(markers)) - - -def test_quoted_column_names_may_hold_the_delimiter(tmp_path: Path) -> None: - path = tmp_path / "cells.csv" - path.write_text('cx,cy,"CD3, mean",DAPI\n1,2,3,4\n', encoding="utf-8") - - table = read_cell_table( - path, "generic", pixel_size_um=None, default_region="S", cells_table_kwargs={"x": "cx", "y": "cy"} - ) + with pytest.warns(UserWarning, match="no 'cycCCC_chHHH' columns"): + table = _read_codex(path, region="S") - assert list(table.var.index) == ["CD3, mean", "DAPI"] + assert list(table.var.index) == ["DAPI", "CD3", "DAPI (2)"] + assert table.X.shape == (1, 3) @pytest.mark.parametrize("cell_id", ["", "1.5", "a"]) def test_missing_or_fractional_instance_ids_raise(tmp_path: Path, cell_id: str) -> None: path = tmp_path / "cells.csv" - path.write_text(f"CellID,CD3,X_centroid,Y_centroid\n1,1,1,1\n{cell_id},1,2,2\n", encoding="utf-8") + path.write_text(f"cell_id,x,y,cyc001_ch001\n1,1,1,1\n{cell_id},2,2,1\n", encoding="utf-8") - with pytest.raises(ValueError, match=r"1 cell\(s\) whose instance id in 'CellID' is missing or not an integer"): - read_cell_table(path, "mcquant", pixel_size_um=None, default_region="S", cells_table_kwargs={}) + with pytest.raises(ValueError, match=r"1 cell\(s\) whose instance id in 'cell_id' is missing or not an integer"): + _read_codex(path, region="S") def test_build_cells_single_table_and_circles(tmp_path: Path) -> None: @@ -273,7 +164,7 @@ def test_build_cells_single_table_and_circles(tmp_path: Path) -> None: with pytest.warns(UserWarning, match=r"No segmentation mask found for region.*circles") as record: anndatas, shapes = build_cells( - tables, {}, {}, {"reg001": {"reg001": Identity()}, "reg002": {"reg002": translation}} + tables, {}, {"reg001": {"reg001": Identity()}, "reg002": {"reg002": translation}} ) assert len(record) == 1 @@ -297,7 +188,7 @@ def test_build_cells_different_panels_split_tables(tmp_path: Path) -> None: tables = [_read_codex(tmp_path / f"reg00{r}.fcs") for r in (1, 2)] with pytest.warns(UserWarning, match="circles"): - anndatas, _ = build_cells(tables, {}, {}, {}) + anndatas, _ = build_cells(tables, {}, {}) assert sorted(anndatas) == ["reg001_table", "reg002_table"] assert anndatas["reg002_table"].n_vars == 2 @@ -324,707 +215,7 @@ def test_build_cells_overlapping_regions_raise(tmp_path: Path, different_panels: tables = [_read_codex(tmp_path / name) for name in ("a.fcs", "b.fcs")] with pytest.raises(ValueError, match="Several cell tables"): - build_cells(tables, {}, {}, {}) - - -_QUPATH_HEADER = ( - "Image\tObject ID\tName\tClass\tCentroid X µm\tCentroid Y µm\tCell: Area µm^2\t" - "CD3: Nucleus: Mean\tCD3: Cell: Mean\tCD3: Cell: Median\tCD8: Nucleus: Mean" -) - - -def _write_qupath(path: Path, image: str = "S_Scan1.qptiff") -> None: - rows = [ - f"{image}\t0a1b\tcell\tTumor\t10\t5\t3.14159\t1\t2\t3\t4", - f"{image}\t0a1c\tcell\tStroma\t20\t15\t12.56637\t5\t6\t7\t8", - ] - path.write_text("\n".join([_QUPATH_HEADER, *rows]) + "\n", encoding="utf-8") - - -def _write_mcquant(path: Path, n: int = 5, extent: tuple[float, float] = (60, 40)) -> pd.DataFrame: - rng = np.random.default_rng(1) - cells = pd.DataFrame( - { - "CellID": np.arange(1, n + 1), - "DAPI": rng.uniform(0, 100, n), - "CD3": rng.uniform(0, 100, n), - "X_centroid": rng.uniform(0, extent[0], n), - "Y_centroid": rng.uniform(0, extent[1], n), - "Area": rng.uniform(10, 50, n), - "Eccentricity": rng.uniform(0, 1, n), - } - ) - cells.to_csv(path, index=False) - return cells - - -@pytest.mark.parametrize( - ("header", "expected"), - [ - ("cell_id,region,x,y,size,cyc001_ch001:DAPI1\n", "codex"), - (_QUPATH_HEADER + "\n", "qupath"), - ("CellID,DAPI,CD3,X_centroid,Y_centroid,Area\n", "mcquant"), - ("ID,DAPI,CD3\n", "sprm"), - # SPRM means have no centroids - ("ID,x,y,CD3\n", "generic"), - ("Path\tSample Name\tCell ID\tCell X Position\tCell Y Position\n", "inform"), - ("label,cx,cy,CD3\n", "generic"), - ], -) -def test_detect_formats(tmp_path: Path, header: str, expected: str) -> None: - path = tmp_path / "cells.csv" - path.write_text(header + "\n", encoding="utf-8") - - assert detect_table_format(path) == expected - - -def test_detect_fcs_is_codex(tmp_path: Path) -> None: - write_fcs(tmp_path / "cells.fcs", _cells()) - - assert detect_table_format(tmp_path / "cells.fcs") == "codex" - - -def test_qupath_um_to_px(tmp_path: Path) -> None: - path = tmp_path / "measurements.tsv" - _write_qupath(path) - - table = read_cell_table(path, "qupath", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) - - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, [[20, 10], [40, 30]]) - assert list(table.var.index) == ["CD3", "CD8"] - assert list(table.var["compartment"]) == ["Cell", "Nucleus"] - assert list(table.var["source_column"]) == ["CD3: Cell: Mean", "CD8: Nucleus: Mean"] - np.testing.assert_array_equal(table.X, np.array([[2, 4], [6, 8]], dtype=np.float32)) - # the other measurements, including the unused compartment, stay in obs - assert {"Class", "Object ID", "CD3: Nucleus: Mean", "CD3: Cell: Median"} <= set(table.obs.columns) - # Object IDs are UUIDs, so the instance ids are row numbers - assert list(table.instance_id) == [1, 2] - assert list(table.region) == ["S_Scan1.qptiff", "S_Scan1.qptiff"] - # areas of pi and 4 pi µm^2 at 0.5 µm/px are circles of radius 2 and 4 px - assert table.radius_px is not None - np.testing.assert_allclose(table.radius_px, [2, 4], rtol=1e-5) - - -def test_qupath_without_pixel_size_raises(tmp_path: Path) -> None: - path = tmp_path / "measurements.tsv" - _write_qupath(path) - - with pytest.raises(ValueError, match="pixel size"): - read_cell_table(path, "qupath", pixel_size_um=None, default_region="S", cells_table_kwargs={}) - - -def test_mcquant_columns(tmp_path: Path) -> None: - path = tmp_path / "cells.csv" - cells = _write_mcquant(path) - - table = read_cell_table(path, "auto", pixel_size_um=None, default_region="S", cells_table_kwargs={}) - - assert list(table.var.index) == ["DAPI", "CD3"] - np.testing.assert_allclose(table.X, cells[["DAPI", "CD3"]], rtol=1e-6) - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, cells[["X_centroid", "Y_centroid"]]) - assert list(table.instance_id) == [1, 2, 3, 4, 5] - assert list(table.region.unique()) == ["S"] - assert "Eccentricity" in table.obs.columns - assert table.radius_px is not None - np.testing.assert_allclose(table.radius_px, np.sqrt(cells["Area"] / np.pi)) - - -def test_mcquant_without_region_raises(tmp_path: Path) -> None: - path = tmp_path / "cells.csv" - _write_mcquant(path) - - with pytest.raises(ValueError, match="needs a region column"): - read_cell_table(path, "mcquant", pixel_size_um=None, default_region=None, cells_table_kwargs={}) - - -def test_generic_kwargs(tmp_path: Path) -> None: - path = tmp_path / "cells.csv" - path.write_text("label,cx,cy,sample,CD3,CD8,score\n7,2,4,A,1,2,0.5\n9,6,8,B,3,4,0.7\n") - kwargs = { - "x": "cx", - "y": "cy", - "instance_id": "label", - "region_column": "sample", - "markers": "CD.*", - "coordinates_unit": "um", - } - - table = read_cell_table(path, "generic", pixel_size_um=2.0, default_region=None, cells_table_kwargs=kwargs) - - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, [[1, 2], [3, 4]]) - assert list(table.var.index) == ["CD3", "CD8"] - assert list(table.instance_id) == [7, 9] - assert list(table.region) == ["A", "B"] - assert "score" in table.obs.columns - assert table.radius_px is None - - listed = read_cell_table( - path, - "generic", - pixel_size_um=None, - default_region="S", - cells_table_kwargs={"x": "cx", "y": "cy", "markers": ["CD8"]}, - ) - assert list(listed.var.index) == ["CD8"] - assert list(listed.instance_id) == [1, 2] - - -def test_row_number_ids_count_the_cells_without_a_region(tmp_path: Path) -> None: - path = tmp_path / "cells.csv" - path.write_text("cx,cy,sample,CD3\n1,2,A,1\n3,4,A,2\n5,6,,3\n7,8,B,4\n9,10,B,5\n") - kwargs = {"x": "cx", "y": "cy", "region_column": "sample"} - - with pytest.warns(UserWarning, match=r"Ignoring the 1 cell\(s\)"): - table = read_cell_table(path, "generic", pixel_size_um=None, default_region=None, cells_table_kwargs=kwargs) - - # the instance ids are the rows of the file - assert list(table.instance_id) == [1, 2, 4, 5] - assert list(table.region) == ["A", "A", "B", "B"] - assert table.row_instance_ids - - -def test_generic_bad_kwargs_raise(tmp_path: Path) -> None: - path = tmp_path / "cells.csv" - path.write_text("cx,cy,CD3\n1,2,3\n") - - with pytest.raises(ValueError, match="'x'"): - read_cell_table(path, "generic", pixel_size_um=None, default_region="S", cells_table_kwargs={}) - with pytest.raises(ValueError, match="unknown"): - read_cell_table(path, "generic", pixel_size_um=None, default_region="S", cells_table_kwargs={"xx": "cx"}) - with pytest.raises(ValueError, match="nope"): - read_cell_table( - path, "generic", pixel_size_um=None, default_region="S", cells_table_kwargs={"x": "cx", "y": "nope"} - ) - - -def _write_sprm(directory: Path, *, centers: bool = True) -> pd.DataFrame: - rng = np.random.default_rng(3) - means = pd.DataFrame({"ID": [1, 2, 4], "DAPI": rng.uniform(0, 1, 3), "CD3": rng.uniform(0, 1, 3)}) - means.to_csv(directory / "reg1.ome.tiff-cell_channel_mean.csv", index=False) - if centers: - # more cells than in the means table, in another order - pd.DataFrame({"ID": [4, 3, 2, 1], "x": [40.0, 30.0, 20.0, 10.0], "y": [4.0, 3.0, 2.0, 1.0]}).to_csv( - directory / "reg1.ome.tiff-cell_centers.csv", index=False - ) - return means - - -def test_sprm_adapter(tmp_path: Path) -> None: - means = _write_sprm(tmp_path) - path = tmp_path / "reg1.ome.tiff-cell_channel_mean.csv" - - assert detect_table_format(path) == "sprm" - table = read_cell_table(path, "auto", pixel_size_um=None, default_region="reg1", cells_table_kwargs={}) - - assert list(table.var.index) == ["DAPI", "CD3"] - np.testing.assert_allclose(table.X, means[["DAPI", "CD3"]], rtol=1e-6) - assert list(table.instance_id) == [1, 2, 4] - assert not table.row_instance_ids - assert list(table.region.unique()) == ["reg1"] - assert table.centroids_px is not None - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, [[10, 1], [20, 2], [40, 4]]) - assert table.radius_px is None - - -def test_sprm_without_centers_warns_and_omits_spatial(tmp_path: Path) -> None: - _write_sprm(tmp_path, centers=False) - path = tmp_path / "reg1.ome.tiff-cell_channel_mean.csv" - - with pytest.warns(UserWarning, match="cell_centers"): - table = read_cell_table(path, "sprm", pixel_size_um=None, default_region="reg1", cells_table_kwargs={}) - assert table.centroids_px is None - - mask = np.zeros((6, 6), dtype=np.uint16) - mask[1:3, 1:3], mask[4, 4] = 1, 2 - labels = Labels2DModel.parse(mask, dims=("y", "x")) - sdata_tables, shapes = build_cells([table], {}, {"reg1": labels}, {}) - assert "spatial" not in sdata_tables["table"].obsm - assert shapes == {} - with pytest.raises(ValueError, match="centroids"): - build_cells([table], {}, {}, {}) - - -def test_sprm_centers_missing_cells_raise(tmp_path: Path) -> None: - _write_sprm(tmp_path) - pd.DataFrame({"ID": [1, 2], "x": [1.0, 2.0], "y": [1.0, 2.0]}).to_csv( - tmp_path / "reg1.ome.tiff-cell_centers.csv", index=False - ) - - with pytest.raises(ValueError, match="cell_centers"): - read_cell_table( - tmp_path / "reg1.ome.tiff-cell_channel_mean.csv", - "sprm", - pixel_size_um=None, - default_region="reg1", - cells_table_kwargs={}, - ) - - -def _write_inform(path: Path, unit: str | None, area_unit: str = "square microns") -> None: - suffix = "" if unit is None else f" ({unit})" - header = [ - "Sample Name", - "Cell ID", - f"Cell X Position{suffix}", - f"Cell Y Position{suffix}", - f"Entire Cell Area ({area_unit})", - "Entire Cell DAPI Mean (Normalized Counts, Total Weighting)", - "Entire Cell CD8 (Opal 520) Mean (Normalized Counts, Total Weighting)", - "Nucleus DAPI Mean (Normalized Counts, Total Weighting)", - "Phenotype", - ] - rows = [ - ["Scan1.im3", "1", "100", "50", "314.159", "1.5", "2.5", "9", "CD8+"], - ["Scan1.im3", "2", "200", "80", "78.54", "3.5", "4.5", "9", "other"], - ] - path.write_text("\n".join("\t".join(row) for row in [header, *rows]) + "\n", encoding="utf-8") - - -def test_inform_adapter_microns(tmp_path: Path) -> None: - path = tmp_path / "Scan1_cell_seg_data.txt" - _write_inform(path, "microns") - - assert detect_table_format(path) == "inform" - table = read_cell_table(path, "inform", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) - - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, [[200, 100], [400, 160]]) - assert list(table.var.index) == ["DAPI", "CD8 (Opal 520)"] - np.testing.assert_allclose(table.X, [[1.5, 2.5], [3.5, 4.5]]) - assert list(table.instance_id) == [1, 2] - assert not table.row_instance_ids - assert list(table.region) == ["Scan1.im3"] * 2 - assert table.radius_px is not None - np.testing.assert_allclose(table.radius_px, np.sqrt(np.array([314.159, 78.54]) / 0.25 / np.pi)) - assert "Phenotype" in table.obs.columns - with pytest.raises(ValueError, match="pixel size"): - read_cell_table(path, "inform", pixel_size_um=None, default_region=None, cells_table_kwargs={}) - - -def test_inform_adapter_pixels(tmp_path: Path) -> None: - path = tmp_path / "Scan1_cell_seg_data.txt" - _write_inform(path, "pixels", area_unit="pixels") - - table = read_cell_table(path, "auto", pixel_size_um=None, default_region="S", cells_table_kwargs={}) - - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, [[100, 50], [200, 80]]) - assert table.radius_px is not None - np.testing.assert_allclose(table.radius_px, np.sqrt(np.array([314.159, 78.54]) / np.pi)) - - -def test_inform_without_unit_is_pixels_and_renames_cell_id(tmp_path: Path) -> None: - path = tmp_path / "Scan1_cell_seg_data.txt" - _write_inform(path, None, area_unit="pixels") - - table = read_cell_table(path, "inform", pixel_size_um=None, default_region=None, cells_table_kwargs={}) - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, [[100, 50], [200, 80]]) - assert table.radius_px is not None - - # mixed units (px positions, µm areas) cannot give a radius - _write_inform(path, "pixels") - table = read_cell_table(path, "inform", pixel_size_um=None, default_region=None, cells_table_kwargs={}) - assert table.radius_px is None - elements, _ = build_cells([table], {}, {}, {}) - assert "source_Cell ID" in elements["table"].obs.columns or "source_Cell_ID" in elements["table"].obs.columns - - -# the header of the inForm micron variant (github.com/lilab-stanford/auto-cell): no unit after `Cell X Position`, -# trailing spaces after the marker columns, CRLF line ends -_INFORM_MICRON_HEADER = [ - "Sample Name", - "Tissue Category", - "Cell ID", - "Total Cells", - "Tissue Category Area (square microns)", - "Cell Density (per square mm)", - "Cell X Position", - "Cell Y Position", - "Distance from Process Region Edge (microns)", - "Nucleus PD1 (PPD520) Mean ", - "Entire Cell Area (square microns)", - "Entire Cell PD1 (PPD520) Mean ", - "Entire Cell PPDDAPI Mean ", - "Lab ID", - "Slide ID", -] - - -def test_inform_micron_variant_without_position_unit(tmp_path: Path) -> None: - path = tmp_path / "sample_cell_seg_data.txt" - rows = [ - ['"TMA_Core[1,1,A]_[4768,30518].im3"', "STROMA", "1", "", "", "", "100", "50", "#N/A", "0", "314.159", "1.5"], - ['"TMA_Core[1,1,A]_[4768,30518].im3"', "TUMOR", "2", "", "", "", "200", "80", "#N/A", "0", "78.54", "3.5"], - ] - rows = [[*row, "2.5" if i == 0 else "4.5", "", ""] for i, row in enumerate(rows)] - path.write_text("\r\n".join("\t".join(row) for row in [_INFORM_MICRON_HEADER, *rows]), encoding="utf-8") - - assert detect_table_format(path) == "inform" - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) - - # the other columns are in microns, so the positions are too - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, [[200, 100], [400, 160]]) - assert list(table.var.index) == ["PD1 (PPD520)", "PPDDAPI"] - np.testing.assert_allclose(table.X, [[1.5, 2.5], [3.5, 4.5]]) - assert list(table.region) == ["TMA_Core[1,1,A]_[4768,30518].im3"] * 2 - assert table.radius_px is not None - np.testing.assert_allclose(table.radius_px, np.sqrt(np.array([314.159, 78.54]) / 0.25 / np.pi)) - assert "Nucleus PD1 (PPD520) Mean" in table.obs.columns - - -def test_inform_position_unit_from_other_columns(tmp_path: Path) -> None: - # the phenoptr sample (inForm 2.3) names the unit of its areas and distances only, here pixels - path = tmp_path / "S_cell_seg_data.txt" - header = ["Sample Name", "Cell ID", "Tissue Category Area (pixels)", "Cell X Position", "Cell Y Position"] - path.write_text("\t".join(header) + "\nS.im3\t1\t5\t10\t20\n", encoding="utf-8") - - table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, [[10, 20]]) - - # columns in both units leave the unit of the positions open - header[2] = "Distance from Process Region Edge (microns)" - path.write_text("\t".join([*header, "Entire Cell Area (pixels)"]) + "\nS.im3\t1\t5\t10\t20\t3\n", encoding="utf-8") - with pytest.raises(ValueError, match="coordinates_unit"): - read_cell_table(path, "inform", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) - table = read_cell_table( - path, "inform", pixel_size_um=0.5, default_region=None, cells_table_kwargs={"coordinates_unit": "um"} - ) - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, [[20, 40]]) - - -# R-exported QuPath tables (dryad doi:10.5061/dryad.hqbzkh1v1): `make.names` turns `Centroid X µm` into -# `Centroid.X.um`; the coordinates and the expression are split into two files -_R_COORDS_HEADER = "Object.ID,Centroid.X.um,Centroid.Y.um,core" - - -def _square(x: float, y: float, size: float = 8) -> dict[str, Any]: - return {"type": "Polygon", "coordinates": [[[x, y], [x + size, y], [x + size, y + size], [x, y + size], [x, y]]]} - - -def test_qupath_r_exported_coordinates_with_geojson(tmp_path: Path) -> None: - path = tmp_path / "C-7_coords.csv" - # µm centroids at 0.5 µm/px: the centres of the 8 px squares at (0, 0) and (20, 20) - path.write_text(f"{_R_COORDS_HEADER}\n0000aa8d-e87f,2,2,C-7\n0001e8f3-07bc,12,12,C-7\n", encoding="utf-8") - annotation = {"objectType": "annotation", "isLocked": True} - features = [ - {"type": "Feature", "id": "c28880d8", "geometry": _square(0, 0, 2270), "properties": annotation}, - *( - { - "type": "Feature", - "id": object_id, - "geometry": _square(x, x), - "nucleusGeometry": _square(x + 2, x + 2, 4), - "properties": {"objectType": "cell"}, - } - # in another order than the table; the last cell was filtered out of the table - for object_id, x in [("0001e8f3-07bc", 20), ("0000aa8d-e87f", 0), ("ffff0000-0000", 40)] - ), - ] - geojson = tmp_path / "C-7.geojson" - geojson.write_text(json.dumps(features), encoding="utf-8") - - assert detect_table_format(path) == "qupath" - with pytest.warns(UserWarning, match=r"1 of the 3 cell features of .*C-7.geojson have no row"): - table = read_cell_table( - path, "auto", pixel_size_um=0.5, default_region="C-7", cells_table_kwargs={"geojson": geojson} - ) - - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, [[4, 4], [24, 24]]) - assert table.polygons_px is not None - assert table.polygons_px[0].equals(shapely.box(0, 0, 8, 8)) - assert table.polygons_px[1].equals(shapely.box(20, 20, 28, 28)) - assert list(table.instance_id) == [1, 2] - assert list(table.obs["Object.ID"]) == ["0000aa8d-e87f", "0001e8f3-07bc"] - - -def test_qupath_r_exported_marker_columns(tmp_path: Path) -> None: - # the expression file joined to the coordinates on `Object.ID`: `__Mean`, and the - # `make.names` form of `CD3: Cell: Mean` - path = tmp_path / "C-7.csv" - header = f"{_R_COORDS_HEADER},DAPI_Nucleus_Mean,HLA.A_Cytoplasm_Mean,Pan.Cytokeratin_Cytoplasm_Mean,CD3..Cell..Mean" - path.write_text(f"{header}\nab,2,2,C-7,1,2,3,4\n", encoding="utf-8") - - table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region="C-7", cells_table_kwargs={}) - - assert list(table.var.index) == ["DAPI", "HLA.A", "Pan.Cytokeratin", "CD3"] - assert list(table.var["compartment"]) == ["Nucleus", "Cytoplasm", "Cytoplasm", "Cell"] - np.testing.assert_allclose(table.X, [[1, 2, 3, 4]]) - - -@pytest.mark.parametrize("area", ["Cell..Area.µm.2", "Cell..Area.um.2"]) -def test_qupath_r_exported_area(tmp_path: Path, area: str) -> None: - # `make.names` of `Cell: Area µm^2`; areas of pi and 4 pi µm^2 at 0.5 µm/px are circles of radius 2 and 4 px - path = tmp_path / "cells.csv" - path.write_text(f"{_R_COORDS_HEADER},{area}\nab,2,2,C-7,3.14159\ncd,12,12,C-7,12.56637\n", encoding="utf-8") - - table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region="C-7", cells_table_kwargs={}) - - assert table.radius_px is not None - np.testing.assert_allclose(table.radius_px, [2, 4], rtol=1e-5) - - # the unit of R-form measurements gives that of `X`/`Y` too - path.write_text(f"Object.ID,X,Y,{area}\nab,2,2,3.14159\n", encoding="utf-8") - with pytest.warns(UserWarning, match="reading them in um"): - table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region="C-7", cells_table_kwargs={}) - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, [[4, 4]]) - assert table.radius_px is not None - - -def test_qupath_expression_without_centroids_raises(tmp_path: Path) -> None: - path = tmp_path / "C-7_expression.csv" - path.write_text("Object.ID,DAPI_Nucleus_Mean,Ki67_Nucleus_Mean\nab,1,2\n", encoding="utf-8") - - assert detect_table_format(path) == "qupath" - with pytest.raises(ValueError, match="no centroid columns"): - read_cell_table(path, "auto", pixel_size_um=0.5, default_region="C-7", cells_table_kwargs={}) - - -# a QuPath detection export with `X`/`Y` centroids and bare marker names (zenodo.org/records/15514605) -_QUPATH_XY_HEADER = ( - "Image,Object ID,Parent,X,Y,Nucleus: Area µm^2,Nucleus: Length µm,Cell: Area µm^2,Nucleus/Cell area ratio," - "DAPI: Nucleus: Median,CD44,HLA-DR-AF647,CD45RO" -) - - -def test_qupath_xy_centroids_and_bare_markers(tmp_path: Path) -> None: - path = tmp_path / "measurement_TA4369.csv" - rows = [ - "STardist,74fcc0e7-d660,A-1,10,5,7.7,11.5,3.14159,0.13,20.5,1.5,34.2,4.3", - "STardist,9b2e5c11-0a3f,A-2,20,15,8.1,12.0,12.56637,0.2,21,2.5,35.2,5.3", - ] - path.write_text("\r\n".join([_QUPATH_XY_HEADER, *rows]) + "\r\n", encoding="utf-8") - - assert detect_table_format(path) == "qupath" - with ( - pytest.warns(UserWarning, match=r"reading the 3 other numeric columns \['CD44', 'HLA-DR-AF647', 'CD45RO'\]"), - pytest.warns(UserWarning, match=r"'X' and 'Y' .* name no unit; reading them in um.*coordinates_unit"), - ): - table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) - - # the other measurements are in µm, so the centroids are too - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, [[20, 10], [40, 30]]) - assert list(table.var.index) == ["CD44", "HLA-DR-AF647", "CD45RO"] - np.testing.assert_allclose(table.X, [[1.5, 34.2, 4.3], [2.5, 35.2, 5.3]], rtol=1e-6) - assert {"Parent", "DAPI: Nucleus: Median", "Nucleus/Cell area ratio", "Object ID"} <= set(table.obs.columns) - assert list(table.region) == ["STardist"] * 2 - assert table.radius_px is not None - np.testing.assert_allclose(table.radius_px, [2, 4], rtol=1e-5) - - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - table = read_cell_table( - path, - "qupath", - pixel_size_um=0.5, - default_region=None, - cells_table_kwargs={"region_column": "Parent", "markers": ["CD44", "CD45RO"], "coordinates_unit": "um"}, - ) - assert list(table.region) == ["A-1", "A-2"] - assert list(table.var.index) == ["CD44", "CD45RO"] - - -@pytest.mark.parametrize("r_names", [False, True]) -def test_qupath_bare_markers_exclude_morphology(tmp_path: Path, r_names: bool) -> None: - path = tmp_path / "detections.csv" - header = [ - "Image", - "Object ID", - "Centroid X µm", - "Centroid Y µm", - "Area µm^2", - "Perimeter µm", - "Max diameter µm", - "Circularity", - "Solidity", - "Eccentricity", - "Detection probability", - "CD44", - "CD45RO", - ] - if r_names: - header = [name.replace(" ", ".").replace("µm^2", "um.2").replace("µm", "um") for name in header] - path.write_text(",".join(header) + "\nS,u1,1,2,3,4,5,0.9,0.8,0.5,0.99,1.5,4.3\n", encoding="utf-8") - - with pytest.warns(UserWarning, match=r"reading the 2 other numeric columns \['CD44', 'CD45RO'\]"): - table = read_cell_table(path, "qupath", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) - - assert list(table.var.index) == ["CD44", "CD45RO"] - assert {header[4], header[7], header[10]} <= set(table.obs.columns) - - -@pytest.mark.parametrize( - ("intensities", "compartments"), - [ - # built-in cell detection (all versions) - (["Nucleus: DAPI mean", "Nucleus: DAPI std dev", "Cytoplasm: CD3 mean", "Cell: CD3 mean"], ["Nucleus", "Cell"]), - # cells of QuPath >= 0.6 - ( - ["Nucleus: DAPI: Mean", "Nucleus: DAPI: Median", "Cell: CD3: Mean", "Nucleus: CD3: Mean"], - ["Nucleus", "Cell"], - ), - # the same in R's `make.names` form - (["Nucleus..DAPI.mean", "Cytoplasm..CD3.mean", "Cell..CD3.mean"], ["Nucleus", "Cell"]), - # detections without compartments - (["DAPI: Mean", "DAPI: Median", "DAPI: Std.Dev.", "CD3: Mean", "CD3: Max"], None), - ], -) -def test_qupath_intensity_column_forms(tmp_path: Path, intensities: list[str], compartments: list[str] | None) -> None: - path = tmp_path / "cells.csv" - header = ["Image", "Object ID", "Name", "Classification", "Centroid X µm", "Centroid Y µm", *intensities] - rows = [f"S,u{i},,,{i},{i},{','.join(str(i + j) for j in range(len(intensities)))}" for i in range(2)] - path.write_text("\n".join([",".join(header), *rows]) + "\n", encoding="utf-8") - - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) - - assert list(table.var.index) == ["DAPI", "CD3"] - if compartments is None: - assert "compartment" not in table.var.columns - else: - assert list(table.var["compartment"]) == compartments - assert {"Name", "Classification"} <= set(table.obs.columns) - - -@pytest.mark.parametrize("r_names", [False, True]) -def test_qupath_bare_markers_exclude_empty_text_and_measurements(tmp_path: Path, r_names: bool) -> None: - path = tmp_path / "cells.csv" - header = ["Image", "Object ID", "Name", "Classification", "Centroid X µm", "Centroid Y µm", "DAPI: Median", "CD44"] - if r_names: - header = [name.replace(": ", "..").replace(" ", ".").replace("µm", "um") for name in header] - path.write_text(",".join(header) + "\nS,u1,,,1,2,3,4\nS,u2,,,1,2,3,4\n", encoding="utf-8") - - with pytest.warns(UserWarning, match=r"reading the 1 other numeric columns \['CD44'\]"): - table = read_cell_table(path, "qupath", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) - - assert list(table.var.index) == ["CD44"] - - -def test_qupath_xy_centroids_need_a_unit(tmp_path: Path) -> None: - path = tmp_path / "cells.csv" - path.write_text("Object ID,X,Y,CD3: Cell: Mean\nab,10,5,1\n", encoding="utf-8") - - with pytest.raises(ValueError, match="coordinates_unit"): - read_cell_table(path, "auto", pixel_size_um=0.5, default_region="S", cells_table_kwargs={}) - table = read_cell_table( - path, "auto", pixel_size_um=0.5, default_region="S", cells_table_kwargs={"coordinates_unit": "px"} - ) - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, [[10, 5]]) - - -def test_cells_outside_the_image_warn(tmp_path: Path) -> None: - # inForm positions in microns relative to the slide, not to the field of the image - write_qptiff(tmp_path / "Scan1.qptiff", biomarkers=["DAPI"], shape=(64, 80)) - _write_inform(tmp_path / "Scan1_cell_seg_data.txt", "microns") - - with pytest.warns(UserWarning, match=r"2 of the 2 cells of region 'Scan1' lie outside its image \(80 x 64 px\)"): - phenocycler( - tmp_path, cells_table=tmp_path / "Scan1_cell_seg_data.txt", image_models_kwargs={"scale_factors": [2]} - ) - - -def _write_inform_seg_maps( - path: Path, shape: tuple[int, int] = (16, 24), first: str = "TissueClassMap", compartments: int = 3 -) -> np.ndarray: - # inForm `*_binary_seg_maps.tif`: the tissue class map, then one page per compartment, each with its XML - descriptions = [ - f'\n<{first}>\n 1\n', - *( - '\n\n 1\n' - f" {compartment}\n" - for compartment in ("Nucleus", "Cytoplasm", "Membrane")[:compartments] - ), - ] - pages = np.stack([np.full(shape, i, dtype=np.uint16) for i in range(len(descriptions))]) - with tifffile.TiffWriter(path) as tw: - for page, description in zip(pages, descriptions, strict=True): - tw.write(page, description=description, metadata=None) - return pages - - -@pytest.mark.parametrize(("mask_channel", "expected"), [(None, 1), ("Cytoplasm", 2), (0, 0)]) -def test_read_labels_inform_seg_maps(tmp_path: Path, mask_channel: int | str | None, expected: int) -> None: - path = tmp_path / "S_binary_seg_maps.tif" - pages = _write_inform_seg_maps(path) - - labels = read_labels(path, {"scale_factors": None}, {"global": Identity()}, mask_channel=mask_channel) - - # by default the nuclei, which hold the `Cell ID` of every cell, not the tissue class map - np.testing.assert_array_equal(labels.values, pages[expected]) - with pytest.raises(ValueError, match=r"'Tissue', 'Nucleus', 'Cytoplasm', 'Membrane'"): - read_labels(path, {}, {"global": Identity()}, mask_channel="cells") - - -def test_read_labels_inform_seg_maps_other_pages(tmp_path: Path) -> None: - # pages are named by their XML root tag; the nuclei are the default without a tissue class map too - path = tmp_path / "S_binary_seg_maps.tif" - pages = _write_inform_seg_maps(path, first="ProcessRegionMap", compartments=2) - - labels = read_labels(path, {"scale_factors": None}, {"global": Identity()}) - - np.testing.assert_array_equal(labels.values, pages[1]) - with pytest.raises(ValueError, match=r"\['ProcessRegionMap', 'Nucleus', 'Cytoplasm'\]"): - read_labels(path, {}, {"global": Identity()}, mask_channel="Membrane") - - -@pytest.mark.parametrize( - ("header", "x", "y"), [("object_id,x,y,CD3", "x", "y"), ("Object Id,XMin,XMax,cx,cy,CD3", "cx", "cy")] -) -def test_object_id_lookalikes_stay_generic(tmp_path: Path, header: str, x: str, y: str) -> None: - # only QuPath's own `Object ID` (or R's `Object.ID`) makes a QuPath table, e.g. not HALO's `Object Id` - path = tmp_path / "cells.csv" - row = "7,10,5,2" if header.startswith("object_id") else "7,0,20,10,5,2" - path.write_text(f"{header}\n{row}\n", encoding="utf-8") - - assert detect_table_format(path) == "generic" - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - table = read_cell_table( - path, "auto", pixel_size_um=0.5, default_region="S", cells_table_kwargs={"x": x, "y": y} - ) - - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, [[10, 5]]) - assert "CD3" in table.var.index - - -def test_auto_with_centroid_kwargs_is_generic(tmp_path: Path) -> None: - # user-given centroid columns win over the detected format - path = tmp_path / "measurements.tsv" - _write_qupath(path) - - table = read_cell_table( - path, - "auto", - pixel_size_um=0.5, - default_region=None, - cells_table_kwargs={"x": "Centroid X µm", "y": "Centroid Y µm", "region_column": "Image", "markers": "CD.*"}, - ) - - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, [[10, 5], [20, 15]]) - - -def test_unknown_format_raises(tmp_path: Path) -> None: - path = tmp_path / "cells.csv" - path.write_text("ID,DAPI\n1,2\n") - - with pytest.raises(ValueError, match="qupath"): - read_cell_table(path, "excel", pixel_size_um=None, default_region="S", cells_table_kwargs={}) + build_cells(tables, {}, {}) def test_resolve_per_region(tmp_path: Path) -> None: @@ -1057,15 +248,14 @@ def test_read_labels_lazy_with_transformation(tmp_path: Path) -> None: assert get_transformation(labels, "global") == translation -@pytest.mark.parametrize(("mask_channel", "expected"), [(None, 0), (1, 1), ("nuclei", 1)]) -def test_read_labels_ome_channels(tmp_path: Path, mask_channel: int | str | None, expected: int) -> None: +def test_read_labels_first_channel(tmp_path: Path) -> None: masks = np.stack([np.full((16, 24), i + 1, dtype=np.int32) for i in range(2)]) path = tmp_path / "mask.ome.tif" tifffile.imwrite(path, masks, metadata={"axes": "CYX", "Channel": {"Name": ["cells", "nuclei"]}}) - labels = read_labels(path, {"scale_factors": None}, {"global": Identity()}, mask_channel=mask_channel) + labels = read_labels(path, {"scale_factors": None}, {"global": Identity()}) - np.testing.assert_array_equal(labels.values, masks[expected]) + np.testing.assert_array_equal(labels.values, masks[0]) def test_read_labels_errors(tmp_path: Path) -> None: @@ -1074,10 +264,6 @@ def test_read_labels_errors(tmp_path: Path) -> None: with pytest.raises(ValueError, match="integer"): read_labels(path, {}, {"global": Identity()}) - tifffile.imwrite(path, np.zeros((2, 8, 8), dtype=np.uint16)) - with pytest.raises(ValueError, match="channel"): - read_labels(path, {}, {"global": Identity()}, mask_channel=5) - @pytest.mark.parametrize("compression", [None, "zlib"]) def test_read_labels_pyramidal_ome_is_lazy(tmp_path: Path, compression: str | None) -> None: @@ -1107,14 +293,11 @@ def test_mask_integer_downsample_autoscaled(tmp_path: Path, mask_shape: tuple[in mask = np.zeros(mask_shape, dtype=np.uint32) mask[2:6, 2:6], mask[10:14, 8:12] = 1, 2 tifffile.imwrite(tmp_path / "B_mask.tif", mask) - # the centroids are in mask pixels, inside the mask - cells = _write_mcquant(tmp_path / "B.csv", n=2, extent=(mask_shape[1], mask_shape[0])) with warnings.catch_warnings(): warnings.simplefilter("error", UserWarning) sdata = phenocycler( tmp_path, - cells_table={"B": tmp_path / "B.csv"}, labels={"A": tmp_path / "A_mask.tif", "B": tmp_path / "B_mask.tif"}, image_models_kwargs={"scale_factors": [2]}, ) @@ -1137,13 +320,6 @@ def test_mask_integer_downsample_autoscaled(tmp_path: Path, mask_shape: tuple[in assert labels_extent[axis][1] == pytest.approx(expected_end) if image_size % mask_size == 0: assert labels_extent[axis][1] == pytest.approx(image_extent[axis][1]) - # mcquant centroids of the mask are in mask pixels, the table's are in image pixels, at the centre of the - # factor x factor image pixels of their mask pixel; obs keeps them as read - table = sdata.tables["table"] - expected = cells[["X_centroid", "Y_centroid"]].to_numpy() * factor + (factor - 1) / 2 - np.testing.assert_allclose(table.obsm["spatial"], expected) - np.testing.assert_allclose(table.obs["X_centroid"], cells["X_centroid"]) - # writing may reorder the axes of the translation, so compare the composed affine maps sdata.write(tmp_path / "x.zarr") read = get_transformation(read_zarr(tmp_path / "x.zarr").labels["B_labels"], "global") @@ -1154,78 +330,15 @@ def test_mask_integer_downsample_autoscaled(tmp_path: Path, mask_shape: tuple[in ) -def test_mask_downsample_keeps_image_pixel_centroids(tmp_path: Path) -> None: - # inForm centroids are in image pixels, so a downsampled mask does not scale them - write_qptiff(tmp_path / "Scan1.qptiff", biomarkers=["DAPI"], shape=(128, 256)) - tifffile.imwrite(tmp_path / "mask.tif", np.zeros((64, 128), dtype=np.uint32)) - _write_inform(tmp_path / "Scan1_cell_seg_data.txt", "pixels", area_unit="pixels") - - sdata = phenocycler( - tmp_path, - cells_table=tmp_path / "Scan1_cell_seg_data.txt", - labels=tmp_path / "mask.tif", - image_models_kwargs={"scale_factors": [2]}, - ) - - labels_transformation = get_transformation(sdata.labels[next(iter(sdata.labels))], "global") - assert isinstance(labels_transformation, Sequence) - assert labels_transformation.transformations[0] == Scale([2, 2], axes=("y", "x")) - np.testing.assert_allclose(sdata.tables["table"].obsm["spatial"], [[100, 50], [200, 80]]) - - -@pytest.mark.parametrize(("space", "factor"), [(None, 1), ("image", 1), ("mask", 2)]) -def test_generic_px_table_coordinates_space(tmp_path: Path, space: str | None, factor: int) -> None: - # generic pixel centroids are in image pixels unless `coordinates_space="mask"` - write_qptiff(tmp_path / "S.qptiff", biomarkers=["DAPI"], shape=(64, 80)) - mask = np.zeros((32, 40), dtype=np.uint32) - mask[2:4, 2:4], mask[5:7, 5:7] = 1, 2 - tifffile.imwrite(tmp_path / "mask.tif", mask) - (tmp_path / "cells.csv").write_text("label,cx,cy,CD3\n1,6,6,1\n2,12,12,2\n") - kwargs = {"x": "cx", "y": "cy", "instance_id": "label"} - if space is not None: - kwargs["coordinates_space"] = space - - sdata = phenocycler( - tmp_path, - cells_table=tmp_path / "cells.csv", - labels=tmp_path / "mask.tif", - image_models_kwargs={"scale_factors": [2]}, - cells_table_kwargs=kwargs, - ) - - expected = np.array([[6, 6], [12, 12]]) * factor + (factor - 1) / 2 - np.testing.assert_allclose(sdata.tables["table"].obsm["spatial"], expected) - - -@pytest.mark.parametrize( - ("fmt", "kwargs", "match"), - [ - ("generic", {"coordinates_space": "labels"}, "must be 'image' or 'mask'"), - ("generic", {"coordinates_space": "mask", "coordinates_unit": "um"}, "in µm"), - ("mcquant", {"coordinates_space": "image"}, "only used with generic"), - ], -) -def test_bad_coordinates_space_raises(tmp_path: Path, fmt: str, kwargs: dict[str, str], match: str) -> None: - path = tmp_path / "cells.csv" - path.write_text("CellID,CD3,X_centroid,Y_centroid\n1,1,6,6\n") - kwargs = {"x": "X_centroid", "y": "Y_centroid", **kwargs} if fmt == "generic" else kwargs - - with pytest.raises(ValueError, match=match): - read_cell_table(path, fmt, pixel_size_um=0.5, default_region="S", cells_table_kwargs=kwargs) - - def test_mask_non_integer_mismatch_warns(tmp_path: Path) -> None: write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"], shape=(64, 80)) tifffile.imwrite(tmp_path / "mask.tif", np.zeros((48, 60), dtype=np.uint32)) - cells = _write_mcquant(tmp_path / "cells.csv", n=2) with pytest.warns(UserWarning, match=r"\(48, 60\).*\(64, 80\).*not align"): sdata = phenocycler( tmp_path, - cells_table=tmp_path / "cells.csv", labels=tmp_path / "mask.tif", image_models_kwargs={"scale_factors": [2]}, ) assert get_transformation(sdata.labels["S_labels"], "global") == Identity() - np.testing.assert_allclose(sdata.tables["table"].obsm["spatial"], cells[["X_centroid", "Y_centroid"]].to_numpy()) diff --git a/tests/test_akoya_real_data.py b/tests/test_akoya_real_data.py index 402f6e33..123e6bbf 100644 --- a/tests/test_akoya_real_data.py +++ b/tests/test_akoya_real_data.py @@ -1,33 +1,24 @@ """Tests of the Akoya readers on real public datasets. They run only when the environment variable ``AKOYA_TEST_DATA`` points to a directory holding the files below (paths -relative to it). The files are public (Zenodo, Figshare, BioImage Archive, Dryad, GitHub, the MCMICRO bucket) but -too large to download in CI. +relative to it). The files are public (Zenodo, Figshare, BioImage Archive, GitHub) but too large to +download in CI. """ from __future__ import annotations import os -import warnings from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING import dask.array as da import numpy as np -import pandas as pd import pytest import tifffile -from scipy.stats import spearmanr -from spatialdata import read_zarr from spatialdata.models import get_channel_names from spatialdata.transformations import get_transformation from spatialdata_io import codex, phenocycler -from spatialdata_io.readers._utils._cell_tables import detect_table_format, read_cell_table - -if TYPE_CHECKING: - from collections.abc import Iterable _ROOT = os.environ.get("AKOYA_TEST_DATA") @@ -281,29 +272,6 @@ def test_real_same_qptiff_through_both_readers() -> None: assert a.attrs["akoya"]["regions"] == b.attrs["akoya"]["regions"] -def _image_at(sdata, region: str, channel: str, xy: np.ndarray) -> np.ndarray: - """The full-resolution values of ``channel`` of the image of ``region`` at the pixel positions ``xy``.""" - image = sdata.images[f"{region}_image"]["scale0"]["image"] - plane = image.sel(c=channel).values - x = np.clip(xy[:, 0].astype(int), 0, plane.shape[1] - 1) - y = np.clip(xy[:, 1].astype(int), 0, plane.shape[0] - 1) - return plane[y, x] - - -def _assert_cells_on_their_signal(sdata, region: str, element: str, pairs: dict[str, str], rho: float) -> None: - """The table's per-cell means correlate with the image at the cell centroids, and drop away from them.""" - table = sdata.tables["table"] - cells = table[table.obs["region"] == element] - xy = cells.obsm["spatial"] - for var, channel in pairs.items(): - values = np.asarray(cells[:, var].X).ravel() - at_cells = spearmanr(values, _image_at(sdata, region, channel, xy)).statistic - shifted = spearmanr(values, _image_at(sdata, region, channel, xy + 25)).statistic - assert at_cells > rho, (var, at_cells) - # clustered markers stay correlated nearby, but less so - assert shifted < at_cells - 0.1, (var, at_cells, shifted) - - def test_real_codex_processor_csv() -> None: # a 500-cell subset of a CODEX Processor CSV with the `cyc001_ch001:DAPI1` header (github.com/igordot/phenomenalist) path = Path(_ROOT) / "github_codex_metadata/phenomenalist/tonsil-akoya-2018-500.csv" # type: ignore[arg-type] @@ -319,190 +287,3 @@ def test_real_codex_processor_csv() -> None: assert list(table.obs["region"].cat.categories) == ["reg001_cells"] assert table.obsm["spatial"].shape == (500, 2) assert len(sdata.shapes["reg001_cells"]) == 500 - - -# public cell-table exports (see the docstrings of `read_cell_table` and `read_labels` for the variants) -_EXPORTS = "exports" -_PHENOPTR = f"{_EXPORTS}/github_phenoptr/repo/inst/extdata/sample/Set4_1-6plex_[16142,55840]" -_DRYAD = f"{_EXPORTS}/dryad_hqbzkh1v1" -# the OME `PhysicalSizeX` of the Dryad images, which no Akoya reader reads -_DRYAD_PIXEL_SIZE_UM = 0.5072844 - - -def _export(relative: str) -> Path: - path = Path(_ROOT) / relative # type: ignore[arg-type] - if not path.exists(): - pytest.skip(f"{path} is missing") - return path - - -def _assert_cells_inside_images(caught: Iterable[warnings.WarningMessage]) -> None: - assert not [w for w in caught if "outside its image" in str(w.message)] - - -def test_real_inform_table_with_seg_maps_and_component_data() -> None: - # inForm 2.3 (github.com/akoyabio/phenoptr): positions in pixels, the `Cell ID`s are the labels of page 1 (nuclei) - # of the segmentation map; page 0 is the tissue class map - image = _export(f"{_PHENOPTR}_component_data.tif") - - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - sdata = phenocycler( - image, - cells_table=_export(f"{_PHENOPTR}_cell_seg_data.txt"), - labels=_export(f"{_PHENOPTR}_binary_seg_maps.tif"), - ) - _assert_cells_inside_images(caught) - - region = "Set4_1-6plex__16142_55840__component_data" - table = sdata.tables["table"] - assert table.shape == (6072, 8) - assert list(table.var_names[:2]) == ["PDL1 (Opal 520)", "CD8 (Opal 540)"] - assert list(table.obs["region"].cat.categories) == [f"{region}_labels"] - labels = sdata.labels[f"{region}_labels"]["scale0"]["image"] - assert labels.shape == (1400, 1868) - assert set(table.obs["cell_id"]) == set(np.unique(labels.values)) - {0} - np.testing.assert_allclose(table.obsm["spatial"][0], [515, 13]) - # the table's centroids are those of its labels, and its DAPI is the image's DAPI there - mask = labels.values - ys, xs = np.nonzero(mask) - counts = np.bincount(mask[ys, xs]) - ids = table.obs["cell_id"].to_numpy() - centroids = np.c_[np.bincount(mask[ys, xs], xs)[ids], np.bincount(mask[ys, xs], ys)[ids]] / counts[ids, None] - # inForm rounds the positions to whole pixels - assert np.median(np.linalg.norm(centroids + 0.5 - table.obsm["spatial"], axis=1)) < 2 - _assert_cells_on_their_signal(sdata, region, f"{region}_labels", {"DAPI": "DAPI"}, rho=0.5) - - -def test_real_inform_micron_variant() -> None: - # github.com/lilab-stanford/auto-cell: positions in microns without a unit in their name, trailing spaces - path = _export(f"{_EXPORTS}/github_auto-cell/repo/cell_annotation/sample_cell_seg_data.txt") - - assert detect_table_format(path) == "inform" - table = read_cell_table(path, "auto", pixel_size_um=0.5, default_region=None, cells_table_kwargs={}) - - assert len(table.obs) == 4 - assert list(table.var.index) == [ - "PD1 (PPD520)", - "FOXP3 (PPD540)", - "CD20 (PPD620)", - "CD3 (PPD650)", - "PANCK (PPD690)", - "PPDDAPI", - "Autofluorescence", - ] - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px[0], [4230.4 / 0.5, 29684.5 / 0.5]) - assert table.radius_px is not None - - -def test_real_qupath_r_exported_coordinates_with_geojson(tmp_path: Path) -> None: - # doi:10.5061/dryad.hqbzkh1v1: R-exported QuPath centroids (`Centroid.X.um`) and measurements in two files, and - # the GeoJSON of all StarDist cells (UUID ids, one `annotation`); 116 cells were filtered out of the tables - coords = _export(f"{_DRYAD}/C-7_coords.csv") - geojson = _export(f"{_DRYAD}/C-7.geojson") - expression = _export(f"{_DRYAD}/C-7_expression.csv") - kwargs = {"geojson": geojson} - - assert detect_table_format(coords) == "qupath" - with pytest.warns(UserWarning, match="116 of the 9531 cell features"): - table = read_cell_table( - coords, "auto", pixel_size_um=_DRYAD_PIXEL_SIZE_UM, default_region="C-7", cells_table_kwargs=kwargs - ) - assert table.polygons_px is not None - assert pd.notna(table.polygons_px).all() - assert len(table.obs) == 9415 - - # joined on the object id, the measurements are the markers: 30 markers in the `Cell` compartment (also 27 - # `Cytoplasm`, 27 `Membrane`, 3 `Nucleus`), the first one taken; the expression file alone has no centroids in - # QuPath's form (its `X_coord`/`Y_coord` are µm, `Y_coord` negated) - joined = tmp_path / "C-7.csv" - pd.read_csv(coords).merge(pd.read_csv(expression), on="Object.ID", validate="1:1").to_csv(joined, index=False) - with pytest.warns(UserWarning, match="116 of the 9531 cell features"): - table = read_cell_table( - joined, "auto", pixel_size_um=_DRYAD_PIXEL_SIZE_UM, default_region="C-7", cells_table_kwargs=kwargs - ) - assert table.X.shape == (9415, 30) - assert {"DAPI", "HLA.A", "Pan.Cytokeratin"} <= set(table.var.index) - assert set(table.var["compartment"]) == {"Cell"} - assert table.var.loc["DAPI", "source_column"] == "DAPI_Cell_Mean" - - -def test_real_qupath_detection_xy_and_bare_markers() -> None: - # zenodo.org/records/15514605: `X`/`Y` centroids in µm, markers named after the marker only, `Parent` = TMA core - path = _export(f"{_EXPORTS}/zenodo_15514605/measurement_TA4369.csv") - - assert detect_table_format(path) == "qupath" - with pytest.warns(UserWarning, match="reading the 48 other numeric columns"): - table = read_cell_table( - path, "auto", pixel_size_um=0.5, default_region=None, cells_table_kwargs={"region_column": "Parent"} - ) - - assert table.X.shape == (348314, 48) - assert list(table.var.index[:3]) == ["CD44", "CD68", "CD31"] - assert table.region.nunique() == 59 - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px[0], [2620.8 / 0.5, 661.3 / 0.5]) - - -def test_real_mcquant_table_with_ome_tiff() -> None: - # MCMICRO exemplar-001: mcquant table in pixels with the OME-TIFF of its first cycle, read as a hyperstack - image = _export(f"{_EXPORTS}/s3_mcmicro_exemplar001/exemplar-001-cycle6.ome.tif") - cells = _export(f"{_EXPORTS}/s3_mcmicro_exemplar001/unmicst-exemplar-001_cell.csv") - - # the OME-XML has no channel names: those of cycle 6 in `markers.csv` - with pytest.warns(UserWarning, match="cells are represented as circles") as caught: - sdata = codex(image, cells_table=cells, channel_names=["DNA_6", "ELANE", "CD57", "CD45"]) - _assert_cells_inside_images(caught) - - table = sdata.tables["table"] - assert table.shape == (11170, 12) - assert list(table.var_names[:4]) == ["DNA_6", "ELANE", "CD57", "CD45"] - assert len(sdata.shapes["exemplar-001-cycle6_cells"]) == 11170 - pairs = {"DNA_6": "DNA_6", "CD57": "CD57"} - _assert_cells_on_their_signal(sdata, "exemplar-001-cycle6", "exemplar-001-cycle6_cells", pairs, rho=0.8) - - -def test_real_qupath_cores_with_geojson_end_to_end(tmp_path: Path) -> None: - # two TMA cores of doi:10.5061/dryad.hqbzkh1v1 in one read: OME-TIFFs, QuPath centroids + measurements, StarDist - # GeoJSON; the pixel size comes from `experiment.json` (the OME `PhysicalSizeX`) - cores = ("C-7", "N-12") - for core in cores: - (tmp_path / f"{core}.ome.tif").symlink_to(_export(f"{_DRYAD}/{core}.ome.tif")) - coords = pd.read_csv(_export(f"{_DRYAD}/{core}_coords.csv")) - expression = pd.read_csv(_export(f"{_DRYAD}/{core}_expression.csv")) - coords.merge(expression, on="Object.ID", validate="1:1").to_csv(tmp_path / f"{core}.csv", index=False) - (tmp_path / "channelnames.txt").symlink_to(_export(f"{_DRYAD}/channelnames.txt")) - (tmp_path / "experiment.json").write_text(f'{{"xyResolution": {_DRYAD_PIXEL_SIZE_UM * 1000}}}') - - with pytest.warns(UserWarning, match="cell features"): - sdata = codex( - tmp_path, - cells_table={core: tmp_path / f"{core}.csv" for core in cores}, - cells_table_kwargs={"geojson": {core: _export(f"{_DRYAD}/{core}.geojson") for core in cores}}, - ) - - assert sorted(sdata.coordinate_systems) == list(cores) - table = sdata.tables["table"] - assert list(table.obs["region"].cat.categories) == [f"{core}_labels" for core in cores] - for core, n_cells in zip(cores, (9415, 14242), strict=True): - cells = table[table.obs["region"] == f"{core}_labels"] - assert cells.n_obs == n_cells - assert sdata.shapes[f"{core}_polygons"].geometry.is_valid.all() - mask = sdata.labels[f"{core}_labels"]["scale0"]["image"].values - assert set(np.unique(mask)) - {0} == set(cells.obs["cell_id"]) - # the rasterised cell under each centroid is that cell - x, y = cells.obsm["spatial"].astype(int).T - assert (mask[y, x] == cells.obs["cell_id"].to_numpy()).mean() > 0.99 - _assert_cells_on_their_signal(sdata, core, f"{core}_labels", {"DAPI": "DAPI"}, rho=0.6) - - sdata.write(tmp_path / "dryad.zarr") - read = read_zarr(tmp_path / "dryad.zarr") - assert read.attrs["akoya"] == sdata.attrs["akoya"] - for name in ("C-7_labels", "N-12_polygons", "N-12_image"): - assert get_transformation(read[name], get_all=True) == get_transformation(sdata[name], get_all=True) - np.testing.assert_array_equal(read.tables["table"].X, table.X) - for name in ("C-7_labels", "N-12_image"): - element = read.labels.get(name) or read.images[name] - expected = sdata.labels.get(name) or sdata.images[name] - np.testing.assert_array_equal(element["scale0"]["image"].values, expected["scale0"]["image"].values) diff --git a/tests/test_codex.py b/tests/test_codex.py index 6c58120f..da8932a3 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -295,9 +295,7 @@ def test_discover_legacy_tables_ranking(tmp_path: Path) -> None: def test_codex_user_cells_table_replaces_discovered(tmp_path: Path) -> None: write_legacy_processor_tree(tmp_path, regions=2, layout="per_marker", n_cells=50) - pd.DataFrame( - {"CellID": [1, 2, 3], "CD3": [1.0, 2.0, 3.0], "X_centroid": [1.0, 2.0, 3.0], "Y_centroid": [4.0, 5.0, 6.0]} - ).to_csv(tmp_path / "mine.csv", index=False) + (tmp_path / "mine.csv").write_text("cell_id,x,y,cyc001_ch001:CD3\n1,1.0,4.0,1.0\n2,2.0,5.0,2.0\n3,3.0,6.0,3.0\n") mask = np.zeros((128, 160), dtype=np.uint16) mask[:3, :3] = [1, 2, 3] tifffile.imwrite(tmp_path / "mask.tif", mask) @@ -414,47 +412,6 @@ def test_codex_labels_with_discovered_tables(tmp_path: Path) -> None: assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["reg001_labels"] -def test_codex_qupath_geojson_labels(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path / "tree", regions=2, layout="per_marker", n_cells=5) - header = "Image\tObject ID\tCentroid X px\tCentroid Y px\tCD3: Cell: Mean" - rows = ["reg002.tif\tu1\t5\t5\t1", "reg002.tif\tu2\t25\t25\t2"] - (tmp_path / "q.tsv").write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") - features = [ - {"type": "Feature", "id": oid, "geometry": {"type": "Polygon", "coordinates": [ring]}, "properties": {}} - for oid, ring in [ - ("u2", [[20, 20], [30, 20], [30, 30], [20, 30], [20, 20]]), - ("u1", [[0, 0], [9, 0], [9, 9], [0, 0]]), - ] - ] - (tmp_path / "polygons.geojson").write_text(json.dumps({"type": "FeatureCollection", "features": features})) - - sdata = codex( - tmp_path / "tree", - regions=["reg002"], - cells_table={"reg002": tmp_path / "q.tsv"}, - image_models_kwargs=_FAST, - cells_table_kwargs={"geojson": {"reg002": tmp_path / "polygons.geojson"}}, - ) - - assert list(sdata.labels) == ["reg002_labels"] - assert list(sdata.shapes) == ["reg002_polygons"] - values = sdata.labels["reg002_labels"]["scale0"]["image"].values - assert values.shape == (128, 160) - assert values[2, 7] == 1 # the triangle below the diagonal - assert values[7, 2] == 0 - assert values[25, 25] == 2 - assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["reg002_labels"] - assert sdata.attrs["akoya"]["geojson"] == {"reg002": str(tmp_path / "polygons.geojson")} - - with pytest.raises(ValueError, match="same form as the tables"): - codex( - tmp_path / "tree", - cells_table=tmp_path / "q.tsv", - image_models_kwargs=_FAST, - cells_table_kwargs={"geojson": {"reg002": tmp_path / "polygons.geojson"}}, - ) - - def test_codex_str_regions_and_channel_names(tmp_path: Path) -> None: write_legacy_processor_tree(tmp_path, regions=2, cycles=1, channels=1, layout="per_marker", n_cells=5) @@ -570,8 +527,6 @@ def test_discover_legacy_regionless_table_ignores_sidecar_csv(tmp_path: Path) -> def test_codex_region_named_non_cell_csv_is_ignored(tmp_path: Path, with_table: bool) -> None: data = write_legacy_processor_tree(tmp_path, cycles=1, channels=2, fcs=False, csv=with_table, n_cells=5) (tmp_path / "reg001_antibodies.csv").write_text("Antibody,Clone\nCD3,UCHT1\n") - # a HuBMAP SPRM output of the same region, which is only read when given as `cells_table` - (tmp_path / "reg001_expr-cell_channel_mean.csv").write_text("ID,CD3\n1,2.0\n") if with_table: with pytest.warns(UserWarning, match="circles"): @@ -813,24 +768,18 @@ def test_codex_old_positional_call(tmp_path: Path) -> None: codex(tmp_path, True, {"nframes": 1}) -@pytest.mark.parametrize("case", ["one_region", "kidney_2019", "regions_subset", "phenocycler_codex_csv"]) +@pytest.mark.parametrize("case", ["one_region", "kidney_2019", "regions_subset"]) def test_single_region_codex_table_write_roundtrip(tmp_path: Path, case: str) -> None: data = tmp_path / "data" data.mkdir() if case == "kidney_2019": tifffile.imwrite(data / "Kidney.tif", np.zeros((2, 16, 16), dtype=np.uint16)) (data / "Kidney.csv").write_text("cell_id,x,y,size,cyc001_ch001\n1,1.0,2.0,3.0,4.0\n2,5.0,6.0,7.0,8.0\n") - elif case == "phenocycler_codex_csv": - write_qptiff(data / "S_Scan1.qptiff", biomarkers=["DAPI"]) - (data / "cells.csv").write_text("cell_id,x,y,size,DAPI\n1,1.0,2.0,3.0,4.0\n2,5.0,6.0,7.0,8.0\n") else: write_legacy_processor_tree(data, regions=1 if case == "one_region" else 2, cycles=1, channels=1, n_cells=5) with pytest.warns(UserWarning, match="circles"): - if case == "phenocycler_codex_csv": - sdata = phenocycler(data, cells_table=data / "cells.csv", image_models_kwargs=_FAST) - else: - sdata = codex(data, regions=["reg001"] if case == "regions_subset" else None, image_models_kwargs=_FAST) + sdata = codex(data, regions=["reg001"] if case == "regions_subset" else None, image_models_kwargs=_FAST) sdata.write(tmp_path / "x.zarr") table = read_zarr(tmp_path / "x.zarr").tables["table"] @@ -862,31 +811,6 @@ def test_codex_discovered_table_of_another_region_raises(tmp_path: Path) -> None codex(tmp_path, image_models_kwargs=_FAST) -def test_codex_cells_table_kwargs_apply_to_discovered_tables(tmp_path: Path) -> None: - tifffile.imwrite(tmp_path / "reg001_montage.tif", np.zeros((2, 16, 16), dtype=np.uint16)) - (tmp_path / "reg001_cells.csv").write_text("cell_id,x,y,CD3,CD4\n1,1.0,2.0,3.0,4.0\n2,3.0,4.0,5.0,6.0\n") - - with pytest.warns(UserWarning, match="circles") as record: - sdata = codex(tmp_path, cells_table_kwargs={"markers": ["CD3"]}, image_models_kwargs=_FAST) - - assert list(sdata.tables["table"].var_names) == ["CD3"] - assert not [warning for warning in record if "cycCCC_chHHH" in str(warning.message)] - - -def test_codex_region_column_of_discovered_regionless_table(tmp_path: Path) -> None: - for region in ("reg001", "reg002"): - tifffile.imwrite(tmp_path / f"{region}_montage.tif", np.zeros((2, 16, 16), dtype=np.uint16)) - (tmp_path / "cells.csv").write_text("cell_id,Sample,x,y,cyc001_ch001\n1,reg001,1.0,2.0,3.0\n1,reg002,3.0,4.0,5.0\n") - - with pytest.raises(ValueError, match=r"cannot be assigned to regions.*cells_table_kwargs\['region_column'\]"): - codex(tmp_path, image_models_kwargs=_FAST) - - with pytest.warns(UserWarning, match="circles"): - sdata = codex(tmp_path, cells_table_kwargs={"region_column": "Sample"}, image_models_kwargs=_FAST) - assert sorted(sdata.shapes) == ["reg001_cells", "reg002_cells"] - assert sdata.tables["table"].n_obs == 2 - - def test_codex_labels_disable_mask_discovery(tmp_path: Path) -> None: for region in ("reg001", "reg002"): tifffile.imwrite(tmp_path / f"{region}_montage.tif", np.zeros((2, 16, 16), dtype=np.uint16)) diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 8783977c..5c7e77e6 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -8,7 +8,6 @@ import dask.array as da import numpy as np -import pandas as pd import pytest import tifffile from click.testing import CliRunner @@ -16,7 +15,7 @@ from spatialdata.models import get_channel_names from spatialdata.transformations import Identity, Scale, Sequence, Translation, get_transformation -from spatialdata_io import codex, phenocycler +from spatialdata_io import phenocycler from spatialdata_io.__main__ import phenocycler_wrapper from tests._akoya_data import forbid_pixel_reads, write_qptiff @@ -117,22 +116,14 @@ def test_read_directory_multiple_scans_and_subset(tmp_path: Path) -> None: def test_phenocycler_unrelated_scans_own_coordinate_systems(tmp_path: Path) -> None: for region, slide in (("A", "SLIDE1"), ("B", "SLIDE2")): write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI", "CD3"], slide_id=slide) - _write_mcquant(tmp_path / f"{region}.csv") _write_mask(tmp_path / "A_mask.tif") - with pytest.warns(UserWarning, match="circles"): - sdata = phenocycler( - tmp_path, - cells_table={"A": tmp_path / "A.csv", "B": tmp_path / "B.csv"}, - labels={"A": tmp_path / "A_mask.tif"}, - image_models_kwargs=_FAST, - ) + sdata = phenocycler(tmp_path, labels={"A": tmp_path / "A_mask.tif"}, image_models_kwargs=_FAST) assert sorted(sdata.coordinate_systems) == ["A", "B"] for name in ("A_image", "A_labels"): assert get_transformation(sdata[name], get_all=True) == {"A": Identity()} - for name in ("B_image", "B_cells"): - assert get_transformation(sdata[name], get_all=True) == {"B": Identity()} + assert get_transformation(sdata["B_image"], get_all=True) == {"B": Identity()} def test_phenocycler_same_slide_scans_placed_in_global(tmp_path: Path) -> None: @@ -217,7 +208,7 @@ def test_read_all_bit_depths_shares_stage_placement(tmp_path: Path) -> None: assert get_transformation(sdata[f"{region}_8bit_image"], get_all=True) == expected -def test_read_all_bit_depths_with_labels_and_cells_table(tmp_path: Path) -> None: +def test_read_all_bit_depths_with_labels(tmp_path: Path) -> None: for suffix, dtype in ((".qptiff", "uint8"), (".er.qptiff", "uint16")): write_qptiff( tmp_path / f"S{suffix}", biomarkers=["DAPI"], shape=(64, 80), dtype=dtype, stage_position_cm=(1, 2) @@ -225,11 +216,9 @@ def test_read_all_bit_depths_with_labels_and_cells_table(tmp_path: Path) -> None mask = np.zeros((32, 40), dtype=np.uint32) mask[2:4, 2:4], mask[5:7, 5:7] = 1, 2 tifffile.imwrite(tmp_path / "mask.tif", mask) - (tmp_path / "cells.csv").write_text("CellID,CD3,X_centroid,Y_centroid,Area\n1,1,3,3,4\n2,2,6,6,4\n") sdata = phenocycler( tmp_path, - cells_table=tmp_path / "cells.csv", labels=tmp_path / "mask.tif", read_all_bit_depths=True, image_models_kwargs=_FAST, @@ -242,10 +231,6 @@ def test_read_all_bit_depths_with_labels_and_cells_table(tmp_path: Path) -> None labels_transformation = get_transformation(sdata["S_labels"], "global") assert isinstance(labels_transformation, Sequence) assert labels_transformation.transformations == [Scale([2, 2], axes=("y", "x")), image_transformations["global"]] - table = sdata.tables["table"] - assert table.uns["spatialdata_attrs"]["region"] == ["S_labels"] - # the centre of the 2 x 2 image pixels of each mask pixel - np.testing.assert_allclose(table.obsm["spatial"], [[6.5, 6.5], [12.5, 12.5]]) with pytest.raises(ValueError, match=r"Unknown region\(s\) \['S_8bit'\]"): phenocycler(tmp_path, labels={"S_8bit": tmp_path / "mask.tif"}, read_all_bit_depths=True) @@ -467,20 +452,6 @@ def test_channel_names_override(tmp_path: Path) -> None: phenocycler(path, channel_names=["only one"]) -def _write_mcquant(path: Path, n: int = 5, markers: tuple[str, ...] = ("DAPI", "CD3")) -> None: - rng = np.random.default_rng(0) - cells = pd.DataFrame( - { - "CellID": np.arange(1, n + 1), - **{marker: rng.uniform(0, 100, n) for marker in markers}, - "X_centroid": rng.uniform(0, 300, n), - "Y_centroid": rng.uniform(0, 250, n), - "Area": rng.uniform(10, 50, n), - } - ) - cells.to_csv(path, index=False) - - def _write_mask(path: Path, n: int = 5, shape: tuple[int, int] = (256, 320)) -> np.ndarray: mask = np.zeros(shape, dtype=np.uint32) for i in range(n): @@ -489,34 +460,6 @@ def _write_mask(path: Path, n: int = 5, shape: tuple[int, int] = (256, 320)) -> return mask -def test_mcquant_with_mask_labels(tmp_path: Path) -> None: - path = tmp_path / "S_Scan1.qptiff" - write_qptiff(path, biomarkers=["DAPI", "CD3"], stage_position_cm=(1.0, 2.0)) - _write_mcquant(tmp_path / "cells.csv") - mask = _write_mask(tmp_path / "mask.tif") - - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - sdata = phenocycler( - path, - cells_table=tmp_path / "cells.csv", - labels=tmp_path / "mask.tif", - image_models_kwargs=_FAST, - labels_models_kwargs=_FAST, - ) - - assert list(sdata.labels) == ["S_labels"] - assert len(sdata.shapes) == 0 - np.testing.assert_array_equal(sdata.labels["S_labels"]["scale0"]["image"].values, mask) - table = sdata.tables["table"] - assert set(table.obs["region"]) == {"S_labels"} - assert table.uns["spatialdata_attrs"]["region"] == ["S_labels"] - assert list(table.obs["cell_id"]) == [1, 2, 3, 4, 5] - assert list(table.var_names) == ["DAPI", "CD3"] - assert sdata.attrs["akoya"]["cell_tables"] == {"S": str(tmp_path / "cells.csv")} - assert sdata.attrs["akoya"]["labels"] == {"S": str(tmp_path / "mask.tif")} - - def test_labels_without_cells_table(tmp_path: Path) -> None: path = tmp_path / "S_Scan1.qptiff" write_qptiff(path, biomarkers=["DAPI"]) @@ -528,111 +471,52 @@ def test_labels_without_cells_table(tmp_path: Path) -> None: assert len(sdata.tables) == 0 -def test_cells_table_without_mask_makes_circles(tmp_path: Path) -> None: - path = tmp_path / "S_Scan1.qptiff" - write_qptiff(path, biomarkers=["DAPI", "CD3"]) - _write_mcquant(tmp_path / "cells.csv") - - with pytest.warns(UserWarning, match="circles"): - sdata = phenocycler(path, cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) - - assert list(sdata.shapes) == ["S_cells"] - assert set(sdata.tables["table"].obs["region"]) == {"S_cells"} - - -def test_phenocycler_cells_mapping_per_region(tmp_path: Path) -> None: - for region in ("A", "B"): - write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) - _write_mcquant(tmp_path / f"{region}.csv") - _write_mask(tmp_path / f"{region}_mask.tif") - - sdata = phenocycler( - tmp_path, - cells_table={"A": tmp_path / "A.csv", "B": str(tmp_path / "B.csv")}, - labels={"A": tmp_path / "A_mask.tif", "B": tmp_path / "B_mask.tif"}, - image_models_kwargs=_FAST, - labels_models_kwargs=_FAST, - ) - - assert list(sdata.tables) == ["table"] - table = sdata.tables["table"] - assert table.n_obs == 10 - assert table.obs_names.is_unique - assert table.uns["spatialdata_attrs"]["region"] == ["A_labels", "B_labels"] - assert sorted(sdata.labels) == ["A_labels", "B_labels"] - - -def test_codex_format_table_uses_region_column(tmp_path: Path) -> None: - for region in ("A", "B"): - write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI"]) - (tmp_path / "cells.csv").write_text("cell_id,Sample,x,y,DAPI\n1,A,1.0,2.0,3.0\n2,B,1.0,2.0,3.0\n3,B,4,5,6\n") - - with pytest.warns(UserWarning, match="circles"): - sdata = phenocycler( - tmp_path, - regions=["A"], - cells_table=tmp_path / "cells.csv", - cells_table_kwargs={"region_column": "Sample"}, - image_models_kwargs=_FAST, - ) - - assert list(sdata.tables["table"].obs["cell_id"]) == [1] - assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["A_cells"] - - def test_mapping_with_unknown_region_raises(tmp_path: Path) -> None: write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI"]) - _write_mcquant(tmp_path / "A.csv", markers=("DAPI",)) + _write_mask(tmp_path / "A_mask.tif") with pytest.raises(ValueError, match="'Z'"): - phenocycler(tmp_path, cells_table={"Z": tmp_path / "A.csv"}) + phenocycler(tmp_path, labels={"Z": tmp_path / "A_mask.tif"}) def test_mapping_keys_of_unselected_regions_are_ignored(tmp_path: Path) -> None: for region in ("A", "B"): write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) - _write_mcquant(tmp_path / f"{region}.csv") _write_mask(tmp_path / f"{region}_mask.tif") sdata = phenocycler( tmp_path, regions=["A"], - cells_table={"A": tmp_path / "A.csv", "B": tmp_path / "B.csv"}, labels={"A": tmp_path / "A_mask.tif", "B": tmp_path / "B_mask.tif"}, image_models_kwargs=_FAST, labels_models_kwargs=_FAST, ) assert list(sdata.labels) == ["A_labels"] - assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["A_labels"] - assert sdata.attrs["akoya"]["cell_tables"] == {"A": str(tmp_path / "A.csv")} + assert sdata.attrs["akoya"]["labels"] == {"A": str(tmp_path / "A_mask.tif")} def test_raw_region_names_select_sanitised_regions(tmp_path: Path) -> None: write_qptiff(tmp_path / "Slide 1.qptiff", biomarkers=["DAPI", "CD3"]) write_qptiff(tmp_path / "Slide 2.qptiff", biomarkers=["DAPI", "CD3"]) - _write_mcquant(tmp_path / "cells.csv") + _write_mask(tmp_path / "mask.tif") - with pytest.warns(UserWarning, match="circles"): - sdata = phenocycler( - tmp_path, - regions=["Slide 1"], - cells_table={"Slide 1": tmp_path / "cells.csv"}, - image_models_kwargs=_FAST, - ) + sdata = phenocycler( + tmp_path, regions=["Slide 1"], labels={"Slide 1": tmp_path / "mask.tif"}, image_models_kwargs=_FAST + ) assert list(sdata.images) == ["Slide_1_image"] - assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["Slide_1_cells"] + assert list(sdata.labels) == ["Slide_1_labels"] def test_mapping_keys_colliding_after_sanitising_raise(tmp_path: Path) -> None: write_qptiff(tmp_path / "Slide_1.qptiff", biomarkers=["DAPI", "CD3"]) - _write_mcquant(tmp_path / "cells.csv") + _write_mask(tmp_path / "mask.tif") with pytest.raises(ValueError, match="same region"): phenocycler( tmp_path, - cells_table={"Slide 1": tmp_path / "cells.csv", "Slide_1": tmp_path / "cells.csv"}, + labels={"Slide 1": tmp_path / "mask.tif", "Slide_1": tmp_path / "mask.tif"}, image_models_kwargs=_FAST, ) @@ -647,155 +531,6 @@ def test_mask_shape_mismatch_warns(tmp_path: Path) -> None: assert list(sdata.labels) == ["S_labels"] -def test_mask_with_row_number_instance_ids_warns(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - _write_mask(tmp_path / "mask.tif") - (tmp_path / "cells.csv").write_text("cx,cy,DAPI\n1,2,3\n4,5,6\n") - - with pytest.warns(UserWarning, match="row numbers"): - phenocycler( - tmp_path, - cells_table=tmp_path / "cells.csv", - cells_table_format="generic", - labels=tmp_path / "mask.tif", - image_models_kwargs=_FAST, - cells_table_kwargs={"x": "cx", "y": "cy"}, - ) - - -@pytest.mark.parametrize( - ("header", "kwargs"), - [ - ("Cell ID,X,Y,CD3", {"x": "X", "y": "Y", "instance_id": "Cell ID"}), - ("Region,X,Y,CD3", {"x": "X", "y": "Y", "region_column": "Region"}), - ("CELL_ID,region,X,Y,CD3", {"x": "X", "y": "Y", "region_column": "region", "markers": ["CD3"]}), - ], -) -def test_generic_columns_clashing_with_reserved_keys(tmp_path: Path, header: str, kwargs: dict[str, str]) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - rows = "\n".join( - ",".join("S" if col.casefold() == "region" else str(i + 1) for col in header.split(",")) for i in range(2) - ) - (tmp_path / "cells.csv").write_text(f"{header}\n{rows}\n") - - with pytest.warns(UserWarning, match="circles"): - sdata = phenocycler( - tmp_path, - cells_table=tmp_path / "cells.csv", - cells_table_format="generic", - image_models_kwargs=_FAST, - cells_table_kwargs=kwargs, - ) - - table = sdata.tables["table"] - assert list(table.obs["region"]) == ["S_cells", "S_cells"] - assert table.obs["cell_id"].dtype == np.int64 - sdata.write(tmp_path / "x.zarr") - assert read_zarr(tmp_path / "x.zarr").tables["table"].n_obs == 2 - - -@pytest.mark.parametrize( - ("header", "row", "fmt", "kwargs"), - [ - ( - "CELL,X,Y,CD3,region,cell_id", - "{i},1,2,3,other,{source}", - "generic", - {"x": "X", "y": "Y", "instance_id": "CELL"}, - ), - ("CellID,DAPI,CD3,X_centroid,Y_centroid,Area,region,cell_id", "{i},1,2,3,4,5,other,{source}", "mcquant", {}), - ], -) -def test_extra_region_and_cell_id_columns_are_kept( - tmp_path: Path, header: str, row: str, fmt: str, kwargs: dict[str, object] -) -> None: - # columns named like the region and instance keys, but not read as them, keep their values as `source_` - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - rows = "\n".join(row.format(i=i, source=100 + i) for i in (1, 2)) - (tmp_path / "cells.csv").write_text(f"{header}\n{rows}\n") - if fmt == "generic": - kwargs = {**kwargs, "markers": ["CD3"]} - - with pytest.warns(UserWarning, match="circles"): - sdata = phenocycler( - tmp_path, - cells_table=tmp_path / "cells.csv", - cells_table_format=fmt, - image_models_kwargs=_FAST, - cells_table_kwargs=kwargs, - ) - - obs = sdata.tables["table"].obs - assert list(obs["region"]) == ["S_cells", "S_cells"] - assert list(obs["cell_id"]) == [1, 2] - assert list(obs["source_region"]) == ["other", "other"] - assert list(obs["source_cell_id"]) == [101, 102] - - -def test_duplicate_instance_ids_raise(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - (tmp_path / "cells.csv").write_text("CellID,CD3,X_centroid,Y_centroid\n1,1,1,1\n2,1,2,2\n2,1,3,3\n") - - with pytest.raises(ValueError, match=r"cells\.csv has 1 duplicate instance id.*\[2\].*region 'S'"): - phenocycler(tmp_path, cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) - - -def test_qupath_without_markers_warns(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - (tmp_path / "cells.tsv").write_text("Object ID\tCentroid X px\tCentroid Y px\nu1\t1\t2\n", encoding="utf-8") - - with pytest.warns(UserWarning, match="no marker intensities"): - sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) - - assert sdata.tables["table"].n_vars == 0 - - -def test_single_table_many_regions_needs_region_column(tmp_path: Path) -> None: - for region in ("A", "B"): - write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) - _write_mcquant(tmp_path / "cells.csv") - - with pytest.raises(ValueError, match="needs a region column"): - phenocycler(tmp_path, cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) - - -def test_single_qupath_table_many_regions(tmp_path: Path) -> None: - for region in ("A", "B"): - write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) - header = "Image\tObject ID\tCentroid X µm\tCentroid Y µm\tCD3: Cell: Mean" - rows = ["A_Scan1.qptiff\tu1\t5\t5\t1", "B_Scan1.qptiff\tu2\t10\t10\t2", "C_Scan1.qptiff\tu3\t1\t1\t3"] - (tmp_path / "cells.tsv").write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") - - with pytest.warns(UserWarning, match="C_Scan1.qptiff"): - sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) - - table = sdata.tables["table"] - assert list(table.obs["region"]) == ["A_cells", "B_cells"] - np.testing.assert_allclose(np.asarray(table.obsm["spatial"]), np.array([[5, 5], [10, 10]]) / 0.5069) - # QuPath column names are not valid spatialdata names, so they are sanitized and the result can be written - assert {"Image", "Object_ID", "Centroid_X_µm"} <= set(table.obs.columns) - sdata.write(tmp_path / "x.zarr") - assert read_zarr(tmp_path / "x.zarr").tables["table"].n_obs == 2 - - -@pytest.mark.parametrize("unit", ["px", "µm"]) -def test_single_qupath_table_matches_raw_names_of_sanitised_regions(tmp_path: Path, unit: str) -> None: - # the region ids are sanitised (`A_[1,2]` -> `A__1_2_`), the table's `Image` column holds the raw file names - for name in ("A_[1,2]", "B [3,4]"): - write_qptiff(tmp_path / f"{name}.qptiff", biomarkers=["DAPI"]) - header = f"Image\tObject ID\tCentroid X {unit}\tCentroid Y {unit}\tCD3: Cell: Mean" - rows = ["A_[1,2].qptiff\tu1\t5\t5\t1", "B [3,4].qptiff\tu2\t10\t10\t2"] - (tmp_path / "cells.tsv").write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") - - with pytest.warns(UserWarning, match="circles"): - sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) - - table = sdata.tables["table"] - assert list(table.obs["region"]) == ["A__1_2__cells", "B__3_4__cells"] - scale = 1 / 0.5069 if unit == "µm" else 1 - np.testing.assert_allclose(np.asarray(table.obsm["spatial"]), np.array([[5, 5], [10, 10]]) * scale) - - def test_regions_colliding_after_sanitising_raise(tmp_path: Path) -> None: write_qptiff(tmp_path / "A [1].qptiff", biomarkers=["DAPI"]) write_qptiff(tmp_path / "A_[1].qptiff", biomarkers=["DAPI"]) @@ -804,43 +539,6 @@ def test_regions_colliding_after_sanitising_raise(tmp_path: Path) -> None: phenocycler(tmp_path, image_models_kwargs=_FAST) -def test_single_qupath_table_regions_with_different_pixel_sizes(tmp_path: Path) -> None: - # scans of the same instrument differ in the 4th decimal; each cell is converted with its own scan's pixel size - sizes = {"A": 0.5069, "B": 0.5079} - for region, size in sizes.items(): - write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI", "CD3"], resolution_px_per_cm=1e4 / size) - header = "Image\tObject ID\tCentroid X µm\tCentroid Y µm\tCell: Area µm^2\tCD3: Cell: Mean" - rows = ["A_Scan1.qptiff\tu1\t5\t6\t3.14159\t1", "B_Scan1.qptiff\tu2\t10\t20\t3.14159\t2"] - (tmp_path / "cells.tsv").write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") - - with pytest.warns(UserWarning, match="circles"): - sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) - - table = sdata.tables["table"] - expected = np.array([[5 / sizes["A"], 6 / sizes["A"]], [10 / sizes["B"], 20 / sizes["B"]]]) - np.testing.assert_allclose(np.asarray(table.obsm["spatial"]), expected) - for region, (x, y) in zip(sizes, expected, strict=True): - circles = sdata.shapes[f"{region}_cells"] - np.testing.assert_allclose([circles.geometry.x.iloc[0], circles.geometry.y.iloc[0]], [x, y]) - np.testing.assert_allclose(circles["radius"].iloc[0], 1 / sizes[region], rtol=1e-5) - - -def test_different_panels_make_per_region_tables(tmp_path: Path) -> None: - write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) - write_qptiff(tmp_path / "B_Scan1.qptiff", biomarkers=["DAPI", "CD8", "CD4"]) - _write_mcquant(tmp_path / "A.csv", markers=("DAPI", "CD3")) - _write_mcquant(tmp_path / "B.csv", markers=("DAPI", "CD8", "CD4")) - - with pytest.warns(UserWarning, match="circles"): - sdata = phenocycler( - tmp_path, cells_table={"A": tmp_path / "A.csv", "B": tmp_path / "B.csv"}, image_models_kwargs=_FAST - ) - - assert sorted(sdata.tables) == ["A_table", "B_table"] - assert list(sdata.tables["B_table"].var_names) == ["DAPI", "CD8", "CD4"] - assert sdata.tables["A_table"].uns["spatialdata_attrs"]["region"] == ["A_cells"] - - def test_single_mask_many_regions_raises(tmp_path: Path) -> None: for region in ("A", "B"): write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI"]) @@ -850,9 +548,8 @@ def test_single_mask_many_regions_raises(tmp_path: Path) -> None: phenocycler(tmp_path, labels=tmp_path / "mask.tif") -def test_cli_cells_table_and_labels_mappings(runner: CliRunner, tmp_path: Path) -> None: +def test_cli_labels_mapping(runner: CliRunner, tmp_path: Path) -> None: write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) - _write_mcquant(tmp_path / "cells.csv") _write_mask(tmp_path / "mask.tif") output = tmp_path / "out.zarr" @@ -863,8 +560,6 @@ def test_cli_cells_table_and_labels_mappings(runner: CliRunner, tmp_path: Path) str(tmp_path / "S_Scan1.qptiff"), "--output", str(output), - "--cells-table", - str(tmp_path / "cells.csv"), "--labels", json.dumps({"S": str(tmp_path / "mask.tif")}), "--image-models-kwargs", @@ -877,7 +572,6 @@ def test_cli_cells_table_and_labels_mappings(runner: CliRunner, tmp_path: Path) assert result.exit_code == 0, result.output sdata = read_zarr(output) assert list(sdata.labels) == ["S_labels"] - assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["S_labels"] @pytest.mark.parametrize("value", ['{"S": 1}', "{not json"]) @@ -885,11 +579,11 @@ def test_cli_bad_mapping_is_bad_parameter(runner: CliRunner, tmp_path: Path, val write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) result = runner.invoke( - phenocycler_wrapper, ["--input", str(tmp_path), "--output", str(tmp_path / "o.zarr"), "--cells-table", value] + phenocycler_wrapper, ["--input", str(tmp_path), "--output", str(tmp_path / "o.zarr"), "--labels", value] ) assert result.exit_code == 2 - assert "cells_table" in result.output + assert "labels" in result.output def test_cli_regions_and_channel_names_may_hold_commas(runner: CliRunner, tmp_path: Path) -> None: @@ -911,7 +605,7 @@ def test_cli_regions_and_channel_names_may_hold_commas(runner: CliRunner, tmp_pa assert sorted(read_zarr(tmp_path / "b.zarr").images) == ["A__1_2__image", "B_image"] -@pytest.mark.parametrize("option", ["--image-models-kwargs", "--cells-table-kwargs"]) +@pytest.mark.parametrize("option", ["--image-models-kwargs", "--labels-models-kwargs"]) def test_cli_kwargs_must_be_json_objects(runner: CliRunner, tmp_path: Path, option: str) -> None: write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) @@ -960,134 +654,3 @@ def test_phenocycler_keyword_only(tmp_path: Path) -> None: write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI"]) with pytest.raises(TypeError, match="positional argument"): phenocycler(tmp_path, ["A"]) # type: ignore[misc] - - -def _write_qupath_px(path: Path, images: dict[str, int]) -> None: - rows = ["Image,Object ID,Centroid X px,Centroid Y px,CD3: Cell: Mean"] - rows += [f"{image},{i},{i},{i},1" for image, n in images.items() for i in range(1, n + 1)] - path.write_text("\n".join(rows) + "\n", encoding="utf-8") - - -def test_region_values_match_image_names_not_prefixes(tmp_path: Path) -> None: - write_qptiff(tmp_path / "Tonsil_Scan1.qptiff", biomarkers=["DAPI"]) - images = {"Tonsil_Scan1.qptiff - resolution #1": 3, "Tonsil_2_Scan1.qptiff": 2, "Spleen_Scan1.qptiff": 2} - _write_qupath_px(tmp_path / "cells.csv", images) - - with pytest.warns(UserWarning, match=r"Ignoring the cells of \['Spleen_Scan1.qptiff', 'Tonsil_2_Scan1.qptiff'\]"): - sdata = phenocycler( - tmp_path / "Tonsil_Scan1.qptiff", cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST - ) - - assert list(sdata.tables["table"].obs["cell_id"]) == [1, 2, 3] - - -@pytest.mark.parametrize("regions", [None, ["A"]]) -def test_region_values_naming_no_image_are_not_assigned_to_the_selected_one( - tmp_path: Path, regions: list[str] | None -) -> None: - for region in ("A", "B"): - write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI"]) - _write_qupath_px(tmp_path / "cells.csv", {"Patient 7 tonsil": 2, "Patient 9 lymph node": 2}) - - with pytest.warns(UserWarning, match="Ignoring the cells"), pytest.raises(ValueError, match="has no cells of"): - phenocycler(tmp_path, regions=regions, cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) - - -def test_duplicate_instance_ids_are_checked_per_assigned_region(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - # two names of the same image - _write_qupath_px(tmp_path / "cells.csv", {"S": 1, "S_Scan1.qptiff": 1}) - - with pytest.raises(ValueError, match=r"cells\.csv has 1 duplicate instance id.*region 'S'"): - phenocycler(tmp_path, cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) - - -def test_codex_region_values_match_qptiff_file_names(tmp_path: Path) -> None: - for region in ("reg001", "reg002"): - write_qptiff(tmp_path / f"exp_{region}.qptiff", biomarkers=["DAPI"], software="Codex Instrument Manager 1.30") - _write_qupath_px(tmp_path / "cells.csv", {"exp_reg001.qptiff": 1, "exp_reg002.qptiff": 2}) - - with pytest.warns(UserWarning, match="circles"): - sdata = codex(tmp_path, regions=["reg001"], cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) - - assert list(sdata.tables["table"].obs["cell_id"]) == [1] - - -@pytest.mark.parametrize("regions", [["A"], ["B"]]) -def test_selected_region_of_qupath_table_uses_each_rows_pixel_size(tmp_path: Path, regions: list[str]) -> None: - rows, features = ["Image,Object ID,Centroid X µm,Centroid Y µm,CD3: Cell: Mean"], [] - for region, pixel_size in (("A", 0.5), ("B", 0.25)): - write_qptiff( - tmp_path / f"{region}_Scan1.qptiff", - biomarkers=["DAPI"], - resolution_px_per_cm=1e4 / pixel_size, - slide_id=region, - ) - for i in range(4): - x, y = 20 + 25 * i, 30 + 15 * i - rows.append(f"{region}_Scan1.qptiff,{region}{i},{x * pixel_size},{y * pixel_size},{i}") - ring = [[x - 3, y - 3], [x + 3, y - 3], [x + 3, y + 3], [x - 3, y + 3], [x - 3, y - 3]] - features.append( - {"type": "Feature", "id": f"{region}{i}", "geometry": {"type": "Polygon", "coordinates": [ring]}} - ) - (tmp_path / "cells.csv").write_text("\n".join(rows) + "\n", encoding="utf-8") - (tmp_path / "cells.geojson").write_text(json.dumps({"type": "FeatureCollection", "features": features})) - - sdata = phenocycler( - tmp_path, - regions=regions, - cells_table=tmp_path / "cells.csv", - image_models_kwargs=_FAST, - labels_models_kwargs=_FAST, - ) - - assert list(sdata.labels) == [f"{regions[0]}_labels"] - np.testing.assert_allclose(np.asarray(sdata.tables["table"].obsm["spatial"])[:, 0], [20, 45, 70, 95]) - - -def test_cells_without_region_are_ignored(tmp_path: Path) -> None: - for region in ("A", "B"): - write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI"]) - _write_qupath_px(tmp_path / "cells.csv", {"A_Scan1.qptiff": 1, "": 1, "B_Scan1.qptiff": 1}) - - with pytest.warns(UserWarning, match=r"Ignoring the 1 cell\(s\) of .*cells\.csv without a region in 'Image'"): - sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) - - assert list(sdata.tables["table"].obs["region"]) == ["A_cells", "B_cells"] - - -def test_tables_with_different_marker_metadata_are_not_merged(tmp_path: Path) -> None: - for region in ("A", "B"): - write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI"]) - header = "Object ID\tCentroid X px\tCentroid Y px\tCD3: Nucleus: Mean" - (tmp_path / "a.tsv").write_text(f"{header}\tCD3: Cell: Mean\na1\t10\t10\t1\t100\n", encoding="utf-8") - (tmp_path / "b.tsv").write_text(f"{header}\nb1\t10\t10\t2\n", encoding="utf-8") - - with pytest.warns(UserWarning, match="circles"): - sdata = phenocycler( - tmp_path, cells_table={"A": tmp_path / "a.tsv", "B": tmp_path / "b.tsv"}, image_models_kwargs=_FAST - ) - - assert sorted(sdata.tables) == ["A_table", "B_table"] - assert list(sdata.tables["A_table"].var["compartment"]) == ["Cell"] - assert list(sdata.tables["B_table"].var["compartment"]) == ["Nucleus"] - - -def test_cells_without_a_positive_area_get_unit_circles(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - (tmp_path / "cells.csv").write_text("CellID,CD3,X_centroid,Y_centroid,Area\n1,1,1,1,0\n2,1,2,2,\n3,1,3,3,3.14159\n") - - with pytest.warns(UserWarning, match="circles"): - sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.csv", image_models_kwargs=_FAST) - - np.testing.assert_allclose(sdata.shapes["S_cells"]["radius"], [1, 1, 1], rtol=1e-5) - - -@pytest.mark.parametrize("reader", [codex, phenocycler]) -def test_cells_table_options_are_checked_without_a_cells_table(tmp_path: Path, reader) -> None: # type: ignore[no-untyped-def] - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - - with pytest.raises(ValueError, match=r"unknown key\(s\) \['mask_chanel'\]"): - reader(tmp_path, cells_table_kwargs={"mask_chanel": 1}) - with pytest.raises(ValueError, match="Unknown cell table format 'excel'"): - reader(tmp_path, cells_table_format="excel") From 06115799c9089dc5af5c1079e98d239002e122f4 Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 21:05:03 +0200 Subject: [PATCH 105/107] refactor(akoya): drop GeoJSON rasterisation --- .../readers/_utils/_rasterize.py | 143 ----- tests/readers/test_rasterize.py | 497 ------------------ 2 files changed, 640 deletions(-) delete mode 100644 src/spatialdata_io/readers/_utils/_rasterize.py delete mode 100644 tests/readers/test_rasterize.py diff --git a/src/spatialdata_io/readers/_utils/_rasterize.py b/src/spatialdata_io/readers/_utils/_rasterize.py deleted file mode 100644 index 6432ea31..00000000 --- a/src/spatialdata_io/readers/_utils/_rasterize.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Lazy, chunk-wise rasterisation of cell polygons into a labels array.""" - -from __future__ import annotations - -import hashlib -from itertools import product -from typing import TYPE_CHECKING, Any - -import dask.array as da -import numpy as np -import shapely -from dask.array.core import normalize_chunks -from dask.base import tokenize - -if TYPE_CHECKING: - from collections.abc import Sequence - -__all__ = ["rasterize_polygons_lazy"] - -_MAX_ID = np.iinfo(np.uint32).max -_POLYGONAL = (shapely.GeometryType.POLYGON, shapely.GeometryType.MULTIPOLYGON) -# the polygon pixels tested at once (at least one row of a polygon), bounding the memory of a block's temporaries -# (~25 MB) -_BATCH_PIXELS = 1 << 18 -_TOKEN_BATCH = 1 << 16 - - -def rasterize_polygons_lazy( - polygons: Sequence[shapely.Geometry], - ids: Sequence[int], - shape: tuple[int, int], - chunks: tuple[int, int] = (4096, 4096), -) -> da.Array: - """Rasterise polygons into a lazy ``uint32`` labels array, one chunk at a time. - - A pixel ``(row, col)`` gets the id of a polygon that contains its centre ``(col + 0.5, row + 0.5)`` or has it on - its boundary: pixel ``(0, 0)`` covers ``[0, 1) x [0, 1)``, as in QuPath. Holes are not part of a polygon. Where - polygons overlap or touch, the one with the higher id wins, independently of their order, so cells that share an - edge leave no gap. Only the polygons that may - intersect each chunk are found here; a chunk draws them when it is computed. - - Parameters - ---------- - polygons - The polygons, in pixels; of other geometries (e.g. a ``GeometryCollection``) only the polygons are drawn. - ids - The label of each polygon, between 1 and ``2**32 - 1``. - shape - The ``(y, x)`` shape of the labels. - chunks - The ``(y, x)`` chunk size. - - Returns - ------- - The ``(y, x)`` labels, 0 for the background. - """ - if len(polygons) != len(ids): - raise ValueError(f"Got {len(polygons)} polygons but {len(ids)} ids.") - labels = np.asarray(ids, dtype=np.int64) - if len(labels) and (labels.min() < 1 or labels.max() > _MAX_ID): - raise ValueError(f"Polygon ids must be between 1 and {_MAX_ID}; got {labels.min()} to {labels.max()}.") - geometries = _polygonal(np.array(polygons, dtype=object)) - labels = labels.astype(np.uint32) - sizes = normalize_chunks(chunks, shape, dtype=np.uint32) - name = f"rasterize-polygons-{_token(geometries, labels, sizes)}" - # every task holds just the polygons that may intersect its chunk, so the polygons are shipped once in all - starts = [np.cumsum((0, *axis[:-1])) for axis in sizes] - blocks = list(product(*(range(len(axis)) for axis in sizes))) - boxes = [ - shapely.box(starts[1][j], starts[0][i], starts[1][j] + sizes[1][j], starts[0][i] + sizes[0][i]) - for i, j in blocks - ] - block_of, polygon = shapely.STRtree(geometries).query(boxes) - counts = np.bincount(block_of, minlength=len(blocks)) - by_block = np.split(polygon[np.argsort(block_of, kind="stable")], np.cumsum(counts)[:-1]) - graph = { - (name, i, j): ( - _draw_block, - geometries[rows], - labels[rows], - (int(starts[0][i]), int(starts[1][j])), - (sizes[0][i], sizes[1][j]), - ) - for (i, j), rows in zip(blocks, by_block, strict=True) - } - return da.Array(graph, name, sizes, meta=np.empty((0, 0), dtype=np.uint32)) - - -def _polygonal(geometries: np.ndarray) -> np.ndarray: - """``geometries`` with only the polygons of collections; lines and points cover no pixels.""" - for i in np.flatnonzero(~np.isin(shapely.get_type_id(geometries), _POLYGONAL)): - parts = shapely.get_parts(shapely.get_parts(geometries[i])) - geometries[i] = shapely.multipolygons(parts[shapely.get_type_id(parts) == shapely.GeometryType.POLYGON]) - return geometries - - -def _token(geometries: np.ndarray, labels: np.ndarray, *args: Any) -> str: - """A deterministic token of the polygons, hashing their WKB in batches to bound the memory.""" - wkb = hashlib.blake2b(digest_size=16) - for start in range(0, len(geometries), _TOKEN_BATCH): - wkb.update(b"".join(shapely.to_wkb(geometries[start : start + _TOKEN_BATCH]))) - return tokenize(wkb.hexdigest(), labels, *args) - - -def _draw_block( - geometries: np.ndarray, labels: np.ndarray, origin: tuple[int, int], shape: tuple[int, int] -) -> np.ndarray: - """The labels of the block of ``shape`` at ``origin``, the highest id of the polygons covering each pixel centre.""" - out = np.zeros(shape, dtype=np.uint32) - if not len(geometries): - return out - # private copies in block coordinates, prepared without racing other threads over the shared polygons - geometries = shapely.transform(geometries, lambda xy: xy - origin[::-1]) - shapely.prepare(geometries) - # the pixel window of each polygon in the block; pixel centres are at +0.5 - minx, miny, maxx, maxy = shapely.bounds(geometries).T - r0, r1 = np.clip([np.floor(miny - 0.5), np.ceil(maxy - 0.5) + 1], 0, shape[0]).astype(np.int64) - c0, c1 = np.clip([np.floor(minx - 0.5), np.ceil(maxx - 0.5) + 1], 0, shape[1]).astype(np.int64) - # a polygon larger than a batch is tested in bands of rows - width = np.maximum(c1 - c0, 0) - band = np.maximum(_BATCH_PIXELS // np.maximum(width, 1), 1) - n_bands = np.maximum(-(-(r1 - r0) // band), 1) - of = np.repeat(np.arange(len(geometries)), n_bands) - nth = np.arange(len(of)) - np.repeat(np.cumsum(n_bands) - n_bands, n_bands) - r0 = r0[of] + nth * band[of] - r1, c0, width = np.minimum(r0 + band[of], r1[of]), c0[of], width[of] - geometries, labels = geometries[of], labels[of] - sizes = np.maximum(r1 - r0, 0) * width - ends = np.cumsum(sizes) - firsts = ends - sizes - flat = out.reshape(-1) - # all candidate pixels of a batch of polygons are tested in one GEOS call, which releases the GIL - start = 0 - while start < len(geometries): - stop = max(int(np.searchsorted(ends, firsts[start] + _BATCH_PIXELS, side="right")), start + 1) - owner = np.repeat(np.arange(start, stop), sizes[start:stop]) - offset = np.arange(len(owner)) - (firsts[owner] - firsts[start]) - rows = r0[owner] + offset // width[owner] - cols = c0[owner] + offset % width[owner] - inside = shapely.intersects_xy(geometries[owner], cols + 0.5, rows + 0.5) - np.maximum.at(flat, rows[inside] * shape[1] + cols[inside], labels[owner[inside]]) - start = stop - return out diff --git a/tests/readers/test_rasterize.py b/tests/readers/test_rasterize.py deleted file mode 100644 index f7c5205b..00000000 --- a/tests/readers/test_rasterize.py +++ /dev/null @@ -1,497 +0,0 @@ -from __future__ import annotations - -import json -import pickle -import warnings -from typing import TYPE_CHECKING, Any - -import dask.array as da -import geopandas as gpd -import numpy as np -import pytest -import shapely -import tifffile -from skimage.draw import polygon as draw_polygon -from spatialdata import read_zarr -from spatialdata.transformations import get_transformation -from tests._akoya_data import write_qptiff - -from spatialdata_io import phenocycler -from spatialdata_io.readers._utils import _rasterize -from spatialdata_io.readers._utils._cell_tables import _read_geojson -from spatialdata_io.readers._utils._rasterize import rasterize_polygons_lazy - -if TYPE_CHECKING: - from pathlib import Path - - -def _eager_reference(polygons: list[shapely.Polygon], ids: list[int], shape: tuple[int, int]) -> np.ndarray: - """Draw on the full canvas at once, in ascending id order; pixel (r, c) is covered when (c + .5, r + .5) is.""" - labels = np.zeros(shape, dtype=np.uint32) - for i in np.argsort(ids, kind="stable"): - inside = np.zeros(shape, dtype=bool) - rings = [(polygons[i].exterior, True), *((ring, False) for ring in polygons[i].interiors)] - for ring, fill in rings: - xy = shapely.get_coordinates(ring) - inside[draw_polygon(xy[:, 1] - 0.5, xy[:, 0] - 0.5, shape=shape)] = fill - labels[inside] = ids[i] - return labels - - -def test_rasterize_matches_eager_reference() -> None: - polygons = [ - shapely.Polygon([(2.3, 3.1), (9.7, 2.2), (8.4, 11.6), (1.2, 9.9)]), - # crosses the chunk borders at x = 16 and y = 16 - shapely.Polygon([(12.2, 10.4), (27.6, 13.3), (24.1, 30.8), (11.7, 25.2)]), - # has a hole - shapely.Polygon( - [(30.1, 20.2), (48.3, 21.4), (47.6, 38.7), (31.4, 37.9)], - holes=[[(35.2, 25.3), (42.4, 25.1), (41.7, 32.6), (36.1, 33.3)]], - ), - ] - ids = [3, 1, 2] - - labels = rasterize_polygons_lazy(polygons, ids, shape=(40, 50), chunks=(16, 16)) - - assert labels.dtype == np.uint32 - assert labels.chunks == ((16, 16, 8), (16, 16, 16, 2)) - result = labels.compute() - np.testing.assert_array_equal(result, _eager_reference(polygons, ids, (40, 50))) - assert set(np.unique(result)) == {0, 1, 2, 3} - assert result[29, 38] == 0 # in the hole - - -def test_rasterize_pixel_convention() -> None: - square = shapely.box(1, 2, 3, 5) # covers the pixels x 1..2, y 2..4 - - expected = np.zeros((8, 8), dtype=np.uint32) - expected[2:5, 1:3] = 7 - np.testing.assert_array_equal(rasterize_polygons_lazy([square], [7], shape=(8, 8)).compute(), expected) - # pixel centres on the shared diagonal go to the higher id - halves = [shapely.Polygon([(0, 0), (4, 0), (0, 4)]), shapely.Polygon([(4, 0), (4, 4), (0, 4)])] - labels = rasterize_polygons_lazy(halves, [1, 2], shape=(4, 4)).compute() - rows, cols = np.indices((4, 4)) - np.testing.assert_array_equal(labels, np.where(rows + cols >= 3, 2, 1)) - - -def test_rasterize_overlap_higher_id_wins() -> None: - left, right = shapely.box(0, 0, 6, 6), shapely.box(3, 3, 9, 9) - # a cell inside the hole of a higher-id ring keeps its pixels - ring = shapely.Polygon(shapely.box(10, 0, 20, 10).exterior, holes=[shapely.box(13, 3, 17, 7).exterior.coords]) - inner = shapely.box(14, 4, 16, 6) - - for order in ([0, 1, 2, 3], [3, 2, 1, 0]): - polygons = [[left, right, ring, inner][i] for i in order] - ids = [[5, 2, 9, 4][i] for i in order] - labels = rasterize_polygons_lazy(polygons, ids, shape=(10, 20), chunks=(4, 4)).compute() - assert labels[4, 4] == 5 - assert labels[1, 1] == 5 - assert labels[8, 8] == 2 - assert labels[5, 15] == 4 - assert labels[3, 13] == 0 - assert labels[1, 11] == 9 - - -def test_rasterize_multipolygon_and_empty() -> None: - multi = shapely.MultiPolygon([shapely.box(0, 0, 2, 2), shapely.box(5, 5, 7, 7)]) - - labels = rasterize_polygons_lazy([multi], [1], shape=(8, 8), chunks=(4, 4)).compute() - - assert labels[0, 0] == labels[6, 6] == 1 - assert labels.sum() == 8 - assert not rasterize_polygons_lazy([], [], shape=(5, 6)).compute().any() - - -def test_rasterize_bounds_the_pixels_of_a_large_polygon(monkeypatch: pytest.MonkeyPatch) -> None: - tested = [] - - def counting_intersects_xy(geometries: np.ndarray, x: np.ndarray, y: np.ndarray) -> np.ndarray: - tested.append(len(x)) - return intersects_xy(geometries, x, y) - - intersects_xy = shapely.intersects_xy - polygons = [shapely.Polygon([(-1.3, -2.2), (40.5, 3.1), (37.2, 41.7), (2.4, 30.3)]), shapely.box(3, 3, 9, 9)] - expected = rasterize_polygons_lazy(polygons, [1, 2], shape=(32, 32), chunks=(32, 32)).compute() - monkeypatch.setattr(_rasterize, "_BATCH_PIXELS", 100) - monkeypatch.setattr(_rasterize.shapely, "intersects_xy", counting_intersects_xy) - - # the first polygon covers the whole chunk: its pixels are tested in bands of rows - labels = rasterize_polygons_lazy(polygons, [1, 2], shape=(32, 32), chunks=(32, 32)).compute() - - np.testing.assert_array_equal(labels, expected) - assert max(tested) <= 100 - assert sum(tested) == 32 * 32 + 8 * 8 - - -def test_rasterize_draws_only_polygonal_parts() -> None: - collection = shapely.GeometryCollection( - [shapely.box(0, 0, 2, 2), shapely.LineString([(4, 0.5), (7, 0.5)]), shapely.Point(6.5, 6.5)] - ) - line = shapely.LineString([(0.5, 5), (0.5, 7)]) - - labels = rasterize_polygons_lazy([collection, line], [1, 2], shape=(8, 8), chunks=(4, 4)).compute() - - expected = np.zeros((8, 8), dtype=np.uint32) - expected[:2, :2] = 1 - np.testing.assert_array_equal(labels, expected) - - -def test_rasterize_errors() -> None: - with pytest.raises(ValueError, match="2 polygons but 1 ids"): - rasterize_polygons_lazy([shapely.box(0, 0, 1, 1)] * 2, [1], shape=(4, 4)) - with pytest.raises(ValueError, match="between 1 and"): - rasterize_polygons_lazy([shapely.box(0, 0, 1, 1)], [0], shape=(4, 4)) - - -def test_rasterize_is_lazy(monkeypatch: pytest.MonkeyPatch) -> None: - calls = [] - - def counting_draw_block(geometries: np.ndarray, *args: Any) -> np.ndarray: - calls.append(len(geometries)) - return draw_block(geometries, *args) - - draw_block = _rasterize._draw_block - monkeypatch.setattr(_rasterize, "_draw_block", counting_draw_block) - polygons = [shapely.box(i, i, i + 3, i + 3) for i in range(0, 60, 6)] - - labels = rasterize_polygons_lazy(polygons, list(range(1, 11)), shape=(64, 64), chunks=(16, 16)) - - assert not calls - labels.blocks[0, 0].compute() - # only the polygons of the chunk are drawn - assert calls == [3] - assert labels.compute().max() == 10 - - -def test_rasterize_tasks_hold_only_their_polygons() -> None: - polygons = [shapely.box(x, y, x + 3, y + 3) for x in range(0, 64, 8) for y in range(0, 64, 8)] - - labels = rasterize_polygons_lazy(polygons, list(range(1, 65)), shape=(64, 64), chunks=(16, 16)) - - # with dask.distributed, every task is shipped on its own - largest = max(len(pickle.dumps(task)) for task in dict(labels.__dask_graph__()).values()) - assert largest < len(pickle.dumps(polygons)) / 4 - - -def _write_qupath_cells(path: Path, object_ids: list[str]) -> None: - # centroids in the 8 px squares `_cell(..., 20 * i, 20 * i)` - header = "Image\tObject ID\tClass\tCentroid X px\tCentroid Y px\tCD3: Cell: Mean" - rows = [f"S_Scan1.qptiff\t{oid}\tTumor\t{4 + 20 * i}\t{4 + 20 * i}\t{i}" for i, oid in enumerate(object_ids)] - path.write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") - - -def _write_geojson(path: Path, features: list[dict], collection: bool = True) -> None: - path.write_text(json.dumps({"type": "FeatureCollection", "features": features} if collection else features)) - - -def _cell(object_id: str | None, x: float, y: float, object_type: str = "cell") -> dict: - feature = { - "type": "Feature", - "geometry": shapely.geometry.mapping(shapely.box(x, y, x + 8, y + 8)), - "properties": {"objectType": object_type, "classification": {"name": "Tumor"}}, - } - if object_id is not None: - feature["id"] = object_id - return feature - - -_FAST = {"scale_factors": [2]} - - -def test_qupath_geojson_to_labels(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"], stage_position_cm=(1.0, 2.0)) - uuids = ["5b1f-aa", "5b1f-bb", "5b1f-cc"] - _write_qupath_cells(tmp_path / "cells.tsv", uuids) - # in any order; the annotation has an id the table does not know - features = [_cell(uuids[2], 40, 40), _cell(uuids[0], 0, 0), _cell("tissue", 0, 0, "annotation")] - features.append(_cell(uuids[1], 20, 20)) - _write_geojson(tmp_path / "cells.geojson", features) - _write_geojson(tmp_path / "S-tissue-boundary.geojson", [_cell(None, 0, 0, "annotation")]) - - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) - - assert list(sdata.labels) == ["S_labels"] - assert list(sdata.shapes) == ["S_polygons"] - labels = sdata.labels["S_labels"]["scale0"]["image"] - assert isinstance(labels.data, da.Array) - assert labels.shape == (256, 320) - assert get_transformation(sdata.labels["S_labels"]) == get_transformation(sdata.images["S_image"]) - table = sdata.tables["table"] - assert table.uns["spatialdata_attrs"]["region"] == ["S_labels"] - # the UUIDs are not integers, so the instance ids are the row numbers; the mask values follow them - assert list(table.obs["cell_id"]) == [1, 2, 3] - assert list(table.obs["Object_ID"]) == uuids - values = labels.values - assert values[4, 4] == 1 - assert values[24, 24] == 2 - assert values[44, 44] == 3 - assert set(np.unique(values)) == {0, 1, 2, 3} - assert sdata.attrs["akoya"]["geojson"] == {"S": str(tmp_path / "cells.geojson")} - - -def test_qupath_geojson_keeps_polygons(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - _write_qupath_cells(tmp_path / "cells.tsv", ["a", "b", "c"]) - # cell "b" has no polygon: it is missing from the shapes and the labels - _write_geojson(tmp_path / "cells.geojson", [_cell("c", 40, 40), _cell("a", 0, 0)]) - - with pytest.warns(UserWarning, match="have no polygon"): - sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) - - assert list(sdata.shapes) == ["S_polygons"] - polygons = sdata.shapes["S_polygons"] - assert list(polygons.index) == [1, 3] - labels = sdata.labels["S_labels"]["scale0"]["image"].values - assert sorted(set(np.unique(labels)) - {0}) == list(polygons.index) - assert polygons.geometry.loc[3].equals(shapely.box(40, 40, 48, 48)) - assert get_transformation(polygons, get_all=True) == get_transformation(sdata.images["S_image"], get_all=True) - assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["S_labels"] - - sdata.write(tmp_path / "out.zarr") - reread = read_zarr(tmp_path / "out.zarr") - assert reread.shapes["S_polygons"].geometry.equals(polygons.geometry) - - -@pytest.mark.parametrize("collection", [True, False]) -def test_qupath_geojson_by_order_explicit_path_in_um(tmp_path: Path, collection: bool) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - header = "Image\tCentroid X µm\tCentroid Y µm\tCD3: Cell: Mean" - rows = ["S_Scan1.qptiff\t4\t4\t1", "S_Scan1.qptiff\t24\t24\t2"] - (tmp_path / "cells.tsv").write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") - # without ids, the cells and detections follow the row order of the table; annotations are skipped - features = [_cell(None, 0, 0, "annotation"), _cell(None, 0, 0), _cell(None, 20, 20, "detection")] - _write_geojson(tmp_path / "polygons.json", features, collection=collection) - - sdata = phenocycler( - tmp_path, - cells_table=tmp_path / "cells.tsv", - image_models_kwargs=_FAST, - cells_table_kwargs={"geojson": tmp_path / "polygons.json", "geojson_units": "um"}, - ) - - values = sdata.labels["S_labels"]["scale0"]["image"].values - # 8 µm squares at 0.5069 µm per pixel - assert values[int(4 / 0.5069), int(4 / 0.5069)] == 1 - assert values[int(24 / 0.5069), int(24 / 0.5069)] == 2 - assert (values == 1).sum() == pytest.approx((8 / 0.5069) ** 2, rel=0.15) - - -def test_qupath_geojson_in_um_regions_with_different_pixel_sizes(tmp_path: Path) -> None: - sizes = {"A": 0.5, "B": 0.25} - for region, size in sizes.items(): - write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI"], resolution_px_per_cm=1e4 / size) - header = "Image\tCentroid X µm\tCentroid Y µm\tCD3: Cell: Mean" - rows = ["A_Scan1.qptiff\t4\t4\t1", "B_Scan1.qptiff\t24\t24\t2"] - (tmp_path / "cells.tsv").write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") - _write_geojson(tmp_path / "polygons.json", [_cell(None, 0, 0), _cell(None, 20, 20)]) - - sdata = phenocycler( - tmp_path, - cells_table=tmp_path / "cells.tsv", - image_models_kwargs=_FAST, - cells_table_kwargs={"geojson": tmp_path / "polygons.json", "geojson_units": "um"}, - ) - - assert sdata.shapes["A_polygons"].geometry.iloc[0].equals(shapely.box(0, 0, 16, 16)) - assert sdata.shapes["B_polygons"].geometry.iloc[0].equals(shapely.box(80, 80, 112, 112)) - - -def test_qupath_geojson_mask_wins(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - _write_qupath_cells(tmp_path / "cells.tsv", ["a", "b"]) - _write_geojson(tmp_path / "cells_cells.geojson", [_cell("a", 0, 0), _cell("b", 20, 20)]) - mask = np.zeros((256, 320), dtype=np.uint32) - mask[100:110, 100:110] = 1 - tifffile.imwrite(tmp_path / "mask.tif", mask) - - with pytest.warns(UserWarning, match="mask .* is used for the labels and the polygons are kept as 'S_polygons'"): - sdata = phenocycler( - tmp_path, cells_table=tmp_path / "cells.tsv", labels=tmp_path / "mask.tif", image_models_kwargs=_FAST - ) - - np.testing.assert_array_equal(sdata.labels["S_labels"]["scale0"]["image"].values, mask) - assert "S_polygons" in sdata.shapes - assert sdata.attrs["akoya"]["geojson"] == {"S": str(tmp_path / "cells_cells.geojson")} - - -def test_qupath_geojson_errors(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - _write_qupath_cells(tmp_path / "cells.tsv", ["a", "b"]) - _write_geojson(tmp_path / "other.geojson", [_cell("x", 0, 0), _cell("y", 20, 20)]) - - with pytest.raises(ValueError, match="none of the feature ids"): - phenocycler( - tmp_path, - cells_table=tmp_path / "cells.tsv", - image_models_kwargs=_FAST, - cells_table_kwargs={"geojson": tmp_path / "other.geojson"}, - ) - with pytest.raises(ValueError, match="geojson_units"): - phenocycler( - tmp_path, - cells_table=tmp_path / "cells.tsv", - image_models_kwargs=_FAST, - cells_table_kwargs={"geojson": tmp_path / "other.geojson", "geojson_units": "mm"}, - ) - with pytest.raises(FileNotFoundError, match="missing.geojson"): - phenocycler( - tmp_path, - cells_table=tmp_path / "cells.tsv", - image_models_kwargs=_FAST, - cells_table_kwargs={"geojson": tmp_path / "missing.geojson"}, - ) - (tmp_path / "cells.csv").write_text("CellID,DAPI,X_centroid,Y_centroid\n1,2,3,4\n") - with pytest.raises(ValueError, match="QuPath"): - phenocycler( - tmp_path, - cells_table=tmp_path / "cells.csv", - image_models_kwargs=_FAST, - cells_table_kwargs={"geojson": tmp_path / "other.geojson"}, - ) - - # without ids, the number of cells must match - (tmp_path / "noid.tsv").write_text("Image\tCentroid X px\tCentroid Y px\nS_Scan1.qptiff\t1\t1\n", encoding="utf-8") - _write_geojson(tmp_path / "noid.geojson", [_cell(None, 0, 0), _cell(None, 20, 20)]) - with pytest.raises(ValueError, match="2 cell polygons .* 1 rows"), pytest.warns(UserWarning, match="marker"): - phenocycler(tmp_path, cells_table=tmp_path / "noid.tsv", image_models_kwargs=_FAST) - - -def _write_order_table(path: Path) -> None: - header = "Image\tCentroid X px\tCentroid Y px\tCD3: Cell: Mean" - rows = ["S_Scan1.qptiff\t4\t4\t1", "S_Scan1.qptiff\t24\t24\t2"] - path.write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") - - -@pytest.mark.parametrize( - ("features", "match"), - [ - # without ids, polygons in another order than the rows - ([_cell(None, 20, 20), _cell(None, 0, 0)], "2 of the 2 polygons .* are more than 1 px"), - # ids, but no `Object ID` in the table: matched in order too - ([_cell("u2", 20, 20), _cell("u1", 0, 0)], "another order"), - # coordinates in µm read as pixels - ([_cell(None, 0, 0), _cell(None, 10, 10)], "geojson_units"), - ], -) -def test_qupath_geojson_polygons_far_from_centroids_raise(tmp_path: Path, features: list[dict], match: str) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - _write_order_table(tmp_path / "cells.tsv") - _write_geojson(tmp_path / "cells.geojson", features) - - with pytest.raises(ValueError, match=match): - phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) - - -def test_qupath_geojson_partial_and_duplicate_ids(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - _write_qupath_cells(tmp_path / "cells.tsv", ["a", "b", "c"]) - _write_geojson(tmp_path / "cells.geojson", [_cell("a", 0, 0), _cell("c", 40, 40)]) - - with pytest.warns(UserWarning, match="1 of the 3 cells .* have no polygon"): - sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) - - assert set(np.unique(sdata.labels["S_labels"]["scale0"]["image"].values)) == {0, 1, 3} - assert sdata.tables["table"].n_obs == 3 - - _write_geojson(tmp_path / "cells.geojson", [_cell("a", 0, 0), _cell("b", 20, 20), _cell("a", 40, 40)]) - with pytest.raises(ValueError, match="same id"): - phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) - - -def test_qupath_geojson_region_without_polygons_gets_circles(tmp_path: Path) -> None: - for region in "AB": - write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI"]) - header = "Image\tObject ID\tCentroid X px\tCentroid Y px\tCD3: Cell: Mean" - rows = ["A_Scan1.qptiff\ta\t4\t4\t1", "B_Scan1.qptiff\tb\t24\t24\t2"] - (tmp_path / "cells.tsv").write_text("\n".join([header, *rows]) + "\n", encoding="utf-8") - # QuPath exports the GeoJSON of one image: B's cells have no polygon - _write_geojson(tmp_path / "cells.geojson", [_cell("a", 0, 0)]) - - with pytest.warns(UserWarning, match="have no polygon"): - sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) - - assert list(sdata.labels) == ["A_labels"] - assert set(sdata.shapes) == {"A_polygons", "B_cells"} - assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["A_labels", "B_cells"] - assert sdata.attrs["akoya"]["geojson"] == {"A": str(tmp_path / "cells.geojson")} - - -@pytest.mark.parametrize("object_ids", [True, False]) -def test_qupath_discovered_annotation_geojson_is_ignored(tmp_path: Path, object_ids: bool) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - if object_ids: - _write_qupath_cells(tmp_path / "cells.tsv", ["a", "b"]) - else: - _write_order_table(tmp_path / "cells.tsv") - # QuPath's "Export objects as GeoJSON" of the tissue annotation, named after the image like the table - _write_geojson(tmp_path / "cells.geojson", [_cell("tissue", 0, 0, "annotation")]) - - with pytest.warns(UserWarning, match="Ignoring .*cells.geojson.* none of the cells"): - sdata = phenocycler(tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST) - - assert list(sdata.shapes) == ["S_cells"] - assert not sdata.labels - assert "geojson" not in sdata.attrs["akoya"] - # an explicit GeoJSON must hold the cells - with pytest.raises(ValueError, match="cell"): - phenocycler( - tmp_path, - cells_table=tmp_path / "cells.tsv", - image_models_kwargs=_FAST, - cells_table_kwargs={"geojson": tmp_path / "cells.geojson"}, - ) - - -def test_qupath_geojson_false_reads_no_outlines(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - _write_qupath_cells(tmp_path / "cells.tsv", ["a", "b"]) - _write_geojson(tmp_path / "cells.geojson", [_cell("a", 0, 0), _cell("b", 20, 20)]) - - sdata = phenocycler( - tmp_path, cells_table=tmp_path / "cells.tsv", image_models_kwargs=_FAST, cells_table_kwargs={"geojson": False} - ) - - assert list(sdata.shapes) == ["S_cells"] - assert not sdata.labels - assert "geojson" not in sdata.attrs["akoya"] - - -def test_read_geojson_reads_only_ids_and_object_types(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - frames = [] - - def spy(*args: Any, **kwargs: Any) -> gpd.GeoDataFrame: - frames.append(read_file(*args, **kwargs)) - return frames[-1] - - read_file = gpd.read_file - monkeypatch.setattr(gpd, "read_file", spy) - feature = _cell("a", 0, 0) - # QuPath also writes the nucleus outline and every measurement - feature["properties"] |= {"nucleusGeometry": feature["geometry"], "measurements": {"CD3: Cell: Mean": 1.0}} - _write_geojson(tmp_path / "cells.geojson", [feature]) - - ids, geometries, kinds = _read_geojson(tmp_path / "cells.geojson") - - assert ids is not None - assert kinds is not None - assert list(ids) == ["a"] - assert list(kinds) == ["cell"] - assert geometries[0].equals(shapely.box(0, 0, 8, 8)) - assert set(frames[0].columns) == {"id", "objectType", "geometry"} - - -def test_read_geojson_array_with_bom_and_properties(tmp_path: Path) -> None: - feature = _cell("a", 0, 0) - feature["properties"] |= {"nucleusGeometry": feature["geometry"], "measurements": {"CD3: Cell: Mean": 1.0}} - path = tmp_path / "cells.json" - path.write_bytes(b"\xef\xbb\xbf" + json.dumps([feature, _cell(None, 20, 20, "annotation")]).encode()) - - ids, geometries, kinds = _read_geojson(path) - - assert ids is not None - assert kinds is not None - assert list(ids) == ["a", None] - assert list(kinds) == ["cell", "annotation"] - assert geometries[1].equals(shapely.box(20, 20, 28, 28)) From 0d367f2accc192b26e696514599acc92b39516ec Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 21:16:26 +0200 Subject: [PATCH 106/107] test(akoya): consolidate tests --- tests/_akoya_data.py | 217 ++----- tests/readers/test_akoya_utils.py | 254 ++------ tests/readers/test_cell_tables.py | 355 +++-------- tests/readers/test_qptiff.py | 346 +++-------- tests/test_akoya_fixtures.py | 73 --- tests/test_akoya_real_data.py | 145 +---- tests/test_codex.py | 981 ++++++++---------------------- tests/test_phenocycler.py | 679 +++++---------------- 8 files changed, 706 insertions(+), 2344 deletions(-) delete mode 100644 tests/test_akoya_fixtures.py diff --git a/tests/_akoya_data.py b/tests/_akoya_data.py index 73a5d99b..b826ce0b 100644 --- a/tests/_akoya_data.py +++ b/tests/_akoya_data.py @@ -16,97 +16,28 @@ import pytest -_XML_DECLARATION = '' - -def qpi_xml( - *, - image_type: str, - name: str, - biomarker: str | None = None, - software: str = "Fusion 2.3.1", - description_version: int = 6, - exposure_us: int = 450000, - color: str = "255,255,0", - scan_profile: str | None = None, - extra: dict[str, str] | None = None, -) -> str: - """Build a ``PerkinElmer-QPI-ImageDescription`` document (CRLF line endings, utf-16 declaration).""" - fields = [ - ("DescriptionVersion", str(description_version)), - ("AcquisitionSoftware", software), - ("ImageType", image_type), - ("Identifier", "00000000-0000-0000-0000-000000000000"), - ("SlideID", "SLIDE1"), - ("IsUnmixedComponent", "False"), - ("ExposureTime", str(exposure_us)), - ("SignalUnits", "64"), - ("Name", name), - ("Color", color), - ] - if biomarker is not None: - fields.append(("Biomarker", biomarker)) - fields.append(("ScaleFactor", "2.0001")) - fields.extend((extra or {}).items()) - if scan_profile is not None: - fields.append(("ScanProfile", scan_profile)) - lines = [_XML_DECLARATION, ""] - lines += [f" <{tag}>{text}" for tag, text in fields] - lines.append("") - return "\r\n".join(lines) +def fusion_scan_profile_json(wells: list[list[tuple[str, str]]]) -> str: + """A Fusion ``ScanProfile`` JSON; each well is a list of ``(markerName, channel)``.""" + items = [[{"markerName": marker, "channel": channel, "dye": channel} for marker, channel in well] for well in wells] + wells_json = [{"wellName": f"A{i + 1}", "items": well} for i, well in enumerate(items)] + return json.dumps({"experimentDescription": {"formatVersion": 2, "wells": wells_json}}) -def fusion_scan_profile_json(wells: list[list[tuple[str, str]]]) -> str: - """Build a Fusion ``ScanProfile`` JSON (formatVersion 2); each well is a list of ``(markerName, channel)``.""" - channels = sorted({channel for well in wells for _, channel in well}) - profile = { - "binning": 2, - "experimentDescription": { - "name": "synthetic", - "formatVersion": 2, - "tissueType": "HumanFFPE", - "channels": [{"name": c, "defaultExposure": 100} for c in channels], - "wells": [ - { - "wellName": f"A{i + 1}", - "items": [ - { - "id": "00000000-0000-0000-0000-000000000000", - "inventoryItemType": "Marker", - "markerName": marker, - "channel": channel, - "dye": channel, - "exposure": 100, - } - for marker, channel in well - ], - } - for i, well in enumerate(wells) - ], - }, - } - return json.dumps(profile, separators=(",", ":")) - - -def cim_experiment_v4_xml(cycles: list[list[str]], resolution_nm: float = 377.39) -> str: - """Build a CODEX Instrument Manager ``ExperimentV4`` document; ``cycles`` lists marker names per channel.""" - cycle_xml = [] - for i, markers in enumerate(cycles, start=1): - channels = "".join( +def cim_experiment_v4_xml(cycles: list[list[str]]) -> str: + """A CODEX Instrument Manager ``ExperimentV4`` document; ``cycles`` lists the marker names per channel.""" + cycle_xml = "".join( + f"{i}" + + "".join( f"{j}{marker}" - f"{'Nuclear' if marker == 'DAPI' else 'Blank' if marker == 'BLANK' else 'Other'}" + f"{'Blank' if marker == 'BLANK' else 'Other'}" f"100{358 + 100 * j}" for j, marker in enumerate(markers, start=1) ) - cycle_xml.append(f"{i}{channels}") - return ( - "" - "synthetic" - f"{len(cycles)}" - f"{resolution_nm}" - f"{''.join(cycle_xml)}" - "" + + "" + for i, markers in enumerate(cycles, start=1) ) + return f"377.39{cycle_xml}" def write_qptiff( @@ -118,7 +49,6 @@ def write_qptiff( dtype: str = "uint8", levels: int = 2, software: str = "Fusion 2.3.1", - description_version: int = 6, scan_profile: str | None = None, resolution_px_per_cm: float | None = 1e4 / 0.5069, resolution_unit: str = "CENTIMETER", @@ -131,8 +61,8 @@ def write_qptiff( ) -> np.ndarray: """Write a synthetic QPTIFF and return the level-0 data, ``(c, y, x)`` or ``(y, x, 3)`` when ``rgb``. - Page order mirrors real files: full-resolution pages, an RGB thumbnail, the reduced-resolution - groups (the last one stripped), an RGB overview and an RGB label. + Page order mirrors real files: full-resolution pages, an RGB thumbnail, the reduced-resolution groups (the last + one stripped), an RGB overview and an RGB label. """ rng = np.random.default_rng(0) @@ -142,26 +72,25 @@ def random(size: tuple[int, ...]) -> np.ndarray: return rng.integers(0, np.iinfo(dtype).max, size, dtype=dtype) n_pages = 1 if rgb else len(biomarkers) - names = ( - names if names is not None else [b if b is not None else f"Channel {i + 1}" for i, b in enumerate(biomarkers)] - ) + names = names or [b if b is not None else f"Channel {i + 1}" for i, b in enumerate(biomarkers)] resolution = None if resolution_px_per_cm is None else (resolution_px_per_cm, resolution_px_per_cm) - shape_c = (*shape, 3) if rgb else shape - - def description(image_type: str, name: str, biomarker: str | None = None, *, first: bool = False) -> str: - xml = qpi_xml( - image_type=image_type, - name=name, - biomarker=biomarker, - software=software, - description_version=description_version, - scan_profile=scan_profile if first else None, - ) - return xml.replace("SLIDE1", f"{slide_id}") - def write_page( - tw: tifffile.TiffWriter, data: np.ndarray, desc: str, *, subfiletype: int = 0, **kwargs: Any - ) -> None: + def description(image_type: str, name: str, biomarker: str | None = None, profile: str | None = None) -> str: + fields = { + "AcquisitionSoftware": software, + "ImageType": image_type, + "SlideID": slide_id, + "ExposureTime": "450000", + "Name": name, + "Color": "255,255,0", + "Biomarker": biomarker, + "ScanProfile": profile, + } + tags = "".join(f"<{tag}>{text}\r\n" for tag, text in fields.items() if text is not None) + root = "PerkinElmer-QPI-ImageDescription" + return f'\r\n<{root}>\r\n{tags}' + + def write_page(tw: tifffile.TiffWriter, data: np.ndarray, desc: str, subfiletype: int = 0, **kwargs: Any) -> None: tw.write( data, software="PerkinElmer-QPI", @@ -174,38 +103,25 @@ def write_page( **kwargs, ) - def rgb_page(side: tuple[int, int] = (64, 64)) -> np.ndarray: - return rng.integers(0, 256, (*side, 3), dtype=np.uint8) + def rgb_page() -> np.ndarray: + return rng.integers(0, 256, (64, 64, 3), dtype=np.uint8) extratags: list[Any] = [] - if stage_position_cm is not None: - for code, value in zip((286, 287), stage_position_cm, strict=True): - extratags.append((code, 5, 1, (round(value * 1e6), 1_000_000), False)) + for code, value in zip((286, 287), stage_position_cm or (), strict=False): + extratags.append((code, 5, 1, (round(value * 1e6), 1_000_000), False)) - data = random(shape_c if rgb else (n_pages, *shape)) + data = random((*shape, 3) if rgb else (n_pages, *shape)) with tifffile.TiffWriter(path, bigtiff=bigtiff) as tw: for i in range(n_pages): - page = data if rgb else data[i] - write_page( - tw, - page, - description("FullResolution", names[i], biomarkers[i], first=i == 0), - tile=tile, - extratags=extratags if i == 0 else None, - ) + desc = description("FullResolution", names[i], biomarkers[i], scan_profile if i == 0 else None) + write_page(tw, data if rgb else data[i], desc, tile=tile, extratags=extratags if i == 0 else None) write_page(tw, rgb_page(), description("Thumbnail", "Thumbnail")) for level in range(1, levels): side = (shape[0] // 2**level, shape[1] // 2**level) - is_last = level == levels - 1 for i in range(n_pages): page = random((*side, 3) if rgb else side) - write_page( - tw, - page, - description("ReducedResolution", names[i], biomarkers[i]), - subfiletype=1, - tile=None if is_last else tile, - ) + desc = description("ReducedResolution", names[i], biomarkers[i]) + write_page(tw, page, desc, subfiletype=1, tile=None if level == levels - 1 else tile) write_page(tw, rgb_page(), description("Overview", "Overview")) if label_page: write_page(tw, rgb_page(), description("Label", "Label")) @@ -215,21 +131,12 @@ def rgb_page(side: tuple[int, int] = (64, 64)) -> np.ndarray: def write_fcs(path: Path, df: pd.DataFrame, marker_names: dict[str, str] | None = None) -> None: """Write a minimal FCS 3.0 file: little-endian float32, ``$PnN`` = column name, ``$PnS`` = marker name.""" marker_names = marker_names or {} - values = np.ascontiguousarray(df.to_numpy(dtype=" bytes: - pairs: list[tuple[str, str]] = [ - ("$BEGINDATA", f"{data_begin:>12}"), - ("$ENDDATA", f"{data_end:>12}"), - ("$PAR", str(n_par)), - ("$TOT", str(len(df))), - ("$DATATYPE", "F"), - ("$BYTEORD", "1,2,3,4"), - ("$MODE", "L"), - ("$NEXTDATA", "0"), - ] + pairs = [("$BEGINDATA", f"{data_begin:>12}"), ("$ENDDATA", f"{data_end:>12}"), ("$PAR", str(df.shape[1]))] + pairs += [("$TOT", str(len(df))), ("$DATATYPE", "F"), ("$BYTEORD", "1,2,3,4"), ("$MODE", "L")] + pairs.append(("$NEXTDATA", "0")) for i, col in enumerate(df.columns, start=1): pairs += [(f"$P{i}N", str(col)), (f"$P{i}B", "32"), (f"$P{i}E", "0,0"), (f"$P{i}R", "262144")] if col in marker_names: @@ -237,10 +144,8 @@ def text_segment(data_begin: int, data_end: int) -> bytes: return ("/" + "/".join(f"{k}/{v}" for k, v in pairs) + "/").encode("ascii") text_start = 58 - text_len = len(text_segment(0, 0)) # offsets are fixed-width, so the length does not depend on their value - text_end = text_start + text_len - 1 - data_begin = text_end + 1 - data_end = data_begin + len(data) - 1 + text_end = text_start + len(text_segment(0, 0)) - 1 # the offsets are fixed-width + data_begin, data_end = text_end + 1, text_end + len(data) header = f"FCS3.0 {text_start:>8}{text_end:>8}{data_begin:>8}{data_end:>8}{0:>8}{0:>8}".encode("ascii") path.write_bytes(header.ljust(text_start) + text_segment(data_begin, data_end) + data) @@ -251,7 +156,6 @@ def write_legacy_processor_tree( regions: int = 1, cycles: int = 2, channels: int = 4, - markers: list[str] | None = None, shape: tuple[int, int] = (128, 160), layout: Literal["per_marker", "hyperstack", "old_flat"] = "per_marker", fcs: bool = True, @@ -260,15 +164,11 @@ def write_legacy_processor_tree( ) -> dict[str, Any]: """Write a CODEX Processor output tree and return ``{"markers", "images", "cells"}``. - ``markers`` has one entry per (cycle, channel), cycle-major. ``images`` maps region name to - ``(n_markers, y, x)`` for ``per_marker`` and ``old_flat`` and to ``(cycles, channels, y, x)`` for - ``hyperstack``. ``cells`` is the table written for the last region. + ``images`` maps region name to ``(n_markers, y, x)`` for ``per_marker`` and ``old_flat`` and to + ``(cycles, channels, y, x)`` for ``hyperstack``. ``cells`` is the table written for the last region. """ n_markers = cycles * channels - if markers is None: - markers = ["DAPI" if c == 0 else f"MARKER{i}" for i in range(n_markers) for c in [i % channels]] - if len(markers) != n_markers: - raise ValueError(f"expected {n_markers} markers (cycles * channels), got {len(markers)}") + markers = ["DAPI" if i % channels == 0 else f"MARKER{i}" for i in range(n_markers)] rng = np.random.default_rng(0) root.mkdir(parents=True, exist_ok=True) processed = root / "processed_2020-01-01" @@ -278,9 +178,7 @@ def write_legacy_processor_tree( if layout != "old_flat": (root / "channelNames.txt").write_text("\n".join(markers) + "\n") - (root / "experiment.json").write_text( - json.dumps({"name": "synthetic", "numCycles": cycles, "numChannels": channels, "regions": regions}) - ) + (root / "experiment.json").write_text(json.dumps({"numCycles": cycles, "numChannels": channels})) for r in range(1, regions + 1): name = f"reg{r:03d}" @@ -289,14 +187,14 @@ def write_legacy_processor_tree( { "cell_id": np.arange(1, n_cells + 1, dtype=float), "region": float(r), - "tile_num": 1.0, "x": rng.uniform(0, shape[1], n_cells), "y": rng.uniform(0, shape[0], n_cells), - "z": 1.0, "size": rng.uniform(10, 100, n_cells), **{col: rng.uniform(0, 1000, n_cells) for col in channel_cols}, } ) + table_dir = processed / "segm" / "segm-1" / "fcs" / "compensated" + table_stem = f"{name}_compensated" if layout == "per_marker": images[name] = stack image_dir = processed / "stitched" / name @@ -304,21 +202,16 @@ def write_legacy_processor_tree( for i, marker in enumerate(markers): cyc, ch = divmod(i, channels) tifffile.imwrite(image_dir / f"{name}_cyc{cyc + 1:03d}_ch{ch + 1:03d}_{marker}.tif", stack[i]) - table_dir = processed / "segm" / "segm-1" / "fcs" / "compensated" - table_stem = f"{name}_compensated" elif layout == "hyperstack": images[name] = stack.reshape(cycles, channels, *shape) (processed / "stitched").mkdir(parents=True, exist_ok=True) tifffile.imwrite( processed / "stitched" / f"{name}_montage.tif", images[name], imagej=True, metadata={"axes": "TCYX"} ) - table_dir = processed / "segm" / "segm-1" / "fcs" / "compensated" - table_stem = f"{name}_compensated" else: images[name] = stack tifffile.imwrite(root / f"exp_{name}.tif", stack[0]) - table_dir = root - table_stem = f"exp_{name}" + table_dir, table_stem = root, f"exp_{name}" table_dir.mkdir(parents=True, exist_ok=True) if fcs: write_fcs(table_dir / f"{table_stem}.fcs", cells, dict(zip(channel_cols, markers, strict=True))) diff --git a/tests/readers/test_akoya_utils.py b/tests/readers/test_akoya_utils.py index 79cdf8c5..64da6dd3 100644 --- a/tests/readers/test_akoya_utils.py +++ b/tests/readers/test_akoya_utils.py @@ -1,227 +1,113 @@ from __future__ import annotations -import json import os import pickle from pathlib import Path -from typing import Any import dask import dask.array as da import numpy as np -import pandas as pd import pytest import tifffile -from spatialdata.transformations import Identity, Translation, get_transformation +from spatialdata.transformations import Identity, Translation from spatialdata_io.readers._utils._akoya import ( apply_channel_names, - build_image_element, dedupe_channel_names, read_tiff, region_transformations, - scans_to_attrs, - select_regions, ) from spatialdata_io.readers._utils._qptiff import AkoyaChannel, AkoyaScan -_NO_PYRAMID = {"scale_factors": None, "chunks": (1, 8, 8)} - def _scan( - region: str = "reg1", - names: tuple[str, ...] = ("DAPI", "CD3", "CD8"), + region: str, stage: tuple[float, float] | None = None, pixel_size_um: float | None = 0.5, slide_id: str | None = "slide", - rgb: bool = False, ) -> AkoyaScan: + channels = [AkoyaChannel(name=name) for name in ("DAPI", "DAPI", "CD3")] return AkoyaScan( - path=Path(f"{region}.qptiff"), - region=region, - generation="fusion2", - software="Fusion", - slide_id=slide_id, - pixel_size_um=pixel_size_um, - nominal_pixel_size_um=0.5, - stage_position_um=stage, - channels=[AkoyaChannel(name=n, cycle=1, extra={"k": [1, 2]}) for n in names], - rgb=rgb, - raw={"page0_xml": "", "scan_profile": None}, + Path(f"{region}.qptiff"), region, "fusion2", None, slide_id, pixel_size_um, None, stage, channels, False, {} ) -def test_dedupe_channel_names(): - assert dedupe_channel_names(["DAPI", "CD3", "DAPI", "DAPI"]) == ["DAPI", "CD3", "DAPI (2)", "DAPI (3)"] - - -def test_dedupe_channel_names_never_collides(): - # a generated suffix must not repeat a name that is already in the list +def test_channel_names() -> None: + # a generated suffix never repeats a name already in the list assert dedupe_channel_names(["DAPI", "DAPI", "DAPI (2)"]) == ["DAPI", "DAPI (3)", "DAPI (2)"] - assert dedupe_channel_names(["A (2)", "A", "A"]) == ["A (2)", "A", "A (3)"] - - -def test_apply_channel_names_override_and_default(): - scan = _scan(names=("DAPI", "DAPI", "CD3")) + scan = _scan("a") assert apply_channel_names(scan, None) == ["DAPI", "DAPI (2)", "CD3"] - assert apply_channel_names(scan, ["a", "b", "c"]) == ["a", "b", "c"] - # an override is de-duplicated like the metadata names, so channels stay selectable by name assert apply_channel_names(scan, ["a", "b", "a"]) == ["a", "b", "a (2)"] - - -@pytest.mark.parametrize("names", [{"a": 0, "b": 1, "c": 2}, {"a", "b", "c"}], ids=["mapping", "set"]) -def test_apply_channel_names_unordered_raises(names: object): - # the keys of a mapping or the iteration order of a set would silently become the channel names - with pytest.raises(TypeError, match="sequence"): - apply_channel_names(_scan(), names) # type: ignore[arg-type] - - -def test_apply_channel_names_length_mismatch(): with pytest.raises(ValueError, match="expected 3"): - apply_channel_names(_scan(), ["a", "b"]) - - -def test_region_transformations_single_is_global(): - assert region_transformations([_scan("a", stage=(5.0, 5.0))]) == {"a": {"global": Identity()}} - - -def test_region_transformations_shared_slide_translation(): - result = region_transformations([_scan("a", stage=(0.0, 0.0)), _scan("b", stage=(100.0, 50.0))]) - assert result == { - "a": {"global": Translation([0.0, 0.0], axes=("x", "y"))}, - "b": {"global": Translation([200.0, 100.0], axes=("x", "y"))}, - } - - -def test_region_transformations_unrelated_get_own_cs(): - result = region_transformations( - [_scan("A", stage=(0.0, 0.0), slide_id="s1"), _scan("B", stage=(100.0, 50.0), slide_id="s2")] - ) - assert result == {"A": {"A": Identity()}, "B": {"B": Identity()}} + apply_channel_names(scan, ["a", "b"]) + # the keys of a mapping or the order of a set would silently become the channel names + with pytest.raises(TypeError, match="sequence"): + apply_channel_names(scan, {"a", "b", "c"}) # type: ignore[arg-type] -def test_region_transformations_mixed_pixel_sizes_own_cs(): - result = region_transformations([_scan("a", stage=(0.0, 0.0)), _scan("b", stage=(1.0, 1.0), pixel_size_um=0.25)]) - assert result == {"a": {"a": Identity()}, "b": {"b": Identity()}} - # pixel sizes equal up to rounding share the sample space - result = region_transformations([_scan("a", stage=(0.0, 0.0)), _scan("b", stage=(1.0, 1.0), pixel_size_um=0.5004)]) - assert set(result["b"]) == {"global"} +_GLOBAL = { + "a": {"global": Translation([0.0, 0.0], axes=("x", "y"))}, + "b": {"global": Translation([200.0, 100.0], axes=("x", "y"))}, +} +_OWN = {"a": {"a": Identity()}, "b": {"b": Identity()}} @pytest.mark.parametrize( - "scans", + ("scans", "expected"), [ - [_scan("a"), _scan("b")], - [_scan("a", stage=(0.0, 0.0), pixel_size_um=None), _scan("b", stage=(1.0, 1.0))], - [_scan("a", stage=(0.0, 0.0), slide_id=None), _scan("b", stage=(1.0, 1.0), slide_id=None)], + ([_scan("a", (0.0, 0.0)), _scan("b", (100.0, 50.0))], _GLOBAL), + # pixel sizes equal up to rounding share the sample space + ([_scan("a", (0.0, 0.0)), _scan("b", (100.0, 50.0), pixel_size_um=0.5004)], None), + ([_scan("a", (0.0, 0.0), slide_id="s1"), _scan("b", (100.0, 50.0), slide_id="s2")], _OWN), + ([_scan("a", (0.0, 0.0)), _scan("b", (1.0, 1.0), pixel_size_um=0.25)], _OWN), + ([_scan("a"), _scan("b")], _OWN), + ([_scan("a", (0.0, 0.0), pixel_size_um=None), _scan("b", (1.0, 1.0))], _OWN), + ([_scan("a", (0.0, 0.0), slide_id=None), _scan("b", (1.0, 1.0), slide_id=None)], _OWN), ], - ids=["no_stage_positions", "missing_pixel_size", "no_slide_id"], -) -def test_region_transformations_unplaceable_get_own_cs(scans: list[AkoyaScan]): - assert region_transformations(scans) == {"a": {"a": Identity()}, "b": {"b": Identity()}} - - -def test_region_transformations_regions_without_image(): - # a region read without an image (e.g. only its cells) is unrelated to the scans - shared = [_scan("a", stage=(0.0, 0.0)), _scan("b", stage=(100.0, 50.0))] - result = region_transformations(shared, ["c", "a"]) - assert set(result["a"]) == set(result["b"]) == {"global"} - assert result["c"] == {"c": Identity()} - assert region_transformations([_scan("a")], ["c"]) == {"a": {"a": Identity()}, "c": {"c": Identity()}} - assert region_transformations([], ["c"]) == {"c": {"global": Identity()}} - - -def test_scans_to_attrs_json_serialisable(): - scans = [_scan("a", stage=(1.0, 2.0)), _scan("b")] - names = {"a": ["DAPI", "CD3", "CD8"], "b": ["x", "y", "z"]} - attrs = scans_to_attrs(scans, names) - json.dumps(attrs) - entry = attrs["regions"]["a"] - assert entry["stage_position_um"] == [1.0, 2.0] - assert entry["path"] == "a.qptiff" - assert [c["name_deduped"] for c in attrs["regions"]["b"]["channels"]] == ["x", "y", "z"] - assert entry["channels"][0]["name"] == "DAPI" - - -def test_select_regions(): - assert select_regions(["a", "b"], None) == ["a", "b"] - assert select_regions(["a", "b"], ["b"]) == ["b"] - - -def test_select_regions_array_like(): - # e.g. `sdata.tables["table"].obs["region"].unique()` or a `pd.Index` - assert select_regions(["a", "b"], np.array(["b"])) == ["b"] # type: ignore[arg-type] - assert select_regions(["a", "b"], pd.Index(["a", "b"])) == ["a", "b"] # type: ignore[arg-type] - with pytest.raises(ValueError, match="empty"): - select_regions(["a", "b"], np.array([], dtype=str)) # type: ignore[arg-type] - - -def test_select_regions_deduplicates(): - assert select_regions(["a", "b"], ["b", "a", "b"]) == ["b", "a"] - assert select_regions(["a", "b"], "b") == ["b"] - - -@pytest.mark.parametrize( - ("regions", "error", "match"), - [ - ({}, TypeError, "sequence of region ids"), - ({"a": 1}, TypeError, "sequence of region ids"), - ([], ValueError, "empty"), + ids=[ + "same_slide", + "rounded_pixel_size", + "other_slide", + "other_pixel_size", + "no_stage", + "no_pixel_size", + "no_slide", ], ) -def test_select_regions_invalid_raises(regions: object, error: type[Exception], match: str): - with pytest.raises(error, match=match): - select_regions(["a", "b"], regions) # type: ignore[arg-type] - - -def test_select_regions_unknown_raises(): - with pytest.raises(ValueError, match="'c'.*Available.*a.*b"): - select_regions(["a", "b"], ["c"]) +def test_region_transformations(scans: list[AkoyaScan], expected: dict | None) -> None: + result = region_transformations(scans) + if expected is None: + assert set(result["b"]) == {"global"} + else: + assert result == expected + # a region without an image is unrelated to the scans + assert region_transformations(scans, ["c"])["c"] == {"c": Identity()} + assert region_transformations([_scan("a", (5.0, 5.0))]) == {"a": {"global": Identity()}} @pytest.mark.parametrize("compression", [None, "zlib"]) -def test_read_tiff_is_picklable(tmp_path: Path, compression: str | None) -> None: +def test_read_tiff_is_picklable_and_keys_depend_on_chunks(tmp_path: Path, compression: str | None) -> None: # compressed tiles are read through tifffile's zarr store, uncompressed data is memory-mapped path = tmp_path / "x.tif" data = np.arange(2 * 64 * 96, dtype=np.uint16).reshape(2, 64, 96) - tifffile.imwrite(path, data, compression=compression, tile=(32, 32) if compression else None) - - image, _, _ = read_tiff(path, (16, 16)) - - restored = pickle.loads(pickle.dumps(image)) - np.testing.assert_array_equal(restored[0, :8, :8].compute(), data[0, :8, :8]) - np.testing.assert_array_equal(restored.compute(), data) - + tifffile.imwrite(path, data, compression=compression, tile=(16, 16) if compression else None) -def test_build_image_rgb_and_multiplex(): - scan = _scan() - img = da.from_array(np.random.default_rng(0).integers(0, 255, (3, 16, 16), dtype=np.uint8), chunks=(1, 8, 8)) - element = build_image_element(img, scan, ["DAPI", "CD3", "CD8"], {"global": Identity()}, _NO_PYRAMID) - assert list(element.coords["c"].values) == ["DAPI", "CD3", "CD8"] - assert isinstance(get_transformation(element, "global"), Identity) + small, _, _ = read_tiff(path, (16, 16)) + large, _, _ = read_tiff(path, (32, 32)) - rgb_scan = _scan(rgb=True) - rgb_img = da.from_array(np.zeros((16, 16, 3), dtype=np.uint8), chunks=(8, 8, 3)) - trans = Translation([4.0, 2.0], axes=("x", "y")) - element = build_image_element(rgb_img, rgb_scan, ["r", "g", "b"], {"reg1": trans}, _NO_PYRAMID) - assert list(element.coords["c"].values) == ["r", "g", "b"] - assert element.shape == (3, 16, 16) - assert get_transformation(element, get_all=True) == {"reg1": trans} + np.testing.assert_array_equal(pickle.loads(pickle.dumps(small)).compute(), data) + # arrays of one file with other chunks must not share dask keys, or computing both mixes their chunks + assert small.name != large.name + np.testing.assert_array_equal(da.stack([small.rechunk(large.chunks), large]).compute(), [data, data]) @pytest.mark.parametrize("imagej", [False, True]) -@pytest.mark.parametrize( - "data", [np.array([[[1, 8], [15, 0]]], dtype=np.uint16), np.array([[[1, 2], [300, 0]]], np.uint16)] -) -def test_read_tiff_big_endian(tmp_path: Path, imagej: bool, data: np.ndarray) -> None: +def test_read_tiff_big_endian(tmp_path: Path, imagej: bool) -> None: # ImageJ/Fiji write big-endian tiffs by default; the memory-mapped path must use the file's byte order - path = tmp_path / "x.tif" - tifffile.imwrite(path, data, byteorder=">", imagej=imagej) - - image, _, _ = read_tiff(path, (16, 16)) + data = np.array([[[1, 2], [300, 0]]], np.uint16) + tifffile.imwrite(tmp_path / "x.tif", data, byteorder=">", imagej=imagej) - assert image.dtype == np.dtype("uint16") - np.testing.assert_array_equal(image.compute(), data) + np.testing.assert_array_equal(read_tiff(tmp_path / "x.tif", (16, 16))[0].compute(), data) @pytest.mark.skipif(not os.path.isdir("/dev/fd"), reason="needs /dev/fd to count open files") @@ -235,39 +121,7 @@ def test_read_tiff_does_not_keep_files_open(tmp_path: Path, compression: str | N images.append(read_tiff(tmp_path / f"{i}.tif", (16, 16))[0]) n_open = len(os.listdir("/dev/fd")) - # the chunks as computed, before dask assembles them (e.g. while ``compute`` or ``sdata.write`` runs) chunks = dask.compute(*[chunk for image in images for chunk in image.to_delayed().ravel()]) assert len(os.listdir("/dev/fd")) - n_open <= 2 np.testing.assert_array_equal(chunks[-1][0], data[48:, 80:]) - - -@pytest.mark.parametrize( - ("layout", "chunksize"), - [({"rowsperstrip": 2}, (1, 16, 64)), ({"tile": (16, 16)}, (1, 32, 32)), ({"tile": (64, 64)}, (1, 64, 64))], - ids=["strips", "small_tiles", "large_tiles"], -) -def test_read_tiff_groups_native_chunks(tmp_path: Path, layout: dict[str, Any], chunksize: tuple[int, ...]) -> None: - # one dask chunk per strip or tile makes huge graphs; chunks group whole strips/tiles to about `chunks` pixels - path = tmp_path / "x.tif" - data = np.arange(256 * 64, dtype=np.uint16).reshape(256, 64) - tifffile.imwrite(path, data, compression="zlib", **layout) - - image, _, _ = read_tiff(path, (32, 32)) - - assert image.chunksize == chunksize - np.testing.assert_array_equal(image.compute()[0], data) - - -@pytest.mark.parametrize("compression", [None, "zlib"]) -def test_read_tiff_names_depend_on_chunks(tmp_path: Path, compression: str | None) -> None: - # arrays of one file with different chunks must not share dask keys, or computing both mixes their chunks - path = tmp_path / "x.tif" - data = np.arange(64 * 64, dtype=np.uint16).reshape(64, 64) - tifffile.imwrite(path, data, compression=compression, tile=(16, 16) if compression else None) - - small, _, _ = read_tiff(path, (16, 16)) - large, _, _ = read_tiff(path, (32, 32)) - - assert small.name != large.name - np.testing.assert_array_equal(da.stack([small.rechunk(large.chunks), large]).compute()[:, 0], [data, data]) diff --git a/tests/readers/test_cell_tables.py b/tests/readers/test_cell_tables.py index 3e57b0dd..185996e6 100644 --- a/tests/readers/test_cell_tables.py +++ b/tests/readers/test_cell_tables.py @@ -1,6 +1,5 @@ from __future__ import annotations -import warnings from typing import TYPE_CHECKING, Any import numpy as np @@ -8,337 +7,153 @@ import pytest import tifffile from spatialdata import get_extent, read_zarr -from spatialdata.models import get_axes_names from spatialdata.transformations import Identity, Scale, Sequence, Translation, get_transformation from tests._akoya_data import forbid_pixel_reads, write_fcs, write_qptiff from spatialdata_io import phenocycler -from spatialdata_io.readers._utils._akoya import dedupe_channel_names -from spatialdata_io.readers._utils._cell_tables import ( - build_cells, - read_cell_table, - read_labels, - resolve_per_region, -) +from spatialdata_io.readers._utils._cell_tables import read_cell_table, read_labels if TYPE_CHECKING: from pathlib import Path - from spatialdata_io.readers._utils._cell_tables import CellTable - -def _cells(n: int = 5, region: int = 1, markers: tuple[str, ...] = ("cyc001_ch001", "cyc001_ch002")) -> pd.DataFrame: +def _cells(columns: tuple[str, ...]) -> pd.DataFrame: rng = np.random.default_rng(0) - return pd.DataFrame( - { - "cell_id": np.arange(1, n + 1, dtype=float), - "region": float(region), - "x": rng.uniform(0, 100, n), - "y": rng.uniform(0, 80, n), - "size": rng.uniform(10, 100, n), - **{marker: rng.uniform(0, 1000, n) for marker in markers}, - } - ) - - -def _read_codex(path: Path, region: str | None = None) -> CellTable: - return read_cell_table(path, default_region=region) - - -def test_read_codex_fcs_marker_names_from_pns(tmp_path: Path) -> None: - cells = _cells(markers=("cyc001_ch001", "cyc001_ch002", "cyc002_ch001")) - path = tmp_path / "reg001_compensated.fcs" - write_fcs(path, cells, {"cyc001_ch001": "DAPI", "cyc001_ch002": "CD3", "cyc002_ch001": "DAPI"}) + meta = {"cell_id": [1.0, 2.0], "region": 1.0, "x": [1.0, 2.0], "y": [3.0, 4.0], "size": [3.0, 12.0]} + return pd.DataFrame(meta | {column: rng.uniform(0, 1000, 2) for column in columns}) + + +@pytest.mark.parametrize( + ("pns", "names"), + [ + ({"cyc001_ch001": "DAPI", "cyc001_ch002": "CD3", "cyc002_ch001": "DAPI"}, ["DAPI", "CD3", "DAPI (2)"]), + ({"cyc001_ch001": " ", "cyc001_ch002": "CD3"}, ["cyc001_ch001", "CD3", "cyc002_ch001"]), + (None, ["cyc001_ch001", "cyc001_ch002", "cyc002_ch001"]), + ], + ids=["pns", "blank_pns", "no_pns"], +) +def test_fcs_marker_names(tmp_path: Path, pns: dict[str, str] | None, names: list[str]) -> None: + columns = ("cyc001_ch001", "cyc001_ch002", "cyc002_ch001") + cells = _cells(columns) + write_fcs(tmp_path / "reg001.fcs", cells, pns) - table = _read_codex(path) + table = read_cell_table(tmp_path / "reg001.fcs", default_region=None) - assert list(table.var.index) == ["DAPI", "CD3", "DAPI (2)"] + assert list(table.var.index) == names assert list(table.var["cycle"]) == [1, 1, 2] - assert list(table.var["channel"]) == [1, 2, 1] - assert list(table.var["source_column"]) == ["cyc001_ch001", "cyc001_ch002", "cyc002_ch001"] - assert table.X.dtype == np.float32 - np.testing.assert_allclose(table.X, cells[["cyc001_ch001", "cyc001_ch002", "cyc002_ch001"]], rtol=1e-6) - assert "cyc001_ch001" not in table.obs.columns - assert list(table.obs.index) == ["1", "2", "3", "4", "5"] - assert list(table.region.unique()) == ["reg001"] - assert table.instance_id.dtype == np.int64 - assert table.centroids_px is not None - np.testing.assert_allclose(table.centroids_px, cells[["x", "y"]], rtol=1e-6) + assert list(table.var["source_column"]) == list(columns) + np.testing.assert_allclose(table.X, cells[list(columns)], rtol=1e-6) + assert list(table.region) == ["reg001", "reg001"] + assert list(table.instance_id) == [1, 2] + np.testing.assert_allclose(table.centroids_px, cells[["x", "y"]]) assert table.radius_px is not None - np.testing.assert_allclose(table.radius_px, np.sqrt(cells["size"] / np.pi), rtol=1e-6) - - -def test_read_codex_fcs_without_pns_keeps_column_names(tmp_path: Path) -> None: - path = tmp_path / "reg001.fcs" - write_fcs(path, _cells()) + np.testing.assert_allclose(table.radius_px, np.sqrt(cells["size"] / np.pi)) - assert list(_read_codex(path).var.index) == ["cyc001_ch001", "cyc001_ch002"] - -def test_read_codex_csv_colon_header(tmp_path: Path) -> None: +def test_csv_header_names_the_markers(tmp_path: Path) -> None: path = tmp_path / "reg002.csv" path.write_text( - "cell_id:cell_id,region:region,x:x,y:y,size:size,cyc001_ch001:DAPI1,cyc001_ch002:CD3\n" - "1,2,10,20,30,100.5,7\n" - "2,2,11,21,31,200.5,8\n" + "cell_id:cell_id,region:region,x:x,y:y,size:size,cyc001_ch001:DAPI1,cyc001_ch002:CD3\n1,2,10,20,30,1,7\n" ) - with warnings.catch_warnings(): - warnings.simplefilter("error") # the markers are the cycCCC_chHHH columns, no heuristic - table = _read_codex(path) + table = read_cell_table(path, default_region=None) assert list(table.var.index) == ["DAPI1", "CD3"] assert list(table.var["source_column"]) == ["cyc001_ch001:DAPI1", "cyc001_ch002:CD3"] - np.testing.assert_array_equal(table.X, np.array([[100.5, 7], [200.5, 8]], dtype=np.float32)) - # the columns read as the instance ids and regions are replaced by the keys of the table element + # the instance ids and regions are stored under the keys of the table element assert list(table.obs.columns) == ["x", "y", "size"] - assert list(table.region) == ["reg002", "reg002"] - np.testing.assert_array_equal(table.centroids_px, [[10, 20], [11, 21]]) - - -def test_read_codex_csv_marker_name_columns(tmp_path: Path) -> None: - path = tmp_path / "cells.csv" - path.write_text("cell_id,x,y,tile_num,CD3,CD8,label\n1,1.0,2.0,1,5.0,6.0,T\n") - - with pytest.warns(UserWarning, match=r"no 'cycCCC_chHHH' columns.*\['CD3', 'CD8'\]"): - table = _read_codex(path, region="Kidney") - - assert list(table.var.index) == ["CD3", "CD8"] - assert "cycle" not in table.var.columns - assert list(table.obs.columns) == ["x", "y", "tile_num", "label"] - assert list(table.region) == ["Kidney"] - assert table.radius_px is None - - -def test_read_codex_table_errors(tmp_path: Path) -> None: - no_xy = tmp_path / "a.csv" - no_xy.write_text("cell_id,cyc001_ch001\n1,2.0\n") - with pytest.raises(ValueError, match="'x' and 'y'"): - _read_codex(no_xy) - no_region = tmp_path / "b.csv" - no_region.write_text("cell_id,x,y,cyc001_ch001\n1,1.0,2.0,2.0\n") - with pytest.raises(ValueError, match="region"): - _read_codex(no_region) + assert list(table.region) == ["reg002"] @pytest.mark.parametrize("mangled", [False, True]) -def test_repeated_column_names_are_deduplicated(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mangled: bool) -> None: - if mangled: # pandas >= 3.1 renames repeated names (DAPI.1) with the pyarrow engine too +def test_csv_without_cycle_columns_reads_numeric_columns( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mangled: bool +) -> None: + if mangled: # pandas >= 3.1 renames repeated names (CD3.1) with the pyarrow engine too read_csv = pd.read_csv def mangling_read_csv(*args: Any, **kwargs: Any) -> pd.DataFrame: df = read_csv(*args, **kwargs) - seen: dict[str, int] = {} - columns = [] - for column in df.columns: - columns.append(f"{column}.{seen[column]}" if column in seen else column) - seen[column] = seen.get(column, 0) + 1 - df.columns = pd.Index(columns) + df.columns = pd.Index([*df.columns[:-2], "CD3.1", df.columns[-1]]) return df monkeypatch.setattr(pd, "read_csv", mangling_read_csv) path = tmp_path / "cells.csv" - path.write_text("cell_id,x,y,size,DAPI,CD3,DAPI\n1,2,3,4,5,6,7\n", encoding="utf-8") - - with pytest.warns(UserWarning, match="no 'cycCCC_chHHH' columns"): - table = _read_codex(path, region="S") - - assert list(table.var.index) == ["DAPI", "CD3", "DAPI (2)"] - assert table.X.shape == (1, 3) - - -@pytest.mark.parametrize("cell_id", ["", "1.5", "a"]) -def test_missing_or_fractional_instance_ids_raise(tmp_path: Path, cell_id: str) -> None: - path = tmp_path / "cells.csv" - path.write_text(f"cell_id,x,y,cyc001_ch001\n1,1,1,1\n{cell_id},2,2,1\n", encoding="utf-8") - - with pytest.raises(ValueError, match=r"1 cell\(s\) whose instance id in 'cell_id' is missing or not an integer"): - _read_codex(path, region="S") - - -def test_build_cells_single_table_and_circles(tmp_path: Path) -> None: - write_fcs(tmp_path / "reg001.fcs", _cells(n=3, region=1)) - write_fcs(tmp_path / "reg002.fcs", _cells(n=4, region=2)) - tables = [_read_codex(tmp_path / f"reg00{r}.fcs") for r in (1, 2)] - translation = Translation([10, 0], axes=("x", "y")) - - with pytest.warns(UserWarning, match=r"No segmentation mask found for region.*circles") as record: - anndatas, shapes = build_cells( - tables, {}, {"reg001": {"reg001": Identity()}, "reg002": {"reg002": translation}} - ) - - assert len(record) == 1 - assert list(anndatas) == ["table"] - table = anndatas["table"] - assert table.n_obs == 7 - assert table.obs_names.is_unique - assert list(table.obs["region"].cat.categories) == ["reg001_cells", "reg002_cells"] - assert table.uns["spatialdata_attrs"]["region"] == ["reg001_cells", "reg002_cells"] - assert table.obs["cell_id"].dtype == np.int64 - assert tables[0].centroids_px is not None - np.testing.assert_allclose(np.asarray(table.obsm["spatial"])[:3], tables[0].centroids_px) - assert sorted(shapes) == ["reg001_cells", "reg002_cells"] - assert list(shapes["reg002_cells"].index) == [1, 2, 3, 4] - assert get_transformation(shapes["reg002_cells"], get_all=True) == {"reg002": translation} - - -def test_build_cells_different_panels_split_tables(tmp_path: Path) -> None: - write_fcs(tmp_path / "reg001.fcs", _cells(region=1, markers=("cyc001_ch001",))) - write_fcs(tmp_path / "reg002.fcs", _cells(region=2, markers=("cyc001_ch001", "cyc001_ch002"))) - tables = [_read_codex(tmp_path / f"reg00{r}.fcs") for r in (1, 2)] - - with pytest.warns(UserWarning, match="circles"): - anndatas, _ = build_cells(tables, {}, {}) - - assert sorted(anndatas) == ["reg001_table", "reg002_table"] - assert anndatas["reg002_table"].n_vars == 2 - assert anndatas["reg002_table"].uns["spatialdata_attrs"]["region"] == ["reg002_cells"] - - -def test_dedupe_matches_akoya_convention(tmp_path: Path) -> None: - write_fcs(tmp_path / "reg001.fcs", _cells(), {"cyc001_ch001": "X", "cyc001_ch002": "X"}) - - assert list(_read_codex(tmp_path / "reg001.fcs").var.index) == dedupe_channel_names(["X", "X"]) + path.write_text("cell_id,x,y,tile_num,CD3,CD8,CD3,label\n1,1.0,2.0,1,5.0,6.0,7.0,T\n") + with pytest.warns(UserWarning, match=r"no 'cycCCC_chHHH' columns.*\['CD3', 'CD8', 'CD3 \(2\)'\]"): + table = read_cell_table(path, default_region="Kidney") -def test_read_codex_fcs_blank_pns_falls_back_to_pnn(tmp_path: Path) -> None: - write_fcs(tmp_path / "reg001.fcs", _cells(), {"cyc001_ch001": " ", "cyc001_ch002": "CD3"}) - - assert list(_read_codex(tmp_path / "reg001.fcs").var.index) == ["cyc001_ch001", "CD3"] - - -@pytest.mark.parametrize("different_panels", [False, True]) -def test_build_cells_overlapping_regions_raise(tmp_path: Path, different_panels: bool) -> None: - write_fcs(tmp_path / "a.fcs", _cells(region=1)) - markers = ("cyc001_ch001",) if different_panels else ("cyc001_ch001", "cyc001_ch002") - write_fcs(tmp_path / "b.fcs", _cells(region=1, markers=markers)) - tables = [_read_codex(tmp_path / name) for name in ("a.fcs", "b.fcs")] - - with pytest.raises(ValueError, match="Several cell tables"): - build_cells(tables, {}, {}) - - -def test_resolve_per_region(tmp_path: Path) -> None: - a, b = tmp_path / "a.csv", tmp_path / "b.csv" - a.touch() - b.touch() - - assert resolve_per_region(None, ["A"]) == {} - assert resolve_per_region(str(a), ["A"]) == {"A": a} - assert resolve_per_region(a, ["A", "B"]) == {"*": a} - assert resolve_per_region({"B": str(b), "A": a}, ["A", "B"]) == {"A": a, "B": b} - with pytest.raises(ValueError, match="'C'"): - resolve_per_region({"C": a}, ["A", "B"]) - with pytest.raises(FileNotFoundError, match="missing.csv"): - resolve_per_region(tmp_path / "missing.csv", ["A"]) - - -def test_read_labels_lazy_with_transformation(tmp_path: Path) -> None: - mask = np.zeros((40, 60), dtype=np.uint32) - mask[5:10, 5:10] = 1 - path = tmp_path / "mask.tif" - tifffile.imwrite(path, mask) - translation = Translation([3, 4], axes=("x", "y")) - - labels = read_labels(path, {"scale_factors": [2]}, {"global": translation}) - - level0 = labels["scale0"]["image"] - assert get_axes_names(labels) == ("y", "x") - np.testing.assert_array_equal(level0.values, mask) - assert get_transformation(labels, "global") == translation - - -def test_read_labels_first_channel(tmp_path: Path) -> None: - masks = np.stack([np.full((16, 24), i + 1, dtype=np.int32) for i in range(2)]) - path = tmp_path / "mask.ome.tif" - tifffile.imwrite(path, masks, metadata={"axes": "CYX", "Channel": {"Name": ["cells", "nuclei"]}}) - - labels = read_labels(path, {"scale_factors": None}, {"global": Identity()}) + assert list(table.var.index) == ["CD3", "CD8", "CD3 (2)"] + assert list(table.obs.columns) == ["x", "y", "tile_num", "label"] + assert list(table.region) == ["Kidney"] - np.testing.assert_array_equal(labels.values, masks[0]) +@pytest.mark.parametrize( + ("content", "match"), + [ + ("cell_id,cyc001_ch001\n1,2.0\n", "'x' and 'y'"), + ("cell_id,x,y,cyc001_ch001\n1,1.0,2.0,2.0\n", "cannot be assigned to regions"), + ("cell_id,region,x,y,cyc001_ch001\n1,1,1,1,1\n1.5,1,2,2,1\n", r"1 cell\(s\) whose instance id in 'cell_id'"), + ( + "cell_id,region,x,y,cyc001_ch001\n1,1,1,1,1\n1,1,2,2,1\n", + r"1 duplicate instance id\(s\) \[1\] in region 'reg001'", + ), + ], + ids=["no_centroids", "no_region", "fractional_id", "duplicate_id"], +) +def test_bad_tables_raise(tmp_path: Path, content: str, match: str) -> None: + (tmp_path / "cells.csv").write_text(content) -def test_read_labels_errors(tmp_path: Path) -> None: - path = tmp_path / "mask.tif" - tifffile.imwrite(path, np.zeros((8, 8), dtype=np.float32)) - with pytest.raises(ValueError, match="integer"): - read_labels(path, {}, {"global": Identity()}) + with pytest.raises(ValueError, match=match): + read_cell_table(tmp_path / "cells.csv", default_region=None) @pytest.mark.parametrize("compression", [None, "zlib"]) -def test_read_labels_pyramidal_ome_is_lazy(tmp_path: Path, compression: str | None) -> None: +def test_read_labels_is_lazy(tmp_path: Path, compression: str | None) -> None: mask = np.repeat(np.arange(64, dtype=np.uint16), 48).reshape(64, 48) path = tmp_path / "mask.ome.tif" tile = (16, 16) if compression else None with tifffile.TiffWriter(path) as tw: - tw.write(mask, subifds=1, tile=tile, compression=compression, metadata={"axes": "YX"}) - tw.write(mask[::2, ::2], subfiletype=1, tile=tile, compression=compression) + tw.write(np.stack([mask, mask + 1]), subifds=1, tile=tile, compression=compression, metadata={"axes": "CYX"}) + tw.write(np.stack([mask, mask + 1])[:, ::2, ::2], subfiletype=1, tile=tile, compression=compression) + translation = Translation([3, 4], axes=("x", "y")) with pytest.MonkeyPatch.context() as monkeypatch: forbid_pixel_reads(monkeypatch) - labels = read_labels(path, {"scale_factors": None}, {"global": Identity()}) + labels = read_labels(path, {"scale_factors": None}, {"global": translation}) - assert labels.shape == (64, 48) + # of several planes, the first np.testing.assert_array_equal(labels.values, mask) + assert get_transformation(labels, "global") == translation + tifffile.imwrite(path, np.zeros((8, 8), dtype=np.float32)) + with pytest.raises(ValueError, match="integer"): + read_labels(path, {}, {"global": Identity()}) @pytest.mark.parametrize(("mask_shape", "factor"), [((32, 40), 2), ((22, 27), 3)]) -def test_mask_integer_downsample_autoscaled(tmp_path: Path, mask_shape: tuple[int, int], factor: int) -> None: - # a mask downsampled by an integer factor (exactly, or with its shape rounded up) is scaled onto the image, - # before the stage translation of its region; the full-resolution mask of the other region is unaffected - image_shape = (64, 80) - write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI"], shape=image_shape, stage_position_cm=(1.0, 2.0)) - write_qptiff(tmp_path / "B_Scan1.qptiff", biomarkers=["DAPI"], shape=image_shape, stage_position_cm=(1.01, 2.0)) - tifffile.imwrite(tmp_path / "A_mask.tif", np.zeros(image_shape, dtype=np.uint32)) - mask = np.zeros(mask_shape, dtype=np.uint32) - mask[2:6, 2:6], mask[10:14, 8:12] = 1, 2 - tifffile.imwrite(tmp_path / "B_mask.tif", mask) - - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - sdata = phenocycler( - tmp_path, - labels={"A": tmp_path / "A_mask.tif", "B": tmp_path / "B_mask.tif"}, - image_models_kwargs={"scale_factors": [2]}, - ) - - assert sdata.coordinate_systems == ["global"] - assert get_transformation(sdata.labels["A_labels"], get_all=True) == get_transformation( - sdata.images["A_image"], get_all=True - ) - image_transformation = get_transformation(sdata.images["B_image"], "global") - assert isinstance(image_transformation, Translation) - labels_transformation = get_transformation(sdata.labels["B_labels"], "global") +def test_downsampled_mask_is_scaled_onto_its_image(tmp_path: Path, mask_shape: tuple[int, int], factor: int) -> None: + # exactly, or with its shape rounded up; before the stage translation of its region + write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"], shape=(64, 80), stage_position_cm=(1.0, 2.0)) + tifffile.imwrite(tmp_path / "mask.tif", np.zeros(mask_shape, dtype=np.uint32)) + + sdata = phenocycler(tmp_path, labels=tmp_path / "mask.tif", image_models_kwargs={"scale_factors": [2]}) + + image_transformation = get_transformation(sdata["S_image"], "global") + labels_transformation = get_transformation(sdata["S_labels"], "global") assert isinstance(labels_transformation, Sequence) assert labels_transformation.transformations == [Scale([factor, factor], axes=("y", "x")), image_transformation] - image_extent = get_extent(sdata.images["B_image"], coordinate_system="global") - labels_extent = get_extent(sdata.labels["B_labels"], coordinate_system="global") - for axis, image_size, mask_size in zip(("y", "x"), image_shape, mask_shape, strict=True): + image_extent = get_extent(sdata["S_image"], coordinate_system="global") + labels_extent = get_extent(sdata["S_labels"], coordinate_system="global") + for axis, mask_size in zip(("y", "x"), mask_shape, strict=True): assert labels_extent[axis][0] == pytest.approx(image_extent[axis][0]) # a mask rounded up overhangs the image by less than one mask pixel - expected_end = image_extent[axis][0] + mask_size * factor - assert labels_extent[axis][1] == pytest.approx(expected_end) - if image_size % mask_size == 0: - assert labels_extent[axis][1] == pytest.approx(image_extent[axis][1]) + assert labels_extent[axis][1] == pytest.approx(image_extent[axis][0] + mask_size * factor) + # writing may reorder the axes of the translation, so compare the composed affine maps sdata.write(tmp_path / "x.zarr") - read = get_transformation(read_zarr(tmp_path / "x.zarr").labels["B_labels"], "global") - assert isinstance(read, Sequence) + read = get_transformation(read_zarr(tmp_path / "x.zarr")["S_labels"], "global") np.testing.assert_allclose( read.to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")), labels_transformation.to_affine_matrix(input_axes=("x", "y"), output_axes=("x", "y")), ) - - -def test_mask_non_integer_mismatch_warns(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"], shape=(64, 80)) - tifffile.imwrite(tmp_path / "mask.tif", np.zeros((48, 60), dtype=np.uint32)) - - with pytest.warns(UserWarning, match=r"\(48, 60\).*\(64, 80\).*not align"): - sdata = phenocycler( - tmp_path, - labels=tmp_path / "mask.tif", - image_models_kwargs={"scale_factors": [2]}, - ) - - assert get_transformation(sdata.labels["S_labels"], "global") == Identity() diff --git a/tests/readers/test_qptiff.py b/tests/readers/test_qptiff.py index d50443dd..06ebf3c2 100644 --- a/tests/readers/test_qptiff.py +++ b/tests/readers/test_qptiff.py @@ -1,5 +1,4 @@ import gc -import json import pickle import warnings from pathlib import Path @@ -8,298 +7,127 @@ import dask.array as da import numpy as np import pytest -import tifffile from tests._akoya_data import cim_experiment_v4_xml, fusion_scan_profile_json, write_qptiff from spatialdata_io.readers._utils import _qptiff from spatialdata_io.readers._utils._qptiff import read_qptiff_image, read_qptiff_metadata - -def test_fusion2_channels_from_biomarker(tmp_path: Path) -> None: - p = tmp_path / "scan.qptiff" - write_qptiff(p, biomarkers=["DAPI", "CD3", "DAPI"], names=["DAPI", "ATTO 550", "DAPI"]) - scan = read_qptiff_metadata(p) - assert [c.name for c in scan.channels] == ["DAPI", "CD3", "DAPI"] - assert scan.channels[1].fluor == "ATTO 550" - assert scan.generation == "fusion2" - assert scan.region == "scan" - assert scan.slide_id == "SLIDE1" - assert scan.software == "Fusion 2.3.1" - assert scan.channels[1].exposure_ms == pytest.approx(450.0) - assert scan.channels[1].color == "255,255,0" - - -def test_opal_mode_uses_name(tmp_path: Path) -> None: - p = tmp_path / "opal.qptiff" - names = ["DAPI", "Opal 480", "Opal 520", "Opal 570", "Opal 620", "Opal 690"] - write_qptiff(p, biomarkers=[None] * 6, names=names) - scan = read_qptiff_metadata(p) - assert [c.name for c in scan.channels] == names - assert all(c.fluor is None for c in scan.channels) - assert scan.generation == "fusion2" +_CIM = "Codex Instrument Manager 1.30.0.12" -def test_fusion1_generation_and_region_override(tmp_path: Path) -> None: - p = tmp_path / "scan.er.qptiff" - write_qptiff(p, biomarkers=["DAPI"], software="Fusion 1.0.5", dtype="uint16") - assert read_qptiff_metadata(p).region == "scan" - scan = read_qptiff_metadata(p, region="reg001") - assert scan.generation == "fusion1" - assert scan.region == "reg001" +@pytest.mark.parametrize( + ("kwargs", "names", "generation"), + [ + ({"biomarkers": ["DAPI", "CD3"], "names": ["DAPI", "ATTO 550"]}, ["DAPI", "CD3"], "fusion2"), + # PhenoImager (Opal) mode has no `` + ({"biomarkers": [None, None], "names": ["DAPI", "Opal 480"]}, ["DAPI", "Opal 480"], "fusion2"), + ({"biomarkers": ["DAPI"], "software": "Fusion 1.0.5"}, ["DAPI"], "fusion1"), + ({"biomarkers": [None], "names": ["FITC"], "software": "VectraPolaris 1.0"}, ["FITC"], "qpi_other"), + ], +) +def test_channel_names_and_generation(tmp_path: Path, kwargs: dict[str, Any], names: list[str], generation: str): + write_qptiff(tmp_path / "S_Scan1.qptiff", **kwargs) + scan = read_qptiff_metadata(tmp_path / "S_Scan1.qptiff") -def test_vectra_is_qpi_other(tmp_path: Path) -> None: - p = tmp_path / "vectra.qptiff" - write_qptiff(p, biomarkers=[None, None], names=["DAPI", "FITC"], software="VectraPolaris 1.0") - assert read_qptiff_metadata(p).generation == "qpi_other" + assert [c.name for c in scan.channels] == names + assert scan.generation == generation -def test_pixel_size_from_xresolution(tmp_path: Path) -> None: +def test_pixel_size_and_stage_position(tmp_path: Path) -> None: p = tmp_path / "x.qptiff" - write_qptiff(p, biomarkers=["DAPI"], resolution_px_per_cm=1e4 / 0.5069) + write_qptiff(p, biomarkers=["DAPI"], stage_position_cm=(1.5, 0.25)) scan = read_qptiff_metadata(p) assert scan.pixel_size_um == pytest.approx(0.5069, rel=1e-6) - assert scan.nominal_pixel_size_um is None - + assert scan.stage_position_um == pytest.approx((15000.0, 2500.0)) -def test_96dpi_resolution_is_unknown(tmp_path: Path) -> None: - p = tmp_path / "x.qptiff" + # 96 dpi is a screen resolution, not a calibration write_qptiff(p, biomarkers=["DAPI"], resolution_px_per_cm=96, resolution_unit="INCH") with pytest.warns(UserWarning, match="Could not determine the pixel size"): - scan = read_qptiff_metadata(p) - assert scan.pixel_size_um is None + assert read_qptiff_metadata(p).pixel_size_um is None -def test_stage_position(tmp_path: Path) -> None: - p = tmp_path / "x.qptiff" - write_qptiff(p, biomarkers=["DAPI"], stage_position_cm=(1.5, 0.25)) - scan = read_qptiff_metadata(p) - assert scan.stage_position_um == pytest.approx((15000.0, 2500.0)) - write_qptiff(p, biomarkers=["DAPI"]) - assert read_qptiff_metadata(p).stage_position_um is None - - -def test_cim_without_label_page_counts_pages_by_imagetype(tmp_path: Path) -> None: +def test_cim_without_label_page_counts_pages_by_image_type(tmp_path: Path) -> None: p = tmp_path / "cim.qptiff" biomarkers: list[str | None] = ["DAPI", "CD3", "CD8", "DAPI", "CD20"] profile = cim_experiment_v4_xml([["DAPI", "CD3", "CD8"], ["DAPI", "CD20"]]) - written = write_qptiff( - p, - biomarkers=biomarkers, - software="Codex Instrument Manager 1.30.0.12", - description_version=3, - scan_profile=profile, - levels=3, - label_page=False, - resolution_px_per_cm=1e4 / 0.37739, - ) - image = read_qptiff_image(read_qptiff_metadata(p)) - assert image.shape == (5, 256, 320) - assert np.array_equal(image.compute(), written) - scan = read_qptiff_metadata(p) - assert scan.generation == "cim_qptiff" - assert [c.name for c in scan.channels] == biomarkers - assert scan.channels[1].cycle == 1 - assert scan.channels[1].channel == 2 - assert scan.channels[1].exposure_ms is not None - assert scan.channels[1].extra["wavelength_nm"] == pytest.approx(558.0) - assert [c.cycle for c in scan.channels] == [1, 1, 1, 2, 2] - assert scan.nominal_pixel_size_um == pytest.approx(0.37739) - - -def test_fusion_scan_profile_enrichment(tmp_path: Path) -> None: - p = tmp_path / "x.qptiff" - wells = [[("DAPI", "DAPI"), ("CD3", "ATTO 550")], [("DAPI", "DAPI"), ("CD8", "AF 750")]] - write_qptiff(p, biomarkers=["DAPI", "CD3", "DAPI", "CD8"], scan_profile=fusion_scan_profile_json(wells)) - scan = read_qptiff_metadata(p) - assert [c.cycle for c in scan.channels] == [1, 1, 2, 2] - assert [c.channel for c in scan.channels] == [1, 2, 1, 2] - assert isinstance(scan.raw["scan_profile"], dict) - - -def test_fusion_reagent_placeholders_are_dropped(tmp_path: Path) -> None: - p = tmp_path / "x.qptiff" - profile = json.loads(fusion_scan_profile_json([[("DAPI", "DAPI"), ("CD3", "ATTO 550")]])) - items = profile["experimentDescription"]["wells"][0]["items"] - items[0].update(reporter="None", barcode="None", clone="None") - items[1].update(reporter="GN65", barcode="GN65", clone="XXX") - write_qptiff(p, biomarkers=["DAPI", "CD3"], scan_profile=json.dumps(profile)) - dapi, cd3 = read_qptiff_metadata(p).channels - assert not {"reporter", "barcode", "clone"} & dapi.extra.keys() - assert (cd3.extra["reporter"], cd3.extra["barcode"], cd3.extra["clone"]) == ("GN65", "GN65", "XXX") - assert cd3.extra["dye"] == "ATTO 550" - - -def test_enrichment_leaves_unmatched_channels_empty(tmp_path: Path) -> None: - p = tmp_path / "x.qptiff" - wells = [[("DAPI", "DAPI"), ("CD3", "ATTO 550")]] - write_qptiff(p, biomarkers=["DAPI", "CD3", "CD4"], scan_profile=fusion_scan_profile_json(wells)) - scan = read_qptiff_metadata(p) - assert [c.name for c in scan.channels] == ["DAPI", "CD3", "CD4"] - assert [c.cycle for c in scan.channels] == [1, 1, None] - - -def test_enrichment_ambiguous_name_left_unset(tmp_path: Path) -> None: - p = tmp_path / "x.qptiff" - wells = [[("DAPI", "DAPI"), ("CD3", "ATTO 550")], [("DAPI", "DAPI"), ("CD8", "AF 750")], [("DAPI", "DAPI")]] - write_qptiff(p, biomarkers=["DAPI", "CD3", "CD8"], scan_profile=fusion_scan_profile_json(wells)) - scan = read_qptiff_metadata(p) - assert [c.cycle for c in scan.channels] == [None, 1, 2] - assert scan.channels[0].channel is None - + written = write_qptiff(p, biomarkers=biomarkers, software=_CIM, scan_profile=profile, levels=3, label_page=False) -def test_enrichment_positional_when_counts_match(tmp_path: Path) -> None: - p = tmp_path / "cim.qptiff" - write_qptiff( - p, - biomarkers=["DAPI", "F4-80", "NK1"], - software="Codex Instrument Manager 1.30.0.12", - scan_profile=cim_experiment_v4_xml([["DAPI", "F4/80"], ["NK1.1", "BLANK"]]), - ) scan = read_qptiff_metadata(p) - assert [c.name for c in scan.channels] == ["DAPI", "F4-80", "NK1"] - assert [(c.cycle, c.channel) for c in scan.channels] == [(1, 1), (1, 2), (2, 1)] - - -def test_enrichment_not_positional_when_a_record_names_another_page(tmp_path: Path) -> None: - # the third record is DAPI, a marker of another page: pairing it with CD4 by position would be a guess - p = tmp_path / "x.qptiff" - wells = [[("DAPI", "DAPI"), ("CD3", "ATTO 550")], [("DAPI", "DAPI")]] - write_qptiff(p, biomarkers=["DAPI", "CD3", "CD4"], scan_profile=fusion_scan_profile_json(wells)) - scan = read_qptiff_metadata(p) - cd4 = scan.channels[2] - assert (cd4.cycle, cd4.channel) == (None, None) - assert (scan.channels[1].cycle, scan.channels[1].channel) == (1, 2) + assert scan.generation == "cim_qptiff" + assert [(c.name, c.cycle, c.channel) for c in scan.channels][:2] == [("DAPI", 1, 1), ("CD3", 1, 2)] + assert scan.nominal_pixel_size_um == pytest.approx(0.37739) + np.testing.assert_array_equal(read_qptiff_image(scan).compute(), written) + + +@pytest.mark.parametrize( + ("biomarkers", "profile", "expected"), + [ + # by name; a page without a record is left unset + (["DAPI", "CD3", "CD4"], [[("DAPI", "DAPI"), ("CD3", "ATTO 550")]], [(1, 1), (1, 2), (None, None)]), + # DAPI is in several cycles, so it is not paired + ( + ["DAPI", "CD3", "CD8"], + [[("DAPI", "D"), ("CD3", "A")], [("DAPI", "D"), ("CD8", "C")]], + [(None, None), (1, 2), (2, 2)], + ), + # names that differ only in non-ASCII letters, written as character references in the page XML + (["TCRγδ", "TCRαβ"], [[("TCRαβ", "A"), ("TCRγδ", "C")]], [(1, 2), (1, 1)]), + ], +) +def test_scan_profile_sets_cycle_and_channel( + tmp_path: Path, biomarkers: list[str | None], profile: list[list[tuple[str, str]]], expected: list[tuple] +) -> None: + p = tmp_path / "x.qptiff" + write_qptiff(p, biomarkers=biomarkers, scan_profile=fusion_scan_profile_json(profile)) + + assert [(c.cycle, c.channel) for c in read_qptiff_metadata(p).channels] == expected + + +@pytest.mark.parametrize( + "kwargs", + [ + {"biomarkers": ["DAPI", "CD3", "CD8"]}, # three channels are not RGB + {"biomarkers": ["DAPI"], "dtype": "uint16"}, + {"biomarkers": [None], "names": ["BF"], "rgb": True, "software": "VectraPolaris 1.0"}, + {"biomarkers": ["DAPI", "CD3"], "bigtiff": False, "tile": None}, + ], + ids=["multiplex", "uint16", "brightfield", "classic_stripped"], +) +def test_image_is_lazy_and_correct(tmp_path: Path, kwargs: dict[str, Any]) -> None: + p = tmp_path / "x.qptiff" + written = write_qptiff(p, **kwargs) -def test_enrichment_not_positional_when_orders_differ(tmp_path: Path) -> None: - p = tmp_path / "x.qptiff" - wells = [[("DAPI", "DAPI"), ("CD3", "ATTO 550"), ("CD8", "CY5"), ("CD20", "AF 750")]] - write_qptiff(p, biomarkers=["DAPI", "CD3", "CD20", "CD8"], scan_profile=fusion_scan_profile_json(wells)) scan = read_qptiff_metadata(p) - assert [c.channel for c in scan.channels] == [1, 2, 4, 3] - + image = read_qptiff_image(scan) -def test_enrichment_keeps_non_ascii_names_apart(tmp_path: Path) -> None: - # names that differ only in non-ASCII letters must not be paired with each other's records; the ASCII page - # descriptions hold them as character references - p = tmp_path / "x.qptiff" - wells = [[("DAPI", "DAPI"), ("TCRαβ", "ATTO 550"), ("TCRγδ", "CY5")]] - biomarkers: list[str | None] = ["DAPI", "TCRγδ", "TCRαβ"] - write_qptiff(p, biomarkers=biomarkers, scan_profile=fusion_scan_profile_json(wells)) - scan = read_qptiff_metadata(p) - assert [c.name for c in scan.channels] == ["DAPI", "TCRγδ", "TCRαβ"] - assert [c.channel for c in scan.channels] == [1, 3, 2] + assert scan.rgb == kwargs.get("rgb", False) + assert isinstance(image, da.Array) + assert image.dtype == written.dtype + np.testing.assert_array_equal(image.compute(), written) -def test_profile_record_without_marker_skipped(tmp_path: Path) -> None: +@pytest.mark.parametrize("series_match", [True, False]) +def test_image_is_picklable_and_closes_its_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, series_match: bool +) -> None: + # dask.distributed pickles the graph, so it must not hold open file handles; also for the per-page fallback, + # used when tifffile's series do not match the full-resolution pages of the XML p = tmp_path / "x.qptiff" - profile = json.loads(fusion_scan_profile_json([[("DAPI", "DAPI"), ("CD3", "ATTO 550")]])) - del profile["experimentDescription"]["wells"][0]["items"][1]["markerName"] - write_qptiff(p, biomarkers=["DAPI", "CD3"], scan_profile=json.dumps(profile)) - dapi, cd3 = read_qptiff_metadata(p).channels - assert dapi.cycle == 1 - assert cd3.cycle is None - - -@pytest.mark.parametrize(("dtype", "name"), [("uint8", "x.qptiff"), ("uint16", "x.er.qptiff")]) -def test_image_is_lazy_and_correct(tmp_path: Path, dtype: str, name: str) -> None: - p = tmp_path / name - written = write_qptiff(p, biomarkers=["DAPI", "CD3"], dtype=dtype) - arr = read_qptiff_image(read_qptiff_metadata(p)) - assert isinstance(arr, da.Array) - assert arr.chunksize != arr.shape - assert arr.dtype == np.dtype(dtype) - assert np.array_equal(arr.compute(), written) + written = write_qptiff(p, biomarkers=["DAPI", "CD3"]) + if not series_match: + real = _qptiff.lazy_tiff_array + def wrong_series(path: Path, *, series: int = 0, page: int | None = None) -> da.Array: + return real(path, series=series, page=page) if page is not None else da.zeros((1, 1, 1)) -def test_image_is_picklable_and_closes_its_file(tmp_path: Path) -> None: - # dask.distributed and process-based schedulers pickle the graph, so it must not hold open file handles - p = tmp_path / "x.qptiff" - written = write_qptiff(p, biomarkers=["DAPI", "CD3"]) + monkeypatch.setattr(_qptiff, "lazy_tiff_array", wrong_series) with warnings.catch_warnings(record=True) as record: warnings.simplefilter("always", ResourceWarning) - arr = read_qptiff_image(read_qptiff_metadata(p)) - restored = pickle.loads(pickle.dumps(arr)) - np.testing.assert_array_equal(restored[0, :8, :8].compute(), written[0, :8, :8]) - np.testing.assert_array_equal(arr.compute(), written) - del arr, restored + image = read_qptiff_image(read_qptiff_metadata(p)) + restored = pickle.loads(pickle.dumps(image)) + np.testing.assert_array_equal(restored.compute(), written) + del image, restored gc.collect() assert not [warning for warning in record if issubclass(warning.category, ResourceWarning)] - - -def test_image_pages_are_picklable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - # the per-page fallback, used when tifffile's series do not match the full-resolution pages of the XML - p = tmp_path / "x.qptiff" - written = write_qptiff(p, biomarkers=["DAPI", "CD3"]) - real = _qptiff.lazy_tiff_array - - def wrong_series(path: Path, *, series: int = 0, page: int | None = None) -> da.Array: - return real(path, series=series, page=page) if page is not None else da.zeros((1, 1, 1)) - - monkeypatch.setattr(_qptiff, "lazy_tiff_array", wrong_series) - arr = read_qptiff_image(read_qptiff_metadata(p)) - assert arr.shape == written.shape - np.testing.assert_array_equal(pickle.loads(pickle.dumps(arr))[1, :8, :8].compute(), written[1, :8, :8]) - - -def test_image_does_not_parse_the_page_xml_again(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - p = tmp_path / "x.qptiff" - written = write_qptiff(p, biomarkers=["DAPI", "CD3"]) - scan = read_qptiff_metadata(p) - parsed = [] - parse = _qptiff._parse_description - - def counting_parse(description: str) -> Any: - parsed.append(description) - return parse(description) - - monkeypatch.setattr(_qptiff, "_parse_description", counting_parse) - - image = read_qptiff_image(scan) - - assert not parsed - np.testing.assert_array_equal(image.compute(), written) - - -def test_three_channel_not_rgb(tmp_path: Path) -> None: - p = tmp_path / "x.qptiff" - written = write_qptiff(p, biomarkers=["DAPI", "CD3", "CD8"]) - scan = read_qptiff_metadata(p) - assert scan.rgb is False - image = read_qptiff_image(read_qptiff_metadata(p)) - assert image.ndim == 3 - assert image.shape == (3, 256, 320) - assert np.array_equal(image.compute(), written) - - -def test_brightfield_rgb(tmp_path: Path) -> None: - p = tmp_path / "x.qptiff" - written = write_qptiff(p, biomarkers=[None], names=["BF"], rgb=True, software="VectraPolaris 1.0") - scan = read_qptiff_metadata(p) - assert scan.rgb is True - image = read_qptiff_image(read_qptiff_metadata(p)) - assert image.shape == (256, 320, 3) - assert np.array_equal(image.compute(), written) - - -def test_not_qpi_raises(tmp_path: Path) -> None: - p = tmp_path / "x.qptiff" - tifffile.imwrite(p, np.zeros((16, 16), dtype=np.uint8)) - with pytest.raises(ValueError, match="not a PerkinElmer"): - read_qptiff_metadata(p) - - -def test_missing_file_raises(tmp_path: Path) -> None: - with pytest.raises(FileNotFoundError): - read_qptiff_metadata(tmp_path / "missing.qptiff") - - -def test_classic_tiff_and_stripped_level(tmp_path: Path) -> None: - p = tmp_path / "x.qptiff" - written = write_qptiff(p, biomarkers=["DAPI", "CD3"], bigtiff=False, tile=None) - with warnings.catch_warnings(): - warnings.simplefilter("error") - scan = read_qptiff_metadata(p) - assert len(scan.channels) == 2 - assert np.array_equal(read_qptiff_image(read_qptiff_metadata(p)).compute(), written) diff --git a/tests/test_akoya_fixtures.py b/tests/test_akoya_fixtures.py deleted file mode 100644 index 470d7e91..00000000 --- a/tests/test_akoya_fixtures.py +++ /dev/null @@ -1,73 +0,0 @@ -from pathlib import Path - -import numpy as np -import pandas as pd -import readfcs -import tifffile - -from tests._akoya_data import write_fcs, write_legacy_processor_tree, write_qptiff - - -def _page(tf: tifffile.TiffFile, index: int) -> tifffile.TiffPage: - page = tf.pages[index] - assert isinstance(page, tifffile.TiffPage) - return page - - -def test_write_qptiff_is_qpi(tmp_path: Path) -> None: - p = tmp_path / "x.qptiff" - data = write_qptiff(p, biomarkers=["DAPI", "CD3", "CD8"], levels=2) - with tifffile.TiffFile(p) as tf: - assert tf.is_qpi - s = tf.series[0] - assert s.shape == (3, 256, 320) - assert len(s.levels) == 2 - assert s.levels[1].shape == (3, 128, 160) - desc = _page(tf, 0).description - assert "DAPI" in desc - assert 'encoding="utf-16"' in desc - assert "CD3" in _page(tf, 1).description - assert data.shape == (3, 256, 320) - - -def test_write_qptiff_rgb_and_stage_position(tmp_path: Path) -> None: - p = tmp_path / "rgb.qptiff" - data = write_qptiff(p, biomarkers=[None], rgb=True, stage_position_cm=(1.5, 2.5), label_page=False, levels=1) - assert data.shape == (256, 320, 3) - with tifffile.TiffFile(p) as tf: - page = _page(tf, 0) - x, y = (page.tags[t].value for t in ("XPosition", "YPosition")) - assert x[0] / x[1] == 1.5 - assert y[0] / y[1] == 2.5 - - -def test_write_fcs_roundtrip(tmp_path: Path) -> None: - p = tmp_path / "x.fcs" - df = pd.DataFrame({"cell_id": [1.0, 2.0], "cyc001_ch001": [0.5, 1.5]}) - write_fcs(p, df, marker_names={"cyc001_ch001": "DAPI"}) - fcs = readfcs.ReadFCS(p) - # readfcs exposes events with positional integer columns; the $PnN names live in ``channels`` - assert list(fcs.channels["PnN"]) == list(df.columns) - assert fcs.data.shape == df.shape - assert "DAPI" in set(fcs.channels["PnS"]) - np.testing.assert_allclose(fcs.data.iloc[:, 1].to_numpy(), [0.5, 1.5]) - - -def test_legacy_tree_per_marker(tmp_path: Path) -> None: - info = write_legacy_processor_tree(tmp_path, markers=["DAPI", "CD3", "CD8", "CD4", "DAPI", "CD20", "CD68", "CD31"]) - assert (tmp_path / "processed_2020-01-01/stitched/reg001/reg001_cyc001_ch001_DAPI.tif").exists() - assert (tmp_path / "processed_2020-01-01/segm/segm-1/fcs/compensated/reg001_compensated.fcs").exists() - assert (tmp_path / "processed_2020-01-01/segm/segm-1/fcs/compensated/reg001_compensated.csv").exists() - assert len(info["markers"]) == 8 - assert info["images"]["reg001"].shape == (8, 128, 160) - assert len(info["cells"]) == 50 - - -def test_legacy_tree_other_layouts(tmp_path: Path) -> None: - info = write_legacy_processor_tree(tmp_path / "h", layout="hyperstack") - with tifffile.TiffFile(tmp_path / "h/processed_2020-01-01/stitched/reg001_montage.tif") as tf: - assert tf.series[0].shape == (2, 4, 128, 160) - assert info["images"]["reg001"].shape == (2, 4, 128, 160) - write_legacy_processor_tree(tmp_path / "f", layout="old_flat", fcs=False) - assert (tmp_path / "f/exp_reg001.csv").exists() - assert (tmp_path / "f/exp_reg001.tif").exists() diff --git a/tests/test_akoya_real_data.py b/tests/test_akoya_real_data.py index 123e6bbf..31516e2c 100644 --- a/tests/test_akoya_real_data.py +++ b/tests/test_akoya_real_data.py @@ -35,174 +35,67 @@ @dataclass(frozen=True) class Scan: - """What a real dataset is expected to read as.""" + """What a real dataset is expected to read as: ``shape`` starts ``(c, y, x)``, ``names`` the first channels.""" path: str - reader: str region: str generation: str - n_channels: int - first_names: tuple[str, ...] + names: tuple[str, ...] pixel_size_um: float | None - shape: tuple[int, int, int] | None = None + shape: tuple[int, ...] dtype: str | None = None +# one per generation and file format _SCANS = [ Scan( - "figshare_25123538/AML2_183_Scan1.qptiff", - "phenocycler", - "AML2_183", - "fusion1", - 54, - ("DAPI", "NPM1C", "ASMA", "CD4"), - 0.5069, - (54, 10800, 7680), - ), - Scan( - "zenodo_7916784/DON126-B3A.qptiff", - "phenocycler", - "DON126-B3A", - "fusion1", - 28, - ("DAPI", "GCG", "CD45", "CD19"), - 0.5079, - ), - Scan( - "zenodo_16732565/VI_CODEX_12_Scan1.qptiff", - "phenocycler", - "VI_CODEX_12", - "fusion2", - 15, - ("DAPI", "GATA6", "S100a2", "LamininG"), - 0.5062, - ), - Scan( - "zenodo_14741083/EWS_Akoya_Slide6.qptiff", - "phenocycler", - "EWS_Akoya_Slide6", - "fusion2", - 42, - ("DAPI", "WGA", "Na/K ATPase", "CD11c"), - 0.5075, + "figshare_25123538/AML2_183_Scan1.qptiff", "AML2_183", "fusion1", ("DAPI", "NPM1C"), 0.5069, (54, 10800, 7680) ), # Fusion 2.x in PhenoImager (Opal) mode: the names come from ``, there is no `` - Scan( - "bia_S-BIAD3467/Tonsil-ctrl_Scan1.qptiff", - "phenocycler", - "Tonsil-ctrl", - "fusion2", - 6, - ("DAPI", "Opal 690", "Opal 620", "Opal 780", "Opal 520", "Sample AF"), - 0.4994, - ), - Scan( - "figshare_31956057/20210825-ipLN-9-hpi-VSV.qptiff", - "codex", - "20210825-ipLN-9-hpi-VSV", - "cim_qptiff", - 37, - ("DAPI", "Runx2", "CXCR4", "VSV"), - 0.3774, - ), + Scan("bia_S-BIAD3467/Tonsil-ctrl_Scan1.qptiff", "Tonsil-ctrl", "fusion2", ("DAPI", "Opal 690"), 0.4994, (6,)), # tifffile's QPI series builder finds only 3 of the 5 levels; the full-resolution shape must still be right Scan( "zenodo_22107401/Arlene_Tonsil_9x9.qptiff", - "codex", "Arlene_Tonsil_9x9", "cim_qptiff", - 43, - ("DAPI-01", "HLA-A", "CD20", "Vimentin"), + ("DAPI-01",), 0.3774, (43, 9508, 12672), ), # CODEX Uploader montage, TCYX (21, 4, y, x); the names come from `channelNames.txt` - Scan( - "bia_S-BIAD859/stitched/6371v1", - "codex", - "reg001", - "legacy_processor", - 84, - ("HOECHST", "CollIV", "NaKATPase", "HLA-ABC"), - None, - (84, 3528, 3360), - ), + Scan("bia_S-BIAD859/stitched/6371v1", "reg001", "legacy_processor", ("HOECHST", "CollIV"), None, (84, 3528, 3360)), # CODEX Processor per-marker stitched images `reg001_cycCCC_chHHH_.tiff` Scan( "zenodo_7742474/human_pancreas_codex_zip_subset", - "codex", "reg001", "legacy_processor", - 2, ("Hoechst", "CGC"), None, (2, 18440, 25816), ), - # Vectra / PhenoImager (generation `qpi_other`): Opal, unmixed uint16, JPEG, brightfield RGB and inForm float32 - Scan( - "zenodo_15265351/P074_Scan1.qptiff", - "phenocycler", - "P074", - "qpi_other", - 8, - ("DAPI", "Opal 570", "Opal 690", "Opal 480"), - 0.4984, - (8, 17280, 17280), - "uint8", - ), - Scan( - "ome_vectra_qptiff/LuCa-7color_Scan1.qptiff", - "phenocycler", - "LuCa-7color", - "qpi_other", - 5, - ("DAPI", "FITC", "CY3", "Texas Red", "CY5"), - 0.498, - (5, 34560, 24960), - "uint8", - ), + # Vectra / PhenoImager (generation `qpi_other`): unmixed uint16, brightfield RGB and inForm float32 component data Scan( "zenodo_22107401/Panel33C_Scan2.unmixed.qptiff", - "phenocycler", "Panel33C_Scan2.unmixed", "qpi_other", - 8, - ("CD21", "CD23", "CD20", "CD4", "CK"), + ("CD21",), 0.4965, (8, 36000, 24960), "uint16", ), - # a third-party (KFBIO) scanner writing JPEG-compressed QPI - Scan( - "zenodo_16569043/2025-07-15_13_13_34.qptiff", - "phenocycler", - "2025-07-15_13_13_34", - "qpi_other", - 3, - ("DAPI", "520", "620"), - 0.25, - (3, 71472, 61440), - "uint8", - ), - # brightfield: one RGB page read as the channels r, g, b Scan( "ome_vectra_qptiff/HandEcompressed_Scan1.qptiff", - "phenocycler", "HandEcompressed", "qpi_other", - 3, ("r", "g", "b"), 0.4989, (3, 26640, 30720), "uint8", ), - # inForm component data: a single `.tif` file with float32 pages Scan( "ome_vectra_qptiff/LuCa-7color_[13860,52919]_1x1component_data.tif", - "phenocycler", "LuCa-7color__13860_52919__1x1component_data", "qpi_other", - 8, ("PDL1 (Opal 520)",), 0.498, (8, 1400, 1868), @@ -216,31 +109,29 @@ def test_real_scan(scan: Scan) -> None: path = Path(_ROOT) / scan.path # type: ignore[arg-type] if not path.exists(): pytest.skip(f"{path} is missing") + reader = codex if scan.generation in ("cim_qptiff", "legacy_processor") else phenocycler - sdata = (phenocycler if scan.reader == "phenocycler" else codex)(path) + sdata = reader(path) image = sdata.images[f"{scan.region}_image"] data = image["scale0"]["image"].data assert isinstance(data, da.Array) - assert data.shape[0] == scan.n_channels - if scan.shape is not None: - assert data.shape == scan.shape - if scan.dtype is not None: - assert data.dtype == scan.dtype + assert data.shape[: len(scan.shape)] == scan.shape + assert scan.dtype is None or data.dtype == scan.dtype names = get_channel_names(image) - assert tuple(names[: len(scan.first_names)]) == scan.first_names + assert tuple(names[: len(scan.names)]) == scan.names assert len(set(names)) == len(names) metadata = sdata.attrs["akoya"]["regions"][scan.region] assert metadata["generation"] == scan.generation # a brightfield scan has one RGB page, which is read as three channels - assert len(metadata["channels"]) == (1 if names == ["r", "g", "b"] else scan.n_channels) + assert len(metadata["channels"]) == (1 if names == ["r", "g", "b"] else scan.shape[0]) if scan.pixel_size_um is None: assert metadata["pixel_size_um"] is None else: assert metadata["pixel_size_um"] == pytest.approx(scan.pixel_size_um, abs=1e-3) - assert sdata.attrs["spatialdata_io_reader"] == scan.reader + assert sdata.attrs["spatialdata_io_reader"] == reader.__name__ if path.is_file(): - _assert_tile_matches_tifffile(path, data, channel=scan.n_channels - 1) + _assert_tile_matches_tifffile(path, data, channel=scan.shape[0] - 1) def _assert_tile_matches_tifffile(path: Path, data: da.Array, channel: int) -> None: diff --git a/tests/test_codex.py b/tests/test_codex.py index da8932a3..b9ed8c07 100644 --- a/tests/test_codex.py +++ b/tests/test_codex.py @@ -13,13 +13,12 @@ import tifffile from spatialdata import read_zarr from spatialdata.models import get_channel_names -from spatialdata.transformations import Identity, get_transformation +from spatialdata.transformations import Identity, Scale, Sequence, get_transformation from spatialdata_io import codex, phenocycler from spatialdata_io.readers._utils._akoya import dedupe_channel_names -from spatialdata_io.readers.codex import _discover_legacy, _read_legacy_image +from spatialdata_io.readers.codex import _discover_legacy from tests._akoya_data import ( - cim_experiment_v4_xml, forbid_pixel_reads, write_fcs, write_legacy_processor_tree, @@ -36,254 +35,160 @@ def _level0(sdata, name): # type: ignore[no-untyped-def] return sdata.images[name]["scale0"]["image"] -def test_codex_per_marker_images(tmp_path: Path) -> None: - data = write_legacy_processor_tree(tmp_path, regions=2, layout="per_marker") +def _codex(path: Path | str, **kwargs): # type: ignore[no-untyped-def] + """`codex()` with fast pyramids, ignoring the warning that cells without a mask are circles.""" + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", "No segmentation mask found", UserWarning) + return codex(path, image_models_kwargs=_FAST, labels_models_kwargs=_FAST, **kwargs) + + +@pytest.mark.parametrize("layout", ["per_marker", "hyperstack"]) +def test_codex_images_are_lazy_and_named(tmp_path: Path, layout: Literal["per_marker", "hyperstack"]) -> None: + data = write_legacy_processor_tree(tmp_path, regions=2, layout=layout) - sdata = codex(tmp_path, image_models_kwargs=_FAST) + with pytest.MonkeyPatch.context() as monkeypatch: + forbid_pixel_reads(monkeypatch) + sdata = _codex(tmp_path) assert sorted(sdata.images) == ["reg001_image", "reg002_image"] for region in ("reg001", "reg002"): - image = sdata.images[f"{region}_image"] - assert get_channel_names(image) == dedupe_channel_names(data["markers"]) - np.testing.assert_array_equal(_level0(sdata, f"{region}_image").values, data["images"][region]) + assert get_channel_names(sdata.images[f"{region}_image"]) == dedupe_channel_names(data["markers"]) + expected = data["images"][region] + np.testing.assert_array_equal(_level0(sdata, f"{region}_image").values, expected.reshape(-1, 128, 160)) attrs = sdata.attrs["akoya"]["regions"][region] assert attrs["generation"] == "legacy_processor" - assert attrs["pixel_size_um"] is None - assert [(c["cycle"], c["channel"]) for c in attrs["channels"][:5]] == [(1, 1), (1, 2), (1, 3), (1, 4), (2, 1)] + assert [(c["cycle"], c["channel"]) for c in attrs["channels"][3:5]] == [(1, 4), (2, 1)] assert sdata.attrs["spatialdata_io_reader"] == "codex" -def test_codex_hyperstack_channel_names_txt(tmp_path: Path) -> None: - data = write_legacy_processor_tree(tmp_path, cycles=2, channels=4, layout="hyperstack") +@pytest.mark.parametrize( + ("labels", "names_txt", "expected"), + [ + # ImageJ labels win over channelNames.txt + (["L0", "L1", "L2", "L3"], "o0\no1\no2\no3\n", ["L0", "L1", "L2", "L3"]), + # ImageJ writes an empty label for every unlabelled slice; such labels do not replace channelNames.txt + (["cyc1.tif", "", "", ""], "DAPI\nCD3\nDAPI2\nCD8\n", ["DAPI", "CD3", "DAPI2", "CD8"]), + (None, "DAPI\nIFNµ\nCD3\nCD8\n".encode("utf-16"), ["DAPI", "IFNµ", "CD3", "CD8"]), + (None, "DAPI\nIFNµ\nCD3\nCD8\n".encode("cp1252"), ["DAPI", "IFNµ", "CD3", "CD8"]), + # a channelNames.txt of another length is ignored with a warning + (None, "DAPI\nCD3\n", ["cyc001_ch001", "cyc001_ch002", "cyc002_ch001", "cyc002_ch002"]), + ], + ids=["imagej_labels", "blank_imagej_labels", "utf16", "cp1252", "wrong_length"], +) +def test_codex_hyperstack_channel_names( + tmp_path: Path, labels: list[str] | None, names_txt: str | bytes, expected: list[str] +) -> None: + metadata = {"axes": "TCYX"} if labels is None else {"axes": "TCYX", "Labels": labels} + tifffile.imwrite( + tmp_path / "reg001_montage.tif", np.zeros((2, 2, 16, 16), np.uint16), imagej=True, metadata=metadata + ) + names = tmp_path / "channelNames.txt" + names.write_bytes(names_txt) if isinstance(names_txt, bytes) else names.write_text(names_txt) - sdata = codex(str(tmp_path), image_models_kwargs=_FAST) + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + sdata = _codex(tmp_path) - assert list(sdata.images) == ["reg001_image"] - image = _level0(sdata, "reg001_image") - assert image.shape[0] == 8 - assert get_channel_names(sdata.images["reg001_image"]) == dedupe_channel_names(data["markers"]) - np.testing.assert_array_equal(image.values, data["images"]["reg001"].reshape(8, *image.shape[1:])) + assert get_channel_names(sdata.images["reg001_image"]) == expected + assert any("channelNames.txt" in str(w.message) for w in record) == (expected[0] == "cyc001_ch001") -@pytest.mark.parametrize("from_subdirectory", [False, True, "relative"]) -def test_codex_sidecar_names_are_case_insensitive( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, from_subdirectory: bool | str -) -> None: +@pytest.mark.parametrize("from_subdirectory", [False, True]) +def test_codex_sidecars_case_insensitive_and_above_path(tmp_path: Path, from_subdirectory: bool) -> None: # SPACEc and the HuBMAP codex-pipeline write `channelnames.txt`; older exports `Experiment.json` - data = write_legacy_processor_tree(tmp_path, cycles=2, channels=4, layout="hyperstack") + data = write_legacy_processor_tree(tmp_path, cycles=2, channels=4, layout="hyperstack", fcs=False, csv=False) (tmp_path / "channelNames.txt").rename(tmp_path / "channelnames.txt") (tmp_path / "experiment.json").unlink() (tmp_path / "Experiment.json").write_text(json.dumps({"xyResolution": 377.5})) - source: Path | str = tmp_path / "processed_2020-01-01" / "stitched" if from_subdirectory else tmp_path - if from_subdirectory == "relative": - monkeypatch.chdir(source) - source = "." - sdata = codex(source, image_models_kwargs=_FAST) + sdata = _codex(tmp_path / "processed_2020-01-01" / "stitched" if from_subdirectory else tmp_path) assert get_channel_names(sdata.images["reg001_image"]) == dedupe_channel_names(data["markers"]) assert sdata.attrs["akoya"]["regions"]["reg001"]["pixel_size_um"] == pytest.approx(0.3775) -@pytest.mark.parametrize("encoding", ["utf-8-sig", "utf-16", "cp1252"]) -def test_codex_channel_names_txt_encodings(tmp_path: Path, encoding: str) -> None: - write_legacy_processor_tree(tmp_path, cycles=1, channels=2, layout="hyperstack", fcs=False, csv=False) - (tmp_path / "channelNames.txt").write_text("DAPI\nIFNµ\n", encoding=encoding) - - sdata = codex(tmp_path, image_models_kwargs=_FAST) - - assert get_channel_names(sdata.images["reg001_image"]) == ["DAPI", "IFNµ"] - - -def test_codex_hyperstack_bad_channel_names_warns(tmp_path: Path) -> None: - data = write_legacy_processor_tree(tmp_path, cycles=2, channels=4, layout="hyperstack") - (tmp_path / "channelNames.txt").write_text("\n".join(data["markers"][:7]) + "\n") - - with pytest.warns(UserWarning, match="channelNames.txt"): - sdata = codex(tmp_path, image_models_kwargs=_FAST) +def test_codex_regions_channel_names_and_positional_call(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, regions=2, cycles=1, channels=2, fcs=False, csv=False) - names = get_channel_names(sdata.images["reg001_image"]) - assert names[:5] == ["cyc001_ch001", "cyc001_ch002", "cyc001_ch003", "cyc001_ch004", "cyc002_ch001"] - assert len(names) == 8 + sdata = _codex(tmp_path, regions=["reg002"], channel_names=["a", "b"]) - -def test_codex_hyperstack_imagej_labels_win(tmp_path: Path) -> None: - labels = [f"L{i}" for i in range(6)] - tifffile.imwrite( - tmp_path / "reg001_X01_Y01_Z01.tif", - np.zeros((3, 2, 32, 48), dtype=np.uint16), - imagej=True, - metadata={"axes": "ZCYX", "Labels": labels}, - ) - (tmp_path / "channelNames.txt").write_text("\n".join(f"other{i}" for i in range(6))) - - sdata = codex(tmp_path, image_models_kwargs=_FAST) - - assert get_channel_names(sdata.images["reg001_image"]) == labels - - -@pytest.mark.parametrize("labels", [["", "", "", ""], ["cyc1.tif", "", "", ""]], ids=["blank", "partly_blank"]) -def test_codex_hyperstack_blank_imagej_labels_ignored(tmp_path: Path, labels: list[str]) -> None: - # ImageJ writes an empty label for every unlabelled slice; such labels must not replace channelNames.txt - tifffile.imwrite( - tmp_path / "reg001_montage.tif", - np.zeros((4, 32, 48), dtype=np.uint16), - imagej=True, - metadata={"axes": "CYX", "Labels": labels}, - ) - (tmp_path / "channelNames.txt").write_text("DAPI\nCD3\nDAPI2\nCD8\n") - - sdata = codex(tmp_path, image_models_kwargs=_FAST) - - assert get_channel_names(sdata.images["reg001_image"]) == ["DAPI", "CD3", "DAPI2", "CD8"] - - -def test_codex_cim_qptiff(tmp_path: Path) -> None: - path = tmp_path / "experiment_reg001.qptiff" - data = write_qptiff( - path, - biomarkers=["DAPI", "CD3", "DAPI", "CD20"], - software="Codex Instrument Manager 1.30.0.12", - scan_profile=cim_experiment_v4_xml([["DAPI", "CD3"], ["DAPI", "CD20"]]), - label_page=False, - ) - - for source in (tmp_path, path): - sdata = codex(source, image_models_kwargs=_FAST) - assert list(sdata.images) == ["reg001_image"] - assert get_channel_names(sdata.images["reg001_image"]) == ["DAPI", "CD3", "DAPI (2)", "CD20"] - np.testing.assert_array_equal(_level0(sdata, "reg001_image").values, data) - assert sdata.attrs["akoya"]["regions"]["reg001"]["generation"] == "cim_qptiff" + assert list(sdata.images) == ["reg002_image"] + assert get_channel_names(sdata.images["reg002_image"]) == ["a", "b"] + assert sdata.coordinate_systems == ["global"] + # `codex(path, fcs, imread_kwargs)` is the signature of earlier releases; `imread_kwargs` is ignored + with pytest.warns(UserWarning, match="imread_kwargs"): + assert sorted(codex(tmp_path, True, {"nframes": 1}).images) == ["reg001_image", "reg002_image"] + for kwargs, error in (({"regions": ["reg003"]}, "reg003"), ({"channel_names": ["a"]}, "channel_names")): + with pytest.raises(ValueError, match=error): + codex(tmp_path, **kwargs) + with pytest.raises(FileNotFoundError, match="reg"): + codex(tmp_path / "processed_2020-01-01" / "segm") -def test_codex_and_phenocycler_name_a_qptiff_alike(tmp_path: Path) -> None: - write_qptiff(tmp_path / "Tonsil_Scan1.qptiff", biomarkers=["DAPI"], dtype="uint8") +def test_codex_and_phenocycler_name_qptiffs_alike(tmp_path: Path) -> None: + write_qptiff(tmp_path / "exp_reg001.qptiff", biomarkers=["DAPI"], dtype="uint8") + write_qptiff(tmp_path / "exp_reg001.er.qptiff", biomarkers=["DAPI"], dtype="uint16") write_qptiff(tmp_path / "Tonsil_Scan1.er.qptiff", biomarkers=["DAPI"], dtype="uint16") + write_qptiff(tmp_path / "Preg12_Scan1.qptiff", biomarkers=["DAPI"]) for reader in (codex, phenocycler): sdata = reader(tmp_path, image_models_kwargs=_FAST) - assert list(sdata.images) == ["Tonsil_image"] - assert sdata.attrs["akoya"]["regions"]["Tonsil"]["path"].endswith("Tonsil_Scan1.er.qptiff") - - -def test_reader_modules_export_only_the_readers() -> None: - from spatialdata_io.readers import codex as codex_module - from spatialdata_io.readers import phenocycler as phenocycler_module - - assert codex_module.__all__ == ["codex"] - assert phenocycler_module.__all__ == ["phenocycler"] - - -def test_codex_old_flat_layout_image(tmp_path: Path) -> None: - data = write_legacy_processor_tree(tmp_path, layout="old_flat") - - sdata = codex(tmp_path, image_models_kwargs=_FAST) - - assert list(sdata.images) == ["reg001_image"] - # only the first plane of the stack is written to disk - np.testing.assert_array_equal(_level0(sdata, "reg001_image").values, data["images"]["reg001"][:1]) - - -def test_codex_empty_dir_raises(tmp_path: Path) -> None: - with pytest.raises(FileNotFoundError, match="reg"): - codex(str(tmp_path)) - - -def test_codex_ignores_raw_tiles(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path, layout="per_marker") - expected = sorted(codex(tmp_path, image_models_kwargs=_FAST).images) - processed = tmp_path / "processed_2020-01-01" - plane = np.zeros((16, 16), dtype=np.uint16) - for raw in ( - tmp_path / "cyc001_reg001" / "1_00001_Z001_CH1.tif", - tmp_path / "cyc001_reg002" / "reg002_X01_Y01_Z01.tif", - processed / "tiles" / "reg002_X01_Y01" / "reg002_X01_Y01_t001_z001_c001.tif", - processed / "bestFocus" / "reg002_X01_Y01_Z08.tif", - ): - raw.parent.mkdir(parents=True, exist_ok=True) - tifffile.imwrite(raw, plane) - - assert sorted(codex(tmp_path, image_models_kwargs=_FAST).images) == expected - - -def test_codex_skips_hidden_files_and_follows_symlinks(tmp_path: Path) -> None: - root = tmp_path / "exp" - write_legacy_processor_tree(root, regions=2, layout="per_marker", fcs=True, csv=False, n_cells=5) - expected = _discover_legacy(root, fcs=True) - stitched = root / "processed_2020-01-01" / "stitched" - # macOS AppleDouble files and copies in __MACOSX/ and .Trash-1000/ - for file in [*stitched.rglob("*.tif"), *root.rglob("*.fcs")]: - (file.parent / f"._{file.name}").write_bytes(b"AppleDouble") - for folder in ("__MACOSX", ".Trash-1000"): - shutil.copytree(stitched, root / folder / "stitched") - # the stitched images moved to another volume, and a link back up the tree - shutil.move(stitched, tmp_path / "scratch") - stitched.symlink_to(tmp_path / "scratch", target_is_directory=True) - (root / "loop").symlink_to(root, target_is_directory=True) - - layout = _discover_legacy(root, fcs=True) - - assert layout.tables == expected.tables - assert {region: [file.name for file in files] for region, files in layout.images.items()} == { - region: [file.name for file in files] for region, files in expected.images.items() - } - - -def test_codex_regions_channel_names_and_pixel_size(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path, regions=2, cycles=1, channels=2, layout="per_marker") - (tmp_path / "experiment.json").write_text(json.dumps({"xyResolution": 377.5, "numChannels": 2})) - - sdata = codex(tmp_path, regions=["reg002"], channel_names=["a", "b"], image_models_kwargs=_FAST) + assert sorted(sdata.images) == ["Preg12_image", "Tonsil_image", "reg001_image"] + assert sdata.attrs["akoya"]["regions"]["reg001"]["path"].endswith("exp_reg001.er.qptiff") + assert list(reader(tmp_path / "exp_reg001.qptiff").images) == ["reg001_image"] - assert list(sdata.images) == ["reg002_image"] - assert get_channel_names(sdata.images["reg002_image"]) == ["a", "b"] - region = sdata.attrs["akoya"]["regions"]["reg002"] - assert region["pixel_size_um"] == pytest.approx(0.3775) - assert region["raw"]["experiment_json"]["numChannels"] == 2 - with pytest.raises(ValueError, match="reg003"): - codex(tmp_path, regions=["reg003"]) - with pytest.raises(ValueError, match="channel_names"): - codex(tmp_path, channel_names=["a"]) + write_qptiff(tmp_path / "other_reg1.qptiff", biomarkers=["DAPI"]) + for reader in (codex, phenocycler): + with pytest.raises(ValueError, match=r"Several qptiff files map to the same region\(s\) \['reg001'\]"): + reader(tmp_path) -def test_discover_legacy_tables_prefer_requested_format(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path, regions=2, layout="per_marker", fcs=True, csv=True) +def test_codex_region_ids_and_ignored_files(tmp_path: Path) -> None: + for name in ("reg001_montage.tif", "exp_reg002.tif", "1_reg003.tif", "Treg2.tif", "coreg1.tif"): + tifffile.imwrite(tmp_path / name, np.zeros((1, 4, 4), dtype=np.uint8)) + # raw acquisition and intermediate folders, macOS AppleDouble files and a link back up the tree + for raw in ("cyc001_reg004/1_00001_Z001_CH1.tif", "tiles/reg005_X01_Y01.tif", "__MACOSX/reg006.tif"): + (tmp_path / raw).parent.mkdir() + tifffile.imwrite(tmp_path / raw, np.zeros((4, 4), dtype=np.uint8)) + (tmp_path / "._reg007.tif").write_bytes(b"AppleDouble") + (tmp_path / "loop").symlink_to(tmp_path, target_is_directory=True) - assert {region: path.suffix for region, path in _discover_legacy(tmp_path, fcs=True).tables.items()} == { - "reg001": ".fcs", - "reg002": ".fcs", - } - assert {region: path.suffix for region, path in _discover_legacy(tmp_path, fcs=False).tables.items()} == { - "reg001": ".csv", - "reg002": ".csv", - } + assert sorted(_discover_legacy(tmp_path, fcs=True).images) == ["reg001", "reg002", "reg003"] -def test_discover_legacy_tables_ranking(tmp_path: Path) -> None: +def test_codex_discovers_tables(tmp_path: Path) -> None: cells = pd.DataFrame({"cell_id": [1.0, 2.0], "x": [1.0, 2.0], "y": [3.0, 4.0], "cyc001_ch001": [5.0, 6.0]}) tifffile.imwrite(tmp_path / "Kidney.tif", np.zeros((2, 16, 16), dtype=np.uint16)) cells.to_csv(tmp_path / "Kidney.csv", index=False) - # an unrelated flow cytometry tube and a file that is no FCS at all do not replace the CODEX table + # not cell tables: sidecar CSVs (also named after a region), flow cytometry tubes and broken FCS files (tmp_path / "flow").mkdir() write_fcs(tmp_path / "flow" / "tube1.fcs", pd.DataFrame({"FSC-A": [1.0], "SSC-A": [2.0]})) (tmp_path / "flow" / "broken.fcs").write_bytes(b"not an fcs file") + # flowio raises an EOFError and a KeyError for these + write_fcs(tmp_path / "flow" / "reg001.fcs", cells) + content = (tmp_path / "flow" / "reg001.fcs").read_bytes() + (tmp_path / "flow" / "reg001.fcs").write_bytes(content[:100]) + (tmp_path / "flow" / "reg002.fcs").write_bytes(content.replace(b"$DATATYPE", b"$DATATYPX")) + (tmp_path / "reg001_antibodies.csv").write_text("Antibody,Clone\nCD3,UCHT1\n") + (tmp_path / "xy.csv").write_text("x,y\n1.0,2.0\n") assert _discover_legacy(tmp_path, fcs=True).tables == {"*": tmp_path / "Kidney.csv"} # a CODEX gate export next to the table: the reader cannot tell which one holds the cells - (tmp_path / "gates").mkdir() - write_fcs(tmp_path / "gates" / "CD3pos.fcs", cells.iloc[:1]) - with pytest.raises(ValueError, match=r"CD3pos\.fcs.*Kidney\.csv|Kidney\.csv.*CD3pos\.fcs"): + write_fcs(tmp_path / "flow" / "CD3pos.fcs", cells.iloc[:1]) + with pytest.raises(ValueError, match="cannot choose"): _discover_legacy(tmp_path, fcs=True) - # compensated beats uncompensated before the file format + # compensated beats uncompensated before the file format; `fcs` picks the format of one table fcs_dir = tmp_path / "exp" / "segm" / "segm-1" / "fcs" for kind in ("compensated", "uncompensated"): (fcs_dir / kind).mkdir(parents=True) cells.to_csv(fcs_dir / "compensated" / "reg001_compensated.csv", index=False) + write_fcs(fcs_dir / "compensated" / "reg001_compensated.fcs", cells) write_fcs(fcs_dir / "uncompensated" / "reg001_uncompensated.fcs", cells) - assert _discover_legacy(tmp_path / "exp", fcs=True).tables == { - "reg001": fcs_dir / "compensated" / "reg001_compensated.csv" - } + for fcs, suffix in ((True, ".fcs"), (False, ".csv")): + assert _discover_legacy(tmp_path / "exp", fcs=fcs).tables == { + "reg001": fcs_dir / f"compensated/reg001_compensated{suffix}" + } # two segmentation runs of one region are not resolved by their path segm_10 = tmp_path / "exp" / "segm" / "segm-10" / "fcs" / "compensated" @@ -293,585 +198,203 @@ def test_discover_legacy_tables_ranking(tmp_path: Path) -> None: _discover_legacy(tmp_path / "exp", fcs=True) -def test_codex_user_cells_table_replaces_discovered(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path, regions=2, layout="per_marker", n_cells=50) - (tmp_path / "mine.csv").write_text("cell_id,x,y,cyc001_ch001:CD3\n1,1.0,4.0,1.0\n2,2.0,5.0,2.0\n3,3.0,6.0,3.0\n") - mask = np.zeros((128, 160), dtype=np.uint16) - mask[:3, :3] = [1, 2, 3] - tifffile.imwrite(tmp_path / "mask.tif", mask) - - sdata = codex( - tmp_path, - cells_table={"reg002": tmp_path / "mine.csv"}, - labels={"reg002": tmp_path / "mask.tif"}, - image_models_kwargs=_FAST, - labels_models_kwargs=_FAST, - ) - - table = sdata.tables["table"] - assert table.n_obs == 3 - assert list(table.var_names) == ["CD3"] - assert table.uns["spatialdata_attrs"]["region"] == ["reg002_labels"] - assert list(sdata.labels) == ["reg002_labels"] - assert len(sdata.shapes) == 0 - assert sdata.attrs["akoya"]["cell_tables"] == {"reg002": str(tmp_path / "mine.csv")} - - -def test_codex_explicit_inputs_disable_discovery(tmp_path: Path) -> None: - # HuBMAP-like: the mask has the region id of its image, and two region-less tables are discoverable - for folder in ("expr", "mask"): - (tmp_path / folder).mkdir() - tifffile.imwrite(tmp_path / "expr" / "reg001_expr.ome.tiff", np.zeros((2, 16, 16), dtype=np.uint16)) - mask = tmp_path / "mask" / "reg001_mask.ome.tiff" - tifffile.imwrite(mask, np.arange(256, dtype=np.uint16).reshape(16, 16)) - header = "cell_id,x,y,cyc001_ch001\n" - (tmp_path / "cells_all.csv").write_text(header + "1,1.0,1.0,1.0\n2,2.0,2.0,2.0\n") - (tmp_path / "cells_filtered.csv").write_text(header + "2,2.0,2.0,2.0\n") - - for labels in (mask, {"reg001": mask}): - sdata = codex( - tmp_path, - cells_table=tmp_path / "cells_filtered.csv", - labels=labels, - image_models_kwargs=_FAST, - labels_models_kwargs=_FAST, - ) - assert list(sdata.images) == ["reg001_image"] - assert list(sdata.labels) == ["reg001_labels"] - assert sdata.tables["table"].n_obs == 1 - - # an explicit table of an unselected region does not bring the discovered tables back - write_legacy_processor_tree(tmp_path / "tree", regions=2, cycles=1, channels=1, n_cells=5) - sdata = codex(tmp_path / "tree", regions=["reg001"], cells_table={"reg002": tmp_path / "cells_all.csv"}) - assert not sdata.tables - - -@pytest.mark.parametrize( - ("image", "mask", "region"), - [ - ("expr/reg001_expr.ome.tiff", "mask/reg001_mask.ome.tiff", "reg001"), - ("Kidney sample.tif", "Kidney sample_labels.tif", "Kidney_sample"), - ], -) -def test_codex_discovers_the_mask_of_a_region(tmp_path: Path, image: str, mask: str, region: str) -> None: - for name in (image, mask): - (tmp_path / name).parent.mkdir(exist_ok=True) - tifffile.imwrite(tmp_path / image, np.zeros((2, 16, 16), dtype=np.uint16)) - tifffile.imwrite(tmp_path / mask, np.arange(256, dtype=np.uint16).reshape(16, 16)) - (tmp_path / "cells.csv").write_text("cell_id,x,y,cyc001_ch001\n1,0.0,0.0,1.0\n2,1.0,0.0,2.0\n") - - sdata = codex(tmp_path, image_models_kwargs=_FAST, labels_models_kwargs=_FAST) - - assert list(sdata.images) == [f"{region}_image"] - assert list(sdata.labels) == [f"{region}_labels"] - assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == [f"{region}_labels"] - assert sdata.attrs["akoya"]["labels"] == {region: str(tmp_path / mask)} - - # an explicit mask wins - other = tmp_path / "other.tif" - tifffile.imwrite(other, np.ones((16, 16), dtype=np.uint16)) - sdata = codex(tmp_path, labels=other, image_models_kwargs=_FAST, labels_models_kwargs=_FAST) - assert list(sdata.images) == [f"{region}_image"] - assert sdata.attrs["akoya"]["labels"] == {region: str(other)} - - -def test_codex_masks_of_no_region_or_one_region_twice(tmp_path: Path) -> None: - tifffile.imwrite(tmp_path / "reg001.tif", np.zeros((2, 16, 16), dtype=np.uint16)) - tifffile.imwrite(tmp_path / "Other_mask.tif", np.zeros((16, 16), dtype=np.uint16)) - - with pytest.warns(UserWarning, match="Ignoring the mask .*Other_mask.tif"): - sdata = codex(tmp_path, image_models_kwargs=_FAST) - assert list(sdata.images) == ["reg001_image"] - assert not sdata.labels - - for name in ("reg001_mask.tif", "reg001_labels.tif"): - tifffile.imwrite(tmp_path / name, np.zeros((16, 16), dtype=np.uint16)) - with pytest.raises(ValueError, match="several masks for region 'reg001'"): - codex(tmp_path, image_models_kwargs=_FAST) - - -def test_codex_regions_of_regionless_table_are_available(tmp_path: Path) -> None: - data = write_legacy_processor_tree(tmp_path, regions=1, fcs=False, csv=False, n_cells=5) - pd.concat([data["cells"], data["cells"].assign(region=2.0)]).to_csv(tmp_path / "cells.csv", index=False) - - with pytest.warns(UserWarning, match="circles"): - sdata = codex(tmp_path, regions=["reg002"], image_models_kwargs=_FAST) - - assert not sdata.images - assert list(sdata.shapes) == ["reg002_cells"] - - -def test_codex_labels_with_discovered_tables(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path, regions=1, layout="per_marker", n_cells=5) - tifffile.imwrite(tmp_path / "mask.tif", np.zeros((128, 160), dtype=np.uint16)) - - sdata = codex(tmp_path, labels=tmp_path / "mask.tif", image_models_kwargs=_FAST, labels_models_kwargs=_FAST) - - assert list(sdata.labels) == ["reg001_labels"] - assert len(sdata.shapes) == 0 - assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == ["reg001_labels"] - - -def test_codex_str_regions_and_channel_names(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path, regions=2, cycles=1, channels=1, layout="per_marker", n_cells=5) - - with pytest.warns(UserWarning, match="circles"): - sdata = codex(tmp_path, regions="reg002", channel_names="nuclei", image_models_kwargs=_FAST) - - assert list(sdata.images) == ["reg002_image"] - assert get_channel_names(sdata.images["reg002_image"]) == ["nuclei"] - - -def test_codex_regions_validation(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path, regions=2, cycles=1, channels=1, layout="per_marker", fcs=False, csv=False) - - with pytest.raises(TypeError, match="sequence of region ids"): - codex(tmp_path, regions={}) # type: ignore[arg-type] - with pytest.raises(ValueError, match="empty"): - codex(tmp_path, regions=[]) - - sdata = codex(tmp_path, regions=["reg002", "reg002"], image_models_kwargs=_FAST) - assert list(sdata.images) == ["reg002_image"] - - -@pytest.mark.parametrize("imagej", [False, True]) -def test_read_legacy_image_tiles_uncompressed_planes(tmp_path: Path, imagej: bool) -> None: - # uncompressed single-strip planes are one zarr chunk each; they must be memory-mapped and tiled instead - stack = np.random.default_rng(0).integers(0, 2**16, (2, 100, 130), dtype=np.uint16) - files = [tmp_path / f"reg001_cyc001_ch{i + 1:03d}_M{i}.tif" for i in range(2)] - for file, plane in zip(files, stack, strict=True): - tifffile.imwrite(file, plane, imagej=imagej, rowsperstrip=plane.shape[0]) - hyperstack = tmp_path / "reg002_montage.tif" - tifffile.imwrite(hyperstack, stack[None], imagej=True, metadata={"axes": "TCYX"}) - - for source in (files, [hyperstack]): - image, _ = _read_legacy_image(source, None, image_models_kwargs={"chunks": (1, 32, 64)}) - assert max(image.chunks[-2]) <= 32 - assert max(image.chunks[-1]) <= 64 - np.testing.assert_array_equal(image.compute(), stack) - - -def test_codex_single_tif_without_region(tmp_path: Path) -> None: - path = tmp_path / "Kidney.tif" - tifffile.imwrite(path, np.zeros((3, 32, 48), dtype=np.uint16), imagej=True, metadata={"axes": "CYX"}) - - sdata = codex(path, image_models_kwargs=_FAST) - - assert list(sdata.images) == ["Kidney_image"] - assert _level0(sdata, "Kidney_image").shape == (3, 32, 48) - - -def test_codex_region_id_needs_reg_without_a_letter_before(tmp_path: Path) -> None: - for name in ("reg001_montage.tif", "exp_reg002.tif", "1_reg003.tif", "Treg2.tif", "Preg12.tif", "coreg1.tif"): - tifffile.imwrite(tmp_path / name, np.zeros((1, 4, 4), dtype=np.uint8)) - assert sorted(_discover_legacy(tmp_path, fcs=True).images) == ["reg001", "reg002", "reg003"] - - -def test_codex_flat_layout_ignores_non_region_tifs(tmp_path: Path) -> None: - tifffile.imwrite(tmp_path / "Kidney sample.tif", np.zeros((2, 16, 16), dtype=np.uint16)) - (tmp_path / "analysis").mkdir() - tifffile.imwrite(tmp_path / "analysis" / "Treg2_density.tif", np.zeros((16, 16), dtype=np.uint16)) - - for regions in (None, ["Kidney sample"], ["Kidney_sample"]): - sdata = codex(tmp_path, regions=regions, image_models_kwargs=_FAST) - assert list(sdata.images) == ["Kidney_sample_image"] - - -def test_codex_and_phenocycler_take_the_region_id_of_a_qptiff(tmp_path: Path) -> None: - write_qptiff(tmp_path / "exp_reg001.qptiff", biomarkers=["DAPI"], dtype="uint8") - write_qptiff(tmp_path / "exp_reg001.er.qptiff", biomarkers=["DAPI"], dtype="uint16") - write_qptiff(tmp_path / "Tonsil_Scan1.qptiff", biomarkers=["DAPI"], dtype="uint8") - - for reader in (codex, phenocycler): - sdata = reader(tmp_path, image_models_kwargs=_FAST) - assert sorted(sdata.images) == ["Tonsil_image", "reg001_image"] - assert sdata.attrs["akoya"]["regions"]["reg001"]["path"].endswith("exp_reg001.er.qptiff") - assert list(reader(tmp_path / "exp_reg001.qptiff", image_models_kwargs=_FAST).images) == ["reg001_image"] - - -def test_codex_and_phenocycler_raise_for_qptiffs_of_one_region_id(tmp_path: Path) -> None: - write_qptiff(tmp_path / "exp_reg001_Scan1.qptiff", biomarkers=["DAPI"]) - write_qptiff(tmp_path / "other_reg1.qptiff", biomarkers=["DAPI"]) - - for reader in (codex, phenocycler): - with pytest.raises(ValueError, match=r"Several qptiff files map to the same region\(s\) \['reg001'\]"): - reader(tmp_path, image_models_kwargs=_FAST) - - -def test_codex_and_phenocycler_name_a_non_region_qptiff_alike(tmp_path: Path) -> None: - write_qptiff(tmp_path / "Preg12_Scan1.qptiff", biomarkers=["DAPI"], dtype="uint8") - - for reader in (codex, phenocycler): - assert list(reader(tmp_path, image_models_kwargs=_FAST).images) == ["Preg12_image"] - - -def test_discover_legacy_regionless_table_ignores_sidecar_csv(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path / "exp", layout="per_marker", fcs=False, csv=False) - (tmp_path / "antibody_list.csv").write_text("marker,clone\nCD3,UCHT1\n") - cells = tmp_path / "exp" / "results" / "cells.csv" - cells.parent.mkdir() - cells.write_text("cell_id,x,y,cyc001_ch001:DAPI\n1,2.0,3.0,4.0\n") - - assert _discover_legacy(tmp_path, fcs=True).tables == {"*": cells} - - (tmp_path / "clusters.csv").write_text("cellid,cluster\n1,2\n") - (tmp_path / "xy.csv").write_text("x,y\n1.0,2.0\n") - assert _discover_legacy(tmp_path, fcs=True).tables == {"*": cells} - - (tmp_path / "other_cells.csv").write_text("cell_id:cell_id,x:x,y:y\n1,1.0,2.0\n") - with pytest.raises(ValueError, match="cannot choose"): - _discover_legacy(tmp_path, fcs=True) - - -@pytest.mark.parametrize("with_table", [True, False]) -def test_codex_region_named_non_cell_csv_is_ignored(tmp_path: Path, with_table: bool) -> None: - data = write_legacy_processor_tree(tmp_path, cycles=1, channels=2, fcs=False, csv=with_table, n_cells=5) - (tmp_path / "reg001_antibodies.csv").write_text("Antibody,Clone\nCD3,UCHT1\n") - - if with_table: - with pytest.warns(UserWarning, match="circles"): - sdata = codex(tmp_path, image_models_kwargs=_FAST) - assert sdata.tables["table"].n_obs == len(data["cells"]) - assert sdata.attrs["akoya"]["cell_tables"]["reg001"].endswith("reg001_compensated.csv") - else: - sdata = codex(tmp_path, image_models_kwargs=_FAST) - assert not sdata.tables - assert list(sdata.images) == ["reg001_image"] - - -@pytest.mark.parametrize("layout", ["per_marker", "hyperstack"]) -def test_codex_is_lazy(tmp_path: Path, layout: Literal["per_marker", "hyperstack"]) -> None: - data = write_legacy_processor_tree(tmp_path, layout=layout) - - with pytest.MonkeyPatch.context() as monkeypatch: - forbid_pixel_reads(monkeypatch) - sdata = codex(tmp_path, image_models_kwargs=_FAST) - - expected = data["images"]["reg001"] - np.testing.assert_array_equal(_level0(sdata, "reg001_image").values, expected.reshape(-1, *expected.shape[-2:])) - - -@pytest.mark.parametrize("qptiff", [False, True]) -def test_codex_imread_kwargs_warns_when_unused(tmp_path: Path, qptiff: bool) -> None: - # every image is read lazily with tifffile, so `imread_kwargs` of the old dask_image reader are ignored - if qptiff: - write_qptiff(tmp_path / "Tonsil_Scan1.qptiff", biomarkers=["DAPI"]) - else: - write_legacy_processor_tree(tmp_path, layout="per_marker") - with pytest.warns(UserWarning, match="imread_kwargs"): - codex(tmp_path, imread_kwargs={"nframes": 1}, image_models_kwargs=_FAST) - - -def test_codex_table_multi_region(tmp_path: Path) -> None: - data = write_legacy_processor_tree(tmp_path, regions=2, layout="per_marker", n_cells=50) +def test_codex_tables_of_several_regions(tmp_path: Path) -> None: + data = write_legacy_processor_tree(tmp_path, regions=2, n_cells=50) + shutil.rmtree(tmp_path / "processed_2020-01-01" / "stitched" / "reg002") - with pytest.warns(UserWarning, match="circles"): + with pytest.warns(UserWarning, match=r"No segmentation mask found for region\(s\) \['reg001', 'reg002'\]"): sdata = codex(tmp_path, image_models_kwargs=_FAST) - assert list(sdata.tables) == ["table"] table = sdata.tables["table"] assert table.n_obs == 100 + assert isinstance(table.X, np.ndarray) + assert table.X.dtype == np.float32 assert list(table.obs["region"].cat.categories) == ["reg001_cells", "reg002_cells"] assert list(table.var_names) == dedupe_channel_names(data["markers"]) - assert sorted(sdata.shapes) == ["reg001_cells", "reg002_cells"] - # the fixture returns the cells of the last region - radius = sdata.shapes["reg002_cells"]["radius"].to_numpy() - np.testing.assert_allclose(radius, np.sqrt(data["cells"]["size"] / np.pi), rtol=1e-6) reg002 = table[table.obs["region"] == "reg002_cells"] np.testing.assert_allclose(reg002.obsm["spatial"], data["cells"][["x", "y"]], rtol=1e-6) + radius = sdata.shapes["reg002_cells"]["radius"].to_numpy() + np.testing.assert_allclose(radius, np.sqrt(data["cells"]["size"] / np.pi), rtol=1e-6) assert sdata.attrs["akoya"]["cell_tables"]["reg002"].endswith("reg002_compensated.fcs") - - -def test_codex_multi_region_own_coordinate_systems(tmp_path: Path) -> None: - # CODEX Processor regions have no stage positions: each gets its own coordinate system - write_legacy_processor_tree(tmp_path, regions=2, cycles=1, layout="per_marker", n_cells=5) - - with pytest.warns(UserWarning, match="circles"): - sdata = codex(tmp_path, image_models_kwargs=_FAST) - + # regions without stage positions, and a region without an image, have their own coordinate systems assert sorted(sdata.coordinate_systems) == ["reg001", "reg002"] - for region in ("reg001", "reg002"): - for name in (f"{region}_image", f"{region}_cells"): - assert get_transformation(sdata[name], get_all=True) == {region: Identity()} - - single = codex(tmp_path, regions=["reg002"], image_models_kwargs=_FAST) - assert single.coordinate_systems == ["global"] - - -def test_codex_region_without_image_own_coordinate_system(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path, regions=2, cycles=1, layout="per_marker", n_cells=5) - shutil.rmtree(tmp_path / "processed_2020-01-01" / "stitched" / "reg002") - - with pytest.warns(UserWarning, match="circles"): - sdata = codex(tmp_path, image_models_kwargs=_FAST) - - assert list(sdata.images) == ["reg001_image"] - assert get_transformation(sdata["reg001_image"], get_all=True) == {"reg001": Identity()} + for name in ("reg001_image", "reg001_cells"): + assert get_transformation(sdata[name], get_all=True) == {"reg001": Identity()} assert get_transformation(sdata["reg002_cells"], get_all=True) == {"reg002": Identity()} - - -@pytest.mark.parametrize("fcs", [True, False]) -def test_codex_old_flat_layout_regression(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fcs: bool) -> None: - # the original reader read the image (and the CSV) relative to the working directory - data = write_legacy_processor_tree(tmp_path / "data", layout="old_flat", fcs=fcs) - monkeypatch.chdir(tmp_path) - - with pytest.warns(UserWarning, match="circles"): - sdata = codex(tmp_path / "data", fcs=fcs, image_models_kwargs=_FAST) - - np.testing.assert_array_equal(_level0(sdata, "reg001_image").values, data["images"]["reg001"][:1]) - table = sdata.tables["table"] - assert table.n_obs == 50 - assert list(table.obs["region"].cat.categories) == ["reg001_cells"] - np.testing.assert_allclose(table.X, data["cells"].filter(regex="^cyc").to_numpy(), rtol=1e-6) - - -def test_codex_fcs_false_falls_back_to_fcs_with_warning(tmp_path: Path) -> None: - data = write_legacy_processor_tree(tmp_path, layout="per_marker", fcs=True, csv=False) - - with pytest.warns(UserWarning, match="CSV"): - sdata = codex(tmp_path, fcs=False, image_models_kwargs=_FAST) - - assert list(sdata.tables["table"].var_names) == dedupe_channel_names(data["markers"]) - - -def test_codex_without_tables_reads_images_only(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path, layout="per_marker", fcs=False, csv=False) - - sdata = codex(tmp_path, image_models_kwargs=_FAST) - - assert list(sdata.images) == ["reg001_image"] - assert not sdata.tables - assert not sdata.shapes - - -def test_codex_table_float32_dense(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path, layout="per_marker", fcs=False, csv=True) - - with pytest.warns(UserWarning, match="circles"): - sdata = codex(tmp_path, fcs=False, image_models_kwargs=_FAST) - - table = sdata.tables["table"] - assert isinstance(table.X, np.ndarray) - assert table.X.dtype == np.float32 - - -def test_codex_regions_subsets_tables(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path, regions=2, layout="per_marker") - - with pytest.warns(UserWarning, match="circles"): - sdata = codex(tmp_path, regions=["reg002"], image_models_kwargs=_FAST) - - assert list(sdata.shapes) == ["reg002_cells"] - assert list(sdata.tables["table"].obs["region"].cat.categories) == ["reg002_cells"] - - -def test_codex_single_fcs_file(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path, layout="per_marker") - fcs = tmp_path / "processed_2020-01-01" / "segm" / "segm-1" / "fcs" / "compensated" / "reg001_compensated.fcs" - - with pytest.warns(UserWarning, match="circles"): - sdata = codex(fcs) - - assert not sdata.images - assert list(sdata.shapes) == ["reg001_cells"] - assert sdata.tables["table"].n_obs == 50 - - -def test_codex_unreadable_regionless_table_raises(tmp_path: Path) -> None: - tifffile.imwrite(tmp_path / "reg001_montage.tif", np.zeros((2, 16, 16), dtype=np.uint16)) - # a region-less table that cannot be assigned to a region: there are two images and no `region` column - tifffile.imwrite(tmp_path / "reg002_montage.tif", np.zeros((2, 16, 16), dtype=np.uint16)) - (tmp_path / "cells.csv").write_text("cell_id,x,y,cyc001_ch001\n1,1.0,2.0,3.0\n") - (tmp_path / "clusters.csv").write_text("cellid,cluster\n1,2\n") - - with pytest.raises(ValueError, match="cannot be assigned to regions"): - codex(tmp_path, image_models_kwargs=_FAST) - - # a CODEX table next to a single image that cannot be read - (tmp_path / "reg002_montage.tif").unlink() - (tmp_path / "cells.csv").write_text("cell_id,x,y,cyc001_ch001\n1,1.0,2.0,3.0\n1,4.0,5.0,6.0\n") - with pytest.raises(ValueError, match="duplicate instance id"): - codex(tmp_path, image_models_kwargs=_FAST) - - # a single table file without a region column and no image - write_fcs(tmp_path / "Kidney.fcs", pd.DataFrame({"cell_id": [1.0], "x": [1.0], "y": [2.0], "cyc001_ch001": [3.0]})) - with pytest.raises(ValueError, match="cannot be assigned to regions"): - codex(tmp_path / "Kidney.fcs") - - -@pytest.mark.parametrize("file_name", ["Kidney.tif", "Kidney.ome.tif", "Kidney.OME.TIFF"]) -def test_codex_regionless_table_belongs_to_single_stem_image(tmp_path: Path, file_name: str) -> None: - # the region is the file name without its image extension, `.ome` included - tifffile.imwrite(tmp_path / file_name, np.zeros((2, 16, 16), dtype=np.uint16)) - (tmp_path / "cells.csv").write_text("cell_id,region,x,y,size,cyc001_ch001\n1,1,1.0,2.0,3.0,4.0\n") - - for regions in (None, ["Kidney"]): - with pytest.warns(UserWarning, match="circles"): - sdata = codex(tmp_path, regions=regions, image_models_kwargs=_FAST) - assert list(sdata.tables["table"].obs["region"].cat.categories) == ["Kidney_cells"] - assert list(sdata.shapes) == ["Kidney_cells"] - assert sdata.attrs["akoya"]["cell_tables"] == {"Kidney": str(tmp_path / "cells.csv")} - - -def test_codex_duplicate_cell_ids_raise(tmp_path: Path) -> None: - data = write_legacy_processor_tree(tmp_path, cycles=1, channels=1, fcs=False, n_cells=3) - table = next(tmp_path.rglob("reg001_compensated.csv")) - cells = data["cells"].assign(cell_id=[1.0, 1.0, 2.0]) - cells.to_csv(table, index=False) - - with pytest.raises(ValueError, match="reg001_compensated.csv has 1 duplicate instance id"): - codex(tmp_path, image_models_kwargs=_FAST) - - -def test_codex_overlapping_tables_raise(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path, layout="per_marker", fcs=False, csv=True) - (tmp_path / "all_cells.csv").write_text("cell_id,region,x,y,cyc001_ch001\n1,1,1.0,2.0,3.0\n") - - with pytest.raises(ValueError, match=r"Several cell tables contain the cells of region\(s\) \['reg001'\]"): - codex(tmp_path, fcs=False, image_models_kwargs=_FAST) - - -def test_codex_write_roundtrip(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path / "data", regions=2, cycles=1, channels=2, n_cells=5) - with pytest.warns(UserWarning, match="circles"): - sdata = codex(tmp_path / "data", image_models_kwargs=_FAST) + assert list(_codex(tmp_path, regions=["reg002"]).shapes) == ["reg002_cells"] sdata.write(tmp_path / "x.zarr") read = read_zarr(tmp_path / "x.zarr") - - assert sorted(read.images) == ["reg001_image", "reg002_image"] - assert sorted(read.shapes) == ["reg001_cells", "reg002_cells"] - for region in ("reg001", "reg002"): - name = f"{region}_image" - assert get_channel_names(read.images[name]) == get_channel_names(sdata.images[name]) - np.testing.assert_array_equal(_level0(read, name).values, _level0(sdata, name).values) - assert get_transformation(read.images[name], get_all=True) == {region: Identity()} - assert get_transformation(read.shapes[f"{region}_cells"], get_all=True) == {region: Identity()} - table = read.tables["table"] - assert table.n_obs == 10 - assert list(table.obs["region"].cat.categories) == ["reg001_cells", "reg002_cells"] - assert list(table.var_names) == list(sdata.tables["table"].var_names) + np.testing.assert_array_equal(_level0(read, "reg001_image").values, _level0(sdata, "reg001_image").values) + assert get_transformation(read["reg002_cells"], get_all=True) == {"reg002": Identity()} + assert read.tables["table"].obs.index.name is None + assert read.tables["table"].obs["cell_id"].dtype == np.int64 assert read.attrs["akoya"] == sdata.attrs["akoya"] assert read.attrs["spatialdata_io_reader"] == "codex" -def test_codex_old_positional_call(tmp_path: Path) -> None: - write_legacy_processor_tree(tmp_path, regions=2, cycles=1, channels=1, layout="per_marker", fcs=False, csv=False) +def test_codex_tables_with_different_panels_or_formats(tmp_path: Path) -> None: + columns = {"cell_id": [1.0], "x": [1.0], "y": [2.0]} + write_fcs(tmp_path / "reg001.fcs", pd.DataFrame(columns | {"cyc001_ch001": [3.0]}), {"cyc001_ch001": "DAPI"}) + pd.DataFrame(columns | {"cyc001_ch001:DAPI": [3.0], "cyc001_ch002:CD3": [4.0]}).to_csv( + tmp_path / "reg002.csv", index=False + ) - # `codex(path, fcs, imread_kwargs)` is the signature of earlier releases - for fcs in (True, False): - sdata = codex(tmp_path, fcs, {}) - assert sorted(sdata.images) == ["reg001_image", "reg002_image"] + sdata = _codex(tmp_path) - with pytest.warns(UserWarning, match="imread_kwargs"): - codex(tmp_path, True, {"nframes": 1}) + assert sorted(sdata.tables) == ["reg001_table", "reg002_table"] + assert list(sdata.tables["reg002_table"].var_names) == ["DAPI", "CD3"] + assert sdata.tables["reg001_table"].uns["spatialdata_attrs"]["region"] == ["reg001_cells"] + # the CSV and FCS tables of one panel share a table; their source columns differ. With `fcs=False`, a region with + # only an FCS table is read from it + pd.DataFrame(columns | {"cyc001_ch001:DAPI": [3.0]}).to_csv(tmp_path / "reg002.csv", index=False) + with pytest.warns(UserWarning, match="no CSV cell table was found"): + sdata = _codex(tmp_path, fcs=False) + assert list(sdata.tables["table"].var_names) == ["DAPI"] + assert "source_column" not in sdata.tables["table"].var -@pytest.mark.parametrize("case", ["one_region", "kidney_2019", "regions_subset"]) -def test_single_region_codex_table_write_roundtrip(tmp_path: Path, case: str) -> None: - data = tmp_path / "data" - data.mkdir() - if case == "kidney_2019": - tifffile.imwrite(data / "Kidney.tif", np.zeros((2, 16, 16), dtype=np.uint16)) - (data / "Kidney.csv").write_text("cell_id,x,y,size,cyc001_ch001\n1,1.0,2.0,3.0,4.0\n2,5.0,6.0,7.0,8.0\n") - else: - write_legacy_processor_tree(data, regions=1 if case == "one_region" else 2, cycles=1, channels=1, n_cells=5) - with pytest.warns(UserWarning, match="circles"): - sdata = codex(data, regions=["reg001"] if case == "regions_subset" else None, image_models_kwargs=_FAST) - sdata.write(tmp_path / "x.zarr") +@pytest.mark.parametrize( + ("images", "rows", "regions", "expected"), + [ + # the region is the file name without its image extension, `.ome` included + (["Kidney.ome.tif"], ["1,1,1,2"], None, ["Kidney_cells"]), + # region values are matched after `sanitize_name`, like the region ids + ( + ["Tonsil 1.tif", "Tonsil 2.tif"], + ["1,Tonsil 1,1,2", "1,Tonsil 2,3,4"], + None, + ["Tonsil_1_cells", "Tonsil_2_cells"], + ), + # a region without an image is still a region + (["reg001.tif"], ["1,1,1,2", "1,2,3,4"], ["reg002"], ["reg002_cells"]), + ], +) +def test_codex_regionless_table_regions( + tmp_path: Path, images: list[str], rows: list[str], regions: list[str] | None, expected: list[str] +) -> None: + for name in images: + tifffile.imwrite(tmp_path / name, np.zeros((2, 16, 16), dtype=np.uint16)) + (tmp_path / "cells.csv").write_text("cell_id,region,x,y,cyc001_ch001\n" + "".join(f"{row},5\n" for row in rows)) - table = read_zarr(tmp_path / "x.zarr").tables["table"] - assert table.obs.index.name is None - assert table.obs["cell_id"].dtype == np.int64 - np.testing.assert_array_equal(table.obs["cell_id"], sdata.tables["table"].obs["cell_id"]) + sdata = _codex(tmp_path, regions=regions) + assert list(sdata.tables["table"].obs["region"].cat.categories) == expected + assert sorted(sdata.shapes) == expected -def test_codex_discovered_tables_assign_regions_like_given_ones(tmp_path: Path) -> None: - for name in ("Tonsil 1", "Tonsil 2"): - tifffile.imwrite(tmp_path / f"{name}.tif", np.zeros((2, 16, 16), dtype=np.uint16)) - (tmp_path / "cells.csv").write_text( - "cell_id,region,x,y,size,cyc001_ch001\n1,Tonsil 1,1.0,2.0,3.0,4.0\n1,Tonsil 2,3.0,4.0,3.0,5.0\n" - ) - with pytest.warns(UserWarning, match="circles"): - sdata = codex(tmp_path, image_models_kwargs=_FAST) +@pytest.mark.parametrize( + ("setup", "match"), + [ + ("two_images", "cannot be assigned to regions"), + ("duplicate_ids", r"reg001_compensated.csv has 1 duplicate instance id"), + ("overlapping", r"Several cell tables contain the cells of region\(s\) \['reg001'\]"), + ("other_region", "has no cells of region 'reg002'"), + ], +) +def test_codex_table_errors(tmp_path: Path, setup: str, match: str) -> None: + data = write_legacy_processor_tree(tmp_path, regions=2, cycles=1, channels=1, fcs=False, n_cells=3) + table = next(tmp_path.rglob("reg002_compensated.csv" if setup == "other_region" else "reg001_compensated.csv")) + if setup == "two_images": + table.rename(tmp_path / "cells.csv") + data["cells"].drop(columns="region").to_csv(tmp_path / "cells.csv", index=False) + elif setup == "duplicate_ids": + data["cells"].assign(cell_id=[1.0, 1.0, 2.0], region=1.0).to_csv(table, index=False) + elif setup == "overlapping": + (tmp_path / "all_cells.csv").write_text("cell_id,region,x,y,cyc001_ch001\n1,1,1.0,2.0,3.0\n") + else: + # the table of reg002 whose region column names reg001 + data["cells"].assign(region=1.0).to_csv(table, index=False) - assert sorted(sdata.shapes) == ["Tonsil_1_cells", "Tonsil_2_cells"] - assert sorted(sdata.coordinate_systems) == ["Tonsil_1", "Tonsil_2"] + with pytest.raises(ValueError, match=match): + codex(tmp_path, fcs=False) -def test_codex_discovered_table_of_another_region_raises(tmp_path: Path) -> None: - data = write_legacy_processor_tree(tmp_path, regions=2, cycles=1, channels=1, fcs=False, n_cells=3) - # the table of reg002 whose region column names reg001 - data["cells"].assign(region=1.0).to_csv(next(tmp_path.rglob("reg002_compensated.csv")), index=False) +@pytest.mark.parametrize( + ("image", "mask", "region"), + [ + ("expr/reg001_expr.ome.tiff", "mask/reg001_mask.ome.tiff", "reg001"), + ("Kidney sample.tif", "Kidney sample_labels.tif", "Kidney_sample"), + ("Tonsil_Scan1.er.qptiff", "Tonsil_Scan1_mask.tif", "Tonsil"), + ("Tonsil_Scan1.er.qptiff", "Tonsil_mask.tif", "Tonsil"), + ], +) +def test_codex_discovers_the_mask_of_a_region(tmp_path: Path, image: str, mask: str, region: str) -> None: + for name in (image, mask): + (tmp_path / name).parent.mkdir(exist_ok=True) + if image.endswith("qptiff"): + write_qptiff(tmp_path / image, biomarkers=["DAPI"], shape=(16, 16)) + else: + tifffile.imwrite(tmp_path / image, np.zeros((2, 16, 16), dtype=np.uint16)) + tifffile.imwrite(tmp_path / mask, np.arange(256, dtype=np.uint16).reshape(16, 16)) + (tmp_path / "cells.csv").write_text("cell_id,x,y,cyc001_ch001\n1,0.0,0.0,1.0\n2,1.0,0.0,2.0\n") + + sdata = _codex(tmp_path) - with pytest.raises(ValueError, match="has no cells of region 'reg002'"): - codex(tmp_path, image_models_kwargs=_FAST) + assert list(sdata.images) == [f"{region}_image"] + assert list(sdata.labels) == [f"{region}_labels"] + assert sdata.tables["table"].uns["spatialdata_attrs"]["region"] == [f"{region}_labels"] + assert sdata.attrs["akoya"]["labels"] == {region: str(tmp_path / mask)} -def test_codex_labels_disable_mask_discovery(tmp_path: Path) -> None: +def test_codex_mask_discovery_errors_and_explicit_labels(tmp_path: Path) -> None: for region in ("reg001", "reg002"): tifffile.imwrite(tmp_path / f"{region}_montage.tif", np.zeros((2, 16, 16), dtype=np.uint16)) - for name in ("reg002_mask.tif", "reg002_labels.tif", "Other_mask.tif", "mask.tif"): + tifffile.imwrite(tmp_path / "Other_mask.tif", np.zeros((16, 16), dtype=np.uint16)) + with pytest.warns(UserWarning, match="Ignoring the mask .*Other_mask.tif"): + assert not _codex(tmp_path).labels + + for name in ("reg002_mask.tif", "reg002_labels.tif"): tifffile.imwrite(tmp_path / name, np.ones((16, 16), dtype=np.uint16)) + with pytest.raises(ValueError, match="several masks for region 'reg002'"): + _codex(tmp_path) + # given labels disable mask discovery and are not read as images + tifffile.imwrite(tmp_path / "reg001_x_mask.tif", np.ones((16, 16), dtype=np.uint16)) with warnings.catch_warnings(): warnings.simplefilter("error") - sdata = codex( - tmp_path, - labels={"reg001": tmp_path / "mask.tif"}, - image_models_kwargs=_FAST, - labels_models_kwargs=_FAST, - ) - + sdata = _codex(tmp_path, labels={"reg001": tmp_path / "reg001_x_mask.tif"}) assert sorted(sdata.images) == ["reg001_image", "reg002_image"] assert list(sdata.labels) == ["reg001_labels"] -@pytest.mark.parametrize("mask", ["Tonsil_mask.tif", "Tonsil_Scan1_mask.tif"]) -def test_codex_discovers_the_mask_of_a_qptiff_region(tmp_path: Path, mask: str) -> None: - write_qptiff(tmp_path / "Tonsil_Scan1.er.qptiff", biomarkers=["DAPI"], shape=(16, 16)) - tifffile.imwrite(tmp_path / mask, np.ones((16, 16), dtype=np.uint16)) - - sdata = codex(tmp_path, image_models_kwargs=_FAST, labels_models_kwargs=_FAST) - - assert list(sdata.images) == ["Tonsil_image"] - assert list(sdata.labels) == ["Tonsil_labels"] - assert sdata.attrs["akoya"]["labels"] == {"Tonsil": str(tmp_path / mask)} - - -def test_codex_csv_and_fcs_tables_of_one_panel_share_a_table(tmp_path: Path) -> None: - data = write_legacy_processor_tree(tmp_path, regions=2, layout="per_marker", fcs=True, csv=True, n_cells=5) - next(tmp_path.rglob("reg002_compensated.csv")).unlink() - # the CODEX Processor names the markers in the CSV header, e.g. `cyc001_ch001:DAPI` - csv = next(tmp_path.rglob("reg001_compensated.csv")) - cells = pd.read_csv(csv) - names = dict(zip(cells.filter(regex="^cyc").columns, data["markers"], strict=True)) - cells.rename(columns={column: f"{column}:{marker}" for column, marker in names.items()}).to_csv(csv, index=False) +def test_codex_explicit_table_and_downsampled_mask(tmp_path: Path) -> None: + write_legacy_processor_tree(tmp_path, regions=2, n_cells=50) + (tmp_path / "mine.csv").write_text("cell_id,x,y,cyc001_ch001:CD3\n1,1.0,4.0,1.0\n2,3.0,6.0,2.0\n") + mask = np.zeros((64, 80), dtype=np.uint16) + mask[2, 0], mask[3, 1] = 1, 2 + tifffile.imwrite(tmp_path / "mask.tif", mask) - with pytest.warns(UserWarning, match="CSV"): - sdata = codex(tmp_path, fcs=False, image_models_kwargs=_FAST) + with warnings.catch_warnings(): + warnings.simplefilter("error") + sdata = _codex( + tmp_path, cells_table={"reg002": tmp_path / "mine.csv"}, labels={"reg002": tmp_path / "mask.tif"} + ) - assert list(sdata.tables) == ["table"] table = sdata.tables["table"] - assert list(table.obs["region"].cat.categories) == ["reg001_cells", "reg002_cells"] - assert list(table.var_names) == dedupe_channel_names(data["markers"]) - # the markers were read from columns of other names - assert "source_column" not in table.var + assert list(table.var_names) == ["CD3"] + assert table.uns["spatialdata_attrs"]["region"] == ["reg002_labels"] + assert not sdata.shapes + assert sdata.attrs["akoya"]["cell_tables"] == {"reg002": str(tmp_path / "mine.csv")} + # the mask is scaled onto the image; the centroids stay in image pixels + labels_transformation = get_transformation(sdata["reg002_labels"], "reg002") + assert isinstance(labels_transformation, Sequence) + assert labels_transformation.transformations[0] == Scale([2, 2], axes=("y", "x")) + np.testing.assert_allclose(table.obsm["spatial"], [[1, 4], [3, 6]]) + # an explicit table of an unselected region does not bring the discovered tables back + assert not _codex(tmp_path, regions=["reg001"], cells_table={"reg002": tmp_path / "mine.csv"}).tables -@pytest.mark.parametrize("corrupt", ["truncated", "no_datatype"]) -def test_codex_ignores_malformed_fcs_files(tmp_path: Path, corrupt: str) -> None: - tifffile.imwrite(tmp_path / "reg001_montage.tif", np.zeros((2, 16, 16), dtype=np.uint16)) - fcs = tmp_path / "reg001.fcs" - write_fcs(fcs, pd.DataFrame({"cell_id": [1.0], "x": [1.0], "y": [2.0], "cyc001_ch001": [3.0]})) - content = fcs.read_bytes() - # flowio raises an EOFError and a KeyError for these - fcs.write_bytes(content[:100] if corrupt == "truncated" else content.replace(b"$DATATYPE", b"$DATATYPX")) +@pytest.mark.parametrize("fcs", [True, False]) +def test_codex_old_flat_layout_regression(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fcs: bool) -> None: + # the original reader read the image (and the CSV) relative to the working directory, and ignored `fcs` + data = write_legacy_processor_tree(tmp_path / "data", layout="old_flat", fcs=fcs) + monkeypatch.chdir(tmp_path) - sdata = codex(tmp_path, image_models_kwargs=_FAST) + sdata = _codex(tmp_path / "data", fcs=fcs) - assert list(sdata.images) == ["reg001_image"] - assert not sdata.tables + np.testing.assert_array_equal(_level0(sdata, "reg001_image").values, data["images"]["reg001"][:1]) + table = sdata.tables["table"] + assert list(table.obs["region"].cat.categories) == ["reg001_cells"] + np.testing.assert_allclose(table.X, data["cells"].filter(regex="^cyc").to_numpy(), rtol=1e-6) diff --git a/tests/test_phenocycler.py b/tests/test_phenocycler.py index 5c7e77e6..de3ccad1 100644 --- a/tests/test_phenocycler.py +++ b/tests/test_phenocycler.py @@ -5,6 +5,7 @@ import json import warnings from pathlib import Path +from typing import Any import dask.array as da import numpy as np @@ -26,299 +27,147 @@ def _level0(sdata, name): # type: ignore[no-untyped-def] return sdata.images[name]["scale0"]["image"] -def test_read_single_file(tmp_path: Path) -> None: +def _write_mask(path: Path, shape: tuple[int, int] = (256, 320)) -> np.ndarray: + mask = np.zeros(shape, dtype=np.uint32) + mask[10:20, 10:20], mask[30:40, 10:20] = 1, 2 + tifffile.imwrite(path, mask) + return mask + + +def test_read_is_lazy_and_round_trips(tmp_path: Path) -> None: path = tmp_path / "Tonsil_Scan1.qptiff" data = write_qptiff(path, biomarkers=["DAPI", "CD3", "CD3"]) - sdata = phenocycler(path, image_models_kwargs=_FAST) + with pytest.MonkeyPatch.context() as monkeypatch: + forbid_pixel_reads(monkeypatch) + sdata = phenocycler(path, image_models_kwargs=_FAST) - assert list(sdata.images) == ["Tonsil_image"] + assert isinstance(_level0(sdata, "Tonsil_image").data, da.Array) assert get_channel_names(sdata.images["Tonsil_image"]) == ["DAPI", "CD3", "CD3 (2)"] np.testing.assert_array_equal(_level0(sdata, "Tonsil_image").values, data) assert sdata.attrs["akoya"]["regions"]["Tonsil"]["generation"] == "fusion2" - assert sdata.attrs["spatialdata_io_reader"] == "phenocycler" - - -def test_vectra_unmixed_float32_component_qptiff(tmp_path: Path) -> None: - path = tmp_path / "Slide_Scan1.unmixed.qptiff" - names = ["PDL1 (Opal 520)", "DAPI"] - data = write_qptiff( - path, - biomarkers=[None, None], - names=names, - dtype="float32", - software="VectraPolaris 1.0", - resolution_px_per_cm=2e4, - ) - - sdata = phenocycler(path, image_models_kwargs=_FAST) - - image = _level0(sdata, "Slide_Scan1.unmixed_image") - assert image.dtype == np.float32 - np.testing.assert_array_equal(image.values, data) - assert get_channel_names(sdata.images["Slide_Scan1.unmixed_image"]) == names - metadata = sdata.attrs["akoya"]["regions"]["Slide_Scan1.unmixed"] - assert metadata["generation"] == "qpi_other" - assert metadata["pixel_size_um"] == pytest.approx(0.5) - - -def test_vectra_brightfield_rgb_qptiff(tmp_path: Path) -> None: - path = tmp_path / "HandE_Scan1.qptiff" - data = write_qptiff(path, biomarkers=[None], names=["BF"], rgb=True, software="VectraPolaris 1.0") - - sdata = phenocycler(path, image_models_kwargs=_FAST) - - assert get_channel_names(sdata.images["HandE_image"]) == ["r", "g", "b"] - np.testing.assert_array_equal(_level0(sdata, "HandE_image").values, np.moveaxis(data, -1, 0)) - assert sdata.attrs["akoya"]["regions"]["HandE"]["generation"] == "qpi_other" - - -def test_inform_component_tif_with_special_characters_in_name(tmp_path: Path) -> None: - # inForm names its component files `_[x,y]_component_data.tif`; the brackets and comma are not valid - # in an element name - path = tmp_path / "LuCa_[13860,52919]_1x1component_data.tif" - data = write_qptiff( - path, biomarkers=[None, None], names=["A (Opal 520)", "B"], dtype="float32", software="inForm", levels=1 - ) - - sdata = phenocycler(path, image_models_kwargs=_FAST) - - assert list(sdata.images) == ["LuCa__13860_52919__1x1component_data_image"] - image = _level0(sdata, "LuCa__13860_52919__1x1component_data_image") - assert image.dtype == np.float32 - np.testing.assert_array_equal(image.values, data) - - -def test_non_qpi_tif_raises(tmp_path: Path) -> None: - path = tmp_path / "plain.tif" - tifffile.imwrite(path, np.zeros((8, 8), dtype=np.uint8)) - - with pytest.raises(ValueError, match="not a PerkinElmer/Akoya QPTIFF"): - phenocycler(path) - - -def test_read_directory_multiple_scans_and_subset(tmp_path: Path) -> None: - write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) - write_qptiff(tmp_path / "B_Scan1.qptiff", biomarkers=["DAPI", "CD8"]) + sdata.write(tmp_path / "x.zarr") + read = read_zarr(tmp_path / "x.zarr") + assert get_channel_names(read.images["Tonsil_image"]) == ["DAPI", "CD3", "CD3 (2)"] + assert read.attrs["akoya"] == sdata.attrs["akoya"] + assert read.attrs["spatialdata_io_reader"] == "phenocycler" - sdata = phenocycler(str(tmp_path), image_models_kwargs=_FAST) - assert sorted(sdata.images) == ["A_image", "B_image"] - assert sorted(sdata.attrs["akoya"]["regions"]) == ["A", "B"] - subset = phenocycler(tmp_path, regions=["B"], image_models_kwargs=_FAST) - assert list(subset.images) == ["B_image"] - assert list(subset.attrs["akoya"]["regions"]) == ["B"] +@pytest.mark.parametrize( + ("name", "kwargs", "region", "channels"), + [ + ( + "Slide_Scan1.unmixed.qptiff", + {"biomarkers": [None, None], "names": ["PDL1 (Opal 520)", "DAPI"], "dtype": "float32"}, + "Slide_Scan1.unmixed", + ["PDL1 (Opal 520)", "DAPI"], + ), + ("HandE_Scan1.qptiff", {"biomarkers": [None], "names": ["BF"], "rgb": True}, "HandE", ["r", "g", "b"]), + # inForm component data; the brackets and comma of its name are not valid in an element name + ( + "LuCa_[13860,52919]_1x1component_data.tif", + {"biomarkers": [None], "names": ["A"], "dtype": "float32", "levels": 1}, + "LuCa__13860_52919__1x1component_data", + ["A"], + ), + ], + ids=["unmixed", "brightfield", "inform_component"], +) +def test_vectra_and_phenoimager_files( + tmp_path: Path, name: str, kwargs: dict[str, Any], region: str, channels: list[str] +) -> None: + data = write_qptiff(tmp_path / name, software="VectraPolaris 1.0", **kwargs) - with pytest.raises(ValueError, match="Z"): - phenocycler(tmp_path, regions=["Z"]) + sdata = phenocycler(tmp_path / name, image_models_kwargs=_FAST) + image = _level0(sdata, f"{region}_image") + assert get_channel_names(sdata.images[f"{region}_image"]) == channels + np.testing.assert_array_equal(image.values, np.moveaxis(data, -1, 0) if kwargs.get("rgb") else data) + assert sdata.attrs["akoya"]["regions"][region]["generation"] == "qpi_other" -def test_phenocycler_unrelated_scans_own_coordinate_systems(tmp_path: Path) -> None: - for region, slide in (("A", "SLIDE1"), ("B", "SLIDE2")): - write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI", "CD3"], slide_id=slide) - _write_mask(tmp_path / "A_mask.tif") - sdata = phenocycler(tmp_path, labels={"A": tmp_path / "A_mask.tif"}, image_models_kwargs=_FAST) +@pytest.mark.parametrize( + ("files", "match"), + [ + ([], r"No \.qptiff files"), + (["A [1].qptiff", "A_[1].qptiff"], r"same region\(s\) \['A__1_'\]"), + (["plain.tif"], "not a PerkinElmer/Akoya QPTIFF"), + ], +) +def test_bad_inputs_raise(tmp_path: Path, files: list[str], match: str) -> None: + for name in files: + if name.endswith(".tif"): + tifffile.imwrite(tmp_path / name, np.zeros((8, 8), dtype=np.uint8)) + else: + write_qptiff(tmp_path / name, biomarkers=["DAPI"]) - assert sorted(sdata.coordinate_systems) == ["A", "B"] - for name in ("A_image", "A_labels"): - assert get_transformation(sdata[name], get_all=True) == {"A": Identity()} - assert get_transformation(sdata["B_image"], get_all=True) == {"B": Identity()} + with pytest.raises((FileNotFoundError, ValueError), match=match): + phenocycler(tmp_path / files[0] if files and files[0].endswith(".tif") else tmp_path) -def test_phenocycler_same_slide_scans_placed_in_global(tmp_path: Path) -> None: +def test_regions_and_coordinate_systems(tmp_path: Path) -> None: write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI"], stage_position_cm=(1.0, 2.0)) write_qptiff(tmp_path / "B_Scan1.qptiff", biomarkers=["DAPI"], stage_position_cm=(1.01, 2.0)) + write_qptiff(tmp_path / "C_Scan1.qptiff", biomarkers=["DAPI"], slide_id="SLIDE2") + write_qptiff(tmp_path / "C_Scan2.qptiff", biomarkers=["DAPI"], slide_id="SLIDE2") - sdata = phenocycler(tmp_path, image_models_kwargs=_FAST) + sdata = phenocycler(tmp_path, regions=["A", "B"], image_models_kwargs=_FAST) + # scans of the same slide are placed at their stage positions assert sdata.coordinate_systems == ["global"] assert get_transformation(sdata["A_image"], get_all=True) == {"global": Translation([0, 0], axes=("x", "y"))} offset = get_transformation(sdata["B_image"], "global") assert isinstance(offset, Translation) np.testing.assert_allclose(offset.translation, [100 / 0.5069, 0], rtol=1e-3) - - -def test_region_collision_keeps_full_stem(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - write_qptiff(tmp_path / "S_Scan2.qptiff", biomarkers=["DAPI"]) - - sdata = phenocycler(tmp_path, image_models_kwargs=_FAST) - - assert sorted(sdata.images) == ["S_Scan1_image", "S_Scan2_image"] - - -def test_er_preferred(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI", "CD3"], dtype="uint8") - write_qptiff(tmp_path / "S_Scan1.er.qptiff", biomarkers=["DAPI", "CD3"], dtype="uint16") - - sdata = phenocycler(tmp_path, image_models_kwargs=_FAST) - - assert list(sdata.images) == ["S_image"] - assert _level0(sdata, "S_image").dtype == np.uint16 - assert sdata.attrs["akoya"]["regions"]["S"]["path"].endswith("S_Scan1.er.qptiff") + # unrelated scans get their own coordinate systems; `_Scan` is kept if it tells regions apart + sdata = phenocycler(tmp_path, regions=["A", "C_Scan1", "C_Scan2"], image_models_kwargs=_FAST) + assert sorted(sdata.coordinate_systems) == ["A", "C_Scan1", "C_Scan2"] + assert get_transformation(sdata["C_Scan1_image"], get_all=True) == {"C_Scan1": Identity()} def test_read_all_bit_depths(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI", "CD3"], dtype="uint8") - write_qptiff(tmp_path / "S_Scan1.er.qptiff", biomarkers=["DAPI", "CD3"], dtype="uint16") - - sdata = phenocycler(tmp_path, read_all_bit_depths=True, image_models_kwargs=_FAST) - - assert sorted(sdata.images) == ["S_8bit_image", "S_image"] - assert _level0(sdata, "S_image").dtype == np.uint16 - assert _level0(sdata, "S_8bit_image").dtype == np.uint8 - assert get_transformation(sdata["S_8bit_image"], get_all=True) == get_transformation(sdata["S_image"], get_all=True) - assert list(get_transformation(sdata["S_image"], get_all=True)) == ["global"] - assert sdata.attrs["akoya"]["regions"]["S"]["path"].endswith("S_Scan1.er.qptiff") - assert sdata.attrs["akoya"]["regions"]["S_8bit"]["path"].endswith("S_Scan1.qptiff") - - -def test_read_all_bit_depths_of_an_extended_range_file(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.QPTIFF", biomarkers=["DAPI"], dtype="uint8") - write_qptiff(tmp_path / "S_Scan1.ER.QPTIFF", biomarkers=["DAPI"], dtype="uint16") - - sdata = phenocycler(tmp_path / "S_Scan1.ER.QPTIFF", read_all_bit_depths=True, image_models_kwargs=_FAST) - - assert sorted(sdata.images) == ["S_8bit_image", "S_image"] - assert _level0(sdata, "S_8bit_image").dtype == np.uint8 - # the 8-bit file alone is read as it is - assert list(phenocycler(tmp_path / "S_Scan1.QPTIFF", read_all_bit_depths=True).images) == ["S_image"] - - -def test_read_all_bit_depths_shares_stage_placement(tmp_path: Path) -> None: - for name, pos in (("A", (0.0, 0.0)), ("B", (0.1, 0.0))): - write_qptiff(tmp_path / f"{name}.qptiff", biomarkers=["DAPI"], dtype="uint8", stage_position_cm=pos) - write_qptiff(tmp_path / f"{name}.er.qptiff", biomarkers=["DAPI"], dtype="uint16", stage_position_cm=pos) - - sdata = phenocycler(tmp_path, read_all_bit_depths=True, regions=["B"], image_models_kwargs=_FAST) - - assert sorted(sdata.images) == ["B_8bit_image", "B_image"] - assert get_transformation(sdata["B_8bit_image"], get_all=True) == get_transformation(sdata["B_image"], get_all=True) - - # with both regions placed by their stage positions, the 8-bit image gets the translation of its region - sdata = phenocycler(tmp_path, read_all_bit_depths=True, image_models_kwargs=_FAST) - - assert sorted(sdata.images) == ["A_8bit_image", "A_image", "B_8bit_image", "B_image"] - translation = get_transformation(sdata["B_image"], "global") - assert isinstance(translation, Translation) - assert translation.translation[0] > 0 - for region in ("A", "B"): - expected = get_transformation(sdata[f"{region}_image"], get_all=True) - assert get_transformation(sdata[f"{region}_8bit_image"], get_all=True) == expected - - -def test_read_all_bit_depths_with_labels(tmp_path: Path) -> None: for suffix, dtype in ((".qptiff", "uint8"), (".er.qptiff", "uint16")): write_qptiff( tmp_path / f"S{suffix}", biomarkers=["DAPI"], shape=(64, 80), dtype=dtype, stage_position_cm=(1, 2) ) + write_qptiff(tmp_path / "T.qptiff", biomarkers=["DAPI"], shape=(64, 80), stage_position_cm=(1.1, 2)) mask = np.zeros((32, 40), dtype=np.uint32) - mask[2:4, 2:4], mask[5:7, 5:7] = 1, 2 tifffile.imwrite(tmp_path / "mask.tif", mask) + assert sorted(phenocycler(tmp_path, image_models_kwargs=_FAST).images) == ["S_image", "T_image"] sdata = phenocycler( - tmp_path, - labels=tmp_path / "mask.tif", - read_all_bit_depths=True, - image_models_kwargs=_FAST, + tmp_path / "S.er.qptiff", labels=tmp_path / "mask.tif", read_all_bit_depths=True, image_models_kwargs=_FAST ) + assert sorted(sdata.images) == ["S_8bit_image", "S_image"] + assert _level0(sdata, "S_image").dtype == np.uint16 + assert _level0(sdata, "S_8bit_image").dtype == np.uint8 + assert sdata.attrs["akoya"]["regions"]["S_8bit"]["path"].endswith("S.qptiff") image_transformations = get_transformation(sdata["S_image"], get_all=True) assert get_transformation(sdata["S_8bit_image"], get_all=True) == image_transformations # the mask is scaled onto the extended-range region only - assert list(sdata.labels) == ["S_labels"] labels_transformation = get_transformation(sdata["S_labels"], "global") assert isinstance(labels_transformation, Sequence) assert labels_transformation.transformations == [Scale([2, 2], axes=("y", "x")), image_transformations["global"]] - with pytest.raises(ValueError, match=r"Unknown region\(s\) \['S_8bit'\]"): phenocycler(tmp_path, labels={"S_8bit": tmp_path / "mask.tif"}, read_all_bit_depths=True) - - -def test_read_all_bit_depths_shape_mismatch_warns(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S.qptiff", biomarkers=["DAPI"], shape=(64, 80), dtype="uint8") - write_qptiff(tmp_path / "S.er.qptiff", biomarkers=["DAPI"], shape=(64, 96), dtype="uint16") - - with pytest.warns(UserWarning, match=r"8-bit scan .*\(64, 80\).*\(64, 96\)"): - sdata = phenocycler(tmp_path, read_all_bit_depths=True, image_models_kwargs=_FAST) - - assert sorted(sdata.images) == ["S_8bit_image", "S_image"] - - -def test_read_all_bit_depths_without_pair_reads_one(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S.qptiff", biomarkers=["DAPI"], dtype="uint8") - - sdata = phenocycler(tmp_path, read_all_bit_depths=True, image_models_kwargs=_FAST) - - assert list(sdata.images) == ["S_image"] - - -def test_read_all_bit_depths_region_name_clash_raises(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S.qptiff", biomarkers=["DAPI"], dtype="uint8") - write_qptiff(tmp_path / "S.er.qptiff", biomarkers=["DAPI"], dtype="uint16") - write_qptiff(tmp_path / "S_8bit.qptiff", biomarkers=["DAPI"], dtype="uint8") - + write_qptiff(tmp_path / "S_8bit.qptiff", biomarkers=["DAPI"]) with pytest.raises(ValueError, match="S_8bit"): - phenocycler(tmp_path, read_all_bit_depths=True, image_models_kwargs=_FAST) + phenocycler(tmp_path, read_all_bit_depths=True) def test_ignores_acquisition_files(tmp_path: Path) -> None: images = tmp_path / "raw" / "images" acquisition = images / "phenocycler" acquisition.mkdir(parents=True) - write_qptiff(images / "S_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) - write_qptiff(acquisition / "S_Scan1_Cycle1.qptiff.raw", biomarkers=["DAPI"]) - write_qptiff(acquisition / "S.raw.qptiff", biomarkers=["DAPI"]) - write_qptiff(acquisition / "S_Scan1_Cycle1.qptiff", biomarkers=["DAPI"]) - write_qptiff(images / "S_Scan1.qptiff.intermediate", biomarkers=["DAPI"]) - write_qptiff(images / "S_Scan1.raw.qptiff", biomarkers=["DAPI"]) - - sdata = phenocycler(tmp_path, image_models_kwargs=_FAST) + write_qptiff(images / "S_Scan1.QPTIFF", biomarkers=["DAPI", "CD3"]) + for name in ("phenocycler/S_Scan1_Cycle1.qptiff", "phenocycler/S.qptiff.raw", "S_Scan1.raw.qptiff"): + write_qptiff(images / name, biomarkers=["DAPI"]) - assert list(sdata.images) == ["S_image"] + assert list(phenocycler(tmp_path, image_models_kwargs=_FAST).images) == ["S_image"] -def test_discovery_ignores_suffix_case(tmp_path: Path) -> None: - write_qptiff(tmp_path / "A_Scan1.QPTIFF", biomarkers=["DAPI"]) - write_qptiff(tmp_path / "B_Scan1.ER.QPTIFF", biomarkers=["DAPI"]) - - sdata = phenocycler(tmp_path, image_models_kwargs=_FAST) - - assert sorted(sdata.images) == ["A_image", "B_image"] - - -def test_discovery_skips_hidden_files_and_follows_symlinks(tmp_path: Path) -> None: - root = tmp_path / "ds" - for folder in (".Trash-1000/files", "__MACOSX"): - (root / folder).mkdir(parents=True) - write_qptiff(root / folder / "A_Scan1.qptiff", biomarkers=["DAPI"]) - write_qptiff(root / "A_Scan1.qptiff", biomarkers=["DAPI"]) - (root / "._A_Scan1.qptiff").write_bytes(b"AppleDouble") - # a scan folder on another volume, and a link back up the tree - (tmp_path / "scratch").mkdir() - write_qptiff(tmp_path / "scratch" / "B_Scan1.qptiff", biomarkers=["DAPI"]) - (root / "scans").symlink_to(tmp_path / "scratch", target_is_directory=True) - (root / "loop").symlink_to(root, target_is_directory=True) - - sdata = phenocycler(root, image_models_kwargs=_FAST) - - assert sorted(sdata.images) == ["A_image", "B_image"] - - -def test_dataset_below_an_acquisition_like_folder_is_read(tmp_path: Path) -> None: - # only the folders below `path` are checked for the HuBMAP acquisition folder `raw/images/phenocycler/` - root = tmp_path / "raw" / "images" / "phenocycler" / "project" - root.mkdir(parents=True) - write_qptiff(root / "S_Scan1.qptiff", biomarkers=["DAPI"]) - - sdata = phenocycler(root, image_models_kwargs=_FAST) - - assert list(sdata.images) == ["S_image"] - - -@pytest.mark.parametrize("relative", [False, True]) -def test_sidecars_hubmap_layout(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, relative: bool) -> None: +def test_sidecars_hubmap_layout(tmp_path: Path) -> None: raw_images = tmp_path / "raw" / "images" processed = tmp_path / "lab_processed" / "images" raw_images.mkdir(parents=True) @@ -328,329 +177,111 @@ def test_sidecars_hubmap_layout(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, {"wellName": "A1", "items": [{"markerName": "DAPI"}]}, {"wellName": "A2", "items": [{"markerName": "DAPI"}, {"markerName": "CD3"}, {"markerName": "--"}]}, ] - xpd = {"formatType": "Protein", "wells": wells} - (raw_images / "S_Scan1.xpd").write_text(json.dumps(xpd)) - (processed / "S_Scan1.qptiff.channels.csv").write_text("Channel,Marker\n1,DAPI\n2,CD3\n") - - if relative: - monkeypatch.chdir(raw_images) + (raw_images / "S_Scan1.xpd").write_text(json.dumps({"wells": wells}), encoding="utf-8-sig") + (processed / "S_Scan1.qptiff.channels.csv").write_text("Channel,Marker\n1,DAPI\n2,CD3\n", encoding="utf-8-sig") # DAPI repeats per cycle and blanks do not count as channels, so the sidecars match the two pages with warnings.catch_warnings(): warnings.simplefilter("error", UserWarning) - sdata = phenocycler("." if relative else tmp_path, image_models_kwargs=_FAST) + sdata = phenocycler(tmp_path, image_models_kwargs=_FAST) raw = sdata.attrs["akoya"]["regions"]["S"]["raw"] - assert raw["xpd"] == xpd + assert raw["xpd"] == {"wells": wells} assert raw["channels_csv"] == [{"Channel": "1", "Marker": "DAPI"}, {"Channel": "2", "Marker": "CD3"}] -def test_sidecar_channel_mismatch_warns(tmp_path: Path) -> None: - path = tmp_path / "S_Scan1.qptiff" - write_qptiff(path, biomarkers=["DAPI", "CD3", "CD8"]) - xpd = {"wells": [{"wellName": "A1", "items": [{"markerName": "DAPI"}, {"markerName": "CD4"}]}]} - (tmp_path / "S_Scan1.xpd").write_text(json.dumps(xpd)) - - with pytest.warns(UserWarning, match="3") as record: - sdata = phenocycler(path, image_models_kwargs=_FAST) - - assert any("2" in str(warning.message) for warning in record) - assert get_channel_names(sdata.images["S_image"]) == ["DAPI", "CD3", "CD8"] - - -def test_channels_csv_mismatch_warns(tmp_path: Path) -> None: - path = tmp_path / "S_Scan1.qptiff" - write_qptiff(path, biomarkers=["DAPI", "CD3"]) - (tmp_path / "S_Scan1.qptiff.channels.csv").write_text("Marker\nDAPI\nCD3\nCD8\n") - - with pytest.warns(UserWarning, match="channels.csv"): - phenocycler(path, image_models_kwargs=_FAST) - - -def test_invalid_xpd_warns_and_is_ignored(tmp_path: Path) -> None: - path = tmp_path / "S_Scan1.qptiff" - write_qptiff(path, biomarkers=["DAPI"]) - (tmp_path / "S_Scan1.xpd").write_text("not json") - - with pytest.warns(UserWarning, match="not valid JSON"): - sdata = phenocycler(path, image_models_kwargs=_FAST) - - assert "xpd" not in sdata.attrs["akoya"]["regions"]["S"]["raw"] - - @pytest.mark.parametrize( - ("content", "match"), + ("name", "content", "match"), [ - (json.dumps({"wells": [{"items": ["DAPI"]}]}).encode(), "no readable markers"), - (json.dumps({"wells": "x"}).encode(), "no readable markers"), - (json.dumps({"wells": [{"items": [{"markerName": "DAPI"}]}]}).encode("utf-16"), "not valid"), + ("S_Scan1.xpd", json.dumps({"wells": [{"items": [{"markerName": "CD4"}]}]}).encode(), "lists 1 distinct"), + ("S_Scan1.qptiff.channels.csv", b"Marker\nDAPI\nCD3\nCD8\n", "channels.csv lists 3 distinct"), + ("S_Scan1.xpd", b"not json", "not valid JSON"), + ("S_Scan1.xpd", json.dumps({"wells": "x"}).encode(), "no readable markers"), + ("S_Scan1.qptiff.channels.csv", "Marker\nDAPI\n".encode("utf-16"), "Ignoring"), ], - ids=["items-not-objects", "wells-not-a-list", "utf-16"], ) -def test_malformed_xpd_warns_and_image_is_read(tmp_path: Path, content: bytes, match: str) -> None: - path = tmp_path / "S_Scan1.qptiff" - data = write_qptiff(path, biomarkers=["DAPI"]) - (tmp_path / "S_Scan1.xpd").write_bytes(content) +def test_bad_sidecars_warn(tmp_path: Path, name: str, content: bytes, match: str) -> None: + data = write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) + (tmp_path / name).write_bytes(content) - with pytest.warns(UserWarning, match=match) as record: - sdata = phenocycler(path, image_models_kwargs=_FAST) + with pytest.warns(UserWarning, match=match): + sdata = phenocycler(tmp_path / "S_Scan1.qptiff", image_models_kwargs=_FAST) - assert not any("distinct markers" in str(warning.message) for warning in record) + # the channel names always come from the qptiff + assert get_channel_names(sdata.images["S_image"]) == ["DAPI", "CD3"] np.testing.assert_array_equal(_level0(sdata, "S_image").values, data) -def test_xpd_without_wells_is_kept_without_warning(tmp_path: Path) -> None: - path = tmp_path / "S_Scan1.qptiff" - write_qptiff(path, biomarkers=["DAPI"]) - # a UTF-8 byte order mark, as Windows tools write it - (tmp_path / "S_Scan1.xpd").write_text(json.dumps({"formatType": "Protein"}), encoding="utf-8-sig") +def test_labels(tmp_path: Path) -> None: + for name in ("Slide 1", "Slide 2"): + write_qptiff(tmp_path / f"{name}.qptiff", biomarkers=["DAPI"]) + mask = _write_mask(tmp_path / "mask.tif") + _write_mask(tmp_path / "small.tif", shape=(200, 300)) - with warnings.catch_warnings(): - warnings.simplefilter("error", UserWarning) - sdata = phenocycler(path, image_models_kwargs=_FAST) - - assert sdata.attrs["akoya"]["regions"]["S"]["raw"]["xpd"] == {"formatType": "Protein"} - - -def test_channels_csv_with_byte_order_mark(tmp_path: Path) -> None: - path = tmp_path / "S_Scan1.qptiff" - write_qptiff(path, biomarkers=["DAPI", "CD3"]) - (tmp_path / "S_Scan1.qptiff.channels.csv").write_text("Marker\nDAPI\nCD3\nCD8\n", encoding="utf-8-sig") - - with pytest.warns(UserWarning, match="channels.csv lists 3 distinct markers"): - sdata = phenocycler(path, image_models_kwargs=_FAST) - - assert sdata.attrs["akoya"]["regions"]["S"]["raw"]["channels_csv"][0] == {"Marker": "DAPI"} - - -def test_unreadable_channels_csv_warns_and_is_ignored(tmp_path: Path) -> None: - path = tmp_path / "S_Scan1.qptiff" - write_qptiff(path, biomarkers=["DAPI"]) - (tmp_path / "S_Scan1.qptiff.channels.csv").write_bytes("Marker\nDAPI\n".encode("utf-16")) - - with pytest.warns(UserWarning, match="Ignoring"): - sdata = phenocycler(path, image_models_kwargs=_FAST) - - assert "channels_csv" not in sdata.attrs["akoya"]["regions"]["S"]["raw"] - - -def test_empty_dir_raises(tmp_path: Path) -> None: - with pytest.raises(FileNotFoundError, match="\\*.qptiff"): - phenocycler(str(tmp_path)) - - -def test_channel_names_override(tmp_path: Path) -> None: - path = tmp_path / "S_Scan1.qptiff" - write_qptiff(path, biomarkers=["DAPI", "CD3"]) - - sdata = phenocycler(path, channel_names=["nuclei", "T cells"], image_models_kwargs=_FAST) - assert get_channel_names(sdata.images["S_image"]) == ["nuclei", "T cells"] - channels = sdata.attrs["akoya"]["regions"]["S"]["channels"] - assert [channel["name_deduped"] for channel in channels] == ["nuclei", "T cells"] - - with pytest.raises(ValueError, match="expected 2"): - phenocycler(path, channel_names=["only one"]) - - -def _write_mask(path: Path, n: int = 5, shape: tuple[int, int] = (256, 320)) -> np.ndarray: - mask = np.zeros(shape, dtype=np.uint32) - for i in range(n): - mask[10 + 20 * i : 20 + 20 * i, 10:20] = i + 1 - tifffile.imwrite(path, mask) - return mask - - -def test_labels_without_cells_table(tmp_path: Path) -> None: - path = tmp_path / "S_Scan1.qptiff" - write_qptiff(path, biomarkers=["DAPI"]) - _write_mask(tmp_path / "mask.tif") - - sdata = phenocycler(path, labels=str(tmp_path / "mask.tif"), image_models_kwargs=_FAST) - - assert list(sdata.labels) == ["S_labels"] - assert len(sdata.tables) == 0 - - -def test_mapping_with_unknown_region_raises(tmp_path: Path) -> None: - write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI"]) - _write_mask(tmp_path / "A_mask.tif") - - with pytest.raises(ValueError, match="'Z'"): - phenocycler(tmp_path, labels={"Z": tmp_path / "A_mask.tif"}) - - -def test_mapping_keys_of_unselected_regions_are_ignored(tmp_path: Path) -> None: - for region in ("A", "B"): - write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) - _write_mask(tmp_path / f"{region}_mask.tif") - - sdata = phenocycler( - tmp_path, - regions=["A"], - labels={"A": tmp_path / "A_mask.tif", "B": tmp_path / "B_mask.tif"}, - image_models_kwargs=_FAST, - labels_models_kwargs=_FAST, - ) - - assert list(sdata.labels) == ["A_labels"] - assert sdata.attrs["akoya"]["labels"] == {"A": str(tmp_path / "A_mask.tif")} - - -def test_raw_region_names_select_sanitised_regions(tmp_path: Path) -> None: - write_qptiff(tmp_path / "Slide 1.qptiff", biomarkers=["DAPI", "CD3"]) - write_qptiff(tmp_path / "Slide 2.qptiff", biomarkers=["DAPI", "CD3"]) - _write_mask(tmp_path / "mask.tif") - - sdata = phenocycler( - tmp_path, regions=["Slide 1"], labels={"Slide 1": tmp_path / "mask.tif"}, image_models_kwargs=_FAST - ) - - assert list(sdata.images) == ["Slide_1_image"] - assert list(sdata.labels) == ["Slide_1_labels"] - - -def test_mapping_keys_colliding_after_sanitising_raise(tmp_path: Path) -> None: - write_qptiff(tmp_path / "Slide_1.qptiff", biomarkers=["DAPI", "CD3"]) - _write_mask(tmp_path / "mask.tif") - - with pytest.raises(ValueError, match="same region"): - phenocycler( + # keys may be raw names; keys of unselected regions are ignored; other shapes warn + with pytest.warns(UserWarning, match=r"\(200, 300\).*\(256, 320\).*not align"): + sdata = phenocycler( tmp_path, - labels={"Slide 1": tmp_path / "mask.tif", "Slide_1": tmp_path / "mask.tif"}, + regions=["Slide 1"], + labels={"Slide 1": tmp_path / "small.tif", "Slide_2": tmp_path / "mask.tif"}, image_models_kwargs=_FAST, ) + assert list(sdata.labels) == ["Slide_1_labels"] + assert get_transformation(sdata["Slide_1_labels"], get_all=True) == {"global": Identity()} + assert sdata.attrs["akoya"]["labels"] == {"Slide_1": str(tmp_path / "small.tif")} - -def test_mask_shape_mismatch_warns(tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - _write_mask(tmp_path / "mask.tif", shape=(200, 300)) - - with pytest.warns(UserWarning, match=r"\(200, 300\).*\(256, 320\)"): - sdata = phenocycler(tmp_path, labels=tmp_path / "mask.tif", image_models_kwargs=_FAST) - - assert list(sdata.labels) == ["S_labels"] - - -def test_regions_colliding_after_sanitising_raise(tmp_path: Path) -> None: - write_qptiff(tmp_path / "A [1].qptiff", biomarkers=["DAPI"]) - write_qptiff(tmp_path / "A_[1].qptiff", biomarkers=["DAPI"]) - - with pytest.raises(ValueError, match=r"same region\(s\) \['A__1_'\]"): - phenocycler(tmp_path, image_models_kwargs=_FAST) + sdata = phenocycler(tmp_path / "Slide 1.qptiff", labels=tmp_path / "mask.tif", labels_models_kwargs=_FAST) + np.testing.assert_array_equal(sdata.labels["Slide_1_labels"]["scale0"]["image"].values, mask) + assert not sdata.tables -def test_single_mask_many_regions_raises(tmp_path: Path) -> None: - for region in ("A", "B"): - write_qptiff(tmp_path / f"{region}_Scan1.qptiff", biomarkers=["DAPI"]) +@pytest.mark.parametrize( + ("labels", "match"), + [ + ("mask.tif", "mapping"), + ({"Z": "mask.tif"}, "'Z'"), + ({"Slide 1": "mask.tif", "Slide_1": "mask.tif"}, "same region"), + ], +) +def test_bad_labels_raise(tmp_path: Path, labels: str | dict[str, str], match: str) -> None: + for name in ("Slide 1", "Slide 2"): + write_qptiff(tmp_path / f"{name}.qptiff", biomarkers=["DAPI"]) _write_mask(tmp_path / "mask.tif") + resolved = tmp_path / labels if isinstance(labels, str) else {k: tmp_path / v for k, v in labels.items()} - with pytest.raises(ValueError, match="mapping"): - phenocycler(tmp_path, labels=tmp_path / "mask.tif") + with pytest.raises(ValueError, match=match): + phenocycler(tmp_path, labels=resolved) -def test_cli_labels_mapping(runner: CliRunner, tmp_path: Path) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI", "CD3"]) +def test_cli(runner: CliRunner, tmp_path: Path) -> None: + write_qptiff(tmp_path / "A_[1,2].qptiff", biomarkers=["DAPI", "HLA-A"]) + write_qptiff(tmp_path / "B.qptiff", biomarkers=["DAPI", "CD3"]) _write_mask(tmp_path / "mask.tif") - output = tmp_path / "out.zarr" + args = ["--input", str(tmp_path), "--image-models-kwargs", json.dumps(_FAST)] + # region ids and channel names may hold commas result = runner.invoke( phenocycler_wrapper, [ - "--input", - str(tmp_path / "S_Scan1.qptiff"), - "--output", - str(output), - "--labels", - json.dumps({"S": str(tmp_path / "mask.tif")}), - "--image-models-kwargs", - json.dumps(_FAST), - "--labels-models-kwargs", - json.dumps(_FAST), + *args, + "-o", + str(tmp_path / "a.zarr"), + "--regions", + "A_[1,2]", + *("--channel-names", "DAPI", "--channel-names", "HLA-A,B,C"), + *("--labels", json.dumps({"A_[1,2]": str(tmp_path / "mask.tif")})), ], ) - - assert result.exit_code == 0, result.output - sdata = read_zarr(output) - assert list(sdata.labels) == ["S_labels"] - - -@pytest.mark.parametrize("value", ['{"S": 1}', "{not json"]) -def test_cli_bad_mapping_is_bad_parameter(runner: CliRunner, tmp_path: Path, value: str) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - - result = runner.invoke( - phenocycler_wrapper, ["--input", str(tmp_path), "--output", str(tmp_path / "o.zarr"), "--labels", value] - ) - - assert result.exit_code == 2 - assert "labels" in result.output - - -def test_cli_regions_and_channel_names_may_hold_commas(runner: CliRunner, tmp_path: Path) -> None: - write_qptiff(tmp_path / "A_[1,2].qptiff", biomarkers=["DAPI", "HLA-A"]) - write_qptiff(tmp_path / "B.qptiff", biomarkers=["DAPI", "CD3"]) - args = ["--input", str(tmp_path), "--image-models-kwargs", json.dumps(_FAST)] - names = ["--channel-names", "DAPI", "--channel-names", "HLA-A,B,C"] - - result = runner.invoke(phenocycler_wrapper, [*args, "-o", str(tmp_path / "a.zarr"), "--regions", "A_[1,2]", *names]) assert result.exit_code == 0, result.output sdata = read_zarr(tmp_path / "a.zarr") - assert list(sdata.images) == ["A__1_2__image"] assert get_channel_names(sdata.images["A__1_2__image"]) == ["DAPI", "HLA-A,B,C"] - - result = runner.invoke( - phenocycler_wrapper, [*args, "-o", str(tmp_path / "b.zarr"), "--regions", "A__1_2_", "--regions", "B"] - ) - assert result.exit_code == 0, result.output - assert sorted(read_zarr(tmp_path / "b.zarr").images) == ["A__1_2__image", "B_image"] - - -@pytest.mark.parametrize("option", ["--image-models-kwargs", "--labels-models-kwargs"]) -def test_cli_kwargs_must_be_json_objects(runner: CliRunner, tmp_path: Path, option: str) -> None: - write_qptiff(tmp_path / "S_Scan1.qptiff", biomarkers=["DAPI"]) - - result = runner.invoke(phenocycler_wrapper, ["-i", str(tmp_path), "-o", str(tmp_path / "o.zarr"), option, "null"]) - - assert result.exit_code == 2 - assert "JSON object" in result.output - - -def test_str_regions_and_channel_names(tmp_path: Path) -> None: - write_qptiff(tmp_path / "AB_Scan1.qptiff", biomarkers=["DAPI"]) - write_qptiff(tmp_path / "C_Scan1.qptiff", biomarkers=["DAPI"]) - - sdata = phenocycler(tmp_path, regions="AB", channel_names="nuclei", image_models_kwargs=_FAST) - - assert list(sdata.images) == ["AB_image"] - assert get_channel_names(sdata.images["AB_image"]) == ["nuclei"] - - -def test_write_roundtrip(tmp_path: Path) -> None: - path = tmp_path / "S_Scan1.qptiff" - write_qptiff(path, biomarkers=["DAPI", "CD3", "CD3"]) - sdata = phenocycler(path, image_models_kwargs=_FAST) - - sdata.write(tmp_path / "x.zarr") - read = read_zarr(tmp_path / "x.zarr") - - assert get_channel_names(read.images["S_image"]) == get_channel_names(sdata.images["S_image"]) - assert read.attrs["akoya"] == sdata.attrs["akoya"] - assert read.attrs["spatialdata_io_reader"] == "phenocycler" - - -def test_is_lazy(tmp_path: Path) -> None: - path = tmp_path / "S_Scan1.qptiff" - written = write_qptiff(path, biomarkers=["DAPI", "CD3"]) - - with pytest.MonkeyPatch.context() as monkeypatch: - forbid_pixel_reads(monkeypatch) - sdata = phenocycler(path, image_models_kwargs=_FAST) - - assert isinstance(_level0(sdata, "S_image").data, da.Array) - np.testing.assert_array_equal(_level0(sdata, "S_image").values, written) - - -def test_phenocycler_keyword_only(tmp_path: Path) -> None: - write_qptiff(tmp_path / "A_Scan1.qptiff", biomarkers=["DAPI"]) - with pytest.raises(TypeError, match="positional argument"): - phenocycler(tmp_path, ["A"]) # type: ignore[misc] + assert list(sdata.labels) == ["A__1_2__labels"] + + for option, value, match in ( + ("--labels", '{"B": 1}', "labels"), + ("--labels", "{not json", "labels"), + ("--labels-models-kwargs", "null", "JSON object"), + ): + result = runner.invoke(phenocycler_wrapper, [*args, "-o", str(tmp_path / "b.zarr"), option, value]) + assert result.exit_code == 2 + assert match in result.output From 67a3d384d3b3b1f35fd5b93bb03bdd5ad4f526fc Mon Sep 17 00:00:00 2001 From: anon Date: Sun, 18 Oct 2026 21:55:44 +0200 Subject: [PATCH 107/107] docs(changelog): shorten the Akoya entries --- CHANGELOG.md | 57 ++++++++++++---------------------------------------- 1 file changed, 13 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23028555..543e616e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,22 +22,10 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. and the raster cell/nucleus labels are reconstructed by rasterizing the boundary polygons. The cell feature matrix falls back to a `cell_feature_matrix/` MatrixMarket directory or `cell_feature_matrix.tar.gz` when `cell_feature_matrix.h5` is missing. -- `phenocycler()` reader (and `phenocycler` CLI command) for Akoya PhenoCycler-Fusion 1.x/2.x `.qptiff` / - `.er.qptiff` scans, including PhenoImager (Opal) mode: lazy level-0 image with a recomputed pyramid, channel names - from the per-page XML, structured and raw scan metadata in `sdata.attrs["akoya"]`, optional `.xpd` / - `*channels.csv` sidecars, and optional segmentation masks (`labels`). The lazy images and masks can be pickled (e.g. - for `dask.distributed`): they open their file in each process on the first chunk read. -- `phenocycler(read_all_bit_depths=True)` (CLI `--read-all-bit-depths`) reads both the 8-bit `.qptiff` and the - extended-range `.er.qptiff` of a scan: the 8-bit one becomes the image `_8bit_image` with the transformations - of the `.er` region (a warning is raised if their shapes differ). By default only the extended-range file is read. -- The Akoya readers (`phenocycler()`, `codex()`) scale a segmentation mask downsampled from its region image by an - integer factor (exactly, or rounded up) onto the image (a `Scale` before the region's transformation); the cell - centroids stay in image pixels. Other shape mismatches warn. -- `phenocycler()` officially supports the PerkinElmer/Akoya QPTIFFs of Vectra Polaris and PhenoImager HT (generation - `"qpi_other"`): Opal and unmixed (uint16) scans, brightfield RGB scans (channels `r`, `g`, `b`) and inForm component - data (float32). A single `.tif` file is accepted if it is a QPI file. Region ids are made valid element names with - `spatialdata.sanitize_name` (e.g. the `[x,y]` of inForm tiles: `A_[1,2]` becomes `A__1_2_`); `regions` and the - keys of the `labels` mapping may hold the raw names. +- `phenocycler()` reader and CLI command for Akoya PhenoCycler-Fusion 1.x/2.x `.qptiff` / `.er.qptiff` scans + (incl. Opal mode) and other PerkinElmer/Akoya QPTIFFs (Vectra Polaris, PhenoImager HT, inForm component data): + lazy images with channel names from the page XML, scan metadata in `sdata.attrs["akoya"]`, optional `.xpd` / + `*channels.csv` sidecars and segmentation masks (`labels`); `read_all_bit_depths=True` also reads the 8-bit scan. ### Changed @@ -46,34 +34,15 @@ Release notes for `v0.7.1` and earlier are available on the [Releases][] page. and Dependabot updates. - Requires `tifffile>=2026.5.2` and `zarr>=3.2`, the first versions whose zarr store the lazy Akoya TIFF reads work with. - Requires `flowio>=1.4`, the first version with `FlowData.pnn_labels`, with which `codex()` detects CODEX FCS tables. -- `codex()` was rewritten to read CODEX Processor, CODEX Uploader and CODEX Instrument Manager outputs. This changes - its output and signature: - - Element names are now `{region}_image`, `{region}_cells` (circles), `{region}_labels` and `table` (previously - `images` and `{region}` for the shapes). - - Every region is read (previously only the first file found); `regions=[...]` selects a subset. - - Marker names in `table.var_names` come from the FCS `$PnS` keywords or the CSV header `cycCCC_chHHH:` - (previously `cycCCC_chHHH`); the image channels are named too. - - The positional signature `codex(path, fcs, imread_kwargs)` is unchanged; all new parameters (`regions`, - `channel_names`, `cells_table`, `labels` and the `*_models_kwargs`) are keyword-only, and the `codex` CLI command - exposes them (`--regions` and `--channel-names` are repeated per value, as in the `phenocycler` command). An empty `regions` raises a `ValueError`. - `imread_kwargs` is ignored, with a warning: every image is read lazily with `tifffile`. - - A single region is stored in `"global"`. Several regions are placed at their stage positions in `"global"` only - when all of them share a sample space (same slide id, stage positions and pixel size); otherwise each region, - including multi-region reads of unrelated regions, gets its own coordinate system named after the region. - - `channelNames.txt` and `experiment.json` are found in any case (e.g. SPACEc's `channelnames.txt`); CSV and FCS - files that are not CODEX Processor cell tables are ignored, also when their name holds a region id. Compensated - tables are preferred; other tables of the same region (e.g. of two segmentation runs) raise a `ValueError`. - A `cells_table` disables table discovery, and the `labels` masks are not read as images. The CSV and FCS tables - of one marker panel (e.g. with `fcs=False` and a region with only an FCS table) share one `table`, without - `var["source_column"]`. A table without `cycCCC_chHHH` columns reads its other numeric columns as the markers, - with a warning listing them. Without `labels`, a `.tif(f)` ending in `_mask` or `_labels` and named after a region - (e.g. `reg001_mask.ome.tiff`) is its mask. - - Region ids `regNNN` are taken from `reg` and digits not preceded by a letter (`exp_reg001`, not `Treg2`), also - by `phenocycler()` for a `.qptiff`. A CIM `.qptiff` without one is named as `phenocycler()` names it (without the - suffix and a trailing `_Scan`), and its `.er.qptiff` is preferred. Top-level `.tif` files of the original - layout are named after their sanitised stems. - - Hidden files and folders (e.g. macOS `._*` files, `__MACOSX/`, `.Trash-*/`) are ignored and symlinked folders - are followed, by `phenocycler()` too. +- `codex()` was rewritten to read CODEX Processor, CODEX Uploader and CODEX Instrument Manager (`.qptiff`) + outputs, sharing the QPTIFF core with `phenocycler()`. Breaking changes: + - Element names are `{region}_image`, `{region}_cells`, `{region}_labels` and `table` (were `images`, `{region}`). + - All regions are read (was the first file found); `regions=[...]` subsets. Unrelated regions get one coordinate + system each; regions of one slide are placed at their stage positions in `"global"`. + - `table.var_names` and image channels carry marker names (FCS `$PnS`, CSV `cycCCC_chHHH:`, + `channelNames.txt`), not `cycCCC_chHHH`. + - New keyword-only parameters `regions`, `channel_names`, `cells_table`, `labels`, `*_models_kwargs`; + `imread_kwargs` is ignored with a warning. ### Fixed