diff --git a/packages/zarr-metadata/changes/415.doc.md b/packages/zarr-metadata/changes/415.doc.md new file mode 100644 index 0000000000..6e4c1d2ce0 --- /dev/null +++ b/packages/zarr-metadata/changes/415.doc.md @@ -0,0 +1,2 @@ +Every public object now has an API reference entry with a docstring, and the +`struct` data type has its reference page. diff --git a/packages/zarr-metadata/docs/api/index.md b/packages/zarr-metadata/docs/api/index.md index 4a23c5c2fb..6dd79e7e89 100644 --- a/packages/zarr-metadata/docs/api/index.md +++ b/packages/zarr-metadata/docs/api/index.md @@ -45,9 +45,13 @@ The model layer's readers are imported from [`zarr_metadata.model`](model.md) directly, and the problem types every reader reports from [`zarr_metadata.problems`](problems.md). -## Common types +## Top-level names Two cross-cutting types are also exported from the top-level `zarr_metadata` namespace: `JSONValue`, documented in [`zarr_metadata.typed_json`](typed_json.md), and `ZarrV3NamedConfigJSON`, documented in [`zarr_metadata.v3`](v3/index.md). + +The package version is exported only from the top level: + +::: zarr_metadata.__version__ diff --git a/packages/zarr-metadata/docs/api/v3/data_type.md b/packages/zarr-metadata/docs/api/v3/data_type.md index 3f8444625a..4653779a9b 100644 --- a/packages/zarr-metadata/docs/api/v3/data_type.md +++ b/packages/zarr-metadata/docs/api/v3/data_type.md @@ -40,6 +40,8 @@ title: data_type ::: zarr_metadata.v3.data_type.string +::: zarr_metadata.v3.data_type.struct + ::: zarr_metadata.v3.data_type.numpy_datetime64 ::: zarr_metadata.v3.data_type.numpy_timedelta64 diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index abd93bc729..24d1ddc7f4 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -57,6 +57,7 @@ test = [ "jsonschema", "hypothesis", "pytest-examples>=0.0.18", + "griffelib", ] docs = [ # Pins match the zarr-python docs environment in the repo-root diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index bc57d90fd0..ecf5a23200 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -247,6 +247,7 @@ ) __version__ = version("zarr-metadata") +"""The installed version of the `zarr-metadata` distribution.""" __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index e866f907b4..b198a0e774 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -73,5 +73,13 @@ class ZarrV3NamedConfigJSON(TypedDict, closed=True): """ name: ReadOnly[str] + """The extension's name.""" configuration: NotRequired[ReadOnly[Mapping[str, JSONValue]]] + """The extension's parameters. May be absent.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this extension must refuse the document. + + Absent means `true`. The spec does not permit `false` for a data type, chunk + grid or chunk key encoding + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581). + """ diff --git a/packages/zarr-metadata/src/zarr_metadata/_json.py b/packages/zarr-metadata/src/zarr_metadata/_json.py index 8cc582c3f8..87a7c9a395 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_json.py @@ -130,14 +130,23 @@ class ValidationProblem: """ loc: tuple[str | int, ...] + """The path from the root of what was judged to the offending value. + + Keys and indices, e.g. `("codecs", 0, "name")` in a document; an empty + `loc` refers to that root. + """ message: str + """The human-readable description of what is wrong.""" kind: ProblemKind + """The kind of problem, for programmatic dispatch.""" input: JSONValue | UNSET = dataclasses.field( default=UNSET, kw_only=True, compare=False, repr=False ) + """The JSON found at `loc`, or `UNSET` when nothing is there or it is not JSON.""" ctx: Mapping[str, JSONValue] = dataclasses.field( default_factory=_no_ctx, kw_only=True, compare=False, repr=False ) + """What was expected, e.g. bounds or a closed set of values; empty when only a type was.""" def __post_init__(self) -> None: # Enforce the annotations at runtime. A rule written without a type @@ -215,6 +224,7 @@ class MetadataValidationError(ValueError): """ problems: tuple[ValidationProblem, ...] + """Every problem found, in the order found.""" def __init__(self, problems: Sequence[ValidationProblem]) -> None: self.problems = tuple(problems) diff --git a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py index 901d15a9b5..d53e6d707d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py @@ -259,8 +259,21 @@ class TypedDictKeys: """ members: Mapping[str, tuple[object, bool]] + """Every key it declares, its bases' included, with its value's type and whether it is required. + + The type has its qualifiers and `Annotated` metadata peeled. + """ extra_items: object + """What any other key may hold. + + `Never` when the TypedDict is closed, `object` when it is open, and the + `extra_items` type otherwise. + """ declared: bool + """Whether the TypedDict or a base said what other keys may hold, rather than defaulting. + + It is said with `closed=` or `extra_items=`. A TypedDict that says nothing is open. + """ @property def required(self) -> frozenset[str]: diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index 3e7ec67ef9..5fa17c14e8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -524,9 +524,13 @@ class Context(Protocol): """ @property - def zarr_format(self) -> Literal[2, 3]: ... + def zarr_format(self) -> Literal[2, 3]: + """The Zarr format of the documents this scope reads.""" + ... - def claimant(self, kind: type[D], name: str) -> D | None: ... + def claimant(self, kind: type[D], name: str) -> D | None: + """The definition of `kind` in scope that reads `name` (a name as a document uses it), or None if there is none.""" + ... @dataclass(frozen=True, slots=True) diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py index 11ad7e1563..d590cc058e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py @@ -51,9 +51,13 @@ class Conflict: """One place where two readings of a name disagree: the key, the definition one reading claimed, the definition the other found, and the location in the document when known.""" key: ClaimKey + """The kind, and the name the two definitions are filed under.""" claimed: Definition[Any] | None + """The definition one reading or scope gives the name; None when it leaves the name unclaimed.""" found: Definition[Any] | None + """The definition the other gives the name; None when it gives none, so the name would lose its meaning.""" loc: Loc | None = None + """Where the name occurs in the document; None when no document is available.""" written: str | None = None """The name the document uses at the conflict location, for example `r16` for a key of `r*`; None when no document is available.""" @@ -90,6 +94,7 @@ class ScopeConflictError(ValueError): def __init__(self, conflicts: Sequence[Conflict]) -> None: self.conflicts: tuple[Conflict, ...] = tuple(conflicts) + """Every conflict, in the order found; the message joins them.""" super().__init__("; ".join(str(conflict) for conflict in self.conflicts)) def __reduce__( diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py index ce749573ef..5ce52849f9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py @@ -420,15 +420,34 @@ class ZarrV2ArrayMetadataUpdate(TypedDict, total=False, closed=True): """ shape: tuple[int, ...] + """The `shape` member: the length of each dimension.""" dtype: ZarrV2DataTypeMetadata + """The `dtype` member: a NumPy typestr, or a list of field records for a structured data type.""" chunks: tuple[int, ...] + """The `chunks` member: the length of each chunk along each dimension, one for each dimension of `shape`.""" fill_value: JSONValue + """The `fill_value` member: the value of an element in a chunk that was never written, or `None` for none.""" order: ZarrV2ArrayOrder + """The `order` member: the layout of elements within each chunk, `"C"` or `"F"`.""" compressor: ZarrV2CodecMetadata | None + """The `compressor` member: the configuration of the codec that compresses each chunk, or `None` for none.""" filters: tuple[ZarrV2CodecMetadata, ...] | None + """The `filters` member: the configuration of each codec applied to each chunk before the compressor, in order, or `None` for none.""" dimension_separator: ZarrV2ArrayDimensionSeparator | UNSET + """The `dimension_separator` member: the separator between a chunk's grid coordinates in its key, `"."` or `"/"`. + + `UNSET` removes it, which means `"."`. + """ attributes: Mapping[str, JSONValue] | UNSET + """The `attributes` member: user attributes, which a store holds in `.zattrs`. + + `UNSET` removes it, so the array has no `.zattrs`. + """ extra_fields: Mapping[str, JSONValue | UNSET] + """The members the spec does not define, by key: each one given is set, or removed when its value is `UNSET`, and any other is kept. + + A key the spec defines, or that the update takes by name, raises `TypeError`. + """ class ZarrV2ArrayMetadata(Keyed): @@ -460,7 +479,12 @@ class ZarrV2ArrayMetadata(Keyed): __slots__ = ("_context", "_document", "_members", "_reading", "_shown") zarr_format: Final[Literal[2]] = 2 # noqa: PYI064 - a bare Final does not narrow a union of models + """The Zarr format version, `2`. A class constant: comparing it narrows a union of models.""" node_type: Final[Literal["array"]] = "array" # noqa: PYI064 - as above + """The kind of node, `"array"`. A class constant: comparing it narrows a union of models. + + A v2 document does not write this member: its `.zarray` key says the node is an array. + """ def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: context = scoped(context, CORE_V2) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 53d12254bc..f5e9b3a665 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -459,15 +459,37 @@ class ZarrV3ArrayMetadataUpdate(TypedDict, total=False, closed=True): """ shape: tuple[int, ...] + """The `shape` member: the length of each dimension.""" data_type: ZarrV3MetadataFieldJSON + """The `data_type` member: the data type of the array's elements.""" chunk_grid: ZarrV3MetadataFieldJSON + """The `chunk_grid` member: how the array is divided into chunks.""" chunk_key_encoding: ZarrV3MetadataFieldJSON + """The `chunk_key_encoding` member: how a chunk's grid coordinates become its store key.""" fill_value: JSONValue + """The `fill_value` member: the value of an element in a chunk that was never written.""" codecs: tuple[ZarrV3MetadataFieldJSON, ...] + """The `codecs` member: the codecs that encode each chunk, in order.""" attributes: Mapping[str, JSONValue] | UNSET + """The `attributes` member: user attributes. + + `UNSET` removes it, so the array has no attributes. + """ storage_transformers: tuple[ZarrV3MetadataFieldJSON, ...] | UNSET + """The `storage_transformers` member: the storage transformers, in order. + + `UNSET` removes it, which means none. + """ dimension_names: tuple[str | None, ...] | UNSET + """The `dimension_names` member: a name for each dimension, `None` for an unnamed one. + + `UNSET` removes it, which means every dimension is unnamed. + """ extra_fields: Mapping[str, ZarrV3ExtensionField | UNSET] + """The members the spec does not define, by key: each one given is set, or removed when its value is `UNSET`, and any other is kept. + + A key the spec defines, or that the update takes by name, raises `TypeError`. + """ class ZarrV3ArrayMetadata(Keyed): @@ -496,7 +518,9 @@ class ZarrV3ArrayMetadata(Keyed): __slots__ = ("_context", "_document", "_members", "_reading", "_shown") zarr_format: Final[Literal[3]] = 3 # noqa: PYI064 - a bare Final does not narrow a union of models + """The Zarr format version, `3`. A class constant: comparing it narrows a union of models.""" node_type: Final[Literal["array"]] = "array" # noqa: PYI064 - as above + """The kind of node, `"array"`. A class constant: comparing it narrows a union of models.""" def __init__(self, document: object, context: ZarrV3Context | None = None) -> None: context = scoped(context, CORE_AND_EXTENSIONS) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py index b26a46ca1b..d391b7f0cb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py @@ -238,6 +238,10 @@ class ZarrV2GroupMetadataUpdate(TypedDict, total=False): """The members `ZarrV2GroupMetadata.update` replaces: `attributes` in the form a `.zattrs` writes them, or `UNSET` for no `.zattrs`.""" attributes: Mapping[str, JSONValue] | UNSET + """The `attributes` member: user attributes, which a store holds in `.zattrs`. + + `UNSET` removes it, so the group has no `.zattrs`. + """ class ZarrV2GroupMetadata(Keyed): @@ -265,7 +269,12 @@ class ZarrV2GroupMetadata(Keyed): _shown: Mapping[str, JSONValue] zarr_format: Final[Literal[2]] = 2 # noqa: PYI064 - a bare Final does not narrow a union of models + """The Zarr format version, `2`. A class constant: comparing it narrows a union of models.""" node_type: Final[Literal["group"]] = "group" # noqa: PYI064 - as above + """The kind of node, `"group"`. A class constant: comparing it narrows a union of models. + + A v2 document does not write this member: its `.zgroup` key says the node is a group. + """ def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: context = scoped(context, CORE_V2) @@ -488,6 +497,7 @@ class ZarrV2ConsolidatedMetadata(Keyed): _metadata: dict[str, ZarrV2NodeMetadata] zarr_consolidated_format: Final = 1 + """The version of the consolidated metadata format, `1`.""" def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: context = scoped(context, CORE_V2) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index daae22cdfe..fd6ac0e845 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -132,8 +132,11 @@ class ZarrV3ConsolidatedMetadataInput(TypedDict, closed=True): """ kind: Literal["inline"] + """How the entries are carried, `"inline"`: in the group's own document.""" must_understand: Literal[False] + """Always `False`: a reader that does not recognize consolidated metadata may ignore it.""" metadata: Mapping[str, ZarrV3NodeMetadataInput] + """The document of each node below the group, or a model of one, by its path from the group.""" class ZarrV3GroupMetadataUpdate(TypedDict, total=False, closed=True): @@ -152,13 +155,25 @@ class ZarrV3GroupMetadataUpdate(TypedDict, total=False, closed=True): """ attributes: Mapping[str, JSONValue] | UNSET + """The `attributes` member: user attributes. + + `UNSET` removes it, so the group has no attributes. + """ consolidated_metadata: ( ZarrV3ConsolidatedMetadataJSON | Mapping[str, JSONValue] | ZarrV3ConsolidatedMetadata | UNSET ) + """The `consolidated_metadata` member: the documents of the nodes below the group, carried in its own document. + + `UNSET` removes it. + """ extra_fields: Mapping[str, ZarrV3ExtensionField | UNSET] + """The members the spec does not define, by key: each one given is set, or removed when its value is `UNSET`, and any other is kept. + + A key the spec defines, or that the update takes by name, raises `TypeError`. + """ class ZarrV3GroupMetadata(Keyed): @@ -182,7 +197,9 @@ class ZarrV3GroupMetadata(Keyed): __slots__ = ("_context", "_document", "_members", "_reading", "_shown") zarr_format: Final[Literal[3]] = 3 # noqa: PYI064 - a bare Final does not narrow a union of models + """The Zarr format version, `3`. A class constant: comparing it narrows a union of models.""" node_type: Final[Literal["group"]] = "group" # noqa: PYI064 - as above + """The kind of node, `"group"`. A class constant: comparing it narrows a union of models.""" def __init__(self, document: object, context: ZarrV3Context | None = None) -> None: context = scoped(context, CORE_AND_EXTENSIONS) @@ -491,7 +508,9 @@ class ZarrV3ConsolidatedMetadata(Keyed): __slots__ = ("_context", "_document", "_metadata") kind: Final = "inline" + """How the entries are carried, `"inline"`: in the group's own document.""" must_understand: Final = False + """Always `False`: a reader that does not recognize consolidated metadata may ignore it.""" def __init__(self, member: object, context: ZarrV3Context | None = None) -> None: context = scoped(context, CORE_AND_EXTENSIONS) diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index f369f2beec..86c8df35d1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -449,6 +449,7 @@ class InScope: """ context: ZarrV3Context | ZarrV2Context + """The scope the field is read in, in place of any scope the validation context holds.""" def __post_init__(self) -> None: if not _is_context(self.context): diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py index 3e3e5ac1b9..df4d336442 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py @@ -140,7 +140,9 @@ class ZarrV2DataTypeDefinition(WithFillValue[C], kind=True, format=2): """ label: ClassVar[str] = "v2 data type" + """The name of the kind in error messages, "v2 data type".""" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV2DataTypeField,) + """The field alias that annotates a configuration member holding a v2 dtype, `ZarrV2DataTypeField`. The type of each struct record is one.""" def _refusal(self) -> str | None: if parse_typestr(self.name) is not None: @@ -231,6 +233,7 @@ class ZarrV2CodecDefinition(Definition[C], kind=True, format=2): """ label: ClassVar[str] = "v2 codec" + """The name of the kind in error messages, "v2 codec".""" @classmethod def _name_problem(cls, name: str, at: Loc) -> ValidationProblem | None: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_field_json.py b/packages/zarr-metadata/src/zarr_metadata/v2/_field_json.py index f7a004bd17..e6f9763b22 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/_field_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_field_json.py @@ -40,6 +40,7 @@ class ZarrV2CodecMetadata(TypedDict, extra_items=JSONValue): """ id: str + """The numcodecs id that names the codec, such as `"blosc"`.""" __all__ = ["ZarrV2CodecMetadata", "ZarrV2DataTypeMetadata"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index f4bd7803a7..1232d01445 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -45,14 +45,39 @@ class ZarrV2ZArrayJSON(TypedDict): """ zarr_format: Literal[2] + """The Zarr format version, `2`.""" shape: tuple[Annotated[int, Ge(0)], ...] + """The length of each dimension of the array.""" chunks: tuple[Annotated[int, Ge(0)], ...] + """The length of each dimension of a chunk. + + Every chunk of the array has this shape, with one length per dimension of `shape`. + """ dtype: ZarrV2DataTypeMetadata + """The data type of the array's elements. + + A NumPy typestr such as `" ZarrV2ZlibParameters: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py index 4c5ad5c94b..f840ef531f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py @@ -23,22 +23,29 @@ class ZarrV2ShuffleParameters(TypedDict, closed=True): """`numcodecs.Shuffle(elementsize=4)`. An element size of 1 or less leaves the bytes as they are. numcodecs passes a larger one to C as a `Py_ssize_t`, so it is at most 2**63 - 1 on a 64-bit platform, and raises `OverflowError` for any other.""" elementsize: NotRequired[ReadOnly[Annotated[int, Le(2**63 - 1)]]] + """The size in bytes of the elements whose bytes are shuffled. Absent means 4.""" class ZarrV2DeltaParameters(TypedDict, closed=True): """`numcodecs.Delta(dtype, astype=None)`. When `astype` is omitted or null, it equals `dtype`.""" dtype: ReadOnly[str] + """The dtype of the decoded data.""" astype: NotRequired[ReadOnly[str | None]] + """The dtype of the encoded data. Absent or null means `dtype`.""" class ZarrV2FixedScaleOffsetParameters(TypedDict, closed=True): """`numcodecs.FixedScaleOffset(offset, scale, dtype, astype=None)`. When `astype` is omitted or null, it equals `dtype`.""" offset: ReadOnly[float | int] + """The value subtracted from each element before it is scaled.""" scale: ReadOnly[float | int] + """The factor that each element is multiplied by after the offset is subtracted. The product is rounded to the nearest integer.""" dtype: ReadOnly[str] + """The dtype of the decoded data.""" astype: NotRequired[ReadOnly[str | None]] + """The dtype of the encoded data. Absent or null means `dtype`.""" class ZarrV2QuantizeParameters(TypedDict, closed=True): @@ -50,21 +57,31 @@ class ZarrV2QuantizeParameters(TypedDict, closed=True): """ digits: ReadOnly[Annotated[int, Interval(ge=-308, le=307)]] + """The number of decimal digits of precision to keep.""" dtype: ReadOnly[str] + """The float dtype of the decoded data.""" astype: NotRequired[ReadOnly[str | None]] + """The float dtype of the encoded data. Absent or null means `dtype`.""" class ZarrV2BitRoundParameters(TypedDict, closed=True): """`numcodecs.BitRound(keepbits)`.""" keepbits: ReadOnly[Annotated[int, Ge(0)]] + """The number of mantissa bits to keep. + + numcodecs refuses, when encoding, a value above the mantissa width of the + data's float type: 10, 23 or 52 bits. + """ class ZarrV2AsTypeParameters(TypedDict, closed=True): """`numcodecs.AsType(encode_dtype, decode_dtype)`.""" encode_dtype: ReadOnly[str] + """The dtype of the encoded data.""" decode_dtype: ReadOnly[str] + """The dtype of the decoded data.""" def _shuffle_canonical(configuration: ZarrV2ShuffleParameters) -> ZarrV2ShuffleParameters: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py index 39b822f098..4497ab64e5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py @@ -15,6 +15,7 @@ class ZarrV2VLenArrayParameters(TypedDict, closed=True): """`numcodecs.VLenArray(dtype)`: the element type of each array.""" dtype: ReadOnly[str] + """The dtype of the elements of each variable-length array.""" def _vlen_array_canonical(configuration: ZarrV2VLenArrayParameters) -> ZarrV2VLenArrayParameters: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py index a6b8e4a58e..236a8616ab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/consolidated.py @@ -35,7 +35,12 @@ class ZarrV2ConsolidatedMetadataJSON(TypedDict): """ zarr_consolidated_format: Literal[1] + """The version of the consolidated metadata format, `1`.""" metadata: Mapping[str, ZarrV2ZArrayJSON | ZarrV2ZGroupJSON | ZarrV2ZAttrsJSON] + """The content of each metadata file in the hierarchy, by its store key. + + See the class docstring for the shape each key's value takes. + """ ZarrV2ConsolidatedMetadataStoreKey = Literal[".zmetadata"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py index 01a1fd42cd..9e4dbf4605 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py @@ -16,6 +16,7 @@ class ZarrV2ObjectConfiguration(TypedDict, closed=True): """The configuration that `|O` carries: a byte order, which the type ignores.""" byteorder: ReadOnly[ZarrV2ByteOrder] + """The byte order that the typestr writes. The canonical form writes `|`, as NumPy does.""" def _canonical(configuration: ZarrV2ObjectConfiguration) -> ZarrV2ObjectConfiguration: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py index 6f94fa793b..2f9ec2393b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py @@ -27,7 +27,14 @@ class ZarrV2ScalarConfiguration(TypedDict, closed=True): """The configuration that a scalar typestr carries: its byte order and its size in bytes, such as `{"byteorder": "<", "itemsize": 4}` for ` Iterator[ValidationProblem]: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py index ffa0379053..ffca4e8662 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py @@ -26,9 +26,17 @@ class ZarrV2TimeConfiguration(TypedDict, closed=True): """The configuration that a time typestr carries. For example, ``.""" itemsize: ReadOnly[Annotated[int, Ge(0)]] + """The size in bytes that the typestr writes, which must be 8.""" unit: NotRequired[ReadOnly[str]] + """The NumPy time unit in brackets, such as `ns`. The canonical form writes `μs` as `us`. + + An array's dtype must have one. Only a codec's dtype parameter may leave + it out, as in ` ZarrV2TimeConfiguration: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/group.py b/packages/zarr-metadata/src/zarr_metadata/v2/group.py index 2ddb5769d1..a1d0cf6530 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/group.py @@ -25,6 +25,7 @@ class ZarrV2ZGroupJSON(TypedDict, closed=True): """ zarr_format: Literal[2] + """The Zarr format version, `2`.""" class ZarrV2GroupMetadataJSON(TypedDict, closed=True): @@ -46,7 +47,12 @@ class ZarrV2GroupMetadataJSON(TypedDict, closed=True): """ zarr_format: Literal[2] + """The Zarr format version, `2`.""" attributes: NotRequired[Mapping[str, JSONValue]] + """User attributes from the sibling `.zattrs` file (not part of `.zgroup`). + + See the class docstring for the rationale behind the merged representation. + """ ZarrV2GroupMetadataStoreKey = Literal[".zgroup"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py index f67b861c9b..83b2418f00 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -188,7 +188,9 @@ class DataTypeDefinition(WithFillValue[C], kind=True, format=3): """ label: ClassVar[str] = "data type" + """The name of the kind in error messages, `"data type"`.""" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3DataTypeField,) + """The field alias that annotates a configuration member holding a data type, `ZarrV3DataTypeField`.""" @classmethod def _filed_under(cls, name: str) -> tuple[str | None, dict[str, JSONValue] | None]: @@ -246,7 +248,9 @@ class ChunkGridDefinition(Definition[C], kind=True, format=3): """ label: ClassVar[str] = "chunk grid" + """The name of the kind in error messages, `"chunk grid"`.""" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3ChunkGridField,) + """The field alias that annotates a configuration member holding a chunk grid, `ZarrV3ChunkGridField`.""" shape_rules: Callable[[C, Nested, tuple[int, ...]], Iterable[ValidationProblem]] = no_rules """The spec violations in this grid over an array of a given shape, located in the configuration.""" @@ -259,7 +263,9 @@ class ChunkKeyEncodingDefinition(Definition[C], kind=True, format=3): """A chunk key encoding.""" label: ClassVar[str] = "chunk key encoding" + """The name of the kind in error messages, `"chunk key encoding"`.""" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3ChunkKeyEncodingField,) + """The field alias that annotates a configuration member holding a chunk key encoding, `ZarrV3ChunkKeyEncodingField`.""" CodecKind = Literal["array_array", "array_bytes", "bytes_bytes"] @@ -320,10 +326,27 @@ class CodecDefinition(Definition[C], kind=True, format=3): """ label: ClassVar[str] = "codec" + """The name of the kind in error messages, `"codec"`.""" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3CodecField, ZarrV3StaticCodecField) + """The field aliases that annotate a configuration member holding a codec. + + `ZarrV3CodecField` admits any codec; `ZarrV3StaticCodecField` admits only + codecs of static size. + """ kind: CodecKind + """What the codec does to what it is handed: array -> array, array -> bytes or bytes -> bytes. + + It fixes where the codec may sit in a pipeline, and which of `chunk_rules`, + `transition` and `pipelines` it may be given: building a definition with + one its kind is never asked raises `TypeError`. + """ size: CodecSize + """Whether the size of what the codec gives out is fixed by the size of what it is handed. + + Where a member takes only codecs of static size, a codec of dynamic size + is a problem. + """ chunk_rules: Callable[[C, Nested, Chunk], Iterable[ValidationProblem]] = no_rules """The spec violations for this codec with a given input chunk, located in the configuration.""" transition: Callable[[C, Nested, Chunk], Chunk] = unknown_chunk @@ -366,7 +389,9 @@ class StorageTransformerDefinition(Definition[C], kind=True, format=3): """A storage transformer.""" label: ClassVar[str] = "storage transformer" + """The name of the kind in error messages, `"storage transformer"`.""" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3StorageTransformerField,) + """The field alias that annotates a configuration member holding a storage transformer, `ZarrV3StorageTransformerField`.""" @dataclass(frozen=True, kw_only=True, slots=True, repr=False) @@ -549,9 +574,11 @@ class ArrayExtraFieldDefinition(ExtraFieldDefinition[C], kind=True, format=3): """An extra field of a v3 array document: a member the spec does not define, read by its key.""" label: ClassVar[str] = "array extra field" + """The name of the kind in error messages, `"array extra field"`.""" spec_members: ClassVar[frozenset[str]] = frozenset( typeddict_keys(ZarrV3ArrayMetadataJSON).members ) + """The members the spec defines in an array document, the keys of `ZarrV3ArrayMetadataJSON`; never extra fields.""" @dataclass(frozen=True, kw_only=True, slots=True, repr=False) @@ -559,9 +586,11 @@ class GroupExtraFieldDefinition(ExtraFieldDefinition[C], kind=True, format=3): """An extra field of a v3 group document: a member the spec does not define, read by its key. `consolidated_metadata` is one.""" label: ClassVar[str] = "group extra field" + """The name of the kind in error messages, `"group extra field"`.""" spec_members: ClassVar[frozenset[str]] = frozenset( typeddict_keys(ZarrV3GroupMetadataJSON).members ) + """The members the spec defines in a group document, the keys of `ZarrV3GroupMetadataJSON`; never extra fields.""" STATIC_SIZE_ALIASES: Final[frozenset[TypeAliasType]] = frozenset({ZarrV3StaticCodecField}) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/array.py b/packages/zarr-metadata/src/zarr_metadata/v3/array.py index 6c815d40fd..36c03eed9a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/array.py @@ -31,16 +31,41 @@ class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=ZarrV3ExtensionField): """ zarr_format: Literal[3] + """The Zarr format version, `3`.""" node_type: Literal["array"] + """The kind of node the document describes, `"array"`.""" data_type: ZarrV3DataTypeField + """The data type of the array's elements, e.g. `"float64"`.""" shape: tuple[Annotated[int, Ge(0)], ...] + """The length of each dimension of the array.""" chunk_grid: ZarrV3ChunkGridField + """How the array is divided into chunks, e.g. a `"regular"` grid of one chunk shape.""" chunk_key_encoding: ZarrV3ChunkKeyEncodingField + """How a chunk's grid coordinates become its key in the store.""" fill_value: JSONValue + """The value of an element in a chunk that was never written. + + Which JSON values it may take depends on `data_type`. + """ codecs: tuple[ZarrV3CodecField, ...] + """The codecs that encode each chunk, in encoding order. + + Zero or more array -> array codecs, then exactly one array -> bytes codec, + then zero or more bytes -> bytes codecs, so the list is never empty + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L974-L983). + """ attributes: NotRequired[Mapping[str, JSONValue]] + """User attributes: arbitrary JSON values by string key.""" storage_transformers: NotRequired[tuple[ZarrV3StorageTransformerField, ...]] + """The storage transformers between the array and its store. + + Absent means none, as does an empty list. + """ dimension_names: NotRequired[tuple[str | None, ...]] + """A name for each dimension, `null` for an unnamed one. + + As long as `shape`; absent means every dimension is unnamed. + """ ZarrV3ArrayMetadataStoreKey = Literal["zarr.json"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 812a15fc9c..a893cae7bf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -38,15 +38,30 @@ class RectilinearChunkGridConfiguration(TypedDict, closed=True): """Configuration for the rectilinear chunk grid.""" kind: Literal["inline"] + """How `chunk_shapes` is encoded; `"inline"` is the only kind the spec defines.""" chunk_shapes: tuple[RectilinearDimSpec, ...] + """The chunk edge lengths along each dimension of the array, one entry per dimension. + + An entry is a bare integer, a step repeated until it covers the + dimension, or a list of edge lengths and `[length, count]` runs, whose + lengths must sum to at least the dimension's length. Every length and + count is at least 1. + """ class RectilinearChunkGridObject(TypedDict, closed=True): """Rectilinear chunk grid metadata in object form.""" name: RectilinearChunkGridName + """The chunk grid's name, `"rectilinear"`.""" configuration: RectilinearChunkGridConfiguration + """The chunk grid's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this chunk grid must refuse the document. + + Absent means `true`. The spec does not permit `false` for a chunk grid + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ RectilinearChunkGridMetadata = RectilinearChunkGridObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 21ebd5cca1..365e27ac74 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -38,14 +38,25 @@ class RegularChunkGridConfiguration(TypedDict, closed=True): """ chunk_shape: tuple[Annotated[int, Ge(1)], ...] + """The length of every chunk along each dimension of the array. + + One entry per dimension, each at least 1. + """ class RegularChunkGridObject(TypedDict, closed=True): """Regular chunk grid metadata in object form.""" name: RegularChunkGridName + """The chunk grid's name, `"regular"`.""" configuration: RegularChunkGridConfiguration + """The chunk grid's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this chunk grid must refuse the document. + + Absent means `true`. The spec does not permit `false` for a chunk grid + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ RegularChunkGridMetadata = RegularChunkGridObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index 25d62fd2d8..1e95a07a19 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -37,14 +37,28 @@ class DefaultChunkKeyEncodingConfiguration(TypedDict, closed=True): """ separator: NotRequired[DefaultChunkKeyEncodingSeparator] + """The character between the parts of a chunk key: `c/1/23/45` or `c.1.23.45`. + + Absent means `"/"`. + """ class DefaultChunkKeyEncodingObject(TypedDict, closed=True): """Default chunk key encoding metadata in object form.""" name: DefaultChunkKeyEncodingName + """The chunk key encoding's name, `"default"`.""" configuration: NotRequired[DefaultChunkKeyEncodingConfiguration] + """The chunk key encoding's parameters. + + Absent means every parameter takes its default: a `separator` of `"/"`. + """ must_understand: NotRequired[bool] + """Whether a reader that does not recognize this chunk key encoding must refuse the document. + + Absent means `true`. The spec does not permit `false` for a chunk key encoding + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ DefaultChunkKeyEncodingMetadata = DefaultChunkKeyEncodingObject | DefaultChunkKeyEncodingName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index 522d7b04b5..1162699a6b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -43,14 +43,28 @@ class V2ChunkKeyEncodingConfiguration(TypedDict, closed=True): """ separator: NotRequired[V2ChunkKeyEncodingSeparator] + """The character between the parts of a chunk key: `1.23.45` or `1/23/45`. + + Absent means `"."`. + """ class V2ChunkKeyEncodingObject(TypedDict, closed=True): """v2-compatibility chunk key encoding metadata in object form.""" name: V2ChunkKeyEncodingName + """The chunk key encoding's name, `"v2"`.""" configuration: NotRequired[V2ChunkKeyEncodingConfiguration] + """The chunk key encoding's parameters. + + Absent means every parameter takes its default: a `separator` of `"."`. + """ must_understand: NotRequired[bool] + """Whether a reader that does not recognize this chunk key encoding must refuse the document. + + Absent means `true`. The spec does not permit `false` for a chunk key encoding + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ V2ChunkKeyEncodingMetadata = V2ChunkKeyEncodingObject | V2ChunkKeyEncodingName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 51aadb5f22..f5672e9e61 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -37,18 +37,46 @@ class BloscCodecConfiguration(TypedDict, closed=True): """Configuration for the Zarr v3 `blosc` codec.""" cname: BloscCName + """The compressor blosc applies to each block, one of `BLOSC_CNAME`.""" clevel: Annotated[int, Interval(ge=0, le=9)] + """The compression level, an integer from 0 to 9. + + 0 turns compression off; 1 is the fastest and compresses least, 9 the + slowest and compresses most. + """ shuffle: BloscShuffle + """The shuffling applied before compression. + + `"noshuffle"` for none, `"shuffle"` for byte-wise, `"bitshuffle"` for + bit-wise. + """ blocksize: Annotated[int, Ge(0)] + """The size in bytes of the blocks the buffer is divided into before compression. + + At least 0; 0 lets blosc choose the size. + """ typesize: NotRequired[int] + """The stride in bytes over which shuffling is performed. + + Required, and at least 1, unless `shuffle` is `"noshuffle"`, under + which the spec ignores it and `canonicalize` drops it. + """ class BloscCodecObject(TypedDict, closed=True): """`blosc` codec metadata in object form.""" name: BloscCodecName + """The codec's name, `"blosc"`.""" configuration: BloscCodecConfiguration + """The codec's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ BloscCodecMetadata = BloscCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index e5b55e7697..037d702ad1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -38,6 +38,12 @@ class BytesCodecConfiguration(TypedDict, closed=True): """ endian: NotRequired[Endianness] + """The byte order in which each value is written. + + Required when the values of the data type the codec is handed take more + than one byte; a single-byte type such as `uint8` or `bool` does not + depend on it. + """ class BytesCodecObject(TypedDict, closed=True): @@ -50,8 +56,19 @@ class BytesCodecObject(TypedDict, closed=True): """ name: BytesCodecName + """The codec's name, `"bytes"`.""" configuration: NotRequired[BytesCodecConfiguration] + """The codec's parameters. + + May be absent when no parameter is set. + """ must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ BytesCodecMetadata = BytesCodecObject | BytesCodecName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 8094321161..50d644d5f9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -72,7 +72,17 @@ class ScalarMap(TypedDict, closed=True): """Optional encode/decode scalar overrides for the cast_value codec.""" encode: NotRequired[tuple[ScalarMapEntry, ...]] + """The `[input, output]` pairs applied when encoding: input in the array's data type, output in `data_type`. + + Each scalar is written in its data type's fill value encoding. Where an + input repeats, the first pair applies. + """ decode: NotRequired[tuple[ScalarMapEntry, ...]] + """The `[input, output]` pairs applied when decoding: input in `data_type`, output in the array's data type. + + Each scalar is written in its data type's fill value encoding. Where an + input repeats, the first pair applies. + """ class CastValueCodecConfiguration(TypedDict, closed=True): @@ -85,17 +95,44 @@ class CastValueCodecConfiguration(TypedDict, closed=True): """ data_type: ZarrV3DataTypeField + """The data type values are cast to when encoding, and the one the decoder is handed. + + The codec is defined only for data types that model real numbers, the + integer and floating-point types: a boolean, raw-bits, string, bytes, + struct or complex type is refused, here and as the data type the codec + is handed. + """ rounding: NotRequired[CastRoundingMode] + """How a value `data_type` cannot represent exactly is rounded. + + Absent means `"nearest-even"`. + """ out_of_range: NotRequired[CastOutOfRangeMode] + """How a value outside the range of `data_type` is brought into it, after rounding. + + `"clamp"` takes the nearest bound; `"wrap"` takes the value congruent + modulo 2^N, N being the width in bits of `data_type`, and is permitted + only when `data_type` is an integral type. Absent means an out-of-range + value is an encoding or decoding error. + """ scalar_map: NotRequired[ScalarMap] + """Scalars mapped explicitly, in place of `rounding` and `out_of_range`.""" class CastValueCodecObject(TypedDict, closed=True): """`cast_value` codec metadata in object form.""" name: CastValueCodecName + """The codec's name, `"cast_value"`.""" configuration: CastValueCodecConfiguration + """The codec's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ CastValueCodecMetadata = CastValueCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 537c8eb48a..044f9fe1b8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -30,8 +30,19 @@ class Crc32cCodecObject(TypedDict, closed=True): """ name: Crc32cCodecName + """The codec's name, `"crc32c"`.""" configuration: NotRequired[EmptyConfiguration] + """The codec's parameters, of which there are none. + + Absent, or an empty mapping. + """ must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ Crc32cCodecMetadata = Crc32cCodecObject | Crc32cCodecName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 66e535a165..1dc7220cfb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -33,14 +33,27 @@ class GzipCodecConfiguration(TypedDict, closed=True): """ level: Annotated[int, Interval(ge=0, le=9)] + """The compression level, an integer from 0 to 9. + + 0 turns compression off; 1 is the fastest and compresses least, 9 the + slowest and compresses most. + """ class GzipCodecObject(TypedDict, closed=True): """`gzip` codec metadata in object form.""" name: GzipCodecName + """The codec's name, `"gzip"`.""" configuration: GzipCodecConfiguration + """The codec's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ GzipCodecMetadata = GzipCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 5816d229f6..9880d07935 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -42,7 +42,17 @@ class ScaleOffsetCodecConfiguration(TypedDict, closed=True): """ offset: NotRequired[JSONValue] + """The quantity subtracted from each value when encoding. + + Absent means the additive identity, e.g. 0. A fill value of the data + type the codec is handed. + """ scale: NotRequired[JSONValue] + """The quantity each value is multiplied by when encoding, after `offset` is subtracted. + + Absent means the multiplicative identity, e.g. 1. A fill value of the + data type the codec is handed. + """ class ScaleOffsetCodecObject(TypedDict, closed=True): @@ -55,8 +65,20 @@ class ScaleOffsetCodecObject(TypedDict, closed=True): """ name: ScaleOffsetCodecName + """The codec's name, `"scale_offset"`.""" configuration: NotRequired[ScaleOffsetCodecConfiguration] + """The codec's parameters. + + May be absent when neither `offset` nor `scale` is set, which makes the + codec a no-op. + """ must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ ScaleOffsetCodecMetadata = ScaleOffsetCodecObject | ScaleOffsetCodecName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 17e702e213..73def2cafa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -60,17 +60,44 @@ class ShardingIndexedCodecConfiguration(TypedDict, closed=True): """ chunk_shape: tuple[Annotated[int, Ge(1)], ...] + """The shape of the inner chunks a shard is divided into. + + One length, at least 1, per axis of the shard the codec is handed, each + dividing the shard's length along that axis. + """ codecs: tuple[ZarrV3CodecField, ...] + """The codec pipeline that encodes each inner chunk. + + Like an array's `codecs`, it holds exactly one array -> bytes codec. + """ index_codecs: tuple[ZarrV3StaticCodecField, ...] + """The codec pipeline that encodes the shard index. + + It holds exactly one array -> bytes codec, and only codecs whose output + size is fixed by the size of their input, so no compressor. The spec + recommends a little-endian `bytes` codec followed by `crc32c`. + """ index_location: NotRequired[ShardingIndexLocation] + """Whether the shard index sits at the `"start"` or the `"end"` of the encoded shard. + + Absent means `"end"`. + """ class ShardingIndexedCodecObject(TypedDict, closed=True): """`sharding_indexed` codec metadata in object form.""" name: ShardingIndexedCodecName + """The codec's name, `"sharding_indexed"`.""" configuration: ShardingIndexedCodecConfiguration + """The codec's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ ShardingIndexedCodecMetadata = ShardingIndexedCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 59ff289def..d32886182e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -32,14 +32,27 @@ class TransposeCodecConfiguration(TypedDict, closed=True): """ order: tuple[int, ...] + """The reordering of axes applied when encoding: axis `i` of the encoded chunk is axis `order[i]` of the chunk the codec is handed. + + A permutation of `0, 1, ..., n-1`, `n` being the number of axes of the + chunk the codec is handed. + """ class TransposeCodecObject(TypedDict, closed=True): """`transpose` codec metadata in object form.""" name: TransposeCodecName + """The codec's name, `"transpose"`.""" configuration: TransposeCodecConfiguration + """The codec's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ TransposeCodecMetadata = TransposeCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index c6fdfd77e0..222346caac 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -36,15 +36,33 @@ class ZstdCodecConfiguration(TypedDict, closed=True): """ level: Annotated[int, Interval(ge=ZSTD_MIN_LEVEL, le=ZSTD_MAX_LEVEL)] + """The compression level, an integer from `ZSTD_MIN_LEVEL` (-131072) to `ZSTD_MAX_LEVEL` (22). + + 0 means zstd's default level; otherwise a higher level compresses more, + more slowly. It has no effect on decoding. + """ checksum: NotRequired[bool] + """Whether a checksum is stored when writing and verified when reading. + + Absent means `false`; the registry says to omit it rather than write + `false`. + """ class ZstdCodecObject(TypedDict, closed=True): """`zstd` codec metadata in object form.""" name: ZstdCodecName + """The codec's name, `"zstd"`.""" configuration: ZstdCodecConfiguration + """The codec's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ ZstdCodecMetadata = ZstdCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index a3aa01afb9..c3a76cdcd9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -42,8 +42,11 @@ class ZarrV3ConsolidatedMetadataEnvelopeJSON(TypedDict, closed=True): """ kind: Literal["inline"] + """How the entries are carried, `"inline"`: in the group's own document.""" must_understand: Literal[False] + """Always `false`: a reader that does not recognize consolidated metadata may ignore it.""" metadata: Mapping[str, UserJSONValue] + """The entries, by each node's path from the group, as any JSON.""" class ZarrV3ConsolidatedMetadataJSON(TypedDict, closed=True): @@ -59,8 +62,14 @@ class ZarrV3ConsolidatedMetadataJSON(TypedDict, closed=True): """ kind: Literal["inline"] + """How the entries are carried, `"inline"`: in the group's own document.""" must_understand: Literal[False] + """Always `false`: a reader that does not recognize consolidated metadata may ignore it.""" metadata: Mapping[str, ZarrV3ArrayMetadataJSON | ZarrV3GroupMetadataJSON] + """The metadata document of each node below the group, by its path from the group. + + A path joins its segments with `"/"`, e.g. `"child/grandchild"`. + """ ZARR_V3_CONSOLIDATED_METADATA_KEY: Final = "consolidated_metadata" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index bb51e3d16a..af4394abb9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -28,27 +28,34 @@ class NumpyDatetime64Configuration(TypedDict, closed=True): - """ - Configuration for the `numpy.datetime64` data type. - - Attributes - ---------- - unit - A string encoding a unit of time. - scale_factor - The multiplier relative to the unit. - """ + """Configuration for the `numpy.datetime64` data type.""" unit: ReadOnly[NumpyTimeUnit] + """A string encoding a unit of time. + + `"us"` and `"μs"` are equivalent spellings of microseconds; `"generic"` + means the unit is unspecified. + """ scale_factor: ReadOnly[NumpyTimeScaleFactor] + """The multiplier relative to the unit. + + An integer in `[1, 2147483647]`. + """ class NumpyDatetime64(TypedDict, closed=True): """`numpy.datetime64` data type metadata.""" name: NumpyDatetime64DataTypeName + """The data type's name, `"numpy.datetime64"`.""" configuration: NumpyDatetime64Configuration + """The data type's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this data type must refuse the document. + + Absent means `true`. The spec does not permit `false` for a data type + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ NumpyDatetime64FillValue = NumpyTimeTicks | Literal["NaT"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index 3e5485235c..f320226ee8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -24,27 +24,34 @@ class NumpyTimedelta64Configuration(TypedDict, closed=True): - """ - Configuration for the `numpy.timedelta64` data type. - - Attributes - ---------- - unit - A string encoding a unit of time. - scale_factor - The multiplier relative to the unit. - """ + """Configuration for the `numpy.timedelta64` data type.""" unit: ReadOnly[NumpyTimeUnit] + """A string encoding a unit of time. + + `"us"` and `"μs"` are equivalent spellings of microseconds; `"generic"` + means the unit is unspecified. + """ scale_factor: ReadOnly[NumpyTimeScaleFactor] + """The multiplier relative to the unit. + + An integer in `[1, 2147483647]`. + """ class NumpyTimedelta64(TypedDict, closed=True): """`numpy.timedelta64` data type metadata.""" name: NumpyTimedelta64DataTypeName + """The data type's name, `"numpy.timedelta64"`.""" configuration: NumpyTimedelta64Configuration + """The data type's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this data type must refuse the document. + + Absent means `true`. The spec does not permit `false` for a data type + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ NumpyTimedelta64FillValue = NumpyTimeTicks | Literal["NaT"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 7c8367349d..ffb478c182 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -62,6 +62,10 @@ class RawBytesConfiguration(TypedDict, closed=True): """ bits: int + """The size in bits that the name writes, `16` for `r16`. + + A positive multiple of 8. + """ def _rules(configuration: RawBytesConfiguration, nested: Nested) -> Iterator[ValidationProblem]: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index ac974198d4..f8916b170d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -31,34 +31,42 @@ class StructField(TypedDict, closed=True): - """ - A single field entry inside a structured dtype. - - Attributes - ---------- - name - The field name (must be unique within a struct and non-empty). - data_type - The field's data type. Recursive: may be a bare-string primitive - or a named-config envelope including another `struct`. - """ + """A single field entry inside a structured dtype.""" name: ReadOnly[str] + """The field name (must be unique within a struct and non-empty).""" data_type: ReadOnly[ZarrV3DataTypeField] + """The field's data type. + + Recursive: may be a bare-string primitive or a named-config envelope + including another `struct`. + """ class StructConfiguration(TypedDict, closed=True): """Configuration for the `struct` data type.""" fields: ReadOnly[tuple[StructField, ...]] + """The record's fields, in the order they are encoded. + + At least one. The order is significant: it is the order in which the + `bytes` codec packs the fields' values. + """ class Struct(TypedDict, closed=True): """`struct` data type metadata.""" name: StructDataTypeName + """The data type's name, `"struct"`.""" configuration: StructConfiguration + """The data type's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this data type must refuse the document. + + Absent means `true`. The spec does not permit `false` for a data type + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ StructFillValue = Mapping[str, JSONValue] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/group.py b/packages/zarr-metadata/src/zarr_metadata/v3/group.py index 73f9c88d87..99c0431192 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/group.py @@ -22,8 +22,11 @@ class ZarrV3GroupMetadataJSON(TypedDict, extra_items=ZarrV3ExtensionField): """ zarr_format: Literal[3] + """The Zarr format version, `3`.""" node_type: Literal["group"] + """The kind of node the document describes, `"group"`.""" attributes: NotRequired[Mapping[str, JSONValue]] + """User attributes: arbitrary JSON values by string key.""" ZarrV3GroupMetadataStoreKey = Literal["zarr.json"] diff --git a/packages/zarr-metadata/tests/test_api_docs.py b/packages/zarr-metadata/tests/test_api_docs.py new file mode 100644 index 0000000000..ccfb4e8cd9 --- /dev/null +++ b/packages/zarr-metadata/tests/test_api_docs.py @@ -0,0 +1,102 @@ +"""Every public module has an API reference page, and every public object the docstring it renders. + +The reference pages render every public name with mkdocstrings, which reads +docstrings statically, through griffe: a field's docstring is the string +literal under it, which the interpreter discards. So these tests read the +package the way the pages do, rather than through `__doc__`. + +Public means: a module whose dotted path has no segment starting with `_`, +each name in its `__all__`, and each member a public class declares itself +whose name does not start with `_`. +""" + +from __future__ import annotations + +import importlib +import pkgutil +import re +from pathlib import Path + +import griffe + +import zarr_metadata + +API_PAGES = Path(__file__).parent.parent / "docs" / "api" + +PACKAGE = griffe.load("zarr_metadata", resolve_aliases=True, resolve_external=False) + +PUBLIC_MODULES = [ + name + for name in [ + "zarr_metadata", + *( + module.name + for module in pkgutil.walk_packages(zarr_metadata.__path__, "zarr_metadata.") + ), + ] + if not any(part.startswith("_") for part in name.split(".")) +] + +# A strict on-disk v2 type whose fields mean what the merged in-memory +# type's same-named fields mean. Their field docstrings are the same text. +MIRRORS = [ + ("zarr_metadata.v2.array.ZarrV2ArrayMetadataJSON", "zarr_metadata.v2.array.ZarrV2ZArrayJSON"), + ("zarr_metadata.v2.group.ZarrV2GroupMetadataJSON", "zarr_metadata.v2.group.ZarrV2ZGroupJSON"), +] + + +def _griffe(path: str) -> griffe.Object: + """The object at a dotted path, followed through any re-export to its definition.""" + found = PACKAGE if path == "zarr_metadata" else PACKAGE[path.removeprefix("zarr_metadata.")] + return found.final_target if isinstance(found, griffe.Alias) else found + + +def _public_objects() -> dict[str, griffe.Object]: + """Each public module, name and class member, keyed by the path it is defined at.""" + found: dict[str, griffe.Object] = {} + for module_name in PUBLIC_MODULES: + module = _griffe(module_name) + found[module.path] = module + for name in importlib.import_module(module_name).__all__: + target = _griffe(f"{module_name}.{name}") + found[target.path] = target + if target.is_class: + for member_name, member in target.members.items(): + if not member_name.startswith("_") and not isinstance(member, griffe.Alias): + found[member.path] = member + return found + + +def test_public_modules_have_reference_pages() -> None: + # The top-level namespace re-exports names whose pages are their defining + # modules'; the API index renders the few it defines itself. + rendered = { + match + for page in API_PAGES.rglob("*.md") + for match in re.findall(r"^::: (\S+)$", page.read_text(), flags=re.MULTILINE) + } + assert sorted(set(PUBLIC_MODULES) - rendered - {"zarr_metadata"}) == [] + + +def test_public_objects_have_docstrings() -> None: + undocumented = sorted( + path + for path, obj in _public_objects().items() + if obj.docstring is None or not obj.docstring.value.strip() + ) + assert undocumented == [] + + +def test_mirrored_fields_share_docstrings() -> None: + drifted = [] + for source_path, mirror_path in MIRRORS: + source, mirror = _griffe(source_path), _griffe(mirror_path) + for name, field in mirror.members.items(): + if ( + name in source.members + and field.is_attribute + and getattr(field.docstring, "value", None) + != getattr(source.members[name].docstring, "value", None) + ): + drifted.append(f"{mirror_path}.{name}") + assert drifted == [] diff --git a/packages/zarr-metadata/uv.lock b/packages/zarr-metadata/uv.lock index eeacb00670..d0ae525310 100644 --- a/packages/zarr-metadata/uv.lock +++ b/packages/zarr-metadata/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.11" [[package]] @@ -1244,6 +1244,7 @@ docs = [ { name = "ruff" }, ] test = [ + { name = "griffelib" }, { name = "hypothesis" }, { name = "jsonschema" }, { name = "pydantic" }, @@ -1268,6 +1269,7 @@ docs = [ { name = "ruff", specifier = "==0.16.8" }, ] test = [ + { name = "griffelib" }, { name = "hypothesis" }, { name = "jsonschema" }, { name = "pydantic", specifier = ">=2.13" },