From 23b3d714d22913f0d4af86f4f16b6e745d2136a9 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 27 Oct 2026 21:17:46 +0100 Subject: [PATCH 1/3] docs(zarr-metadata): document every public object and give struct its reference page Every public module, `__all__` name, and member of a public class now carries the docstring its mkdocstrings page renders: TypedDict fields, model attributes and methods, the metadata key sets, and the top-level module and `__version__`. Numpy `Attributes` sections become per-field docstrings. The `struct` data type, which no page rendered, joins the data type page. tests/test_api_docs.py reads the package through griffe, as the pages do, and fails on an undocumented public object, a public module no page renders, or a Partial (or on-disk v2) type whose field docstrings drift from its counterpart's. Assisted-by: ClaudeCode:claude-opus-5-5 --- packages/zarr-metadata/docs/api/index.md | 8 +- .../zarr-metadata/docs/api/v3/data_type.md | 2 + packages/zarr-metadata/pyproject.toml | 2 +- .../src/zarr_metadata/__init__.py | 12 + .../src/zarr_metadata/_common.py | 8 + .../zarr-metadata/src/zarr_metadata/_json.py | 17 +- .../src/zarr_metadata/_typed_json.py | 24 +- .../src/zarr_metadata/model/_array.py | 228 +++++++++++++++++- .../src/zarr_metadata/model/_group.py | 167 +++++++++++++ .../src/zarr_metadata/model/_validation.py | 19 ++ .../src/zarr_metadata/v2/array.py | 84 ++++++- .../src/zarr_metadata/v2/codec.py | 1 + .../src/zarr_metadata/v2/consolidated.py | 5 + .../src/zarr_metadata/v2/group.py | 11 + .../src/zarr_metadata/v3/_definition.py | 22 +- .../src/zarr_metadata/v3/_registry.py | 1 + .../src/zarr_metadata/v3/array.py | 50 ++++ .../v3/chunk_grid/rectilinear.py | 15 ++ .../zarr_metadata/v3/chunk_grid/regular.py | 12 + .../v3/chunk_key_encoding/default.py | 14 ++ .../zarr_metadata/v3/chunk_key_encoding/v2.py | 14 ++ .../src/zarr_metadata/v3/codec/blosc.py | 28 +++ .../src/zarr_metadata/v3/codec/bytes.py | 17 ++ .../src/zarr_metadata/v3/codec/cast_value.py | 37 +++ .../src/zarr_metadata/v3/codec/crc32c.py | 11 + .../src/zarr_metadata/v3/codec/gzip.py | 13 + .../zarr_metadata/v3/codec/scale_offset.py | 22 ++ .../v3/codec/sharding_indexed.py | 27 +++ .../src/zarr_metadata/v3/codec/transpose.py | 13 + .../src/zarr_metadata/v3/codec/zstd.py | 18 ++ .../src/zarr_metadata/v3/consolidated.py | 6 + .../v3/data_type/numpy_datetime64.py | 26 +- .../v3/data_type/numpy_timedelta64.py | 26 +- .../src/zarr_metadata/v3/data_type/raw.py | 4 + .../src/zarr_metadata/v3/data_type/struct.py | 30 ++- .../src/zarr_metadata/v3/group.py | 6 + packages/zarr-metadata/tests/test_api_docs.py | 123 ++++++++++ packages/zarr-metadata/uv.lock | 4 +- 38 files changed, 1064 insertions(+), 63 deletions(-) create mode 100644 packages/zarr-metadata/tests/test_api_docs.py diff --git a/packages/zarr-metadata/docs/api/index.md b/packages/zarr-metadata/docs/api/index.md index 66c7bda75b..abf4a04ff9 100644 --- a/packages/zarr-metadata/docs/api/index.md +++ b/packages/zarr-metadata/docs/api/index.md @@ -32,11 +32,13 @@ are re-exported at the top level, so The model layer's validators, parsers, type guards, and metadata key sets are imported from [`zarr_metadata.model`](model.md) directly. -## Common types +## Top-level names -A few cross-cutting aliases are exported only from the top-level -`zarr_metadata` namespace: +A few names are exported only from the top-level `zarr_metadata` +namespace: two cross-cutting aliases, and the package version. ::: zarr_metadata.JSONValue ::: zarr_metadata.ZarrV3NamedConfigJSON + +::: zarr_metadata.__version__ diff --git a/packages/zarr-metadata/docs/api/v3/data_type.md b/packages/zarr-metadata/docs/api/v3/data_type.md index f482c33201..5ae699e5d9 100644 --- a/packages/zarr-metadata/docs/api/v3/data_type.md +++ b/packages/zarr-metadata/docs/api/v3/data_type.md @@ -40,6 +40,8 @@ title: data_type ::: zarr_metadata.v3.data_type.string +::: zarr_metadata.v3.data_type.struct + ::: zarr_metadata.v3.data_type.numpy_datetime64 ::: zarr_metadata.v3.data_type.numpy_timedelta64 diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 31d548bf2a..0030cbaefd 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -47,7 +47,7 @@ Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/z Documentation = "https://zarr-metadata.readthedocs.io/" [dependency-groups] -test = ["pytest", "pydantic>=2.13", "jsonschema", "hypothesis"] +test = ["pytest", "pydantic>=2.13", "jsonschema", "hypothesis", "griffelib"] docs = [ # Pins match the zarr-python docs environment in the repo-root # pyproject.toml so every site renders with the same toolchain, and the diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index 1a6b39f04d..96bd0487cb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -1,3 +1,14 @@ +"""Spec-defined metadata types, models, and validators for Zarr v2 and v3. + +The document types, models, and spec vocabulary — store keys, extension +names, and the literal types of their fields — are re-exported here from the +modules that define them, so `from zarr_metadata import ZarrV3ArrayMetadataJSON` +and `from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON` are +equivalent. Validators, parsers, and definitions are imported from their own +modules: `zarr_metadata.model`, `zarr_metadata.typed_json`, and +`zarr_metadata.v3.definition`. +""" + from importlib.metadata import version from zarr_metadata._common import JSONValue, ZarrV3NamedConfigJSON @@ -227,6 +238,7 @@ from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON, ZarrV3GroupMetadataJSONPartial __version__ = version("zarr-metadata") +"""The installed version of the `zarr-metadata` distribution.""" __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index 0751d56563..e18243754f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -60,5 +60,13 @@ class ZarrV3NamedConfigJSON(TypedDict, closed=True): """ name: ReadOnly[str] + """The extension's name.""" configuration: NotRequired[ReadOnly[Mapping[str, JSONValue]]] + """The extension's parameters. May be absent.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this extension must refuse the document. + + Absent means `true`. The spec does not permit `false` for a data type, chunk + grid or chunk key encoding + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581). + """ diff --git a/packages/zarr-metadata/src/zarr_metadata/_json.py b/packages/zarr-metadata/src/zarr_metadata/_json.py index a5c0a9b6af..70a06aee4c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_json.py @@ -37,18 +37,18 @@ @dataclass(frozen=True, slots=True) class ValidationProblem: - """A single problem found in a value: where it is, what is wrong, and what kind of wrong. - - `loc` is the path from the root of what was judged to the offending - value, e.g. `("codecs", 0, "name")` in a document, and an empty `loc` - refers to that root. - `kind` classifies the failure mode for programmatic dispatch; `message` - is the human-readable description. - """ + """A single problem found in a value: where it is, what is wrong, and what kind of wrong.""" loc: tuple[str | int, ...] + """The path from the root of what was judged to the offending value. + + Keys and indices, e.g. `("codecs", 0, "name")` in a document; an empty + `loc` refers to that root. + """ message: str + """The human-readable description of what is wrong.""" kind: ProblemKind + """What kind of wrong it is, for programmatic dispatch.""" def __post_init__(self) -> None: # The runtime half of the annotations: a rule written without a type @@ -84,6 +84,7 @@ class MetadataValidationError(ValueError): """ problems: tuple[ValidationProblem, ...] + """Every problem found, in the order found.""" def __init__(self, problems: Sequence[ValidationProblem]) -> None: self.problems = tuple(problems) diff --git a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py index b8dbc1cd80..c392ee5ba0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py @@ -199,20 +199,24 @@ def alias_value(alias: typing_extensions.TypeAliasType) -> object: @dataclass(frozen=True, slots=True) class TypedDictKeys: - """What a TypedDict says of an object's keys, read as the typing spec defines it. - - `members` holds every key it declares, its bases' included, with the - type of the key's value -- qualifiers and `Annotated` metadata peeled - -- and whether the key is required. `extra_items` is what any other - key may hold: `Never` when the TypedDict is closed, `object` when it - is open, and the `extra_items` type otherwise. `declared` is whether - that was said, by the TypedDict or a base, rather than defaulted: a - TypedDict that says nothing is open. - """ + """What a TypedDict says of an object's keys, read as the typing spec defines it.""" members: Mapping[str, tuple[object, bool]] + """Every key it declares, its bases' included, with its value's type and whether it is required. + + The type has its qualifiers and `Annotated` metadata peeled. + """ extra_items: object + """What any other key may hold. + + `Never` when the TypedDict is closed, `object` when it is open, and the + `extra_items` type otherwise. + """ declared: bool + """Whether the TypedDict or a base said what other keys may hold, rather than defaulting. + + It is said with `closed=` or `extra_items=`. A TypedDict that says nothing is open. + """ @property def required(self) -> frozenset[str]: diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 6d3979dba3..16e6be8bfe 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -59,10 +59,30 @@ class ZarrV3NamedConfig: """ name: str + """The extension's name, e.g. `"gzip"` or `"regular"`.""" configuration: dict[str, JSONValue] + """The extension's parameters. + + `{}` when the document gave a bare name or omitted `configuration`. + """ must_understand: bool = True + """Whether a reader that does not recognize this extension must refuse the document. + + Absent means `True`. The spec does not permit `False` for a data type, + chunk grid or chunk key encoding + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581), + and `ZarrV3ArrayMetadata.from_json` refuses it at every extension point, + codecs and storage transformers included. + """ def to_json(self) -> ZarrV3MetadataFieldJSON: + """Return the model as a v3 metadata field, in its shortest spelling. + + The bare name when `configuration` is empty and `must_understand` is + `True`; otherwise an object with `name`, with `configuration` only when + it is not empty, and with `must_understand: false` only when it is + `False`. The result shares no mutable state with the model. + """ if not self.configuration and self.must_understand: return self.name # `configuration` is ReadOnly, so it is set in the literal rather than @@ -79,6 +99,17 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: @classmethod def from_json(cls, data: object) -> ZarrV3NamedConfig: + """Return the model of the v3 metadata field `data`, or raise `MetadataValidationError`. + + `data` is a bare name string, or an object with a string `name` and + optional `configuration` (an object of string keys and JSON values) + and boolean `must_understand` members; any other member is a problem, + and every problem found is raised together. A bare name or an absent + `configuration` reads as `{}`, and an absent `must_understand` as + `True`. Only the envelope is judged: the configuration is not read + through any definition, and `must_understand: false` is accepted. The + model shares no mutable state with `data`. + """ field = parse_metadata_field_v3(data) if isinstance(field, str): return cls(name=field, configuration={}, must_understand=True) @@ -141,15 +172,52 @@ class ZarrV3ArrayMetadataPartial(TypedDict, total=False): """ shape: tuple[int, ...] + """The length of each dimension of the array.""" fill_value: JSONValue + """The JSON value of an element no chunk has written. + + Its permitted spellings depend on `data_type`; the model holds it as the + document wrote it, uninterpreted. + """ data_type: ZarrV3MetadataField + """The array's data type.""" chunk_grid: ZarrV3MetadataField + """How the array is divided into chunks.""" codecs: tuple[ZarrV3MetadataField, ...] + """The codecs that encode each chunk, in the order they encode it. + + A valid document holds an array-to-bytes codec, so this is not empty + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L582-L583). + """ chunk_key_encoding: ZarrV3MetadataField + """How a chunk's grid coordinates map to its store key.""" dimension_names: tuple[str | None, ...] | UNSET + """A name, or `None` for an unnamed dimension, for each dimension of `shape`. + + `UNSET` when the document omits the key, which leaves every dimension + unnamed. + """ attributes: dict[str, JSONValue] + """User attributes. + + A document without the key and one with an empty object both read as + `{}`, and `to_json` omits an empty mapping. + """ storage_transformers: tuple[ZarrV3MetadataField, ...] + """The storage transformers, in order. + + A document without the key and one with an empty list both read as `()`, + which the spec gives the same meaning + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L627-L628), + and `to_json` omits an empty tuple. + """ extra_fields: dict[str, ZarrV3ExtensionField] + """Each top-level member outside the standard array keys, by name, as the document wrote it. + + A key in `ARRAY_METADATA_STANDARD_KEYS_V3` is refused on construction with + `MetadataValidationError`. `must_understand_fields` is the subset a reader + must recognize. + """ @dataclass(frozen=True, slots=True, kw_only=True) @@ -170,17 +238,56 @@ class ZarrV3ArrayMetadata: """ zarr_format: Literal[3] = field(default=3, init=False) + """The Zarr format version, always `3`.""" node_type: Literal["array"] = field(default="array", init=False) + """The node type, always `"array"`.""" shape: tuple[int, ...] + """The length of each dimension of the array.""" fill_value: JSONValue + """The JSON value of an element no chunk has written. + + Its permitted spellings depend on `data_type`; the model holds it as the + document wrote it, uninterpreted. + """ data_type: ZarrV3MetadataField + """The array's data type.""" chunk_grid: ZarrV3MetadataField + """How the array is divided into chunks.""" codecs: tuple[ZarrV3MetadataField, ...] + """The codecs that encode each chunk, in the order they encode it. + + A valid document holds an array-to-bytes codec, so this is not empty + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L582-L583). + """ chunk_key_encoding: ZarrV3MetadataField + """How a chunk's grid coordinates map to its store key.""" dimension_names: tuple[str | None, ...] | UNSET + """A name, or `None` for an unnamed dimension, for each dimension of `shape`. + + `UNSET` when the document omits the key, which leaves every dimension + unnamed. + """ attributes: dict[str, JSONValue] + """User attributes. + + A document without the key and one with an empty object both read as + `{}`, and `to_json` omits an empty mapping. + """ storage_transformers: tuple[ZarrV3MetadataField, ...] + """The storage transformers, in order. + + A document without the key and one with an empty list both read as `()`, + which the spec gives the same meaning + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L627-L628), + and `to_json` omits an empty tuple. + """ extra_fields: dict[str, ZarrV3ExtensionField] + """Each top-level member outside the standard array keys, by name, as the document wrote it. + + A key in `ARRAY_METADATA_STANDARD_KEYS_V3` is refused on construction with + `MetadataValidationError`. `must_understand_fields` is the subset a reader + must recognize. + """ @classmethod def create_default(cls, **overrides: Unpack[ZarrV3ArrayMetadataPartial]) -> ZarrV3ArrayMetadata: @@ -261,6 +368,15 @@ def __post_init__(self) -> None: ) def to_json(self) -> ZarrV3ArrayMetadataJSON: + """Return the model as a v3 array document. + + Each extension point is written by `ZarrV3NamedConfig.to_json`, in its + shortest spelling. `dimension_names` is written unless it is `UNSET`, + `attributes` and `storage_transformers` only when they are not empty, + and each entry of `extra_fields` as a top-level member. The document is + not validated (`to_key_value` validates it) and shares no mutable + state with the model. + """ # to_json output shares no mutable state with the model: every value # that can hold a mutable container is deep-copied. out: ZarrV3ArrayMetadataJSON = { @@ -293,6 +409,15 @@ def to_json(self) -> ZarrV3ArrayMetadataJSON: def from_json( cls, data: object, *, context: Context = CORE_AND_EXTENSIONS ) -> ZarrV3ArrayMetadata: + """Return the model of the v3 array document `data`, or raise `MetadataValidationError`. + + `data` is a decoded `zarr.json` document. It is validated as + `parse_array_metadata_v3` validates it, each extension point read + through the definition that claims its name in `context`, and every + problem found is raised together. Each top-level member outside the + standard keys lands in `extra_fields`. The model shares no mutable + state with `data`. + """ # A read model shares no mutable state with what it read. parsed = copy.deepcopy(parse_array_metadata_v3(data, context=context)) extra_fields: dict[str, ZarrV3ExtensionField] = { @@ -329,6 +454,14 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: def from_key_value( cls, mapping: Mapping[StoreKey, bytes], *, context: Context = CORE_AND_EXTENSIONS ) -> ZarrV3ArrayMetadata: + """Return the model of the v3 array stored at `zarr.json` in `mapping`, or raise `MetadataValidationError`. + + `mapping` maps store keys to the bytes stored there; other keys are + ignored. A missing `zarr.json` (a `missing_key` problem), a value that + is not `bytes` (`invalid_type`), or bytes that do not decode as JSON + (`invalid_json`) raise, as does every problem `from_json` finds with + the document in `context`. + """ return cls.from_json( load_store_json(mapping, ZARR_V3_ARRAY_METADATA_STORE_KEY), context=context ) @@ -336,6 +469,14 @@ def from_key_value( def to_key_value( self, *, indent: int | str | None = None, context: Context = CORE_AND_EXTENSIONS ) -> Mapping[ZarrV3ArrayMetadataStoreKey, bytes]: + """Return the model as the bytes of its `zarr.json` document, or raise `MetadataValidationError`. + + The document `to_json` returns is validated as `from_json` validates + one in `context`, since a model built or updated by hand is not, and + every problem found is raised together. `indent` is `json.dumps`'s: + `None` writes the document on one line, and an int or a string + pretty-prints it with that indent. + """ # A model built by hand is not validated: its document is written only # if it reads as `from_json` reads one in `context`, and every problem # is raised. @@ -357,22 +498,48 @@ class ZarrV2ArrayMetadataPartial(TypedDict, total=False): """ shape: tuple[int, ...] + """The length of each dimension of the array.""" dtype: ZarrV2DataTypeMetadata + """The array's data type, as the document spells it. + + A type string such as `" ZarrV2ArrayMetadata: """ @@ -466,6 +659,17 @@ def to_json(self) -> ZarrV2ArrayMetadataJSON: @classmethod def from_json(cls, data: object) -> ZarrV2ArrayMetadata: + """Return the model of the merged v2 array document `data`, or raise `MetadataValidationError`. + + `data` is a decoded `.zarray` document, with the `.zattrs` content + folded in as an `attributes` member when there is one (the + `ZarrV2ArrayMetadataJSON` form). It is validated as + `parse_array_metadata_v2` validates it, structure only, and every + problem found is raised together. An absent `dimension_separator` + reads as `"."` and an absent `attributes` as `UNSET`. Any other member + is checked to be JSON and then dropped: the model does not hold it. + The model shares no mutable state with `data`. + """ # A read model shares no mutable state with what it read. parsed = copy.deepcopy(parse_array_metadata_v2(data)) return cls( @@ -482,6 +686,16 @@ def from_json(cls, data: object) -> ZarrV2ArrayMetadata: @classmethod def from_key_value(cls, mapping: Mapping[StoreKey, bytes]) -> ZarrV2ArrayMetadata: + """Return the model of the v2 array stored at `.zarray` and `.zattrs` in `mapping`, or raise `MetadataValidationError`. + + `mapping` maps store keys to the bytes stored there; other keys are + ignored. `.zattrs`, when present, becomes `attributes`; when absent, + `attributes` is `UNSET`. A missing `.zarray` (a `missing_key` + problem), a value that is not `bytes` (`invalid_type`), bytes that do + not decode as JSON (`invalid_json`), or an `attributes` member inside + `.zarray` (`invalid_value`) raise, as does every problem `from_json` + finds with the merged document. + """ zarray_raw = load_store_json(mapping, ZARR_V2_ARRAY_METADATA_STORE_KEY) if not isinstance(zarray_raw, Mapping): return cls.from_json(zarray_raw) @@ -504,6 +718,16 @@ def from_key_value(cls, mapping: Mapping[StoreKey, bytes]) -> ZarrV2ArrayMetadat def to_key_value( self, *, indent: int | str | None = None ) -> Mapping[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: + """Return the model as the bytes of its `.zarray` and `.zattrs` documents, or raise `MetadataValidationError`. + + The merged document `to_json` returns is validated as `from_json` + validates one, since a model built or updated by hand is not, and + every problem found is raised together. `.zarray` holds every member + but `attributes`; `.zattrs` holds `attributes`, and is present exactly + when `attributes` is not `UNSET`, even when it is empty. `indent` is + `json.dumps`'s: `None` writes each document on one line, and an int or + a string pretty-prints it with that indent. + """ # Attributes live only in the sibling `.zattrs` file; the `.zarray` # document must exclude them. The `.zattrs` key is present exactly # when attributes are set (even empty) — UNSET emits no file. A model diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index fa56f633aa..8fe8176935 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -64,8 +64,24 @@ class ZarrV3GroupMetadataPartial(TypedDict, total=False): """ attributes: dict[str, JSONValue] + """User attributes. + + A document without the key and one with an empty object both read as + `{}`, and `to_json` omits an empty mapping. + """ consolidated_metadata: ZarrV3ConsolidatedMetadata | UNSET + """The group's inline consolidated metadata, or `UNSET` when the document has none. + + A document's `consolidated_metadata: null` reads as `UNSET` and is not + written back. + """ extra_fields: dict[str, ZarrV3ExtensionField] + """Each top-level member outside the standard group keys and `consolidated_metadata`, by name, as the document wrote it. + + A key in `GROUP_METADATA_STANDARD_KEYS_V3`, or `consolidated_metadata`, is + refused on construction with `MetadataValidationError`. + `must_understand_fields` is the subset a reader must recognize. + """ @dataclass(frozen=True, slots=True, kw_only=True) @@ -80,10 +96,28 @@ class ZarrV3GroupMetadata: """ zarr_format: Literal[3] = field(default=3, init=False) + """The Zarr format version, always `3`.""" node_type: Literal["group"] = field(default="group", init=False) + """The node type, always `"group"`.""" attributes: dict[str, JSONValue] + """User attributes. + + A document without the key and one with an empty object both read as + `{}`, and `to_json` omits an empty mapping. + """ consolidated_metadata: ZarrV3ConsolidatedMetadata | UNSET + """The group's inline consolidated metadata, or `UNSET` when the document has none. + + A document's `consolidated_metadata: null` reads as `UNSET` and is not + written back. + """ extra_fields: dict[str, ZarrV3ExtensionField] + """Each top-level member outside the standard group keys and `consolidated_metadata`, by name, as the document wrote it. + + A key in `GROUP_METADATA_STANDARD_KEYS_V3`, or `consolidated_metadata`, is + refused on construction with `MetadataValidationError`. + `must_understand_fields` is the subset a reader must recognize. + """ def __post_init__(self) -> None: reserved = GROUP_METADATA_STANDARD_KEYS_V3 | {ZARR_V3_CONSOLIDATED_METADATA_KEY} @@ -122,6 +156,14 @@ def update(self, **kwargs: Unpack[ZarrV3GroupMetadataPartial]) -> ZarrV3GroupMet return dataclasses.replace(self, **kwargs) def to_json(self) -> ZarrV3GroupMetadataJSON: + """Return the model as a v3 group document. + + `attributes` is written only when it is not empty, + `consolidated_metadata` unless it is `UNSET`, and each entry of + `extra_fields` as a top-level member. The document is not validated + (`to_key_value` validates it) and shares no mutable state with the + model. + """ # to_json output shares no mutable state with the model: every value # that can hold a mutable container is deep-copied. out: ZarrV3GroupMetadataJSON = { @@ -141,6 +183,16 @@ def to_json(self) -> ZarrV3GroupMetadataJSON: def from_json( cls, data: object, *, context: Context = CORE_AND_EXTENSIONS ) -> ZarrV3GroupMetadata: + """Return the model of the v3 group document `data`, or raise `MetadataValidationError`. + + `data` is a decoded `zarr.json` document. It is validated as + `parse_group_metadata_v3` validates it, each array in its consolidated + metadata read in `context`, and every problem found is raised + together. `consolidated_metadata` becomes a + `ZarrV3ConsolidatedMetadata` (`null` reads as `UNSET`), and each other + top-level member outside the standard keys lands in `extra_fields`. + The model shares no mutable state with `data`. + """ # A read model shares no mutable state with what it read. parsed = copy.deepcopy(parse_group_metadata_v3(data, context=context)) consolidated_raw: object = parsed.get(ZARR_V3_CONSOLIDATED_METADATA_KEY, UNSET) @@ -179,6 +231,14 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: def from_key_value( cls, mapping: Mapping[StoreKey, bytes], *, context: Context = CORE_AND_EXTENSIONS ) -> ZarrV3GroupMetadata: + """Return the model of the v3 group stored at `zarr.json` in `mapping`, or raise `MetadataValidationError`. + + `mapping` maps store keys to the bytes stored there; other keys are + ignored. A missing `zarr.json` (a `missing_key` problem), a value that + is not `bytes` (`invalid_type`), or bytes that do not decode as JSON + (`invalid_json`) raise, as does every problem `from_json` finds with + the document in `context`. + """ return cls.from_json( load_store_json(mapping, ZARR_V3_GROUP_METADATA_STORE_KEY), context=context ) @@ -186,6 +246,14 @@ def from_key_value( def to_key_value( self, *, indent: int | str | None = None, context: Context = CORE_AND_EXTENSIONS ) -> Mapping[ZarrV3GroupMetadataStoreKey, bytes]: + """Return the model as the bytes of its `zarr.json` document, or raise `MetadataValidationError`. + + The document `to_json` returns is validated as `from_json` validates + one in `context`, since a model built or updated by hand is not, and + every problem found is raised together. `indent` is `json.dumps`'s: + `None` writes the document on one line, and an int or a string + pretty-prints it with that indent. + """ # A model built by hand is not validated: its document is written only # if it reads as `from_json` reads one in `context`, and every problem # is raised. @@ -205,8 +273,15 @@ class ZarrV3ConsolidatedMetadata: """ kind: Literal["inline"] = field(default="inline", init=False) + """The consolidation kind, always `"inline"`.""" must_understand: bool = False + """Whether a reader that does not recognize consolidated metadata must refuse the group. + + Always `False`: any other value is refused on construction with + `MetadataValidationError`. + """ metadata: dict[str, ZarrV3ArrayMetadata | ZarrV3GroupMetadata] + """The metadata of each node below the group, keyed by its path relative to the group.""" def __post_init__(self) -> None: if self.must_understand is not False: @@ -222,6 +297,11 @@ def __post_init__(self) -> None: ) def to_json(self) -> ZarrV3ConsolidatedMetadataJSON: + """Return the model as an inline consolidated metadata envelope. + + Each entry is written by its model's `to_json`. The envelope is not + validated. + """ # `must_understand` is emitted as the literal False: the field is typed # permissively as `bool`, but `__post_init__` guarantees the value. return { @@ -234,6 +314,16 @@ def to_json(self) -> ZarrV3ConsolidatedMetadataJSON: def from_json( cls, data: object, *, context: Context = CORE_AND_EXTENSIONS ) -> ZarrV3ConsolidatedMetadata: + """Return the model of the inline consolidated metadata `data`, or raise `MetadataValidationError`. + + `data` is the decoded value of a v3 group's `consolidated_metadata` + member. It is validated as `validate_consolidated_metadata_v3` + validates it: exactly the members `kind`, which must be `"inline"`, + `must_understand`, which must be `false`, and `metadata`, whose every + entry is a v3 array or group document read in `context`. Every problem + found is raised together. Each entry becomes a `ZarrV3ArrayMetadata` or + a `ZarrV3GroupMetadata` by its `node_type`. + """ normalized = arrays_to_tuples(data) problems = validate_consolidated_metadata_v3(normalized, context=context) if len(problems) != 0: @@ -263,6 +353,11 @@ class ZarrV2GroupMetadataPartial(TypedDict, total=False): """ attributes: dict[str, JSONValue] | UNSET + """User attributes, from the sibling `.zattrs` document. + + `UNSET` when there is no `.zattrs`; `{}` is an empty `.zattrs`, which + `to_key_value` writes. + """ @dataclass(frozen=True, slots=True, kw_only=True) @@ -278,7 +373,13 @@ class ZarrV2GroupMetadata: """ zarr_format: Literal[2] = field(default=2, init=False) + """The Zarr format version, always `2`.""" attributes: dict[str, JSONValue] | UNSET + """User attributes, from the sibling `.zattrs` document. + + `UNSET` when there is no `.zattrs`; `{}` is an empty `.zattrs`, which + `to_key_value` writes. + """ @classmethod def create_default(cls, **overrides: Unpack[ZarrV2GroupMetadataPartial]) -> ZarrV2GroupMetadata: @@ -320,12 +421,32 @@ def to_json(self) -> ZarrV2GroupMetadataJSON: @classmethod def from_json(cls, data: object) -> ZarrV2GroupMetadata: + """Return the model of the merged v2 group document `data`, or raise `MetadataValidationError`. + + `data` is a decoded `.zgroup` document, with the `.zattrs` content + folded in as an `attributes` member when there is one (the + `ZarrV2GroupMetadataJSON` form). It is validated as + `parse_group_metadata_v2` validates it, and every problem found is + raised together; the document is closed, so a member other than + `zarr_format` and `attributes` is a problem. An absent `attributes` + reads as `UNSET`. The model shares no mutable state with `data`. + """ # A read model shares no mutable state with what it read. parsed = copy.deepcopy(parse_group_metadata_v2(data)) return cls(attributes=(dict(parsed["attributes"]) if "attributes" in parsed else UNSET)) @classmethod def from_key_value(cls, mapping: Mapping[StoreKey, bytes]) -> ZarrV2GroupMetadata: + """Return the model of the v2 group stored at `.zgroup` and `.zattrs` in `mapping`, or raise `MetadataValidationError`. + + `mapping` maps store keys to the bytes stored there; other keys are + ignored. `.zattrs`, when present, becomes `attributes`; when absent, + `attributes` is `UNSET`. A missing `.zgroup` (a `missing_key` + problem), a value that is not `bytes` (`invalid_type`), bytes that do + not decode as JSON (`invalid_json`), or an `attributes` member inside + `.zgroup` (`invalid_value`) raise, as does every problem `from_json` + finds with the merged document. + """ zgroup_raw = load_store_json(mapping, ZARR_V2_GROUP_METADATA_STORE_KEY) if not isinstance(zgroup_raw, Mapping): return cls.from_json(zgroup_raw) @@ -348,6 +469,16 @@ def from_key_value(cls, mapping: Mapping[StoreKey, bytes]) -> ZarrV2GroupMetadat def to_key_value( self, *, indent: int | str | None = None ) -> Mapping[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: + """Return the model as the bytes of its `.zgroup` and `.zattrs` documents, or raise `MetadataValidationError`. + + The merged document `to_json` returns is validated as `from_json` + validates one, since a model built or updated by hand is not, and + every problem found is raised together. `.zgroup` holds every member + but `attributes`; `.zattrs` holds `attributes`, and is present exactly + when `attributes` is not `UNSET`, even when it is empty. `indent` is + `json.dumps`'s: `None` writes each document on one line, and an int or + a string pretty-prints it with that indent. + """ # Attributes live only in the sibling `.zattrs` file; the `.zgroup` # document must exclude them. The `.zattrs` key is present exactly # when attributes are set (even empty) — UNSET emits no file. A model @@ -377,9 +508,20 @@ class ZarrV2ConsolidatedMetadata: """ zarr_consolidated_format: Literal[1] = field(default=1, init=False) + """The consolidated metadata format version, always `1`.""" metadata: dict[str, JSONValue] + """Each consolidated document, keyed by its store path (`"a/.zarray"`, `"a/.zattrs"`, ...). + + Each entry is JSON, and a `.zattrs` entry, being user data, may also hold + `NaN` or an infinity. Entries are not read as array or group documents. + """ def to_json(self) -> dict[str, JSONValue]: + """Return the model as a `.zmetadata` document. + + The document is not validated (`to_key_value` validates it) and shares + no mutable state with the model. + """ # to_json output shares no mutable state with the model. return { "zarr_consolidated_format": self.zarr_consolidated_format, @@ -388,6 +530,15 @@ def to_json(self) -> dict[str, JSONValue]: @classmethod def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: + """Return the model of the `.zmetadata` document `data`, or raise `MetadataValidationError`. + + `data` must be an object with exactly the members + `zarr_consolidated_format`, which must be the integer `1`, and + `metadata`, an object of string keys whose every entry is JSON; a + `.zattrs` entry, being user data, may also hold `NaN` or an infinity. + Every problem found is raised together. Entries are held as written, + not read as array or group documents. + """ normalized = arrays_to_tuples(data) if not isinstance(normalized, Mapping): raise MetadataValidationError( @@ -447,11 +598,27 @@ def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: @classmethod def from_key_value(cls, mapping: Mapping[StoreKey, bytes]) -> ZarrV2ConsolidatedMetadata: + """Return the model of the consolidated metadata stored at `.zmetadata` in `mapping`, or raise `MetadataValidationError`. + + `mapping` maps store keys to the bytes stored there; other keys are + ignored. A missing `.zmetadata` (a `missing_key` problem), a value + that is not `bytes` (`invalid_type`), or bytes that do not decode as + JSON (`invalid_json`) raise, as does every problem `from_json` finds + with the document. + """ return cls.from_json(load_store_json(mapping, ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY)) def to_key_value( self, *, indent: int | str | None = None ) -> Mapping[ZarrV2ConsolidatedMetadataStoreKey, bytes]: + """Return the model as the bytes of its `.zmetadata` document, or raise `MetadataValidationError`. + + The document `to_json` returns is validated as `from_json` validates + one, since a model built or updated by hand is not, and every problem + found is raised together. `indent` is `json.dumps`'s: `None` writes the + document on one line, and an int or a string pretty-prints it with + that indent. + """ # A model built by hand is not validated: it is written only as # `from_json` reads it, and every problem is raised. document = ZarrV2ConsolidatedMetadata.from_json(self.to_json()).to_json() diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index d1ae10b83a..45b663c586 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -63,44 +63,63 @@ ARRAY_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( ZarrV3ArrayMetadataJSON.__required_keys__ ) +"""The top-level keys every v3 array document holds.""" ARRAY_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( ZarrV3ArrayMetadataJSON.__optional_keys__ ) +"""The top-level keys the v3 spec defines for an array document that it may omit.""" ARRAY_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( ARRAY_METADATA_REQUIRED_KEYS_V3 | ARRAY_METADATA_OPTIONAL_KEYS_V3 ) +"""Every top-level key the v3 spec defines for an array document. + +Any other top-level key is an extension field, held in `extra_fields`. +""" ARRAY_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( ZarrV2ArrayMetadataJSON.__required_keys__ ) +"""The keys every v2 array document holds.""" ARRAY_METADATA_OPTIONAL_KEYS_V2: Final[frozenset[str]] = frozenset( ZarrV2ArrayMetadataJSON.__optional_keys__ ) +"""The keys of a merged v2 array document that it may omit: `dimension_separator` and `attributes`.""" ARRAY_METADATA_STANDARD_KEYS_V2: Final[frozenset[str]] = ( ARRAY_METADATA_REQUIRED_KEYS_V2 | ARRAY_METADATA_OPTIONAL_KEYS_V2 ) +"""Every key of a merged v2 array document; any other is not held by the model.""" # The standard top-level keys of a v3 group metadata document. Anything outside # this set is an extension field. GROUP_METADATA_REQUIRED_KEYS_V3: Final[frozenset[str]] = frozenset( ZarrV3GroupMetadataJSON.__required_keys__ ) +"""The top-level keys every v3 group document holds.""" GROUP_METADATA_OPTIONAL_KEYS_V3: Final[frozenset[str]] = frozenset( ZarrV3GroupMetadataJSON.__optional_keys__ ) +"""The top-level keys the v3 spec defines for a group document that it may omit.""" GROUP_METADATA_STANDARD_KEYS_V3: Final[frozenset[str]] = ( GROUP_METADATA_REQUIRED_KEYS_V3 | GROUP_METADATA_OPTIONAL_KEYS_V3 ) +"""Every top-level key the v3 spec defines for a group document. + +`consolidated_metadata` is not among them: `ZarrV3GroupMetadata` holds it in a +field of its own, and any other top-level key in `extra_fields`. +""" GROUP_METADATA_REQUIRED_KEYS_V2: Final[frozenset[str]] = frozenset( ZarrV2GroupMetadataJSON.__required_keys__ ) +"""The keys every v2 group document holds.""" GROUP_METADATA_OPTIONAL_KEYS_V2: Final[frozenset[str]] = frozenset( ZarrV2GroupMetadataJSON.__optional_keys__ ) +"""The keys of a merged v2 group document that it may omit: `attributes`.""" GROUP_METADATA_STANDARD_KEYS_V2: Final[frozenset[str]] = ( GROUP_METADATA_REQUIRED_KEYS_V2 | GROUP_METADATA_OPTIONAL_KEYS_V2 ) +"""Every key of a merged v2 group document; the document is closed, so any other is a problem.""" def _missing_keys( diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index 1b02e1ffc7..65e111253e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -67,14 +67,40 @@ class ZarrV2ZArrayJSON(TypedDict): """ zarr_format: Literal[2] + """The Zarr format version, `2`.""" shape: tuple[int, ...] + """The length of each dimension of the array.""" chunks: tuple[int, ...] + """The length of each dimension of a chunk. + + Every chunk has this shape, so a chunk at the array's edge overhangs it + where a length does not divide evenly; what the overhang holds is undefined. + """ dtype: ZarrV2DataTypeMetadata + """The data type of the array's elements. + + A NumPy typestr such as `" array, array -> bytes or bytes -> bytes. + + It fixes where the codec may sit in a pipeline, and which of `chunk_rules`, + `transition` and `pipelines` it may be given: building a definition with + one its kind is never asked raises `TypeError`. + """ size: CodecSize + """Whether the size of what the codec gives out is fixed by the size of what it is handed. + + Where a member takes only codecs of static size, a codec of dynamic size + is a problem. + """ chunk_rules: Callable[[C, Nested, Chunk], Iterable[ValidationProblem]] = no_rules """What the spec disallows in this codec handed a chunk, located in the configuration.""" transition: Callable[[C, Nested, Chunk], Chunk] = unknown_chunk @@ -760,10 +771,8 @@ def _nothing_nested() -> Nested: class Resolved(Generic[D]): """One metadata field, as read in a scope: its JSON, and what the scope made of it. - `resolution` is what became of the configuration: read by the - definition that claims the name, claimed by nothing, or not readable. - A problem with the envelope around it -- a stray member, a - `must_understand` of `false` -- is reported with the field, and leaves + A problem with the envelope around the configuration -- a stray member, + a `must_understand` of `false` -- is reported with the field, and leaves the resolution as it is; so is a problem of a field the configuration holds, which is that field's own, with its own resolution in `nested`. """ @@ -771,6 +780,11 @@ class Resolved(Generic[D]): json: JSONValue """The field as written, refined: arrays as tuples. `None` for a value that was not JSON, as for `null`.""" resolution: Resolution + """What became of the configuration. + + `"read"` by the definition that claims the name, `"out_of_scope"` when + nothing in scope claims it, or `"invalid"` when it could not be read. + """ definition: D | None """The definition that claims the field's name; None when nothing in scope does, or it names none.""" configuration: Mapping[str, JSONValue] | None diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index e5beff8c64..63bd579fb1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -74,6 +74,7 @@ class Context: """ tables: Tables + """The definitions in scope, by kind, then by the name each is filed under.""" @classmethod def of(cls, *definitions: Definition[Any]) -> Context: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/array.py b/packages/zarr-metadata/src/zarr_metadata/v3/array.py index 31a5f6b755..9b2db769dd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/array.py @@ -27,16 +27,41 @@ class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=ZarrV3ExtensionField): """ zarr_format: Literal[3] + """The Zarr format version, `3`.""" node_type: Literal["array"] + """The kind of node the document describes, `"array"`.""" data_type: ZarrV3MetadataFieldJSON + """The data type of the array's elements, e.g. `"float64"`.""" shape: tuple[int, ...] + """The length of each dimension of the array.""" chunk_grid: ZarrV3MetadataFieldJSON + """How the array is divided into chunks, e.g. a `"regular"` grid of one chunk shape.""" chunk_key_encoding: ZarrV3MetadataFieldJSON + """How a chunk's grid coordinates become its key in the store.""" fill_value: JSONValue + """The value of an element in a chunk that was never written. + + Which JSON values it may take depends on `data_type`. + """ codecs: tuple[ZarrV3MetadataFieldJSON, ...] + """The codecs that encode each chunk, in encoding order. + + Zero or more array -> array codecs, then exactly one array -> bytes codec, + then zero or more bytes -> bytes codecs, so the list is never empty + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L974-L983). + """ attributes: NotRequired[Mapping[str, JSONValue]] + """User attributes: arbitrary JSON values by string key.""" storage_transformers: NotRequired[tuple[ZarrV3MetadataFieldJSON, ...]] + """The storage transformers between the array and its store. + + Absent means none, as does an empty list. + """ dimension_names: NotRequired[tuple[str | None, ...]] + """A name for each dimension, `null` for an unnamed one. + + As long as `shape`; absent means every dimension is unnamed. + """ class ZarrV3ArrayMetadataJSONPartial(TypedDict, total=False, extra_items=ZarrV3ExtensionField): @@ -63,16 +88,41 @@ class ZarrV3ArrayMetadataJSONPartial(TypedDict, total=False, extra_items=ZarrV3E """ zarr_format: Literal[3] + """The Zarr format version, `3`.""" node_type: Literal["array"] + """The kind of node the document describes, `"array"`.""" data_type: ZarrV3MetadataFieldJSON + """The data type of the array's elements, e.g. `"float64"`.""" shape: tuple[int, ...] + """The length of each dimension of the array.""" chunk_grid: ZarrV3MetadataFieldJSON + """How the array is divided into chunks, e.g. a `"regular"` grid of one chunk shape.""" chunk_key_encoding: ZarrV3MetadataFieldJSON + """How a chunk's grid coordinates become its key in the store.""" fill_value: JSONValue + """The value of an element in a chunk that was never written. + + Which JSON values it may take depends on `data_type`. + """ codecs: tuple[ZarrV3MetadataFieldJSON, ...] + """The codecs that encode each chunk, in encoding order. + + Zero or more array -> array codecs, then exactly one array -> bytes codec, + then zero or more bytes -> bytes codecs, so the list is never empty + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L974-L983). + """ attributes: NotRequired[Mapping[str, JSONValue]] + """User attributes: arbitrary JSON values by string key.""" storage_transformers: NotRequired[tuple[ZarrV3MetadataFieldJSON, ...]] + """The storage transformers between the array and its store. + + Absent means none, as does an empty list. + """ dimension_names: NotRequired[tuple[str | None, ...]] + """A name for each dimension, `null` for an unnamed one. + + As long as `shape`; absent means every dimension is unnamed. + """ ZarrV3ArrayMetadataStoreKey = Literal["zarr.json"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index a8191727a8..4c8128985f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -32,15 +32,30 @@ class RectilinearChunkGridConfiguration(TypedDict, closed=True): """Configuration for the rectilinear chunk grid.""" kind: Literal["inline"] + """How `chunk_shapes` is encoded; `"inline"` is the only kind the spec defines.""" chunk_shapes: tuple[RectilinearDimSpec, ...] + """The chunk edge lengths along each dimension of the array, one entry per dimension. + + An entry is a bare integer, a step repeated until it covers the + dimension, or a list of edge lengths and `[length, count]` runs, whose + lengths must sum to at least the dimension's length. Every length and + count is at least 1. + """ class RectilinearChunkGridObject(TypedDict, closed=True): """Rectilinear chunk grid metadata in object form.""" name: RectilinearChunkGridName + """The chunk grid's name, `"rectilinear"`.""" configuration: RectilinearChunkGridConfiguration + """The chunk grid's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this chunk grid must refuse the document. + + Absent means `true`. The spec does not permit `false` for a chunk grid + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ RectilinearChunkGridMetadata = RectilinearChunkGridObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 444ec11f4a..7128b705fe 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -23,14 +23,26 @@ class RegularChunkGridConfiguration(TypedDict, closed=True): """Configuration for the regular chunk grid.""" chunk_shape: tuple[int, ...] + """The length of every chunk along each dimension of the array. + + One entry per dimension, each greater than zero + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-grids/regular-grid/index.rst#L40). + """ class RegularChunkGridObject(TypedDict, closed=True): """Regular chunk grid metadata in object form.""" name: RegularChunkGridName + """The chunk grid's name, `"regular"`.""" configuration: RegularChunkGridConfiguration + """The chunk grid's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this chunk grid must refuse the document. + + Absent means `true`. The spec does not permit `false` for a chunk grid + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ RegularChunkGridMetadata = RegularChunkGridObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index c0e49ee543..0213f915a2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -37,14 +37,28 @@ class DefaultChunkKeyEncodingConfiguration(TypedDict, closed=True): """ separator: NotRequired[DefaultChunkKeyEncodingSeparator] + """The character between the parts of a chunk key: `c/1/23/45` or `c.1.23.45`. + + Absent means `"/"`. + """ class DefaultChunkKeyEncodingObject(TypedDict, closed=True): """Default chunk key encoding metadata in object form.""" name: DefaultChunkKeyEncodingName + """The chunk key encoding's name, `"default"`.""" configuration: NotRequired[DefaultChunkKeyEncodingConfiguration] + """The chunk key encoding's parameters. + + Absent means every parameter takes its default: a `separator` of `"/"`. + """ must_understand: NotRequired[bool] + """Whether a reader that does not recognize this chunk key encoding must refuse the document. + + Absent means `true`. The spec does not permit `false` for a chunk key encoding + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ DefaultChunkKeyEncodingMetadata = DefaultChunkKeyEncodingObject | DefaultChunkKeyEncodingName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index 7c178a599e..6e134afe52 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -43,14 +43,28 @@ class V2ChunkKeyEncodingConfiguration(TypedDict, closed=True): """ separator: NotRequired[V2ChunkKeyEncodingSeparator] + """The character between the parts of a chunk key: `1.23.45` or `1/23/45`. + + Absent means `"."`. + """ class V2ChunkKeyEncodingObject(TypedDict, closed=True): """v2-compatibility chunk key encoding metadata in object form.""" name: V2ChunkKeyEncodingName + """The chunk key encoding's name, `"v2"`.""" configuration: NotRequired[V2ChunkKeyEncodingConfiguration] + """The chunk key encoding's parameters. + + Absent means every parameter takes its default: a `separator` of `"."`. + """ must_understand: NotRequired[bool] + """Whether a reader that does not recognize this chunk key encoding must refuse the document. + + Absent means `true`. The spec does not permit `false` for a chunk key encoding + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ V2ChunkKeyEncodingMetadata = V2ChunkKeyEncodingObject | V2ChunkKeyEncodingName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index fb162c537d..411d927ca2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -35,18 +35,46 @@ class BloscCodecConfiguration(TypedDict, closed=True): """Configuration for the Zarr v3 `blosc` codec.""" cname: BloscCName + """The compressor blosc applies to each block, one of `BLOSC_CNAME`.""" clevel: int + """The compression level, an integer from 0 to 9. + + 0 turns compression off; 1 is the fastest and compresses least, 9 the + slowest and compresses most. + """ shuffle: BloscShuffle + """The shuffling applied before compression. + + `"noshuffle"` for none, `"shuffle"` for byte-wise, `"bitshuffle"` for + bit-wise. + """ blocksize: int + """The size in bytes of the blocks the buffer is divided into before compression. + + At least 0; 0 lets blosc choose the size. + """ typesize: NotRequired[int] + """The stride in bytes over which shuffling is performed. + + Required, and at least 1, unless `shuffle` is `"noshuffle"`, under + which the spec ignores it and `canonicalize` drops it. + """ class BloscCodecObject(TypedDict, closed=True): """`blosc` codec metadata in object form.""" name: BloscCodecName + """The codec's name, `"blosc"`.""" configuration: BloscCodecConfiguration + """The codec's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ BloscCodecMetadata = BloscCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index c98f1d712d..d0f5f8e105 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -39,6 +39,12 @@ class BytesCodecConfiguration(TypedDict, closed=True): """ endian: NotRequired[Endianness] + """The byte order in which each value is written. + + Required when the values of the data type the codec is handed take more + than one byte; a single-byte type such as `uint8` or `bool` does not + depend on it. + """ class BytesCodecObject(TypedDict, closed=True): @@ -51,8 +57,19 @@ class BytesCodecObject(TypedDict, closed=True): """ name: BytesCodecName + """The codec's name, `"bytes"`.""" configuration: NotRequired[BytesCodecConfiguration] + """The codec's parameters. + + May be absent when no parameter is set. + """ must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ BytesCodecMetadata = BytesCodecObject | BytesCodecName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index d5a018a1e6..98b2473a38 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -75,7 +75,17 @@ class ScalarMap(TypedDict, closed=True): """Optional encode/decode scalar overrides for the cast_value codec.""" encode: NotRequired[tuple[ScalarMapEntry, ...]] + """The `[input, output]` pairs applied when encoding: input in the array's data type, output in `data_type`. + + Each scalar is written in its data type's fill value encoding. Where an + input repeats, the first pair applies. + """ decode: NotRequired[tuple[ScalarMapEntry, ...]] + """The `[input, output]` pairs applied when decoding: input in `data_type`, output in the array's data type. + + Each scalar is written in its data type's fill value encoding. Where an + input repeats, the first pair applies. + """ class CastValueCodecConfiguration(TypedDict, closed=True): @@ -88,17 +98,44 @@ class CastValueCodecConfiguration(TypedDict, closed=True): """ data_type: DataTypeField + """The data type values are cast to when encoding, and the one the decoder is handed. + + The codec is defined only for data types that model real numbers, the + integer and floating-point types: a boolean, raw-bits, string, bytes, + struct or complex type is refused, here and as the data type the codec + is handed. + """ rounding: NotRequired[CastRoundingMode] + """How a value `data_type` cannot represent exactly is rounded. + + Absent means `"nearest-even"`. + """ out_of_range: NotRequired[CastOutOfRangeMode] + """How a value outside the range of `data_type` is brought into it, after rounding. + + `"clamp"` takes the nearest bound; `"wrap"` takes the value congruent + modulo 2^N, N being the width in bits of `data_type`, and is permitted + only when `data_type` is an integral type. Absent means an out-of-range + value is an encoding or decoding error. + """ scalar_map: NotRequired[ScalarMap] + """Scalars mapped explicitly, in place of `rounding` and `out_of_range`.""" class CastValueCodecObject(TypedDict, closed=True): """`cast_value` codec metadata in object form.""" name: CastValueCodecName + """The codec's name, `"cast_value"`.""" configuration: CastValueCodecConfiguration + """The codec's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ CastValueCodecMetadata = CastValueCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 454c0d7a1b..5564d59076 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -33,8 +33,19 @@ class Crc32cCodecObject(TypedDict, closed=True): """ name: Crc32cCodecName + """The codec's name, `"crc32c"`.""" configuration: NotRequired[Empty] + """The codec's parameters, of which there are none. + + Absent, or an empty mapping. + """ must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ Crc32cCodecMetadata = Crc32cCodecObject | Crc32cCodecName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index aa07d18421..88b89749f9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -34,14 +34,27 @@ class GzipCodecConfiguration(TypedDict, closed=True): """ level: int + """The compression level, an integer from 0 to 9. + + 0 turns compression off; 1 is the fastest and compresses least, 9 the + slowest and compresses most. + """ class GzipCodecObject(TypedDict, closed=True): """`gzip` codec metadata in object form.""" name: GzipCodecName + """The codec's name, `"gzip"`.""" configuration: GzipCodecConfiguration + """The codec's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ GzipCodecMetadata = GzipCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index e4807fe6fe..e1f0dd1e33 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -39,7 +39,17 @@ class ScaleOffsetCodecConfiguration(TypedDict, closed=True): """ offset: NotRequired[JSONValue] + """The quantity subtracted from each value when encoding. + + Absent means the additive identity, e.g. 0. A fill value of the data + type the codec is handed. + """ scale: NotRequired[JSONValue] + """The quantity each value is multiplied by when encoding, after `offset` is subtracted. + + Absent means the multiplicative identity, e.g. 1. A fill value of the + data type the codec is handed. + """ class ScaleOffsetCodecObject(TypedDict, closed=True): @@ -52,8 +62,20 @@ class ScaleOffsetCodecObject(TypedDict, closed=True): """ name: ScaleOffsetCodecName + """The codec's name, `"scale_offset"`.""" configuration: NotRequired[ScaleOffsetCodecConfiguration] + """The codec's parameters. + + May be absent when neither `offset` nor `scale` is set, which makes the + codec a no-op. + """ must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ ScaleOffsetCodecMetadata = ScaleOffsetCodecObject | ScaleOffsetCodecName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 98dd1560cf..83496d1349 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -54,17 +54,44 @@ class ShardingIndexedCodecConfiguration(TypedDict, closed=True): """ chunk_shape: tuple[int, ...] + """The shape of the inner chunks a shard is divided into. + + One length, at least 1, per axis of the shard the codec is handed, each + dividing the shard's length along that axis. + """ codecs: tuple[CodecField, ...] + """The codec pipeline that encodes each inner chunk. + + Like an array's `codecs`, it holds exactly one array -> bytes codec. + """ index_codecs: tuple[StaticCodecField, ...] + """The codec pipeline that encodes the shard index. + + It holds exactly one array -> bytes codec, and only codecs whose output + size is fixed by the size of their input, so no compressor. The spec + recommends a little-endian `bytes` codec followed by `crc32c`. + """ index_location: NotRequired[ShardingIndexLocation] + """Whether the shard index sits at the `"start"` or the `"end"` of the encoded shard. + + Absent means `"end"`. + """ class ShardingIndexedCodecObject(TypedDict, closed=True): """`sharding_indexed` codec metadata in object form.""" name: ShardingIndexedCodecName + """The codec's name, `"sharding_indexed"`.""" configuration: ShardingIndexedCodecConfiguration + """The codec's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ ShardingIndexedCodecMetadata = ShardingIndexedCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 2fb88338f9..69280ee3f4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -28,14 +28,27 @@ class TransposeCodecConfiguration(TypedDict, closed=True): """ order: tuple[int, ...] + """The reordering of axes applied when encoding: axis `i` of the encoded chunk is axis `order[i]` of the chunk the codec is handed. + + A permutation of `0, 1, ..., n-1`, `n` being the number of axes of the + chunk the codec is handed. + """ class TransposeCodecObject(TypedDict, closed=True): """`transpose` codec metadata in object form.""" name: TransposeCodecName + """The codec's name, `"transpose"`.""" configuration: TransposeCodecConfiguration + """The codec's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ TransposeCodecMetadata = TransposeCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index efeefb7edb..ef5ef09f67 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -31,15 +31,33 @@ class ZstdCodecConfiguration(TypedDict, closed=True): """ level: int + """The compression level, an integer from `ZSTD_MIN_LEVEL` (-131072) to `ZSTD_MAX_LEVEL` (22). + + 0 means zstd's default level; otherwise a higher level compresses more, + more slowly. It has no effect on decoding. + """ checksum: NotRequired[bool] + """Whether a checksum is stored when writing and verified when reading. + + Absent means `false`; the registry says to omit it rather than write + `false`. + """ class ZstdCodecObject(TypedDict, closed=True): """`zstd` codec metadata in object form.""" name: ZstdCodecName + """The codec's name, `"zstd"`.""" configuration: ZstdCodecConfiguration + """The codec's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this codec must refuse the document. + + Absent means `true`. The spec permits `false` for a codec + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1581), + though `resolve` reports a `false` as a problem at every extension point. + """ ZstdCodecMetadata = ZstdCodecObject diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index 274af507b7..54a4680949 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -33,8 +33,14 @@ class ZarrV3ConsolidatedMetadataJSON(TypedDict, closed=True): """ kind: Literal["inline"] + """How the entries are carried, `"inline"`: in the group's own document.""" must_understand: Literal[False] + """Always `false`: a reader that does not recognize consolidated metadata may ignore it.""" metadata: Mapping[str, ZarrV3ArrayMetadataJSON | ZarrV3GroupMetadataJSON] + """The metadata document of each node below the group, by its path from the group. + + A path joins its segments with `"/"`, e.g. `"child/grandchild"`. + """ ZARR_V3_CONSOLIDATED_METADATA_KEY: Final = "consolidated_metadata" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index f8b2e21280..dc3c98763c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -28,27 +28,33 @@ class NumpyDatetime64Configuration(TypedDict, closed=True): - """ - Configuration for the `numpy.datetime64` data type. - - Attributes - ---------- - unit - A string encoding a unit of time. - scale_factor - The multiplier relative to the unit. - """ + """Configuration for the `numpy.datetime64` data type.""" unit: ReadOnly[NumpyTimeUnit] + """A string encoding a unit of time. + + `"us"` and `"μs"` are equivalent spellings of microseconds. + """ scale_factor: ReadOnly[int] + """The multiplier relative to the unit. + + An integer in `[1, 2147483647]`. + """ class NumpyDatetime64(TypedDict, closed=True): """`numpy.datetime64` data type metadata.""" name: NumpyDatetime64DataTypeName + """The data type's name, `"numpy.datetime64"`.""" configuration: NumpyDatetime64Configuration + """The data type's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this data type must refuse the document. + + Absent means `true`. The spec does not permit `false` for a data type + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ NumpyDatetime64FillValue = int | Literal["NaT"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index 5b0ee23626..09f79ab8ec 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -47,27 +47,33 @@ class NumpyTimedelta64Configuration(TypedDict, closed=True): - """ - Configuration for the `numpy.timedelta64` data type. - - Attributes - ---------- - unit - A string encoding a unit of time. - scale_factor - The multiplier relative to the unit. - """ + """Configuration for the `numpy.timedelta64` data type.""" unit: ReadOnly[NumpyTimeUnit] + """A string encoding a unit of time. + + `"us"` and `"μs"` are equivalent spellings of microseconds. + """ scale_factor: ReadOnly[int] + """The multiplier relative to the unit. + + An integer in `[1, 2147483647]`. + """ class NumpyTimedelta64(TypedDict, closed=True): """`numpy.timedelta64` data type metadata.""" name: NumpyTimedelta64DataTypeName + """The data type's name, `"numpy.timedelta64"`.""" configuration: NumpyTimedelta64Configuration + """The data type's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this data type must refuse the document. + + Absent means `true`. The spec does not permit `false` for a data type + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ NumpyTimedelta64FillValue = int | Literal["NaT"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index e23e523f8c..4f7a247da1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -62,6 +62,10 @@ class RawBytesConfiguration(TypedDict, closed=True): """ bits: int + """The size in bits that the name writes, `16` for `r16`. + + A positive multiple of 8. + """ def _rules(configuration: RawBytesConfiguration, nested: Nested) -> Iterator[ValidationProblem]: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 3f3d284204..a204d105bc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -29,34 +29,42 @@ class StructField(TypedDict, closed=True): - """ - A single field entry inside a structured dtype. - - Attributes - ---------- - name - The field name (must be unique within a struct and non-empty). - data_type - The field's data type. Recursive: may be a bare-string primitive - or a named-config envelope including another `struct`. - """ + """A single field entry inside a structured dtype.""" name: ReadOnly[str] + """The field name (must be unique within a struct and non-empty).""" data_type: ReadOnly[DataTypeField] + """The field's data type. + + Recursive: may be a bare-string primitive or a named-config envelope + including another `struct`. + """ class StructConfiguration(TypedDict, closed=True): """Configuration for the `struct` data type.""" fields: ReadOnly[tuple[StructField, ...]] + """The record's fields, in the order they are encoded. + + At least one. The order is significant: it is the order in which the + `bytes` codec packs the fields' values. + """ class Struct(TypedDict, closed=True): """`struct` data type metadata.""" name: StructDataTypeName + """The data type's name, `"struct"`.""" configuration: StructConfiguration + """The data type's parameters.""" must_understand: NotRequired[bool] + """Whether a reader that does not recognize this data type must refuse the document. + + Absent means `true`. The spec does not permit `false` for a data type + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1580-L1581). + """ StructFillValue = Mapping[str, JSONValue] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/group.py b/packages/zarr-metadata/src/zarr_metadata/v3/group.py index 37bfdd6934..ce6ca5cfb6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/group.py @@ -22,8 +22,11 @@ class ZarrV3GroupMetadataJSON(TypedDict, extra_items=ZarrV3ExtensionField): """ zarr_format: Literal[3] + """The Zarr format version, `3`.""" node_type: Literal["group"] + """The kind of node the document describes, `"group"`.""" attributes: NotRequired[Mapping[str, JSONValue]] + """User attributes: arbitrary JSON values by string key.""" class ZarrV3GroupMetadataJSONPartial(TypedDict, total=False, extra_items=ZarrV3ExtensionField): @@ -50,8 +53,11 @@ class ZarrV3GroupMetadataJSONPartial(TypedDict, total=False, extra_items=ZarrV3E """ zarr_format: Literal[3] + """The Zarr format version, `3`.""" node_type: Literal["group"] + """The kind of node the document describes, `"group"`.""" attributes: NotRequired[Mapping[str, JSONValue]] + """User attributes: arbitrary JSON values by string key.""" ZarrV3GroupMetadataStoreKey = Literal["zarr.json"] diff --git a/packages/zarr-metadata/tests/test_api_docs.py b/packages/zarr-metadata/tests/test_api_docs.py new file mode 100644 index 0000000000..633f454cd2 --- /dev/null +++ b/packages/zarr-metadata/tests/test_api_docs.py @@ -0,0 +1,123 @@ +"""Every public module has an API reference page, and every public object the docstring it renders. + +The reference pages render every public name with mkdocstrings, which reads +docstrings statically, through griffe: a field's docstring is the string +literal under it, which the interpreter discards. So these tests read the +package the way the pages do, rather than through `__doc__`. + +Public means: a module whose dotted path has no segment starting with `_`, +each name in its `__all__`, and each member a public class declares itself +whose name does not start with `_`. +""" + +from __future__ import annotations + +import importlib +import pkgutil +import re +from pathlib import Path + +import griffe + +import zarr_metadata + +API_PAGES = Path(__file__).parent.parent / "docs" / "api" + +PACKAGE = griffe.load("zarr_metadata", resolve_aliases=True, resolve_external=False) + +PUBLIC_MODULES = [ + name + for name in [ + "zarr_metadata", + *( + module.name + for module in pkgutil.walk_packages(zarr_metadata.__path__, "zarr_metadata.") + ), + ] + if not any(part.startswith("_") for part in name.split(".")) +] + +# A type whose fields mean what another type's same-named fields mean: a +# partial form, or a strict on-disk form of a merged in-memory one. Their +# field docstrings are the same text. +MIRRORS = [ + ( + "zarr_metadata.v2.array.ZarrV2ArrayMetadataJSON", + "zarr_metadata.v2.array.ZarrV2ArrayMetadataJSONPartial", + ), + ("zarr_metadata.v2.array.ZarrV2ArrayMetadataJSON", "zarr_metadata.v2.array.ZarrV2ZArrayJSON"), + ( + "zarr_metadata.v2.group.ZarrV2GroupMetadataJSON", + "zarr_metadata.v2.group.ZarrV2GroupMetadataJSONPartial", + ), + ("zarr_metadata.v2.group.ZarrV2GroupMetadataJSON", "zarr_metadata.v2.group.ZarrV2ZGroupJSON"), + ( + "zarr_metadata.v3.array.ZarrV3ArrayMetadataJSON", + "zarr_metadata.v3.array.ZarrV3ArrayMetadataJSONPartial", + ), + ( + "zarr_metadata.v3.group.ZarrV3GroupMetadataJSON", + "zarr_metadata.v3.group.ZarrV3GroupMetadataJSONPartial", + ), + ("zarr_metadata.model.ZarrV2ArrayMetadata", "zarr_metadata.model.ZarrV2ArrayMetadataPartial"), + ("zarr_metadata.model.ZarrV2GroupMetadata", "zarr_metadata.model.ZarrV2GroupMetadataPartial"), + ("zarr_metadata.model.ZarrV3ArrayMetadata", "zarr_metadata.model.ZarrV3ArrayMetadataPartial"), + ("zarr_metadata.model.ZarrV3GroupMetadata", "zarr_metadata.model.ZarrV3GroupMetadataPartial"), +] + + +def _griffe(path: str) -> griffe.Object: + """The object at a dotted path, followed through any re-export to its definition.""" + found = PACKAGE if path == "zarr_metadata" else PACKAGE[path.removeprefix("zarr_metadata.")] + return found.final_target if isinstance(found, griffe.Alias) else found + + +def _public_objects() -> dict[str, griffe.Object]: + """Each public module, name and class member, keyed by the path it is defined at.""" + found: dict[str, griffe.Object] = {} + for module_name in PUBLIC_MODULES: + module = _griffe(module_name) + found[module.path] = module + for name in importlib.import_module(module_name).__all__: + target = _griffe(f"{module_name}.{name}") + found[target.path] = target + if target.is_class: + for member_name, member in target.members.items(): + if not member_name.startswith("_") and not isinstance(member, griffe.Alias): + found[member.path] = member + return found + + +def test_public_modules_have_reference_pages() -> None: + # The top-level namespace re-exports names whose pages are their defining + # modules'; the API index renders the few it defines itself. + rendered = { + match + for page in API_PAGES.rglob("*.md") + for match in re.findall(r"^::: (\S+)$", page.read_text(), flags=re.MULTILINE) + } + assert sorted(set(PUBLIC_MODULES) - rendered - {"zarr_metadata"}) == [] + + +def test_public_objects_have_docstrings() -> None: + undocumented = sorted( + path + for path, obj in _public_objects().items() + if obj.docstring is None or not obj.docstring.value.strip() + ) + assert undocumented == [] + + +def test_mirrored_fields_share_docstrings() -> None: + drifted = [] + for source_path, mirror_path in MIRRORS: + source, mirror = _griffe(source_path), _griffe(mirror_path) + for name, field in mirror.members.items(): + if ( + name in source.members + and field.is_attribute + and getattr(field.docstring, "value", None) + != getattr(source.members[name].docstring, "value", None) + ): + drifted.append(f"{mirror_path}.{name}") + assert drifted == [] diff --git a/packages/zarr-metadata/uv.lock b/packages/zarr-metadata/uv.lock index 07ad9a0ebe..feba04dbeb 100644 --- a/packages/zarr-metadata/uv.lock +++ b/packages/zarr-metadata/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.11" [[package]] @@ -1148,6 +1148,7 @@ docs = [ { name = "ruff" }, ] test = [ + { name = "griffelib" }, { name = "hypothesis" }, { name = "jsonschema" }, { name = "pydantic" }, @@ -1167,6 +1168,7 @@ docs = [ { name = "ruff", specifier = "==0.16.8" }, ] test = [ + { name = "griffelib" }, { name = "hypothesis" }, { name = "jsonschema" }, { name = "pydantic", specifier = ">=2.13" }, From 3f79a1cb96355a69d20c573fe70b61e59679db72 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 27 Oct 2026 21:18:26 +0100 Subject: [PATCH 2/3] docs(zarr-metadata): add changelog fragment for #415 Assisted-by: ClaudeCode:claude-opus-5-5 --- packages/zarr-metadata/changes/415.doc.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 packages/zarr-metadata/changes/415.doc.md diff --git a/packages/zarr-metadata/changes/415.doc.md b/packages/zarr-metadata/changes/415.doc.md new file mode 100644 index 0000000000..6e4c1d2ce0 --- /dev/null +++ b/packages/zarr-metadata/changes/415.doc.md @@ -0,0 +1,2 @@ +Every public object now has an API reference entry with a docstring, and the +`struct` data type has its reference page. From b37e4de209c35738a61e887da4227510b46860d3 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 28 Oct 2026 21:36:46 +0100 Subject: [PATCH 3/3] docs(zarr-metadata): document the public API main added The merge of main brought the v2 codec and data type modules, the split model modules, the shared `definition` package and the problem fields `input` and `ctx`, 142 of whose public objects had no docstring. Each now has one. The per-field docstrings that conflicting hunks dropped (v3 array fields, codec configurations, `StructField`, the numpy time configurations) are restored where the field kept its meaning, and the API index renders `__version__` again. The mirrored-docstring check follows main's types: the Partial forms are gone, leaving the on-disk v2 `.zarray` and `.zgroup` types. Assisted-by: ClaudeCode:claude-opus-5-5 --- packages/zarr-metadata/docs/api/index.md | 4 +++ .../zarr-metadata/src/zarr_metadata/_json.py | 3 ++ .../zarr_metadata/definition/_definition.py | 8 ++++-- .../src/zarr_metadata/definition/_scope.py | 5 ++++ .../src/zarr_metadata/model/_array_v2.py | 24 ++++++++++++++++ .../src/zarr_metadata/model/_array_v3.py | 24 ++++++++++++++++ .../src/zarr_metadata/model/_group_v2.py | 10 +++++++ .../src/zarr_metadata/model/_group_v3.py | 19 +++++++++++++ .../src/zarr_metadata/pydantic.py | 1 + .../src/zarr_metadata/v2/_definition.py | 3 ++ .../src/zarr_metadata/v2/_field_json.py | 1 + .../src/zarr_metadata/v2/array.py | 12 ++++++++ .../src/zarr_metadata/v2/codec/checksum.py | 4 +++ .../src/zarr_metadata/v2/codec/compression.py | 28 +++++++++++++++++++ .../src/zarr_metadata/v2/codec/filters.py | 17 +++++++++++ .../src/zarr_metadata/v2/codec/vlen.py | 1 + .../src/zarr_metadata/v2/data_type/object.py | 1 + .../src/zarr_metadata/v2/data_type/scalar.py | 7 +++++ .../src/zarr_metadata/v2/data_type/struct.py | 5 ++++ .../src/zarr_metadata/v2/data_type/time.py | 8 ++++++ .../src/zarr_metadata/v3/_definition.py | 18 ++++++++++++ .../src/zarr_metadata/v3/array.py | 20 +++++++++++++ .../zarr_metadata/v3/chunk_grid/regular.py | 4 +++ .../src/zarr_metadata/v3/codec/blosc.py | 15 ++++++++++ .../src/zarr_metadata/v3/codec/cast_value.py | 7 +++++ .../src/zarr_metadata/v3/codec/crc32c.py | 5 ++++ .../src/zarr_metadata/v3/codec/gzip.py | 5 ++++ .../v3/codec/sharding_indexed.py | 15 ++++++++++ .../src/zarr_metadata/v3/codec/zstd.py | 5 ++++ .../src/zarr_metadata/v3/consolidated.py | 3 ++ .../v3/data_type/numpy_datetime64.py | 9 ++++++ .../v3/data_type/numpy_timedelta64.py | 9 ++++++ .../src/zarr_metadata/v3/data_type/struct.py | 6 ++++ packages/zarr-metadata/tests/test_api_docs.py | 25 ++--------------- 34 files changed, 306 insertions(+), 25 deletions(-) diff --git a/packages/zarr-metadata/docs/api/index.md b/packages/zarr-metadata/docs/api/index.md index 8d5cd77902..6dd79e7e89 100644 --- a/packages/zarr-metadata/docs/api/index.md +++ b/packages/zarr-metadata/docs/api/index.md @@ -51,3 +51,7 @@ Two cross-cutting types are also exported from the top-level `zarr_metadata` namespace: `JSONValue`, documented in [`zarr_metadata.typed_json`](typed_json.md), and `ZarrV3NamedConfigJSON`, documented in [`zarr_metadata.v3`](v3/index.md). + +The package version is exported only from the top level: + +::: zarr_metadata.__version__ diff --git a/packages/zarr-metadata/src/zarr_metadata/_json.py b/packages/zarr-metadata/src/zarr_metadata/_json.py index 78f4934903..87a7c9a395 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_json.py @@ -138,12 +138,15 @@ class ValidationProblem: message: str """The human-readable description of what is wrong.""" kind: ProblemKind + """The kind of problem, for programmatic dispatch.""" input: JSONValue | UNSET = dataclasses.field( default=UNSET, kw_only=True, compare=False, repr=False ) + """The JSON found at `loc`, or `UNSET` when nothing is there or it is not JSON.""" ctx: Mapping[str, JSONValue] = dataclasses.field( default_factory=_no_ctx, kw_only=True, compare=False, repr=False ) + """What was expected, e.g. bounds or a closed set of values; empty when only a type was.""" def __post_init__(self) -> None: # Enforce the annotations at runtime. A rule written without a type diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index 3e7ec67ef9..5fa17c14e8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -524,9 +524,13 @@ class Context(Protocol): """ @property - def zarr_format(self) -> Literal[2, 3]: ... + def zarr_format(self) -> Literal[2, 3]: + """The Zarr format of the documents this scope reads.""" + ... - def claimant(self, kind: type[D], name: str) -> D | None: ... + def claimant(self, kind: type[D], name: str) -> D | None: + """The definition of `kind` in scope that reads `name` (a name as a document uses it), or None if there is none.""" + ... @dataclass(frozen=True, slots=True) diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py index 11ad7e1563..d590cc058e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py @@ -51,9 +51,13 @@ class Conflict: """One place where two readings of a name disagree: the key, the definition one reading claimed, the definition the other found, and the location in the document when known.""" key: ClaimKey + """The kind, and the name the two definitions are filed under.""" claimed: Definition[Any] | None + """The definition one reading or scope gives the name; None when it leaves the name unclaimed.""" found: Definition[Any] | None + """The definition the other gives the name; None when it gives none, so the name would lose its meaning.""" loc: Loc | None = None + """Where the name occurs in the document; None when no document is available.""" written: str | None = None """The name the document uses at the conflict location, for example `r16` for a key of `r*`; None when no document is available.""" @@ -90,6 +94,7 @@ class ScopeConflictError(ValueError): def __init__(self, conflicts: Sequence[Conflict]) -> None: self.conflicts: tuple[Conflict, ...] = tuple(conflicts) + """Every conflict, in the order found; the message joins them.""" super().__init__("; ".join(str(conflict) for conflict in self.conflicts)) def __reduce__( diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py index ce749573ef..5ce52849f9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py @@ -420,15 +420,34 @@ class ZarrV2ArrayMetadataUpdate(TypedDict, total=False, closed=True): """ shape: tuple[int, ...] + """The `shape` member: the length of each dimension.""" dtype: ZarrV2DataTypeMetadata + """The `dtype` member: a NumPy typestr, or a list of field records for a structured data type.""" chunks: tuple[int, ...] + """The `chunks` member: the length of each chunk along each dimension, one for each dimension of `shape`.""" fill_value: JSONValue + """The `fill_value` member: the value of an element in a chunk that was never written, or `None` for none.""" order: ZarrV2ArrayOrder + """The `order` member: the layout of elements within each chunk, `"C"` or `"F"`.""" compressor: ZarrV2CodecMetadata | None + """The `compressor` member: the configuration of the codec that compresses each chunk, or `None` for none.""" filters: tuple[ZarrV2CodecMetadata, ...] | None + """The `filters` member: the configuration of each codec applied to each chunk before the compressor, in order, or `None` for none.""" dimension_separator: ZarrV2ArrayDimensionSeparator | UNSET + """The `dimension_separator` member: the separator between a chunk's grid coordinates in its key, `"."` or `"/"`. + + `UNSET` removes it, which means `"."`. + """ attributes: Mapping[str, JSONValue] | UNSET + """The `attributes` member: user attributes, which a store holds in `.zattrs`. + + `UNSET` removes it, so the array has no `.zattrs`. + """ extra_fields: Mapping[str, JSONValue | UNSET] + """The members the spec does not define, by key: each one given is set, or removed when its value is `UNSET`, and any other is kept. + + A key the spec defines, or that the update takes by name, raises `TypeError`. + """ class ZarrV2ArrayMetadata(Keyed): @@ -460,7 +479,12 @@ class ZarrV2ArrayMetadata(Keyed): __slots__ = ("_context", "_document", "_members", "_reading", "_shown") zarr_format: Final[Literal[2]] = 2 # noqa: PYI064 - a bare Final does not narrow a union of models + """The Zarr format version, `2`. A class constant: comparing it narrows a union of models.""" node_type: Final[Literal["array"]] = "array" # noqa: PYI064 - as above + """The kind of node, `"array"`. A class constant: comparing it narrows a union of models. + + A v2 document does not write this member: its `.zarray` key says the node is an array. + """ def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: context = scoped(context, CORE_V2) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 53d12254bc..f5e9b3a665 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -459,15 +459,37 @@ class ZarrV3ArrayMetadataUpdate(TypedDict, total=False, closed=True): """ shape: tuple[int, ...] + """The `shape` member: the length of each dimension.""" data_type: ZarrV3MetadataFieldJSON + """The `data_type` member: the data type of the array's elements.""" chunk_grid: ZarrV3MetadataFieldJSON + """The `chunk_grid` member: how the array is divided into chunks.""" chunk_key_encoding: ZarrV3MetadataFieldJSON + """The `chunk_key_encoding` member: how a chunk's grid coordinates become its store key.""" fill_value: JSONValue + """The `fill_value` member: the value of an element in a chunk that was never written.""" codecs: tuple[ZarrV3MetadataFieldJSON, ...] + """The `codecs` member: the codecs that encode each chunk, in order.""" attributes: Mapping[str, JSONValue] | UNSET + """The `attributes` member: user attributes. + + `UNSET` removes it, so the array has no attributes. + """ storage_transformers: tuple[ZarrV3MetadataFieldJSON, ...] | UNSET + """The `storage_transformers` member: the storage transformers, in order. + + `UNSET` removes it, which means none. + """ dimension_names: tuple[str | None, ...] | UNSET + """The `dimension_names` member: a name for each dimension, `None` for an unnamed one. + + `UNSET` removes it, which means every dimension is unnamed. + """ extra_fields: Mapping[str, ZarrV3ExtensionField | UNSET] + """The members the spec does not define, by key: each one given is set, or removed when its value is `UNSET`, and any other is kept. + + A key the spec defines, or that the update takes by name, raises `TypeError`. + """ class ZarrV3ArrayMetadata(Keyed): @@ -496,7 +518,9 @@ class ZarrV3ArrayMetadata(Keyed): __slots__ = ("_context", "_document", "_members", "_reading", "_shown") zarr_format: Final[Literal[3]] = 3 # noqa: PYI064 - a bare Final does not narrow a union of models + """The Zarr format version, `3`. A class constant: comparing it narrows a union of models.""" node_type: Final[Literal["array"]] = "array" # noqa: PYI064 - as above + """The kind of node, `"array"`. A class constant: comparing it narrows a union of models.""" def __init__(self, document: object, context: ZarrV3Context | None = None) -> None: context = scoped(context, CORE_AND_EXTENSIONS) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py index b26a46ca1b..d391b7f0cb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py @@ -238,6 +238,10 @@ class ZarrV2GroupMetadataUpdate(TypedDict, total=False): """The members `ZarrV2GroupMetadata.update` replaces: `attributes` in the form a `.zattrs` writes them, or `UNSET` for no `.zattrs`.""" attributes: Mapping[str, JSONValue] | UNSET + """The `attributes` member: user attributes, which a store holds in `.zattrs`. + + `UNSET` removes it, so the group has no `.zattrs`. + """ class ZarrV2GroupMetadata(Keyed): @@ -265,7 +269,12 @@ class ZarrV2GroupMetadata(Keyed): _shown: Mapping[str, JSONValue] zarr_format: Final[Literal[2]] = 2 # noqa: PYI064 - a bare Final does not narrow a union of models + """The Zarr format version, `2`. A class constant: comparing it narrows a union of models.""" node_type: Final[Literal["group"]] = "group" # noqa: PYI064 - as above + """The kind of node, `"group"`. A class constant: comparing it narrows a union of models. + + A v2 document does not write this member: its `.zgroup` key says the node is a group. + """ def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: context = scoped(context, CORE_V2) @@ -488,6 +497,7 @@ class ZarrV2ConsolidatedMetadata(Keyed): _metadata: dict[str, ZarrV2NodeMetadata] zarr_consolidated_format: Final = 1 + """The version of the consolidated metadata format, `1`.""" def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: context = scoped(context, CORE_V2) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index daae22cdfe..fd6ac0e845 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -132,8 +132,11 @@ class ZarrV3ConsolidatedMetadataInput(TypedDict, closed=True): """ kind: Literal["inline"] + """How the entries are carried, `"inline"`: in the group's own document.""" must_understand: Literal[False] + """Always `False`: a reader that does not recognize consolidated metadata may ignore it.""" metadata: Mapping[str, ZarrV3NodeMetadataInput] + """The document of each node below the group, or a model of one, by its path from the group.""" class ZarrV3GroupMetadataUpdate(TypedDict, total=False, closed=True): @@ -152,13 +155,25 @@ class ZarrV3GroupMetadataUpdate(TypedDict, total=False, closed=True): """ attributes: Mapping[str, JSONValue] | UNSET + """The `attributes` member: user attributes. + + `UNSET` removes it, so the group has no attributes. + """ consolidated_metadata: ( ZarrV3ConsolidatedMetadataJSON | Mapping[str, JSONValue] | ZarrV3ConsolidatedMetadata | UNSET ) + """The `consolidated_metadata` member: the documents of the nodes below the group, carried in its own document. + + `UNSET` removes it. + """ extra_fields: Mapping[str, ZarrV3ExtensionField | UNSET] + """The members the spec does not define, by key: each one given is set, or removed when its value is `UNSET`, and any other is kept. + + A key the spec defines, or that the update takes by name, raises `TypeError`. + """ class ZarrV3GroupMetadata(Keyed): @@ -182,7 +197,9 @@ class ZarrV3GroupMetadata(Keyed): __slots__ = ("_context", "_document", "_members", "_reading", "_shown") zarr_format: Final[Literal[3]] = 3 # noqa: PYI064 - a bare Final does not narrow a union of models + """The Zarr format version, `3`. A class constant: comparing it narrows a union of models.""" node_type: Final[Literal["group"]] = "group" # noqa: PYI064 - as above + """The kind of node, `"group"`. A class constant: comparing it narrows a union of models.""" def __init__(self, document: object, context: ZarrV3Context | None = None) -> None: context = scoped(context, CORE_AND_EXTENSIONS) @@ -491,7 +508,9 @@ class ZarrV3ConsolidatedMetadata(Keyed): __slots__ = ("_context", "_document", "_metadata") kind: Final = "inline" + """How the entries are carried, `"inline"`: in the group's own document.""" must_understand: Final = False + """Always `False`: a reader that does not recognize consolidated metadata may ignore it.""" def __init__(self, member: object, context: ZarrV3Context | None = None) -> None: context = scoped(context, CORE_AND_EXTENSIONS) diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index f369f2beec..86c8df35d1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -449,6 +449,7 @@ class InScope: """ context: ZarrV3Context | ZarrV2Context + """The scope the field is read in, in place of any scope the validation context holds.""" def __post_init__(self) -> None: if not _is_context(self.context): diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py index 3e3e5ac1b9..df4d336442 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_definition.py @@ -140,7 +140,9 @@ class ZarrV2DataTypeDefinition(WithFillValue[C], kind=True, format=2): """ label: ClassVar[str] = "v2 data type" + """The name of the kind in error messages, "v2 data type".""" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV2DataTypeField,) + """The field alias that annotates a configuration member holding a v2 dtype, `ZarrV2DataTypeField`. The type of each struct record is one.""" def _refusal(self) -> str | None: if parse_typestr(self.name) is not None: @@ -231,6 +233,7 @@ class ZarrV2CodecDefinition(Definition[C], kind=True, format=2): """ label: ClassVar[str] = "v2 codec" + """The name of the kind in error messages, "v2 codec".""" @classmethod def _name_problem(cls, name: str, at: Loc) -> ValidationProblem | None: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_field_json.py b/packages/zarr-metadata/src/zarr_metadata/v2/_field_json.py index f7a004bd17..e6f9763b22 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/_field_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_field_json.py @@ -40,6 +40,7 @@ class ZarrV2CodecMetadata(TypedDict, extra_items=JSONValue): """ id: str + """The numcodecs id that names the codec, such as `"blosc"`.""" __all__ = ["ZarrV2CodecMetadata", "ZarrV2DataTypeMetadata"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index d97c44ff9e..1232d01445 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -45,8 +45,14 @@ class ZarrV2ZArrayJSON(TypedDict): """ zarr_format: Literal[2] + """The Zarr format version, `2`.""" shape: tuple[Annotated[int, Ge(0)], ...] + """The length of each dimension of the array.""" chunks: tuple[Annotated[int, Ge(0)], ...] + """The length of each dimension of a chunk. + + Every chunk of the array has this shape, with one length per dimension of `shape`. + """ dtype: ZarrV2DataTypeMetadata """The data type of the array's elements. @@ -94,8 +100,14 @@ class ZarrV2ArrayMetadataJSON(TypedDict, extra_items=JSONValue): """ zarr_format: Literal[2] + """The Zarr format version, `2`.""" shape: tuple[Annotated[int, Ge(0)], ...] + """The length of each dimension of the array.""" chunks: tuple[Annotated[int, Ge(0)], ...] + """The length of each dimension of a chunk. + + Every chunk of the array has this shape, with one length per dimension of `shape`. + """ dtype: ZarrV2DataTypeMetadata """The data type of the array's elements. diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec/checksum.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec/checksum.py index 2f10849821..af0cb4ed79 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec/checksum.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec/checksum.py @@ -14,6 +14,10 @@ class ZarrV2Checksum32Parameters(TypedDict, closed=True): """`numcodecs.CRC32(location=None)` and the other 32-bit checksums. `location` is where the checksum is placed. When it is omitted or null, numcodecs places it at the start for crc32 and adler32, and at the end for crc32c.""" location: NotRequired[ReadOnly[Literal["start", "end"] | None]] + """Where the checksum is placed in the encoded chunk, `"start"` or `"end"`. + + Absent or null means the start for `crc32` and `adler32`, and the end for `crc32c`. + """ def _fill_location( diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec/compression.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec/compression.py index 14d7e7998f..2b81338c4e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec/compression.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec/compression.py @@ -29,27 +29,44 @@ class ZarrV2ZlibParameters(TypedDict, closed=True): """`numcodecs.Zlib(level=1)`.""" level: NotRequired[ReadOnly[ZarrV2CompressionLevel]] + """The zlib compression level. Absent means 1.""" class ZarrV2GzipParameters(TypedDict, closed=True): """`numcodecs.GZip(level=1)`.""" level: NotRequired[ReadOnly[ZarrV2CompressionLevel]] + """The zlib compression level. Absent means 1.""" class ZarrV2Bz2Parameters(TypedDict, closed=True): """`numcodecs.BZ2(level=1)`: 1 to 9.""" level: NotRequired[ReadOnly[Annotated[int, Interval(ge=1, le=9)]]] + """The bzip2 compression level. Absent means 1.""" class ZarrV2LzmaParameters(TypedDict, closed=True): """`numcodecs.LZMA(format=1, check=-1, preset=None, filters=None)`, with the parameters as the `lzma` module accepts them.""" format: NotRequired[ReadOnly[Annotated[int, Interval(ge=0, le=3)]]] + """The container format, an `lzma.FORMAT_*` code. Absent means 1, `FORMAT_XZ`. + + 2 is `FORMAT_ALONE`, and 3 is `FORMAT_RAW`, which requires `filters`. 0, + `FORMAT_AUTO`, only decompresses: `lzma` refuses it when compressing. + """ check: NotRequired[ReadOnly[int]] + """The integrity check, an `lzma.CHECK_*` code, or -1 for the format's default. Absent means -1. + + Only `FORMAT_XZ` takes a check other than -1 or `CHECK_NONE` (0). + """ preset: NotRequired[ReadOnly[Annotated[int, Ge(0)] | None]] + """The compression preset, 0 to 9, optionally ORed with `lzma.PRESET_EXTREME`, or null for lzma's default, 6. + + Absent means null. `lzma` refuses a preset together with `filters`. + """ filters: NotRequired[ReadOnly[tuple[JSONValue, ...] | None]] + """A custom filter chain, as `lzma` filter specifiers, or null to use `preset`. Absent means null.""" ZarrV2BloscCName = Literal["blosclz", "lz4", "lz4hc", "snappy", "zlib", "zstd"] @@ -70,23 +87,34 @@ class ZarrV2BloscParameters(TypedDict, closed=True): """ cname: NotRequired[ReadOnly[ZarrV2BloscCName]] + """The compressor that Blosc runs. Absent means `"lz4"`.""" clevel: NotRequired[ReadOnly[Annotated[int, Interval(ge=0, le=9)]]] + """The compression level. Absent means 5. At 0, Blosc stores the data uncompressed.""" shuffle: NotRequired[ReadOnly[Literal[-1, 0, 1, 2] | bool]] + """How Blosc shuffles the data before compressing it. Absent means 1, byte shuffle.""" blocksize: NotRequired[ReadOnly[_CInt]] + """The size in bytes of the blocks that Blosc compresses each chunk in. Absent means 0, which lets Blosc choose.""" typesize: NotRequired[ReadOnly[Annotated[int, Interval(ge=1, le=2**31 - 1)] | None]] + """The size in bytes of one element, the unit that Blosc shuffles by. + + Absent or null means the item size of the buffer that numcodecs is handed. + """ class ZarrV2ZstdParameters(TypedDict, closed=True): """`numcodecs.Zstd(level=0, checksum=False)`. Level 0 is zstd's default level, and negative levels give up compression ratio for speed. zstd clamps a level outside its own range, -131072 to 22, to that range, so numcodecs reads any level that fits in a C `int`.""" level: NotRequired[ReadOnly[_CInt]] + """The zstd compression level. Absent means 0, zstd's default level.""" checksum: NotRequired[ReadOnly[bool]] + """Whether zstd writes a checksum of the content into each frame. Absent means false.""" class ZarrV2Lz4Parameters(TypedDict, closed=True): """`numcodecs.LZ4(acceleration=1)`. numcodecs passes `acceleration` to C as an `int`, and lz4 reads one below 1 as 1.""" acceleration: NotRequired[ReadOnly[_CInt]] + """How far lz4 trades compression ratio for speed: a larger value is faster. Absent means 1.""" def _zlib_canonical(configuration: ZarrV2ZlibParameters) -> ZarrV2ZlibParameters: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py index 4c5ad5c94b..f840ef531f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec/filters.py @@ -23,22 +23,29 @@ class ZarrV2ShuffleParameters(TypedDict, closed=True): """`numcodecs.Shuffle(elementsize=4)`. An element size of 1 or less leaves the bytes as they are. numcodecs passes a larger one to C as a `Py_ssize_t`, so it is at most 2**63 - 1 on a 64-bit platform, and raises `OverflowError` for any other.""" elementsize: NotRequired[ReadOnly[Annotated[int, Le(2**63 - 1)]]] + """The size in bytes of the elements whose bytes are shuffled. Absent means 4.""" class ZarrV2DeltaParameters(TypedDict, closed=True): """`numcodecs.Delta(dtype, astype=None)`. When `astype` is omitted or null, it equals `dtype`.""" dtype: ReadOnly[str] + """The dtype of the decoded data.""" astype: NotRequired[ReadOnly[str | None]] + """The dtype of the encoded data. Absent or null means `dtype`.""" class ZarrV2FixedScaleOffsetParameters(TypedDict, closed=True): """`numcodecs.FixedScaleOffset(offset, scale, dtype, astype=None)`. When `astype` is omitted or null, it equals `dtype`.""" offset: ReadOnly[float | int] + """The value subtracted from each element before it is scaled.""" scale: ReadOnly[float | int] + """The factor that each element is multiplied by after the offset is subtracted. The product is rounded to the nearest integer.""" dtype: ReadOnly[str] + """The dtype of the decoded data.""" astype: NotRequired[ReadOnly[str | None]] + """The dtype of the encoded data. Absent or null means `dtype`.""" class ZarrV2QuantizeParameters(TypedDict, closed=True): @@ -50,21 +57,31 @@ class ZarrV2QuantizeParameters(TypedDict, closed=True): """ digits: ReadOnly[Annotated[int, Interval(ge=-308, le=307)]] + """The number of decimal digits of precision to keep.""" dtype: ReadOnly[str] + """The float dtype of the decoded data.""" astype: NotRequired[ReadOnly[str | None]] + """The float dtype of the encoded data. Absent or null means `dtype`.""" class ZarrV2BitRoundParameters(TypedDict, closed=True): """`numcodecs.BitRound(keepbits)`.""" keepbits: ReadOnly[Annotated[int, Ge(0)]] + """The number of mantissa bits to keep. + + numcodecs refuses, when encoding, a value above the mantissa width of the + data's float type: 10, 23 or 52 bits. + """ class ZarrV2AsTypeParameters(TypedDict, closed=True): """`numcodecs.AsType(encode_dtype, decode_dtype)`.""" encode_dtype: ReadOnly[str] + """The dtype of the encoded data.""" decode_dtype: ReadOnly[str] + """The dtype of the decoded data.""" def _shuffle_canonical(configuration: ZarrV2ShuffleParameters) -> ZarrV2ShuffleParameters: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py index 39b822f098..4497ab64e5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec/vlen.py @@ -15,6 +15,7 @@ class ZarrV2VLenArrayParameters(TypedDict, closed=True): """`numcodecs.VLenArray(dtype)`: the element type of each array.""" dtype: ReadOnly[str] + """The dtype of the elements of each variable-length array.""" def _vlen_array_canonical(configuration: ZarrV2VLenArrayParameters) -> ZarrV2VLenArrayParameters: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py index 01a1fd42cd..9e4dbf4605 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/object.py @@ -16,6 +16,7 @@ class ZarrV2ObjectConfiguration(TypedDict, closed=True): """The configuration that `|O` carries: a byte order, which the type ignores.""" byteorder: ReadOnly[ZarrV2ByteOrder] + """The byte order that the typestr writes. The canonical form writes `|`, as NumPy does.""" def _canonical(configuration: ZarrV2ObjectConfiguration) -> ZarrV2ObjectConfiguration: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py index 6f94fa793b..2f9ec2393b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/scalar.py @@ -27,7 +27,14 @@ class ZarrV2ScalarConfiguration(TypedDict, closed=True): """The configuration that a scalar typestr carries: its byte order and its size in bytes, such as `{"byteorder": "<", "itemsize": 4}` for ` Iterator[ValidationProblem]: diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py index ffa0379053..ffca4e8662 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/data_type/time.py @@ -26,9 +26,17 @@ class ZarrV2TimeConfiguration(TypedDict, closed=True): """The configuration that a time typestr carries. For example, ``.""" itemsize: ReadOnly[Annotated[int, Ge(0)]] + """The size in bytes that the typestr writes, which must be 8.""" unit: NotRequired[ReadOnly[str]] + """The NumPy time unit in brackets, such as `ns`. The canonical form writes `μs` as `us`. + + An array's dtype must have one. Only a codec's dtype parameter may leave + it out, as in ` ZarrV2TimeConfiguration: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py index 3cf371f4b4..83b2418f00 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -188,7 +188,9 @@ class DataTypeDefinition(WithFillValue[C], kind=True, format=3): """ label: ClassVar[str] = "data type" + """The name of the kind in error messages, `"data type"`.""" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3DataTypeField,) + """The field alias that annotates a configuration member holding a data type, `ZarrV3DataTypeField`.""" @classmethod def _filed_under(cls, name: str) -> tuple[str | None, dict[str, JSONValue] | None]: @@ -246,7 +248,9 @@ class ChunkGridDefinition(Definition[C], kind=True, format=3): """ label: ClassVar[str] = "chunk grid" + """The name of the kind in error messages, `"chunk grid"`.""" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3ChunkGridField,) + """The field alias that annotates a configuration member holding a chunk grid, `ZarrV3ChunkGridField`.""" shape_rules: Callable[[C, Nested, tuple[int, ...]], Iterable[ValidationProblem]] = no_rules """The spec violations in this grid over an array of a given shape, located in the configuration.""" @@ -259,7 +263,9 @@ class ChunkKeyEncodingDefinition(Definition[C], kind=True, format=3): """A chunk key encoding.""" label: ClassVar[str] = "chunk key encoding" + """The name of the kind in error messages, `"chunk key encoding"`.""" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3ChunkKeyEncodingField,) + """The field alias that annotates a configuration member holding a chunk key encoding, `ZarrV3ChunkKeyEncodingField`.""" CodecKind = Literal["array_array", "array_bytes", "bytes_bytes"] @@ -320,7 +326,13 @@ class CodecDefinition(Definition[C], kind=True, format=3): """ label: ClassVar[str] = "codec" + """The name of the kind in error messages, `"codec"`.""" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3CodecField, ZarrV3StaticCodecField) + """The field aliases that annotate a configuration member holding a codec. + + `ZarrV3CodecField` admits any codec; `ZarrV3StaticCodecField` admits only + codecs of static size. + """ kind: CodecKind """What the codec does to what it is handed: array -> array, array -> bytes or bytes -> bytes. @@ -377,7 +389,9 @@ class StorageTransformerDefinition(Definition[C], kind=True, format=3): """A storage transformer.""" label: ClassVar[str] = "storage transformer" + """The name of the kind in error messages, `"storage transformer"`.""" field_aliases: ClassVar[tuple[TypeAliasType, ...]] = (ZarrV3StorageTransformerField,) + """The field alias that annotates a configuration member holding a storage transformer, `ZarrV3StorageTransformerField`.""" @dataclass(frozen=True, kw_only=True, slots=True, repr=False) @@ -560,9 +574,11 @@ class ArrayExtraFieldDefinition(ExtraFieldDefinition[C], kind=True, format=3): """An extra field of a v3 array document: a member the spec does not define, read by its key.""" label: ClassVar[str] = "array extra field" + """The name of the kind in error messages, `"array extra field"`.""" spec_members: ClassVar[frozenset[str]] = frozenset( typeddict_keys(ZarrV3ArrayMetadataJSON).members ) + """The members the spec defines in an array document, the keys of `ZarrV3ArrayMetadataJSON`; never extra fields.""" @dataclass(frozen=True, kw_only=True, slots=True, repr=False) @@ -570,9 +586,11 @@ class GroupExtraFieldDefinition(ExtraFieldDefinition[C], kind=True, format=3): """An extra field of a v3 group document: a member the spec does not define, read by its key. `consolidated_metadata` is one.""" label: ClassVar[str] = "group extra field" + """The name of the kind in error messages, `"group extra field"`.""" spec_members: ClassVar[frozenset[str]] = frozenset( typeddict_keys(ZarrV3GroupMetadataJSON).members ) + """The members the spec defines in a group document, the keys of `ZarrV3GroupMetadataJSON`; never extra fields.""" STATIC_SIZE_ALIASES: Final[frozenset[TypeAliasType]] = frozenset({ZarrV3StaticCodecField}) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/array.py b/packages/zarr-metadata/src/zarr_metadata/v3/array.py index fbbf6895f2..36c03eed9a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/array.py @@ -33,14 +33,34 @@ class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=ZarrV3ExtensionField): zarr_format: Literal[3] """The Zarr format version, `3`.""" node_type: Literal["array"] + """The kind of node the document describes, `"array"`.""" data_type: ZarrV3DataTypeField + """The data type of the array's elements, e.g. `"float64"`.""" shape: tuple[Annotated[int, Ge(0)], ...] + """The length of each dimension of the array.""" chunk_grid: ZarrV3ChunkGridField + """How the array is divided into chunks, e.g. a `"regular"` grid of one chunk shape.""" chunk_key_encoding: ZarrV3ChunkKeyEncodingField + """How a chunk's grid coordinates become its key in the store.""" fill_value: JSONValue + """The value of an element in a chunk that was never written. + + Which JSON values it may take depends on `data_type`. + """ codecs: tuple[ZarrV3CodecField, ...] + """The codecs that encode each chunk, in encoding order. + + Zero or more array -> array codecs, then exactly one array -> bytes codec, + then zero or more bytes -> bytes codecs, so the list is never empty + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L974-L983). + """ attributes: NotRequired[Mapping[str, JSONValue]] + """User attributes: arbitrary JSON values by string key.""" storage_transformers: NotRequired[tuple[ZarrV3StorageTransformerField, ...]] + """The storage transformers between the array and its store. + + Absent means none, as does an empty list. + """ dimension_names: NotRequired[tuple[str | None, ...]] """A name for each dimension, `null` for an unnamed one. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 29544ec713..365e27ac74 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -38,6 +38,10 @@ class RegularChunkGridConfiguration(TypedDict, closed=True): """ chunk_shape: tuple[Annotated[int, Ge(1)], ...] + """The length of every chunk along each dimension of the array. + + One entry per dimension, each at least 1. + """ class RegularChunkGridObject(TypedDict, closed=True): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 64185104d7..f5672e9e61 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -37,9 +37,24 @@ class BloscCodecConfiguration(TypedDict, closed=True): """Configuration for the Zarr v3 `blosc` codec.""" cname: BloscCName + """The compressor blosc applies to each block, one of `BLOSC_CNAME`.""" clevel: Annotated[int, Interval(ge=0, le=9)] + """The compression level, an integer from 0 to 9. + + 0 turns compression off; 1 is the fastest and compresses least, 9 the + slowest and compresses most. + """ shuffle: BloscShuffle + """The shuffling applied before compression. + + `"noshuffle"` for none, `"shuffle"` for byte-wise, `"bitshuffle"` for + bit-wise. + """ blocksize: Annotated[int, Ge(0)] + """The size in bytes of the blocks the buffer is divided into before compression. + + At least 0; 0 lets blosc choose the size. + """ typesize: NotRequired[int] """The stride in bytes over which shuffling is performed. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 801a1c841e..50d644d5f9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -95,6 +95,13 @@ class CastValueCodecConfiguration(TypedDict, closed=True): """ data_type: ZarrV3DataTypeField + """The data type values are cast to when encoding, and the one the decoder is handed. + + The codec is defined only for data types that model real numbers, the + integer and floating-point types: a boolean, raw-bits, string, bytes, + struct or complex type is refused, here and as the data type the codec + is handed. + """ rounding: NotRequired[CastRoundingMode] """How a value `data_type` cannot represent exactly is rounded. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 05c4e95a1e..044f9fe1b8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -30,7 +30,12 @@ class Crc32cCodecObject(TypedDict, closed=True): """ name: Crc32cCodecName + """The codec's name, `"crc32c"`.""" configuration: NotRequired[EmptyConfiguration] + """The codec's parameters, of which there are none. + + Absent, or an empty mapping. + """ must_understand: NotRequired[bool] """Whether a reader that does not recognize this codec must refuse the document. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 4a5948c0e8..1dc7220cfb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -33,6 +33,11 @@ class GzipCodecConfiguration(TypedDict, closed=True): """ level: Annotated[int, Interval(ge=0, le=9)] + """The compression level, an integer from 0 to 9. + + 0 turns compression off; 1 is the fastest and compresses least, 9 the + slowest and compresses most. + """ class GzipCodecObject(TypedDict, closed=True): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index d85af2650b..73def2cafa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -60,8 +60,23 @@ class ShardingIndexedCodecConfiguration(TypedDict, closed=True): """ chunk_shape: tuple[Annotated[int, Ge(1)], ...] + """The shape of the inner chunks a shard is divided into. + + One length, at least 1, per axis of the shard the codec is handed, each + dividing the shard's length along that axis. + """ codecs: tuple[ZarrV3CodecField, ...] + """The codec pipeline that encodes each inner chunk. + + Like an array's `codecs`, it holds exactly one array -> bytes codec. + """ index_codecs: tuple[ZarrV3StaticCodecField, ...] + """The codec pipeline that encodes the shard index. + + It holds exactly one array -> bytes codec, and only codecs whose output + size is fixed by the size of their input, so no compressor. The spec + recommends a little-endian `bytes` codec followed by `crc32c`. + """ index_location: NotRequired[ShardingIndexLocation] """Whether the shard index sits at the `"start"` or the `"end"` of the encoded shard. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index d2d3dcd2fb..222346caac 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -36,6 +36,11 @@ class ZstdCodecConfiguration(TypedDict, closed=True): """ level: Annotated[int, Interval(ge=ZSTD_MIN_LEVEL, le=ZSTD_MAX_LEVEL)] + """The compression level, an integer from `ZSTD_MIN_LEVEL` (-131072) to `ZSTD_MAX_LEVEL` (22). + + 0 means zstd's default level; otherwise a higher level compresses more, + more slowly. It has no effect on decoding. + """ checksum: NotRequired[bool] """Whether a checksum is stored when writing and verified when reading. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index 2ec8afc994..c3a76cdcd9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -42,8 +42,11 @@ class ZarrV3ConsolidatedMetadataEnvelopeJSON(TypedDict, closed=True): """ kind: Literal["inline"] + """How the entries are carried, `"inline"`: in the group's own document.""" must_understand: Literal[False] + """Always `false`: a reader that does not recognize consolidated metadata may ignore it.""" metadata: Mapping[str, UserJSONValue] + """The entries, by each node's path from the group, as any JSON.""" class ZarrV3ConsolidatedMetadataJSON(TypedDict, closed=True): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 78d4b81aca..af4394abb9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -31,7 +31,16 @@ class NumpyDatetime64Configuration(TypedDict, closed=True): """Configuration for the `numpy.datetime64` data type.""" unit: ReadOnly[NumpyTimeUnit] + """A string encoding a unit of time. + + `"us"` and `"μs"` are equivalent spellings of microseconds; `"generic"` + means the unit is unspecified. + """ scale_factor: ReadOnly[NumpyTimeScaleFactor] + """The multiplier relative to the unit. + + An integer in `[1, 2147483647]`. + """ class NumpyDatetime64(TypedDict, closed=True): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index bf29523551..f320226ee8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -27,7 +27,16 @@ class NumpyTimedelta64Configuration(TypedDict, closed=True): """Configuration for the `numpy.timedelta64` data type.""" unit: ReadOnly[NumpyTimeUnit] + """A string encoding a unit of time. + + `"us"` and `"μs"` are equivalent spellings of microseconds; `"generic"` + means the unit is unspecified. + """ scale_factor: ReadOnly[NumpyTimeScaleFactor] + """The multiplier relative to the unit. + + An integer in `[1, 2147483647]`. + """ class NumpyTimedelta64(TypedDict, closed=True): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 402f6ce1fa..f8916b170d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -34,7 +34,13 @@ class StructField(TypedDict, closed=True): """A single field entry inside a structured dtype.""" name: ReadOnly[str] + """The field name (must be unique within a struct and non-empty).""" data_type: ReadOnly[ZarrV3DataTypeField] + """The field's data type. + + Recursive: may be a bare-string primitive or a named-config envelope + including another `struct`. + """ class StructConfiguration(TypedDict, closed=True): diff --git a/packages/zarr-metadata/tests/test_api_docs.py b/packages/zarr-metadata/tests/test_api_docs.py index 633f454cd2..ccfb4e8cd9 100644 --- a/packages/zarr-metadata/tests/test_api_docs.py +++ b/packages/zarr-metadata/tests/test_api_docs.py @@ -37,32 +37,11 @@ if not any(part.startswith("_") for part in name.split(".")) ] -# A type whose fields mean what another type's same-named fields mean: a -# partial form, or a strict on-disk form of a merged in-memory one. Their -# field docstrings are the same text. +# A strict on-disk v2 type whose fields mean what the merged in-memory +# type's same-named fields mean. Their field docstrings are the same text. MIRRORS = [ - ( - "zarr_metadata.v2.array.ZarrV2ArrayMetadataJSON", - "zarr_metadata.v2.array.ZarrV2ArrayMetadataJSONPartial", - ), ("zarr_metadata.v2.array.ZarrV2ArrayMetadataJSON", "zarr_metadata.v2.array.ZarrV2ZArrayJSON"), - ( - "zarr_metadata.v2.group.ZarrV2GroupMetadataJSON", - "zarr_metadata.v2.group.ZarrV2GroupMetadataJSONPartial", - ), ("zarr_metadata.v2.group.ZarrV2GroupMetadataJSON", "zarr_metadata.v2.group.ZarrV2ZGroupJSON"), - ( - "zarr_metadata.v3.array.ZarrV3ArrayMetadataJSON", - "zarr_metadata.v3.array.ZarrV3ArrayMetadataJSONPartial", - ), - ( - "zarr_metadata.v3.group.ZarrV3GroupMetadataJSON", - "zarr_metadata.v3.group.ZarrV3GroupMetadataJSONPartial", - ), - ("zarr_metadata.model.ZarrV2ArrayMetadata", "zarr_metadata.model.ZarrV2ArrayMetadataPartial"), - ("zarr_metadata.model.ZarrV2GroupMetadata", "zarr_metadata.model.ZarrV2GroupMetadataPartial"), - ("zarr_metadata.model.ZarrV3ArrayMetadata", "zarr_metadata.model.ZarrV3ArrayMetadataPartial"), - ("zarr_metadata.model.ZarrV3GroupMetadata", "zarr_metadata.model.ZarrV3GroupMetadataPartial"), ]