feat(zarr-metadata): export base64_fill_value_rules for data type definitions

packages/zarr-metadata/changes/428.feature.md

@@ -0,0 +1 @@
+Export `base64_fill_value_rules` from `zarr_metadata.v3.data_type.bytes`, the rule that rejects a fill value string that is not standard-alphabet base64, so a data type defined outside the package can pass it as `fill_value_rules` for a `Base64Bytes` fill value, or yield from it in a rule of its own. The guide to parsing and validation now shows how to define a data type with `fill_value_rules`.

packages/zarr-metadata/docs/parsing-validation.md

@@ -227,4 +227,50 @@ reading = read_array_metadata_v3(raw, context=context)
 
 `update` replaces each slot it is given and keeps the others. A slot can also be given as a mapping from each definition's name to it, so `CORE_AND_EXTENSIONS_CONTEXT.update(codecs={**CORE_AND_EXTENSIONS_CONTEXT.codecs, "gzip": MY_GZIP})` replaces the gzip definition with your own.
 
+A data type's definition also says what its fill value may be. `fill_value` is the fill value's JSON shape, as a type annotation. `fill_value_rules` is a function of the configuration, the nested fields and the fill value, that yields each problem the shape cannot express. For example, `Base64Bytes` is a string, so an annotation accepts any string, and the rule `base64_fill_value_rules` rejects one that is not base64. A rule of your own can yield from it, then check what depends on the configuration:
+
+```python test="true" session="parsing-validation"
+import base64
+
+from typing_extensions import TypedDict
+
+from zarr_metadata.definition import fill_value_problems
+from zarr_metadata.problems import ValidationProblem
+from zarr_metadata.v3.data_type.bytes import Base64Bytes, base64_fill_value_rules
+from zarr_metadata.v3.definition import DataTypeDefinition
+
+
+class PackedConfiguration(TypedDict, closed=True):
+    length_bytes: int
+
+
+def packed_fill_value_rules(configuration, nested, value):
+    # The fill value is base64, of at most `length_bytes` bytes.
+    problems = list(base64_fill_value_rules(configuration, nested, value))
+    yield from problems
+    if not problems and len(base64.b64decode(value)) > configuration["length_bytes"]:
+        yield ValidationProblem((), "expected fewer bytes than the type holds", "invalid_value")
+
+
+PACKED = DataTypeDefinition(
+    name="acme.packed",
+    configuration=PackedConfiguration,
+    fill_value=Base64Bytes,
+    fill_value_rules=packed_fill_value_rules,
+)
+context = CORE_AND_EXTENSIONS_CONTEXT.update(
+    data_types=[*CORE_AND_EXTENSIONS_CONTEXT.data_types.values(), PACKED]
+)
+data_type, _ = resolve(
+    {"name": "acme.packed", "configuration": {"length_bytes": 2}}, DataTypeDefinition, context
+)
+for fill_value in ("YWI=", "not base64!", "YWJj"):
+    print(fill_value, [problem.message for problem in fill_value_problems(data_type, fill_value)])
+# YWI= []
+# not base64! ['expected standard-alphabet base64, got "not base64!"']
+# YWJj ['expected fewer bytes than the type holds']
+```
+
+A reader of a document in this scope checks its fill value with these rules, and so does a pydantic field. A JSON Schema written for the scope does not contain the rules, so it accepts any string here.
+
 [^1]: The one exception is the `r*` data type, which is identified by a templated string with no `configuration` object. This mixes the identifier with the configuration in the same JSON value, which is not good design. So this library normalizes `r*` into the same form as all the other metadata objects: the field `"r16"` keeps its name, and is read by the `r*` definition with the configuration `{"bits": 16}`.

packages/zarr-metadata/src/zarr_metadata/_numpy.py

@@ -5,10 +5,13 @@
 """
 
 import re
+from collections.abc import Iterator
 from typing import Annotated, Final, Literal, NewType
 
 from annotated_types import Interval
 
+from zarr_metadata._json import ValidationProblem, shown
+
 NUMPY_TIME_UNITS: Final = (
     "Y",
     "M",
@@ -75,6 +78,29 @@ def base64_bytes(value: str) -> Base64Bytes:
     return Base64Bytes(value)
 
 
+def base64_fill_value_rules(
+    configuration: object, nested: object, value: object
+) -> Iterator[ValidationProblem]:
+    """Check that a fill value written as a string is standard-alphabet base64: a data type's `fill_value_rules`.
+
+    `Base64Bytes` is a `NewType` of `str`, so a fill value annotated with
+    it accepts any string. Pass this rule as the data type's
+    `fill_value_rules` to reject a string that is not base64. A value of
+    any other shape is not checked here, so the rule also serves a fill
+    value that may be written another way, such as
+    `tuple[ByteValue, ...] | Base64Bytes`. A rule that checks more, such
+    as the number of bytes, yields from this one first.
+    """
+    if not isinstance(value, str):
+        return
+    try:
+        base64_bytes(value)
+    except ValueError:
+        yield ValidationProblem(
+            (), f"expected standard-alphabet base64, got {shown(value)}", "invalid_value"
+        )
+
+
 __all__ = [
     "NOT_A_TIME_TICKS",
     "NUMPY_TIME_MAX_SCALE_FACTOR",
@@ -83,5 +109,6 @@ def base64_bytes(value: str) -> Base64Bytes:
     "NumpyTimeScaleFactor",
     "NumpyTimeTicks",
     "base64_bytes",
+    "base64_fill_value_rules",
     "numpy_time_fill_value_canonical",
 ]

packages/zarr-metadata/src/zarr_metadata/v2/data_type/_rules.py

@@ -6,8 +6,8 @@
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Literal
 
-from zarr_metadata._json import ValidationProblem, shown
-from zarr_metadata._numpy import NUMPY_TIME_UNITS, base64_bytes
+from zarr_metadata._json import ValidationProblem
+from zarr_metadata._numpy import NUMPY_TIME_UNITS, base64_fill_value_rules
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
@@ -139,12 +139,9 @@ def sized_base64_rules(
     """Check that a fill value that is not null is standard-alphabet base64 of exactly `size` bytes, or of at most `size` bytes when `exact` is False. Any size passes when `size` is None, which happens only when the item size is unknown."""
     if value is None:
         return
-    try:
-        base64_bytes(value)
-    except ValueError:
-        yield ValidationProblem(
-            (), f"expected standard-alphabet base64, got {shown(value)}", "invalid_value"
-        )
+    problems = list(base64_fill_value_rules(None, (), value))
+    if problems:
+        yield from problems
         return
     if size is None:
         return

packages/zarr-metadata/src/zarr_metadata/v3/data_type/_byte.py

@@ -1,30 +1,15 @@
-"""A byte value, as used in the fill values of raw bits and of `bytes`, and the rules of a fill value written in base64.
+"""A byte value, as used in the fill values of raw bits and of `bytes`.
 
-Each byte value is an integer in `[0, 255]`
+Each is an integer in `[0, 255]`
 (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L97-L99;
 the reference for the `bytes` type is https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/bytes/README.md?plain=1#L8).
 """
 
-from collections.abc import Iterator
 from typing import Annotated
 
 from annotated_types import Interval
 
-from zarr_metadata._json import ValidationProblem, shown
-from zarr_metadata._numpy import base64_bytes
-
 ByteValue = Annotated[int, Interval(ge=0, le=255)]
 """One byte of a fill value: a JSON integer in `[0, 255]`."""
 
-
-def base64_problems(value: str) -> Iterator[ValidationProblem]:
-    """Check that a fill value written as a string is standard-alphabet base64."""
-    try:
-        base64_bytes(value)
-    except ValueError:
-        yield ValidationProblem(
-            (), f"expected standard-alphabet base64, got {shown(value)}", "invalid_value"
-        )
-
-
-__all__ = ["ByteValue", "base64_problems"]
+__all__ = ["ByteValue"]

packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py

@@ -5,11 +5,9 @@
 """
 
 import base64
-from collections.abc import Iterator
 from typing import Final, Literal
 
-from zarr_metadata._json import ValidationProblem
-from zarr_metadata._numpy import Base64Bytes, base64_bytes
+from zarr_metadata._numpy import Base64Bytes, base64_bytes, base64_fill_value_rules
 from zarr_metadata.definition._definition import (
     EmptyConfiguration,
     Nested,
@@ -18,7 +16,7 @@
     DataTypeDefinition,
     variable_length,
 )
-from zarr_metadata.v3.data_type._byte import ByteValue, base64_problems
+from zarr_metadata.v3.data_type._byte import ByteValue
 
 BYTES_DATA_TYPE_NAME: Final = "bytes"
 """The `data_type` value for the variable-length `bytes` type."""
@@ -34,14 +32,6 @@
 """
 
 
-def _fill_value_rules(
-    configuration: EmptyConfiguration, nested: Nested, value: BytesFillValue
-) -> Iterator[ValidationProblem]:
-    """Check that a string fill value is standard-alphabet base64. A fill value written as byte values is not checked here."""
-    if isinstance(value, str):
-        yield from base64_problems(value)
-
-
 def _fill_value_canonical(
     configuration: EmptyConfiguration, nested: Nested, value: BytesFillValue
 ) -> str:
@@ -60,7 +50,7 @@ def _fill_value_canonical(
     name=BYTES_DATA_TYPE_NAME,
     configuration=EmptyConfiguration,
     fill_value=BytesFillValue,
-    fill_value_rules=_fill_value_rules,
+    fill_value_rules=base64_fill_value_rules,
     fill_value_canonical=_fill_value_canonical,
     storage=variable_length,
 )
@@ -79,4 +69,5 @@ def _fill_value_canonical(
     "BytesDataTypeName",
     "BytesFillValue",
     "base64_bytes",
+    "base64_fill_value_rules",
 ]

packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py

@@ -15,10 +15,9 @@
 from typing_extensions import ReadOnly, TypedDict
 
 from zarr_metadata._json import ValidationProblem
-from zarr_metadata._numpy import Base64Bytes
+from zarr_metadata._numpy import Base64Bytes, base64_fill_value_rules
 from zarr_metadata.definition._definition import Nested
 from zarr_metadata.v3._definition import DataTypeDefinition, single_byte
-from zarr_metadata.v3.data_type._byte import base64_problems
 
 NULL_TERMINATED_BYTES_DATA_TYPE_NAME: Final = "null_terminated_bytes"
 """The `name` field value of the `null_terminated_bytes` data type."""
@@ -69,7 +68,7 @@ def _fill_value_rules(
     value: NullTerminatedBytesFillValue,
 ) -> Iterator[ValidationProblem]:
     """Check that the fill value is base64, and that the bytes it encodes fit in `length_bytes`."""
-    problems = list(base64_problems(value))
+    problems = list(base64_fill_value_rules(configuration, nested, value))
     if problems:
         yield from problems
         return

packages/zarr-metadata/src/zarr_metadata/v3/data_type/variable_length_bytes.py

@@ -10,14 +10,11 @@
 """
 
 import base64
-from collections.abc import Iterator
 from typing import Final, Literal
 
-from zarr_metadata._json import ValidationProblem
-from zarr_metadata._numpy import Base64Bytes
+from zarr_metadata._numpy import Base64Bytes, base64_fill_value_rules
 from zarr_metadata.definition._definition import EmptyConfiguration, Nested
 from zarr_metadata.v3._definition import DataTypeDefinition, variable_length
-from zarr_metadata.v3.data_type._byte import base64_problems
 
 VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME: Final = "variable_length_bytes"
 """The `data_type` value for the `variable_length_bytes` type."""
@@ -29,13 +26,6 @@
 """Permitted JSON shape of the `fill_value` field for `variable_length_bytes`: a `Base64Bytes` string, which is what zarr-python writes and reads."""
 
 
-def _fill_value_rules(
-    configuration: EmptyConfiguration, nested: Nested, value: VariableLengthBytesFillValue
-) -> Iterator[ValidationProblem]:
-    """Check that the fill value is standard-alphabet base64."""
-    yield from base64_problems(value)
-
-
 def _fill_value_canonical(
     configuration: EmptyConfiguration, nested: Nested, value: VariableLengthBytesFillValue
 ) -> str:
@@ -47,7 +37,7 @@ def _fill_value_canonical(
     name=VARIABLE_LENGTH_BYTES_DATA_TYPE_NAME,
     configuration=EmptyConfiguration,
     fill_value=VariableLengthBytesFillValue,
-    fill_value_rules=_fill_value_rules,
+    fill_value_rules=base64_fill_value_rules,
     fill_value_canonical=_fill_value_canonical,
     storage=variable_length,
 )

packages/zarr-metadata/tests/v3/data_type/bytes/test_validators.py

@@ -1,16 +1,16 @@
-"""Cover the `base64_bytes` brand validator.
+"""Cover the `base64_bytes` brand validator and the `base64_fill_value_rules` fill value rule.
 
 The pydantic-driven fixture tests don't enforce the base64 alphabet or
 length-multiple-of-4 constraint because `Base64Bytes` is a `NewType`,
 which pydantic treats as plain `str`. Direct coverage of the validator
-function lives here.
+function and the rule built on it lives here.
 """
 
 from __future__ import annotations
 
 import pytest
 
-from zarr_metadata.v3.data_type.bytes import base64_bytes
+from zarr_metadata.v3.data_type.bytes import base64_bytes, base64_fill_value_rules
 
 VALID = [
     "",  # empty is valid base64 (length 0, multiple of 4)
@@ -41,3 +41,16 @@ def test_valid(value: str) -> None:
 def test_invalid(value: str) -> None:
     with pytest.raises(ValueError, match="standard-alphabet base64"):
         base64_bytes(value)
+
+
+# A value written in another shape, such as byte values, is the annotation's to check.
+@pytest.mark.parametrize("value", [*VALID, (1, 2, 3), None])
+def test_base64_fill_value_rules_accepts(value: object) -> None:
+    assert list(base64_fill_value_rules({}, (), value)) == []
+
+
+@pytest.mark.parametrize("value", INVALID)
+def test_error_base64_fill_value_rules_rejects_a_string_that_is_not_base64(value: str) -> None:
+    assert [(p.loc, p.kind) for p in base64_fill_value_rules({}, (), value)] == [
+        ((), "invalid_value")
+    ]