feat(zarr-metadata)!: replace scope update with copy(update=...) and add definitions with union

packages/zarr-metadata/changes/427.removal.md

@@ -0,0 +1 @@
+Replace `ZarrV3Context.update(**slots)` and `ZarrV2Context.update(**slots)` with `copy(*, update=None)`, after pydantic's `model_copy(update=...)`: `context.update(codecs=[...])` becomes `context.copy(update={"codecs": [...]})`, and still replaces each slot it is given whole. To add definitions to a scope and keep the ones it has, use `union`: `CORE_AND_EXTENSIONS.union(ZarrV3Context(data_types=[MY_DATA_TYPE]))`, which the docs now show in place of splicing a slot's definitions back in.

packages/zarr-metadata/docs/context.md

@@ -73,10 +73,10 @@ In this example, our context object only contained one codec definition, for the
 After resolution the `gzip` codec was explicitly flagged as unrecognized. But we can actually check that codec declaration against a real codec definition by expanding our context to include the `GZIP_CODEC` definition:
 
 ```python test="true" session="context"
-new_context = context.update(codecs={**context.codecs, "gzip": GZIP_CODEC})
+new_context = context.union(ZarrV3Context(codecs=[GZIP_CODEC]))
 ```
 
-`update` returns a new context with the `codecs` slot replaced. We pass the codecs the old context already has, plus `gzip`, so the new context claims both `bytes` and `gzip`. The old context is unchanged.
+`union` returns a new context that files every definition of both, so the new context claims both `bytes` and `gzip`. The old context is unchanged. To replace a slot instead of adding to it, use `copy(update=...)`.
 
 Now read the same document in the new context. `read_array_metadata_v3` returns everything the read found, instead of raising, so we can look at each codec:
 

packages/zarr-metadata/docs/parsing-validation.md

@@ -210,21 +210,21 @@ unclaimed = None if reading.metadata is None else reading.metadata.unclaimed_fie
 can_open = unclaimed is not None and len(unclaimed) == 0
 ```
 
-To make an unknown data type or codec known, add its definition to the scope's slot and read the document again:
+To make an unknown data type or codec known, add its definition to the scope with `union` and read the document again:
 
 ```python test="true" session="parsing-validation"
 from zarr_metadata.definition import EmptyConfiguration
-from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition
+from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition, ZarrV3Context
 
 # The definition of the codec that was unknown: here, a codec named
 # `acme.codec` that takes no configuration.
 MY_CODEC = CodecDefinition(
     name="acme.codec", configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic"
 )
-context = CORE_AND_EXTENSIONS.update(codecs=[*CORE_AND_EXTENSIONS.codecs.values(), MY_CODEC])
+context = CORE_AND_EXTENSIONS.union(ZarrV3Context(codecs=[MY_CODEC]))
 reading = read_array_metadata_v3(raw, context=context)
 ```
 
-`update` replaces each slot it is given and keeps the others. A slot can also be given as a mapping from each definition's name to it, so `CORE_AND_EXTENSIONS.update(codecs={**CORE_AND_EXTENSIONS.codecs, "gzip": MY_GZIP})` replaces the gzip definition with your own.
+`union` keeps every definition the scope has, and raises `ScopeConflictError` if `MY_CODEC` has a name the scope already files with another definition. To replace a definition on purpose, use `copy`, which, like pydantic's `model_copy(update=...)`, replaces whole each slot it is given and keeps the others. A slot can be given as a mapping from each definition's name to it, so `CORE_AND_EXTENSIONS.copy(update={"codecs": {**CORE_AND_EXTENSIONS.codecs, "gzip": MY_GZIP}})` replaces the gzip definition with your own. Passing `copy(update={"codecs": [MY_CODEC]})` instead would leave `MY_CODEC` as the only codec.
 
 [^1]: The one exception is the `r*` data type, which is identified by a templated string with no `configuration` object. This mixes the identifier with the configuration in the same JSON value, which is not good design. So this library normalizes `r*` into the same form as all the other metadata objects: the field `"r16"` keeps its name, and is read by the `r*` definition with the configuration `{"bits": 16}`.

packages/zarr-metadata/src/zarr_metadata/definition/__init__.py

@@ -124,7 +124,7 @@
 
     from zarr_metadata import ValidationProblem
     from zarr_metadata.definition import Nested
-    from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition
+    from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition, ZarrV3Context
 
 
     class AcmeLz4Configuration(TypedDict, closed=True):
@@ -149,7 +149,7 @@ def acme_lz4_rules(
         size="dynamic",
         rules=acme_lz4_rules,
     )
-    CONTEXT = CORE_AND_EXTENSIONS.update(codecs=[*CORE_AND_EXTENSIONS.codecs.values(), ACME_LZ4])
+    CONTEXT = CORE_AND_EXTENSIONS.union(ZarrV3Context(codecs=[ACME_LZ4]))
 
 The TypedDict declares how keys it does not declare are handled. With
 `closed=True`, such a key is a problem, as above. With `extra_items=`,
@@ -231,11 +231,12 @@ def acme_lz4_rules(
 format and file the same definitions, and equal scopes have equal
 hashes. `context.union(*others)` returns the smallest scope that
 contains each of them, or raises a `ScopeConflictError` naming each name
-filed with two different definitions. `update` replaces the
-definitions in the slots it is given and keeps the others, and takes a
-slot as definitions or as a mapping by name, so
-`context.update(codecs={**context.codecs, "gzip": MY_GZIP})` replaces
-one definition on purpose. A model's `refined_in` moves it to a scope
+filed with two different definitions. It is how to add definitions to a
+scope, keeping the ones it has. `context.copy(update={...})`, like
+pydantic's `model_copy`, replaces whole each slot it is given and keeps
+the others. It takes a slot as definitions or as a mapping by name, so
+`context.copy(update={"codecs": {**context.codecs, "gzip": MY_GZIP}})`
+replaces one definition on purpose. A model's `refined_in` moves it to a scope
 that claims more names and contradicts nothing. `refines` orders two
 models by how much they know. A name that was unclaimed and is now read
 by a definition is a gain. The reverse is a loss. One name read by two

packages/zarr-metadata/src/zarr_metadata/v2/_context.py

@@ -51,7 +51,7 @@ def is_v2_kind(kind: type[Definition[Any]]) -> TypeIs[type[ZarrV2Kind]]:
 
 
 class ZarrV2ContextSlots(TypedDict, total=False, closed=True):
-    """The definitions in each slot of a v2 scope, as `ZarrV2Context` and its `update` take them.
+    """The definitions in each slot of a v2 scope, as `ZarrV2Context` and its `copy` take them.
 
     This is where a v2 scope's slots are declared: each key is a slot,
     and its value the definitions of the kind that slot holds, as an
@@ -77,13 +77,13 @@ class ZarrV2Context:
     document holds, as `ZarrV2ContextSlots` declares them: `data_types`,
     read from `dtype`, and `codecs`, read from `compressor` and `filters`.
     Each slot is a read-only mapping from name to definition. The
-    constructor and `update` take, for each slot, an iterable of
+    constructor and `copy` take, for each slot, an iterable of
     definitions or such a mapping, so one scope's slot can be given to
     another. A definition of another kind than its slot holds, and a key
     that is not its definition's name, raise `TypeError`, and two
     different definitions under one name in a slot raise
-    `ScopeConflictError`. To replace a definition, or a whole slot, use
-    `update`.
+    `ScopeConflictError`. To add definitions, use `union`. To replace a
+    definition, or a whole slot, use `copy`.
     `ZarrV2Context()` is the scope that claims nothing. Two scopes are
     equal when they file the same definitions in each slot, and equal
     scopes have equal hashes.
@@ -139,26 +139,28 @@ def claimant(self, kind: type[D], name: str) -> D | None:
         """
         return claimant_in(self._tables, kind, name)
 
-    def update(self, **slots: Unpack[ZarrV2ContextSlots]) -> ZarrV2Context:
-        """This scope, with each slot given here holding only the definitions given for it. The other slots are kept.
+    def copy(self, *, update: ZarrV2ContextSlots | None = None) -> ZarrV2Context:
+        """This scope, with each slot in `update` holding only the definitions given for it, as pydantic's `model_copy(update=...)` replaces each field it is given. The other slots are kept.
 
-        A slot can be given as a mapping from each definition's name to
-        it, so a slot of this scope passes back in changed:
-        `context.update(codecs={**context.codecs, "gzip": MY_GZIP})`
-        replaces the gzip definition and keeps the other codecs, and
-        `context.update(codecs=[*context.codecs.values(), MY_CODEC])` adds
-        one, raising `ScopeConflictError` if the scope already files
-        another definition under its name.
+        A slot given here is replaced whole: `context.copy(update={"codecs": [MY_CODEC]})`
+        is a scope whose only codec is `MY_CODEC`. To add definitions to a
+        scope, use `union`: `context.union(ZarrV2Context(codecs=[MY_CODEC]))`.
+        A slot can be given as a mapping from each definition's name to it,
+        so a slot of this scope passes back in changed:
+        `context.copy(update={"codecs": {**context.codecs, "gzip": MY_GZIP}})`
+        replaces the gzip definition and keeps the other codecs.
         """
-        return self._of(updated(self._tables, self.kinds, slots, 2))
+        return self._of(updated(self._tables, self.kinds, {} if update is None else update, 2))
 
     def union(self, *others: ZarrV2Context) -> ZarrV2Context:
         """The smallest scope that files everything this scope and each of `others` file.
 
-        Raises `ScopeConflictError` when two of them file different
-        definitions under one name in a slot. To replace a definition on
-        purpose, use `update`. Raises `TypeError` for a scope of another
-        format.
+        This is how to add definitions to a scope:
+        `CORE_V2.union(ZarrV2Context(codecs=[MY_CODEC]))` keeps every
+        definition of `CORE_V2` and adds `MY_CODEC`. Raises
+        `ScopeConflictError` when two of them file different definitions
+        under one name in a slot. To replace a definition on purpose, use
+        `copy`. Raises `TypeError` for a scope of another format.
         """
         refuse_others(others, ZarrV2Context)
         return self._of(

packages/zarr-metadata/src/zarr_metadata/v3/_context.py

@@ -67,7 +67,7 @@ def is_v3_kind(kind: type[Definition[Any]]) -> TypeIs[type[ZarrV3Kind]]:
 
 
 class ZarrV3ContextSlots(TypedDict, total=False, closed=True):
-    """The definitions in each slot of a v3 scope, as `ZarrV3Context` and its `update` take them.
+    """The definitions in each slot of a v3 scope, as `ZarrV3Context` and its `copy` take them.
 
     This is where a v3 scope's slots are declared: each key is a slot,
     and its value the definitions of the kind that slot holds, as an
@@ -114,13 +114,13 @@ class ZarrV3Context:
     `group_extra_fields` hold the definitions of members the spec does not
     define, by key. Each slot is a read-only mapping from the name each
     definition is filed under (`r*` for raw bits) to the definition. The
-    constructor and `update` take, for each slot, an iterable of
+    constructor and `copy` take, for each slot, an iterable of
     definitions or such a mapping, so one scope's slot can be given to
     another. A definition of another kind than its slot holds, and a key
     that is not its definition's name, raise `TypeError`, and two
     different definitions under one name in a slot raise
-    `ScopeConflictError`. To replace a definition, or a whole slot, use
-    `update`.
+    `ScopeConflictError`. To add definitions, use `union`. To replace a
+    definition, or a whole slot, use `copy`.
     `ZarrV3Context()` is the scope that claims nothing. Two scopes are
     equal when they file the same definitions in each slot, and equal
     scopes have equal hashes.
@@ -201,26 +201,28 @@ def claimant(self, kind: type[D], name: str) -> D | None:
         """
         return claimant_in(self._tables, kind, name)
 
-    def update(self, **slots: Unpack[ZarrV3ContextSlots]) -> ZarrV3Context:
-        """This scope, with each slot given here holding only the definitions given for it. The other slots are kept.
+    def copy(self, *, update: ZarrV3ContextSlots | None = None) -> ZarrV3Context:
+        """This scope, with each slot in `update` holding only the definitions given for it, as pydantic's `model_copy(update=...)` replaces each field it is given. The other slots are kept.
 
-        A slot can be given as a mapping from each definition's name to
-        it, so a slot of this scope passes back in changed:
-        `context.update(codecs={**context.codecs, "gzip": MY_GZIP})`
-        replaces the gzip definition and keeps the other codecs, and
-        `context.update(codecs=[*context.codecs.values(), MY_CODEC])` adds
-        one, raising `ScopeConflictError` if the scope already files
-        another definition under its name.
+        A slot given here is replaced whole: `context.copy(update={"codecs": [MY_CODEC]})`
+        is a scope whose only codec is `MY_CODEC`. To add definitions to a
+        scope, use `union`: `context.union(ZarrV3Context(codecs=[MY_CODEC]))`.
+        A slot can be given as a mapping from each definition's name to it,
+        so a slot of this scope passes back in changed:
+        `context.copy(update={"codecs": {**context.codecs, "gzip": MY_GZIP}})`
+        replaces the gzip definition and keeps the other codecs.
         """
-        return self._of(updated(self._tables, self.kinds, slots, 3))
+        return self._of(updated(self._tables, self.kinds, {} if update is None else update, 3))
 
     def union(self, *others: ZarrV3Context) -> ZarrV3Context:
         """The smallest scope that files everything this scope and each of `others` file.
 
-        Raises `ScopeConflictError` when two of them file different
-        definitions under one name in a slot. To replace a definition on
-        purpose, use `update`. Raises `TypeError` for a scope of another
-        format.
+        This is how to add definitions to a scope:
+        `CORE_AND_EXTENSIONS.union(ZarrV3Context(codecs=[MY_CODEC]))` keeps every
+        definition of `CORE_AND_EXTENSIONS` and adds `MY_CODEC`. Raises
+        `ScopeConflictError` when two of them file different definitions
+        under one name in a slot. To replace a definition on purpose, use
+        `copy`. Raises `TypeError` for a scope of another format.
         """
         refuse_others(others, ZarrV3Context)
         return self._of(

packages/zarr-metadata/src/zarr_metadata/v3/_core.py

@@ -80,17 +80,18 @@
 )
 """A scope with only what the Zarr v3 specification defines, including the envelope of a group's `consolidated_metadata`, which the spec fixes."""
 
-CORE_AND_EXTENSIONS: Final = CORE.update(
-    data_types=[
-        *CORE.data_types.values(),
-        BYTES_DATA_TYPE,
-        STRING_DATA_TYPE,
-        NUMPY_DATETIME64_DATA_TYPE,
-        NUMPY_TIMEDELTA64_DATA_TYPE,
-        STRUCT_DATA_TYPE,
-    ],
-    chunk_grids=[*CORE.chunk_grids.values(), RECTILINEAR_CHUNK_GRID],
-    codecs=[*CORE.codecs.values(), CAST_VALUE_CODEC, SCALE_OFFSET_CODEC, ZSTD_CODEC],
+CORE_AND_EXTENSIONS: Final = CORE.union(
+    ZarrV3Context(
+        data_types=[
+            BYTES_DATA_TYPE,
+            STRING_DATA_TYPE,
+            NUMPY_DATETIME64_DATA_TYPE,
+            NUMPY_TIMEDELTA64_DATA_TYPE,
+            STRUCT_DATA_TYPE,
+        ],
+        chunk_grids=[RECTILINEAR_CHUNK_GRID],
+        codecs=[CAST_VALUE_CODEC, SCALE_OFFSET_CODEC, ZSTD_CODEC],
+    )
 )
 """A scope with what the specification defines, plus the extensions that `zarr-extensions` registers and this package defines."""
 

packages/zarr-metadata/tests/model/test_consolidating.py

@@ -41,7 +41,7 @@
 MY_BYTES = CodecDefinition(
     name="bytes", configuration=EmptyConfiguration, kind="array_bytes", size="static"
 )
-PRIVATE = CORE.update(codecs={**CORE.codecs, "bytes": MY_BYTES})
+PRIVATE = CORE.copy(update={"codecs": {**CORE.codecs, "bytes": MY_BYTES}})
 INLINE: dict[str, Any] = {"kind": "inline", "must_understand": False}
 
 
@@ -310,7 +310,7 @@ def test_error_a_data_type_conflict_names_the_kind_and_the_written_name() -> Non
     )
     child = ZarrV3ArrayMetadata(
         {**ARRAY, "data_type": "r16", "fill_value": [0, 0], "codecs": ("bytes",)},
-        context=CORE.update(data_types={**CORE.data_types, "r*": strict}),
+        context=CORE.copy(update={"data_types": {**CORE.data_types, "r*": strict}}),
     )
     with pytest.raises(MetadataValidationError) as raised:
         _consolidated(CORE, a=child)
@@ -328,7 +328,7 @@ def test_error_a_conflict_between_definitions_alike_says_they_differ() -> None:
     strict = dataclasses.replace(GZIP_CODEC, rules=lambda configuration, nested: iter(()))
     child = ZarrV3ArrayMetadata(
         {**ARRAY, "codecs": ("bytes", {"name": "gzip", "configuration": {"level": 1}})},
-        context=CORE.update(codecs={**CORE.codecs, "gzip": strict}),
+        context=CORE.copy(update={"codecs": {**CORE.codecs, "gzip": strict}}),
     )
     with pytest.raises(MetadataValidationError) as raised:
         _consolidated(CORE, a=child)

packages/zarr-metadata/tests/model/test_construction.py

@@ -195,8 +195,8 @@ def counted(
         yield from ()
 
     counting = dataclasses.replace(INT8_DATA_TYPE, fill_value_rules=counted)
-    context = CORE_AND_EXTENSIONS.update(
-        data_types={**CORE_AND_EXTENSIONS.data_types, "int8": counting}
+    context = CORE_AND_EXTENSIONS.copy(
+        update={"data_types": {**CORE_AND_EXTENSIONS.data_types, "int8": counting}}
     )
     model = ZarrV3ArrayMetadata.create_default(context=context, data_type="int8", fill_value=3)
     assert values == [3]
@@ -456,10 +456,12 @@ def counting(configuration: GzipCodecConfiguration) -> GzipCodecConfiguration:
         asked.append(1)
         return configuration
 
-    context = CORE_AND_EXTENSIONS.update(
-        codecs={
-            **CORE_AND_EXTENSIONS.codecs,
-            "gzip": dataclasses.replace(GZIP_CODEC, canonical=counting),
+    context = CORE_AND_EXTENSIONS.copy(
+        update={
+            "codecs": {
+                **CORE_AND_EXTENSIONS.codecs,
+                "gzip": dataclasses.replace(GZIP_CODEC, canonical=counting),
+            }
         }
     )
     document = {
@@ -489,10 +491,12 @@ def test_error_a_canonical_that_raises_is_raised_when_the_model_is_first_compare
     def refuses(configuration: GzipCodecConfiguration) -> GzipCodecConfiguration:
         raise ValueError("no")
 
-    context = CORE_AND_EXTENSIONS.update(
-        codecs={
-            **CORE_AND_EXTENSIONS.codecs,
-            "gzip": dataclasses.replace(GZIP_CODEC, canonical=refuses),
+    context = CORE_AND_EXTENSIONS.copy(
+        update={
+            "codecs": {
+                **CORE_AND_EXTENSIONS.codecs,
+                "gzip": dataclasses.replace(GZIP_CODEC, canonical=refuses),
+            }
         }
     )
     document = {

packages/zarr-metadata/tests/model/test_extension_points.py

@@ -75,7 +75,7 @@ class LenientGzipConfiguration(TypedDict, closed=True):
         (_document(codecs=[BYTES, {"name": "zstd", "configuration": {"level": 99}}]), CORE),
         (
             _document(codecs=[BYTES, {"name": "gzip", "configuration": {"level": 99}}]),
-            CORE.update(codecs={**CORE.codecs, "gzip": LENIENT_GZIP}),
+            CORE.copy(update={"codecs": {**CORE.codecs, "gzip": LENIENT_GZIP}}),
         ),
         (dict(ZarrV3ArrayMetadata.create_default(shape=(0, 4)).to_json()), CORE_AND_EXTENSIONS),
     ],
@@ -173,7 +173,7 @@ def test_error_an_array_in_consolidated_metadata() -> None:
 def test_an_array_in_consolidated_metadata_reads_in_the_scope_it_is_read_in() -> None:
     """An array listed in consolidated metadata is read in the scope the member is read in: a lenient `gzip` accepts the level the core `gzip` refuses, and the model round-trips in that scope."""
     member = {"kind": "inline", "must_understand": False, "metadata": {"a": BAD_GZIP_ARRAY}}
-    lenient = CORE.update(codecs={**CORE.codecs, "gzip": LENIENT_GZIP})
+    lenient = CORE.copy(update={"codecs": {**CORE.codecs, "gzip": LENIENT_GZIP}})
     reading = read_consolidated_metadata_v3(member, context=lenient)
     assert reading.problems == ()
     assert reading.metadata is not None