diff --git a/changes/4412.bugfix.md b/changes/4412.bugfix.md new file mode 100644 index 0000000000..3fefca87cf --- /dev/null +++ b/changes/4412.bugfix.md @@ -0,0 +1 @@ +`Group.empty`, `zeros`, `ones`, `full` and their `*_like` versions now create arrays in the zarr format of the group. Previously a zarr v2 group got zarr v3 arrays, which were written into the group without becoming members of it. Creating an array like an array of the other zarr format (for example `zarr.zeros_like(v2_array)`) no longer fails. The new array uses the default codecs of its own format, and top-level `*_like` functions inherit the zarr format of the source array unless `zarr_format` is passed. diff --git a/changes/4457.removal.md b/changes/4457.removal.md new file mode 100644 index 0000000000..9bda60640c --- /dev/null +++ b/changes/4457.removal.md @@ -0,0 +1 @@ +`zarr.empty`, `zeros`, `ones` and `full`, their `zarr.api.asynchronous` versions, and `Group`/`AsyncGroup` `empty`, `zeros`, `ones` and `full` now list their parameters instead of taking `**kwargs`. The parameters are those of `zarr.create`. The group methods do not take `store`, `path` or `storage_options`, which the group supplies. An unknown keyword now raises a `TypeError` naming the function called. Passing `data` to these functions is deprecated; it fills the new array with `data` instead of the value the function is named for. To create an array from existing data, use `zarr.create_array(data=...)`. Passing `mode` to the group methods is deprecated, because it never had an effect there. diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index bef4d5c78f..49a61e25e5 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -121,6 +121,17 @@ def _warn_unimplemented_kwargs(kwargs: dict[str, Any]) -> None: warnings.warn(f"{name} is not yet implemented", ZarrRuntimeWarning, stacklevel=3) +def _warn_helper_data_kwarg(func_name: str) -> None: + """ + Emit the deprecation warning for `data` passed to `empty`, `zeros`, `ones` or `full`. + """ + msg = ( + f"Passing `data` to `{func_name}` is deprecated and will be removed in a future version. " + "To create an array from existing data, use `create_array(data=...)`." + ) + warnings.warn(msg, ZarrDeprecationWarning, stacklevel=3) + + def _get_shape_chunks(a: ArrayLike | Any) -> tuple[tuple[int, ...] | None, tuple[int, ...] | None]: """Helper function to get the shape and chunks from an array-like object""" shape = None @@ -148,11 +159,18 @@ class _LikeArgs(TypedDict): compressor: NotRequired[CompressorLikev2] codecs: NotRequired[tuple[Codec, ...]] fill_value: NotRequired[Any] + zarr_format: NotRequired[ZarrFormat] -def _like_args(a: ArrayLike) -> _LikeArgs: - """Set default values for shape and chunks if they are not present in the array-like object""" +def _like_args(a: ArrayLike, zarr_format: ZarrFormat | None) -> _LikeArgs: + """ + Arguments for creating an array like `a` in `zarr_format`. + If `a` is a zarr array, the new array has the zarr format of `a` unless `zarr_format` + requests another. The storage settings of `a` (memory order and codecs) are specific to + its zarr format, so they are copied only when the new array has the format of `a`. + Otherwise the new array uses the defaults of its own format. + """ new: _LikeArgs = {} shape, chunks = _get_shape_chunks(a) @@ -166,14 +184,17 @@ def _like_args(a: ArrayLike) -> _LikeArgs: if isinstance(a, AsyncArray | Array): new["fill_value"] = a.metadata.fill_value - if isinstance(a.metadata, ArrayV2Metadata): - new["order"] = a.order - new["compressor"] = a.metadata.compressor - new["filters"] = a.metadata.filters - else: - # TODO: Remove type: ignore statement when type inference improves. - # mypy cannot correctly infer the type of a.metadata here for some reason. - new["codecs"] = a.metadata.codecs + if zarr_format is None: + zarr_format = a.metadata.zarr_format + if a.metadata.zarr_format == zarr_format: + if isinstance(a.metadata, ArrayV2Metadata): + new["order"] = a.order + new["compressor"] = a.metadata.compressor + new["filters"] = a.metadata.filters + else: + # TODO: Remove type: ignore statement when type inference improves. + # mypy cannot correctly infer the type of a.metadata here for some reason. + new["codecs"] = a.metadata.codecs else: # TODO: set default values compressor/codecs @@ -181,6 +202,9 @@ def _like_args(a: ArrayLike) -> _LikeArgs: # new["compressor"] = "default" pass + if zarr_format is not None: + new["zarr_format"] = zarr_format + return new @@ -1126,16 +1150,61 @@ async def create( ) -async def empty(shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: +async def empty( + shape: tuple[int, ...] | int, + *, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + fill_value: Any | None = DEFAULT_FILL_VALUE, + order: MemoryOrder | None = None, + store: StoreLike | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + path: PathLike | None = None, + chunk_store: StoreLike | None = None, + filters: Iterable[dict[str, JSON] | Numcodec] | None = None, + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunk_shape: tuple[int, ...] | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNamesLike = None, + storage_options: dict[str, Any] | None = None, + config: ArrayConfigLike | None = None, + mode: AccessModeLiteral | None = None, + data: npt.ArrayLike | None = None, +) -> AnyAsyncArray: """Create an empty array with the specified shape. The contents will be filled with the specified fill value or zeros if no fill value is provided. + Parameters not listed below are those of + [`create`][zarr.api.asynchronous.create], with the same meaning. + Parameters ---------- shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [`create`][zarr.api.asynchronous.create]. + Shape of the array. + data : array-like, optional + Deprecated. Values written into the new array after it is created. To create an + array from existing data, use `create_array(data=...)`. + + Returns + ------- + AsyncArray + The new array. Notes ----- @@ -1143,10 +1212,44 @@ async def empty(shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: retrieve data from an empty Zarr array, any values may be returned, and these are not guaranteed to be stable from one access to the next. """ - return await create(shape=shape, **kwargs) + if data is not None: + _warn_helper_data_kwarg("empty") + return await create( + shape=shape, + chunks=chunks, + dtype=dtype, + compressor=compressor, + fill_value=fill_value, + order=order, + store=store, + synchronizer=synchronizer, + overwrite=overwrite, + path=path, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + storage_options=storage_options, + config=config, + mode=mode, + data=data, + ) -async def empty_like(a: ArrayLike, **kwargs: Any) -> AnyAsyncArray: +async def empty_like( + a: ArrayLike, *, zarr_format: ZarrFormat | None = None, **kwargs: Any +) -> AnyAsyncArray: """Create an empty array like `a`. The contents will be filled with the array's fill value or zeros if no fill value is provided. @@ -1154,6 +1257,9 @@ async def empty_like(a: ArrayLike, **kwargs: Any) -> AnyAsyncArray: ---------- a : array-like The array to create an empty array like. + zarr_format : {2, 3, None}, optional + The zarr format of the new array. If `None` (default), the zarr format of `a` if it + is a zarr array, otherwise the default zarr format. **kwargs Keyword arguments passed to [`create`][zarr.api.asynchronous.create]. @@ -1168,40 +1274,116 @@ async def empty_like(a: ArrayLike, **kwargs: Any) -> AnyAsyncArray: retrieve data from an empty Zarr array, any values may be returned, and these are not guaranteed to be stable from one access to the next. """ - like_kwargs = _like_args(a) | kwargs + like_kwargs = _like_args(a, zarr_format) | kwargs return await empty(**like_kwargs) # type: ignore[arg-type] -# TODO: add type annotations for fill_value and kwargs -async def full(shape: tuple[int, ...], fill_value: Any, **kwargs: Any) -> AnyAsyncArray: +async def full( + shape: tuple[int, ...] | int, + fill_value: Any, + *, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + order: MemoryOrder | None = None, + store: StoreLike | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + path: PathLike | None = None, + chunk_store: StoreLike | None = None, + filters: Iterable[dict[str, JSON] | Numcodec] | None = None, + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunk_shape: tuple[int, ...] | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNamesLike = None, + storage_options: dict[str, Any] | None = None, + config: ArrayConfigLike | None = None, + mode: AccessModeLiteral | None = None, + data: npt.ArrayLike | None = None, +) -> AnyAsyncArray: """Create an array, with `fill_value` being used as the default value for uninitialized portions of the array. + Parameters not listed below are those of + [`create`][zarr.api.asynchronous.create], with the same meaning. + Parameters ---------- shape : int or tuple of int - Shape of the empty array. + Shape of the array. fill_value : scalar Fill value. - **kwargs - Keyword arguments passed to [`create`][zarr.api.asynchronous.create]. + data : array-like, optional + Deprecated. Values written into the new array after it is created. To create an + array from existing data, use `create_array(data=...)`. Returns ------- - Array + AsyncArray The new array. """ - return await create(shape=shape, fill_value=fill_value, **kwargs) + if data is not None: + _warn_helper_data_kwarg("full") + return await create( + shape=shape, + fill_value=fill_value, + chunks=chunks, + dtype=dtype, + compressor=compressor, + order=order, + store=store, + synchronizer=synchronizer, + overwrite=overwrite, + path=path, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + storage_options=storage_options, + config=config, + mode=mode, + data=data, + ) # TODO: add type annotations for kwargs -async def full_like(a: ArrayLike, **kwargs: Any) -> AnyAsyncArray: +async def full_like( + a: ArrayLike, *, zarr_format: ZarrFormat | None = None, **kwargs: Any +) -> AnyAsyncArray: """Create a filled array like `a`. Parameters ---------- a : array-like The array to create an empty array like. + zarr_format : {2, 3, None}, optional + The zarr format of the new array. If `None` (default), the zarr format of `a` if it + is a zarr array, otherwise the default zarr format. **kwargs Keyword arguments passed to [`zarr.api.asynchronous.create`][]. @@ -1210,36 +1392,112 @@ async def full_like(a: ArrayLike, **kwargs: Any) -> AnyAsyncArray: Array The new array. """ - like_kwargs = _like_args(a) | kwargs + like_kwargs = _like_args(a, zarr_format) | kwargs return await full(**like_kwargs) # type: ignore[arg-type] -async def ones(shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: +async def ones( + shape: tuple[int, ...] | int, + *, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + order: MemoryOrder | None = None, + store: StoreLike | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + path: PathLike | None = None, + chunk_store: StoreLike | None = None, + filters: Iterable[dict[str, JSON] | Numcodec] | None = None, + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunk_shape: tuple[int, ...] | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNamesLike = None, + storage_options: dict[str, Any] | None = None, + config: ArrayConfigLike | None = None, + mode: AccessModeLiteral | None = None, + data: npt.ArrayLike | None = None, +) -> AnyAsyncArray: """Create an array, with one being used as the default value for uninitialized portions of the array. + Parameters not listed below are those of + [`create`][zarr.api.asynchronous.create], with the same meaning. + Parameters ---------- shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [`zarr.api.asynchronous.create`][]. + Shape of the array. + data : array-like, optional + Deprecated. Values written into the new array after it is created. To create an + array from existing data, use `create_array(data=...)`. Returns ------- - Array + AsyncArray The new array. """ - return await create(shape=shape, fill_value=1, **kwargs) + if data is not None: + _warn_helper_data_kwarg("ones") + return await create( + shape=shape, + fill_value=1, + chunks=chunks, + dtype=dtype, + compressor=compressor, + order=order, + store=store, + synchronizer=synchronizer, + overwrite=overwrite, + path=path, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + storage_options=storage_options, + config=config, + mode=mode, + data=data, + ) -async def ones_like(a: ArrayLike, **kwargs: Any) -> AnyAsyncArray: +async def ones_like( + a: ArrayLike, *, zarr_format: ZarrFormat | None = None, **kwargs: Any +) -> AnyAsyncArray: """Create an array of ones like `a`. Parameters ---------- a : array-like The array to create an empty array like. + zarr_format : {2, 3, None}, optional + The zarr format of the new array. If `None` (default), the zarr format of `a` if it + is a zarr array, otherwise the default zarr format. **kwargs Keyword arguments passed to [`zarr.api.asynchronous.create`][]. @@ -1248,7 +1506,7 @@ async def ones_like(a: ArrayLike, **kwargs: Any) -> AnyAsyncArray: Array The new array. """ - like_args = _like_args(a) + like_args = _like_args(a, zarr_format) # `ones` supplies its own fill_value, so drop any inherited from `a`. like_args.pop("fill_value", None) like_kwargs = like_args | kwargs @@ -1309,7 +1567,9 @@ async def open_array( raise ArrayNotFoundError(msg) from err -async def open_like(a: ArrayLike, path: str, **kwargs: Any) -> AnyAsyncArray: +async def open_like( + a: ArrayLike, path: str, *, zarr_format: ZarrFormat | None = None, **kwargs: Any +) -> AnyAsyncArray: """Open a persistent array like `a`. Parameters @@ -1318,6 +1578,10 @@ async def open_like(a: ArrayLike, path: str, **kwargs: Any) -> AnyAsyncArray: The shape and data-type of a define these same attributes of the returned array. path : str The path to the new array. + zarr_format : {2, 3, None}, optional + The zarr format of the array to open or create. If `None` (default), an existing + array of either format is opened, and a missing one is created in the default zarr + format. The zarr format of `a` is not inherited. **kwargs Additional keyword arguments passed to `open_array`. If `mode` is omitted or `None`, it defaults to `"a"`. Pass `mode="r"` when @@ -1328,38 +1592,118 @@ async def open_like(a: ArrayLike, path: str, **kwargs: Any) -> AnyAsyncArray: AsyncArray The opened array. """ - like_kwargs = _like_args(a) | kwargs + # The zarr format of `a` is not inherited: `open_array` would then look for an existing + # array in only that format. + like_args = _like_args(a, zarr_format or _default_zarr_format()) + like_args.pop("zarr_format") + like_kwargs = like_args | kwargs if like_kwargs.get("mode") is None: like_kwargs["mode"] = "a" - return await open_array(path=path, **like_kwargs) # type: ignore[arg-type] + return await open_array(path=path, zarr_format=zarr_format, **like_kwargs) # type: ignore[arg-type] -async def zeros(shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: +async def zeros( + shape: tuple[int, ...] | int, + *, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + order: MemoryOrder | None = None, + store: StoreLike | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + path: PathLike | None = None, + chunk_store: StoreLike | None = None, + filters: Iterable[dict[str, JSON] | Numcodec] | None = None, + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunk_shape: tuple[int, ...] | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNamesLike = None, + storage_options: dict[str, Any] | None = None, + config: ArrayConfigLike | None = None, + mode: AccessModeLiteral | None = None, + data: npt.ArrayLike | None = None, +) -> AnyAsyncArray: """Create an array, with zero being used as the default value for uninitialized portions of the array. + Parameters not listed below are those of + [`create`][zarr.api.asynchronous.create], with the same meaning. + Parameters ---------- shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [`zarr.api.asynchronous.create`][]. + Shape of the array. + data : array-like, optional + Deprecated. Values written into the new array after it is created. To create an + array from existing data, use `create_array(data=...)`. Returns ------- - Array + AsyncArray The new array. """ - return await create(shape=shape, fill_value=0, **kwargs) + if data is not None: + _warn_helper_data_kwarg("zeros") + return await create( + shape=shape, + fill_value=0, + chunks=chunks, + dtype=dtype, + compressor=compressor, + order=order, + store=store, + synchronizer=synchronizer, + overwrite=overwrite, + path=path, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + storage_options=storage_options, + config=config, + mode=mode, + data=data, + ) -async def zeros_like(a: ArrayLike, **kwargs: Any) -> AnyAsyncArray: +async def zeros_like( + a: ArrayLike, *, zarr_format: ZarrFormat | None = None, **kwargs: Any +) -> AnyAsyncArray: """Create an array of zeros like `a`. Parameters ---------- a : array-like The array to create an empty array like. + zarr_format : {2, 3, None}, optional + The zarr format of the new array. If `None` (default), the zarr format of `a` if it + is a zarr array, otherwise the default zarr format. **kwargs Keyword arguments passed to [`create`][zarr.api.asynchronous.create]. @@ -1368,7 +1712,7 @@ async def zeros_like(a: ArrayLike, **kwargs: Any) -> AnyAsyncArray: Array The new array. """ - like_args = _like_args(a) + like_args = _like_args(a, zarr_format) # `zeros` supplies its own fill_value, so drop any inherited from `a`. like_args.pop("fill_value", None) like_kwargs = like_args | kwargs diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 6b387dafa5..727d9ef90d 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -1250,17 +1250,56 @@ def from_array( ) -# TODO: add type annotations for kwargs -def empty(shape: tuple[int, ...], **kwargs: Any) -> AnyArray: +def empty( + shape: tuple[int, ...] | int, + *, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + fill_value: Any | None = DEFAULT_FILL_VALUE, + order: MemoryOrder | None = None, + store: StoreLike | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + path: PathLike | None = None, + chunk_store: StoreLike | None = None, + filters: Iterable[dict[str, JSON] | Numcodec] | None = None, + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunk_shape: tuple[int, ...] | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNamesLike = None, + storage_options: dict[str, Any] | None = None, + config: ArrayConfigLike | None = None, + mode: AccessModeLiteral | None = None, + data: npt.ArrayLike | None = None, +) -> AnyArray: """Create an empty array with the specified shape. The contents will be filled with the - array's fill value or zeros if no fill value is provided. + specified fill value or zeros if no fill value is provided. + + Parameters not listed below are those of + [`create`][zarr.api.asynchronous.create], with the same meaning. Parameters ---------- shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [`create`][zarr.api.asynchronous.create]. + Shape of the array. + data : array-like, optional + Deprecated. Values written into the new array after it is created. To create an + array from existing data, use `create_array(data=...)`. Returns ------- @@ -1273,12 +1312,46 @@ def empty(shape: tuple[int, ...], **kwargs: Any) -> AnyArray: retrieve data from an empty Zarr array, any values may be returned, and these are not guaranteed to be stable from one access to the next. """ - return Array(sync(async_api.empty(shape, **kwargs))) + return Array( + sync( + async_api.empty( + shape=shape, + chunks=chunks, + dtype=dtype, + compressor=compressor, + fill_value=fill_value, + order=order, + store=store, + synchronizer=synchronizer, + overwrite=overwrite, + path=path, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + storage_options=storage_options, + config=config, + mode=mode, + data=data, + ) + ) + ) # TODO: move ArrayLike to common module # TODO: add type annotations for kwargs -def empty_like(a: ArrayLike, **kwargs: Any) -> AnyArray: +def empty_like(a: ArrayLike, *, zarr_format: ZarrFormat | None = None, **kwargs: Any) -> AnyArray: """Create an empty array like another array. The contents will be filled with the array's fill value or zeros if no fill value is provided. @@ -1286,6 +1359,9 @@ def empty_like(a: ArrayLike, **kwargs: Any) -> AnyArray: ---------- a : array-like The array to create an empty array like. + zarr_format : {2, 3, None}, optional + The zarr format of the new array. If `None` (default), the zarr format of `a` if it + is a zarr array, otherwise the default zarr format. **kwargs Keyword arguments passed to [`create`][zarr.api.asynchronous.create]. @@ -1300,39 +1376,116 @@ def empty_like(a: ArrayLike, **kwargs: Any) -> AnyArray: retrieve data from an empty Zarr array, any values may be returned, and these are not guaranteed to be stable from one access to the next. """ - return Array(sync(async_api.empty_like(a, **kwargs))) + return Array(sync(async_api.empty_like(a, zarr_format=zarr_format, **kwargs))) + +def full( + shape: tuple[int, ...] | int, + fill_value: Any, + *, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + order: MemoryOrder | None = None, + store: StoreLike | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + path: PathLike | None = None, + chunk_store: StoreLike | None = None, + filters: Iterable[dict[str, JSON] | Numcodec] | None = None, + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunk_shape: tuple[int, ...] | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNamesLike = None, + storage_options: dict[str, Any] | None = None, + config: ArrayConfigLike | None = None, + mode: AccessModeLiteral | None = None, + data: npt.ArrayLike | None = None, +) -> AnyArray: + """Create an array, with `fill_value` being used as the default value for + uninitialized portions of the array. -# TODO: add type annotations for kwargs and fill_value -def full(shape: tuple[int, ...], fill_value: Any, **kwargs: Any) -> AnyArray: - """Create an array with a default fill value. + Parameters not listed below are those of + [`create`][zarr.api.asynchronous.create], with the same meaning. Parameters ---------- shape : int or tuple of int - Shape of the empty array. + Shape of the array. fill_value : scalar Fill value. - **kwargs - Keyword arguments passed to [`create`][zarr.api.asynchronous.create]. + data : array-like, optional + Deprecated. Values written into the new array after it is created. To create an + array from existing data, use `create_array(data=...)`. Returns ------- Array The new array. """ - return Array(sync(async_api.full(shape=shape, fill_value=fill_value, **kwargs))) + return Array( + sync( + async_api.full( + shape=shape, + fill_value=fill_value, + chunks=chunks, + dtype=dtype, + compressor=compressor, + order=order, + store=store, + synchronizer=synchronizer, + overwrite=overwrite, + path=path, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + storage_options=storage_options, + config=config, + mode=mode, + data=data, + ) + ) + ) # TODO: move ArrayLike to common module # TODO: add type annotations for kwargs -def full_like(a: ArrayLike, **kwargs: Any) -> AnyArray: +def full_like(a: ArrayLike, *, zarr_format: ZarrFormat | None = None, **kwargs: Any) -> AnyArray: """Create a filled array like another array. Parameters ---------- a : array-like The array to create an empty array like. + zarr_format : {2, 3, None}, optional + The zarr format of the new array. If `None` (default), the zarr format of `a` if it + is a zarr array, otherwise the default zarr format. **kwargs Keyword arguments passed to [`zarr.api.asynchronous.create`][]. @@ -1341,36 +1494,111 @@ def full_like(a: ArrayLike, **kwargs: Any) -> AnyArray: Array The new array. """ - return Array(sync(async_api.full_like(a, **kwargs))) + return Array(sync(async_api.full_like(a, zarr_format=zarr_format, **kwargs))) -# TODO: add type annotations for kwargs -def ones(shape: tuple[int, ...], **kwargs: Any) -> AnyArray: - """Create an array with a fill value of one. +def ones( + shape: tuple[int, ...] | int, + *, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + order: MemoryOrder | None = None, + store: StoreLike | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + path: PathLike | None = None, + chunk_store: StoreLike | None = None, + filters: Iterable[dict[str, JSON] | Numcodec] | None = None, + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunk_shape: tuple[int, ...] | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNamesLike = None, + storage_options: dict[str, Any] | None = None, + config: ArrayConfigLike | None = None, + mode: AccessModeLiteral | None = None, + data: npt.ArrayLike | None = None, +) -> AnyArray: + """Create an array, with one being used as the default value for + uninitialized portions of the array. + + Parameters not listed below are those of + [`create`][zarr.api.asynchronous.create], with the same meaning. Parameters ---------- shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [`zarr.api.asynchronous.create`][]. + Shape of the array. + data : array-like, optional + Deprecated. Values written into the new array after it is created. To create an + array from existing data, use `create_array(data=...)`. Returns ------- Array The new array. """ - return Array(sync(async_api.ones(shape, **kwargs))) + return Array( + sync( + async_api.ones( + shape=shape, + chunks=chunks, + dtype=dtype, + compressor=compressor, + order=order, + store=store, + synchronizer=synchronizer, + overwrite=overwrite, + path=path, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + storage_options=storage_options, + config=config, + mode=mode, + data=data, + ) + ) + ) # TODO: add type annotations for kwargs -def ones_like(a: ArrayLike, **kwargs: Any) -> AnyArray: +def ones_like(a: ArrayLike, *, zarr_format: ZarrFormat | None = None, **kwargs: Any) -> AnyArray: """Create an array of ones like another array. Parameters ---------- a : array-like The array to create an empty array like. + zarr_format : {2, 3, None}, optional + The zarr format of the new array. If `None` (default), the zarr format of `a` if it + is a zarr array, otherwise the default zarr format. **kwargs Keyword arguments passed to [`zarr.api.asynchronous.create`][]. @@ -1379,7 +1607,7 @@ def ones_like(a: ArrayLike, **kwargs: Any) -> AnyArray: Array The new array. """ - return Array(sync(async_api.ones_like(a, **kwargs))) + return Array(sync(async_api.ones_like(a, zarr_format=zarr_format, **kwargs))) # TODO: update this once async_api.open_array is fully implemented @@ -1429,7 +1657,9 @@ def open_array( # TODO: add type annotations for kwargs -def open_like(a: ArrayLike, path: str, **kwargs: Any) -> AnyArray: +def open_like( + a: ArrayLike, path: str, *, zarr_format: ZarrFormat | None = None, **kwargs: Any +) -> AnyArray: """Open a persistent array like another array. Parameters @@ -1438,6 +1668,10 @@ def open_like(a: ArrayLike, path: str, **kwargs: Any) -> AnyArray: The shape and data-type of a define these same attributes of the returned array. path : str The path to the new array. + zarr_format : {2, 3, None}, optional + The zarr format of the array to open or create. If `None` (default), an existing + array of either format is opened, and a missing one is created in the default zarr + format. The zarr format of `a` is not inherited. **kwargs Additional keyword arguments passed to `open_array`. If `mode` is omitted or `None`, it defaults to `"a"`. Pass `mode="r"` when @@ -1448,36 +1682,111 @@ def open_like(a: ArrayLike, path: str, **kwargs: Any) -> AnyArray: Array The opened array. """ - return Array(sync(async_api.open_like(a, path=path, **kwargs))) + return Array(sync(async_api.open_like(a, path=path, zarr_format=zarr_format, **kwargs))) -# TODO: add type annotations for kwargs -def zeros(shape: tuple[int, ...], **kwargs: Any) -> AnyArray: - """Create an array with a fill value of zero. +def zeros( + shape: tuple[int, ...] | int, + *, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + order: MemoryOrder | None = None, + store: StoreLike | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + path: PathLike | None = None, + chunk_store: StoreLike | None = None, + filters: Iterable[dict[str, JSON] | Numcodec] | None = None, + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunk_shape: tuple[int, ...] | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNamesLike = None, + storage_options: dict[str, Any] | None = None, + config: ArrayConfigLike | None = None, + mode: AccessModeLiteral | None = None, + data: npt.ArrayLike | None = None, +) -> AnyArray: + """Create an array, with zero being used as the default value for + uninitialized portions of the array. + + Parameters not listed below are those of + [`create`][zarr.api.asynchronous.create], with the same meaning. Parameters ---------- shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [`zarr.api.asynchronous.create`][]. + Shape of the array. + data : array-like, optional + Deprecated. Values written into the new array after it is created. To create an + array from existing data, use `create_array(data=...)`. Returns ------- Array The new array. """ - return Array(sync(async_api.zeros(shape=shape, **kwargs))) + return Array( + sync( + async_api.zeros( + shape=shape, + chunks=chunks, + dtype=dtype, + compressor=compressor, + order=order, + store=store, + synchronizer=synchronizer, + overwrite=overwrite, + path=path, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + storage_options=storage_options, + config=config, + mode=mode, + data=data, + ) + ) + ) # TODO: add type annotations for kwargs -def zeros_like(a: ArrayLike, **kwargs: Any) -> AnyArray: +def zeros_like(a: ArrayLike, *, zarr_format: ZarrFormat | None = None, **kwargs: Any) -> AnyArray: """Create an array of zeros like another array. Parameters ---------- a : array-like The array to create an empty array like. + zarr_format : {2, 3, None}, optional + The zarr format of the new array. If `None` (default), the zarr format of `a` if it + is a zarr array, otherwise the default zarr format. **kwargs Keyword arguments passed to [`create`][zarr.api.asynchronous.create]. @@ -1486,4 +1795,4 @@ def zeros_like(a: ArrayLike, **kwargs: Any) -> AnyArray: Array The new array. """ - return Array(sync(async_api.zeros_like(a, **kwargs))) + return Array(sync(async_api.zeros_like(a, zarr_format=zarr_format, **kwargs))) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 952c750f27..9ac3c531a6 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -59,6 +59,7 @@ GroupNotFoundError, MetadataValidationError, NodeTypeValidationError, + ZarrDeprecationWarning, ZarrUserWarning, ) from zarr.storage import StoreLike, StorePath @@ -77,16 +78,33 @@ ) from typing import Any + import numpy.typing as npt + + from zarr.abc.codec import Codec + from zarr.abc.numcodec import Numcodec from zarr.core.array_spec import ArrayConfigLike from zarr.core.buffer import Buffer, BufferPrototype - from zarr.core.chunk_key_encodings import ChunkKeyEncodingLike - from zarr.core.common import MemoryOrder + from zarr.core.chunk_key_encodings import ChunkKeyEncoding, ChunkKeyEncodingLike + from zarr.core.common import AccessModeLiteral, MemoryOrder from zarr.core.dtype import ZDTypeLike from zarr.types import AnyArray, AnyAsyncArray, ArrayV2, ArrayV3, AsyncArrayV2, AsyncArrayV3 logger = logging.getLogger("zarr.group") +def _warn_group_helper_mode_kwarg(method_name: str) -> None: + """ + Emit the deprecation warning for `mode` passed to a group's `empty`, `zeros`, `ones` or + `full`, where it has no effect. + """ + msg = ( + f"Passing `mode` to `Group.{method_name}` is deprecated and will be removed in a future " + "version. It has no effect, because the group's store is already open. " + "Use `overwrite=True` to replace an existing array." + ) + warnings.warn(msg, ZarrDeprecationWarning, stacklevel=3) + + def parse_zarr_format(data: Any) -> ZarrFormat: """Parse the zarr_format field from metadata.""" return cast("ZarrFormat", parse_field(data, Literal[2, 3], "zarr_format")) @@ -1666,18 +1684,83 @@ async def tree( raise NotImplementedError("'expand' is not yet implemented.") return await group_tree_async(self, max_depth=level, max_nodes=max_nodes, plain=plain) - async def empty(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: - """Create an empty array with the specified shape in this Group. The contents will - be filled with the array's fill value or zeros if no fill value is provided. + def _member_zarr_format(self, zarr_format: ZarrFormat | None) -> ZarrFormat: + """ + The zarr format of a new array in this group, which is the format of the group. + + A group only lists members of its own format, so an array of another format + would be written into the group without becoming a member of it. + """ + if zarr_format is not None and zarr_format != self.metadata.zarr_format: + raise ValueError( + f"Cannot create a zarr_format={zarr_format} array in a " + f"zarr_format={self.metadata.zarr_format} group." + ) + return self.metadata.zarr_format + + async def empty( + self, + *, + name: str, + shape: tuple[int, ...] | int, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + fill_value: Any | None = DEFAULT_FILL_VALUE, + order: MemoryOrder | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + chunk_store: StoreLike | None = None, + filters: Iterable[dict[str, JSON] | Numcodec] | None = None, + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunk_shape: tuple[int, ...] | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNamesLike = None, + config: ArrayConfigLike | None = None, + mode: AccessModeLiteral | None = None, + data: npt.ArrayLike | None = None, + ) -> AnyAsyncArray: + """Create an empty array with the specified shape. The contents will be filled with the + specified fill value or zeros if no fill value is provided. + + Parameters not listed below are those of + [`create`][zarr.api.asynchronous.create], with the same meaning. + The group supplies `store` and `path`. Parameters ---------- name : str Name of the array. shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. + Shape of the array. + zarr_format : {2, 3, None}, optional + The zarr format of the array. It must be the format of the group, which is + the default. + data : array-like, optional + Deprecated. Values written into the new array after it is created. To create an + array from existing data, use `create_array(data=...)`. + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional + Deprecated. Has no effect, because the group's store is already open. Use + `overwrite=True` to replace an existing array. + + Returns + ------- + AsyncArray + The new array. Notes ----- @@ -1685,73 +1768,322 @@ async def empty(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> An retrieve data from an empty Zarr array, any values may be returned, and these are not guaranteed to be stable from one access to the next. """ - return await async_api.empty(shape=shape, store=self.store_path, path=name, **kwargs) + if mode is not None: + _warn_group_helper_mode_kwarg("empty") + return await async_api.empty( + shape=shape, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(zarr_format), + chunks=chunks, + dtype=dtype, + compressor=compressor, + fill_value=fill_value, + order=order, + synchronizer=synchronizer, + overwrite=overwrite, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + config=config, + data=data, + ) - async def zeros(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: - """Create an array, with zero being used as the default value for uninitialized portions of the array. + async def zeros( + self, + *, + name: str, + shape: tuple[int, ...] | int, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + order: MemoryOrder | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + chunk_store: StoreLike | None = None, + filters: Iterable[dict[str, JSON] | Numcodec] | None = None, + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunk_shape: tuple[int, ...] | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNamesLike = None, + config: ArrayConfigLike | None = None, + mode: AccessModeLiteral | None = None, + data: npt.ArrayLike | None = None, + ) -> AnyAsyncArray: + """Create an array, with zero being used as the default value for + uninitialized portions of the array. + + Parameters not listed below are those of + [`create`][zarr.api.asynchronous.create], with the same meaning. + The group supplies `store` and `path`. Parameters ---------- name : str Name of the array. shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. + Shape of the array. + zarr_format : {2, 3, None}, optional + The zarr format of the array. It must be the format of the group, which is + the default. + data : array-like, optional + Deprecated. Values written into the new array after it is created. To create an + array from existing data, use `create_array(data=...)`. + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional + Deprecated. Has no effect, because the group's store is already open. Use + `overwrite=True` to replace an existing array. Returns ------- AsyncArray The new array. """ - return await async_api.zeros(shape=shape, store=self.store_path, path=name, **kwargs) + if mode is not None: + _warn_group_helper_mode_kwarg("zeros") + return await async_api.zeros( + shape=shape, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(zarr_format), + chunks=chunks, + dtype=dtype, + compressor=compressor, + order=order, + synchronizer=synchronizer, + overwrite=overwrite, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + config=config, + data=data, + ) + + async def ones( + self, + *, + name: str, + shape: tuple[int, ...] | int, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + order: MemoryOrder | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + chunk_store: StoreLike | None = None, + filters: Iterable[dict[str, JSON] | Numcodec] | None = None, + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunk_shape: tuple[int, ...] | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNamesLike = None, + config: ArrayConfigLike | None = None, + mode: AccessModeLiteral | None = None, + data: npt.ArrayLike | None = None, + ) -> AnyAsyncArray: + """Create an array, with one being used as the default value for + uninitialized portions of the array. - async def ones(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyAsyncArray: - """Create an array, with one being used as the default value for uninitialized portions of the array. + Parameters not listed below are those of + [`create`][zarr.api.asynchronous.create], with the same meaning. + The group supplies `store` and `path`. Parameters ---------- name : str Name of the array. shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. + Shape of the array. + zarr_format : {2, 3, None}, optional + The zarr format of the array. It must be the format of the group, which is + the default. + data : array-like, optional + Deprecated. Values written into the new array after it is created. To create an + array from existing data, use `create_array(data=...)`. + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional + Deprecated. Has no effect, because the group's store is already open. Use + `overwrite=True` to replace an existing array. Returns ------- AsyncArray The new array. """ - return await async_api.ones(shape=shape, store=self.store_path, path=name, **kwargs) + if mode is not None: + _warn_group_helper_mode_kwarg("ones") + return await async_api.ones( + shape=shape, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(zarr_format), + chunks=chunks, + dtype=dtype, + compressor=compressor, + order=order, + synchronizer=synchronizer, + overwrite=overwrite, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + config=config, + data=data, + ) async def full( - self, *, name: str, shape: tuple[int, ...], fill_value: Any | None, **kwargs: Any + self, + *, + name: str, + shape: tuple[int, ...] | int, + fill_value: Any | None, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + order: MemoryOrder | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + chunk_store: StoreLike | None = None, + filters: Iterable[dict[str, JSON] | Numcodec] | None = None, + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunk_shape: tuple[int, ...] | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNamesLike = None, + config: ArrayConfigLike | None = None, + mode: AccessModeLiteral | None = None, + data: npt.ArrayLike | None = None, ) -> AnyAsyncArray: - """Create an array, with "fill_value" being used as the default value for uninitialized portions of the array. + """Create an array, with `fill_value` being used as the default value for + uninitialized portions of the array. + + Parameters not listed below are those of + [`create`][zarr.api.asynchronous.create], with the same meaning. + The group supplies `store` and `path`. Parameters ---------- name : str Name of the array. shape : int or tuple of int - Shape of the empty array. + Shape of the array. fill_value : scalar - Value to fill the array with. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. + Fill value. + zarr_format : {2, 3, None}, optional + The zarr format of the array. It must be the format of the group, which is + the default. + data : array-like, optional + Deprecated. Values written into the new array after it is created. To create an + array from existing data, use `create_array(data=...)`. + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional + Deprecated. Has no effect, because the group's store is already open. Use + `overwrite=True` to replace an existing array. Returns ------- AsyncArray The new array. """ + if mode is not None: + _warn_group_helper_mode_kwarg("full") return await async_api.full( shape=shape, fill_value=fill_value, store=self.store_path, path=name, - **kwargs, + zarr_format=self._member_zarr_format(zarr_format), + chunks=chunks, + dtype=dtype, + compressor=compressor, + order=order, + synchronizer=synchronizer, + overwrite=overwrite, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + config=config, + data=data, ) async def empty_like( @@ -1774,7 +2106,13 @@ async def empty_like( AsyncArray The new array. """ - return await async_api.empty_like(a=data, store=self.store_path, path=name, **kwargs) + return await async_api.empty_like( + a=data, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) async def zeros_like( self, *, name: str, data: async_api.ArrayLike, **kwargs: Any @@ -1795,7 +2133,13 @@ async def zeros_like( AsyncArray The new array. """ - return await async_api.zeros_like(a=data, store=self.store_path, path=name, **kwargs) + return await async_api.zeros_like( + a=data, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) async def ones_like( self, *, name: str, data: async_api.ArrayLike, **kwargs: Any @@ -1816,7 +2160,13 @@ async def ones_like( AsyncArray The new array. """ - return await async_api.ones_like(a=data, store=self.store_path, path=name, **kwargs) + return await async_api.ones_like( + a=data, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) async def full_like( self, *, name: str, data: async_api.ArrayLike, **kwargs: Any @@ -1837,7 +2187,13 @@ async def full_like( AsyncArray The new array. """ - return await async_api.full_like(a=data, store=self.store_path, path=name, **kwargs) + return await async_api.full_like( + a=data, + store=self.store_path, + path=name, + zarr_format=self._member_zarr_format(kwargs.pop("zarr_format", None)), + **kwargs, + ) async def move(self, source: str, dest: str) -> None: """Move a sub-group or sub-array from one path to another. @@ -2905,18 +3261,69 @@ def require_array( self._sync(self._async_group.require_array(name, shape=shape, config=config, **kwargs)) ) - def empty(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: - """Create an empty array with the specified shape in this Group. The contents will be filled with - the array's fill value or zeros if no fill value is provided. + def empty( + self, + *, + name: str, + shape: tuple[int, ...] | int, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + fill_value: Any | None = DEFAULT_FILL_VALUE, + order: MemoryOrder | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + chunk_store: StoreLike | None = None, + filters: Iterable[dict[str, JSON] | Numcodec] | None = None, + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunk_shape: tuple[int, ...] | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNamesLike = None, + config: ArrayConfigLike | None = None, + mode: AccessModeLiteral | None = None, + data: npt.ArrayLike | None = None, + ) -> AnyArray: + """Create an empty array with the specified shape. The contents will be filled with the + specified fill value or zeros if no fill value is provided. + + Parameters not listed below are those of + [`create`][zarr.api.asynchronous.create], with the same meaning. + The group supplies `store` and `path`. Parameters ---------- name : str Name of the array. shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. + Shape of the array. + zarr_format : {2, 3, None}, optional + The zarr format of the array. It must be the format of the group, which is + the default. + data : array-like, optional + Deprecated. Values written into the new array after it is created. To create an + array from existing data, use `create_array(data=...)`. + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional + Deprecated. Has no effect, because the group's store is already open. Use + `overwrite=True` to replace an existing array. + + Returns + ------- + Array + The new array. Notes ----- @@ -2924,61 +3331,292 @@ def empty(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray retrieve data from an empty Zarr array, any values may be returned, and these are not guaranteed to be stable from one access to the next. """ - return Array(self._sync(self._async_group.empty(name=name, shape=shape, **kwargs))) + return Array( + self._sync( + self._async_group.empty( + name=name, + shape=shape, + chunks=chunks, + dtype=dtype, + compressor=compressor, + fill_value=fill_value, + order=order, + synchronizer=synchronizer, + overwrite=overwrite, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + config=config, + mode=mode, + data=data, + ) + ) + ) - def zeros(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: - """Create an array, with zero being used as the default value for uninitialized portions of the array. + def zeros( + self, + *, + name: str, + shape: tuple[int, ...] | int, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + order: MemoryOrder | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + chunk_store: StoreLike | None = None, + filters: Iterable[dict[str, JSON] | Numcodec] | None = None, + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunk_shape: tuple[int, ...] | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNamesLike = None, + config: ArrayConfigLike | None = None, + mode: AccessModeLiteral | None = None, + data: npt.ArrayLike | None = None, + ) -> AnyArray: + """Create an array, with zero being used as the default value for + uninitialized portions of the array. + + Parameters not listed below are those of + [`create`][zarr.api.asynchronous.create], with the same meaning. + The group supplies `store` and `path`. Parameters ---------- name : str Name of the array. shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. + Shape of the array. + zarr_format : {2, 3, None}, optional + The zarr format of the array. It must be the format of the group, which is + the default. + data : array-like, optional + Deprecated. Values written into the new array after it is created. To create an + array from existing data, use `create_array(data=...)`. + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional + Deprecated. Has no effect, because the group's store is already open. Use + `overwrite=True` to replace an existing array. Returns ------- Array The new array. """ - return Array(self._sync(self._async_group.zeros(name=name, shape=shape, **kwargs))) + return Array( + self._sync( + self._async_group.zeros( + name=name, + shape=shape, + chunks=chunks, + dtype=dtype, + compressor=compressor, + order=order, + synchronizer=synchronizer, + overwrite=overwrite, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + config=config, + mode=mode, + data=data, + ) + ) + ) - def ones(self, *, name: str, shape: tuple[int, ...], **kwargs: Any) -> AnyArray: - """Create an array, with one being used as the default value for uninitialized portions of the array. + def ones( + self, + *, + name: str, + shape: tuple[int, ...] | int, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + order: MemoryOrder | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + chunk_store: StoreLike | None = None, + filters: Iterable[dict[str, JSON] | Numcodec] | None = None, + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunk_shape: tuple[int, ...] | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNamesLike = None, + config: ArrayConfigLike | None = None, + mode: AccessModeLiteral | None = None, + data: npt.ArrayLike | None = None, + ) -> AnyArray: + """Create an array, with one being used as the default value for + uninitialized portions of the array. + + Parameters not listed below are those of + [`create`][zarr.api.asynchronous.create], with the same meaning. + The group supplies `store` and `path`. Parameters ---------- name : str Name of the array. shape : int or tuple of int - Shape of the empty array. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. + Shape of the array. + zarr_format : {2, 3, None}, optional + The zarr format of the array. It must be the format of the group, which is + the default. + data : array-like, optional + Deprecated. Values written into the new array after it is created. To create an + array from existing data, use `create_array(data=...)`. + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional + Deprecated. Has no effect, because the group's store is already open. Use + `overwrite=True` to replace an existing array. Returns ------- Array The new array. """ - return Array(self._sync(self._async_group.ones(name=name, shape=shape, **kwargs))) + return Array( + self._sync( + self._async_group.ones( + name=name, + shape=shape, + chunks=chunks, + dtype=dtype, + compressor=compressor, + order=order, + synchronizer=synchronizer, + overwrite=overwrite, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + config=config, + mode=mode, + data=data, + ) + ) + ) def full( - self, *, name: str, shape: tuple[int, ...], fill_value: Any | None, **kwargs: Any + self, + *, + name: str, + shape: tuple[int, ...] | int, + fill_value: Any | None, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: ZDTypeLike | None = None, + compressor: CompressorLike = "auto", + order: MemoryOrder | None = None, + synchronizer: Any | None = None, + overwrite: bool = False, + chunk_store: StoreLike | None = None, + filters: Iterable[dict[str, JSON] | Numcodec] | None = None, + cache_metadata: bool | None = None, + cache_attrs: bool | None = None, + read_only: bool | None = None, + object_codec: Codec | None = None, + dimension_separator: Literal[".", "/"] | None = None, + write_empty_chunks: bool | None = None, + zarr_format: ZarrFormat | None = None, + meta_array: Any | None = None, + attributes: dict[str, JSON] | None = None, + chunk_shape: tuple[int, ...] | int | None = None, + chunk_key_encoding: ( + ChunkKeyEncoding + | tuple[Literal["default"], Literal[".", "/"]] + | tuple[Literal["v2"], Literal[".", "/"]] + | None + ) = None, + codecs: Iterable[Codec | dict[str, JSON]] | None = None, + dimension_names: DimensionNamesLike = None, + config: ArrayConfigLike | None = None, + mode: AccessModeLiteral | None = None, + data: npt.ArrayLike | None = None, ) -> AnyArray: - """Create an array, with "fill_value" being used as the default value for uninitialized portions of the array. + """Create an array, with `fill_value` being used as the default value for + uninitialized portions of the array. + + Parameters not listed below are those of + [`create`][zarr.api.asynchronous.create], with the same meaning. + The group supplies `store` and `path`. Parameters ---------- name : str Name of the array. shape : int or tuple of int - Shape of the empty array. + Shape of the array. fill_value : scalar - Value to fill the array with. - **kwargs - Keyword arguments passed to [zarr.api.asynchronous.create][]. + Fill value. + zarr_format : {2, 3, None}, optional + The zarr format of the array. It must be the format of the group, which is + the default. + data : array-like, optional + Deprecated. Values written into the new array after it is created. To create an + array from existing data, use `create_array(data=...)`. + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional + Deprecated. Has no effect, because the group's store is already open. Use + `overwrite=True` to replace an existing array. Returns ------- @@ -2987,7 +3625,35 @@ def full( """ return Array( self._sync( - self._async_group.full(name=name, shape=shape, fill_value=fill_value, **kwargs) + self._async_group.full( + name=name, + shape=shape, + fill_value=fill_value, + chunks=chunks, + dtype=dtype, + compressor=compressor, + order=order, + synchronizer=synchronizer, + overwrite=overwrite, + chunk_store=chunk_store, + filters=filters, + cache_metadata=cache_metadata, + cache_attrs=cache_attrs, + read_only=read_only, + object_codec=object_codec, + dimension_separator=dimension_separator, + write_empty_chunks=write_empty_chunks, + zarr_format=zarr_format, + meta_array=meta_array, + attributes=attributes, + chunk_shape=chunk_shape, + chunk_key_encoding=chunk_key_encoding, + codecs=codecs, + dimension_names=dimension_names, + config=config, + mode=mode, + data=data, + ) ) ) diff --git a/tests/test_api.py b/tests/test_api.py index 0c0ecff77f..edcf095700 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -20,6 +20,7 @@ from zarr.core.common import JSON, AccessModeLiteral, MemoryOrder, ZarrFormat from zarr.types import AnyArray +import asyncio import contextlib from typing import Literal @@ -472,6 +473,60 @@ def test_array_open_array_not_found_sync() -> None: Array.open(store) +@pytest.mark.parametrize("func", [zarr.empty_like, zarr.zeros_like, zarr.ones_like, zarr.full_like]) +@pytest.mark.parametrize("source_format", [2, 3]) +@pytest.mark.parametrize("zarr_format", [None, 2, 3]) +def test_like_zarr_format( + func: Callable[..., AnyArray], source_format: ZarrFormat, zarr_format: ZarrFormat | None +) -> None: + """ + An array created like a zarr array has the zarr format of the source unless another is + requested, and keeps the codecs of the source only when the formats match. + """ + source = zarr.create_array( + {}, + shape=(4,), + dtype="int32", + compressors=None, + zarr_format=source_format, + fill_value=7, + ) + new = func(source, zarr_format=zarr_format) + expected_format = source_format if zarr_format is None else zarr_format + assert new.metadata.zarr_format == expected_format + assert new.shape == source.shape + assert new.dtype == source.dtype + if expected_format == source_format: + assert new.compressors == source.compressors + else: + assert ( + new.compressors + == zarr.create_array( + {}, shape=(4,), dtype="int32", zarr_format=expected_format + ).compressors + ) + + +@pytest.mark.parametrize("source_format", [2, 3]) +@pytest.mark.parametrize("target_format", [None, 2, 3]) +def test_open_like_zarr_format(source_format: ZarrFormat, target_format: ZarrFormat | None) -> None: + """ + open_like does not inherit the zarr format of the source: it opens an existing array of + any format, and creates a missing one in the default zarr format. + """ + source = zarr.zeros(store={}, shape=(4,), dtype="int32", zarr_format=source_format) + store = MemoryStore() + if target_format is not None: + zarr.create_array( + store, name="existing", shape=(4,), dtype="int32", zarr_format=target_format + ) + opened = zarr.open_like(source, path="existing", store=store) + assert opened.metadata.zarr_format == (target_format or zarr.config.get("default_zarr_format")) + assert set(store._store_dict) & {"existing/zarr.json", "existing/.zarray"} == { + "existing/zarr.json" if opened.metadata.zarr_format == 3 else "existing/.zarray" + } + + @pytest.mark.parametrize("store", ["memory", "local", "zip"], indirect=True) def test_v2_and_v3_exist_at_same_path(store: Store) -> None: zarr.create_array(store, shape=(10,), dtype="uint8", zarr_format=3) @@ -1767,3 +1822,98 @@ def test_unimplemented_kwarg_warnings(kwarg_name: str) -> None: kwargs = {kwarg_name: 1} with pytest.warns(RuntimeWarning, match=".* is not yet implemented"): zarr.create(shape=(1,), **kwargs) # type: ignore[arg-type] + + +HELPER_NAMES = ("empty", "zeros", "ones", "full") +HELPER_FILL_VALUES: dict[str, int] = {"zeros": 0, "ones": 1} + + +def _create_parameters() -> dict[str, inspect.Parameter]: + """ + The named parameters of `create`, plus `mode` and `data`, which `create` takes. + """ + params = { + name: p + for name, p in inspect.signature(zarr.api.asynchronous.create).parameters.items() + if p.kind is not inspect.Parameter.VAR_KEYWORD + } + for name in ("mode", "data"): + params.setdefault( + name, inspect.Parameter(name, inspect.Parameter.KEYWORD_ONLY, default=None) + ) + return params + + +@pytest.mark.parametrize("module", [zarr.api.asynchronous, zarr.api.synchronous]) +@pytest.mark.parametrize("func_name", HELPER_NAMES) +def test_helper_signatures_mirror_create(module: Any, func_name: str) -> None: + """ + `empty`, `zeros`, `ones` and `full` take the parameters of `create`, with the same + defaults, except `fill_value` where the function fixes it, and no `**kwargs`. + """ + expected = _create_parameters() + if func_name in HELPER_FILL_VALUES: + del expected["fill_value"] + if func_name == "full": + expected["fill_value"] = inspect.Parameter( + "fill_value", inspect.Parameter.POSITIONAL_OR_KEYWORD + ) + actual = inspect.signature(getattr(module, func_name)).parameters + assert set(actual) == set(expected) + assert all(p.kind is not inspect.Parameter.VAR_KEYWORD for p in actual.values()) + for name, param in actual.items(): + assert param.default == expected[name].default, name + + +@pytest.mark.parametrize("func_name", HELPER_NAMES) +async def test_helpers_forward_every_parameter( + monkeypatch: pytest.MonkeyPatch, func_name: str +) -> None: + """ + `empty`, `zeros`, `ones` and `full` pass every argument on to `create` unchanged, and + the synchronous versions pass every argument on to the asynchronous ones. + """ + received: dict[str, Any] = {} + real = await zarr.api.asynchronous.create(shape=(1,), store=MemoryStore()) + + async def record(**kwargs: Any) -> Any: + received.update(kwargs) + return real + + func = getattr(zarr.api.asynchronous, func_name) + args = {name: object() for name in inspect.signature(func).parameters} + monkeypatch.setattr(zarr.api.asynchronous, "create", record) + with pytest.warns(ZarrDeprecationWarning, match="Passing `data`"): + await func(**args) + expected = args | ( + {"fill_value": HELPER_FILL_VALUES[func_name]} if func_name in HELPER_FILL_VALUES else {} + ) + assert received == expected + + received.clear() + monkeypatch.setattr(zarr.api.asynchronous, func_name, record) + sync_func = getattr(zarr.api.synchronous, func_name) + sync_args = {name: object() for name in inspect.signature(sync_func).parameters} + await asyncio.to_thread(sync_func, **sync_args) + assert received == sync_args + + +@pytest.mark.parametrize("func_name", HELPER_NAMES) +def test_helpers_data_deprecated(func_name: str) -> None: + """ + Passing `data` to `empty`, `zeros`, `ones` or `full` warns, and the data is still written. + """ + fill = {"fill_value": 5} if func_name == "full" else {} + with pytest.warns(ZarrDeprecationWarning, match=f"Passing `data` to `{func_name}`"): + z = getattr(zarr, func_name)((3,), data=np.arange(3), **fill) + assert_array_equal(z[:], np.arange(3)) + + +@pytest.mark.parametrize("func_name", HELPER_NAMES) +def test_helpers_unknown_keyword(func_name: str) -> None: + """ + `empty`, `zeros`, `ones` and `full` reject keyword arguments that `create` does not take. + """ + fill = {"fill_value": 5} if func_name == "full" else {} + with pytest.raises(TypeError, match=rf"^{func_name}\(\) got an unexpected keyword argument"): + getattr(zarr, func_name)((3,), typo=1, **fill) diff --git a/tests/test_api/test_asynchronous.py b/tests/test_api/test_asynchronous.py index 6ebec36bbd..66fd6819dd 100644 --- a/tests/test_api/test_asynchronous.py +++ b/tests/test_api/test_asynchronous.py @@ -19,6 +19,7 @@ import numpy.typing as npt from zarr.core.array import AsyncArray + from zarr.core.common import ZarrFormat from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.types import AnyArray @@ -56,31 +57,48 @@ def test_get_shape_chunks( assert _get_shape_chunks(observed) == expected +_V2_ARRAY = create_array( + {}, + chunks=(10,), + shape=(100,), + dtype="f8", + compressors=None, + filters=None, + zarr_format=2, +)._async_array +_V2_ARRAY_ARGS = { + "chunks": (10,), + "shape": (100,), + "dtype": np.dtype("f8"), + "fill_value": np.float64(0.0), +} + + @pytest.mark.parametrize( - ("observed", "expected"), + ("observed", "zarr_format", "expected"), [ - (np.arange(10, dtype=np.dtype("int64")), {"shape": (10,), "dtype": np.dtype("int64")}), - (WithChunks(shape=(1, 2), chunks=(1, 2)), {"chunks": (1, 2), "shape": (1, 2)}), ( - create_array( - {}, - chunks=(10,), - shape=(100,), - dtype="f8", - compressors=None, - filters=None, - zarr_format=2, - )._async_array, - { - "chunks": (10,), - "shape": (100,), - "dtype": np.dtype("f8"), - "fill_value": np.float64(0.0), - "compressor": None, - "filters": None, - "order": "C", - }, + np.arange(10, dtype=np.dtype("int64")), + None, + {"shape": (10,), "dtype": np.dtype("int64")}, + ), + ( + np.arange(10, dtype=np.dtype("int64")), + 2, + {"shape": (10,), "dtype": np.dtype("int64"), "zarr_format": 2}, + ), + (WithChunks(shape=(1, 2), chunks=(1, 2)), None, {"chunks": (1, 2), "shape": (1, 2)}), + ( + _V2_ARRAY, + None, + _V2_ARRAY_ARGS | {"zarr_format": 2, "compressor": None, "filters": None, "order": "C"}, + ), + ( + _V2_ARRAY, + 2, + _V2_ARRAY_ARGS | {"zarr_format": 2, "compressor": None, "filters": None, "order": "C"}, ), + (_V2_ARRAY, 3, _V2_ARRAY_ARGS | {"zarr_format": 3}), ], ) def test_like_args( @@ -88,12 +106,13 @@ def test_like_args( | AsyncArray[ArrayV3Metadata] | AnyArray | npt.NDArray[Any], + zarr_format: ZarrFormat | None, expected: object, ) -> None: """ Test the like_args function """ - assert _like_args(observed) == expected + assert _like_args(observed, zarr_format) == expected async def test_open_no_array() -> None: diff --git a/tests/test_group.py b/tests/test_group.py index f8a9ca1a9b..a0b5bed13a 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import contextlib import inspect import json @@ -50,6 +51,7 @@ GroupNotFoundError, MetadataValidationError, NodeTypeValidationError, + ZarrDeprecationWarning, ZarrUserWarning, ) from zarr.storage import LocalStore, MemoryStore, StorePath, ZipStore @@ -911,6 +913,131 @@ def test_group_array_like_creation( assert np.all(new_arr[:] == expect_fill) +@pytest.mark.parametrize("group_format", [2, 3]) +@pytest.mark.parametrize( + "method_name", + ["empty", "zeros", "ones", "full", "empty_like", "zeros_like", "ones_like", "full_like"], +) +@pytest.mark.parametrize("source_format", [2, 3]) +@pytest.mark.parametrize("pass_zarr_format", [False, True]) +def test_group_array_helpers_use_group_format( + group_format: ZarrFormat, + method_name: str, + source_format: ZarrFormat, + pass_zarr_format: bool, +) -> None: + """ + Group.{empty, zeros, ones, full} and their *_like versions create an array in the zarr + format of the group, which makes it a member of the group. For the *_like versions this + holds whatever the format of the source array. + """ + group = Group.from_store(MemoryStore(), zarr_format=group_format) + kwargs: dict[str, Any] = {"zarr_format": group_format} if pass_zarr_format else {} + if method_name == "full": + kwargs["fill_value"] = 3 + if method_name.endswith("_like"): + source = zarr.zeros(store={}, shape=(4,), dtype="int32", zarr_format=source_format) + arr = getattr(group, method_name)(name="a", data=source, **kwargs) + else: + arr = getattr(group, method_name)(name="a", shape=(4,), **kwargs) + assert arr.metadata.zarr_format == group_format + assert list(group.array_keys()) == ["a"] + + +def test_group_array_helpers_other_format() -> None: + """ + Asking a group helper for an array in a zarr format other than the group's raises. + """ + group = Group.from_store(MemoryStore(), zarr_format=2) + with pytest.raises(ValueError, match="zarr_format=3 array in a zarr_format=2 group"): + group.zeros(name="a", shape=(4,), zarr_format=3) + + +GROUP_HELPER_NAMES = ("empty", "zeros", "ones", "full") + + +@pytest.mark.parametrize("method_name", GROUP_HELPER_NAMES) +def test_group_helper_signatures_mirror_create(method_name: str) -> None: + """ + The group versions of `empty`, `zeros`, `ones` and `full` take the parameters of the + top-level functions, except `store`, `path` and `storage_options`, which the group + supplies, plus `name`. They take no `**kwargs`. + """ + top = inspect.signature(getattr(zarr.api.asynchronous, method_name)).parameters + expected = { + k: v.default for k, v in top.items() if k not in {"store", "path", "storage_options"} + } + expected["name"] = inspect.Parameter.empty + if method_name == "full": + expected["fill_value"] = inspect.Parameter.empty + for cls in (AsyncGroup, Group): + params = inspect.signature(getattr(cls, method_name)).parameters + actual = {k: v.default for k, v in params.items() if k != "self"} + assert actual == expected, cls + assert all(p.kind is not inspect.Parameter.VAR_KEYWORD for p in params.values()) + + +@pytest.mark.parametrize("method_name", GROUP_HELPER_NAMES) +async def test_group_helpers_forward_every_parameter( + monkeypatch: pytest.MonkeyPatch, method_name: str +) -> None: + """ + The group versions of `empty`, `zeros`, `ones` and `full` pass every argument on to the + top-level function, adding the group's store and the array's path, and `Group` passes + every argument on to `AsyncGroup`. + """ + received: dict[str, Any] = {} + real = await zarr.api.asynchronous.create(shape=(1,), store=MemoryStore()) + + async def record(*args: Any, **kwargs: Any) -> Any: + received.update(kwargs) + return real + + group = await AsyncGroup.from_store(MemoryStore(), zarr_format=3) + method = getattr(group, method_name) + args: dict[str, Any] = {name: object() for name in inspect.signature(method).parameters} + args |= {"zarr_format": 3, "mode": None} + monkeypatch.setattr(zarr.api.asynchronous, method_name, record) + await method(**args) + expected = {k: v for k, v in args.items() if k not in {"name", "mode"}} + expected |= {"store": group.store_path, "path": args["name"]} + assert received == expected + + received.clear() + monkeypatch.setattr(AsyncGroup, method_name, record) + sync_method = getattr(Group(group), method_name) + sync_args = {name: object() for name in inspect.signature(sync_method).parameters} + await asyncio.to_thread(sync_method, **sync_args) + assert received == sync_args + + +@pytest.mark.parametrize("method_name", GROUP_HELPER_NAMES) +def test_group_helpers_mode_deprecated(method_name: str) -> None: + """ + Passing `mode` to a group's `empty`, `zeros`, `ones` or `full` warns that it has no + effect, and the array is created as without it. + """ + group = Group.from_store(MemoryStore(), zarr_format=3) + fill = {"fill_value": 5} if method_name == "full" else {} + with pytest.warns(ZarrDeprecationWarning, match=f"Passing `mode` to `Group.{method_name}`"): + arr = getattr(group, method_name)(name="a", shape=(3,), mode="w", **fill) + assert list(group.array_keys()) == ["a"] + assert arr.shape == (3,) + + +@pytest.mark.parametrize("method_name", GROUP_HELPER_NAMES) +def test_group_helpers_unknown_keyword(method_name: str) -> None: + """ + The group versions of `empty`, `zeros`, `ones` and `full` reject keyword arguments they + do not take, including `store` and `path`, which the group supplies. + """ + group = Group.from_store(MemoryStore(), zarr_format=3) + fill = {"fill_value": 5} if method_name == "full" else {} + for name in ("typo", "store", "path"): + with pytest.raises(TypeError, match=f"unexpected keyword argument '{name}'"): + getattr(group, method_name)(name="a", shape=(3,), **{name: 1}, **fill) + + def test_group_array_creation( store: Store, zarr_format: ZarrFormat,