From d78d177ff0d54aec4cc1bcf272375eef91221e33 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Sun, 13 Oct 2024 12:24:56 -0700 Subject: [PATCH 01/17] fix(consolidated metadata): skip .zmetadata key in members search --- src/zarr/core/group.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 0b15e2f08..110d12758 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -1139,7 +1139,8 @@ async def _members( raise ValueError(msg) # would be nice to make these special keys accessible programmatically, # and scoped to specific zarr versions - _skip_keys = ("zarr.json", ".zgroup", ".zattrs") + # especially true for `.zmetadata` which is configurable + _skip_keys = ("zarr.json", ".zgroup", ".zattrs", ".zmetadata") # hmm lots of I/O and logic interleaved here. # We *could* have an async gen over self.metadata.consolidated_metadata.metadata.keys() From b6e89350e9376751cb136a6d4c32d51081d80323 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Sun, 13 Oct 2024 12:50:12 -0700 Subject: [PATCH 02/17] test + userwarning --- src/zarr/core/group.py | 8 +++++--- tests/v3/test_group.py | 11 +++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 110d12758..e85057e2f 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -4,6 +4,7 @@ import itertools import json import logging +import warnings from collections import defaultdict from dataclasses import asdict, dataclass, field, fields, replace from typing import TYPE_CHECKING, Literal, TypeVar, assert_never, cast, overload @@ -1170,9 +1171,10 @@ async def _members( # keyerror is raised when `key` names an object (in the object storage sense), # as opposed to a prefix, in the store under the prefix associated with this group # in which case `key` cannot be the name of a sub-array or sub-group. - logger.warning( - "Object at %s is not recognized as a component of a Zarr hierarchy.", - key, + warnings.warn( + f"Object at {key} is not recognized as a component of a Zarr hierarchy.", + UserWarning, + stacklevel=1, ) def _members_consolidated( diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 90933abea..7aa9e0c93 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -1,6 +1,7 @@ from __future__ import annotations import pickle +import warnings from typing import TYPE_CHECKING, Any, Literal, cast import numpy as np @@ -1091,8 +1092,8 @@ async def test_require_array(store: Store, zarr_format: ZarrFormat) -> None: @pytest.mark.parametrize("consolidate", [True, False]) -def test_members_name(store: Store, consolidate: bool): - group = Group.from_store(store=store) +async def test_members_name(store: Store, consolidate: bool, zarr_format: ZarrFormat): + group = Group.from_store(store=store, zarr_format=zarr_format) a = group.create_group(name="a") a.create_array("array", shape=(1,)) b = a.create_group(name="b") @@ -1108,6 +1109,12 @@ def test_members_name(store: Store, consolidate: bool): expected = ["/a", "/a/array", "/a/b", "/a/b/array"] assert paths == expected + # regression test for https://github.com/zarr-developers/zarr-python/pull/2356 + g = zarr.open_group(store, use_consolidated=False) + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert list(g) + async def test_open_mutable_mapping(): group = await zarr.api.asynchronous.open_group(store={}, mode="w") From 2b1e90b9e3566bf2a2da1d20dad258c0ba816b18 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Sun, 13 Oct 2024 13:18:25 -0700 Subject: [PATCH 03/17] fixup tests --- tests/v3/test_group.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 7aa9e0c93..20960f034 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextlib import pickle import warnings from typing import TYPE_CHECKING, Any, Literal, cast @@ -178,22 +179,33 @@ def test_group_members(store: Store, zarr_format: ZarrFormat, consolidated_metad ) ) + # this warning shows up when extra objects show up in the hierarchy + warn_context = pytest.warns( + UserWarning, match=r"Object at .* is not recognized as a component of a Zarr hierarchy." + ) if consolidated_metadata: - zarr.consolidate_metadata(store=store, zarr_format=zarr_format) + with warn_context: + zarr.consolidate_metadata(store=store, zarr_format=zarr_format) + # now that we've consolidated the store, we shouldn't get the warnings from the unrecognized objects anymore + # we use a nullcontext to handle these cases + warn_context = contextlib.nullcontext() group = zarr.open_consolidated(store=store, zarr_format=zarr_format) - members_observed = group.members() + with warn_context: + members_observed = group.members() # members are not guaranteed to be ordered, so sort before comparing assert sorted(dict(members_observed)) == sorted(members_expected) # partial - members_observed = group.members(max_depth=1) + with warn_context: + members_observed = group.members(max_depth=1) members_expected["subgroup/subsubgroup"] = subsubgroup # members are not guaranteed to be ordered, so sort before comparing assert sorted(dict(members_observed)) == sorted(members_expected) # total - members_observed = group.members(max_depth=None) + with warn_context: + members_observed = group.members(max_depth=None) members_expected["subgroup/subsubgroup/subsubsubgroup"] = subsubsubgroup # members are not guaranteed to be ordered, so sort before comparing assert sorted(dict(members_observed)) == sorted(members_expected) From 016ec25d867125db3b1c13a9e13a4f13eb4d4dbf Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Mon, 14 Oct 2024 19:55:42 +0200 Subject: [PATCH 04/17] Update deprecated stage names (#2362) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fa1e8bf75..0dd5bd73d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ ci: autoupdate_commit_msg: "chore: update pre-commit hooks" autofix_commit_msg: "style: pre-commit fixes" autofix_prs: false -default_stages: [commit, push] +default_stages: [pre-commit, pre-push] default_language_version: python: python3 repos: From 243ee1b200ba874a28d3e94af7d0940ddaabb26c Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Mon, 14 Oct 2024 19:56:07 +0200 Subject: [PATCH 05/17] Multiple imports for an import name (#2361) --- src/zarr/abc/store.py | 3 +-- src/zarr/api/asynchronous.py | 2 -- src/zarr/storage/logging.py | 1 - tests/v3/test_indexing.py | 1 - 4 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index 261e56dd0..85e335089 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -2,9 +2,8 @@ from abc import ABC, abstractmethod from asyncio import gather -from collections.abc import AsyncGenerator, Iterable from types import TracebackType -from typing import TYPE_CHECKING, Any, NamedTuple, Protocol, runtime_checkable +from typing import TYPE_CHECKING, NamedTuple, Protocol, runtime_checkable if TYPE_CHECKING: from collections.abc import AsyncGenerator, Iterable diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index a6363bc18..559049ae4 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -10,8 +10,6 @@ from zarr.abc.store import Store from zarr.core.array import Array, AsyncArray, get_array_metadata -from zarr.core.buffer import NDArrayLike -from zarr.core.chunk_key_encodings import ChunkKeyEncoding from zarr.core.common import ( JSON, AccessModeLiteral, diff --git a/src/zarr/storage/logging.py b/src/zarr/storage/logging.py index 195c94630..59a796dc1 100644 --- a/src/zarr/storage/logging.py +++ b/src/zarr/storage/logging.py @@ -8,7 +8,6 @@ from typing import TYPE_CHECKING, Any, Self from zarr.abc.store import AccessMode, ByteRangeRequest, Store -from zarr.core.buffer import Buffer if TYPE_CHECKING: from collections.abc import AsyncGenerator, Generator, Iterable diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index 38ca24a7b..0ea9cda39 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -1769,7 +1769,6 @@ async def test_accessed_chunks( # chunks: chunk size # ops: list of tuples with (optype, tuple of slices) # optype = "__getitem__" or "__setitem__", tuple length must match number of dims - import itertools # Use a counting dict as the backing store so we can track the items access store = await CountingDict.open() From c646028fe24e21a2744ceff8a6fe60f16d0186ba Mon Sep 17 00:00:00 2001 From: David Stansby Date: Mon, 14 Oct 2024 18:56:19 +0100 Subject: [PATCH 06/17] Fix logo height in doc navbar (#2352) --- docs/_static/custom.css | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 04f99c680..1d32606f9 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -1,12 +1,5 @@ @import url('https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,400;0,700;0,900;1,400;1,700;1,900&family=Open+Sans:ital,wght@0,400;0,600;1,400;1,600&display=swap'); -.navbar-brand img { - height: 75px; -} -.navbar-brand { - height: 75px; -} - body { font-family: 'Open Sans', sans-serif; } From 840a3f7bd7a4952cce1281981aac555959e66e18 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:11:58 -0700 Subject: [PATCH 07/17] Bump sphinx from 8.0.2 to 8.1.3 in the actions group across 1 directory (#2360) Bumps the actions group with 1 update in the / directory: [sphinx](https://github.com/sphinx-doc/sphinx). Updates `sphinx` from 8.0.2 to 8.1.3 - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES.rst) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v8.0.2...v8.1.3) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2bc2b526e..593be36b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ gpu = [ "cupy-cuda12x", ] docs = [ - 'sphinx==8.0.2', + 'sphinx==8.1.3', 'sphinx-autobuild>=2021.3.14', 'sphinx-autoapi==3.3.2', 'sphinx_design', From fe752a2a2b0e91dbe566ab9874da5bb56957a688 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 15 Oct 2024 08:52:19 -0700 Subject: [PATCH 08/17] test(ci): change branch name in v3 workflows (#2368) --- .github/workflows/gpu_test.yml | 6 +++--- .github/workflows/hypothesis.yaml | 2 -- .github/workflows/test.yml | 6 +++--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index e0be89753..f9e36e9c2 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -1,13 +1,13 @@ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: GPU Test V3 +name: GPU Test on: push: - branches: [ v3 ] + branches: [ main ] pull_request: - branches: [ v3 ] + branches: [ main ] workflow_dispatch: env: diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index c5a239c27..85d48bddb 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -3,11 +3,9 @@ on: push: branches: - "main" - - "v3" pull_request: branches: - "main" - - "v3" types: [opened, reopened, synchronize, labeled] schedule: - cron: "0 0 * * *" # Daily “At 00:00” UTC diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5683b62df..ae24fca6f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,13 +1,13 @@ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Test V3 +name: Test on: push: - branches: [ v3 ] + branches: [ main ] pull_request: - branches: [ v3 ] + branches: [ main ] workflow_dispatch: concurrency: From d43b6c75b3b640ad7e9ec654db855d82c79e1fb4 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Tue, 15 Oct 2024 18:20:36 +0200 Subject: [PATCH 09/17] Use lazy % formatting in logging functions (#2366) * Use lazy % formatting in logging functions * f-string should be more efficient * Space before unit symbol From "SI Unit rules and style conventions": https://physics.nist.gov/cuu/Units/checklist.html There is a space between the numerical value and unit symbol, even when the value is used in an adjectival sense, except in the case of superscript units for plane angle. * Enforce ruff/flake8-logging-format rules (G) --------- Co-authored-by: Joe Hamman --- pyproject.toml | 1 + src/zarr/core/sync.py | 2 +- src/zarr/storage/logging.py | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 593be36b7..059fa8fdb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -211,6 +211,7 @@ extend-select = [ "B", # flake8-bugbear "C4", # flake8-comprehensions "FLY", # flynt + "G", # flake8-logging-format "I", # isort "ISC", # flake8-implicit-str-concat "PGH", # pygrep-hooks diff --git a/src/zarr/core/sync.py b/src/zarr/core/sync.py index 20f04f543..8c5bc9c39 100644 --- a/src/zarr/core/sync.py +++ b/src/zarr/core/sync.py @@ -133,7 +133,7 @@ def sync( finished, unfinished = wait([future], return_when=asyncio.ALL_COMPLETED, timeout=timeout) if len(unfinished) > 0: - raise TimeoutError(f"Coroutine {coro} failed to finish in within {timeout}s") + raise TimeoutError(f"Coroutine {coro} failed to finish in within {timeout} s") assert len(finished) == 1 return_result = next(iter(finished)).result() diff --git a/src/zarr/storage/logging.py b/src/zarr/storage/logging.py index 59a796dc1..a29661729 100644 --- a/src/zarr/storage/logging.py +++ b/src/zarr/storage/logging.py @@ -83,15 +83,15 @@ def log(self, hint: Any = "") -> Generator[None, None, None]: method = inspect.stack()[2].function op = f"{type(self._store).__name__}.{method}" if hint: - op += f"({hint})" - self.logger.info(f"Calling {op}") + op = f"{op}({hint})" + self.logger.info("Calling %s", op) start_time = time.time() try: self.counter[method] += 1 yield finally: end_time = time.time() - self.logger.info(f"Finished {op} [{end_time - start_time:.2f}s]") + self.logger.info("Finished %s [%.2f s]", op, end_time - start_time) @property def supports_writes(self) -> bool: From e0c4f6e2026292b37b110e2cc970be2a8164fd71 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 15 Oct 2024 11:02:30 -0700 Subject: [PATCH 10/17] Move roadmap and v3-design documument to docs (#2354) * move roadmap to docs * formatting and minor copy editing --- docs/index.rst | 1 + docs/roadmap.rst | 696 +++++++++++++++++++++++++++++++++++++++ v3-roadmap-and-design.md | 429 ------------------------ 3 files changed, 697 insertions(+), 429 deletions(-) create mode 100644 docs/roadmap.rst delete mode 100644 v3-roadmap-and-design.md diff --git a/docs/index.rst b/docs/index.rst index 6b90b5a77..d0b41ed63 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,7 @@ Zarr-Python release license contributing + roadmap **Version**: |version| diff --git a/docs/roadmap.rst b/docs/roadmap.rst new file mode 100644 index 000000000..93f2a2689 --- /dev/null +++ b/docs/roadmap.rst @@ -0,0 +1,696 @@ +Roadmap +======= + +- Status: active +- Author: Joe Hamman +- Created On: October 31, 2023 +- Input from: + + - Davis Bennett / @d-v-b + - Norman Rzepka / @normanrz + - Deepak Cherian @dcherian + - Brian Davis / @monodeldiablo + - Oliver McCormack / @olimcc + - Ryan Abernathey / @rabernat + - Jack Kelly / @JackKelly + - Martin Durrant / @martindurant + +.. note:: + + This document was written in the early stages of the 3.0 refactor. Some + aspects of the design have changed since this was originally written. + Questions and discussion about the contents of this document should be directed to + `this GitHub Discussion `__. + +Introduction +------------ + +This document lays out a design proposal for version 3.0 of the +`Zarr-Python `__ package. A +specific focus of the design is to bring Zarr-Python’s API up to date +with the `Zarr V3 +specification `__, +with the hope of enabling the development of the many features and +extensions that motivated the V3 Spec. The ideas presented here are +expected to result in a major release of Zarr-Python (version 3.0) +including significant a number of breaking API changes. For clarity, +“V3” will be used to describe the version of the Zarr specification and +“3.0” will be used to describe the release tag of the Zarr-Python +project. + +Current status of V3 in Zarr-Python +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +During the development of the V3 Specification, a `prototype +implementation `__ +was added to the Zarr-Python library. Since that implementation, the V3 +spec evolved in significant ways and as a result, the Zarr-Python +library is now out of sync with the approved spec. Downstream libraries +(e.g. `Xarray `__) have added support +for this implementation and will need to migrate to the accepted spec +when its available in Zarr-Python. + +Goals +----- + +- Provide a complete implementation of Zarr V3 through the Zarr-Python + API +- Clear the way for exciting extensions / ZEPs + (i.e. `sharding `__, + `variable chunking `__, + etc.) +- Provide a developer API that can be used to implement and register V3 + extensions +- Improve the performance of Zarr-Python by streamlining the interface + between the Store layer and higher level APIs (e.g. Groups and + Arrays) +- Clean up the internal and user facing APIs +- Improve code quality and robustness (e.g. achieve 100% type hint + coverage) +- Align the Zarr-Python array API with the `array API + Standard `__ + +Examples of what 3.0 will enable? +--------------------------------- + +1. Reading and writing V3 spec-compliant groups and arrays +2. V3 extensions including sharding and variable chunking. +3. Improved performance by leveraging concurrency when + creating/reading/writing to stores (imagine a + ``create_hierarchy(zarr_objects)`` function). +4. User-developed extensions (e.g. storage-transformers) can be + registered with Zarr-Python at runtime + +Non-goals (of this document) +---------------------------- + +- Implementation of any unaccepted Zarr V3 extensions +- Major revisions to the Zarr V3 spec + +Requirements +------------ + +1. Read and write spec compliant V2 and V3 data +2. Limit unnecessary traffic to/from the store +3. Cleanly define the Array/Group/Store abstractions +4. Cleanly define how V2 will be supported going forward +5. Provide a clear roadmap to help users upgrade to 3.0 +6. Developer tools / hooks for registering extensions + +Design +------ + +Async API +~~~~~~~~~ + +Zarr-Python is an IO library. As such, supporting concurrent action +against the storage layer is critical to achieving acceptable +performance. The Zarr-Python 2 was not designed with asynchronous +computation in mind and as a result has struggled to effectively +leverage the benefits of concurrency. At one point, ``getitems`` and +``setitems`` support was added to the Zarr store model but that is only +used for operating on a set of chunks in a single variable. + +With Zarr-Python 3.0, we have the opportunity to revisit this design. +The proposal here is as follows: + +1. The ``Store`` interface will be entirely async. +2. On top of the async ``Store`` interface, we will provide an + ``AsyncArray`` and ``AsyncGroup`` interface. +3. Finally, the primary user facing API will be synchronous ``Array`` + and ``Group`` classes that wrap the async equivalents. + +**Examples** + +- **Store** + + .. code:: python + + class Store: + ... + async def get(self, key: str) -> bytes: + ... + async def get_partial_values(self, key_ranges: List[Tuple[str, Tuple[int, Optional[int]]]]) -> bytes: + ... + # (no sync interface here) + +- **Array** + + .. code:: python + + class AsyncArray: + ... + + async def getitem(self, selection: Selection) -> np.ndarray: + # the core logic for getitem goes here + + class Array: + _async_array: AsyncArray + + def __getitem__(self, selection: Selection) -> np.ndarray: + return sync(self._async_array.getitem(selection)) + +- **Group** + + .. code:: python + + class AsyncGroup: + ... + + async def create_group(self, path: str, **kwargs) -> AsyncGroup: + # the core logic for create_group goes here + + class Group: + _async_group: AsyncGroup + + def create_group(self, path: str, **kwargs) -> Group: + return sync(self._async_group.create_group(path, **kwargs)) + + **Internal Synchronization API** + +With the ``Store`` and core ``AsyncArray``/ ``AsyncGroup`` classes being +predominantly async, Zarr-Python will need an internal API to provide a +synchronous API. The proposal here is to use the approach in +`fsspec `__ +to provide a high-level ``sync`` function that takes an ``awaitable`` +and runs it in its managed IO Loop / thread. + +| **FAQ** 1. Why two levels of Arrays/groups? a. First, this is an + intentional decision and departure from the current Zarrita + implementation b. The idea is that users rarely want to mix + interfaces. Either they are working within an async context (currently + quite rare) or they are in a typical synchronous context. c. Splitting + the two will allow us to clearly define behavior on the ``AsyncObj`` + and simply wrap it in the ``SyncObj``. 2. What if a store is only has + a synchronous backend? a. First off, this is expected to be a fairly + rare occurrence. Most storage backends have async interfaces. b. But + in the event a storage backend doesn’t have a async interface, there + is nothing wrong with putting synchronous code in ``async`` methods. + There are approaches to enabling concurrent action through wrappers + like AsyncIO’s ``loop.run_in_executor`` (`ref + 1 `__, + `ref 2 `__, `ref + 3 `__, + `ref + 4 `__. +| 3. Will Zarr help manage the async contexts encouraged by some + libraries + (e.g. `AioBotoCore `__)? + a. Many async IO libraries require entering an async context before + interacting with the API. We expect some experimentation to be needed + here but the initial design will follow something close to what fsspec + does (`example in + s3fs `__). + 4. Why not provide a synchronous Store interface? a. We could but this + design is simpler. It would mean supporting it in the ``AsyncGroup`` + and ``AsyncArray`` classes which, may be more trouble than its worth. + Storage backends that do not have an async API will be encouraged to + wrap blocking calls in an async wrapper + (e.g. ``loop.run_in_executor``). + +Store API +~~~~~~~~~ + +The ``Store`` API is specified directly in the V3 specification. All V3 +stores should implement this abstract API, omitting Write and List +support as needed. As described above, all stores will be expected to +expose the required methods as async methods. + +**Example** + +.. code:: python + + class ReadWriteStore: + ... + async def get(self, key: str) -> bytes: + ... + + async def get_partial_values(self, key_ranges: List[Tuple[str, int, int]) -> bytes: + ... + + async def set(self, key: str, value: Union[bytes, bytearray, memoryview]) -> None: + ... # required for writable stores + + async def set_partial_values(self, key_start_values: List[Tuple[str, int, Union[bytes, bytearray, memoryview]]]) -> None: + ... # required for writable stores + + async def list(self) -> List[str]: + ... # required for listable stores + + async def list_prefix(self, prefix: str) -> List[str]: + ... # required for listable stores + + async def list_dir(self, prefix: str) -> List[str]: + ... # required for listable stores + + # additional (optional methods) + async def getsize(self, prefix: str) -> int: + ... + + async def rename(self, src: str, dest: str) -> None + ... + + +Recognizing that there are many Zarr applications today that rely on the +``MutableMapping`` interface supported by Zarr-Python 2, a wrapper store +will be developed to allow existing stores to plug directly into this +API. + +Array API +~~~~~~~~~ + +The user facing array interface will implement a subset of the `Array +API Standard `__. Most of the +computational parts of the Array API Standard don’t fit into Zarr right +now. That’s okay. What matters most is that we ensure we can give +downstream applications a compliant API. + +*Note, Zarr already does most of this so this is more about formalizing +the relationship than a substantial change in API.* + ++------------------------+------------------------+-------------------------+-------------------------+ +| | Included | Not Included | Unknown / Maybe Possible| ++========================+========================+=========================+=========================+ +| **Attributes** | ``dtype`` | ``mT`` | ``device`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``ndim`` | ``T`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``shape`` | | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``size`` | | | ++------------------------+------------------------+-------------------------+-------------------------+ +| **Methods** | ``__getitem__`` | ``__array_namespace__`` | ``to_device`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``__setitem__`` | ``__abs__`` | ``__bool__`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``__eq__`` | ``__add__`` | ``__complex__`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``__bool__`` | ``__and__`` | ``__dlpack__`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__floordiv__`` | ``__dlpack_device__`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__ge__`` | ``__float__`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__gt__`` | ``__index__`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__invert__`` | ``__int__`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__le__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__lshift__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__lt__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__matmul__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__mod__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__mul__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__ne__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__neg__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__or__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__pos__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__pow__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__rshift__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__sub__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__truediv__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | | ``__xor__`` | | ++------------------------+------------------------+-------------------------+-------------------------+ +| **Creation functions** | ``zeros`` | | ``arange`` | +| (``zarr.creation``) | | | | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``zeros_like`` | | ``asarray`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``ones`` | | ``eye`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``ones_like`` | | ``from_dlpack`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``full`` | | ``linspace`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``full_like`` | | ``meshgrid`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``empty`` | | ``tril`` | ++------------------------+------------------------+-------------------------+-------------------------+ +| | ``empty_like`` | | ``triu`` | ++------------------------+------------------------+-------------------------+-------------------------+ + +In addition to the core array API defined above, the Array class should +have the following Zarr specific properties: + +- ``.metadata`` (see Metadata Interface below) +- ``.attrs`` - (pulled from metadata object) +- ``.info`` - (repolicated from existing property †) + +*† In Zarr-Python 2, the info property listed the store to identify +initialized chunks. By default this will be turned off in 3.0 but will +be configurable.* + +**Indexing** + +Zarr-Python currently supports ``__getitem__`` style indexing and the +special ``oindex`` and ``vindex`` indexers. These are not part of the +current Array API standard (see +`data-apis/array-api#669 `__) +but they have been `proposed as a +NEP `__. +Zarr-Python will maintain these in 3.0. + +We are also exploring a new high-level indexing API that will enabled +optimized batch/concurrent loading of many chunks. We expect this to be +important to enable performant loading of data in the context of +sharding. See `this +discussion `__ +for more detail. + +Concurrent indexing across multiple arrays will be possible using the +AsyncArray API. + +**Async and Sync Array APIs** + +Most the logic to support Zarr Arrays will live in the ``AsyncArray`` +class. There are a few notable differences that should be called out. + +=============== ============ +Sync Method Async Method +=============== ============ +``__getitem__`` ``getitem`` +``__setitem__`` ``setitem`` +``__eq__`` ``equals`` +=============== ============ + +**Metadata interface** + +Zarr-Python 2.\* closely mirrors the V2 spec metadata schema in the +Array and Group classes. In 3.0, we plan to move the underlying metadata +representation to a separate interface (e.g. ``Array.metadata``). This +interface will return either a ``V2ArrayMetadata`` or +``V3ArrayMetadata`` object (both will inherit from a parent +``ArrayMetadataABC`` class. The ``V2ArrayMetadata`` and +``V3ArrayMetadata`` classes will be responsible for producing valid JSON +representations of their metadata, and yielding a consistent view to the +``Array`` or ``Group`` class. + +Group API +~~~~~~~~~ + +The main question is how closely we should follow the existing +Zarr-Python implementation / ``MutableMapping`` interface. The table +below shows the primary ``Group`` methods in Zarr-Python 2 and attempts +to identify if and how they would be implemented in 3.0. + ++---------------------+------------------+------------------+-----------------------+ +| V2 Group Methods | ``AsyncGroup`` | ``Group`` | ``h5py_compat.Group`` | ++=====================+==================+==================+=======================+ +| ``__len__`` | ``length`` | ``__len__`` | ``__len__`` | ++---------------------+------------------+------------------+-----------------------+ +| ``__iter__`` | ``__aiter__`` | ``__iter__`` | ``__iter__`` | ++---------------------+------------------+------------------+-----------------------+ +| ``__contains__`` | ``contains`` | ``__contains__`` | ``__contains__`` | ++---------------------+------------------+------------------+-----------------------+ +| ``__getitem__`` | ``getitem`` | ``__getitem__`` | ``__getitem__`` | ++---------------------+------------------+------------------+-----------------------+ +| ``__enter__`` | N/A | N/A | ``__enter__`` | ++---------------------+------------------+------------------+-----------------------+ +| ``__exit__`` | N/A | N/A | ``__exit__`` | ++---------------------+------------------+------------------+-----------------------+ +| ``group_keys`` | ``group_keys`` | ``group_keys`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``groups`` | ``groups`` | ``groups`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``array_keys`` | ``array_key`` | ``array_keys`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``arrays`` | ``arrays`` | ``arrays`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``visit`` | ? | ? | ``visit`` | ++---------------------+------------------+------------------+-----------------------+ +| ``visitkeys`` | ? | ? | ? | ++---------------------+------------------+------------------+-----------------------+ +| ``visitvalues`` | ? | ? | ? | ++---------------------+------------------+------------------+-----------------------+ +| ``visititems`` | ? | ? | ``visititems`` | ++---------------------+------------------+------------------+-----------------------+ +| ``tree`` | ``tree`` | ``tree`` | ``Both`` | ++---------------------+------------------+------------------+-----------------------+ +| ``create_group`` | ``create_group`` | ``create_group`` | ``create_group`` | ++---------------------+------------------+------------------+-----------------------+ +| ``require_group`` | N/A | N/A | ``require_group`` | ++---------------------+------------------+------------------+-----------------------+ +| ``create_groups`` | ? | ? | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``require_groups`` | ? | ? | ? | ++---------------------+------------------+------------------+-----------------------+ +| ``create_dataset`` | N/A | N/A | ``create_dataset`` | ++---------------------+------------------+------------------+-----------------------+ +| ``require_dataset`` | N/A | N/A | ``require_dataset`` | ++---------------------+------------------+------------------+-----------------------+ +| ``create`` | ``create_array`` | ``create_array`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``empty`` | ``empty`` | ``empty`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``zeros`` | ``zeros`` | ``zeros`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``ones`` | ``ones`` | ``ones`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``full`` | ``full`` | ``full`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``array`` | ``create_array`` | ``create_array`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``empty_like`` | ``empty_like`` | ``empty_like`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``zeros_like`` | ``zeros_like`` | ``zeros_like`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``ones_like`` | ``ones_like`` | ``ones_like`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``full_like`` | ``full_like`` | ``full_like`` | N/A | ++---------------------+------------------+------------------+-----------------------+ +| ``move`` | ``move`` | ``move`` | ``move`` | ++---------------------+------------------+------------------+-----------------------+ + +**``zarr.h5compat.Group``** +-- +Zarr-Python 2.\* made an attempt to align its API with that of +`h5py `__. With 3.0, we will +relax this alignment in favor of providing an explicit compatibility +module (``zarr.h5py_compat``). This module will expose the ``Group`` and +``Dataset`` APIs that map to Zarr-Python’s ``Group`` and ``Array`` +objects. + +Creation API +~~~~~~~~~~~~ + +Zarr-Python 2.\* bundles together the creation and serialization of Zarr +objects. Zarr-Python 3.\* will make it possible to create objects in +memory separate from serializing them. This will specifically enable +writing hierarchies of Zarr objects in a single batch step. For example: + +.. code:: python + + + arr1 = Array(shape=(10, 10), path="foo/bar", dtype="i4", store=store) + arr2 = Array(shape=(10, 10), path="foo/spam", dtype="f8", store=store) + + arr1.save() + arr2.save() + + # or equivalently + + zarr.save_many([arr1 ,arr2]) + +*Note: this batch creation API likely needs additional design effort +prior to implementation.* + +Plugin API +~~~~~~~~~~ + +Zarr V3 was designed to be extensible at multiple layers. Zarr-Python +will support these extensions through a combination of `Abstract Base +Classes `__ (ABCs) and +`Entrypoints `__. + +**ABCs** + +Zarr V3 will expose Abstract base classes for the following objects: + +- ``Store``, ``ReadStore``, ``ReadWriteStore``, ``ReadListStore``, and + ``ReadWriteListStore`` +- ``BaseArray``, ``SynchronousArray``, and ``AsynchronousArray`` +- ``BaseGroup``, ``SynchronousGroup``, and ``AsynchronousGroup`` +- ``Codec``, ``ArrayArrayCodec``, ``ArrayBytesCodec``, + ``BytesBytesCodec`` + +**Entrypoints** + +Lots more thinking here but the idea here is to provide entrypoints for +``data type``, ``chunk grid``, ``chunk key encoding``, ``codecs``, +``storage_transformers`` and ``stores``. These might look something +like: + +:: + + entry_points=""" + [zarr.codecs] + blosc_codec=codec_plugin:make_blosc_codec + zlib_codec=codec_plugin:make_zlib_codec + """ + +Python type hints and static analysis +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Target 100% Mypy coverage in 3.0 source. + +Observability +~~~~~~~~~~~~~ + +A persistent problem in Zarr-Python is diagnosing problems that span +many parts of the stack. To address this in 3.0, we will add a basic +logging framework that can be used to debug behavior at various levels +of the stack. We propose to add the separate loggers for the following +namespaces: + +- ``array`` +- ``group`` +- ``store`` +- ``codec`` + +These should be documented such that users know how to activate them and +developers know how to use them when developing extensions. + +Dependencies +~~~~~~~~~~~~ + +Today, Zarr-Python has the following required dependencies: + +.. code:: python + + dependencies = [ + 'asciitree', + 'numpy>=1.20,!=1.21.0', + 'fasteners', + 'numcodecs>=0.10.0', + ] + +What other dependencies should be considered? + +1. Attrs - Zarrita makes extensive use of the Attrs library +2. Fsspec - Zarrita has a hard dependency on Fsspec. This could be + easily relaxed though. + +Breaking changes relative to Zarr-Python 2.\* +--------------------------------------------- + +1. H5py compat moved to a stand alone module? +2. ``Group.__getitem__`` support moved to ``Group.members.__getitem__``? +3. Others? + +Open questions +-------------- + +1. How to treat V2 + + a. Note: Zarrita currently implements a separate ``V2Array`` and + ``V3Array`` classes. This feels less than ideal. + b. We could easily convert metadata from v2 to the V3 Array, but what + about writing? + c. Ideally, we don’t have completely separate code paths. But if its + too complicated to support both within one interface, its probably + better. + +2. How and when to remove the current implementation of V3. + + a. It’s hidden behind a hard-to-use feature flag so we probably don’t + need to do anything. + +3. How to model runtime configuration? +4. Which extensions belong in Zarr-Python and which belong in separate + packages? + + a. We don’t need to take a strong position on this here. It’s likely + that someone will want to put Sharding in. That will be useful to + develop in parallel because it will give us a good test case for + the plugin interface. + +Testing +------- + +Zarr-python 3.0 adds a major new dimension to Zarr: Async support. This +also comes with a compatibility risk, we will need to thoroughly test +support in key execution environments. Testing plan: - Reuse the +existing test suite for testing the ``v3`` API. - ``xfail`` tests that +expose breaking changes with ``3.0 - breaking change`` description. This +will help identify additional and/or unintentional breaking changes - +Rework tests that were only testing internal APIs. - Add a set of +functional / integration tests targeting real-world workflows in various +contexts (e.g. w/ Dask) + +Development process +------------------- + +Zarr-Python 3.0 will introduce a number of new APIs and breaking changes +to existing APIs. In order to facilitate ongoing support for Zarr-Python +2.*, we will take on the following development process: + +- Create a ``v3`` branch that can be use for developing the core + functionality apart from the ``main`` branch. This will allow us to + support ongoing work and bug fixes on the ``main`` branch. +- Put the ``3.0`` APIs inside a ``zarr.v3`` module. Imports from this + namespace will all be new APIs that users can develop and test + against once the ``v3`` branch is merged to ``main``. +- Kickstart the process by pulling in the current state of ``zarrita`` + - which has many of the features described in this design. +- Release a series of 2.\* releases with the ``v3`` namespace +- When ``v3`` is complete, move contents of ``v3`` to the package root + +**Milestones** + +Below are a set of specific milestones leading toward the completion of +this process. As work begins, we expect this list to grow in +specificity. + +1. Port current version of Zarrita to Zarr-Python +2. Formalize Async interface by splitting ``Array`` and ``Group`` + objects into Sync and Async versions +3. Implement “fancy” indexing operations on the ``AsyncArray`` +4. Implement an abstract base class for the ``Store`` interface and a + wrapper ``Store`` to make use of existing ``MutableMapping`` stores. +5. Rework the existing unit test suite to use the ``v3`` namespace. +6. Develop a plugin interface for extensions +7. Develop a set of functional and integration tests +8. Work with downstream libraries (Xarray, Dask, etc.) to test new APIs + +TODOs +----- + +The following subjects are not covered in detail above but perhaps +should be. Including them here so they are not forgotten. + +1. [Store] Should Zarr provide an API for caching objects after first + read/list/etc. Read only stores? +2. [Array] buffer protocol support +3. [Array] ``meta_array`` support +4. [Extensions] Define how Zarr-Python will consume the various plugin + types +5. [Misc] H5py compatibility requires a bit more work and a champion to + drive it forward. +6. [Misc] Define ``chunk_store`` API in 3.0 +7. [Misc] Define ``synchronizer`` API in 3.0 + +References +---------- + +1. `Zarr-Python + repository `__ +2. `Zarr core specification (version 3.0) — Zarr specs + documentation `__ +3. `Zarrita repository `__ +4. `Async-Zarr `__ +5. `Zarr-Python Discussion + Topic `__ diff --git a/v3-roadmap-and-design.md b/v3-roadmap-and-design.md deleted file mode 100644 index 696799e56..000000000 --- a/v3-roadmap-and-design.md +++ /dev/null @@ -1,429 +0,0 @@ -# Zarr Python Roadmap - -- Status: draft -- Author: Joe Hamman -- Created On: October 31, 2023 -- Input from: - - Davis Bennett / @d-v-b - - Norman Rzepka / @normanrz - - Deepak Cherian @dcherian - - Brian Davis / @monodeldiablo - - Oliver McCormack / @olimcc - - Ryan Abernathey / @rabernat - - Jack Kelly / @JackKelly - - Martin Durrant / @martindurant - -## Introduction - -This document lays out a design proposal for version 3.0 of the [Zarr-Python](https://zarr.readthedocs.io/en/stable/) package. A specific focus of the design is to bring Zarr-Python's API up to date with the [Zarr V3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html), with the hope of enabling the development of the many features and extensions that motivated the V3 Spec. The ideas presented here are expected to result in a major release of Zarr-Python (version 3.0) including significant a number of breaking API changes. -For clarity, “V3” will be used to describe the version of the Zarr specification and “3.0” will be used to describe the release tag of the Zarr-Python project. - -### Current status of V3 in Zarr-Python - -During the development of the V3 Specification, a [prototype implementation](https://github.com/zarr-developers/zarr-python/pull/898) was added to the Zarr-Python library. Since that implementation, the V3 spec evolved in significant ways and as a result, the Zarr-Python library is now out of sync with the approved spec. Downstream libraries (e.g. [Xarray](https://github.com/pydata/xarray)) have added support for this implementation and will need to migrate to the accepted spec when its available in Zarr-Python. - -## Goals - -- Provide a complete implementation of Zarr V3 through the Zarr-Python API -- Clear the way for exciting extensions / ZEPs (i.e. [sharding](https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/v1.0.html), [variable chunking](https://zarr.dev/zeps/draft/ZEP0003.html), etc.) -- Provide a developer API that can be used to implement and register V3 extensions -- Improve the performance of Zarr-Python by streamlining the interface between the Store layer and higher level APIs (e.g. Groups and Arrays) -- Clean up the internal and user facing APIs -- Improve code quality and robustness (e.g. achieve 100% type hint coverage) -- Align the Zarr-Python array API with the [array API Standard](https://data-apis.org/array-api/latest/) - -## Examples of what 3.0 will enable? -1. Reading and writing V3 spec-compliant groups and arrays -2. V3 extensions including sharding and variable chunking. -3. Improved performance by leveraging concurrency when creating/reading/writing to stores (imagine a `create_hierarchy(zarr_objects)` function). -4. User-developed extensions (e.g. storage-transformers) can be registered with Zarr-Python at runtime - -## Non-goals (of this document) - -- Implementation of any unaccepted Zarr V3 extensions -- Major revisions to the Zarr V3 spec - -## Requirements - -1. Read and write spec compliant V2 and V3 data -2. Limit unnecessary traffic to/from the store -3. Cleanly define the Array/Group/Store abstractions -4. Cleanly define how V2 will be supported going forward -5. Provide a clear roadmap to help users upgrade to 3.0 -6. Developer tools / hooks for registering extensions - -## Design - -### Async API - -Zarr-Python is an IO library. As such, supporting concurrent action against the storage layer is critical to achieving acceptable performance. The Zarr-Python 2 was not designed with asynchronous computation in mind and as a result has struggled to effectively leverage the benefits of concurrency. At one point, `getitems` and `setitems` support was added to the Zarr store model but that is only used for operating on a set of chunks in a single variable. - -With Zarr-Python 3.0, we have the opportunity to revisit this design. The proposal here is as follows: - -1. The `Store` interface will be entirely async. -2. On top of the async `Store` interface, we will provide an `AsyncArray` and `AsyncGroup` interface. -3. Finally, the primary user facing API will be synchronous `Array` and `Group` classes that wrap the async equivalents. - -**Examples** - -- **Store** - - ```python - class Store: - ... - async def get(self, key: str) -> bytes: - ... - async def get_partial_values(self, key_ranges: List[Tuple[str, Tuple[int, Optional[int]]]]) -> bytes: - ... - # (no sync interface here) - ``` -- **Array** - - ```python - class AsyncArray: - ... - - async def getitem(self, selection: Selection) -> np.ndarray: - # the core logic for getitem goes here - - class Array: - _async_array: AsyncArray - - def __getitem__(self, selection: Selection) -> np.ndarray: - return sync(self._async_array.getitem(selection)) - ``` -- **Group** - - ```python - class AsyncGroup: - ... - - async def create_group(self, path: str, **kwargs) -> AsyncGroup: - # the core logic for create_group goes here - - class Group: - _async_group: AsyncGroup - - def create_group(self, path: str, **kwargs) -> Group: - return sync(self._async_group.create_group(path, **kwargs)) - ``` -**Internal Synchronization API** - -With the `Store` and core `AsyncArray`/ `AsyncGroup` classes being predominantly async, Zarr-Python will need an internal API to provide a synchronous API. The proposal here is to use the approach in [fsspec](https://github.com/fsspec/filesystem_spec/blob/master/fsspec/asyn.py) to provide a high-level `sync` function that takes an `awaitable` and runs it in its managed IO Loop / thread. - -**FAQ** -1. Why two levels of Arrays/groups? - a. First, this is an intentional decision and departure from the current Zarrita implementation - b. The idea is that users rarely want to mix interfaces. Either they are working within an async context (currently quite rare) or they are in a typical synchronous context. - c. Splitting the two will allow us to clearly define behavior on the `AsyncObj` and simply wrap it in the `SyncObj`. -2. What if a store is only has a synchronous backend? - a. First off, this is expected to be a fairly rare occurrence. Most storage backends have async interfaces. - b. But in the event a storage backend doesn’t have a async interface, there is nothing wrong with putting synchronous code in `async` methods. There are approaches to enabling concurrent action through wrappers like AsyncIO's `loop.run_in_executor` ([ref 1](https://stackoverflow.com/questions/38865050/is-await-in-python3-cooperative-multitasking ), [ref 2](https://stackoverflow.com/a/43263397/732596), [ref 3](https://bbc.github.io/cloudfit-public-docs/asyncio/asyncio-part-5.html), [ref 4](https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.run_in_executor). -3. Will Zarr help manage the async contexts encouraged by some libraries (e.g. [AioBotoCore](https://aiobotocore.readthedocs.io/en/latest/tutorial.html#using-botocore))? - a. Many async IO libraries require entering an async context before interacting with the API. We expect some experimentation to be needed here but the initial design will follow something close to what fsspec does ([example in s3fs](https://github.com/fsspec/s3fs/blob/949442693ec940b35cda3420c17a864fbe426567/s3fs/core.py#L527)). -4. Why not provide a synchronous Store interface? - a. We could but this design is simpler. It would mean supporting it in the `AsyncGroup` and `AsyncArray` classes which, may be more trouble than its worth. Storage backends that do not have an async API will be encouraged to wrap blocking calls in an async wrapper (e.g. `loop.run_in_executor`). - -### Store API - -The `Store` API is specified directly in the V3 specification. All V3 stores should implement this abstract API, omitting Write and List support as needed. As described above, all stores will be expected to expose the required methods as async methods. - -**Example** - -```python -class ReadWriteStore: - ... - async def get(self, key: str) -> bytes: - ... - - async def get_partial_values(self, key_ranges: List[Tuple[str, int, int]) -> bytes: - ... - - async def set(self, key: str, value: Union[bytes, bytearray, memoryview]) -> None: - ... # required for writable stores - - async def set_partial_values(self, key_start_values: List[Tuple[str, int, Union[bytes, bytearray, memoryview]]]) -> None: - ... # required for writable stores - - async def list(self) -> List[str]: - ... # required for listable stores - - async def list_prefix(self, prefix: str) -> List[str]: - ... # required for listable stores - - async def list_dir(self, prefix: str) -> List[str]: - ... # required for listable stores - - # additional (optional methods) - async def getsize(self, prefix: str) -> int: - ... - - async def rename(self, src: str, dest: str) -> None - ... - -``` - -Recognizing that there are many Zarr applications today that rely on the `MutableMapping` interface supported by Zarr-Python 2, a wrapper store will be developed to allow existing stores to plug directly into this API. - -### Array API - -The user facing array interface will implement a subset of the [Array API Standard](https://data-apis.org/array-api/latest/). Most of the computational parts of the Array API Standard don’t fit into Zarr right now. That’s okay. What matters most is that we ensure we can give downstream applications a compliant API. - -*Note, Zarr already does most of this so this is more about formalizing the relationship than a substantial change in API.* - -| | Included | Not Included | Unknown / Maybe possible? | -| --- | --- | --- | --- | -| Attributes | `dtype` | `mT` | `device` | -| | `ndim` | `T` | | -| | `shape` | | | -| | `size` | | | -| Methods | `__getitem__` | `__array_namespace__` | `to_device` | -| | `__setitem__` | `__abs__` | `__bool__` | -| | `__eq__` | `__add__` | `__complex__` | -| | `__bool__` | `__and__` | `__dlpack__` | -| | | `__floordiv__` | `__dlpack_device__` | -| | | `__ge__` | `__float__` | -| | | `__gt__` | `__index__` | -| | | `__invert__` | `__int__` | -| | | `__le__` | | -| | | `__lshift__` | | -| | | `__lt__` | | -| | | `__matmul__` | | -| | | `__mod__` | | -| | | `__mul__` | | -| | | `__ne__` | | -| | | `__neg__` | | -| | | `__or__` | | -| | | `__pos__` | | -| | | `__pow__` | | -| | | `__rshift__` | | -| | | `__sub__` | | -| | | `__truediv__` | | -| | | `__xor__` | | -| Creation functions (`zarr.creation`) | `zeros` | | `arange` | -| | `zeros_like` | | `asarray` | -| | `ones` | | `eye` | -| | `ones_like` | | `from_dlpack` | -| | `full` | | `linspace` | -| | `full_like` | | `meshgrid` | -| | `empty` | | `tril` | -| | `empty_like` | | `triu` | - -In addition to the core array API defined above, the Array class should have the following Zarr specific properties: - -- `.metadata` (see Metadata Interface below) -- `.attrs` - (pull from metadata object) -- `.info` - (pull from existing property †) - -*† In Zarr-Python 2, the info property lists the store to identify initialized chunks. By default this will be turned off in 3.0 but will be configurable.* - -**Indexing** - -Zarr-Python currently supports `__getitem__` style indexing and the special `oindex` and `vindex` indexers. These are not part of the current Array API standard (see [data-apis/array-api\#669](https://github.com/data-apis/array-api/issues/669)) but they have been [proposed as a NEP](https://numpy.org/neps/nep-0021-advanced-indexing.html). Zarr-Python will maintain these in 3.0. - -We are also exploring a new high-level indexing API that will enabled optimized batch/concurrent loading of many chunks. We expect this to be important to enable performant loading of data in the context of sharding. See [this discussion](https://github.com/zarr-developers/zarr-python/discussions/1569) for more detail. - -Concurrent indexing across multiple arrays will be possible using the AsyncArray API. - -**Async and Sync Array APIs** - -Most the logic to support Zarr Arrays will live in the `AsyncArray` class. There are a few notable differences that should be called out. - -| Sync Method | Async Method | -| --- | --- | -| `__getitem__` | `getitem` | -| `__setitem__` | `setitem` | -| `__eq__` | `equals` | - -**Metadata interface** - -Zarr-Python 2.* closely mirrors the V2 spec metadata schema in the Array and Group classes. In 3.0, we plan to move the underlying metadata representation to a separate interface (e.g. `Array.metadata`). This interface will return either a `V2ArrayMetadata` or `V3ArrayMetadata` object (both will inherit from a parent `ArrayMetadataABC` class. The `V2ArrayMetadata` and `V3ArrayMetadata` classes will be responsible for producing valid JSON representations of their metadata, and yielding a consistent view to the `Array` or `Group` class. - -### Group API - -The main question is how closely we should follow the existing Zarr-Python implementation / `MutableMapping` interface. The table below shows the primary `Group` methods in Zarr-Python 2 and attempts to identify if and how they would be implemented in 3.0. - -| V2 Group Methods | `AsyncGroup` | `Group` | `h5py_compat.Group`` | -| --- | --- | --- | --- | -| `__len__` | `length` | `__len__` | `__len__` | -| `__iter__` | `__aiter__` | `__iter__` | `__iter__` | -| `__contains__` | `contains` | `__contains__` | `__contains__` | -| `__getitem__` | `getitem` | `__getitem__` | `__getitem__` | -| `__enter__` | N/A | N/A | `__enter__` | -| `__exit__` | N/A | N/A | `__exit__` | -| `group_keys` | `group_keys` | `group_keys` | N/A | -| `groups` | `groups` | `groups` | N/A | -| `array_keys` | `array_key` | `array_keys` | N/A | -| `arrays` | `arrays`* | `arrays` | N/A | -| `visit` | ? | ? | `visit` | -| `visitkeys` | ? | ? | ? | -| `visitvalues` | ? | ? | ? | -| `visititems` | ? | ? | `visititems` | -| `tree` | `tree` | `tree` | `Both` | -| `create_group` | `create_group` | `create_group` | `create_group` | -| `require_group` | N/A | N/A | `require_group` | -| `create_groups` | ? | ? | N/A | -| `require_groups` | ? | ? | ? | -| `create_dataset` | N/A | N/A | `create_dataset` | -| `require_dataset` | N/A | N/A | `require_dataset` | -| `create` | `create_array` | `create_array` | N/A | -| `empty` | `empty` | `empty` | N/A | -| `zeros` | `zeros` | `zeros` | N/A | -| `ones` | `ones` | `ones` | N/A | -| `full` | `full` | `full` | N/A | -| `array` | `create_array` | `create_array` | N/A | -| `empty_like` | `empty_like` | `empty_like` | N/A | -| `zeros_like` | `zeros_like` | `zeros_like` | N/A | -| `ones_like` | `ones_like` | `ones_like` | N/A | -| `full_like` | `full_like` | `full_like` | N/A | -| `move` | `move` | `move` | `move` | - -**`zarr.h5compat.Group`** - -Zarr-Python 2.* made an attempt to align its API with that of [h5py](https://docs.h5py.org/en/stable/index.html). With 3.0, we will relax this alignment in favor of providing an explicit compatibility module (`zarr.h5py_compat`). This module will expose the `Group` and `Dataset` APIs that map to Zarr-Python’s `Group` and `Array` objects. - -### Creation API - -Zarr-Python 2.* bundles together the creation and serialization of Zarr objects. Zarr-Python 3.* will make it possible to create objects in memory separate from serializing them. This will specifically enable writing hierarchies of Zarr objects in a single batch step. For example: - -```python - -arr1 = Array(shape=(10, 10), path="foo/bar", dtype="i4", store=store) -arr2 = Array(shape=(10, 10), path="foo/spam", dtype="f8", store=store) - -arr1.save() -arr2.save() - -# or equivalently - -zarr.save_many([arr1 ,arr2]) -``` - -*Note: this batch creation API likely needs additional design effort prior to implementation.* - -### Plugin API - -Zarr V3 was designed to be extensible at multiple layers. Zarr-Python will support these extensions through a combination of [Abstract Base Classes](https://docs.python.org/3/library/abc.html) (ABCs) and [Entrypoints](https://packaging.python.org/en/latest/specifications/entry-points/). - -**ABCs** - -Zarr V3 will expose Abstract base classes for the following objects: - -- `Store`, `ReadStore`, `ReadWriteStore`, `ReadListStore`, and `ReadWriteListStore` -- `BaseArray`, `SynchronousArray`, and `AsynchronousArray` -- `BaseGroup`, `SynchronousGroup`, and `AsynchronousGroup` -- `Codec`, `ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec` - -**Entrypoints** - -Lots more thinking here but the idea here is to provide entrypoints for `data type`, `chunk grid`, `chunk key encoding`, `codecs`, `storage_transformers` and `stores`. These might look something like: - -``` -entry_points=""" - [zarr.codecs] - blosc_codec=codec_plugin:make_blosc_codec - zlib_codec=codec_plugin:make_zlib_codec -""" -``` - -### Python type hints and static analysis - -Target 100% Mypy coverage in 3.0 source. - -### Observability - -A persistent problem in Zarr-Python is diagnosing problems that span many parts of the stack. To address this in 3.0, we will add a basic logging framework that can be used to debug behavior at various levels of the stack. We propose to add the separate loggers for the following namespaces: - -- `array` -- `group` -- `store` -- `codec` - -These should be documented such that users know how to activate them and developers know how to use them when developing extensions. - -### Dependencies - -Today, Zarr-Python has the following required dependencies: - -```python -dependencies = [ - 'asciitree', - 'numpy>=1.20,!=1.21.0', - 'fasteners', - 'numcodecs>=0.10.0', -] -``` - -What other dependencies should be considered? - -1. Attrs - Zarrita makes extensive use of the Attrs library -2. Fsspec - Zarrita has a hard dependency on Fsspec. This could be easily relaxed though. - -## Breaking changes relative to Zarr-Python 2.* - -1. H5py compat moved to a stand alone module? -2. `Group.__getitem__` support moved to `Group.members.__getitem__`? -3. Others? - -## Open questions - -1. How to treat V2 - a. Note: Zarrita currently implements a separate `V2Array` and `V3Array` classes. This feels less than ideal. - b. We could easily convert metadata from v2 to the V3 Array, but what about writing? - c. Ideally, we don’t have completely separate code paths. But if its too complicated to support both within one interface, its probably better. -2. How and when to remove the current implementation of V3. - a. It's hidden behind a hard-to-use feature flag so we probably don't need to do anything. -4. How to model runtime configuration? -5. Which extensions belong in Zarr-Python and which belong in separate packages? - a. We don't need to take a strong position on this here. It's likely that someone will want to put Sharding in. That will be useful to develop in parallel because it will give us a good test case for the plugin interface. - -## Testing - -Zarr-python 3.0 adds a major new dimension to Zarr: Async support. This also comes with a compatibility risk, we will need to thoroughly test support in key execution environments. Testing plan: -- Reuse the existing test suite for testing the `v3` API. - - `xfail` tests that expose breaking changes with `3.0 - breaking change` description. This will help identify additional and/or unintentional breaking changes - - Rework tests that were only testing internal APIs. -- Add a set of functional / integration tests targeting real-world workflows in various contexts (e.g. w/ Dask) - -## Development process - -Zarr-Python 3.0 will introduce a number of new APIs and breaking changes to existing APIs. In order to facilitate ongoing support for Zarr-Python 2.*, we will take on the following development process: - -- Create a `v3` branch that can be use for developing the core functionality apart from the `main` branch. This will allow us to support ongoing work and bug fixes on the `main` branch. -- Put the `3.0` APIs inside a `zarr.v3` module. Imports from this namespace will all be new APIs that users can develop and test against once the `v3` branch is merged to `main`. -- Kickstart the process by pulling in the current state of `zarrita` - which has many of the features described in this design. -- Release a series of 2.* releases with the `v3` namespace -- When `v3` is complete, move contents of `v3` to the package root - -**Milestones** - -Below are a set of specific milestones leading toward the completion of this process. As work begins, we expect this list to grow in specificity. - -1. Port current version of Zarrita to Zarr-Python -2. Formalize Async interface by splitting `Array` and `Group` objects into Sync and Async versions -4. Implement "fancy" indexing operations on the `AsyncArray` -6. Implement an abstract base class for the `Store` interface and a wrapper `Store` to make use of existing `MutableMapping` stores. -7. Rework the existing unit test suite to use the `v3` namespace. -8. Develop a plugin interface for extensions -9. Develop a set of functional and integration tests -10. Work with downstream libraries (Xarray, Dask, etc.) to test new APIs - -## TODOs - -The following subjects are not covered in detail above but perhaps should be. Including them here so they are not forgotten. - -1. [Store] Should Zarr provide an API for caching objects after first read/list/etc. Read only stores? -2. [Array] buffer protocol support -3. [Array] `meta_array` support -4. [Extensions] Define how Zarr-Python will consume the various plugin types -5. [Misc] H5py compatibility requires a bit more work and a champion to drive it forward. -6. [Misc] Define `chunk_store` API in 3.0 -7. [Misc] Define `synchronizer` API in 3.0 - -## References - -1. [Zarr-Python repository](https://github.com/zarr-developers/zarr-python) -2. [Zarr core specification (version 3.0) — Zarr specs documentation](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#) -3. [Zarrita repository](https://github.com/scalableminds/zarrita) -4. [Async-Zarr](https://github.com/martindurant/async-zarr) -5. [Zarr-Python Discussion Topic](https://github.com/zarr-developers/zarr-python/discussions/1569) From 1197bbe3f8d992c7abc961fc7da59be94b3869dd Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Tue, 15 Oct 2024 20:21:19 +0200 Subject: [PATCH 11/17] Multiple imports for an import name (#2367) Co-authored-by: Joe Hamman --- src/zarr/abc/store.py | 1 - src/zarr/storage/remote.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index 85e335089..40c8129af 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -2,7 +2,6 @@ from abc import ABC, abstractmethod from asyncio import gather -from types import TracebackType from typing import TYPE_CHECKING, NamedTuple, Protocol, runtime_checkable if TYPE_CHECKING: diff --git a/src/zarr/storage/remote.py b/src/zarr/storage/remote.py index 6945f129e..0a0ec7f7c 100644 --- a/src/zarr/storage/remote.py +++ b/src/zarr/storage/remote.py @@ -5,7 +5,6 @@ import fsspec from zarr.abc.store import ByteRangeRequest, Store -from zarr.core.buffer import Buffer from zarr.storage.common import _dereference_path if TYPE_CHECKING: From da675fe7b64beb83f0c27395b0ed00f4dddfdfbf Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Tue, 15 Oct 2024 20:30:29 +0200 Subject: [PATCH 12/17] Enforce ruff/pycodestyle warnings (W) (#2369) * Apply ruff/pycodestyle rule W291 W291 Trailing whitespace * Enforce ruff/pycodestyle warnings (W) It looks like `ruff format` does not catch all trailing spaces. --------- Co-authored-by: Joe Hamman --- pyproject.toml | 1 + src/zarr/registry.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 059fa8fdb..9e890f3a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -223,6 +223,7 @@ extend-select = [ "TCH", # flake8-type-checking "TRY", # tryceratops "UP", # pyupgrade + "W", # pycodestyle warnings ] ignore = [ "ANN003", diff --git a/src/zarr/registry.py b/src/zarr/registry.py index fcea834f0..12b073801 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -48,9 +48,9 @@ def register(self, cls: type[T]) -> None: __ndbuffer_registry: Registry[NDBuffer] = Registry() """ -The registry module is responsible for managing implementations of codecs, pipelines, buffers and ndbuffers and -collecting them from entrypoints. -The implementation used is determined by the config +The registry module is responsible for managing implementations of codecs, +pipelines, buffers and ndbuffers and collecting them from entrypoints. +The implementation used is determined by the config. """ From 41a567e3029093b7e82f5dd490b335fb7fd1ae20 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Wed, 16 Oct 2024 00:04:10 +0200 Subject: [PATCH 13/17] Apply ruff/pycodestyle preview rule E262 (#2370) E262 Inline comment should start with `# ` Co-authored-by: Joe Hamman --- tests/v3/test_indexing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index 0ea9cda39..b3a199068 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -683,7 +683,7 @@ def test_get_orthogonal_selection_2d(store: StorePath) -> None: with pytest.raises(IndexError): z.get_orthogonal_selection(selection_2d_bad) # type: ignore[arg-type] with pytest.raises(IndexError): - z.oindex[selection_2d_bad] # type: ignore[index] + z.oindex[selection_2d_bad] # type: ignore[index] def _test_get_orthogonal_selection_3d( From 77976c8b7db64af1bf003dca98a9ae87e58ffb85 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Wed, 16 Oct 2024 00:19:39 +0200 Subject: [PATCH 14/17] Fix typo (#2382) Co-authored-by: Joe Hamman --- src/zarr/core/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zarr/core/sync.py b/src/zarr/core/sync.py index 8c5bc9c39..f7d452947 100644 --- a/src/zarr/core/sync.py +++ b/src/zarr/core/sync.py @@ -133,7 +133,7 @@ def sync( finished, unfinished = wait([future], return_when=asyncio.ALL_COMPLETED, timeout=timeout) if len(unfinished) > 0: - raise TimeoutError(f"Coroutine {coro} failed to finish in within {timeout} s") + raise TimeoutError(f"Coroutine {coro} failed to finish within {timeout} s") assert len(finished) == 1 return_result = next(iter(finished)).result() From 428d6da9cd18581835b5ff90b6ec71d32228db8a Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Wed, 16 Oct 2024 00:21:27 +0200 Subject: [PATCH 15/17] Imported name is not used anywhere in the module (#2379) --- docs/conf.py | 1 - tests/v3/test_array.py | 1 - tests/v3/test_v2.py | 2 -- 3 files changed, 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0a328ac25..72c6130a1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -21,7 +21,6 @@ from importlib.metadata import version as get_version -import zarr import sphinx # If extensions (or modules to document with autodoc) are in another directory, diff --git a/tests/v3/test_array.py b/tests/v3/test_array.py index b558c826d..829a04d30 100644 --- a/tests/v3/test_array.py +++ b/tests/v3/test_array.py @@ -6,7 +6,6 @@ import pytest import zarr.api.asynchronous -import zarr.storage from zarr import Array, AsyncArray, Group from zarr.codecs import BytesCodec, VLenBytesCodec from zarr.core.array import chunks_initialized diff --git a/tests/v3/test_v2.py b/tests/v3/test_v2.py index 729ed0533..439b15b64 100644 --- a/tests/v3/test_v2.py +++ b/tests/v3/test_v2.py @@ -9,8 +9,6 @@ from numcodecs.blosc import Blosc import zarr -import zarr.core.buffer.cpu -import zarr.core.metadata import zarr.storage from zarr import Array from zarr.storage import MemoryStore, StorePath From 879d3d7adab4adea1f7c7f9c905d200d9f2cf852 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Wed, 16 Oct 2024 00:44:01 +0200 Subject: [PATCH 16/17] Missing mandatory keyword argument `shape` (#2376) --- src/zarr/core/group.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index e85057e2f..e25f70eef 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -932,7 +932,11 @@ async def create_array( @deprecated("Use AsyncGroup.create_array instead.") async def create_dataset( - self, name: str, **kwargs: Any + self, + name: str, + *, + shape: ShapeLike, + **kwargs: Any, ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: """Create an array. @@ -943,6 +947,8 @@ async def create_dataset( ---------- name : str Array name. + shape : int or tuple of ints + Array shape. kwargs : dict Additional arguments passed to :func:`zarr.AsyncGroup.create_array`. @@ -953,7 +959,7 @@ async def create_dataset( .. deprecated:: 3.0.0 The h5py compatibility methods will be removed in 3.1.0. Use `AsyncGroup.create_array` instead. """ - return await self.create_array(name, **kwargs) + return await self.create_array(name, shape=shape, **kwargs) @deprecated("Use AsyncGroup.require_array instead.") async def require_dataset( @@ -1621,7 +1627,13 @@ def create_dataset(self, name: str, **kwargs: Any) -> Array: return Array(self._sync(self._async_group.create_dataset(name, **kwargs))) @deprecated("Use Group.require_array instead.") - def require_dataset(self, name: str, **kwargs: Any) -> Array: + def require_dataset( + self, + name: str, + *, + shape: ShapeLike, + **kwargs: Any, + ) -> Array: """Obtain an array, creating if it doesn't exist. Arrays are known as "datasets" in HDF5 terminology. For compatibility @@ -1648,9 +1660,15 @@ def require_dataset(self, name: str, **kwargs: Any) -> Array: .. deprecated:: 3.0.0 The h5py compatibility methods will be removed in 3.1.0. Use `Group.require_array` instead. """ - return Array(self._sync(self._async_group.require_array(name, **kwargs))) + return Array(self._sync(self._async_group.require_array(name, shape=shape, **kwargs))) - def require_array(self, name: str, **kwargs: Any) -> Array: + def require_array( + self, + name: str, + *, + shape: ShapeLike, + **kwargs: Any, + ) -> Array: """Obtain an array, creating if it doesn't exist. @@ -1672,7 +1690,7 @@ def require_array(self, name: str, **kwargs: Any) -> Array: ------- a : Array """ - return Array(self._sync(self._async_group.require_array(name, **kwargs))) + return Array(self._sync(self._async_group.require_array(name, shape=shape, **kwargs))) def empty(self, *, name: str, shape: ChunkCoords, **kwargs: Any) -> Array: return Array(self._sync(self._async_group.empty(name=name, shape=shape, **kwargs))) From 818746c7351ffd3667937c4a3cb99f939370deca Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Wed, 16 Oct 2024 00:44:18 +0200 Subject: [PATCH 17/17] Update ruff rules to ignore (#2374) Co-authored-by: Joe Hamman --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 9e890f3a7..3dc46a0f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -231,6 +231,7 @@ ignore = [ "ANN102", "ANN401", "PT004", # deprecated + "PT005", # deprecated "PT011", # TODO: apply this rule "PT012", # TODO: apply this rule "PYI013", @@ -238,6 +239,8 @@ ignore = [ "RET506", "RUF005", "TRY003", + "UP027", # deprecated + "UP038", # https://github.com/astral-sh/ruff/issues/7871 # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules "W191", "E111",