Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

First check for compatibility for zarr version 3 in storage backend #88

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 18 additions & 4 deletions modelrunner/storage/backend/zarr.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,18 @@
import numpy as np
import zarr
from numpy.typing import ArrayLike, DTypeLike
from zarr._storage.store import Store

try:
# try import path from zarr version 3
from zarr.abc.store import Store

is_zarr2 = False

Check warning on line 22 in modelrunner/storage/backend/zarr.py

View check run for this annotation

Codecov / codecov/patch

modelrunner/storage/backend/zarr.py#L22

Added line #L22 was not covered by tests

except ImportError:
# try import path from zarr version 2
from zarr._storage.store import Store

is_zarr2 = True

from ..access_modes import ModeType
from ..attributes import AttrsLike
Expand Down Expand Up @@ -55,7 +66,10 @@
if self.mode.file_mode == "r":
self._logger.info("DirectoryStore is always opened writable")

self._store = zarr.DirectoryStore(path)
if is_zarr2:
self._store = zarr.DirectoryStore(path)
else:
self._store = zarr.LocalStore(path)

Check warning on line 72 in modelrunner/storage/backend/zarr.py

View check run for this annotation

Codecov / codecov/patch

modelrunner/storage/backend/zarr.py#L72

Added line #L72 was not covered by tests

elif path.suffix == ".zip":
# create a ZipStore
Expand All @@ -65,7 +79,7 @@
path.unlink()
self._store = zarr.storage.ZipStore(path, mode=file_mode)

elif path.suffix == ".sqldb":
elif is_zarr2 and path.suffix == ".sqldb":
# create a SQLiteStore
if self.mode.file_mode == "w" and path.exists():
self._logger.info("Delete file `%s`", path)
Expand Down Expand Up @@ -135,7 +149,7 @@
return self._root.keys() # type: ignore

def is_group(self, loc: Sequence[str], *, ignore_cls: bool = False) -> bool:
return isinstance(self[loc], zarr.hierarchy.Group)
return isinstance(self[loc], zarr.Group)

def _create_group(self, loc: Sequence[str]) -> None:
parent, name = self._get_parent(loc)
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ all = [
"h5py>=3.5",
"pandas>=1.3",
"PyYAML>=5",
"zarr>=2,<3",
"zarr>=2,<4",
]
test = [
"black>=19",
Expand Down
2 changes: 1 addition & 1 deletion requirements_full.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
-r requirements.txt
h5py>=3.5
pandas>=1.3
zarr>=2,<3
zarr>=2,<4
6 changes: 4 additions & 2 deletions tests/helpers/storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ def storage_extensions(
dot (bool):
Indicates whether the returned extensions are prepended with a dot (`.`)
exclude (list, optional):
Extensions (without dots) that should be explicitely excluded
Extensions (without dots) that should be explicitly excluded

Returns:
sequence: sorted list of extensions
Expand All @@ -53,10 +53,12 @@ def storage_extensions(
if module_available("h5py"):
exts.append("hdf")
if module_available("zarr"):
from modelrunner.storage.backend.zarr import is_zarr2

exts.append("zip")
if incl_folder:
exts.extend(["", "zarr"])
if module_available("sqlite3"):
if is_zarr2 and module_available("sqlite3"):
exts.append("sqldb")

if dot:
Expand Down
Loading