From ba2b1b5f6d97b22d7129ae4c3743e9a71f253da9 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Mon, 27 Jan 2025 20:39:37 -0500 Subject: [PATCH 1/5] Support Zarr-Python 3 --- src/titiler/xarray/pyproject.toml | 15 ++++- src/titiler/xarray/titiler/xarray/io.py | 81 +++++-------------------- 2 files changed, 26 insertions(+), 70 deletions(-) diff --git a/src/titiler/xarray/pyproject.toml b/src/titiler/xarray/pyproject.toml index e0e1e2112..e7f29cd3b 100644 --- a/src/titiler/xarray/pyproject.toml +++ b/src/titiler/xarray/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ [project.optional-dependencies] full = [ - "zarr>=2,<3", + "zarr", "h5netcdf", "fsspec", "s3fs", @@ -46,7 +46,7 @@ full = [ "gcsfs", ] minimal = [ - "zarr>=2,<3", + "zarr", "h5netcdf", "fsspec", ] @@ -64,7 +64,16 @@ test = [ "pytest-cov", "pytest-asyncio", "httpx", - "zarr>=2,<3", + "zarr", + "h5netcdf", + "fsspec", +] +test-zarr-v2 = [ + "pytest", + "pytest-cov", + "pytest-asyncio", + "httpx", + "zarr<3.0", "h5netcdf", "fsspec", ] diff --git a/src/titiler/xarray/titiler/xarray/io.py b/src/titiler/xarray/titiler/xarray/io.py index 370fa5e97..0b619324b 100644 --- a/src/titiler/xarray/titiler/xarray/io.py +++ b/src/titiler/xarray/titiler/xarray/io.py @@ -9,6 +9,7 @@ from morecantile import TileMatrixSet from rio_tiler.constants import WEB_MERCATOR_TMS from rio_tiler.io.xarray import XarrayReader +from xarray.namedarray.utils import module_available def xarray_open_dataset( # noqa: C901 @@ -29,21 +30,6 @@ def xarray_open_dataset( # noqa: C901 """ import fsspec # noqa - try: - import gcsfs - except ImportError: # pragma: nocover - gcsfs = None # type: ignore - - try: - import s3fs - except ImportError: # pragma: nocover - s3fs = None # type: ignore - - try: - import aiohttp - except ImportError: # pragma: nocover - aiohttp = None # type: ignore - try: import h5netcdf except ImportError: # pragma: nocover @@ -66,55 +52,8 @@ def xarray_open_dataset( # noqa: C901 else: assert zarr is not None, "'zarr' must be installed to read Zarr dataset" - xr_engine = "zarr" - - if protocol in ["", "file"]: - filesystem = fsspec.filesystem(protocol) # type: ignore - file_handler = ( - filesystem.open(src_path) - if xr_engine == "h5netcdf" - else filesystem.get_mapper(src_path) - ) - - elif protocol == "s3": - assert ( - s3fs is not None - ), "'aiohttp' must be installed to read dataset stored online" - - s3_filesystem = s3fs.S3FileSystem() - file_handler = ( - s3_filesystem.open(src_path) - if xr_engine == "h5netcdf" - else s3fs.S3Map(root=src_path, s3=s3_filesystem) - ) - - elif protocol == "gs": - assert ( - gcsfs is not None - ), "'gcsfs' must be installed to read dataset stored in Google Cloud Storage" - - gcs_filesystem = gcsfs.GCSFileSystem() - file_handler = ( - gcs_filesystem.open(src_path) - if xr_engine == "h5netcdf" - else gcs_filesystem.get_mapper(root=src_path) - ) - - elif protocol in ["http", "https"]: - assert ( - aiohttp is not None - ), "'aiohttp' must be installed to read dataset stored online" - - filesystem = fsspec.filesystem(protocol) # type: ignore - file_handler = ( - filesystem.open(src_path) - if xr_engine == "h5netcdf" - else filesystem.get_mapper(src_path) - ) - - else: - raise ValueError(f"Unsupported protocol: {protocol}, for {src_path}") + _zarr_v3 = module_available("zarr", minversion="3.0") # Arguments for xarray.open_dataset # Default args @@ -135,13 +74,21 @@ def xarray_open_dataset( # noqa: C901 "lock": False, } ) - - ds = xarray.open_dataset(file_handler, **xr_open_args) + fs = fsspec.filesystem(protocol) + ds = xarray.open_dataset(fs.open(src_path), **xr_open_args) # Fallback to Zarr else: - ds = xarray.open_zarr(file_handler, **xr_open_args) - + if _zarr_v3: + if protocol == "file": + store = zarr.storage.LocalStore(parsed.path, read_only=True) + else: + fs = fsspec.filesystem(protocol, storage_options={"asynchronous": True}) + store = zarr.storage.FsspecStore(fs, path=src_path, read_only=True) + ds = xarray.open_zarr(store, **xr_open_args) + else: + fs = fsspec.filesystem(protocol) + ds = xarray.open_zarr(fs.get_mapper(src_path), **xr_open_args) return ds From a37907a49c1669981dc1b2af314da12fb0667e9b Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Tue, 28 Jan 2025 09:37:21 -0500 Subject: [PATCH 2/5] Remove test-zarr-v2 deps --- src/titiler/xarray/pyproject.toml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/titiler/xarray/pyproject.toml b/src/titiler/xarray/pyproject.toml index e7f29cd3b..e451c9653 100644 --- a/src/titiler/xarray/pyproject.toml +++ b/src/titiler/xarray/pyproject.toml @@ -68,15 +68,6 @@ test = [ "h5netcdf", "fsspec", ] -test-zarr-v2 = [ - "pytest", - "pytest-cov", - "pytest-asyncio", - "httpx", - "zarr<3.0", - "h5netcdf", - "fsspec", -] [project.urls] Homepage = "https://developmentseed.org/titiler/" From 8e353f45c3bf5ae08f72ccba1a699f446690d68a Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Tue, 28 Jan 2025 11:18:10 -0500 Subject: [PATCH 3/5] Update src/titiler/xarray/titiler/xarray/io.py Co-authored-by: Vincent Sarago --- src/titiler/xarray/titiler/xarray/io.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/titiler/xarray/titiler/xarray/io.py b/src/titiler/xarray/titiler/xarray/io.py index 0b619324b..a5546172e 100644 --- a/src/titiler/xarray/titiler/xarray/io.py +++ b/src/titiler/xarray/titiler/xarray/io.py @@ -79,16 +79,17 @@ def xarray_open_dataset( # noqa: C901 # Fallback to Zarr else: - if _zarr_v3: + if module_available("zarr", minversion="3.0"): if protocol == "file": store = zarr.storage.LocalStore(parsed.path, read_only=True) else: fs = fsspec.filesystem(protocol, storage_options={"asynchronous": True}) store = zarr.storage.FsspecStore(fs, path=src_path, read_only=True) - ds = xarray.open_zarr(store, **xr_open_args) + else: - fs = fsspec.filesystem(protocol) - ds = xarray.open_zarr(fs.get_mapper(src_path), **xr_open_args) + store = fsspec.filesystem(protocol).get_mapper(src_path) + + ds = xarray.open_zarr(store, **xr_open_args) return ds From e081d0f1452e4a950d8e9c13c939f0c5627956aa Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Tue, 28 Jan 2025 11:18:43 -0500 Subject: [PATCH 4/5] Update io.py --- src/titiler/xarray/titiler/xarray/io.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/titiler/xarray/titiler/xarray/io.py b/src/titiler/xarray/titiler/xarray/io.py index a5546172e..7efd0650d 100644 --- a/src/titiler/xarray/titiler/xarray/io.py +++ b/src/titiler/xarray/titiler/xarray/io.py @@ -53,7 +53,6 @@ def xarray_open_dataset( # noqa: C901 else: assert zarr is not None, "'zarr' must be installed to read Zarr dataset" xr_engine = "zarr" - _zarr_v3 = module_available("zarr", minversion="3.0") # Arguments for xarray.open_dataset # Default args From 4961ef2996dddb619392361944ba775138fb9a56 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Wed, 29 Jan 2025 11:25:31 +0100 Subject: [PATCH 5/5] update changelog --- CHANGES.md | 6 ++++++ src/titiler/xarray/titiler/xarray/io.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 7515ca276..5971b89dc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,11 @@ # Release Notes +## Unreleased + +### titiler.xarray + +* Support Zarr-Python >=3 (author @maxrjones, https://github.com/developmentseed/titiler/pull/1082) + ## 0.21.0 (2025-01-24) ### Misc diff --git a/src/titiler/xarray/titiler/xarray/io.py b/src/titiler/xarray/titiler/xarray/io.py index 7efd0650d..830c0e520 100644 --- a/src/titiler/xarray/titiler/xarray/io.py +++ b/src/titiler/xarray/titiler/xarray/io.py @@ -84,7 +84,7 @@ def xarray_open_dataset( # noqa: C901 else: fs = fsspec.filesystem(protocol, storage_options={"asynchronous": True}) store = zarr.storage.FsspecStore(fs, path=src_path, read_only=True) - + else: store = fsspec.filesystem(protocol).get_mapper(src_path)