From e35c4fcfcf91e2f693dd789379657eacddc4332d Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Fri, 18 Oct 2024 17:27:04 +0200 Subject: [PATCH 1/2] move v3/tests to tests (#2396) * move v3/tests to tests and fix various mypy issues * test(ci): change branch name in v3 workflows (#2368) * Use lazy % formatting in logging functions (#2366) * Use lazy % formatting in logging functions * f-string should be more efficient * Space before unit symbol From "SI Unit rules and style conventions": https://physics.nist.gov/cuu/Units/checklist.html There is a space between the numerical value and unit symbol, even when the value is used in an adjectival sense, except in the case of superscript units for plane angle. * Enforce ruff/flake8-logging-format rules (G) --------- Co-authored-by: Joe Hamman * Move roadmap and v3-design documument to docs (#2354) * move roadmap to docs * formatting and minor copy editing * Multiple imports for an import name (#2367) Co-authored-by: Joe Hamman * Enforce ruff/pycodestyle warnings (W) (#2369) * Apply ruff/pycodestyle rule W291 W291 Trailing whitespace * Enforce ruff/pycodestyle warnings (W) It looks like `ruff format` does not catch all trailing spaces. --------- Co-authored-by: Joe Hamman * Apply ruff/pycodestyle preview rule E262 (#2370) E262 Inline comment should start with `# ` Co-authored-by: Joe Hamman * Fix typo (#2382) Co-authored-by: Joe Hamman * Imported name is not used anywhere in the module (#2379) * Missing mandatory keyword argument `shape` (#2376) * Update ruff rules to ignore (#2374) Co-authored-by: Joe Hamman * Docstrings for arraymodule (#2276) * start to docstrings for arraymodule * incorporating toms edits, overriding mypy error... * fix attrs * Update src/zarr/core/array.py Co-authored-by: Sanket Verma * fix store -> storage * remove properties from asyncarray docstring --------- Co-authored-by: Sanket Verma Co-authored-by: Joe Hamman * fix/normalize storage paths (#2384) * bring in path normalization function from v2, and add a failing test * rephrase comment * simplify storepath creation * Update tests/v3/test_api.py Co-authored-by: Joe Hamman * refactor: remove redundant zarr format fixture * replace assertion with an informative error message * fix incorrect path concatenation in make_store_path, and refactor store_path tests * remove upath import because we don't need it * apply suggestions from code review --------- Co-authored-by: Joe Hamman * Enforce ruff/flake8-pyi rule PYI013 (#2389) PYI013 Non-empty class body must not contain `...` Note that documentation is enough to fill the class body. * deps: remove fasteners from list of dependencies (#2386) * Enforce ruff/flake8-annotations rule ANN003 (#2388) ANN003 Missing type annotation Co-authored-by: Joe Hamman * Enforce ruff/Perflint rules (PERF) (#2372) * Apply ruff/Perflint rule PERF401 PERF401 Use a list comprehension to create a transformed list * Enforce ruff/Perflint rules (PERF) * chore: update package maintainers (#2387) * chore: update package maintainers * Update pyproject.toml Co-authored-by: David Stansby --------- Co-authored-by: David Stansby * Fixed consolidated Group getitem with multi-part key (#2363) * Fixed consolidated Group getitem with multi-part key This fixes `Group.__getitem__` when indexing with a key like 'subgroup/array'. The basic idea is to rewrite the indexing operation as `group['subgroup']['array']` by splitting the key and doing each operation independently. Closes https://github.com/zarr-developers/zarr-python/issues/2358 --------- Co-authored-by: Joe Hamman * chore: add python 3.13 to ci / pyproject.toml (#2385) * chore: add python 3.13 to ci / pyproject.toml * update hatch matrix * remove references to dead test dir in pyproject.toml * remove v3 reference in test --------- Co-authored-by: Joe Hamman Co-authored-by: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Co-authored-by: Emma Marshall <55526386+e-marshall@users.noreply.github.com> Co-authored-by: Sanket Verma Co-authored-by: David Stansby Co-authored-by: Tom Augspurger --- pyproject.toml | 27 +++++++++---------- src/zarr/testing/utils.py | 17 +++++++++--- tests/{v3 => }/conftest.py | 0 .../entry_points.txt | 0 .../package_with_entrypoint/__init__.py | 0 tests/{v3 => }/test_api.py | 0 tests/{v3 => }/test_array.py | 0 tests/{v3 => }/test_attributes.py | 0 tests/{v3 => }/test_buffer.py | 0 tests/{v3 => }/test_chunk_grids.py | 0 tests/{v3 => }/test_codec_entrypoints.py | 0 tests/{v3 => test_codecs}/__init__.py | 0 tests/{v3 => }/test_codecs/test_blosc.py | 0 tests/{v3 => }/test_codecs/test_codecs.py | 0 tests/{v3 => }/test_codecs/test_endian.py | 0 tests/{v3 => }/test_codecs/test_gzip.py | 0 tests/{v3 => }/test_codecs/test_sharding.py | 0 tests/{v3 => }/test_codecs/test_transpose.py | 0 tests/{v3 => }/test_codecs/test_vlen.py | 0 tests/{v3 => }/test_codecs/test_zstd.py | 0 tests/{v3 => }/test_common.py | 0 tests/{v3 => }/test_config.py | 2 +- tests/{v3 => }/test_group.py | 0 tests/{v3 => }/test_indexing.py | 0 .../test_codecs => test_metadata}/__init__.py | 0 .../test_metadata/test_consolidated.py | 0 tests/{v3 => }/test_metadata/test_v2.py | 0 tests/{v3 => }/test_metadata/test_v3.py | 0 tests/{v3 => }/test_properties.py | 0 .../test_metadata => test_store}/__init__.py | 0 tests/{v3 => }/test_store/test_core.py | 0 tests/{v3 => }/test_store/test_local.py | 0 tests/{v3 => }/test_store/test_logging.py | 0 tests/{v3 => }/test_store/test_memory.py | 0 tests/{v3 => }/test_store/test_remote.py | 0 .../test_store/test_stateful_store.py | 0 tests/{v3 => }/test_store/test_zip.py | 0 tests/{v3 => }/test_sync.py | 0 tests/{v3 => }/test_v2.py | 0 tests/v3/test_store/__init__.py | 0 40 files changed, 27 insertions(+), 19 deletions(-) rename tests/{v3 => }/conftest.py (100%) rename tests/{v3 => }/package_with_entrypoint-0.1.dist-info/entry_points.txt (100%) rename tests/{v3 => }/package_with_entrypoint/__init__.py (100%) rename tests/{v3 => }/test_api.py (100%) rename tests/{v3 => }/test_array.py (100%) rename tests/{v3 => }/test_attributes.py (100%) rename tests/{v3 => }/test_buffer.py (100%) rename tests/{v3 => }/test_chunk_grids.py (100%) rename tests/{v3 => }/test_codec_entrypoints.py (100%) rename tests/{v3 => test_codecs}/__init__.py (100%) rename tests/{v3 => }/test_codecs/test_blosc.py (100%) rename tests/{v3 => }/test_codecs/test_codecs.py (100%) rename tests/{v3 => }/test_codecs/test_endian.py (100%) rename tests/{v3 => }/test_codecs/test_gzip.py (100%) rename tests/{v3 => }/test_codecs/test_sharding.py (100%) rename tests/{v3 => }/test_codecs/test_transpose.py (100%) rename tests/{v3 => }/test_codecs/test_vlen.py (100%) rename tests/{v3 => }/test_codecs/test_zstd.py (100%) rename tests/{v3 => }/test_common.py (100%) rename tests/{v3 => }/test_config.py (98%) rename tests/{v3 => }/test_group.py (100%) rename tests/{v3 => }/test_indexing.py (100%) rename tests/{v3/test_codecs => test_metadata}/__init__.py (100%) rename tests/{v3 => }/test_metadata/test_consolidated.py (100%) rename tests/{v3 => }/test_metadata/test_v2.py (100%) rename tests/{v3 => }/test_metadata/test_v3.py (100%) rename tests/{v3 => }/test_properties.py (100%) rename tests/{v3/test_metadata => test_store}/__init__.py (100%) rename tests/{v3 => }/test_store/test_core.py (100%) rename tests/{v3 => }/test_store/test_local.py (100%) rename tests/{v3 => }/test_store/test_logging.py (100%) rename tests/{v3 => }/test_store/test_memory.py (100%) rename tests/{v3 => }/test_store/test_remote.py (100%) rename tests/{v3 => }/test_store/test_stateful_store.py (100%) rename tests/{v3 => }/test_store/test_zip.py (100%) rename tests/{v3 => }/test_sync.py (100%) rename tests/{v3 => }/test_v2.py (100%) delete mode 100644 tests/v3/test_store/__init__.py diff --git a/pyproject.toml b/pyproject.toml index 116a646276..574b09b076 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -153,7 +153,7 @@ run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pypro run = "run-coverage --no-cov" run-verbose = "run-coverage --verbose" run-mypy = "mypy src" -run-hypothesis = "pytest --hypothesis-profile ci tests/v3/test_properties.py tests/v3/test_store/test_stateful*" +run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*" list-env = "pip list" [tool.hatch.envs.gputest] @@ -173,7 +173,7 @@ run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov=tests" run = "run-coverage --no-cov" run-verbose = "run-coverage --verbose" run-mypy = "mypy src" -run-hypothesis = "pytest --hypothesis-profile ci tests/v3/test_properties.py tests/v3/test_store/test_stateful*" +run-hypothesis = "pytest --hypothesis-profile ci tests/test_properties.py tests/test_store/test_stateful*" list-env = "pip list" [tool.hatch.envs.docs] @@ -282,18 +282,17 @@ ignore_errors = true [[tool.mypy.overrides]] module = [ - "tests.v2.*", - "tests.v3.package_with_entrypoint.*", - "tests.v3.test_codecs.test_codecs", - "tests.v3.test_codecs.test_transpose", - "tests.v3.test_metadata.*", - "tests.v3.test_store.*", - "tests.v3.test_config", - "tests.v3.test_group", - "tests.v3.test_indexing", - "tests.v3.test_properties", - "tests.v3.test_sync", - "tests.v3.test_v2", + "tests.package_with_entrypoint.*", + "tests.test_codecs.test_codecs", + "tests.test_codecs.test_transpose", + "tests.test_metadata.*", + "tests.test_store.*", + "tests.test_config", + "tests.test_group", + "tests.test_indexing", + "tests.test_properties", + "tests.test_sync", + "tests.test_v2", ] ignore_errors = true diff --git a/src/zarr/testing/utils.py b/src/zarr/testing/utils.py index 9d6dfa7e18..c7b6e7939c 100644 --- a/src/zarr/testing/utils.py +++ b/src/zarr/testing/utils.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, cast +from collections.abc import Callable, Coroutine +from typing import TYPE_CHECKING, Any, TypeVar, cast import pytest @@ -37,8 +38,16 @@ def has_cupy() -> bool: return False +T_Callable = TypeVar("T_Callable", bound=Callable[[], Coroutine[Any, Any, None]]) + + # Decorator for GPU tests -def gpu_test(func: Any) -> Any: - return pytest.mark.gpu( - pytest.mark.skipif(not has_cupy(), reason="CuPy not installed or no GPU available")(func) +def gpu_test(func: T_Callable) -> T_Callable: + return cast( + T_Callable, + pytest.mark.gpu( + pytest.mark.skipif(not has_cupy(), reason="CuPy not installed or no GPU available")( + func + ) + ), ) diff --git a/tests/v3/conftest.py b/tests/conftest.py similarity index 100% rename from tests/v3/conftest.py rename to tests/conftest.py diff --git a/tests/v3/package_with_entrypoint-0.1.dist-info/entry_points.txt b/tests/package_with_entrypoint-0.1.dist-info/entry_points.txt similarity index 100% rename from tests/v3/package_with_entrypoint-0.1.dist-info/entry_points.txt rename to tests/package_with_entrypoint-0.1.dist-info/entry_points.txt diff --git a/tests/v3/package_with_entrypoint/__init__.py b/tests/package_with_entrypoint/__init__.py similarity index 100% rename from tests/v3/package_with_entrypoint/__init__.py rename to tests/package_with_entrypoint/__init__.py diff --git a/tests/v3/test_api.py b/tests/test_api.py similarity index 100% rename from tests/v3/test_api.py rename to tests/test_api.py diff --git a/tests/v3/test_array.py b/tests/test_array.py similarity index 100% rename from tests/v3/test_array.py rename to tests/test_array.py diff --git a/tests/v3/test_attributes.py b/tests/test_attributes.py similarity index 100% rename from tests/v3/test_attributes.py rename to tests/test_attributes.py diff --git a/tests/v3/test_buffer.py b/tests/test_buffer.py similarity index 100% rename from tests/v3/test_buffer.py rename to tests/test_buffer.py diff --git a/tests/v3/test_chunk_grids.py b/tests/test_chunk_grids.py similarity index 100% rename from tests/v3/test_chunk_grids.py rename to tests/test_chunk_grids.py diff --git a/tests/v3/test_codec_entrypoints.py b/tests/test_codec_entrypoints.py similarity index 100% rename from tests/v3/test_codec_entrypoints.py rename to tests/test_codec_entrypoints.py diff --git a/tests/v3/__init__.py b/tests/test_codecs/__init__.py similarity index 100% rename from tests/v3/__init__.py rename to tests/test_codecs/__init__.py diff --git a/tests/v3/test_codecs/test_blosc.py b/tests/test_codecs/test_blosc.py similarity index 100% rename from tests/v3/test_codecs/test_blosc.py rename to tests/test_codecs/test_blosc.py diff --git a/tests/v3/test_codecs/test_codecs.py b/tests/test_codecs/test_codecs.py similarity index 100% rename from tests/v3/test_codecs/test_codecs.py rename to tests/test_codecs/test_codecs.py diff --git a/tests/v3/test_codecs/test_endian.py b/tests/test_codecs/test_endian.py similarity index 100% rename from tests/v3/test_codecs/test_endian.py rename to tests/test_codecs/test_endian.py diff --git a/tests/v3/test_codecs/test_gzip.py b/tests/test_codecs/test_gzip.py similarity index 100% rename from tests/v3/test_codecs/test_gzip.py rename to tests/test_codecs/test_gzip.py diff --git a/tests/v3/test_codecs/test_sharding.py b/tests/test_codecs/test_sharding.py similarity index 100% rename from tests/v3/test_codecs/test_sharding.py rename to tests/test_codecs/test_sharding.py diff --git a/tests/v3/test_codecs/test_transpose.py b/tests/test_codecs/test_transpose.py similarity index 100% rename from tests/v3/test_codecs/test_transpose.py rename to tests/test_codecs/test_transpose.py diff --git a/tests/v3/test_codecs/test_vlen.py b/tests/test_codecs/test_vlen.py similarity index 100% rename from tests/v3/test_codecs/test_vlen.py rename to tests/test_codecs/test_vlen.py diff --git a/tests/v3/test_codecs/test_zstd.py b/tests/test_codecs/test_zstd.py similarity index 100% rename from tests/v3/test_codecs/test_zstd.py rename to tests/test_codecs/test_zstd.py diff --git a/tests/v3/test_common.py b/tests/test_common.py similarity index 100% rename from tests/v3/test_common.py rename to tests/test_common.py diff --git a/tests/v3/test_config.py b/tests/test_config.py similarity index 98% rename from tests/v3/test_config.py rename to tests/test_config.py index 62907588c7..c4cf794c5f 100644 --- a/tests/v3/test_config.py +++ b/tests/test_config.py @@ -87,7 +87,7 @@ class MockClass: assert ( fully_qualified_name(MockClass) - == "tests.v3.test_config.test_fully_qualified_name..MockClass" + == "tests.test_config.test_fully_qualified_name..MockClass" ) diff --git a/tests/v3/test_group.py b/tests/test_group.py similarity index 100% rename from tests/v3/test_group.py rename to tests/test_group.py diff --git a/tests/v3/test_indexing.py b/tests/test_indexing.py similarity index 100% rename from tests/v3/test_indexing.py rename to tests/test_indexing.py diff --git a/tests/v3/test_codecs/__init__.py b/tests/test_metadata/__init__.py similarity index 100% rename from tests/v3/test_codecs/__init__.py rename to tests/test_metadata/__init__.py diff --git a/tests/v3/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py similarity index 100% rename from tests/v3/test_metadata/test_consolidated.py rename to tests/test_metadata/test_consolidated.py diff --git a/tests/v3/test_metadata/test_v2.py b/tests/test_metadata/test_v2.py similarity index 100% rename from tests/v3/test_metadata/test_v2.py rename to tests/test_metadata/test_v2.py diff --git a/tests/v3/test_metadata/test_v3.py b/tests/test_metadata/test_v3.py similarity index 100% rename from tests/v3/test_metadata/test_v3.py rename to tests/test_metadata/test_v3.py diff --git a/tests/v3/test_properties.py b/tests/test_properties.py similarity index 100% rename from tests/v3/test_properties.py rename to tests/test_properties.py diff --git a/tests/v3/test_metadata/__init__.py b/tests/test_store/__init__.py similarity index 100% rename from tests/v3/test_metadata/__init__.py rename to tests/test_store/__init__.py diff --git a/tests/v3/test_store/test_core.py b/tests/test_store/test_core.py similarity index 100% rename from tests/v3/test_store/test_core.py rename to tests/test_store/test_core.py diff --git a/tests/v3/test_store/test_local.py b/tests/test_store/test_local.py similarity index 100% rename from tests/v3/test_store/test_local.py rename to tests/test_store/test_local.py diff --git a/tests/v3/test_store/test_logging.py b/tests/test_store/test_logging.py similarity index 100% rename from tests/v3/test_store/test_logging.py rename to tests/test_store/test_logging.py diff --git a/tests/v3/test_store/test_memory.py b/tests/test_store/test_memory.py similarity index 100% rename from tests/v3/test_store/test_memory.py rename to tests/test_store/test_memory.py diff --git a/tests/v3/test_store/test_remote.py b/tests/test_store/test_remote.py similarity index 100% rename from tests/v3/test_store/test_remote.py rename to tests/test_store/test_remote.py diff --git a/tests/v3/test_store/test_stateful_store.py b/tests/test_store/test_stateful_store.py similarity index 100% rename from tests/v3/test_store/test_stateful_store.py rename to tests/test_store/test_stateful_store.py diff --git a/tests/v3/test_store/test_zip.py b/tests/test_store/test_zip.py similarity index 100% rename from tests/v3/test_store/test_zip.py rename to tests/test_store/test_zip.py diff --git a/tests/v3/test_sync.py b/tests/test_sync.py similarity index 100% rename from tests/v3/test_sync.py rename to tests/test_sync.py diff --git a/tests/v3/test_v2.py b/tests/test_v2.py similarity index 100% rename from tests/v3/test_v2.py rename to tests/test_v2.py diff --git a/tests/v3/test_store/__init__.py b/tests/v3/test_store/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 From ee112b93bd9ca1a59c87a48d81156b95ca049242 Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Fri, 18 Oct 2024 17:36:47 +0200 Subject: [PATCH 2/2] fix reading partial shards (#2397) --- src/zarr/codecs/sharding.py | 4 ++-- tests/test_codecs/test_sharding.py | 36 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index 2181e9eb76..d01e116f9a 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -129,7 +129,7 @@ def get_chunk_slice(self, chunk_coords: ChunkCoords) -> tuple[int, int] | None: if (chunk_start, chunk_len) == (MAX_UINT_64, MAX_UINT_64): return None else: - return (int(chunk_start), int(chunk_start) + int(chunk_len)) + return (int(chunk_start), int(chunk_len)) def set_chunk_slice(self, chunk_coords: ChunkCoords, chunk_slice: slice | None) -> None: localized_chunk = self._localize_chunk(chunk_coords) @@ -203,7 +203,7 @@ def create_empty( def __getitem__(self, chunk_coords: ChunkCoords) -> Buffer: chunk_byte_slice = self.index.get_chunk_slice(chunk_coords) if chunk_byte_slice: - return self.buf[chunk_byte_slice[0] : chunk_byte_slice[1]] + return self.buf[chunk_byte_slice[0] : (chunk_byte_slice[0] + chunk_byte_slice[1])] raise KeyError def __len__(self) -> int: diff --git a/tests/test_codecs/test_sharding.py b/tests/test_codecs/test_sharding.py index c0dcfbf350..f827a0720e 100644 --- a/tests/test_codecs/test_sharding.py +++ b/tests/test_codecs/test_sharding.py @@ -118,6 +118,42 @@ def test_sharding_partial( assert np.array_equal(data, read_data) +@pytest.mark.parametrize("index_location", ["start", "end"]) +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +@pytest.mark.parametrize( + "array_fixture", + [ + ArrayRequest(shape=(128,) * 3, dtype="uint16", order="F"), + ], + indirect=["array_fixture"], +) +def test_sharding_partial_readwrite( + store: Store, array_fixture: npt.NDArray[Any], index_location: ShardingCodecIndexLocation +) -> None: + data = array_fixture + spath = StorePath(store) + a = Array.create( + spath, + shape=data.shape, + chunk_shape=data.shape, + dtype=data.dtype, + fill_value=0, + codecs=[ + ShardingCodec( + chunk_shape=(1, data.shape[1], data.shape[2]), + codecs=[BytesCodec()], + index_location=index_location, + ) + ], + ) + + a[:] = data + + for x in range(data.shape[0]): + read_data = a[x, :, :] + assert np.array_equal(data[x], read_data) + + @pytest.mark.parametrize( "array_fixture", [