From ac9a01fb345ec048655957eb8e19c39c852cc360 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 27 Dec 2023 14:07:47 +0000 Subject: [PATCH] Type dimension separator --- zarr/_storage/absstore.py | 5 ++- zarr/_storage/v3.py | 7 ++-- zarr/_storage/v3_storage_transformers.py | 3 +- zarr/creation.py | 2 +- zarr/storage.py | 45 +++++++++++++++++------- 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/zarr/_storage/absstore.py b/zarr/_storage/absstore.py index c9a113148c..2eb6dc9f9c 100644 --- a/zarr/_storage/absstore.py +++ b/zarr/_storage/absstore.py @@ -1,9 +1,12 @@ """This module contains storage classes related to Azure Blob Storage (ABS)""" +from typing import Optional import warnings + from numcodecs.compat import ensure_bytes from zarr.util import normalize_storage_path from zarr._storage.store import _get_metadata_suffix, data_root, meta_root, Store, StoreV3 +from zarr.types import DIMENSION_SEPARATOR __doctest_requires__ = { ("ABSStore", "ABSStore.*"): ["azure.storage.blob"], @@ -67,7 +70,7 @@ def __init__( account_name=None, account_key=None, blob_service_kwargs=None, - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, client=None, ): self._dimension_separator = dimension_separator diff --git a/zarr/_storage/v3.py b/zarr/_storage/v3.py index 32e78f7a34..300d0e5914 100644 --- a/zarr/_storage/v3.py +++ b/zarr/_storage/v3.py @@ -3,13 +3,14 @@ from collections import OrderedDict from collections.abc import MutableMapping from threading import Lock -from typing import Union, Dict, Any +from typing import Union, Dict, Any, Optional from zarr.errors import ( MetadataError, ReadOnlyError, ) from zarr.util import buffer_size, json_loads, normalize_storage_path +from zarr.types import DIMENSION_SEPARATOR from zarr._storage.absstore import ABSStoreV3 # noqa: F401 from zarr._storage.store import ( # noqa: F401 @@ -224,7 +225,9 @@ def get_partial_values(self, key_ranges): class MemoryStoreV3(MemoryStore, StoreV3): - def __init__(self, root=None, cls=dict, dimension_separator=None): + def __init__( + self, root=None, cls=dict, dimension_separator: Optional[DIMENSION_SEPARATOR] = None + ): if root is None: self.root = cls() else: diff --git a/zarr/_storage/v3_storage_transformers.py b/zarr/_storage/v3_storage_transformers.py index 3afc3823a3..37e56f8ecd 100644 --- a/zarr/_storage/v3_storage_transformers.py +++ b/zarr/_storage/v3_storage_transformers.py @@ -8,6 +8,7 @@ from zarr._storage.store import StorageTransformer, StoreV3, _rmdir_from_keys_v3 from zarr.util import normalize_storage_path +from zarr.types import DIMENSION_SEPARATOR MAX_UINT_64 = 2**64 - 1 @@ -118,7 +119,7 @@ def _copy_for_array(self, array, inner_store): return transformer_copy @property - def dimension_separator(self) -> str: + def dimension_separator(self) -> DIMENSION_SEPARATOR: assert ( self._dimension_separator is not None ), "dimension_separator is not initialized, first get a copy via _copy_for_array." diff --git a/zarr/creation.py b/zarr/creation.py index d4f570895a..25878dcc44 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -470,7 +470,7 @@ def open_array( write_empty_chunks=True, *, zarr_version=None, - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, meta_array=None, **kwargs, ): diff --git a/zarr/storage.py b/zarr/storage.py index aa27e98e6f..0c619936fe 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -40,7 +40,8 @@ from numcodecs.compat import ensure_bytes, ensure_text, ensure_contiguous_ndarray_like from numcodecs.registry import codec_registry from zarr.context import Context -from zarr.types import PathLike as Path +from zarr.types import PathLike as Path, DIMENSION_SEPARATOR +from zarr.util import NoLock from zarr.errors import ( MetadataError, @@ -326,7 +327,7 @@ def init_array( chunk_store: Optional[StoreLike] = None, filters=None, object_codec=None, - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, storage_transformers=(), ): """Initialize an array store with the given configuration. Note that this is a low-level @@ -480,7 +481,7 @@ def _init_array_metadata( chunk_store: Optional[StoreLike] = None, filters=None, object_codec=None, - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, storage_transformers=(), ): store_version = getattr(store, "_store_version", 2) @@ -1053,7 +1054,9 @@ class DirectoryStore(Store): """ - def __init__(self, path, normalize_keys=False, dimension_separator=None): + def __init__( + self, path, normalize_keys=False, dimension_separator: Optional[DIMENSION_SEPARATOR] = None + ): # guard conditions path = os.path.abspath(path) if os.path.exists(path) and not os.path.isdir(path): @@ -1348,7 +1351,7 @@ def __init__( key_separator=None, mode="w", exceptions=(KeyError, PermissionError, IOError), - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, fs=None, check=False, create=False, @@ -1567,7 +1570,12 @@ class TempStore(DirectoryStore): # noinspection PyShadowingBuiltins def __init__( - self, suffix="", prefix="zarr", dir=None, normalize_keys=False, dimension_separator=None + self, + suffix="", + prefix="zarr", + dir=None, + normalize_keys=False, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, ): path = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dir) atexit.register(atexit_rmtree, path) @@ -1651,7 +1659,9 @@ class NestedDirectoryStore(DirectoryStore): """ - def __init__(self, path, normalize_keys=False, dimension_separator="/"): + def __init__( + self, path, normalize_keys=False, dimension_separator: Optional[DIMENSION_SEPARATOR] = "/" + ): super().__init__(path, normalize_keys=normalize_keys) if dimension_separator is None: dimension_separator = "/" @@ -1764,7 +1774,7 @@ def __init__( compression=zipfile.ZIP_STORED, allowZip64=True, mode="a", - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, ): # store properties path = os.path.abspath(path) @@ -2057,7 +2067,7 @@ def __init__( mode=0o666, open=None, write_lock=True, - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **open_kwargs, ): if open is None: @@ -2072,6 +2082,7 @@ def __init__( self.mode = mode self.open = open self.write_lock = write_lock + self.write_mutex: Union[Lock, NoLock] if write_lock: # This may not be required as some dbm implementations manage their own # locks, but err on the side of caution. @@ -2228,7 +2239,13 @@ class LMDBStore(Store): """ - def __init__(self, path, buffers=True, dimension_separator=None, **kwargs): + def __init__( + self, + path, + buffers=True, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, + **kwargs, + ): import lmdb # set default memory map size to something larger than the lmdb default, which is @@ -2579,7 +2596,7 @@ class SQLiteStore(Store): >>> store.close() # don't forget to call this when you're done """ - def __init__(self, path, dimension_separator=None, **kwargs): + def __init__(self, path, dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **kwargs): import sqlite3 self._dimension_separator = dimension_separator @@ -2777,7 +2794,7 @@ def __init__( self, database="mongodb_zarr", collection="zarr_collection", - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **kwargs, ): import pymongo @@ -2852,7 +2869,9 @@ class RedisStore(Store): """ - def __init__(self, prefix="zarr", dimension_separator=None, **kwargs): + def __init__( + self, prefix="zarr", dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **kwargs + ): import redis self._prefix = prefix