Merge pull request #1437 from girder/more-typing

Add more type annotations

CHANGELOG.md

@@ -7,7 +7,7 @@
 - Optimizing when reading arrays rather than images from tiff files ([#1423](../../pull/1423))
 - Better filter DICOM adjacent files to ensure they share series instance IDs ([#1424](../../pull/1424), [#1436](../../pull/1436))
 - Optimizing small getRegion calls and some tiff tile fetches ([#1427](../../pull/1427))
-- Started adding python types to the core library ([#1432](../../pull/1432), [#1433](../../pull/1433))
+- Started adding python types to the core library ([#1432](../../pull/1432), [#1433](../../pull/1433), [#1437](../../pull/1437))
 - Use parallelism in computing tile frames ([#1434](../../pull/1434))
 
 ### Changed

large_image/cache_util/__init__.py

@@ -1,4 +1,4 @@
-###############################################################################
+#############################################################################
 #  Copyright Kitware Inc.
 #
 #  Licensed under the Apache License, Version 2.0 ( the "License" );
@@ -12,25 +12,26 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-###############################################################################
+#############################################################################
 
 import atexit
+from typing import Any, Callable, Dict, List
 
 from .cache import (CacheProperties, LruCacheMetaclass, getTileCache,
                     isTileCacheSetup, methodcache, strhash)
+from .cachefactory import CacheFactory, pickAvailableCache
 
+MemCache: Any
 try:
     from .memcache import MemCache
 except ImportError:
     MemCache = None
 
-from .cachefactory import CacheFactory, pickAvailableCache
-
-_cacheClearFuncs = []
+_cacheClearFuncs: List[Callable] = []
 
 
 @atexit.register
-def cachesClearExceptTile(*args, **kwargs):
+def cachesClearExceptTile(*args, **kwargs) -> None:
     """
     Clear the tilesource caches and the load model cache.  Note that this does
     not clear memcached (which could be done with tileCache._client.flush_all,
@@ -43,21 +44,24 @@ def cachesClearExceptTile(*args, **kwargs):
         func()
 
 
-def cachesClear(*args, **kwargs):
+def cachesClear(*args, **kwargs) -> None:
     """
     Clear the tilesource caches, the load model cache, and the tile cache.
     """
     cachesClearExceptTile()
     if isTileCacheSetup():
         tileCache, tileLock = getTileCache()
         try:
-            with tileLock:
+            if tileLock:
+                with tileLock:
+                    tileCache.clear()
+            else:
                 tileCache.clear()
         except Exception:
             pass
 
 
-def cachesInfo(*args, **kwargs):
+def cachesInfo(*args, **kwargs) -> Dict[str, Dict[str, int]]:
     """
     Report on each cache.
 
@@ -75,7 +79,15 @@ def cachesInfo(*args, **kwargs):
     if isTileCacheSetup():
         tileCache, tileLock = getTileCache()
         try:
-            with tileLock:
+            if tileLock:
+                with tileLock:
+                    info['tileCache'] = {
+                        'maxsize': tileCache.maxsize,
+                        'used': tileCache.currsize,
+                        'items': getattr(tileCache, 'curritems' if hasattr(
+                            tileCache, 'curritems') else 'currsize', None),
+                    }
+            else:
                 info['tileCache'] = {
                     'maxsize': tileCache.maxsize,
                     'used': tileCache.currsize,

large_image/cache_util/base.py

@@ -1,20 +1,25 @@
 import hashlib
 import threading
 import time
-from typing import Tuple
+from typing import Any, Callable, Dict, Optional, Tuple, TypeVar
 
 import cachetools
 
+_VT = TypeVar('_VT')
+
 
 class BaseCache(cachetools.Cache):
     """Base interface to cachetools.Cache for use with large-image."""
 
-    def __init__(self, *args, getsizeof=None, **kwargs):
-        super().__init__(*args, getsizeof=getsizeof, **kwargs)
-        self.lastError = {}
+    def __init__(
+            self, maxsize: float,
+            getsizeof: Optional[Callable[[_VT], float]] = None,
+            **kwargs) -> None:
+        super().__init__(maxsize=maxsize, getsizeof=getsizeof, **kwargs)
+        self.lastError: Dict[Tuple[Any, Callable], Dict[str, Any]] = {}
         self.throttleErrors = 10  # seconds between logging errors
 
-    def logError(self, err, func, msg):
+    def logError(self, err: Any, func: Callable, msg: str) -> None:
         """
         Log errors, but throttle them so as not to spam the logs.
 
@@ -40,16 +45,16 @@ def __repr__(self):
     def __iter__(self):
         raise NotImplementedError
 
-    def __len__(self):
+    def __len__(self) -> int:
         raise NotImplementedError
 
-    def __contains__(self, key):
+    def __contains__(self, item) -> bool:
         raise NotImplementedError
 
     def __delitem__(self, key):
         raise NotImplementedError
 
-    def _hashKey(self, key):
+    def _hashKey(self, key) -> str:
         return hashlib.sha256(key.encode()).hexdigest()
 
     def __getitem__(self, key):
@@ -61,21 +66,21 @@ def __setitem__(self, key, value):
         raise NotImplementedError
 
     @property
-    def curritems(self):
+    def curritems(self) -> int:
         raise NotImplementedError
 
     @property
-    def currsize(self):
+    def currsize(self) -> int:
         raise NotImplementedError
 
     @property
-    def maxsize(self):
+    def maxsize(self) -> int:
         raise NotImplementedError
 
-    def clear(self):
+    def clear(self) -> None:
         raise NotImplementedError
 
     @staticmethod
-    def getCache() -> Tuple['BaseCache', threading.Lock]:
+    def getCache() -> Tuple[Optional['BaseCache'], threading.Lock]:
         # return cache, cacheLock
         raise NotImplementedError

large_image/cache_util/cache.py

@@ -2,8 +2,9 @@
 import pickle
 import threading
 import uuid
-from typing import Any, Callable, Dict, Optional, TypeVar
+from typing import Any, Callable, Dict, Optional, Tuple, TypeVar
 
+import cachetools
 from typing_extensions import ParamSpec
 
 try:
@@ -18,8 +19,8 @@
 P = ParamSpec('P')
 T = TypeVar('T')
 
-_tileCache = None
-_tileLock = None
+_tileCache: Optional[cachetools.Cache] = None
+_tileLock: Optional[threading.Lock] = None
 
 _cacheLockKeyToken = '_cacheLock_key'
 
@@ -175,7 +176,7 @@ def __new__(metacls, name, bases, namespace, **kwargs):
 
         return cls
 
-    def __call__(cls, *args, **kwargs):  # noqa - N805
+    def __call__(cls, *args, **kwargs) -> Any:  # noqa - N805
         if kwargs.get('noCache') or (
                 kwargs.get('noCache') is None and config.getConfig('cache_sources') is False):
             instance = super().__call__(*args, **kwargs)
@@ -259,7 +260,7 @@ def __call__(cls, *args, **kwargs):  # noqa - N805
         return instance
 
 
-def getTileCache():
+def getTileCache() -> Tuple[cachetools.Cache, Optional[threading.Lock]]:
     """
     Get the preferred tile cache and lock.
 

large_image/cache_util/cachefactory.py

@@ -16,30 +16,30 @@
 
 import math
 import threading
+from typing import Dict, Optional, Tuple, Type
 
 import cachetools
 
 try:
     import psutil
+    HAS_PSUTIL = True
 except ImportError:
-    psutil = None
+    HAS_PSUTIL = False
 
 from importlib.metadata import entry_points
 
 from .. import config
 from ..exceptions import TileCacheError
-
-try:
-    from .memcache import MemCache
-except ImportError:
-    MemCache = None
+from .memcache import MemCache
 
 # DO NOT MANUALLY ADD ANYTHING TO `_availableCaches`
 #  use entrypoints and let loadCaches fill in `_availableCaches`
-_availableCaches = {}
+_availableCaches: Dict[str, Type[cachetools.Cache]] = {}
 
 
-def loadCaches(entryPointName='large_image.cache', sourceDict=_availableCaches):
+def loadCaches(
+        entryPointName: str = 'large_image.cache',
+        sourceDict: Dict[str, Type[cachetools.Cache]] = _availableCaches) -> None:
     """
     Load all caches from entrypoints and add them to the
     availableCaches dictionary.
@@ -69,7 +69,9 @@ def loadCaches(entryPointName='large_image.cache', sourceDict=_availableCaches):
     # NOTE: `python` cache is viewed as a fallback and isn't listed in `availableCaches`
 
 
-def pickAvailableCache(sizeEach, portion=8, maxItems=None, cacheName=None):
+def pickAvailableCache(
+        sizeEach: int, portion: int = 8, maxItems: Optional[int] = None,
+        cacheName: Optional[str] = None) -> int:
     """
     Given an estimated size of an item, return how many of those items would
     fit in a fixed portion of the available virtual memory.
@@ -90,7 +92,7 @@ def pickAvailableCache(sizeEach, portion=8, maxItems=None, cacheName=None):
         if configMaxItems > 0:
             maxItems = configMaxItems
     # Estimate usage based on (1 / portion) of the total virtual memory.
-    if psutil:
+    if HAS_PSUTIL:
         memory = psutil.virtual_memory().total
     else:
         memory = 1024 ** 3
@@ -100,7 +102,7 @@ def pickAvailableCache(sizeEach, portion=8, maxItems=None, cacheName=None):
     return numItems
 
 
-def getFirstAvailableCache():
+def getFirstAvailableCache() -> Tuple[cachetools.Cache, Optional[threading.Lock]]:
     cacheBackend = config.getConfig('cache_backend', None)
     if cacheBackend is not None:
         msg = 'cache_backend already set'
@@ -109,7 +111,7 @@ def getFirstAvailableCache():
     cache, cacheLock = None, None
     for cacheBackend in _availableCaches:
         try:
-            cache, cacheLock = _availableCaches[cacheBackend].getCache()
+            cache, cacheLock = _availableCaches[cacheBackend].getCache()  # type: ignore
             break
         except TileCacheError:
             continue
@@ -124,7 +126,7 @@ def getFirstAvailableCache():
 class CacheFactory:
     logged = False
 
-    def getCacheSize(self, numItems, cacheName=None):
+    def getCacheSize(self, numItems: Optional[int], cacheName: Optional[str] = None) -> int:
         if numItems is None:
             defaultPortion = 32
             try:
@@ -145,7 +147,10 @@ def getCacheSize(self, numItems, cacheName=None):
                 pass
         return numItems
 
-    def getCache(self, numItems=None, cacheName=None, inProcess=False):
+    def getCache(
+            self, numItems: Optional[int] = None,
+            cacheName: Optional[str] = None,
+            inProcess: bool = False) -> Tuple[cachetools.Cache, Optional[threading.Lock]]:
         loadCaches()
 
         # Default to `python` cache for inProcess
@@ -156,7 +161,7 @@ def getCache(self, numItems=None, cacheName=None, inProcess=False):
 
         cache = None
         if not inProcess and cacheBackend in _availableCaches:
-            cache, cacheLock = _availableCaches[cacheBackend].getCache()
+            cache, cacheLock = _availableCaches[cacheBackend].getCache()  # type: ignore
         elif not inProcess and cacheBackend is None:
             cache, cacheLock = getFirstAvailableCache()