From b25afbf03a685af1598456518c6969db4a8f21b1 Mon Sep 17 00:00:00 2001 From: David Manthey Date: Tue, 24 Oct 2023 11:57:15 -0400 Subject: [PATCH] Start adding types using monkeytype. We need to drop support for older pythons for this to be useful. After running `tox -e monkeytype-py311`, you can then list which modules have type data via `build/tox/monkeytype-py311/bin/monkeytype list-modules` and apply the type annotations to a specific modules with `build/tox/monkeytype-py311/bin/monkeytype apply `. Check types using `tox -e type`. The next steps are to turn on a few more mypy warnings, then start adding types to more of the source files. --- .circleci/config.yml | 17 ++ .circleci/dcm4chee/upload_example_data.py | 9 +- CHANGELOG.md | 1 + large_image/config.py | 10 +- large_image/exceptions.py | 2 +- large_image/tilesource/__init__.py | 35 ++- large_image/tilesource/base.py | 262 ++++++++++-------- .../large_image_source_dicom/__init__.py | 2 + sources/dicom/test_dicom/test_web_client.py | 2 +- tox.ini | 115 ++++++++ 10 files changed, 326 insertions(+), 129 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 31b2a4832..6c88d4694 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -226,6 +226,13 @@ jobs: path: build/tox/compare.txt - store_artifacts: path: build/tox/compare.yaml + type: + executor: toxandnode + resource_class: large + steps: + - checkout + - tox: + env: type wheels: executor: toxandnode steps: @@ -333,6 +340,13 @@ workflows: branches: ignore: - gh-pages + - type: + filters: + tags: + only: /^v.*/ + branches: + ignore: + - gh-pages - compare: filters: tags: @@ -355,6 +369,7 @@ workflows: - py311 - py312 - lint_and_docs + - type - wheels filters: tags: @@ -369,6 +384,7 @@ workflows: - py311 - py312 - lint_and_docs + - type - wheels filters: tags: @@ -393,5 +409,6 @@ workflows: - py311 - py312 - lint_and_docs + - type - compare - wheels diff --git a/.circleci/dcm4chee/upload_example_data.py b/.circleci/dcm4chee/upload_example_data.py index 3ddcb41a5..810aa90ca 100755 --- a/.circleci/dcm4chee/upload_example_data.py +++ b/.circleci/dcm4chee/upload_example_data.py @@ -11,9 +11,9 @@ def upload_example_data(server_url): # This is TCGA-AA-3697 sha512s = [ - '48cb562b94d0daf4060abd9eef150c851d3509d9abbff4bea11d00832955720bf1941073a51e6fb68fb5cc23704dec2659fc0c02360a8ac753dc523dca2c8c36', # noqa - '36432183380eb7d44417a2210a19d550527abd1181255e19ed5c1d17695d8bb8ca42f5b426a63fa73b84e0e17b770401a377ae0c705d0ed7fdf30d571ef60e2d', # noqa - '99bd3da4b8e11ce7b4f7ed8a294ed0c37437320667a06c40c383f4b29be85fe8e6094043e0600bee0ba879f2401de4c57285800a4a23da2caf2eb94e5b847ee0', # noqa + '48cb562b94d0daf4060abd9eef150c851d3509d9abbff4bea11d00832955720bf1941073a51e6fb68fb5cc23704dec2659fc0c02360a8ac753dc523dca2c8c36', # noqa + '36432183380eb7d44417a2210a19d550527abd1181255e19ed5c1d17695d8bb8ca42f5b426a63fa73b84e0e17b770401a377ae0c705d0ed7fdf30d571ef60e2d', # noqa + '99bd3da4b8e11ce7b4f7ed8a294ed0c37437320667a06c40c383f4b29be85fe8e6094043e0600bee0ba879f2401de4c57285800a4a23da2caf2eb94e5b847ee0', # noqa ] download_urls = [ f'https://data.kitware.com/api/v1/file/hashsum/sha512/{x}/download' for x in sha512s @@ -35,6 +35,7 @@ def upload_example_data(server_url): url = os.getenv('DICOMWEB_TEST_URL') if url is None: - raise Exception('DICOMWEB_TEST_URL must be set') + msg = 'DICOMWEB_TEST_URL must be set' + raise Exception(msg) upload_example_data(url) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cfd5f546..78c9f35dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Optimizing when reading arrays rather than images from tiff files ([#1423](../../pull/1423)) - Better filter DICOM adjacent files to ensure they share series instance IDs ([#1424](../../pull/1424)) - Optimizing small getRegion calls and some tiff tile fetches ([#1427](../../pull/1427) +- Started adding python types to the core library ([#1430](../../pull/1430) ### Changed - Cleanup some places where get was needlessly used ([#1428](../../pull/1428) diff --git a/large_image/config.py b/large_image/config.py index 9eb676ed1..fa350fd61 100644 --- a/large_image/config.py +++ b/large_image/config.py @@ -2,7 +2,7 @@ import logging import os import re -from typing import cast +from typing import Any, Optional, Union, cast from . import exceptions @@ -59,7 +59,8 @@ } -def getConfig(key=None, default=None): +def getConfig(key: Optional[str] = None, + default: Optional[Union[str, bool, int, logging.Logger]] = None) -> Any: """ Get the config dictionary or a value from the cache config settings. @@ -83,7 +84,8 @@ def getConfig(key=None, default=None): return ConfigValues.get(key, default) -def getLogger(key=None, default=None): +def getLogger(key: Optional[str] = None, + default: Optional[logging.Logger] = None) -> logging.Logger: """ Get a logger from the config. Ensure that it is a valid logger. @@ -97,7 +99,7 @@ def getLogger(key=None, default=None): return logger -def setConfig(key, value): +def setConfig(key: str, value: Optional[Union[str, bool, int, logging.Logger]]) -> None: """ Set a value in the config settings. diff --git a/large_image/exceptions.py b/large_image/exceptions.py index 3c9911ecd..a79ad6d3a 100644 --- a/large_image/exceptions.py +++ b/large_image/exceptions.py @@ -22,7 +22,7 @@ class TileSourceInefficientError(TileSourceError): class TileSourceFileNotFoundError(TileSourceError, FileNotFoundError): - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: return super().__init__(errno.ENOENT, *args, **kwargs) diff --git a/large_image/tilesource/__init__.py b/large_image/tilesource/__init__.py index edcb409d2..503cee457 100644 --- a/large_image/tilesource/__init__.py +++ b/large_image/tilesource/__init__.py @@ -2,6 +2,8 @@ import re import uuid from importlib.metadata import entry_points +from pathlib import PosixPath +from typing import Dict, List, Optional, Tuple, Type, Union from .. import config from ..constants import NEW_IMAGE_PATH_FLAG, SourcePriority @@ -13,10 +15,10 @@ FileTileSource, TileOutputMimeTypes, TileSource, dictToEtree, etreeToDict, nearPowerOfTwo) -AvailableTileSources = {} +AvailableTileSources: Dict[str, Type[FileTileSource]] = {} -def isGeospatial(path): +def isGeospatial(path: Union[str, PosixPath]) -> bool: """ Check if a path is likely to be a geospatial file. @@ -38,7 +40,8 @@ def isGeospatial(path): return False -def loadTileSources(entryPointName='large_image.source', sourceDict=AvailableTileSources): +def loadTileSources(entryPointName: str = 'large_image.source', + sourceDict: Dict[str, Type[FileTileSource]] = AvailableTileSources) -> None: """ Load all tilesources from entrypoints and add them to the AvailableTileSources dictionary. @@ -61,7 +64,10 @@ def loadTileSources(entryPointName='large_image.source', sourceDict=AvailableTil 'Failed to loaded tile source %s' % entryPoint.name) -def getSortedSourceList(availableSources, pathOrUri, mimeType=None, *args, **kwargs): +def getSortedSourceList( + availableSources: Dict[str, Type[FileTileSource]], pathOrUri: Union[str, PosixPath], + mimeType: Optional[str] = None, *args, **kwargs, +) -> List[Tuple[bool, bool, SourcePriority, str]]: """ Get an ordered list of sources where earlier sources are more likely to work for a specified path or uri. @@ -108,7 +114,9 @@ def getSortedSourceList(availableSources, pathOrUri, mimeType=None, *args, **kwa return sourceList -def getSourceNameFromDict(availableSources, pathOrUri, mimeType=None, *args, **kwargs): +def getSourceNameFromDict( + availableSources: Dict[str, Type[FileTileSource]], pathOrUri: Union[str, PosixPath], + mimeType: Optional[str] = None, *args, **kwargs) -> Optional[str]: """ Get a tile source based on a ordered dictionary of known sources and a path name or URI. Additional parameters are passed to the tile source and can @@ -125,9 +133,12 @@ def getSourceNameFromDict(availableSources, pathOrUri, mimeType=None, *args, **k for _clash, _fallback, _priority, sourceName in sorted(sourceList): if availableSources[sourceName].canRead(pathOrUri, *args, **kwargs): return sourceName + return None -def getTileSourceFromDict(availableSources, pathOrUri, *args, **kwargs): +def getTileSourceFromDict( + availableSources: Dict[str, Type[FileTileSource]], pathOrUri: Union[str, PosixPath], + *args, **kwargs) -> FileTileSource: """ Get a tile source based on a ordered dictionary of known sources and a path name or URI. Additional parameters are passed to the tile source and can @@ -146,7 +157,7 @@ def getTileSourceFromDict(availableSources, pathOrUri, *args, **kwargs): raise TileSourceError('No available tilesource for %s' % pathOrUri) -def getTileSource(*args, **kwargs): +def getTileSource(*args, **kwargs) -> FileTileSource: """ Get a tilesource using the known sources. If tile sources have not yet been loaded, load them. @@ -158,7 +169,7 @@ def getTileSource(*args, **kwargs): return getTileSourceFromDict(AvailableTileSources, *args, **kwargs) -def open(*args, **kwargs): +def open(*args, **kwargs) -> FileTileSource: """ Alternate name of getTileSource. @@ -170,7 +181,7 @@ def open(*args, **kwargs): return getTileSource(*args, **kwargs) -def canRead(*args, **kwargs): +def canRead(*args, **kwargs) -> bool: """ Check if large_image can read a path or uri. @@ -187,7 +198,9 @@ def canRead(*args, **kwargs): return False -def canReadList(pathOrUri, mimeType=None, *args, **kwargs): +def canReadList( + pathOrUri: Union[str, PosixPath], mimeType: Optional[str] = None, + *args, **kwargs) -> List[Tuple[str, bool]]: """ Check if large_image can read a path or uri via each source. @@ -210,7 +223,7 @@ def canReadList(pathOrUri, mimeType=None, *args, **kwargs): return result -def new(*args, **kwargs): +def new(*args, **kwargs) -> TileSource: """ Create a new image. diff --git a/large_image/tilesource/base.py b/large_image/tilesource/base.py index 9c29a6f90..7df320426 100644 --- a/large_image/tilesource/base.py +++ b/large_image/tilesource/base.py @@ -1,3 +1,4 @@ +import functools import io import json import math @@ -8,8 +9,10 @@ import time import types import uuid +from typing import Any, Dict, Iterator, List, Optional, Tuple, Union, cast import numpy as np +import numpy.typing as npt import PIL import PIL.Image import PIL.ImageCms @@ -23,10 +26,11 @@ TileOutputPILFormat) from .jupyter import IPyLeafletMixin from .tiledict import LazyTileDict -from .utilities import (JSONDict, _addRegionTileToTiled, # noqa: F401 - _addSubimageToImage, _calculateWidthHeight, - _encodeImage, _encodeImageBinary, _imageToNumpy, - _imageToPIL, _letterboxImage, _makeSameChannelDepth, +from .utilities import (ImageBytes, JSONDict, # noqa: F401 + _addRegionTileToTiled, _addSubimageToImage, + _calculateWidthHeight, _encodeImage, + _encodeImageBinary, _imageToNumpy, _imageToPIL, + _letterboxImage, _makeSameChannelDepth, _vipsAddAlphaBand, _vipsCast, _vipsParameters, dictToEtree, etreeToDict, getPaletteColors, histogramThreshold, nearPowerOfTwo) @@ -39,21 +43,21 @@ class TileSource(IPyLeafletMixin): # A dictionary of known file extensions and the ``SourcePriority`` given # to each. It must contain a None key with a priority for the tile source # when the extension does not match. - extensions = { + extensions: Dict[Optional[str], SourcePriority] = { None: SourcePriority.FALLBACK, } # A dictionary of common mime-types handled by the source and the # ``SourcePriority`` given to each. This are used in place of or in # additional to extensions. - mimeTypes = { + mimeTypes: Dict[Optional[str], SourcePriority] = { None: SourcePriority.FALLBACK, } # A dictionary with regex strings as the keys and the ``SourcePriority`` # given to names that match that expression. This is used in addition to # extensions and mimeTypes, with the highest priority match taken. - nameMatches = { + nameMatches: Dict[str, SourcePriority] = { } geospatial = False @@ -66,9 +70,15 @@ class TileSource(IPyLeafletMixin): # _maxSkippedLevels, such large gaps are composited in stages. _maxSkippedLevels = 3 - def __init__(self, encoding='JPEG', jpegQuality=95, jpegSubsampling=0, - tiffCompression='raw', edge=False, style=None, noCache=None, - *args, **kwargs): + _initValues: Tuple[Tuple[Any, ...], Dict[str, Any]] + _iccprofilesObjects: List[Any] + + def __init__(self, encoding: str = 'JPEG', jpegQuality: int = 95, + jpegSubsampling: int = 0, tiffCompression: str = 'raw', + edge: Union[bool, str] = False, + style: Optional[Union[str, Dict[str, int]]] = None, + noCache: Optional[bool] = None, + *args, **kwargs) -> None: """ Initialize the tile class. @@ -148,14 +158,14 @@ def __init__(self, encoding='JPEG', jpegQuality=95, jpegSubsampling=0, self.logger = config.getLogger() self.cache, self.cache_lock = getTileCache() - self.tileWidth = None - self.tileHeight = None - self.levels = None - self.sizeX = None - self.sizeY = None + self.tileWidth: int = 0 + self.tileHeight: int = 0 + self.levels: int = 0 + self.sizeX: int = 0 + self.sizeY: int = 0 self._sourceLock = threading.RLock() - self._dtype = None - self._bandCount = None + self._dtype: Optional[Union[npt.DTypeLike, str]] = None + self._bandCount: Optional[int] = None if encoding not in TileOutputMimeTypes: raise ValueError('Invalid encoding "%s"' % encoding) @@ -176,7 +186,7 @@ def __getstate__(self): """ return None - def __reduce__(self): + def __reduce__(self) -> Tuple[functools.partial, Tuple[str]]: """ Allow pickling. @@ -191,13 +201,13 @@ def __reduce__(self): raise pickle.PicklingError(msg) return functools.partial(type(self), **self._initValues[1]), self._initValues[0] - def __repr__(self): + def __repr__(self) -> str: return self.getState() def _repr_png_(self): return self.getThumbnail(encoding='PNG')[0] - def _setStyle(self, style): + def _setStyle(self, style: Any) -> None: """ Check and set the specified style from a json string or a dictionary. @@ -209,11 +219,11 @@ def _setStyle(self, style): except Exception: pass if not hasattr(self, '_bandRanges'): - self._bandRanges = {} + self._bandRanges: Dict[int, Any] = {} self._jsonstyle = style if style is not None: if isinstance(style, dict): - self._style = JSONDict(style) + self._style: Optional[JSONDict] = JSONDict(style) self._jsonstyle = json.dumps(style, sort_keys=True, separators=(',', ':')) else: try: @@ -273,9 +283,10 @@ def dtype(self): with self._sourceLock: if not self._dtype: self._dtype = 'check' - sample, _ = getattr(self, '_unstyledInstance', self).getRegion( - region=dict(left=0, top=0, width=1, height=1), - format=TILE_FORMAT_NUMPY) + sample, _ = cast(Tuple[np.ndarray, Any], getattr( + self, '_unstyledInstance', self).getRegion( + region=dict(left=0, top=0, width=1, height=1), + format=TILE_FORMAT_NUMPY)) self._dtype = sample.dtype self._bandCount = len( getattr(getattr(self, '_unstyledInstance', self), '_bandInfo', [])) @@ -291,7 +302,7 @@ def bandCount(self): return self._bandCount @staticmethod - def getLRUHash(*args, **kwargs): + def getLRUHash(*args, **kwargs) -> str: """ Return a string hash used as a key in the recently-used cache for tile sources. @@ -609,7 +620,7 @@ def _tileIteratorInfo(self, **kwargs): # noqa metadata, desiredMagnification=mag, **kwargs.get('region', {})) regionWidth = right - left regionHeight = bottom - top - magRequestedScale = None + magRequestedScale: Optional[float] = None if maxWidth is None and maxHeight is None and mag: if mag.get('scale') in (1.0, None): maxWidth, maxHeight = regionWidth, regionHeight @@ -748,7 +759,7 @@ def _tileIteratorInfo(self, **kwargs): # noqa } return info - def _tileIterator(self, iterInfo): + def _tileIterator(self, iterInfo: Dict[str, Any]) -> Iterator[LazyTileDict]: """ Given tile iterator information, iterate through the tiles. Each tile is returned as part of a dictionary that includes @@ -935,7 +946,7 @@ def _tileIterator(self, iterInfo): tile['gheight'] = tile['height'] * scale yield tile - def _pilFormatMatches(self, image, match=True, **kwargs): + def _pilFormatMatches(self, image, match=True, **kwargs) -> bool: """ Determine if the specified PIL image matches the format of the tile source with the specified arguments. @@ -1008,7 +1019,7 @@ def histogram(self, dtype=None, onlyMinMax=False, bins=256, # noqa lastlog = time.time() kwargs = kwargs.copy() histRange = kwargs.pop('range', None) - results = None + results: Optional[Dict[str, Any]] = None for tile in self.tileIterator(format=TILE_FORMAT_NUMPY, **kwargs): if time.time() - lastlog > 10: self.logger.info( @@ -1025,9 +1036,9 @@ def histogram(self, dtype=None, onlyMinMax=False, bins=256, # noqa np.amin(tile[:, :, idx]) for idx in range(tile.shape[2])], tile.dtype) tilemax = np.array([ np.amax(tile[:, :, idx]) for idx in range(tile.shape[2])], tile.dtype) - tilesum = np.array([ + tilesum: np.ndarray = np.array([ np.sum(tile[:, :, idx]) for idx in range(tile.shape[2])], float) - tilesum2 = np.array([ + tilesum2: np.ndarray = np.array([ np.sum(np.array(tile[:, :, idx], float) ** 2) for idx in range(tile.shape[2])], float) tilecount = tile.shape[0] * tile.shape[1] @@ -1045,6 +1056,8 @@ def histogram(self, dtype=None, onlyMinMax=False, bins=256, # noqa results['sum'] += tilesum[:len(results['min'])] results['sum2'] += tilesum2[:len(results['min'])] results['count'] += tilecount + if results is None: + return {} results['mean'] = results['sum'] / results['count'] results['stdev'] = np.maximum( results['sum2'] / results['count'] - results['mean'] ** 2, @@ -1125,7 +1138,9 @@ def _scanForMinMax(self, dtype, frame=None, analysisSize=1024, onlyMinMax=True, k: v for k, v in self._bandRanges[frame].items() if k in { 'min', 'max', 'mean', 'stdev'}}) - def _validateMinMaxValue(self, value, frame, dtype): + def _validateMinMaxValue( + self, value: Union[str, float], frame: int, dtype: npt.DTypeLike, + ) -> Tuple[Union[str, int, float], Union[float, int]]: """ Validate the min/max setting and return a specific string or float value and with any threshold. @@ -1140,12 +1155,13 @@ def _validateMinMaxValue(self, value, frame, dtype): :param frame: the frame to use for auto-ranging. :returns: the validated value and a threshold from [0-1]. """ - threshold = 0 + threshold: float = 0 if value not in {'min', 'max', 'auto', 'full'}: try: - if ':' in str(value) and value.split(':', 1)[0] in {'min', 'max', 'auto'}: - threshold = float(value.split(':', 1)[1]) - value = value.split(':', 1)[0] + if ':' in str(value) and cast(str, value).split(':', 1)[0] in { + 'min', 'max', 'auto'}: + threshold = float(cast(str, value).split(':', 1)[1]) + value = cast(str, value).split(':', 1)[0] else: value = float(value) except ValueError: @@ -1157,7 +1173,9 @@ def _validateMinMaxValue(self, value, frame, dtype): self._scanForMinMax(dtype, frame, onlyMinMax=not threshold) return value, threshold - def _getMinMax(self, minmax, value, dtype, bandidx=None, frame=None): # noqa + def _getMinMax( # noqa + self, minmax: str, value: Union[str, float], dtype: np.dtype, + bandidx: Optional[int] = None, frame: Optional[int] = None) -> float: """ Get an appropriate minimum or maximum for a band. @@ -1217,7 +1235,9 @@ def _getMinMax(self, minmax, value, dtype, bandidx=None, frame=None): # noqa value = 255 return float(value) - def _applyStyleFunction(self, image, sc, stage, function=None): + def _applyStyleFunction( + self, image: np.ndarray, sc: types.SimpleNamespace, stage: str, + function: Optional[Dict[str, Any]] = None) -> np.ndarray: """ Check if a style ahs a style function for the current stage. If so, apply it. @@ -1254,7 +1274,7 @@ def _applyStyleFunction(self, image, sc, stage, function=None): if function is None: return image if isinstance(function, (list, tuple)): - for func in function: + for func in cast(Union[List, Tuple], function): image = self._applyStyleFunction(image, sc, stage, func) return image if isinstance(function, str): @@ -1322,7 +1342,7 @@ def getICCProfiles(self, idx=None, onlyInfo=False): if prof else None for prof in results] return results - def _applyICCProfile(self, sc, frame): + def _applyICCProfile(self, sc: types.SimpleNamespace, frame: int) -> np.ndarray: """ Apply an ICC profile to an image. @@ -1330,6 +1350,8 @@ def _applyICCProfile(self, sc, frame): :param frame: the frame to use for auto ranging. :returns: an image with the icc profile, if any, applied. """ + if not hasattr(self, '_iccprofiles'): + return sc.image profileIdx = frame if frame and len(self._iccprofiles) >= frame + 1 else 0 sc.iccimage = sc.image sc.iccapplied = False @@ -1344,7 +1366,7 @@ def _applyICCProfile(self, sc, frame): PIL.ImageCms.Intent.PERCEPTUAL) else: intent = getattr(PIL.ImageCms, 'INTENT_' + str(sc.style.get('icc')).upper(), - PIL.ImageCms.INTENT_PERCEPTUAL) + PIL.ImageCms.INTENT_PERCEPTUAL) # type: ignore[attr-defined] if not hasattr(self, '_iccsrgbprofile'): try: self._iccsrgbprofile = PIL.ImageCms.createProfile('sRGB') @@ -1381,7 +1403,9 @@ def _applyICCProfile(self, sc, frame): self.logger.exception('Failed to apply ICC profile') return sc.iccimage - def _applyStyle(self, image, style, x, y, z, frame=None): # noqa + def _applyStyle( # noqa + self, image: np.ndarray, style: Optional[JSONDict], x: int, y: int, + z: int, frame: Optional[int] = None) -> np.ndarray: """ Apply a style to a numpy image. @@ -1397,7 +1421,7 @@ def _applyStyle(self, image, style, x, y, z, frame=None): # noqa image=image, originalStyle=style, x=x, y=y, z=z, frame=frame, mainImage=image, mainFrame=frame, dtype=None, axis=None) if not style or ('icc' in style and len(style) == 1): - sc.style = {'icc': (style or {}).get( + sc.style = {'icc': (style or cast(JSONDict, {})).get( 'icc', config.getConfig('icc_correction', True)), 'bands': []} else: sc.style = style if 'bands' in style else {'bands': [style]} @@ -1405,7 +1429,7 @@ def _applyStyle(self, image, style, x, y, z, frame=None): # noqa sc.axis = style.get('axis') if hasattr(self, '_iccprofiles') and sc.style.get( 'icc', config.getConfig('icc_correction', True)): - image = self._applyICCProfile(sc, frame) + image = self._applyICCProfile(sc, frame or 0) if not style or ('icc' in style and len(style) == 1): sc.output = image else: @@ -1428,7 +1452,7 @@ def _applyStyle(self, image, style, x, y, z, frame=None): # noqa elif sc.mainImage.dtype.kind == 'f': sc.dtype = 'float' sc.axis = sc.axis if sc.axis is not None else entry.get('axis') - sc.bandidx = 0 if image.shape[2] <= 2 else 1 + sc.bandidx = 0 if image.shape[2] <= 2 else 1 # type: ignore[misc] sc.band = None if ((entry.get('frame') is None and not entry.get('framedelta')) or entry.get('frame') == sc.mainFrame): @@ -1443,21 +1467,23 @@ def _applyStyle(self, image, style, x, y, z, frame=None): # noqa :sc.mainImage.shape[1], :sc.mainImage.shape[2]] if (isinstance(entry.get('band'), int) and - entry['band'] >= 1 and entry['band'] <= image.shape[2]): + entry['band'] >= 1 and entry['band'] <= image.shape[2]): # type: ignore[misc] sc.bandidx = entry['band'] - 1 sc.composite = entry.get('composite', 'lighten') if (hasattr(self, '_bandnames') and entry.get('band') and str(entry['band']).lower() in self._bandnames and - image.shape[2] > self._bandnames[str(entry['band']).lower()]): + image.shape[2] > self._bandnames[ # type: ignore[misc] + str(entry['band']).lower()]): sc.bandidx = self._bandnames[str(entry['band']).lower()] - if entry.get('band') == 'red' and image.shape[2] > 2: + if entry.get('band') == 'red' and image.shape[2] > 2: # type: ignore[misc] sc.bandidx = 0 - elif entry.get('band') == 'blue' and image.shape[2] > 2: + elif entry.get('band') == 'blue' and image.shape[2] > 2: # type: ignore[misc] sc.bandidx = 2 sc.band = image[:, :, 2] elif entry.get('band') == 'alpha': - sc.bandidx = image.shape[2] - 1 if image.shape[2] in (2, 4) else None - sc.band = (image[:, :, -1] if image.shape[2] in (2, 4) else + sc.bandidx = (image.shape[2] - 1 if image.shape[2] in (2, 4) # type: ignore[misc] + else None) + sc.band = (image[:, :, -1] if image.shape[2] in (2, 4) else # type: ignore[misc] np.full(image.shape[:2], 255, np.uint8)) sc.composite = entry.get('composite', 'multiply') if sc.band is None: @@ -1493,7 +1519,7 @@ def _applyStyle(self, image, style, x, y, z, frame=None): # noqa # divide. # See https://docs.gimp.org/en/gimp-concepts-layer-modes.html for # some details. - for channel in range(sc.output.shape[2]): + for channel in range(sc.output.shape[2]): # type: ignore[misc] if np.all(sc.palette[:, channel] == sc.palette[0, channel]): if ((sc.palette[0, channel] == 0 and sc.composite != 'multiply') or (sc.palette[0, channel] == 255 and sc.composite == 'multiply')): @@ -1534,7 +1560,7 @@ def _applyStyle(self, image, style, x, y, z, frame=None): # noqa sc.output = (sc.output * 65535 / 255).astype(np.uint16) elif sc.dtype == 'float': sc.output /= 255 - if sc.axis is not None and 0 <= int(sc.axis) < sc.output.shape[2]: + if sc.axis is not None and 0 <= int(sc.axis) < sc.output.shape[2]: # type: ignore[misc] sc.output = sc.output[:, :, sc.axis:sc.axis + 1] sc.output = self._applyStyleFunction(sc.output, sc, 'post') return sc.output @@ -1643,7 +1669,7 @@ def _outputTile(self, tile, tileEncoding, x, y, z, pilImageAllowed=False, tile, self.encoding, self.jpegQuality, self.jpegSubsampling, self.tiffCompression) return result - def _getAssociatedImage(self, imageKey): + def _getAssociatedImage(self, imageKey: str): """ Get an associated image in PIL format. @@ -1662,7 +1688,7 @@ def canRead(cls, *args, **kwargs): """ return False - def getMetadata(self): + def getMetadata(self) -> JSONDict: """ Return metadata about this tile source. This contains @@ -1723,10 +1749,11 @@ def getMetadata(self): }) @property - def metadata(self): + def metadata(self) -> JSONDict: return self.getMetadata() - def _addMetadataFrameInformation(self, metadata, channels=None): + def _addMetadataFrameInformation( + self, metadata: JSONDict, channels: Optional[List[str]] = None) -> None: """ Given a metadata response that has a `frames` list, where each frame has some of `Index(XY|Z|C|T)`, populate the `Frame`, `Index` and @@ -1740,7 +1767,7 @@ def _addMetadataFrameInformation(self, metadata, channels=None): """ if 'frames' not in metadata: return - maxref = {} + maxref: Dict[str, int] = {} refkeys = {'IndexC'} index = 0 for idx, frame in enumerate(metadata['frames']): @@ -1822,7 +1849,7 @@ def getBandInformation(self, statistics=False, **kwargs): resample=False, **kwargs) bands = histogram['min'].shape[0] - interp = bandInterp.get(bands, 3) + interp = bandInterp.get(bands, bandInterp[3]) bandInfo = { idx + 1: {'interpretation': interp[idx] if idx < len(interp) else 'unknown'} for idx in range(bands)} @@ -1833,7 +1860,7 @@ def getBandInformation(self, statistics=False, **kwargs): self._bandInfo = bandInfo return self._bandInfo - def _getFrame(self, frame=None, **kwargs): + def _getFrame(self, frame: Optional[int] = None, **kwargs) -> int: """ Get the current frame number. If a style is used that completely specified the frame, use that value instead. @@ -1890,7 +1917,7 @@ def _xyzToCorners(self, x, y, z): y1 = min((y + 1) * step * self.tileHeight, self.sizeY) return x0, y0, x1, y1, step - def _nonemptyLevelsList(self, frame=0): + def _nonemptyLevelsList(self, frame: Optional[int] = 0) -> List[bool]: """ Return a list of one value per level where the value is None if the level does not exist in the file and any other value if it does. @@ -1900,7 +1927,7 @@ def _nonemptyLevelsList(self, frame=0): """ return [True] * self.levels - def _getTileFromEmptyLevel(self, x, y, z, **kwargs): + def _getTileFromEmptyLevel(self, x: int, y: int, z: int, **kwargs) -> PIL.Image.Image: """ Given the x, y, z tile location in an unpopulated level, get tiles from higher resolution levels to make the lower-res tile. @@ -1998,7 +2025,7 @@ def getThumbnail(self, width=None, height=None, **kwargs): params.pop('region', None) return self.getRegion(**params) - def getPreferredLevel(self, level): + def getPreferredLevel(self, level: int) -> int: """ Given a desired level (0 is minimum resolution, self.levels - 1 is max resolution), return the level that contains actual data that is no @@ -2108,7 +2135,8 @@ def convertRegionScale( del targetRegion[key] return targetRegion - def getRegion(self, format=(TILE_FORMAT_IMAGE, ), **kwargs): + def getRegion(self, format: Union[str, Tuple[str]] = (TILE_FORMAT_IMAGE, ), **kwargs) -> Tuple[ + Union[np.ndarray, PIL.Image.Image, ImageBytes, bytes, pathlib.Path], str]: """ Get a rectangular region from the current tile source. Aspect ratio is preserved. If neither width nor height is given, the original size of @@ -2148,7 +2176,7 @@ def getRegion(self, format=(TILE_FORMAT_IMAGE, ), **kwargs): mode = None if TILE_FORMAT_NUMPY in format else iterInfo['mode'] outWidth = iterInfo['output']['width'] outHeight = iterInfo['output']['height'] - image = None + image: Optional[np.ndarray] = None tiledimage = None for tile in self._tileIterator(iterInfo): # Add each tile to the image @@ -2167,9 +2195,10 @@ def getRegion(self, format=(TILE_FORMAT_IMAGE, ), **kwargs): outWidth = int(math.floor(outWidth)) outHeight = int(math.floor(outHeight)) if tiled: - return self._encodeTiledImage(tiledimage, outWidth, outHeight, iterInfo, **kwargs) + return self._encodeTiledImage( + cast(Dict[str, Any], tiledimage), outWidth, outHeight, iterInfo, **kwargs) if outWidth != regionWidth or outHeight != regionHeight: - dtype = image.dtype + dtype = cast(np.ndarray, image).dtype image = _imageToPIL(image, mode).resize( (outWidth, outHeight), getattr(PIL.Image, 'Resampling', PIL.Image).BICUBIC @@ -2183,7 +2212,9 @@ def getRegion(self, format=(TILE_FORMAT_IMAGE, ), **kwargs): image = _letterboxImage(_imageToPIL(image, mode), maxWidth, maxHeight, kwargs['fill']) return _encodeImage(image, format=format, **kwargs) - def _encodeTiledImage(self, image, outWidth, outHeight, iterInfo, **kwargs): + def _encodeTiledImage( + self, image: Dict[str, Any], outWidth: int, outHeight: int, + iterInfo: Dict[str, Any], **kwargs) -> Tuple[pathlib.Path, str]: """ Given an image record of a set of vips image strips, generate a tiled tiff file at the specified output size. @@ -2206,7 +2237,9 @@ def _encodeTiledImage(self, image, outWidth, outHeight, iterInfo, **kwargs): a variety of options similar to the converter utility. :returns: a pathlib.Path of the output file and the output mime type. """ - vimg = image['strips'][0] + import pyvips + + vimg = cast(pyvips.Image, image['strips'][0]) for y in sorted(image['strips'].keys())[1:]: if image['strips'][y].bands + 1 == vimg.bands: image['strips'][y] = _vipsAddAlphaBand(image['strips'][y], vimg) @@ -2226,7 +2259,9 @@ def _encodeTiledImage(self, image, outWidth, outHeight, iterInfo, **kwargs): if image['magnification'] else image['magnification']) return self._encodeTiledImageFromVips(vimg, iterInfo, image, **kwargs) - def _encodeTiledImageFromVips(self, vimg, iterInfo, image, **kwargs): + def _encodeTiledImageFromVips( + self, vimg: Any, iterInfo: Dict[str, Any], image: Dict[str, Any], + **kwargs) -> Tuple[pathlib.Path, str]: """ Save a vips image as a tiled tiff. @@ -2245,13 +2280,14 @@ def _encodeTiledImageFromVips(self, vimg, iterInfo, image, **kwargs): import pyvips convertParams = _vipsParameters(defaultCompression='lzw', **kwargs) - vimg = _vipsCast(vimg, convertParams['compression'] in {'webp', 'jpeg'}) + vimg = _vipsCast(cast(pyvips.Image, vimg), convertParams['compression'] in {'webp', 'jpeg'}) maxWidth = kwargs.get('output', {}).get('maxWidth') maxHeight = kwargs.get('output', {}).get('maxHeight') if (kwargs.get('fill') and str(kwargs.get('fill')).lower() != 'none' and maxWidth and maxHeight and (maxWidth > image['width'] or maxHeight > image['height'])): - corner, fill = False, kwargs.get('fill') + corner: bool = False + fill: str = str(kwargs.get('fill')) if fill.lower().startswith('corner:'): corner, fill = True, fill.split(':', 1)[1] color = PIL.ImageColor.getcolor( @@ -2322,8 +2358,8 @@ def tileFrames(self, format=(TILE_FORMAT_IMAGE, ), frameList=None, tiled = TILE_FORMAT_IMAGE in format and kwargs.get('encoding') == 'TILED' iterInfo = self._tileIteratorInfo(frame=frameList[0], **kwargs) if iterInfo is None: - image = PIL.Image.new('RGB', (0, 0)) - return _encodeImage(image, format=format, **kwargs) + pilimage = PIL.Image.new('RGB', (0, 0)) + return _encodeImage(pilimage, format=format, **kwargs) frameWidth = iterInfo['output']['width'] frameHeight = iterInfo['output']['height'] maxWidth = kwargs.get('output', {}).get('maxWidth') @@ -2334,6 +2370,7 @@ def tileFrames(self, format=(TILE_FORMAT_IMAGE, ), frameList=None, outHeight = frameHeight * framesHigh tile = next(self._tileIterator(iterInfo)) image = None + tiledimage = None for idx, frame in enumerate(frameList): subimage, _ = self.getRegion(format=TILE_FORMAT_NUMPY, frame=frame, **kwargs) offsetX = (idx % framesAcross) * frameWidth @@ -2348,13 +2385,14 @@ def tileFrames(self, format=(TILE_FORMAT_IMAGE, ), frameList=None, 'Tiling frame %d (%d/%d), offset %dx%d', frame, idx, len(frameList), offsetX, offsetY) if tiled: - image = _addRegionTileToTiled( - image, subimage, offsetX, offsetY, outWidth, outHeight, tile, **kwargs) + tiledimage = _addRegionTileToTiled( + tiledimage, subimage, offsetX, offsetY, outWidth, outHeight, tile, **kwargs) else: image = _addSubimageToImage( image, subimage, offsetX, offsetY, outWidth, outHeight) if tiled: - return self._encodeTiledImage(image, outWidth, outHeight, iterInfo, **kwargs) + return self._encodeTiledImage( + cast(Dict[str, Any], tiledimage), outWidth, outHeight, iterInfo, **kwargs) return _encodeImage(image, format=format, **kwargs) def getRegionAtAnotherScale(self, sourceRegion, sourceScale=None, @@ -2406,7 +2444,7 @@ def getNativeMagnification(self): 'mm_y': None, } - def getMagnificationForLevel(self, level=None): + def getMagnificationForLevel(self, level: Optional[float] = None) -> Dict[str, float]: """ Get the magnification at a particular level. @@ -2425,13 +2463,15 @@ def getMagnificationForLevel(self, level=None): mag['mm_y'] *= mag['scale'] if self.levels: mag['level'] = level if level is not None else self.levels - 1 - if mag.get('level') == self.levels - 1: - mag['scale'] = 1.0 + if mag.get('level') == self.levels - 1: + mag['scale'] = 1.0 return mag - def getLevelForMagnification(self, magnification=None, exact=False, - mm_x=None, mm_y=None, rounding='round', - **kwargs): + def getLevelForMagnification( + self, magnification: Optional[float] = None, exact: bool = False, + mm_x: Optional[float] = None, mm_y: Optional[float] = None, + rounding: Optional[Union[str, bool]] = 'round', **kwargs, + ) -> Optional[Union[int, float]]: """ Get the level for a specific magnification or pixel size. If the magnification is unknown or no level is sufficient resolution, and an @@ -2718,7 +2758,7 @@ def getTileCount(self, *args, **kwargs): return tile['iterator_range']['position'] return 0 - def getAssociatedImagesList(self): + def getAssociatedImagesList(self) -> List[str]: """ Return a list of associated images. @@ -2726,7 +2766,8 @@ def getAssociatedImagesList(self): """ return [] - def getAssociatedImage(self, imageKey, *args, **kwargs): + def getAssociatedImage( + self, imageKey: str, *args, **kwargs) -> Optional[Tuple[ImageBytes, str]]: """ Return an associated image. @@ -2738,7 +2779,7 @@ def getAssociatedImage(self, imageKey, *args, **kwargs): """ image = self._getAssociatedImage(imageKey) if not image: - return + return None imageWidth, imageHeight = image.size width = kwargs.get('width') height = kwargs.get('height') @@ -2752,7 +2793,7 @@ def getAssociatedImage(self, imageKey, *args, **kwargs): getattr(PIL.Image, 'Resampling', PIL.Image).LANCZOS) return _encodeImage(image, **kwargs) - def getPixel(self, includeTileRecord=False, **kwargs): + def getPixel(self, includeTileRecord: bool = False, **kwargs) -> JSONDict: """ Get a single pixel from the current tile source. @@ -2769,27 +2810,30 @@ def getPixel(self, includeTileRecord=False, **kwargs): regionArgs['region'] = regionArgs.get('region', {}).copy() regionArgs['region']['width'] = regionArgs['region']['height'] = 1 regionArgs['region']['unitsWH'] = 'base_pixels' - pixel = {} + pixel: Dict[str, Any] = {} # This could be # img, format = self.getRegion(format=TILE_FORMAT_PIL, **regionArgs) # where img is the PIL image (rather than tile['tile'], but using # _tileIteratorInfo and the _tileIterator is slightly more efficient. iterInfo = self._tileIteratorInfo(format=TILE_FORMAT_NUMPY, **regionArgs) - if iterInfo is not None: - tile = next(self._tileIterator(iterInfo), None) - if includeTileRecord: - pixel['tile'] = tile - pixel['value'] = [v.item() for v in tile['tile'][0][0]] - img = _imageToPIL(tile['tile']) - if img.size[0] >= 1 and img.size[1] >= 1: - if len(img.mode) > 1: - pixel.update(dict(zip(img.mode.lower(), img.load()[0, 0]))) - else: - pixel.update(dict(zip([img.mode.lower()], [img.load()[0, 0]]))) + if iterInfo is None: + return JSONDict(pixel) + tile = next(self._tileIterator(iterInfo), None) + if tile is None: + return JSONDict(pixel) + if includeTileRecord: + pixel['tile'] = tile + pixel['value'] = [v.item() for v in tile['tile'][0][0]] + img = _imageToPIL(tile['tile']) + if img.size[0] >= 1 and img.size[1] >= 1: + if len(img.mode) > 1: + pixel.update(dict(zip(img.mode.lower(), img.load()[0, 0]))) + else: + pixel.update(dict(zip([img.mode.lower()], [img.load()[0, 0]]))) return JSONDict(pixel) @property - def frames(self): + def frames(self) -> int: """A property with the number of frames.""" if not hasattr(self, '_frameCount'): self._frameCount = len(self.getMetadata().get('frames', [])) or 1 @@ -2798,7 +2842,8 @@ def frames(self): class FileTileSource(TileSource): - def __init__(self, path, *args, **kwargs): + def __init__( + self, path: Union[str, pathlib.Path, Dict[Any, Any]], *args, **kwargs) -> None: """ Initialize the tile class. See the base class for other available parameters. @@ -2808,22 +2853,23 @@ def __init__(self, path, *args, **kwargs): super().__init__(*args, **kwargs) # Expand the user without converting datatype of path. try: - path = (path.expanduser() if callable(getattr(path, 'expanduser', None)) else - os.path.expanduser(path)) + path = (cast(pathlib.Path, path).expanduser() + if callable(getattr(path, 'expanduser', None)) else + os.path.expanduser(cast(str, path))) except TypeError: # Don't fail if the path is unusual -- maybe a source can handle it pass self.largeImagePath = path @staticmethod - def getLRUHash(*args, **kwargs): + def getLRUHash(*args, **kwargs) -> str: return strhash( args[0], kwargs.get('encoding', 'JPEG'), kwargs.get('jpegQuality', 95), kwargs.get('jpegSubsampling', 0), kwargs.get('tiffCompression', 'raw'), kwargs.get('edge', False), '__STYLESTART__', kwargs.get('style', None), '__STYLEEND__') - def getState(self): + def getState(self) -> str: if hasattr(self, '_classkey'): return self._classkey return '%s,%s,%s,%s,%s,%s,__STYLESTART__,%s,__STYLE_END__' % ( @@ -2835,11 +2881,11 @@ def getState(self): self.edge, self._jsonstyle) - def _getLargeImagePath(self): + def _getLargeImagePath(self) -> Union[str, pathlib.Path, Dict[Any, Any]]: return self.largeImagePath @classmethod - def canRead(cls, path, *args, **kwargs): + def canRead(cls, path: Union[str, pathlib.Path, Dict[Any, Any]], *args, **kwargs) -> bool: """ Check if we can read the input. This takes the same parameters as __init__. diff --git a/sources/dicom/large_image_source_dicom/__init__.py b/sources/dicom/large_image_source_dicom/__init__.py index d44efea66..b1a22fea5 100644 --- a/sources/dicom/large_image_source_dicom/__init__.py +++ b/sources/dicom/large_image_source_dicom/__init__.py @@ -265,6 +265,8 @@ def getNativeMagnification(self): mm_y = self._dicom.levels[0].pixel_spacing.height or None except Exception: pass + mm_x = float(mm_x) if mm_x else None + mm_y = float(mm_y) if mm_y else None # Estimate the magnification; we don't have a direct value mag = 0.01 / mm_x if mm_x else None return { diff --git a/sources/dicom/test_dicom/test_web_client.py b/sources/dicom/test_dicom/test_web_client.py index 4f75d1d52..f93e7a8af 100644 --- a/sources/dicom/test_dicom/test_web_client.py +++ b/sources/dicom/test_dicom/test_web_client.py @@ -22,7 +22,7 @@ def testDICOMWebClient(boundServer, fsAssetstore, db): spec = os.path.join(os.path.dirname(__file__), 'web_client_specs', 'dicomWebSpec.js') # Replace the template variables - with open(spec, 'r') as rf: + with open(spec) as rf: data = rf.read() dicomweb_test_url = os.environ['DICOMWEB_TEST_URL'] diff --git a/tox.ini b/tox.ini index 730956a91..f083ec2f7 100644 --- a/tox.ini +++ b/tox.ini @@ -4,6 +4,7 @@ envlist = docs lint lintclient + type notebook skip_missing_interpreters = true toxworkdir = {toxinidir}/build/tox @@ -71,6 +72,41 @@ allowlist_externals = {[testenv:test]allowlist_externals} commands = {[testenv:test]commands} setenv = {[testenv:test]setenv} +[testenv:monkeytype-py{37,38,39,310,311,312}] +passenv = {[testenv:test]passenv} +deps = + -rrequirements-dev.txt + coverage + mock + pooch + pymongo<4 + pytest>=3.6 + pytest-cov>=2.6 + pytest-custom-exit-code + pytest-girder>=3.1.25.dev9 + pytest-monkeytype + pytest-rerunfailures + pytest-xdist +allowlist_externals = {[testenv:test]allowlist_externals} +commands = + rm -rf build/test/coverage/web_temp + girder build --dev + pytest --numprocesses 0 -m 'not notebook' --no-cov --suppress-no-test-exit-code --monkeytype-output=./monkeytype.sqlite3 {posargs} + - npx nyc report --temp-dir build/test/coverage/web_temp --report-dir build/test/coverage --reporter cobertura --reporter text-summary +# After running tox, you can do +# build/tox/monkeytype-py311/bin/monkeytype list-modules +# and apply the results via +# build/tox/monkeytype-py311/bin/monkeytype apply +setenv = + NPM_CONFIG_FUND=false + NPM_CONFIG_AUDIT=false + NPM_CONFIG_AUDIT_LEVEL=high + NPM_CONFIG_LOGLEVEL=warn + NPM_CONFIG_PROGRESS=false + NPM_CONFIG_PREFER_OFFLINE=true + PIP_FIND_LINKS=https://girder.github.io/large_image_wheels + GDAL_PAM_ENABLED=no + [testenv:server] description = Run all tests except Girder client deps = {[testenv:test]deps} @@ -138,6 +174,23 @@ commands = ruff large_image sources utilities girder girder_annotation examples docs test flake8 +[testenv:type] +description = Check python types +skipsdist = true +deps = + -rrequirements-dev.txt + mypy + types-pillow + types-psutil +commands = + mypy --config-file tox.ini {posargs} + +[testenv:type-py{38,39,310,311,312}] +description = {[testenv:type]description} +skipsdist = true +deps = {[testenv:type]deps} +commands = {[testenv:type]commands} + [testenv:flake8] description = Lint python code skipsdist = true @@ -347,3 +400,65 @@ title = Large image Coverage Report [coverage:xml] output = build/test/coverage/py_coverage.xml + +[mypy] +python_version = 3.8 +install_types = true +non_interactive = true +ignore_missing_imports = true + +follow_imports = silent + +# Turn these all to true as we can +strict = True + +# Start off with these +warn_unused_configs = True +warn_redundant_casts = True +warn_unused_ignores = True + +# Getting these passing should be easy +strict_equality = True +strict_concatenate = True + +# Strongly recommend enabling this one as soon as you can +check_untyped_defs = True + +# These shouldn't be too much additional work, but may be tricky to +# get passing if you use a lot of untyped libraries +disallow_subclassing_any = False +disallow_untyped_decorators = True +disallow_any_generics = False + +# These next few are various gradations of forcing use of type annotations +disallow_untyped_calls = False +disallow_incomplete_defs = False +disallow_untyped_defs = False + +# This one isn't too hard to get passing, but return on investment is lower +no_implicit_reexport = False + +# This one can be tricky to get passing if you use a lot of untyped libraries +warn_return_any = False + +# files = . +files = + large_image/config.py, + large_image/constants.py, + large_image/exceptions.py, + large_image/tilesource/__init__.py, + large_image/tilesource/base.py +# large_image/, +# sources/, +# girder/, +# girder_annotation/, +# utilities/ +exclude = (?x)( + (^|/)build/ + | (^|/)docs/ + | (^|/)examples/ + | (^|/).*\.egg-info/ + | (^|/)setup\.py$ + | (^|/)test/ + | (^|/)test_.*/ + )