diff --git a/.circleci/make_wheels.sh b/.circleci/make_wheels.sh index 562592223..a8a74b6a2 100755 --- a/.circleci/make_wheels.sh +++ b/.circleci/make_wheels.sh @@ -31,6 +31,8 @@ cd "$ROOTPATH/sources/gdal" pip wheel . --no-deps -w ~/wheels && rm -rf build cd "$ROOTPATH/sources/mapnik" pip wheel . --no-deps -w ~/wheels && rm -rf build +cd "$ROOTPATH/sources/multi" +pip wheel . --no-deps -w ~/wheels && rm -rf build cd "$ROOTPATH/sources/nd2" pip wheel . --no-deps -w ~/wheels && rm -rf build cd "$ROOTPATH/sources/ometiff" diff --git a/.circleci/release_pypi.sh b/.circleci/release_pypi.sh index d6002e37f..0e6e48962 100755 --- a/.circleci/release_pypi.sh +++ b/.circleci/release_pypi.sh @@ -64,6 +64,12 @@ cp "$ROOTPATH/LICENSE" . python setup.py sdist pip wheel . --no-deps -w dist twine upload --verbose dist/* +cd "$ROOTPATH/sources/multi" +cp "$ROOTPATH/README.rst" . +cp "$ROOTPATH/LICENSE" . +python setup.py sdist +pip wheel . --no-deps -w dist +twine upload --verbose dist/* cd "$ROOTPATH/sources/nd2" cp "$ROOTPATH/README.rst" . cp "$ROOTPATH/LICENSE" . diff --git a/CHANGELOG.md b/CHANGELOG.md index ee881e4fd..0e41b1a08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +### Features +- Initial implementation of multi-source tile source ([#764](../../pull/764)) + ### Improvements - Add more opacity support for image overlays ([#761](../../pull/761)) - Make annotation schema more uniform ([#763](../../pull/763)) diff --git a/README.rst b/README.rst index 449120d36..17a505df0 100644 --- a/README.rst +++ b/README.rst @@ -96,6 +96,8 @@ Large Image consists of several Python modules designed to work together. These - ``large-image-source-deepzoom``: A tile source for reading Deepzoom tiles. + - ``large-image-source-multi``: A tile source for compisiting other tile sources into a single multi-frame source. + - ``large-image-source-test``: A tile source that generates test tiles, including a simple fractal pattern. Useful for testing extreme zoom levels. - ``large-image-source-dummy``: A tile source that does nothing. diff --git a/docs/index.rst b/docs/index.rst index f42975c96..7ecd6a350 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,6 +20,8 @@ _build/large_image_source_dummy/modules _build/large_image_source_gdal/modules _build/large_image_source_mapnik/modules + _build/large_image_source_multi/modules + multi_source_specification _build/large_image_source_nd2/modules _build/large_image_source_ometiff/modules _build/large_image_source_openjpeg/modules diff --git a/docs/make_docs.sh b/docs/make_docs.sh index c09c77ca8..bd915eaf0 100755 --- a/docs/make_docs.sh +++ b/docs/make_docs.sh @@ -15,6 +15,7 @@ ln -s ../build/docs-work _build large_image_converter --help > _build/large_image_converter.txt python -c 'from girder_large_image_annotation.models import annotation;import json;print(json.dumps(annotation.AnnotationSchema.annotationSchema, indent=2))' > _build/annotation_schema.json +python -c 'import large_image_source_multi, json;print(json.dumps(large_image_source_multi.MultiSourceSchema, indent=2))' > _build/multi_source_schema.json sphinx-apidoc -f -o _build/large_image ../large_image sphinx-apidoc -f -o _build/large_image_source_bioformats ../sources/bioformats/large_image_source_bioformats @@ -22,6 +23,7 @@ sphinx-apidoc -f -o _build/large_image_source_deepzoom ../sources/deepzoom/large sphinx-apidoc -f -o _build/large_image_source_dummy ../sources/dummy/large_image_source_dummy sphinx-apidoc -f -o _build/large_image_source_gdal ../sources/gdal/large_image_source_gdal sphinx-apidoc -f -o _build/large_image_source_mapnik ../sources/mapnik/large_image_source_mapnik +sphinx-apidoc -f -o _build/large_image_source_multi ../sources/multi/large_image_source_multi sphinx-apidoc -f -o _build/large_image_source_nd2 ../sources/nd2/large_image_source_nd2 sphinx-apidoc -f -o _build/large_image_source_ometiff ../sources/ometiff/large_image_source_ometiff sphinx-apidoc -f -o _build/large_image_source_openjpeg ../sources/openjpeg/large_image_source_openjpeg diff --git a/docs/multi_source_specification.rst b/docs/multi_source_specification.rst new file mode 100644 index 000000000..60958c678 --- /dev/null +++ b/docs/multi_source_specification.rst @@ -0,0 +1,6 @@ +.. include:: ../sources/multi/docs/specification.rst + +This returns the following: + +.. include:: ../build/docs-work/multi_source_schema.json + :literal: diff --git a/large_image/tilesource/base.py b/large_image/tilesource/base.py index 8e0e2f898..44df53931 100644 --- a/large_image/tilesource/base.py +++ b/large_image/tilesource/base.py @@ -20,9 +20,10 @@ TileOutputPILFormat, dtypeToGValue) from .tiledict import LazyTileDict from .utilities import (_encodeImage, _gdalParameters, # noqa: F401 - _imageToNumpy, _imageToPIL, _letterboxImage, _vipsCast, - _vipsParameters, dictToEtree, etreeToDict, - getPaletteColors, nearPowerOfTwo) + _imageToNumpy, _imageToPIL, _letterboxImage, + _makeSameChannelDepth, _vipsCast, _vipsParameters, + dictToEtree, etreeToDict, getPaletteColors, + nearPowerOfTwo) class TileSource: @@ -1668,12 +1669,8 @@ def _addRegionTileToImage( raise exceptions.TileSourceError( 'Insufficient memory to get region of %d x %d pixels.' % ( width, height)) - if subimage.shape[2] > image.shape[2]: - newimage = numpy.ones((image.shape[0], image.shape[1], subimage.shape[2])) - newimage[:, :, :image.shape[2]] = image - image = newimage - image[y:y + subimage.shape[0], x:x + subimage.shape[1], - :subimage.shape[2]] = subimage + image, subimage = _makeSameChannelDepth(image, subimage) + image[y:y + subimage.shape[0], x:x + subimage.shape[1], :] = subimage return image def _vipsAddAlphaBand(self, vimg, *otherImages): diff --git a/large_image/tilesource/utilities.py b/large_image/tilesource/utilities.py index ad82d64a0..ca820de0a 100644 --- a/large_image/tilesource/utilities.py +++ b/large_image/tilesource/utilities.py @@ -543,3 +543,46 @@ def getAvailableNamedPalettes(includeColors=True, reduced=False): (key.rsplit('_', 1)[0] + '_' + str(int( key.rsplit('_', 1)[-1]) + 1)) not in palettes)} return sorted(palettes) + + +def _makeSameChannelDepth(arr1, arr2): + """ + Given two numpy arrays that are either two or three dimensions, make the + third dimension the same for both of them. Specifically, if they are two + dimensions, first convert to trhee dimensions with a single final value. + Otherwise, the dimensions are assumed to be channels of L, LA, RGB, RGBA, + or . If L is needed to change to RGB, it is repeated (LLL). + Missing A channels are filled with 1. + + :param arr1: one array to compare. + :param arr2: a second array to compare. + :returns: the two arrays, possibly modified. + """ + arrays = { + 'arr1': arr1, + 'arr2': arr2, + } + # Make sure we have 3 dimensional arrays + for key, arr in arrays.items(): + if len(arr.shape) == 2: + arrays[key] = numpy.resize(arr, (arr.shape[0], arr.shape[1], 1)) + # If any array is RGB, make sure all arrays are RGB. + for key, arr in arrays.items(): + other = arrays['arr1' if key == 'arr2' else 'arr2'] + if arr.shape[2] < 3 and other.shape[2] >= 3: + newarr = numpy.ones((arr.shape[0], arr.shape[1], arr.shape[2] + 2)) + newarr[:, :, 0:1] = arr[:, :, 0:1] + newarr[:, :, 1:2] = arr[:, :, 0:1] + newarr[:, :, 2:3] = arr[:, :, 0:1] + if arr.shape[2] == 2: + newarr[:, :, 3:4] = arr[:, :, 1:2] + arrays[key] = newarr + # If only one array has an A channel, make sure all arrays have an A + # channel + for key, arr in arrays.items(): + other = arrays['arr1' if key == 'arr2' else 'arr2'] + if arr.shape[2] < other.shape[2]: + newarr = numpy.ones((arr.shape[0], arr.shape[1], other.shape[2])) + newarr[:, :, :arr.shape[2]] = arr + arrays[key] = newarr + return arrays['arr1'], arrays['arr2'] diff --git a/requirements-dev-core.txt b/requirements-dev-core.txt index 5426cf02c..6df730b0d 100644 --- a/requirements-dev-core.txt +++ b/requirements-dev-core.txt @@ -3,6 +3,7 @@ -e sources/deepzoom -e sources/dummy -e sources/gdal +-e sources/multi -e sources/nd2 -e sources/openjpeg -e sources/openslide diff --git a/requirements-dev.txt b/requirements-dev.txt index 8b15b0d4d..2af38611d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,6 +6,7 @@ girder-jobs>=3.0.3 -e sources/deepzoom -e sources/dummy -e sources/gdal +-e sources/multi -e sources/nd2 -e sources/openjpeg -e sources/openslide diff --git a/requirements-worker.txt b/requirements-worker.txt index 375dedcbd..e90090059 100644 --- a/requirements-worker.txt +++ b/requirements-worker.txt @@ -2,6 +2,7 @@ -e sources/deepzoom -e sources/dummy -e sources/gdal +-e sources/multi -e sources/nd2 -e sources/openjpeg -e sources/openslide diff --git a/setup.py b/setup.py index 9d57c5a36..48f96b900 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ 'dummy': ['large-image-source-dummy'], 'gdal': ['large-image-source-gdal'], 'mapnik': ['large-image-source-mapnik'], + 'multi': ['large-image-source-multi'], 'nd2': ['large-image-source-nd2'], 'ometiff': ['large-image-source-ometiff'], 'openjpeg': ['large-image-source-openjpeg'], diff --git a/sources/multi/docs/specification.rst b/sources/multi/docs/specification.rst new file mode 100644 index 000000000..32cadbbd1 --- /dev/null +++ b/sources/multi/docs/specification.rst @@ -0,0 +1,136 @@ +Multi Source Schema +=================== + +A multi-source tile source is used to composite multiple other sources into a +single conceptual tile source. It is specified by a yaml or json file that +conforms to the appropriate schema. + +Examples +-------- + +All of the examples presented here are in yaml; json works just as well. + +Multi Z-position +~~~~~~~~~~~~~~~~ + +For example, if you have a set of individual files that you wish to treat as +multiple z slices in a single file, you can do something like: + +:: + + --- + sources: + - path: ./test_orient1.tif + z: 0 + - path: ./test_orient2.tif + z: 1 + - path: ./test_orient3.tif + z: 2 + - path: ./test_orient4.tif + z: 3 + - path: ./test_orient5.tif + z: 4 + - path: ./test_orient6.tif + z: 5 + - path: ./test_orient7.tif + z: 6 + - path: ./test_orient8.tif + z: 7 + +Here, each of the files is explicitly listed with a specific ``z`` value. +Since these files are ordered, this could equivalently be done in a simpler +manner using a ``pathPattern``, which is a regular expression that can match +multiple files. + +:: + + --- + sources: + - path: . + pathPattern: 'test_orient[1-8]\.tif' + zStep: 1 + +Since the ``z`` value will default to 0, this works. The files are sorted in +C-sort order (lexically using the ASCII or UTF code points). This sorting will +break down if you have files with variable length numbers (e.g., ``file10.tif`` +will appear before ``file9.tiff``. You can instead assign values from the +file name using named expressions: + +:: + + --- + sources: + - path: . + pathPattern: 'test_orient(?P[1-8])\.tif' + +Note that the name in the expression (``z1`` in this example) is the name of +the value in the schema. If a ``1`` is added, then it is assumed to be 1-based +indexed. Without the ``1``, it is assumed to be zero-indexed. + +Composite To A Single Frame +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Multiple sources can be made to appear as a single frame. For instance: + +:: + + --- + width: 360 + height: 360 + sources: + - path: ./test_orient1.tif + position: + x: 0 + y: 0 + - path: ./test_orient2.tif + position: + x: 180 + y: 0 + - path: ./test_orient3.tif + position: + x: 0 + y: 180 + - path: ./test_orient4.tif + position: + x: 180 + y: 180 + +Here, the total width and height of the final image is specified, along with +the upper-left position of each image in the frame. + +Composite With Scaling +~~~~~~~~~~~~~~~~~~~~~~ + +Transforms can be applied to scale the individual sources: + +:: + + --- + width: 720 + height: 720 + sources: + - path: ./test_orient1.tif + position: + scale: 2 + - path: ./test_orient2.tif + position: + scale: 2 + x: 360 + - path: ./test_orient3.tif + position: + scale: 2 + y: 360 + - path: ./test_orient4.tif + position: + scale: 360 + x: 180 + y: 180 + +Note that the zero values from the previous example have been omitted as they +are unnecessary. + +Full Schema +----------- + +The full schema (jsonschema Draft6 standard) can be obtained by referencing the +Python at ``large_image_source_multi.MultiSourceSchema``. diff --git a/sources/multi/large_image_source_multi/__init__.py b/sources/multi/large_image_source_multi/__init__.py new file mode 100644 index 000000000..0b2193d83 --- /dev/null +++ b/sources/multi/large_image_source_multi/__init__.py @@ -0,0 +1,928 @@ +import builtins +import copy +import itertools +import json +import math +import os +import re +import warnings +from pathlib import Path + +import jsonschema +import numpy +import yaml +from pkg_resources import DistributionNotFound, get_distribution + +import large_image +from large_image.cache_util import LruCacheMetaclass, methodcache +from large_image.constants import TILE_FORMAT_NUMPY, SourcePriority +from large_image.exceptions import TileSourceError, TileSourceFileNotFoundError +from large_image.tilesource import FileTileSource +from large_image.tilesource.utilities import _makeSameChannelDepth + +try: + __version__ = get_distribution(__name__).version +except DistributionNotFound: + # package is not installed + pass + + +warnings.filterwarnings('ignore', category=UserWarning, module='glymur') + + +MultiSourceSchema = { + '$schema': 'http://json-schema.org/schema#', + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'name': {'type': 'string'}, + 'description': {'type': 'string'}, + 'width': {'type': 'integer', 'exclusiveMinimum': 0}, + 'height': {'type': 'integer', 'exclusiveMinimum': 0}, + 'tileWidth': {'type': 'integer', 'exclusiveMinimum': 0}, + 'tileHeight': {'type': 'integer', 'exclusiveMinimum': 0}, + 'channels': { + 'description': 'A list of channel names', + 'type': 'array', + 'items': {'type': 'string'}, + 'minItems': 1, + }, + 'scale': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'mm_x': {'type': 'number', 'exclusiveMinimum': 0}, + 'mm_y': {'type': 'number', 'exclusiveMinimum': 0}, + 'magnification': {'type': 'integer', 'exclusiveMinimum': 0}, + }, + }, + # 'projection': { + # 'description': 'If specified, sources are treated as ' + # 'non-geospatial and then this projection is added', + # 'type': 'string', + # }, + # corner points in the projection? + 'backgroundColor': { + 'description': 'A list of background color values in the same ' + 'scale and band order as the first tile source', + 'type': 'array', + 'items': {'type': 'number'}, + }, + 'uniformSources': { + 'description': + 'If true and the first two sources are similar in frame ' + 'layout and size, assume all sources are so similar', + 'type': 'boolean', + }, + 'sources': { + 'type': 'array', + 'items': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'name': {'type': 'string'}, + 'description': {'type': 'string'}, + 'path': { + 'decription': + 'The relative path, including file name if ' + 'pathPattern is not specified. The relative path ' + 'excluding file name if pathPttern is specified. ' + 'Or, girder://id for Girder sources.', + 'type': 'string', + }, + 'pathPattern': { + 'description': + 'If specified, file names in the path are matched ' + 'to this regular expression, sorted in C-sort ' + 'order. This can populate other properties via ' + 'named expressions, e.g., base_(?\\d+).png.', + 'type': 'string', + }, + # 'projection': { + # 'description': + # 'If specified, the source is treated as ' + # 'non-geospatial and then a projection is added. ' + # 'Set to None/null to use its own projection if a ' + # 'overall projection was specified.', + # 'type': 'string', + # }, + # corner points in the projection? + 'frame': { + 'description': + 'Base value for all frames; only use this if the ' + 'data does not conceptually have z, t, xy, or c ' + 'arrangement.', + 'type': 'integer', + 'minimum': 0, + }, + 'z': { + 'description': 'Base value for all frames', + 'type': 'integer', + 'minimum': 0, + }, + 't': { + 'description': 'Base value for all frames', + 'type': 'integer', + 'minimum': 0, + }, + 'xy': { + 'description': 'Base value for all frames', + 'type': 'integer', + 'minimum': 0, + }, + 'c': { + 'description': 'Base value for all frames', + 'type': 'integer', + 'minimum': 0, + }, + 'zValues': { + 'description': + 'The numerical z position of the different z ' + 'indices of the source. If only one value is ' + 'specified, other indices are shifted based on the ' + 'source. If fewer values are given than z ' + 'indices, the last two value given imply a stride ' + 'for the remainder.', + 'type': 'array', + 'items': {'type': 'number'}, + 'minItems': 1, + }, + 'tValues': { + 'description': + 'The numerical t position of the different t ' + 'indices of the source. If only one value is ' + 'specified, other indices are shifted based on the ' + 'source. If fewer values are given than t ' + 'indices, the last two value given imply a stride ' + 'for the remainder.', + 'type': 'array', + 'items': {'type': 'number'}, + 'minItems': 1, + }, + 'xyValues': { + 'description': + 'The numerical xy position of the different xy ' + 'indices of the source. If only one value is ' + 'specified, other indices are shifted based on the ' + 'source. If fewer values are given than xy ' + 'indices, the last two value given imply a stride ' + 'for the remainder.', + 'type': 'array', + 'items': {'type': 'number'}, + 'minItems': 1, + }, + 'cValues': { + 'description': + 'The numerical c position of the different c ' + 'indices of the source. If only one value is ' + 'specified, other indices are shifted based on the ' + 'source. If fewer values are given than c ' + 'indices, the last two value given imply a stride ' + 'for the remainder.', + 'type': 'array', + 'items': {'type': 'number'}, + 'minItems': 1, + }, + 'frameValues': { + 'description': + 'The numerical frame position of the different ' + 'frame indices of the source. If only one value ' + 'is specified, other indices are shifted based ' + 'on the source. If fewer values are given than ' + 'frame indices, the last two value given imply a ' + 'stride for the remainder.', + 'type': 'array', + 'items': {'type': 'number'}, + 'minItems': 1, + }, + 'channel': { + 'description': + 'A channel name to correspond with the main ' + 'image. Ignored if c, cValues, or channels is ' + 'specified.', + 'type': 'string', + }, + 'channels': { + 'description': + 'A list of channel names used to correspond ' + 'channels in this source with the main image. ' + 'Ignored if c or cValues is specified.', + 'type': 'array', + 'items': {'type': 'string'}, + 'minItems': 1, + }, + 'zStep': { + 'description': + 'Step value for multiple files included via ' + 'pathPattern. Applies to z or zValues', + 'type': 'integer', + 'exclusiveMinimum': 0, + }, + 'tStep': { + 'description': + 'Step value for multiple files included via ' + 'pathPattern. Applies to t or tValues', + 'type': 'integer', + 'exclusiveMinimum': 0, + }, + 'xyStep': { + 'description': + 'Step value for multiple files included via ' + 'pathPattern. Applies to x or xyValues', + 'type': 'integer', + 'exclusiveMinimum': 0, + }, + 'xStep': { + 'description': + 'Step value for multiple files included via ' + 'pathPattern. Applies to c or cValues', + 'type': 'integer', + 'exclusiveMinimum': 0, + }, + 'position': { + 'type': 'object', + 'additionalProperties': False, + 'description': + 'The image can be translated with x, y offset, ' + 'apply an affine transform, and scaled. If only ' + 'part of the source is desired, a crop can be ' + 'applied before the transformation.', + 'properties': { + 'x': {'type': 'number'}, + 'y': {'type': 'number'}, + 'crop': { + 'description': + 'Crop the source before applying a ' + 'position transform', + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'left': {'type': 'integer'}, + 'top': {'type': 'integer'}, + 'right': {'type': 'integer'}, + 'bottom': {'type': 'integer'}, + }, + # TODO: Add polygon option + # TODO: Add postTransform option + }, + 'scale': { + 'description': + 'Values less than 1 will downsample the ' + 'source. Values greater than 1 will ' + 'upsampled it.', + 'type': 'number', + 'exclusiveMinimum': 0, + }, + 's11': {'type': 'number'}, + 's12': {'type': 'number'}, + 's21': {'type': 'number'}, + 's22': {'type': 'number'}, + }, + }, + 'frames': { + 'description': 'List of frames to use from source', + 'type': 'array', + 'items': {'type': 'integer'}, + }, + 'style': {'type': 'object'}, + 'params': { + 'description': + 'Additional parameters to pass to the base tile ' + 'source', + 'type': 'object', + }, + }, + 'required': [ + 'path', + ], + }, + }, + # TODO: add merge method for cases where the are pixels from multiple + # sources in the same output location. + }, + 'required': [ + 'sources', + ], +} + + +class MultiFileTileSource(FileTileSource, metaclass=LruCacheMetaclass): + """ + Provides tile access to a composite of other tile sources. + """ + + cacheName = 'tilesource' + name = 'multifile' + extensions = { + None: SourcePriority.MEDIUM, + 'json': SourcePriority.PREFERRED, + 'yaml': SourcePriority.PREFERRED, + 'yml': SourcePriority.PREFERRED, + } + mimeTypes = { + None: SourcePriority.FALLBACK, + 'application/json': SourcePriority.PREFERRED, + 'application/yaml': SourcePriority.PREFERRED, + } + + _minTileSize = 64 + _maxTileSize = 4096 + _defaultTileSize = 256 + _maxOpenHandles = 6 + + _validator = jsonschema.Draft6Validator(MultiSourceSchema) + + def __init__(self, path, **kwargs): + """ + Initialize the tile class. See the base class for other available + parameters. + + :param path: a filesystem path for the tile source. + """ + super().__init__(path, **kwargs) + + self._largeImagePath = self._getLargeImagePath() + try: + with builtins.open(self._largeImagePath) as fptr: + start = fptr.read(1024).strip() + if start[:1] not in ('{', '#', '-') and (start[:1] < 'a' or start[:1] > 'z'): + raise TileSourceError('File cannot be opened via multi-source reader.') + fptr.seek(0) + try: + self._info = json.load(fptr) + except (json.JSONDecodeError, UnicodeDecodeError): + fptr.seek(0) + self._info = yaml.safe_load(fptr) + except (json.JSONDecodeError, yaml.YAMLError, UnicodeDecodeError): + raise TileSourceError('File cannot be opened via multi-source reader.') + except FileNotFoundError: + if not os.path.isfile(self._largeImagePath): + raise TileSourceFileNotFoundError(self._largeImagePath) from None + raise + self._validator.validate(self._info) + self._basePath = Path(self._largeImagePath).parent + self._collectFrames() + + def _resolvePathPatterns(self, sources, source): + """ + Given a source resolve pathPattern entries to specific paths. + Ensure that all paths exist. + + :param sources: a list to append found sources to. + :param source: the specific source record with a pathPattern to + resolve. + """ + kept = [] + pattern = re.compile(source['pathPattern']) + basedir = self._basePath / source['path'] + if (self._basePath.name == Path(self._largeImagePath).name and + (self._basePath.parent / source['path']).is_dir()): + basedir = self._basePath.parent / source['path'] + basedir = basedir.resolve() + for entry in basedir.iterdir(): + match = pattern.search(entry.name) + if match: + if entry.is_file(): + kept.append((entry.name, entry, match)) + elif entry.is_dir() and (entry / entry.name).is_file(): + kept.append((entry.name, entry / entry.name, match)) + for idx, (_, entry, match) in enumerate(sorted(kept)): + subsource = copy.deepcopy(source) + # Use named match groups to augment source values. + for k, v in match.groupdict().items(): + if v.isdigit(): + v = int(v) + if k.endswith('1'): + v -= 1 + if '.' in k: + subsource.setdefault(k.split('.', 1)[0], {})[k.split('.', 1)[1]] = v + else: + subsource[k] = v + subsource['path'] = entry + for axis in ['z', 't', 'xy', 'c']: + stepKey = '%sStep' % axis + valuesKey = '%sValues' % axis + if stepKey in source: + if axis in source or valuesKey not in source: + subsource[axis] = subsource.get(axis, 0) + idx * source[stepKey] + else: + subsource[valuesKey] = [ + val + idx * source[stepKey] for val in subsource[valuesKey]] + del subsource['pathPattern'] + sources.append(subsource) + + def _resolveFramePaths(self, sourceList): + """ + Given a list of sources, resolve path and pathPattern entries to + specific paths. + Ensure that all paths exist. + + :param sourceList: a list of source entries to resolve and check. + :returns: sourceList: a expanded and checked list of sources. + """ + # we want to work with both _basePath / and + # _basePath / .. / / to be compatible with Girder + # resource layouts. + sources = [] + for source in sourceList: + if source.get('pathPattern'): + self._resolvePathPatterns(sources, source) + else: + source = copy.deepcopy(source) + sourcePath = Path(source['path']) + source['path'] = self._basePath / sourcePath + if not source['path'].is_file(): + altpath = self._basePath.parent / sourcePath / sourcePath.name + if altpath.is_file(): + source['path'] = altpath + if not source['path'].is_file(): + raise TileSourceFileNotFoundError(str(source['path'])) + sources.append(source) + return sources + + def _sourceBoundingBox(self, source, width, height): + """ + Given a source with a possible transform and an image width and height, + compute the bounding box for the source. If a crop is used, it is + included in the results. If a non-identify transform is used, both it + and its inverse are included in the results. + + :param source: a dictionary that may have a position record. + :param width: the width of the source to transform. + :param height: the height of the source to transform. + :returns: a dictionary with left, top, right, bottom of the bounding + box in the final coordinate space. + """ + pos = source.get('position') + bbox = {'left': 0, 'top': 0, 'right': width, 'bottom': height} + if not pos: + return bbox + x0, y0, x1, y1 = 0, 0, width, height + if 'crop' in pos: + x0 = min(max(pos['crop'].get('left', x0), 0), width) + y0 = min(max(pos['crop'].get('top', y0), 0), height) + x1 = min(max(pos['crop'].get('right', x1), x0), width) + y1 = min(max(pos['crop'].get('bottom', y1), y0), height) + bbox['crop'] = {'left': x0, 'top': y0, 'right': x1, 'bottom': y1} + corners = numpy.array([[x0, y0, 1], [x1, y0, 1], [x0, y1, 1], [x1, y1, 1]]) + m = numpy.identity(3) + m[0][0] = pos.get('s11', 1) * pos.get('scale', 1) + m[0][1] = pos.get('s12', 0) * pos.get('scale', 1) + m[0][2] = pos.get('x', 0) + m[1][0] = pos.get('s21', 0) * pos.get('scale', 1) + m[1][1] = pos.get('s22', 1) * pos.get('scale', 1) + m[1][2] = pos.get('y', 0) + if not numpy.array_equal(m, numpy.identity(3)): + bbox['transform'] = m + try: + bbox['inverse'] = numpy.linalg.inv(m) + except numpy.linalg.LinAlgError: + raise TileSourceError('The position for a source is not invertable (%r)', pos) + transcorners = numpy.dot(m, corners.T) + bbox['left'] = min(transcorners[0]) + bbox['top'] = min(transcorners[1]) + bbox['right'] = max(transcorners[0]) + bbox['bottom'] = max(transcorners[1]) + return bbox + + def _axisKey(self, source, value, key): + """ + Get the value for a particular axis given the source specification. + + :param source: a source specification. + :param value: a default or initial value. + :param key: the axis key. One of frame, c, z, t, xy. + :returns: the axis key (an integer). + """ + vals = source.get('%sValues' % key) or [] + if not vals: + axisKey = value + source.get(key, 0) + elif len(vals) == 1: + axisKey = vals[0] + value + source.get(key, 0) + elif value < len(vals): + axisKey = vals[value] + source.get(key, 0) + else: + axisKey = (vals[len(vals) - 1] + (vals[len(vals) - 1] - vals[len(vals) - 2]) * + (value - len(vals) + source.get(key, 0))) + return axisKey + + def _addSourceToFrames(self, tsMeta, source, sourceIdx, frameDict): + """ + Add a source to the all appropriate frames. + + :param tsMeta: metadata from the source or from a matching uniform + source. + :param source: the source record. + :param sourceIdx: the index of the source. + :param frameDict: a dictionary to log the found frames. + """ + frames = tsMeta.get('frames', [{'Frame': 0, 'Index': 0}]) + # Channel names + channels = tsMeta.get('channels', []) + if source.get('channels'): + channels[:len(source['channels'])] = source['channels'] + elif source.get('channel'): + channels[:1] = [source['channel']] + if len(channels) > len(self._channels): + self._channels += channels[len(self._channels):] + if not any(key in source for key in { + 'frame', 'c', 'z', 't', 'xy', + 'frameValues', 'cValues', 'zValues', 'tValues', 'xyValues'}): + source = source.copy() + if len(frameDict['byFrame']): + source['frame'] = max(frameDict['byFrame'].keys()) + 1 + if len(frameDict['byAxes']): + source['z'] = max(aKey[1] for aKey in frameDict['byAxes']) + 1 + for frameIdx, frame in enumerate(frames): + if 'frames' in source and frameIdx not in source['frames']: + continue + fKey = self._axisKey(source, frameIdx, 'frame') + cIdx = frame.get('IndexC', 0) + zIdx = frame.get('IndexZ', 0) + tIdx = frame.get('IndexT', 0) + xyIdx = frame.get('IndexXY', 0) + aKey = (self._axisKey(source, cIdx, 'c'), + self._axisKey(source, zIdx, 'z'), + self._axisKey(source, tIdx, 't'), + self._axisKey(source, xyIdx, 'xy')) + channel = channels[cIdx] if cIdx < len(channels) else None + if channel and channel not in self._channels and ( + 'channel' in source or 'channels' in source): + self._channels.append(channel) + if (channel and channel in self._channels and + 'c' not in source and 'cValues' not in source): + aKey = (self._channels.index(channel), aKey[1], aKey[2], aKey[3]) + kwargs = source.get('params', {}).copy() + if 'style' in source: + kwargs['style'] = source['style'] + kwargs.pop('frame', None) + kwargs.pop('encoding', None) + frameDict['byFrame'].setdefault(fKey, []) + frameDict['byFrame'][fKey].append({ + 'sourcenum': sourceIdx, + 'frame': frameIdx, + 'kwargs': kwargs, + }) + frameDict['axesAllowed'] = frameDict['axesAllowed'] and ( + len(frames) <= 1 or 'IndexRange' in tsMeta) + frameDict['byAxes'].setdefault(aKey, []) + frameDict['byAxes'][aKey].append({ + 'sourcenum': sourceIdx, + 'frame': frameIdx, + 'kwargs': kwargs, + }) + + def _frameDictToFrames(self, frameDict): + """ + Given a frame dictionary, populate a frame list. + + :param frameDict: a dictionary with known frames stored in byAxes if + axesAllowed is True or byFrame if it is False. + :returns: a list of frames with enough information to generate them. + """ + frames = [] + if not frameDict['axesAllowed']: + frameCount = max(frameDict['byFrame']) + 1 + for frameIdx in range(frameCount): + frame = {'sources': frameDict['byFrame'].get(frameIdx, [])} + frames.append(frame) + else: + axesCount = [max(aKey[idx] for aKey in frameDict['byAxes']) + 1 for idx in range(4)] + for xy, t, z, c in itertools.product( + range(axesCount[3]), range(axesCount[2]), + range(axesCount[1]), range(axesCount[0])): + aKey = (c, z, t, xy) + frame = { + 'sources': frameDict['byAxes'].get(aKey, []), + } + if axesCount[0] > 1: + frame['IndexC'] = c + if axesCount[1] > 1: + frame['IndexZ'] = z + if axesCount[2] > 1: + frame['IndexT'] = t + if axesCount[3] > 1: + frame['IndexXY'] = xy + frames.append(frame) + return frames + + def _collectFrames(self, checkAll=False): + """ + Using the specification in _info, enumerate the source files and open + at least the first two of them to build up the frame specifications. + + :param checkAll: if True, open all source files. + """ + self._sources = sources = self._resolveFramePaths(self._info['sources']) + self.logger.debug('Sources: %r', sources) + + frameDict = {'byFrame': {}, 'byAxes': {}, 'axesAllowed': True} + numChecked = 0 + + self._associatedImages = {} + self._sourcePaths = {} + self._channels = self._info.get('channels') or [] + + absLargeImagePath = os.path.abspath(self._largeImagePath) + computedWidth = computedHeight = 0 + self.tileWidth = self._info.get('tileWidth') + self.tileHeight = self._info.get('tileHeight') + self._nativeMagnification = { + 'mm_x': self._info.get('scale', {}).get('mm_x') or None, + 'mm_y': self._info.get('scale', {}).get('mm_y') or None, + 'magnification': self._info.get('scale', {}).get('magnification') or None, + } + # Walk through the sources, opening at least the first two, and + # construct a frame list. Each frame is a list of sources that affect + # it along with the frame number from that source. + for sourceIdx, source in enumerate(sources): + path = source['path'] + if os.path.abspath(path) == absLargeImagePath: + raise TileSourceError('Multi source specification is self-referential') + if numChecked < 2 or checkAll or not self._info.get('uniformSources'): + # need kwargs of frame, style? + ts = large_image.open(path, **source.get('params', {})) + self.tileWidth = self.tileWidth or ts.tileWidth + self.tileHeight = self.tileHeight or ts.tileHeight + if not numChecked: + tsMag = ts.getNativeMagnification() + for key in self._nativeMagnification: + self._nativeMagnification[key] = ( + self._nativeMagnification[key] or tsMag.get(key)) + numChecked += 1 + tsMeta = ts.getMetadata() + bbox = self._sourceBoundingBox(source, tsMeta['sizeX'], tsMeta['sizeY']) + computedWidth = max(computedWidth, int(math.ceil(bbox['right']))) + computedHeight = max(computedHeight, int(math.ceil(bbox['bottom']))) + # Record this path + if path not in self._sourcePaths: + self._sourcePaths[path] = { + 'frames': set(), + 'sourcenum': set(), + } + # collect associated images + for basekey in ts.getAssociatedImagesList(): + key = basekey + keyidx = 0 + while key in self._associatedImages: + keyidx += 1 + key = '%s-%d' % (basekey, keyidx) + self._associatedImages[key] = { + 'sourcenum': sourceIdx, + 'key': key + } + source['metadata'] = tsMeta + source['bbox'] = bbox + self._sourcePaths[path]['sourcenum'].add(sourceIdx) + # process metadata to determine what frames are used, etc. + self._addSourceToFrames(tsMeta, source, sourceIdx, frameDict) + # Check frameDict and create frame record + self._frames = self._frameDictToFrames(frameDict) + self.tileWidth = min(max(self.tileWidth, self._minTileSize), self._maxTileSize) + self.tileHeight = min(max(self.tileHeight, self._minTileSize), self._maxTileSize) + self.sizeX = self._info.get('width') or computedWidth + self.sizeY = self._info.get('height') or computedHeight + self.levels = int(max(1, math.ceil(math.log( + max(self.sizeX / self.tileWidth, self.sizeY / self.tileHeight)) / math.log(2)) + 1)) + + def getNativeMagnification(self): + """ + Get the magnification at a particular level. + + :return: magnification, width of a pixel in mm, height of a pixel in mm. + """ + return self._nativeMagnification.copy() + + def getAssociatedImage(self, imageKey, *args, **kwargs): + """ + Return an associated image. + + :param imageKey: the key of the associated image to retrieve. + :param kwargs: optional arguments. Some options are width, height, + encoding, jpegQuality, jpegSubsampling, and tiffCompression. + :returns: imageData, imageMime: the image data and the mime type, or + None if the associated image doesn't exist. + """ + if imageKey not in self._associatedImages: + return + source = self._sources[self._associatedImages[imageKey]['sourcenum']] + ts = large_image.open(source['path'], **source.get('params', {})) + return ts.getAssociatedImage(self._associatedImages[imageKey]['key'], *args, **kwargs) + + def getAssociatedImagesList(self): + """ + Return a list of associated images. + + :return: the list of image keys. + """ + return list(sorted(self._associatedImages.keys())) + + def getMetadata(self): + """ + Return a dictionary of metadata containing levels, sizeX, sizeY, + tileWidth, tileHeight, magnification, mm_x, mm_y, and frames. + + :returns: metadata dictionary. + """ + result = super().getMetadata() + if len(self._frames) > 1: + result['frames'] = [ + {k: v for k, v in frame.items() if k.startswith('Index')} + for frame in self._frames] + self._addMetadataFrameInformation(result, self._channels) + return result + + def getInternalMetadata(self, **kwargs): + """ + Return additional known metadata about the tile source. Data returned + from this method is not guaranteed to be in any particular format or + have specific values. + + :returns: a dictionary of data or None. + """ + result = { + 'frames': copy.deepcopy(self._frames), + 'sources': copy.deepcopy(self._sources), + 'sourceFiles': [], + } + for path in self._sourcePaths.values(): + source = self._sources[min(path['sourcenum'])] + ts = large_image.open(source['path'], **source.get('params', {})) + result['sourceFiles'].append({ + 'path': source['path'], + 'internal': ts.getInternalMetadata(), + }) + return result + + def _mergeTiles(self, base, tile, x, y): + """ + Add a tile to an existing tile. The existing tile is expanded as + needed, and the number of channels will always be the greater of the + two. + + :param base: numpy array base tile. May be None. May be modified. + :param tile: numpy tile to add. + :param x: location to add the tile. + :param y: location to add the tile. + :returns: a numpy tile. + """ + # Replace non blank pixels, aggregating opacity appropriately + x = int(round(x)) + y = int(round(y)) + if base is None and not x and not y: + return tile + if base is None: + base = numpy.zeros((0, 0, tile.shape[2]), dtype=tile.dtype) + base, tile = _makeSameChannelDepth(base, tile) + if base.shape[0] < tile.shape[0] + y: + vfill = numpy.zeros((tile.shape[0] + y - base.shape[0], base.shape[1], base.shape[2])) + if base.shape[2] == 2 or base.shape[2] == 4: + vfill[:, :, -1] = 1 + base = numpy.vstack((base, vfill)) + if base.shape[1] < tile.shape[1] + x: + hfill = numpy.zeros((base.shape[0], tile.shape[1] + x - base.shape[1], base.shape[2])) + if base.shape[2] == 2 or base.shape[2] == 4: + hfill[:, :, -1] = 1 + base = numpy.hstack((base, hfill)) + base[y:y + tile.shape[0], x:x + tile.shape[1], :] = tile + return base + + def _addSourceToTile(self, tile, sourceEntry, corners, scale): + """ + Add a source to the current tile. + + :param tile: a numpy array with the tile, or None if there is no data + yet. + :param sourceEntry: the current record from the sourceList. This + contains the sourcenum, kwargs to apply when opening the source, + and the frame within the source to fetch. + :param corners: the four corners of the tile in the main image space + coordinates. + :param scale: power of 2 scale of the output; this is tne number of + pixels that are conceptually aggregated from the source for one + output pixel. + :returns: a numpy array of the tile. + """ + source = self._sources[sourceEntry['sourcenum']] + ts = large_image.open(source['path'], **sourceEntry['kwargs'], format=TILE_FORMAT_NUMPY) + # If tile is outside of bounding box, skip it + bbox = source['bbox'] + if (corners[2][0] <= bbox['left'] or corners[0][0] >= bbox['right'] or + corners[2][1] <= bbox['top'] or corners[0][1] >= bbox['bottom']): + return tile + transform = bbox.get('transform') + srccorners = ( + list(numpy.dot(bbox['inverse'], numpy.array(corners).T).T) + if transform is not None else corners) + x = y = 0 + # If there is no transform or the diagonals are positive and there is + # no sheer, use getRegion with an appropriate size (be wary of edges) + if (transform is None or + transform[0][0] > 0 and transform[0][1] == 0 and + transform[1][0] == 0 and transform[1][1] > 0): + scaleX = transform[0][0] if transform is not None else 1 + scaleY = transform[1][1] if transform is not None else 1 + region = { + 'left': srccorners[0][0], 'top': srccorners[0][1], + 'right': srccorners[2][0], 'bottom': srccorners[2][1] + } + output = { + 'maxWidth': (corners[2][0] - corners[0][0]) // scale, + 'maxHeight': (corners[2][1] - corners[0][1]) // scale, + } + if region['left'] < 0: + x -= region['left'] * scaleX // scale + output['maxWidth'] += int(region['left'] * scaleX // scale) + region['left'] = 0 + if region['top'] < 0: + y -= region['top'] * scaleY // scale + output['maxHeight'] += int(region['top'] * scaleY // scale) + region['top'] = 0 + if region['right'] > source['metadata']['sizeX']: + output['maxWidth'] -= int( + (region['right'] - source['metadata']['sizeX']) * scaleX // scale) + region['right'] = source['metadata']['sizeX'] + if region['bottom'] > source['metadata']['sizeY']: + output['maxHeight'] -= int( + (region['bottom'] - source['metadata']['sizeY']) * scaleY // scale) + region['bottom'] = source['metadata']['sizeY'] + for key in region: + region[key] = int(round(region[key])) + self.logger.debug('getRegion: ts: %r, region: %r, output: %r', ts, region, output) + sourceTile, _ = ts.getRegion( + region=region, output=output, frame=source.get('frame', 0), + format=TILE_FORMAT_NUMPY) + # Otherwise, get an area twice as big as needed and use + # scipy.ndimage.affine_transform to transform it + else: + # TODO + raise TileSourceError('Not implemented') + # Crop + # TODO + tile = self._mergeTiles(tile, sourceTile, x, y) + return tile + + @methodcache() + def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs): + frame = int(kwargs.get('frame') or 0) + self._xyzInRange(x, y, z, frame, len(self._frames) if hasattr(self, '_frames') else None) + scale = 2 ** (self.levels - 1 - z) + corners = [[ + x * self.tileWidth * scale, + y * self.tileHeight * scale, + 1, + ], [ + min((x + 1) * self.tileWidth * scale, self.sizeX), + y * self.tileHeight * scale, + 1, + ], [ + min((x + 1) * self.tileWidth * scale, self.sizeX), + min((y + 1) * self.tileHeight * scale, self.sizeY), + 1, + ], [ + x * self.tileWidth * scale, + min((y + 1) * self.tileHeight * scale, self.sizeY), + 1, + ]] + sourceList = self._frames[frame]['sources'] + tile = None + # If the first source does not completely cover the output tile or uses + # a transformation, create a tile that is the desired size and fill it + # with the background color. + fill = not len(sourceList) + if not fill: + firstsource = self._sources[sourceList[0]['sourcenum']] + fill = 'transform' in firstsource['bbox'] or any( + cx < firstsource['bbox']['left'] or + cx > firstsource['bbox']['right'] or + cy < firstsource['bbox']['top'] or + cy > firstsource['bbox']['bottom'] for cx, cy, _ in corners) + if fill: + colors = self._info.get('backgroundColor') + if colors: + tile = numpy.full((self.tileWidth, self.tileHeight, len(colors)), colors) + # Add each source to the tile + for sourceEntry in sourceList: + tile = self._addSourceToTile(tile, sourceEntry, corners, scale) + if tile is None: + # TODO number of channels? + colors = self._info.get('backgroundColor', [0]) + if colors: + tile = numpy.full((self.tileWidth, self.tileHeight, len(colors)), colors) + # We should always have a tile + return self._outputTile(tile, TILE_FORMAT_NUMPY, x, y, z, + pilImageAllowed, numpyAllowed, **kwargs) + + +def open(*args, **kwargs): + """ + Create an instance of the module class. + """ + return MultiFileTileSource(*args, **kwargs) + + +def canRead(*args, **kwargs): + """ + Check if an input can be read by the module class. + """ + return MultiFileTileSource.canRead(*args, **kwargs) diff --git a/sources/multi/large_image_source_multi/girder_source.py b/sources/multi/large_image_source_multi/girder_source.py new file mode 100644 index 000000000..e84391551 --- /dev/null +++ b/sources/multi/large_image_source_multi/girder_source.py @@ -0,0 +1,15 @@ +from girder_large_image.girder_tilesource import GirderTileSource + +from . import MultiFileTileSource + + +class MultiGirderTileSource(MultiFileTileSource, GirderTileSource): + """ + Provides tile access to Girder items with files that the multi source can + read. + """ + + cacheName = 'tilesource' + name = 'multi' + + _mayHaveAdjacentFiles = True diff --git a/sources/multi/setup.py b/sources/multi/setup.py new file mode 100644 index 000000000..9ce58718e --- /dev/null +++ b/sources/multi/setup.py @@ -0,0 +1,66 @@ +import os + +from setuptools import find_packages, setup + +description = 'A tilesource for large_image to composite other tile sources' +long_description = description + '\n\nSee the large-image package for more details.' + + +def prerelease_local_scheme(version): + """ + Return local scheme version unless building on master in CircleCI. + + This function returns the local scheme version number + (e.g. 0.0.0.dev+g) unless building on CircleCI for a + pre-release in which case it ignores the hash and produces a + PEP440 compliant pre-release version number (e.g. 0.0.0.dev). + """ + from setuptools_scm.version import get_local_node_and_date + + if os.getenv('CIRCLE_BRANCH') in ('master', ): + return '' + else: + return get_local_node_and_date(version) + + +setup( + name='large-image-source-multi', + use_scm_version={'root': '../..', 'local_scheme': prerelease_local_scheme}, + setup_requires=['setuptools-scm'], + description=description, + long_description=long_description, + license='Apache Software License 2.0', + author='Kitware, Inc.', + author_email='kitware@kitware.com', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + ], + install_requires=[ + 'jsonschema', + 'large-image>=1.0.0', + 'pyyaml', + 'scipy', + ], + extras_require={ + 'girder': 'girder-large-image>=1.0.0', + }, + keywords='large_image, tile source', + packages=find_packages(exclude=['test', 'test.*']), + url='https://github.com/girder/large_image', + python_requires='>=3.6', + entry_points={ + 'large_image.source': [ + 'multi = large_image_source_multi:MultiFileTileSource' + ], + 'girder_large_image.source': [ + 'multi = large_image_source_multi.girder_source:MultiGirderTileSource' + ] + }, +) diff --git a/test/datastore.py b/test/datastore.py index 0eb10049e..64ee7c797 100644 --- a/test/datastore.py +++ b/test/datastore.py @@ -72,6 +72,9 @@ # Geospatial file without a projection # Source: generated from geojs oahu sample and a script 'oahu-dense.tiff': 'sha512:414b7807f14991d6f8229134ad6ccbc2cc2d4b05423ccebfd3ede7d7323dfcf04ef1a7b2c2b4c45c31f8a36bccd390782af3e7d3e99f01f1b95650c5da1f122b', # noqa + # Multi source file using different sources + # Source: manually generated. + 'multi_source.yml': 'sha512:81d7768b06eca6903082daa5b91706beaac8557ba4cede7f826524303df69a33478d6bb205c56af7ee2b45cd7d75897cc4b5704f743ddbf71bb3537ed3b9e8a8', # noqa } diff --git a/test/test_files/multi1.yml b/test/test_files/multi1.yml new file mode 100644 index 000000000..6626f43b1 --- /dev/null +++ b/test/test_files/multi1.yml @@ -0,0 +1,24 @@ +--- +name: Multi orientation +description: A test multi file +scale: + mm_x: 0.0005 + mm_y: 0.0005 +sources: + - name: First image + path: ./test_orient1.tif + z: 0 + - path: ./test_orient2.tif + z: 1 + - path: ./test_orient3.tif + z: 2 + - path: ./test_orient4.tif + z: 3 + - path: ./test_orient5.tif + z: 4 + - path: ./test_orient6.tif + z: 5 + - path: ./test_orient7.tif + z: 6 + - path: ./test_orient8.tif + z: 7 diff --git a/test/test_files/multi2.yml b/test/test_files/multi2.yml new file mode 100644 index 000000000..3ee9997a6 --- /dev/null +++ b/test/test_files/multi2.yml @@ -0,0 +1,10 @@ +--- +name: Multi orientation +description: A test multi file +scale: + mm_x: 0.0005 + mm_y: 0.0005 +sources: + - path: . + pathPattern: 'test_orient[1-8]\.tif' + zStep: 1 diff --git a/test/test_files/multi3.yml b/test/test_files/multi3.yml new file mode 100644 index 000000000..d56f27321 --- /dev/null +++ b/test/test_files/multi3.yml @@ -0,0 +1,9 @@ +--- +name: Multi orientation +description: A test multi file +scale: + mm_x: 0.0005 + mm_y: 0.0005 +sources: + - path: . + pathPattern: 'test_orient(?P[1-8])\.tif' diff --git a/test/test_files/multi_channels.yml b/test/test_files/multi_channels.yml new file mode 100644 index 000000000..d525db494 --- /dev/null +++ b/test/test_files/multi_channels.yml @@ -0,0 +1,27 @@ +--- +name: Multi orientationa +sources: + - path: ./test_orient1.tif + channel: CY3 + z: 0 + - path: ./test_orient2.tif + channel: A594 + z: 0 + - path: ./test_orient3.tif + channel: CY5 + z: 0 + - path: ./test_orient4.tif + channel: DAPI + z: 0 + - path: ./test_orient5.tif + channel: CY3 + z: 1 + - path: ./test_orient6.tif + channel: A594 + z: 1 + - path: ./test_orient7.tif + channel: CY5 + z: 1 + - path: ./test_orient8.tif + channel: DAPI + z: 1 diff --git a/test/test_files/multi_composite.yml b/test/test_files/multi_composite.yml new file mode 100644 index 000000000..ab9740466 --- /dev/null +++ b/test/test_files/multi_composite.yml @@ -0,0 +1,50 @@ +--- +name: Multi orientation +description: A test multi file +scale: + mm_x: 0.0005 + mm_y: 0.0005 +width: 360 +height: 360 +sources: + - name: First image + path: ./test_orient1.tif + z: 0 + position: + x: 0 + y: 0 + - path: ./test_orient2.tif + z: 0 + position: + x: 180 + y: 0 + - path: ./test_orient3.tif + z: 0 + position: + x: 0 + y: 180 + - path: ./test_orient4.tif + z: 0 + position: + x: 180 + y: 180 + - path: ./test_orient5.tif + z: 1 + position: + x: 0 + y: 0 + - path: ./test_orient6.tif + z: 1 + position: + x: 180 + y: 0 + - path: ./test_orient7.tif + z: 1 + position: + x: 0 + y: 180 + - path: ./test_orient8.tif + z: 1 + position: + x: 180 + y: 180 diff --git a/test/test_files/multi_simple_scaling.yml b/test/test_files/multi_simple_scaling.yml new file mode 100644 index 000000000..22f22310e --- /dev/null +++ b/test/test_files/multi_simple_scaling.yml @@ -0,0 +1,39 @@ +--- +name: Multi orientation +description: A test multi file +scale: + mm_x: 0.0005 + mm_y: 0.0005 +width: 2048 +height: 1540 +tileWidth: 256 +tileHeight: 256 +backgroundColor: + - 0 + - 0 + - 255 +sources: + - path: ./test_orient1.tif + z: 0 + - path: ./test_orient2.tif + z: 1 + position: + scale: 4 + - path: ./test_orient3.tif + z: 2 + position: + scale: 0.25 + - path: ./test_orient4.tif + z: 3 + position: + x: 137 + y: 32 + scale: 4 + - path: ./test_orient5.tif + z: 4 + - path: ./test_orient6.tif + z: 5 + - path: ./test_orient7.tif + z: 6 + - path: ./test_orient8.tif + z: 7 diff --git a/test/test_source_base.py b/test/test_source_base.py index 98bc80c81..19bd20392 100644 --- a/test/test_source_base.py +++ b/test/test_source_base.py @@ -13,6 +13,9 @@ # In general, if there is something in skipTiles, the reader should be improved # to either indicate that the file can't be read or changed to handle reading # with correct exceptions. +# 'skip' is used to exclude testing specific paths. This might be necessary +# if a file is dependent on other files as these generalized tests don't ensure +# a download order. SourceAndFiles = { 'bioformats': { 'read': r'\.(czi|jp2|svs|scn)$', @@ -30,6 +33,10 @@ 'noread': r'(huron\.image2_jpeg2k|sample_jp2k_33003|TCGA-DU-6399|\.(ome.tiff)$)', # we should only test this with a projection 'skipTiles': r''}, + 'multi': { + 'read': r'\.(yml|yaml)$', + 'skip': r'(multi_source\.yml)$', + }, 'nd2': {'read': r'\.(nd2)$'}, 'ometiff': {'read': r'\.(ome\.tif.*)$'}, 'openjpeg': {'read': r'\.(jp2)$'}, @@ -88,6 +95,8 @@ def testBaseFileNotFound(): @pytest.mark.parametrize('source', SourceAndFiles) def testSourcesCanRead(source, filename): sourceInfo = SourceAndFiles[source] + if re.search(sourceInfo.get('skip', r'^$'), filename): + pytest.skip('this file needs more complex tests') canRead = sourceInfo.get('any') or ( re.search(sourceInfo.get('read', r'^$'), filename) and not re.search(sourceInfo.get('noread', r'^$'), filename)) @@ -101,6 +110,8 @@ def testSourcesCanRead(source, filename): @pytest.mark.parametrize('source', SourceAndFiles) def testSourcesCanReadPath(source, filename): sourceInfo = SourceAndFiles[source] + if re.search(sourceInfo.get('skip', r'^$'), filename): + pytest.skip('this file needs more complex tests') canRead = sourceInfo.get('any') or ( re.search(sourceInfo.get('read', r'^$'), filename) and not re.search(sourceInfo.get('noread', r'^$'), filename)) @@ -114,6 +125,8 @@ def testSourcesCanReadPath(source, filename): @pytest.mark.parametrize('source', SourceAndFiles) def testSourcesTilesAndMethods(source, filename): sourceInfo = SourceAndFiles[source] + if re.search(sourceInfo.get('skip', r'^$'), filename): + pytest.skip('this file needs more complex tests') canRead = sourceInfo.get('any') or ( re.search(sourceInfo.get('read', r'^$'), filename) and not re.search(sourceInfo.get('noread', r'^$'), filename)) diff --git a/test/test_source_multi.py b/test/test_source_multi.py new file mode 100644 index 000000000..31e529a15 --- /dev/null +++ b/test/test_source_multi.py @@ -0,0 +1,109 @@ +import os + +import large_image_source_multi +import pytest + +from . import utilities +from .datastore import datastore + + +@pytest.fixture +def multiSourceImagePath(): + """ + Make sure we have the components for the multi_source.yml test. + """ + datastore.fetch('TCGA-AA-A02O-11A-01-BS1.8b76f05c-4a8b-44ba-b581-6b8b4f437367.svs') + datastore.fetch('DDX58_AXL_EGFR_well2_XY01.ome.tif') + datastore.fetch('ITGA3Hi_export_crop2.nd2') + datastore.fetch('sample_Easy1.png') + yield datastore.fetch('multi_source.yml') + + +@pytest.mark.parametrize('filename', [ + 'multi1.yml', + 'multi2.yml', + 'multi3.yml', + 'multi_channels.yml', +]) +def testTilesFromMulti(filename): + testDir = os.path.dirname(os.path.realpath(__file__)) + imagePath = os.path.join(testDir, 'test_files', filename) + source = large_image_source_multi.open(imagePath) + tileMetadata = source.getMetadata() + assert tileMetadata['tileWidth'] == 64 + assert tileMetadata['tileHeight'] == 64 + assert tileMetadata['sizeX'] == 180 + assert tileMetadata['sizeY'] == 180 + assert tileMetadata['levels'] == 3 + assert len(tileMetadata['frames']) == 8 + + utilities.checkTilesZXY(source, tileMetadata) + + +def testTilesFromMultiComposite(): + testDir = os.path.dirname(os.path.realpath(__file__)) + imagePath = os.path.join(testDir, 'test_files', 'multi_composite.yml') + source = large_image_source_multi.open(imagePath) + tileMetadata = source.getMetadata() + assert tileMetadata['tileWidth'] == 64 + assert tileMetadata['tileHeight'] == 64 + assert tileMetadata['sizeX'] == 360 + assert tileMetadata['sizeY'] == 360 + assert tileMetadata['levels'] == 4 + assert len(tileMetadata['frames']) == 2 + + utilities.checkTilesZXY(source, tileMetadata) + + +def testTilesFromMultiSimpleScaling(): + testDir = os.path.dirname(os.path.realpath(__file__)) + imagePath = os.path.join(testDir, 'test_files', 'multi_simple_scaling.yml') + source = large_image_source_multi.open(imagePath) + tileMetadata = source.getMetadata() + assert tileMetadata['tileWidth'] == 256 + assert tileMetadata['tileHeight'] == 256 + assert tileMetadata['sizeX'] == 2048 + assert tileMetadata['sizeY'] == 1540 + assert tileMetadata['levels'] == 4 + assert len(tileMetadata['frames']) == 8 + + for frame in range(len(tileMetadata['frames'])): + utilities.checkTilesZXY(source, tileMetadata, tileParams={'frame': frame}) + + +def testTilesFromMultiMultiSource(multiSourceImagePath): + imagePath = multiSourceImagePath + source = large_image_source_multi.open(imagePath) + tileMetadata = source.getMetadata() + assert tileMetadata['tileWidth'] == 256 + assert tileMetadata['tileHeight'] == 256 + assert tileMetadata['sizeX'] == 55988 + assert tileMetadata['sizeY'] == 16256 + assert tileMetadata['levels'] == 9 + assert len(tileMetadata['frames']) == 300 + + utilities.checkTilesZXY(source, tileMetadata) + utilities.checkTilesZXY(source, tileMetadata, tileParams={'frame': 50}) + + +def testInternalMetadata(multiSourceImagePath): + imagePath = multiSourceImagePath + source = large_image_source_multi.open(imagePath) + metadata = source.getInternalMetadata() + assert 'frames' in metadata + + +def testAssociatedImages(multiSourceImagePath): + imagePath = multiSourceImagePath + source = large_image_source_multi.open(imagePath) + assert 'label' in source.getAssociatedImagesList() + image, mimeType = source.getAssociatedImage('label') + assert image[:len(utilities.JPEGHeader)] == utilities.JPEGHeader + + +def testCanRead(): + testDir = os.path.dirname(os.path.realpath(__file__)) + imagePath = os.path.join(testDir, 'test_files', 'multi_composite.yml') + assert large_image_source_multi.canRead(imagePath) is True + imagePath2 = os.path.join(testDir, 'test_files', 'test_orient1.tif') + assert large_image_source_multi.canRead(imagePath2) is False