Force reshape tiff data by the adapter (#797)

* ENH: force reshape tiff data if any unitary dimensions are added/missing

* MNT: fix doctring format

* MNT: fix a small bug

* MNT: fix a small bug

* TST: tests for tiff reshaping

* MNT: add changelog entry

* MNT: lint

* MNT: fix comment

* MNT: move helper funcions to utils

* ENH: use ndindex instead of a custom function

* MNT: lint

* MNT: refactor to avoid importing numpy in utils

---------

Co-authored-by: Dan Allan <[email protected]>

CHANGELOG.md

@@ -9,6 +9,8 @@ Write the date in place of the "Unreleased" in the case a new version is release
 
 - Add adapters for reading back assets with the image/jpeg and
   multipart/related;type=image/jpeg mimetypes.
+- Automatic reshaping of tiff data by the adapter to account for
+  extra/missing singleton dimensions
 
 ### Changed
 

tiled/_tests/test_tiff.py

@@ -10,9 +10,11 @@
 from ..client import Context, from_context
 from ..client.register import IMG_SEQUENCE_EMPTY_NAME_ROOT, register
 from ..server.app import build_app
+from ..structures.array import ArrayStructure, BuiltinDtype
 from ..utils import ensure_uri
 
 COLOR_SHAPE = (11, 17, 3)
+rng = numpy.random.default_rng(12345)
 
 
 @pytest.fixture(scope="module")
@@ -21,20 +23,27 @@ def client(tmpdir_module):
     sequence_directory.mkdir()
     filepaths = []
     for i in range(3):
-        data = numpy.random.random((5, 7, 4))
+        data = rng.integers(0, 255, size=(5, 7, 4), dtype="uint8")
         filepath = sequence_directory / f"temp{i:05}.tif"
         tf.imwrite(filepath, data)
         filepaths.append(filepath)
-    color_data = numpy.random.randint(0, 255, COLOR_SHAPE, dtype="uint8")
+    color_data = rng.integers(0, 255, size=COLOR_SHAPE, dtype="uint8")
     path = Path(tmpdir_module, "color.tif")
     tf.imwrite(path, color_data)
-
     tree = MapAdapter(
         {
             "color": TiffAdapter(ensure_uri(path)),
             "sequence": TiffSequenceAdapter.from_uris(
                 [ensure_uri(filepath) for filepath in filepaths]
             ),
+            "5d_sequence": TiffSequenceAdapter.from_uris(
+                [ensure_uri(filepath) for filepath in filepaths],
+                structure=ArrayStructure(
+                    shape=(3, 1, 5, 7, 4),
+                    chunks=((1, 1, 1), (1,), (5,), (7,), (4,)),
+                    data_type=BuiltinDtype.from_numpy_dtype(numpy.dtype("uint8")),
+                ),
+            ),
         }
     )
     app = build_app(tree)
@@ -62,6 +71,27 @@ def test_tiff_sequence(client, slice_input, correct_shape):
     assert arr.shape == correct_shape
 
 
+@pytest.mark.parametrize(
+    "slice_input, correct_shape",
+    [
+        (None, (3, 1, 5, 7, 4)),
+        (..., (3, 1, 5, 7, 4)),
+        ((), (3, 1, 5, 7, 4)),
+        (0, (1, 5, 7, 4)),
+        (slice(0, 3, 2), (2, 1, 5, 7, 4)),
+        ((1, slice(0, 10), slice(0, 3), slice(0, 3)), (1, 3, 3, 4)),
+        ((slice(0, 3), 0, slice(0, 3), slice(0, 3)), (3, 3, 3, 4)),
+        ((..., 0, 0, 0, 0), (3,)),
+        ((0, slice(0, 1), slice(0, 1), slice(0, 2), ...), (1, 1, 2, 4)),
+        ((0, ..., slice(0, 2)), (1, 5, 7, 2)),
+        ((..., slice(0, 1)), (3, 1, 5, 7, 1)),
+    ],
+)
+def test_forced_reshaping(client, slice_input, correct_shape):
+    arr = client["5d_sequence"].read(slice=slice_input)
+    assert arr.shape == correct_shape
+
+
 @pytest.mark.parametrize("block_input, correct_shape", [((0, 0, 0, 0), (1, 5, 7, 4))])
 def test_tiff_sequence_block(client, block_input, correct_shape):
     arr = client["sequence"].read_block(block_input)

tiled/adapters/sequence.py

@@ -1,9 +1,12 @@
 import builtins
+import math
+import warnings
 from abc import abstractmethod
 from pathlib import Path
 from typing import Any, List, Optional, Tuple, Union
 
 import numpy as np
+from ndindex import ndindex
 from numpy._typing import NDArray
 
 from ..structures.array import ArrayStructure, BuiltinDtype
@@ -13,6 +16,47 @@
 from .type_alliases import JSON, NDSlice
 
 
+def force_reshape(arr: np.array, desired_shape: Tuple[int, ...]) -> np.array:
+    """Reshape a numpy array to match the desited shape, if possible.
+
+    Parameters
+    ----------
+
+    arr : np.array
+        The original ND array to be reshaped
+    desired_shape : Tuple[int, ...]
+        The desired shape of the resulting array
+
+    Returns
+    -------
+
+    A view of the original array
+    """
+
+    if arr.shape == desired_shape:
+        # Nothing to do here
+        return arr
+
+    if arr.size == math.prod(desired_shape):
+        if len(arr.shape) != len(desired_shape):
+            # Missing or extra singleton dimensions
+            warnings.warn(
+                f"Forcefully reshaping {arr.shape} to {desired_shape}",
+                category=RuntimeWarning,
+            )
+            return arr.reshape(desired_shape)
+        else:
+            # Some dimensions might be swapped or completely wrong
+            # TODO: needs to be treated more carefully
+            pass
+
+    warnings.warn(
+        f"Can not reshape array of {arr.shape} to match {desired_shape}; proceeding without changes",
+        category=RuntimeWarning,
+    )
+    return arr
+
+
 class FileSequenceAdapter:
     """Base adapter class for image (and other file) sequences
 
@@ -124,12 +168,12 @@ def metadata(self) -> JSON:
     def read(self, slice: Optional[NDSlice] = ...) -> NDArray[Any]:
         """Return a numpy array
 
-        Receives a sequence of values to select from a collection of image files
-        that were saved in a folder The input order is defined as: files -->
+        Receives a sequence of values to select from a collection of data files
+        that were saved in a folder. The input order is defined as: files -->
         vertical slice --> horizontal slice --> color slice --> ... read() can
         receive one value or one slice to select all the data from one file or
         a sequence of files; or it can receive a tuple (int or slice) to select
-        a more specific sequence of pixels of a group of images.
+        a more specific sequence of pixels of a group of images, for example.
 
         Parameters
         ----------
@@ -165,11 +209,15 @@ def read(self, slice: Optional[NDSlice] = ...) -> NDArray[Any]:
                     the_rest.insert(0, Ellipsis)  # Include any leading dimensions
                 elif isinstance(left_axis, builtins.slice):
                     arr = self.read(slice=left_axis)
+
+                sliced_shape = ndindex(left_axis).newshape(self.structure().shape)
+                arr = force_reshape(arr, sliced_shape)
                 arr = np.atleast_1d(arr[tuple(the_rest)])
         else:
             raise RuntimeError(f"Unsupported slice type, {type(slice)} in {slice}")
 
-        return arr
+        sliced_shape = ndindex(slice).newshape(self.structure().shape)
+        return force_reshape(arr, sliced_shape)
 
     def read_block(
         self, block: Tuple[int, ...], slice: Optional[NDSlice] = ...

tiled/server/router.py

@@ -327,7 +327,7 @@ async def metadata(
     entry: Any = SecureEntry(scopes=["read:metadata"]),
     root_path: bool = Query(False),
 ):
-    "Fetch the metadata and structure information for one entry."
+    """Fetch the metadata and structure information for one entry"""
 
     request.state.endpoint = "metadata"
     base_url = get_base_url(request)