From 04d776bf66ce83ea290403c52182e3fa136f28a7 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 10 Oct 2023 23:31:34 -0700
Subject: [PATCH 001/115] added ims_to_zarr function

---
 .../convert_to_n5/ims_to_ngff.py              | 828 ++++++++++++++++++
 1 file changed, 828 insertions(+)
 create mode 100644 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
new file mode 100644
index 0000000..e183362
--- /dev/null
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -0,0 +1,828 @@
+#!/usr/bin/env python
+
+import concurrent.futures
+import dataclasses
+#import itertools
+import math
+#import pathlib
+
+import imageio
+#from natsort import natsorted
+import numpy
+import skimage
+
+import h5py
+import zarr
+from numcodecs import Blosc
+import argschema
+
+import acpreprocessing.utils.convert
+import acpreprocessing.stitching_modules.convert_to_n5.psdeskew as psd
+
+
+def iterate_chunks(ds, slice_length, interleaved_channels=1, channel=0, interleaving_offset=0):
+    """given an iterator, iterate over tuples of a
+    given length from that iterator
+
+    Parameters
+    ----------
+    it : iterable
+        iterable object to iterate over in slices of slice_length
+    slice_length : int
+        number of items in iterator per chunk
+
+    Yields
+    ------
+    chunk : tuple
+        tuple of items from iterator of size slice_length. Yields
+        remainder of iterator when exhausted
+    """
+    start = 0
+    stop = slice_length
+    chunk = ds[start:stop]
+    while start<ds.shape[0]:
+        yield chunk
+        start = stop
+        stop = start + slice_length
+        if stop < ds.shape[0]:
+            chunk = ds[start:stop]
+        else:
+            chunk = ds[start:]
+
+
+def iter_arrays(r, interleaved_channels=1, channel=0, interleaving_offset=0):
+    """iterate arrays from an imageio tiff reader.  Allows the last image
+    of the array to be None, as is the case for data with 'dropped frames'.
+
+    Parameters
+    ----------
+    r : imageio.reader
+        imageio tiff reader from which to iterate multitiff arrays
+    interleaved_channels : int
+        number of channels interleaved in tiff when selecting which
+        arrays to read
+    channel : int
+        channel to iterate over in case of interleaved tiffs
+
+    Yields
+    -------
+    arr : numpy.ndarray
+        constituent page array of reader r
+    """
+    for i, p in enumerate(r._tf.pages):
+        page_channel = (i + interleaving_offset) % interleaved_channels
+        if page_channel != channel:
+            continue
+        arr = p.asarray()
+        if arr is not None:
+            yield arr
+        elif i == r.get_length() - 1:
+            continue
+        else:
+            raise ValueError
+
+
+def iterate_2d_arrays_from_mimgfns(mimgfns, interleaved_channels=1, channel=0):
+    """iterate constituent arrays from an iterator of image filenames
+    that can be opened as an imageio multi-image.
+
+    Parameters
+    ----------
+    mimgfns : list of str
+        imageio-compatible name inputs to be opened as multi-images
+
+    Yields
+    ------
+    array : numpy.ndarray
+        constituent array from ordered iterator of multi-image files
+    """
+    offset = 0
+    for mimgfn in mimgfns:
+        with imageio.get_reader(mimgfn, mode="I") as r:
+            yield from iter_arrays(r, interleaved_channels, channel, offset)
+            offset = (offset + r.get_length()) % interleaved_channels
+
+
+def iterate_numpy_chunks_from_dataset(
+        dataset, slice_length=None, pad=True, *args, **kwargs):
+    """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
+
+    Parameters
+    ----------
+    dataset : hdf5 dataset
+        imageio-compatible name inputs to be opened as multi-images
+    slice_length : int
+        number of 2d arrays from mimgfns included per chunk
+    pad : bool, optional
+        whether to extend final returned chunks with zeros
+
+    Yields
+    ------
+    arr : numpy.ndarray
+        3D numpy array representing a consecutive chunk of 2D arrays
+    """
+    #array_gen = iterate_2d_arrays_from_dataset(mimgfns, *args, **kwargs)
+    for chunk in iterate_chunks(dataset, slice_length,*args,**kwargs):
+        arr = numpy.asarray(chunk)
+        if pad:
+            if arr.shape[0] != slice_length:
+                newarr = numpy.zeros((slice_length, *arr.shape[1:]),
+                                     dtype=arr.dtype)
+                newarr[:arr.shape[0], :, :] = arr[:, :, :]
+                yield newarr
+            else:
+                yield arr
+        else:
+            yield arr
+
+
+def length_to_interleaved_length(length, interleaved_channels):
+    """get length of each interleaved channel that contributes to a
+    total array length
+
+    Parameters
+    ----------
+    length : int
+        length of interleaved array
+    interleaved_channels : int
+        number of constituent channels in interleaved array
+
+    Returns
+    -------
+    channel_lengths : list of int
+        length of each constituent channel in interleaved array
+    """
+    interleaved_length, r = divmod(length, interleaved_channels)
+    return [
+        interleaved_length + (1 if ic < r else 0)
+        for ic in range(interleaved_channels)
+    ]
+
+
+def mimg_shape_from_fn(mimg_fn, only_length_tup=False):
+    """get the shape of an imageio multi-image file without
+    reading it as a volume
+
+    Parameters
+    ----------
+    mimg_fn : str
+        imageio-compatible name input to be opened as multi-image
+    only_length_tup : bool, optional
+        whether to return only the number of frames or
+        include the dimensions of a representative frame
+
+    Returns
+    -------
+    shape : tuple of int
+        shape of array defined by mimg_fn
+    """
+    with imageio.get_reader(mimg_fn, mode="I") as r:
+        l = r.get_length()
+
+        if only_length_tup:
+            s = (l,)
+        else:
+            s = (l, *r.get_data(0).shape)
+    return s
+
+
+def array_downsample_chunks(arr, ds_factors, block_shape, **kwargs):
+    """downsample a numpy array by inputting subvolumes
+    to acpreprocessing.utils.convert.downsample_stack_volume
+
+    Parameters
+    ----------
+    arr : numpy.ndarray
+        array to downsample
+    ds_factors : tuple of int
+        integer downsample factor by which arr should be downsamples.
+        Length must match arr.ndim
+    block_shape : tuple of int
+        shape of individual blocks to process
+
+    Returns
+    -------
+    output_arr : numpy.ndarray
+        downsampled version of arr
+    """
+    blocks = skimage.util.view_as_blocks(arr, block_shape)
+
+    # construct output array and blocks
+    output_shape = tuple(
+        int(math.ceil(s / f)) for s, f in zip(arr.shape, ds_factors))
+    output_block_shape = tuple(
+        int(math.ceil(s / f)) for s, f in zip(block_shape, ds_factors))
+    output_arr = numpy.empty(output_shape, dtype=arr.dtype)
+    output_blocks = skimage.util.view_as_blocks(
+        output_arr, output_block_shape)
+
+    # process
+    for idxs in numpy.ndindex(blocks.shape[:arr.ndim]):
+        output_blocks[idxs] = (
+            acpreprocessing.utils.convert.downsample_stack_volume(
+                blocks[idxs], ds_factors, **kwargs))
+
+    return output_arr
+
+
+def array_downsample_threaded(arr, ds_factors, block_shape,
+                              n_threads=3, **kwargs):
+    """downsample a numpy array by concurrently inputting subvolumes
+    to acpreprocessing.utils.convert.downsample_stack_volume
+
+    Parameters
+    ----------
+    arr : numpy.ndarray
+        array to downsample
+    ds_factors : tuple of int
+        integer downsample factor by which arr should be downsamples.
+        Length must match arr.ndim
+    block_shape : tuple of int
+        shape of individual blocks to process
+    n_threads :  int
+        number of threads used to concurrently downsample chunks
+
+    Returns
+    -------
+    output_arr : numpy.ndarray
+        downsampled version of arr
+    """
+    blocks = skimage.util.view_as_blocks(arr, block_shape)
+
+    # construct output array and blocks
+    output_shape = tuple(
+        int(math.ceil(s / f)) for s, f in zip(arr.shape, ds_factors))
+    output_block_shape = tuple(
+        int(math.ceil(s / f)) for s, f in zip(block_shape, ds_factors))
+    output_arr = numpy.empty(output_shape, dtype=arr.dtype)
+    output_blocks = skimage.util.view_as_blocks(
+        output_arr, output_block_shape)
+
+    # process
+    with concurrent.futures.ThreadPoolExecutor(max_workers=n_threads) as e:
+        fut_to_idxs = {}
+        for idxs in numpy.ndindex(blocks.shape[:arr.ndim]):
+            fut_to_idxs[e.submit(
+                acpreprocessing.utils.convert.downsample_stack_volume,
+                blocks[idxs], ds_factors, **kwargs)] = idxs
+
+        for fut in concurrent.futures.as_completed(fut_to_idxs.keys()):
+            idxs = fut_to_idxs[fut]
+            output_blocks[idxs] = fut.result()
+    return output_arr
+
+
+class ArrayChunkingError(ValueError):
+    """ValueError caused by incorrect array chunking"""
+
+
+def downsample_array(arr, *args, n_threads=None, block_shape=None,
+                     block_divs=None, allow_failover=True, **kwargs):
+    """downsample array with optional subvolume chunking and threading
+
+    Parameters
+    ----------
+    arr : numpy.ndarray
+        array to downsample
+    ds_factors : tuple of int
+        integer downsample factor by which arr should be downsamples.
+        Length must match arr.ndim
+    block_shape : tuple of int
+        shape of individual blocks to process.
+        Supersedes block_divs if both are provided
+    block_divs : tuple of int
+        number of divisions for array chunking.
+    n_threads :  int
+        number of threads used to concurrently downsample chunks
+    allow_failover : bool, optional
+        whether to default to using
+        acpreprocessing.utils.convert.downsample_stack_volume when
+        block_shape or block_divs is not actionable.  (default: True)
+
+    Returns
+    -------
+    output_arr : numpy.ndarray
+        downsampled version of arr
+    """
+    try:
+        if block_shape is not None or block_divs is not None:
+            if block_shape is None:
+                block_shape = [a_s / d for a_s, d in zip(
+                    arr.shape, block_divs)]
+                if not all([s.is_integer() for s in block_shape]):
+                    raise ArrayChunkingError(
+                        "cannot divide array of shape {} by {}".format(
+                            arr.shape, block_divs))
+                block_shape = tuple(int(s) for s in block_shape)
+            if n_threads is not None:
+                return array_downsample_threaded(
+                    arr, *args, n_threads=n_threads,
+                    block_shape=block_shape, **kwargs)
+            return array_downsample_chunks(
+                arr, *args, block_shape=block_shape, **kwargs)
+    except ArrayChunkingError:
+        if not allow_failover:
+            raise
+    return acpreprocessing.utils.convert.downsample_stack_volume(
+        arr, *args, **kwargs)
+
+
+def mip_level_shape(lvl, lvl_0_shape):
+    """get the expected shape of a MIP level downsample for a given
+    full resolution shape.  Expects 2x downsampling on each dimension.
+
+    Parameters
+    ----------
+    lvl : int
+        mip level for which shape is calculated
+    lvl_0_shape : tuple of int
+        shape tuple of level 0
+
+    Returns
+    -------
+    lvl_shape : tuple of int
+        shape tuple for lvl
+    """
+    return tuple([*map(lambda x: math.ceil(x / (2**lvl)), lvl_0_shape)])
+
+
+def dswrite_chunk(ds, start, end, arr, silent_overflow=True):
+    """write array arr into array-like n5 dataset defined by
+    start and end points on 0 axis.
+
+    Parameters
+    ----------
+    ds : z5py.dataset.Dataset (n5 3D) or zarr.dataset (zarr 5D)
+        array-like dataset to fill in
+    start : int
+        start index along 0 axis of ds to fill in
+    end : int
+        end index along 0 axis of ds to fill in
+    arr : numpy.ndarray
+        array from which ds values will be taken
+    silent_overflow : bool, optional
+        whether to shrink the end index to match the
+        shape of ds (default: True)
+    """
+    if len(ds.shape) == 5:  # dataset dimensions should be 3 or 5
+        if end >= ds.shape[2] and silent_overflow:
+            end = ds.shape[2]
+        if end > start:
+            ds[0, 0, start:end, :, :] = arr[:(end - start), :, :]
+    elif len(ds.shape) == 3:
+        if end >= ds.shape[0] and silent_overflow:
+            end = ds.shape[0]
+        if end > start:
+            ds[start:end, :, :] = arr[:(end - start), :, :]
+
+
+@dataclasses.dataclass
+class MIPArray:
+    lvl: int
+    array: numpy.ndarray
+    start: int
+    end: int
+
+
+def iterate_mip_levels_from_dataset(
+        dataset, lvl, block_size, slice_length, downsample_factor,
+        downsample_method=None, lvl_to_mip_kwargs=None,
+        interleaved_channels=1, channel=0, deskew_kwargs={}):
+    """recursively generate MIPmap levels from an iterator of multi-image files
+
+    Parameters
+    ----------
+    dataset : hdf5 dataset
+        imageio-compatible name inputs to be opened as multi-images
+    lvl : int
+        integer mip level to generate
+    block_size : int
+        number of 2D arrays in chunk to process for a chunk at lvl
+    slice_length : int
+        number of 2D arrays to gather from tiff stacks
+    downsample_factor : tuple of int
+        integer downsampling factor for MIP levels
+    downsample_method : str, optional
+        downsampling method for
+        acpreprocessing.utils.convert.downsample_stack_volume
+    lvl_to_mip_kwargs :  dict, optional
+        mapping of MIP level to kwargs used in MIPmap generation
+    interleaved_channels : int
+        number of channels interleaved in the tiff files (default 1)
+    channel : int, optional
+        channel from which interleaved data should be read (default 0)
+    deskew_kwargs : dict, optional
+        parameters for pixel shifting deskew
+
+    Yields
+    ------
+    ma : acpreprocessing.stitching_modules.convert_to_n5.tiff_to_n5.MipArray
+        object describing chunked array, MIP level of origin, and chunk indices
+    """
+    lvl_to_mip_kwargs = ({} if lvl_to_mip_kwargs is None
+                         else lvl_to_mip_kwargs)
+    mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
+    start_index = 0
+    chunk_index = 0
+    if lvl > 0:
+        num_chunks = downsample_factor[0]
+
+        i = 0
+        for ma in iterate_mip_levels_from_dataset(
+                dataset, lvl-1, block_size, slice_length,
+                downsample_factor, downsample_method,
+                lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
+                channel=channel, deskew_kwargs=deskew_kwargs):
+            chunk = ma.array
+            # throw array up for further processing
+            yield ma
+            # ignore if not parent resolution
+            if ma.lvl != lvl-1:
+                continue
+
+            try:
+                temp_lminus1_arr
+            except NameError:
+                temp_lminus1_arr = numpy.empty(
+                    (num_chunks*chunk.shape[0], *chunk.shape[1:]),
+                    dtype=chunk.dtype)
+
+            # fill in temporary block according to index
+            chunk_size = chunk.shape[0]
+            block_offset = i*block_size
+
+            temp_lminus1_arr[
+                block_offset:block_offset+chunk_size, :, :] = chunk[:, :, :]
+
+            # copy op only for uneven final chunk
+            if chunk_size != block_size:
+                temp_lminus1_arr = temp_lminus1_arr[
+                    :block_offset+chunk_size, :, :]
+
+            if i == num_chunks - 1:
+                temp_arr = (
+                    downsample_array(
+                        temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
+                        method=downsample_method, **mip_kwargs))
+                end_index = start_index + temp_arr.shape[0]
+                yield MIPArray(lvl, temp_arr, start_index, end_index)
+                start_index += temp_arr.shape[0]
+                i = 0
+            else:
+                i += 1
+        # add any leftovers
+        if i > 0:
+            temp_arr = acpreprocessing.utils.convert.downsample_stack_volume(
+                temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
+                method=downsample_method)
+            end_index = start_index + temp_arr.shape[0]
+            yield MIPArray(lvl, temp_arr, start_index, end_index)
+    else:
+        # get level 0 chunks
+        # block_size is the number of slices to read from tiffs
+        for chunk in iterate_numpy_chunks_from_dataset(
+                dataset, slice_length, pad=False,
+                interleaved_channels=interleaved_channels,
+                channel=channel):
+            # deskew level 0 chunk
+            if deskew_kwargs:
+                chunk = numpy.transpose(psd.deskew_block(
+                    chunk, chunk_index, **deskew_kwargs), (2, 1, 0))
+            end_index = start_index + chunk.shape[0]
+            yield MIPArray(lvl, chunk, start_index, end_index)
+            start_index += chunk.shape[0]
+            chunk_index += 1
+
+
+
+def omezarr_attrs(name, position_xyz, lvl0_xyz_res, max_lvl):
+    # TODO only single channel implemented
+    datasets = []
+    for i in range(max_lvl+1):
+        s = [(2**i)*r for r in lvl0_xyz_res]
+        d = {"coordinateTransformations": [
+            {"scale": [
+                1.0,
+                1.0,
+                s[2],
+                s[1],
+                s[0]
+            ],
+                "type": "scale"
+            }
+        ],
+            "path": str(i)
+        }
+        datasets.append(d)
+    attrs = {}
+    attrs["multiscales"] = [
+        {
+            "axes": [
+                {
+                    "name": "t",
+                    "type": "time",
+                    "unit": "millisecond"
+                },
+                {
+                    "name": "c",
+                    "type": "channel"
+                },
+                {
+                    "name": "z",
+                    "type": "space",
+                    "unit": "micrometer"
+                },
+                {
+                    "name": "y",
+                    "type": "space",
+                    "unit": "micrometer"
+                },
+                {
+                    "name": "x",
+                    "type": "space",
+                    "unit": "micrometer"
+                }
+            ],
+            "datasets": datasets,
+            "name": "/" + name,
+            "translation": [
+                0,
+                0,
+                0.0,
+                0.0,
+                0.0
+            ],
+            "version": "0.4",
+            "coordinateTransformations": [
+                {
+                    "translation": [
+                        0.0,
+                        0.0,
+                        position_xyz[2],
+                        position_xyz[1],
+                        position_xyz[0]
+                    ],
+                    "type": "translation"
+                }
+            ]
+        }
+    ]
+    attrs["omero"] = {
+        "channels": [
+            {
+                "active": True,
+                "coefficient": 1,
+                "color": "000000",
+                "family": "linear",
+                "inverted": False,
+                "label": "Channel:" + name + ":0",
+                "window": {
+                    "end": 1.0,
+                    "max": 1.0,
+                    "min": 0.0,
+                    "start": 0.0
+                }
+            }
+        ],
+        "id": 1,
+        "name": name,
+        "rdefs": {
+            "defaultT": 0,
+            "defaultZ": 288,
+            "model": "color"
+        },
+        "version": "0.4"
+    }
+    return attrs
+
+
+class TiffToNGFFException(Exception):
+    """class to describe exceptions with TiffToNGFF module"""
+
+
+class TiffToNGFFValueError(TiffToNGFFException, ValueError):
+    """value error in TiffToNgff"""
+
+
+def write_ims_to_zarr(
+        ims_fn, output_n5, group_names, group_attributes=None, max_mip=0,
+        mip_dsfactor=(2, 2, 2), chunk_size=(1, 1, 64, 64, 64),
+        concurrency=10, slice_concurrency=1,
+        compression="raw", dtype="uint16", lvl_to_mip_kwargs=None,
+        interleaved_channels=1, channel=0, deskew_options=None, **kwargs):
+    """write a stack represented by an iterator of multi-image files as a zarr
+    volume with ome-ngff metadata
+
+    Parameters
+    ----------
+    ims_fn : str
+        imageio-compatible name inputs to be opened as multi-images
+    output_n5 : str
+        output zarr directory
+    group_names : list of str
+        names of groups to generate within n5
+    group_attributes : list of dict, optional
+        attribute dictionaries corresponding to group with matching index
+    max_mip : int
+        maximum MIP level to generate
+    mip_dsfactor : tuple of int
+        integer downsampling factor for MIP levels
+    chunk_size : tuple of int
+        chunk size for n5 datasets
+    concurrency : int
+        total concurrency used for writing arrays to n5
+        (python threads = concurrency // slice_concurrency)
+    slice_concurrency : int
+        threads used by z5py
+    compression : str, optional
+        compression for n5 (default: raw)
+    dtype : str, optional
+        dtype for n5 (default: uint16)
+    lvl_to_mip_kwargs :  dict, optional
+        mapping of MIP level to kwargs used in MIPmap generation
+    interleaved_channels : int
+        number of channels interleaved in the tiff files (default 1)
+    channel : int, optional
+        channel from which interleaved data should be read (default 0)
+    deskew_options : dict, optional
+        dictionary of parameters to run pixel shifting deskew (default None)
+    """
+    group_attributes = ([] if group_attributes is None else group_attributes)
+    deskew_options = ({} if deskew_options is None else deskew_options)
+    
+    with h5py.File(ims_fn, 'r') as f:
+        dataset = f['DataSet']['ResolutionLevel 0']['TimePoint 0']['Channel 0']['Data']
+    
+    joined_shapes = dataset.shape
+
+    if deskew_options and deskew_options["deskew_method"] == "ps":
+        block_size = chunk_size[2]
+        slice_length = int(chunk_size[2]/deskew_options['deskew_stride'])
+        deskew_kwargs = psd.psdeskew_kwargs(skew_dims_zyx=(slice_length, joined_shapes[1], joined_shapes[2]),
+                                            **deskew_options
+                                            )
+        joined_shapes = psd.reshape_joined_shapes(
+            joined_shapes, deskew_options['deskew_stride'], **deskew_kwargs)
+    else:
+        block_size = chunk_size[2]
+        slice_length = block_size
+        deskew_kwargs = {}
+
+    # TODO DESKEW: does this generally work regardless of skew?
+
+    workers = concurrency // slice_concurrency
+
+    zstore = zarr.DirectoryStore(output_n5, dimension_separator='/')
+    with zarr.open(zstore, mode='a') as f:
+        mip_ds = {}
+        # create groups with attributes according to omezarr spec
+        if len(group_names) == 1:
+            group_name = group_names[0]
+            try:
+                g = f.create_group(f"{group_name}")
+            except KeyError:
+                g = f[f"{group_name}"]
+            try:
+                attributes = group_attributes[0]
+            except IndexError:
+                print('attributes error')
+
+            if "pixelResolution" in attributes:
+                if deskew_options:
+                    attributes["pixelResolution"]["dimensions"][2] /= deskew_options["deskew_stride"]
+                attributes = omezarr_attrs(
+                    group_name, attributes["position"], attributes["pixelResolution"]["dimensions"], max_mip)
+            if attributes:
+                for k, v in attributes.items():
+                    g.attrs[k] = v
+        else:
+            raise TiffToNGFFValueError("only one group name expected")
+        scales = []
+
+        # shuffle=Blosc.BITSHUFFLE)
+        compression = Blosc(cname='zstd', clevel=1)
+        for mip_lvl in range(max_mip + 1):
+            mip_3dshape = mip_level_shape(mip_lvl, joined_shapes)
+            ds_lvl = g.create_dataset(
+                f"{mip_lvl}",
+                chunks=chunk_size,
+                shape=(1, 1, mip_3dshape[0], mip_3dshape[1], mip_3dshape[2]),
+                compression=compression,
+                dtype=dtype
+            )
+            dsfactors = [int(i)**mip_lvl for i in mip_dsfactor]
+            mip_ds[mip_lvl] = ds_lvl
+            scales.append(dsfactors)
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as e:
+            futs = []
+            for miparr in iterate_mip_levels_from_dataset(
+                    dataset, max_mip, block_size, slice_length, mip_dsfactor,
+                    lvl_to_mip_kwargs=lvl_to_mip_kwargs,
+                    interleaved_channels=interleaved_channels,
+                    channel=channel, deskew_kwargs=deskew_kwargs):
+                futs.append(e.submit(
+                    dswrite_chunk, mip_ds[miparr.lvl],
+                    miparr.start, miparr.end, miparr.array))
+            for fut in concurrent.futures.as_completed(futs):
+                _ = fut.result()
+
+
+def ims_to_ngff_group(ims_fn, output, *args, **kwargs):
+    """convert directory of natsort-consecutive multitiffs to an n5 or zarr pyramid
+
+    Parameters
+    ----------
+    tiffdir : str
+        directory of consecutive multitiffs to convert
+    """
+
+    if output == 'zarr':
+        print('converting to zarr')
+        return write_ims_to_zarr(ims_fn, *args, **kwargs)
+    else:
+        print('unknown output format: ' + output)
+
+
+class DownsampleOptions(argschema.schemas.DefaultSchema):
+    block_divs = argschema.fields.List(
+        argschema.fields.Int, required=False, allow_none=True)
+    n_threads = argschema.fields.Int(required=False, allow_none=True)
+
+
+class DeskewOptions(argschema.schemas.DefaultSchema):
+    deskew_method = argschema.fields.Str(required=False, default='')
+    deskew_stride = argschema.fields.Int(required=False, default=None)
+    deskew_flip = argschema.fields.Bool(required=False, default=True)
+    deskew_crop = argschema.fields.Float(required=False, default=1.0)
+
+
+class NGFFGenerationParameters(argschema.schemas.DefaultSchema):
+    output_format = argschema.fields.Str(required=True)
+    output_file = argschema.fields.Str(required=True)
+    max_mip = argschema.fields.Int(required=False, default=0)
+    concurrency = argschema.fields.Int(required=False, default=10)
+    compression = argschema.fields.Str(required=False, default="raw")
+    lvl_to_mip_kwargs = argschema.fields.Dict(
+        keys=argschema.fields.Int(),
+        values=argschema.fields.Nested(DownsampleOptions))
+
+    # FIXME argschema supports lists and tuples,
+    #   but has some version differences
+
+    mip_dsfactor = argschema.fields.Tuple((
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int()), required=False, default=(2, 2, 2))
+    deskew_options = argschema.fields.Nested(
+        DeskewOptions, required=False)
+
+
+class NGFFGroupGenerationParameters(NGFFGenerationParameters):
+    group_names = argschema.fields.List(
+        argschema.fields.Str, required=True)
+    group_attributes = argschema.fields.List(
+        argschema.fields.Dict(required=False, default={}), default=[],
+        required=False)
+
+
+class IMSToNGFFParameters(NGFFGroupGenerationParameters):
+    input_file = argschema.fields.Str(required=True)
+    interleaved_channels = argschema.fields.Int(required=False, default=1)
+    channel = argschema.fields.Int(required=False, default=0)
+
+
+class IMSToZarrInputParameters(argschema.ArgSchema,
+                                   IMSToNGFFParameters):
+    chunk_size = argschema.fields.Tuple((
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int()), required=False, default=(1, 1, 64, 64, 64))
+
+
+
+class IMSToZarr(argschema.ArgSchemaParser):
+    default_schema = IMSToZarrInputParameters
+
+    def run(self):
+        deskew_options = (self.args["deskew_options"]
+                          if "deskew_options" in self.args else {})
+        ims_to_ngff_group(
+            self.args["input_file"], self.args["output_format"],
+            self.args["output_file"], self.args["group_names"],
+            self.args["group_attributes"],
+            self.args["max_mip"],
+            self.args["mip_dsfactor"],
+            self.args["chunk_size"],
+            concurrency=self.args["concurrency"],
+            compression=self.args["compression"],
+            lvl_to_mip_kwargs=self.args["lvl_to_mip_kwargs"],
+            deskew_options=deskew_options)
+
+
+
+if __name__ == "__main__":
+    mod = IMSToZarr()
+    mod.run()

From 55a5b8dfcf5b414a02408202f47ee14bae76b2a4 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 10 Oct 2023 23:58:03 -0700
Subject: [PATCH 002/115] debugging

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index e183362..d12469b 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -818,7 +818,7 @@ def run(self):
             self.args["chunk_size"],
             concurrency=self.args["concurrency"],
             compression=self.args["compression"],
-            lvl_to_mip_kwargs=self.args["lvl_to_mip_kwargs"],
+            #lvl_to_mip_kwargs=self.args["lvl_to_mip_kwargs"],
             deskew_options=deskew_options)
 
 

From 0a9eab6015c04263fb14beb81f411a795f69d78b Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 11 Oct 2023 00:07:36 -0700
Subject: [PATCH 003/115] debugging

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py          | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index d12469b..3051a07 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -650,8 +650,8 @@ def write_ims_to_zarr(
     group_attributes = ([] if group_attributes is None else group_attributes)
     deskew_options = ({} if deskew_options is None else deskew_options)
     
-    with h5py.File(ims_fn, 'r') as f:
-        dataset = f['DataSet']['ResolutionLevel 0']['TimePoint 0']['Channel 0']['Data']
+    f = h5py.File(ims_fn, 'r')
+    dataset = f['DataSet']['ResolutionLevel 0']['TimePoint 0']['Channel 0']['Data']
     
     joined_shapes = dataset.shape
 
@@ -726,6 +726,8 @@ def write_ims_to_zarr(
                     miparr.start, miparr.end, miparr.array))
             for fut in concurrent.futures.as_completed(futs):
                 _ = fut.result()
+    print("conversion complete, closing file")
+    f.close()
 
 
 def ims_to_ngff_group(ims_fn, output, *args, **kwargs):

From 0dc39712618f7e8590208fed4f581898216b2ae8 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 11 Oct 2023 00:10:01 -0700
Subject: [PATCH 004/115] debugging

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py    | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 3051a07..bf37784 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -682,10 +682,13 @@ def write_ims_to_zarr(
                 g = f.create_group(f"{group_name}")
             except KeyError:
                 g = f[f"{group_name}"]
-            try:
-                attributes = group_attributes[0]
-            except IndexError:
-                print('attributes error')
+            if not group_attributes is None:
+                try:
+                    attributes = group_attributes[0]
+                except IndexError:
+                    print('attributes error')
+            else:
+                attributes = {}
 
             if "pixelResolution" in attributes:
                 if deskew_options:

From c4ae7961432b8a1e23cc936d83612a452b111c8f Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 11 Oct 2023 00:12:36 -0700
Subject: [PATCH 005/115] debugging

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index bf37784..5b7723e 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -682,7 +682,7 @@ def write_ims_to_zarr(
                 g = f.create_group(f"{group_name}")
             except KeyError:
                 g = f[f"{group_name}"]
-            if not group_attributes is None:
+            if group_attributes:
                 try:
                     attributes = group_attributes[0]
                 except IndexError:

From 062a8d01af57618a470360df5b96b65f55fc4e6d Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 11 Oct 2023 00:16:05 -0700
Subject: [PATCH 006/115] debugging

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 5b7723e..02876a1 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -12,6 +12,7 @@
 import skimage
 
 import h5py
+import hdf5plugin
 import zarr
 from numcodecs import Blosc
 import argschema

From c21c6ee6c8faf937bd12c2b80fed9f314256687c Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Fri, 16 Feb 2024 13:51:28 -0800
Subject: [PATCH 007/115] adding support for permuting stage axes coordinates
 in group position attribute

---
 .../convert_to_n5/acquisition_dir_to_ngff.py  | 21 +++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/acquisition_dir_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/acquisition_dir_to_ngff.py
index 3a4edef..9334e63 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/acquisition_dir_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/acquisition_dir_to_ngff.py
@@ -69,10 +69,13 @@ def get_number_interleaved_channels_from_rootdir(
     return interleaved_channels
 
 
-def acquisition_to_ngff(acquisition_dir, output, out_dir, concurrency=5,
+def acquisition_to_ngff(acquisition_dir, output, out_dir, concurrency=5, acq_parameters=None,
                         ngff_generation_kwargs=None, copy_top_level_files=True):
     """
     """
+    acq_parameters = (
+        {} if acq_parameters is None
+        else acq_parameters)
     ngff_generation_kwargs = (
         {} if ngff_generation_kwargs is None
         else ngff_generation_kwargs)
@@ -87,6 +90,10 @@ def acquisition_to_ngff(acquisition_dir, output, out_dir, concurrency=5,
     interleaved_channels = get_number_interleaved_channels_from_rootdir(
         acquisition_path)
     positionList = get_strip_positions_from_rootdir(acquisition_path)
+    if acq_parameters and "stage_axes" in acq_parameters:
+        axes = acq_parameters["stage_axes"]
+    else:
+        axes = (0,1,2)
 
     try:
         setup_group_attributes = [{
@@ -95,7 +102,7 @@ def acquisition_to_ngff(acquisition_dir, output, out_dir, concurrency=5,
                     acquisition_path),
                 "unit": "um"
             },
-            "position": p
+            "position": tuple(p[i] for i in axes)
         } for p in positionList]
     except (KeyError, FileNotFoundError):
         setup_group_attributes = {}
@@ -138,6 +145,13 @@ def acquisition_to_ngff(acquisition_dir, output, out_dir, concurrency=5,
         for tlf_path in top_level_files_paths:
             out_tlf_path = out_path / tlf_path.name
             shutil.copy(str(tlf_path), str(out_tlf_path))
+            
+            
+class AcquisitionParameters(argschema.schemas.DefaultSchema):
+    stage_axes = argschema.fields.Tuple((
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int()), required=False, default=(0,1,2))
 
 
 class AcquisitionDirToNGFFParameters(
@@ -147,6 +161,8 @@ class AcquisitionDirToNGFFParameters(
     # output_dir = argschema.fields.Str(required=True)
     copy_top_level_files = argschema.fields.Bool(required=False, default=True)
     position_concurrency = argschema.fields.Int(required=False, default=5)
+    acq_parameters = argschema.fields.Nested(
+        AcquisitionParameters, required=False, default=None)
 
 
 class AcquisitionDirToNGFF(argschema.ArgSchemaParser):
@@ -164,6 +180,7 @@ def run(self):
         acquisition_to_ngff(
             self.args["input_dir"], self.args["output_format"], self.args["output_file"],
             concurrency=self.args["position_concurrency"],
+            acq_parameters=self.args["acq_parameters"],
             ngff_generation_kwargs=ngff_kwargs,
             copy_top_level_files=self.args["copy_top_level_files"]
         )

From 94d746f8d4f9697fb00a0a5e4eb811571e94be1e Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Fri, 16 Feb 2024 14:38:58 -0800
Subject: [PATCH 008/115] having trouble with argschema implementation of axes
 swapping

---
 .../convert_to_n5/acquisition_dir_to_ngff.py          | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/acquisition_dir_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/acquisition_dir_to_ngff.py
index 9334e63..c79fcc8 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/acquisition_dir_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/acquisition_dir_to_ngff.py
@@ -91,7 +91,11 @@ def acquisition_to_ngff(acquisition_dir, output, out_dir, concurrency=5, acq_par
         acquisition_path)
     positionList = get_strip_positions_from_rootdir(acquisition_path)
     if acq_parameters and "stage_axes" in acq_parameters:
-        axes = acq_parameters["stage_axes"]
+        axesStr = acq_parameters["stage_axes"]
+        if axesStr=="yxz":
+            axes = (1,0,2)
+        else:
+            axes = (0,1,2)
     else:
         axes = (0,1,2)
 
@@ -148,10 +152,7 @@ def acquisition_to_ngff(acquisition_dir, output, out_dir, concurrency=5, acq_par
             
             
 class AcquisitionParameters(argschema.schemas.DefaultSchema):
-    stage_axes = argschema.fields.Tuple((
-        argschema.fields.Int(),
-        argschema.fields.Int(),
-        argschema.fields.Int()), required=False, default=(0,1,2))
+    stage_axes = argschema.fields.Str(required=False,default='')
 
 
 class AcquisitionDirToNGFFParameters(

From ca92b217c79300e737f63d481986c3c709a9e7e1 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Mon, 3 Jun 2024 20:56:18 -0700
Subject: [PATCH 009/115] module for visor-zarr conversion

---
 .../convert_to_n5/tiff_to_visor.py            | 972 ++++++++++++++++++
 1 file changed, 972 insertions(+)
 create mode 100644 acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
new file mode 100644
index 0000000..7671d9a
--- /dev/null
+++ b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
@@ -0,0 +1,972 @@
+#!/usr/bin/env python
+
+import concurrent.futures
+import dataclasses
+import itertools
+import math
+import pathlib
+
+import imageio
+from natsort import natsorted
+import numpy
+import skimage
+
+import z5py
+import zarr
+from numcodecs import Blosc
+import argschema
+
+import acpreprocessing.utils.convert
+import acpreprocessing.stitching_modules.convert_to_n5.psdeskew as psd
+
+
+def iterate_chunks(it, slice_length):
+    """given an iterator, iterate over tuples of a
+    given length from that iterator
+
+    Parameters
+    ----------
+    it : iterable
+        iterable object to iterate over in slices of slice_length
+    slice_length : int
+        number of items in iterator per chunk
+
+    Yields
+    ------
+    chunk : tuple
+        tuple of items from iterator of size slice_length. Yields
+        remainder of iterator when exhausted
+    """
+    it = iter(it)
+    chunk = tuple(itertools.islice(it, slice_length))
+    while chunk:
+        yield chunk
+        chunk = tuple(itertools.islice(it, slice_length))
+
+
+def iter_arrays(r, interleaved_channels=1, channel=0, interleaving_offset=0):
+    """iterate arrays from an imageio tiff reader.  Allows the last image
+    of the array to be None, as is the case for data with 'dropped frames'.
+
+    Parameters
+    ----------
+    r : imageio.reader
+        imageio tiff reader from which to iterate multitiff arrays
+    interleaved_channels : int
+        number of channels interleaved in tiff when selecting which
+        arrays to read
+    channel : int
+        channel to iterate over in case of interleaved tiffs
+
+    Yields
+    -------
+    arr : numpy.ndarray
+        constituent page array of reader r
+    """
+    for i, p in enumerate(r._tf.pages):
+        page_channel = (i + interleaving_offset) % interleaved_channels
+        if page_channel != channel:
+            continue
+        arr = p.asarray()
+        if arr is not None:
+            yield arr
+        elif i == r.get_length() - 1:
+            continue
+        else:
+            raise ValueError
+
+
+def iterate_2d_arrays_from_mimgfns(mimgfns, interleaved_channels=1, channel=0):
+    """iterate constituent arrays from an iterator of image filenames
+    that can be opened as an imageio multi-image.
+
+    Parameters
+    ----------
+    mimgfns : list of str
+        imageio-compatible name inputs to be opened as multi-images
+
+    Yields
+    ------
+    array : numpy.ndarray
+        constituent array from ordered iterator of multi-image files
+    """
+    offset = 0
+    for mimgfn in mimgfns:
+        with imageio.get_reader(mimgfn, mode="I") as r:
+            yield from iter_arrays(r, interleaved_channels, channel, offset)
+            offset = (offset + r.get_length()) % interleaved_channels
+
+
+def iterate_numpy_chunks_from_mimgfns(
+        mimgfns, slice_length=None, pad=True, *args, **kwargs):
+    """iterate over a contiguous iterator of imageio multi-image files as
+    chunks of numpy arrays
+
+    Parameters
+    ----------
+    mimgfns : list of str
+        imageio-compatible name inputs to be opened as multi-images
+    slice_length : int
+        number of 2d arrays from mimgfns included per chunk
+    pad : bool, optional
+        whether to extend final returned chunks with zeros
+
+    Yields
+    ------
+    arr : numpy.ndarray
+        3D numpy array representing a consecutive chunk of 2D arrays
+    """
+    array_gen = iterate_2d_arrays_from_mimgfns(mimgfns, *args, **kwargs)
+    for chunk in iterate_chunks(array_gen, slice_length):
+        arr = numpy.array(chunk)
+        if pad:
+            if arr.shape[0] != slice_length:
+                newarr = numpy.zeros((slice_length, *arr.shape[1:]),
+                                     dtype=arr.dtype)
+                newarr[:arr.shape[0], :, :] = arr[:, :, :]
+                yield newarr
+            else:
+                yield arr
+        else:
+            yield arr
+
+
+def length_to_interleaved_length(length, interleaved_channels):
+    """get length of each interleaved channel that contributes to a
+    total array length
+
+    Parameters
+    ----------
+    length : int
+        length of interleaved array
+    interleaved_channels : int
+        number of constituent channels in interleaved array
+
+    Returns
+    -------
+    channel_lengths : list of int
+        length of each constituent channel in interleaved array
+    """
+    interleaved_length, r = divmod(length, interleaved_channels)
+    return [
+        interleaved_length + (1 if ic < r else 0)
+        for ic in range(interleaved_channels)
+    ]
+
+
+def mimg_shape_from_fn(mimg_fn, only_length_tup=False):
+    """get the shape of an imageio multi-image file without
+    reading it as a volume
+
+    Parameters
+    ----------
+    mimg_fn : str
+        imageio-compatible name input to be opened as multi-image
+    only_length_tup : bool, optional
+        whether to return only the number of frames or
+        include the dimensions of a representative frame
+
+    Returns
+    -------
+    shape : tuple of int
+        shape of array defined by mimg_fn
+    """
+    with imageio.get_reader(mimg_fn, mode="I") as r:
+        l = r.get_length()
+
+        if only_length_tup:
+            s = (l,)
+        else:
+            s = (l, *r.get_data(0).shape)
+    return s
+
+
+def joined_mimg_shape_from_fns(mimg_fns, concurrency=1,
+                               interleaved_channels=1, channel=0,
+                               *args, **kwargs):
+    """concurrently read image shapes from tiff files representing a
+    contiguous stack to get the shape of the combined stack.
+
+    Parameters
+    ----------
+    mimg_fns : list of str
+        imageio-compatible name inputs to be opened as multi-images
+    concurrency : int, optional
+        number of threads to use when getting image shape metadata
+    interleaved_channels : int, optional
+        number of channels interleaved in the tiff files (default 1)
+    channel : int, optional
+        channel from which interleaved data should be read (default 0)
+
+    Returns
+    -------
+    shape : tuple of int
+        shape of 3D array represented by concatenating mimg_fns
+    """
+    # join shapes while reading concurrently
+    with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as e:
+        futs = [e.submit(mimg_shape_from_fn, fn, *args, **kwargs)
+                for fn in mimg_fns]
+        shapes = [fut.result() for fut
+                  in concurrent.futures.as_completed(futs)]
+    return (
+        length_to_interleaved_length(
+            sum([s[0] for s in shapes]), interleaved_channels)[channel],
+        shapes[0][1],
+        shapes[0][2]
+    )
+
+
+def array_downsample_chunks(arr, ds_factors, block_shape, **kwargs):
+    """downsample a numpy array by inputting subvolumes
+    to acpreprocessing.utils.convert.downsample_stack_volume
+
+    Parameters
+    ----------
+    arr : numpy.ndarray
+        array to downsample
+    ds_factors : tuple of int
+        integer downsample factor by which arr should be downsamples.
+        Length must match arr.ndim
+    block_shape : tuple of int
+        shape of individual blocks to process
+
+    Returns
+    -------
+    output_arr : numpy.ndarray
+        downsampled version of arr
+    """
+    blocks = skimage.util.view_as_blocks(arr, block_shape)
+
+    # construct output array and blocks
+    output_shape = tuple(
+        int(math.ceil(s / f)) for s, f in zip(arr.shape, ds_factors))
+    output_block_shape = tuple(
+        int(math.ceil(s / f)) for s, f in zip(block_shape, ds_factors))
+    output_arr = numpy.empty(output_shape, dtype=arr.dtype)
+    output_blocks = skimage.util.view_as_blocks(
+        output_arr, output_block_shape)
+
+    # process
+    for idxs in numpy.ndindex(blocks.shape[:arr.ndim]):
+        output_blocks[idxs] = (
+            acpreprocessing.utils.convert.downsample_stack_volume(
+                blocks[idxs], ds_factors, **kwargs))
+
+    return output_arr
+
+
+def array_downsample_threaded(arr, ds_factors, block_shape,
+                              n_threads=3, **kwargs):
+    """downsample a numpy array by concurrently inputting subvolumes
+    to acpreprocessing.utils.convert.downsample_stack_volume
+
+    Parameters
+    ----------
+    arr : numpy.ndarray
+        array to downsample
+    ds_factors : tuple of int
+        integer downsample factor by which arr should be downsamples.
+        Length must match arr.ndim
+    block_shape : tuple of int
+        shape of individual blocks to process
+    n_threads :  int
+        number of threads used to concurrently downsample chunks
+
+    Returns
+    -------
+    output_arr : numpy.ndarray
+        downsampled version of arr
+    """
+    blocks = skimage.util.view_as_blocks(arr, block_shape)
+
+    # construct output array and blocks
+    output_shape = tuple(
+        int(math.ceil(s / f)) for s, f in zip(arr.shape, ds_factors))
+    output_block_shape = tuple(
+        int(math.ceil(s / f)) for s, f in zip(block_shape, ds_factors))
+    output_arr = numpy.empty(output_shape, dtype=arr.dtype)
+    output_blocks = skimage.util.view_as_blocks(
+        output_arr, output_block_shape)
+
+    # process
+    with concurrent.futures.ThreadPoolExecutor(max_workers=n_threads) as e:
+        fut_to_idxs = {}
+        for idxs in numpy.ndindex(blocks.shape[:arr.ndim]):
+            fut_to_idxs[e.submit(
+                acpreprocessing.utils.convert.downsample_stack_volume,
+                blocks[idxs], ds_factors, **kwargs)] = idxs
+
+        for fut in concurrent.futures.as_completed(fut_to_idxs.keys()):
+            idxs = fut_to_idxs[fut]
+            output_blocks[idxs] = fut.result()
+    return output_arr
+
+
+class ArrayChunkingError(ValueError):
+    """ValueError caused by incorrect array chunking"""
+
+
+def downsample_array(arr, *args, n_threads=None, block_shape=None,
+                     block_divs=None, allow_failover=True, **kwargs):
+    """downsample array with optional subvolume chunking and threading
+
+    Parameters
+    ----------
+    arr : numpy.ndarray
+        array to downsample
+    ds_factors : tuple of int
+        integer downsample factor by which arr should be downsamples.
+        Length must match arr.ndim
+    block_shape : tuple of int
+        shape of individual blocks to process.
+        Supersedes block_divs if both are provided
+    block_divs : tuple of int
+        number of divisions for array chunking.
+    n_threads :  int
+        number of threads used to concurrently downsample chunks
+    allow_failover : bool, optional
+        whether to default to using
+        acpreprocessing.utils.convert.downsample_stack_volume when
+        block_shape or block_divs is not actionable.  (default: True)
+
+    Returns
+    -------
+    output_arr : numpy.ndarray
+        downsampled version of arr
+    """
+    try:
+        if block_shape is not None or block_divs is not None:
+            if block_shape is None:
+                block_shape = [a_s / d for a_s, d in zip(
+                    arr.shape, block_divs)]
+                if not all([s.is_integer() for s in block_shape]):
+                    raise ArrayChunkingError(
+                        "cannot divide array of shape {} by {}".format(
+                            arr.shape, block_divs))
+                block_shape = tuple(int(s) for s in block_shape)
+            if n_threads is not None:
+                return array_downsample_threaded(
+                    arr, *args, n_threads=n_threads,
+                    block_shape=block_shape, **kwargs)
+            return array_downsample_chunks(
+                arr, *args, block_shape=block_shape, **kwargs)
+    except ArrayChunkingError:
+        if not allow_failover:
+            raise
+    return acpreprocessing.utils.convert.downsample_stack_volume(
+        arr, *args, **kwargs)
+
+
+def mip_level_shape(lvl, lvl_0_shape):
+    """get the expected shape of a MIP level downsample for a given
+    full resolution shape.  Expects 2x downsampling on each dimension.
+
+    Parameters
+    ----------
+    lvl : int
+        mip level for which shape is calculated
+    lvl_0_shape : tuple of int
+        shape tuple of level 0
+
+    Returns
+    -------
+    lvl_shape : tuple of int
+        shape tuple for lvl
+    """
+    return tuple([*map(lambda x: math.ceil(x / (2**lvl)), lvl_0_shape)])
+
+
+def dswrite_chunk(ds, start, end, arr, silent_overflow=True):
+    """write array arr into array-like n5 dataset defined by
+    start and end points on 0 axis.
+
+    Parameters
+    ----------
+    ds : z5py.dataset.Dataset (n5 3D) or zarr.dataset (zarr 5D)
+        array-like dataset to fill in
+    start : int
+        start index along 0 axis of ds to fill in
+    end : int
+        end index along 0 axis of ds to fill in
+    arr : numpy.ndarray
+        array from which ds values will be taken
+    silent_overflow : bool, optional
+        whether to shrink the end index to match the
+        shape of ds (default: True)
+    """
+    if len(ds.shape) == 5:  # dataset dimensions should be 3 or 5
+        if end >= ds.shape[2] and silent_overflow:
+            end = ds.shape[2]
+        if end > start:
+            ds[0, 0, start:end, :, :] = arr[:(end - start), :, :]
+    elif len(ds.shape) == 3:
+        if end >= ds.shape[0] and silent_overflow:
+            end = ds.shape[0]
+        if end > start:
+            ds[start:end, :, :] = arr[:(end - start), :, :]
+
+
+@dataclasses.dataclass
+class MIPArray:
+    lvl: int
+    array: numpy.ndarray
+    start: int
+    end: int
+
+
+def iterate_mip_levels_from_mimgfns(
+        mimgfns, lvl, block_size, slice_length, downsample_factor,
+        downsample_method=None, lvl_to_mip_kwargs=None,
+        interleaved_channels=1, channel=0, deskew_kwargs={}):
+    """recursively generate MIPmap levels from an iterator of multi-image files
+
+    Parameters
+    ----------
+    mimgfns : list of str
+        imageio-compatible name inputs to be opened as multi-images
+    lvl : int
+        integer mip level to generate
+    block_size : int
+        number of 2D arrays in chunk to process for a chunk at lvl
+    slice_length : int
+        number of 2D arrays to gather from tiff stacks
+    downsample_factor : tuple of int
+        integer downsampling factor for MIP levels
+    downsample_method : str, optional
+        downsampling method for
+        acpreprocessing.utils.convert.downsample_stack_volume
+    lvl_to_mip_kwargs :  dict, optional
+        mapping of MIP level to kwargs used in MIPmap generation
+    interleaved_channels : int
+        number of channels interleaved in the tiff files (default 1)
+    channel : int, optional
+        channel from which interleaved data should be read (default 0)
+    deskew_kwargs : dict, optional
+        parameters for pixel shifting deskew
+
+    Yields
+    ------
+    ma : acpreprocessing.stitching_modules.convert_to_n5.tiff_to_n5.MipArray
+        object describing chunked array, MIP level of origin, and chunk indices
+    """
+    lvl_to_mip_kwargs = ({} if lvl_to_mip_kwargs is None
+                         else lvl_to_mip_kwargs)
+    mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
+    start_index = 0
+    chunk_index = 0
+    if lvl > 0:
+        num_chunks = downsample_factor[0]
+
+        i = 0
+        for ma in iterate_mip_levels_from_mimgfns(
+                mimgfns, lvl-1, block_size, slice_length,
+                downsample_factor, downsample_method,
+                lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
+                channel=channel, deskew_kwargs=deskew_kwargs):
+            chunk = ma.array
+            # throw array up for further processing
+            yield ma
+            # ignore if not parent resolution
+            if ma.lvl != lvl-1:
+                continue
+
+            try:
+                temp_lminus1_arr
+            except NameError:
+                temp_lminus1_arr = numpy.empty(
+                    (num_chunks*chunk.shape[0], *chunk.shape[1:]),
+                    dtype=chunk.dtype)
+
+            # fill in temporary block according to index
+            chunk_size = chunk.shape[0]
+            block_offset = i*block_size
+
+            temp_lminus1_arr[
+                block_offset:block_offset+chunk_size, :, :] = chunk[:, :, :]
+
+            # copy op only for uneven final chunk
+            if chunk_size != block_size:
+                temp_lminus1_arr = temp_lminus1_arr[
+                    :block_offset+chunk_size, :, :]
+
+            if i == num_chunks - 1:
+                temp_arr = (
+                    downsample_array(
+                        temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
+                        method=downsample_method, **mip_kwargs))
+                end_index = start_index + temp_arr.shape[0]
+                yield MIPArray(lvl, temp_arr, start_index, end_index)
+                start_index += temp_arr.shape[0]
+                i = 0
+            else:
+                i += 1
+        # add any leftovers
+        if i > 0:
+            temp_arr = acpreprocessing.utils.convert.downsample_stack_volume(
+                temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
+                method=downsample_method)
+            end_index = start_index + temp_arr.shape[0]
+            yield MIPArray(lvl, temp_arr, start_index, end_index)
+    else:
+        # get level 0 chunks
+        # block_size is the number of slices to read from tiffs
+        for chunk in iterate_numpy_chunks_from_mimgfns(
+                mimgfns, slice_length, pad=False,
+                interleaved_channels=interleaved_channels,
+                channel=channel):
+            # deskew level 0 chunk
+            if deskew_kwargs:
+                chunk = numpy.transpose(psd.deskew_block(
+                    chunk, chunk_index, **deskew_kwargs), (2, 1, 0))
+            end_index = start_index + chunk.shape[0]
+            yield MIPArray(lvl, chunk, start_index, end_index)
+            start_index += chunk.shape[0]
+            chunk_index += 1
+
+
+def write_mimgfns_to_n5(
+        mimgfns, output_n5, group_names, group_attributes=None, max_mip=0,
+        mip_dsfactor=(2, 2, 2), chunk_size=(32, 32, 32),
+        concurrency=10, slice_concurrency=1,
+        compression="raw", dtype="uint16", lvl_to_mip_kwargs=None,
+        interleaved_channels=1, channel=0, deskew_options=None, **kwargs):
+    """write a stack represented by an iterator of multi-image files as an n5
+    volume
+
+    Parameters
+    ----------
+    mimgfns : list of str
+        imageio-compatible name inputs to be opened as multi-images
+    output_n5 : str
+        output n5 directory
+    group_names : list of str
+        names of groups to generate within n5
+    group_attributes : list of dict, optional
+        attribute dictionaries corresponding to group with matching index
+    max_mip : int
+        maximum MIP level to generate
+    mip_dsfactor : tuple of int
+        integer downsampling factor for MIP levels
+    chunk_size : tuple of int
+        chunk size for n5 datasets
+    concurrency : int
+        total concurrency used for writing arrays to n5
+        (python threads = concurrency // slice_concurrency)
+    slice_concurrency : int
+        threads used by z5py
+    compression : str, optional
+        compression for n5 (default: raw)
+    dtype : str, optional
+        dtype for n5 (default: uint16)
+    lvl_to_mip_kwargs :  dict, optional
+        mapping of MIP level to kwargs used in MIPmap generation
+    interleaved_channels : int
+        number of channels interleaved in the tiff files (default 1)
+    channel : int, optional
+        channel from which interleaved data should be read (default 0)
+    deskew_options : dict, optional
+        dictionary of parameters to run pixel shifting deskew (default None)
+    """
+    group_attributes = ([] if group_attributes is None else group_attributes)
+    deskew_options = ({} if deskew_options is None else deskew_options)
+
+    joined_shapes = joined_mimg_shape_from_fns(
+        mimgfns, concurrency=concurrency,
+        interleaved_channels=interleaved_channels, channel=channel)
+    if deskew_options and deskew_options["deskew_method"] == "ps":
+        block_size = chunk_size[0]
+        slice_length = int(chunk_size[0]/deskew_options['deskew_stride'])
+        deskew_kwargs = psd.psdeskew_kwargs(skew_dims_zyx=(slice_length, joined_shapes[1], joined_shapes[2]),
+                                            **deskew_options
+                                            )
+        joined_shapes = psd.reshape_joined_shapes(
+            joined_shapes, deskew_options['deskew_stride'], **deskew_kwargs)
+    else:
+        block_size = chunk_size[0]
+        slice_length = block_size
+        deskew_kwargs = {}
+
+    # TODO DESKEW: does this generally work regardless of skew?
+
+    workers = concurrency // slice_concurrency
+
+    with z5py.File(output_n5) as f:
+        mip_ds = {}
+        # create groups with custom attributes
+        group_objs = []
+        for i, group_name in enumerate(group_names):
+            try:
+                g = group_objs[-1].create_group(f"{group_name}")
+            except IndexError:
+                try:
+                    g = f.create_group(f"{group_name}")
+                except KeyError:
+                    g = f[f"{group_name}"]
+            group_objs.append(g)
+            try:
+                attributes = group_attributes[i]
+            except IndexError:
+                continue
+            if deskew_options:
+                if "pixelResolution" in attributes:
+                    attributes["pixelResolution"]["dimensions"][0] /= deskew_options["deskew_stride"]
+            for k, v in attributes.items():
+                g.attrs[k] = v
+        scales = []
+        for mip_lvl in range(max_mip + 1):
+            ds_lvl = g.create_dataset(
+                f"s{mip_lvl}",
+                chunks=chunk_size,
+                shape=mip_level_shape(mip_lvl, joined_shapes),
+                compression=compression,
+                dtype=dtype,
+                n_threads=slice_concurrency)
+            dsfactors = [int(i)**mip_lvl for i in mip_dsfactor]
+            ds_lvl.attrs["downsamplingFactors"] = dsfactors
+            mip_ds[mip_lvl] = ds_lvl
+            scales.append(dsfactors)
+        g.attrs["scales"] = scales
+        group_objs[0].attrs["downsamplingFactors"] = scales
+        group_objs[0].attrs["dataType"] = dtype
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as e:
+            futs = []
+            for miparr in iterate_mip_levels_from_mimgfns(
+                    mimgfns, max_mip, block_size, slice_length, mip_dsfactor,
+                    lvl_to_mip_kwargs=lvl_to_mip_kwargs,
+                    interleaved_channels=interleaved_channels,
+                    channel=channel, deskew_kwargs=deskew_kwargs):
+                futs.append(e.submit(
+                    dswrite_chunk, mip_ds[miparr.lvl],
+                    miparr.start, miparr.end, miparr.array))
+            for fut in concurrent.futures.as_completed(futs):
+                _ = fut.result()
+
+
+def omezarr_attrs(name, position_xyz, lvl0_xyz_res, max_lvl):
+    # TODO only single channel implemented
+    datasets = []
+    for i in range(max_lvl+1):
+        s = [(2**i)*r for r in lvl0_xyz_res]
+        d = {"coordinateTransformations": [
+            {"scale": [
+                1.0,
+                1.0,
+                s[2],
+                s[1],
+                s[0]
+            ],
+                "type": "scale"
+            }
+        ],
+            "path": str(i)
+        }
+        datasets.append(d)
+    attrs = {}
+    attrs["multiscales"] = [
+        {
+            "axes": [
+                {
+                    "name": "t",
+                    "type": "time",
+                    "unit": "millisecond"
+                },
+                {
+                    "name": "c",
+                    "type": "channel"
+                },
+                {
+                    "name": "z",
+                    "type": "space",
+                    "unit": "micrometer"
+                },
+                {
+                    "name": "y",
+                    "type": "space",
+                    "unit": "micrometer"
+                },
+                {
+                    "name": "x",
+                    "type": "space",
+                    "unit": "micrometer"
+                }
+            ],
+            "datasets": datasets,
+            "name": "/" + name,
+            "translation": [
+                0,
+                0,
+                0.0,
+                0.0,
+                0.0
+            ],
+            "version": "0.4",
+            "coordinateTransformations": [
+                {
+                    "translation": [
+                        0.0,
+                        0.0,
+                        position_xyz[2],
+                        position_xyz[1],
+                        position_xyz[0]
+                    ],
+                    "type": "translation"
+                }
+            ]
+        }
+    ]
+    attrs["omero"] = {
+        "channels": [
+            {
+                "active": True,
+                "coefficient": 1,
+                "color": "000000",
+                "family": "linear",
+                "inverted": False,
+                "label": "Channel:" + name + ":0",
+                "window": {
+                    "end": 1.0,
+                    "max": 1.0,
+                    "min": 0.0,
+                    "start": 0.0
+                }
+            }
+        ],
+        "id": 1,
+        "name": name,
+        "rdefs": {
+            "defaultT": 0,
+            "defaultZ": 288,
+            "model": "color"
+        },
+        "version": "0.4"
+    }
+    return attrs
+
+
+class TiffToNGFFException(Exception):
+    """class to describe exceptions with TiffToNGFF module"""
+
+
+class TiffToNGFFValueError(TiffToNGFFException, ValueError):
+    """value error in TiffToNgff"""
+
+
+def write_mimgfns_to_zarr(
+        mimgfns, output_n5, group_names, group_attributes=None, max_mip=0,
+        mip_dsfactor=(2, 2, 2), chunk_size=(256, 256, 256),
+        concurrency=10, slice_concurrency=1,
+        compression="raw", dtype="uint16", lvl_to_mip_kwargs=None,
+        interleaved_channels=1, channel=0, deskew_options=None, **kwargs):
+    """write a stack represented by an iterator of multi-image files as a zarr
+    volume with ome-ngff metadata
+
+    Parameters
+    ----------
+    mimgfns : list of str
+        imageio-compatible name inputs to be opened as multi-images
+    output_n5 : str
+        output zarr directory
+    group_names : list of str
+        names of groups to generate within n5
+    group_attributes : list of dict, optional
+        attribute dictionaries corresponding to group with matching index
+    max_mip : int
+        maximum MIP level to generate
+    mip_dsfactor : tuple of int
+        integer downsampling factor for MIP levels
+    chunk_size : tuple of int
+        chunk size for n5 datasets
+    concurrency : int
+        total concurrency used for writing arrays to n5
+        (python threads = concurrency // slice_concurrency)
+    slice_concurrency : int
+        threads used by z5py
+    compression : str, optional
+        compression for n5 (default: raw)
+    dtype : str, optional
+        dtype for n5 (default: uint16)
+    lvl_to_mip_kwargs :  dict, optional
+        mapping of MIP level to kwargs used in MIPmap generation
+    interleaved_channels : int
+        number of channels interleaved in the tiff files (default 1)
+    channel : int, optional
+        channel from which interleaved data should be read (default 0)
+    deskew_options : dict, optional
+        dictionary of parameters to run pixel shifting deskew (default None)
+    """
+    group_attributes = ([] if group_attributes is None else group_attributes)
+    deskew_options = ({} if deskew_options is None else deskew_options)
+
+    joined_shapes = joined_mimg_shape_from_fns(
+        mimgfns, concurrency=concurrency,
+        interleaved_channels=interleaved_channels, channel=channel)
+    if deskew_options and deskew_options["deskew_method"] == "ps":
+        block_size = chunk_size[0]
+        slice_length = int(chunk_size[0]/deskew_options['deskew_stride'])
+        deskew_kwargs = psd.psdeskew_kwargs(skew_dims_zyx=(slice_length, joined_shapes[1], joined_shapes[2]),
+                                            **deskew_options
+                                            )
+        joined_shapes = psd.reshape_joined_shapes(
+            joined_shapes, deskew_options['deskew_stride'], **deskew_kwargs)
+    else:
+        block_size = chunk_size[0]
+        slice_length = block_size
+        deskew_kwargs = {}
+
+    # TODO DESKEW: does this generally work regardless of skew?
+
+    workers = concurrency // slice_concurrency
+
+    zstore = zarr.DirectoryStore(output_n5, dimension_separator='/')
+    with zarr.open(zstore, mode='a') as f:
+        mip_ds = {}
+        # create groups with attributes according to omezarr spec
+        stack_group = group_names[0]
+        channel_group = group_names[1]
+        scales = []
+
+        # shuffle=Blosc.BITSHUFFLE)
+        compression = Blosc(cname='zstd', clevel=1)
+        for mip_lvl in range(max_mip + 1):
+            g = f.create_group(f"Resolution_Level_{mip_lvl}")
+            g = g.create_group(f"Stack_{stack_group}")
+            mip_3dshape = mip_level_shape(mip_lvl, joined_shapes)
+            ds_lvl = g.create_dataset(
+                f"{channel_group}",
+                chunks=chunk_size,
+                shape=(mip_3dshape[0], mip_3dshape[1], mip_3dshape[2]),
+                compression=compression,
+                dtype=dtype
+            )
+            dsfactors = [int(i)**mip_lvl for i in mip_dsfactor]
+            mip_ds[mip_lvl] = ds_lvl
+            scales.append(dsfactors)
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as e:
+            futs = []
+            for miparr in iterate_mip_levels_from_mimgfns(
+                    mimgfns, max_mip, block_size, slice_length, mip_dsfactor,
+                    lvl_to_mip_kwargs=lvl_to_mip_kwargs,
+                    interleaved_channels=interleaved_channels,
+                    channel=channel, deskew_kwargs=deskew_kwargs):
+                futs.append(e.submit(
+                    dswrite_chunk, mip_ds[miparr.lvl],
+                    miparr.start, miparr.end, miparr.array))
+            for fut in concurrent.futures.as_completed(futs):
+                _ = fut.result()
+
+
+def tiffdir_to_ngff_group(tiffdir, output, *args, **kwargs):
+    """convert directory of natsort-consecutive multitiffs to an n5 or zarr pyramid
+
+    Parameters
+    ----------
+    tiffdir : str
+        directory of consecutive multitiffs to convert
+    """
+    mimgfns = [str(p) for p in natsorted(
+        pathlib.Path(tiffdir).iterdir(), key=lambda x:str(x))
+        if p.is_file()]
+
+    if output == 'zarr':
+        print('converting to zarr')
+        return write_mimgfns_to_zarr(mimgfns, *args, **kwargs)
+    else:
+        print('converting to n5')
+        return write_mimgfns_to_n5(mimgfns, *args, **kwargs)
+
+
+class DownsampleOptions(argschema.schemas.DefaultSchema):
+    block_divs = argschema.fields.List(
+        argschema.fields.Int, required=False, allow_none=True)
+    n_threads = argschema.fields.Int(required=False, allow_none=True)
+
+
+class DeskewOptions(argschema.schemas.DefaultSchema):
+    deskew_method = argschema.fields.Str(required=False, default='')
+    deskew_stride = argschema.fields.Int(required=False, default=None)
+    deskew_flip = argschema.fields.Bool(required=False, default=True)
+    deskew_crop = argschema.fields.Float(required=False, default=1.0)
+
+
+class NGFFGenerationParameters(argschema.schemas.DefaultSchema):
+    output_format = argschema.fields.Str(required=True)
+    output_file = argschema.fields.Str(required=True)
+    max_mip = argschema.fields.Int(required=False, default=0)
+    concurrency = argschema.fields.Int(required=False, default=10)
+    compression = argschema.fields.Str(required=False, default="raw")
+    lvl_to_mip_kwargs = argschema.fields.Dict(
+        keys=argschema.fields.Int(),
+        values=argschema.fields.Nested(DownsampleOptions))
+
+    # FIXME argschema supports lists and tuples,
+    #   but has some version differences
+
+    mip_dsfactor = argschema.fields.Tuple((
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int()), required=False, default=(2, 2, 2))
+    deskew_options = argschema.fields.Nested(
+        DeskewOptions, required=False)
+
+
+class NGFFGroupGenerationParameters(NGFFGenerationParameters):
+    group_names = argschema.fields.List(
+        argschema.fields.Str, required=True)
+    group_attributes = argschema.fields.List(
+        argschema.fields.Dict(required=False, default={}), default=[],
+        required=False)
+
+
+class TiffDirToNGFFParameters(NGFFGroupGenerationParameters):
+    input_dir = argschema.fields.InputDir(required=True)
+    interleaved_channels = argschema.fields.Int(required=False, default=1)
+    channel = argschema.fields.Int(required=False, default=0)
+
+
+class TiffDirToZarrInputParameters(argschema.ArgSchema,
+                                   TiffDirToNGFFParameters):
+    chunk_size = argschema.fields.Tuple((
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int()), required=False, default=(1, 1, 64, 64, 64))
+
+
+class TiffDirToN5LegacyParameters(argschema.ArgSchema,
+                                  TiffDirToNGFFParameters):
+    chunk_size = argschema.fields.Tuple((
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int()), required=False, default=(64, 64, 64))
+
+
+class TiffDirToZarr(argschema.ArgSchemaParser):
+    default_schema = TiffDirToZarrInputParameters
+
+    def run(self):
+        deskew_options = (self.args["deskew_options"]
+                          if "deskew_options" in self.args else {})
+        tiffdir_to_ngff_group(
+            self.args["input_dir"], self.args["output_format"],
+            self.args["output_file"], self.args["group_names"],
+            self.args["group_attributes"],
+            self.args["max_mip"],
+            self.args["mip_dsfactor"],
+            self.args["chunk_size"],
+            concurrency=self.args["concurrency"],
+            compression=self.args["compression"],
+            lvl_to_mip_kwargs=self.args["lvl_to_mip_kwargs"],
+            deskew_options=deskew_options)
+
+
+class TiffDirToN5(TiffDirToZarr):
+    default_schema = TiffDirToN5LegacyParameters
+
+
+if __name__ == "__main__":
+    mod = TiffDirToZarr()
+    mod.run()

From 09aa81e8cf18125aa5987924ad6cb8d0c16448c0 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Mon, 3 Jun 2024 21:08:24 -0700
Subject: [PATCH 010/115] cleaning up module

---
 .../convert_to_n5/tiff_to_visor.py            | 225 +-----------------
 1 file changed, 1 insertion(+), 224 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
index 7671d9a..34ac700 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
@@ -11,7 +11,6 @@
 import numpy
 import skimage
 
-import z5py
 import zarr
 from numcodecs import Blosc
 import argschema
@@ -525,224 +524,7 @@ def iterate_mip_levels_from_mimgfns(
             chunk_index += 1
 
 
-def write_mimgfns_to_n5(
-        mimgfns, output_n5, group_names, group_attributes=None, max_mip=0,
-        mip_dsfactor=(2, 2, 2), chunk_size=(32, 32, 32),
-        concurrency=10, slice_concurrency=1,
-        compression="raw", dtype="uint16", lvl_to_mip_kwargs=None,
-        interleaved_channels=1, channel=0, deskew_options=None, **kwargs):
-    """write a stack represented by an iterator of multi-image files as an n5
-    volume
 
-    Parameters
-    ----------
-    mimgfns : list of str
-        imageio-compatible name inputs to be opened as multi-images
-    output_n5 : str
-        output n5 directory
-    group_names : list of str
-        names of groups to generate within n5
-    group_attributes : list of dict, optional
-        attribute dictionaries corresponding to group with matching index
-    max_mip : int
-        maximum MIP level to generate
-    mip_dsfactor : tuple of int
-        integer downsampling factor for MIP levels
-    chunk_size : tuple of int
-        chunk size for n5 datasets
-    concurrency : int
-        total concurrency used for writing arrays to n5
-        (python threads = concurrency // slice_concurrency)
-    slice_concurrency : int
-        threads used by z5py
-    compression : str, optional
-        compression for n5 (default: raw)
-    dtype : str, optional
-        dtype for n5 (default: uint16)
-    lvl_to_mip_kwargs :  dict, optional
-        mapping of MIP level to kwargs used in MIPmap generation
-    interleaved_channels : int
-        number of channels interleaved in the tiff files (default 1)
-    channel : int, optional
-        channel from which interleaved data should be read (default 0)
-    deskew_options : dict, optional
-        dictionary of parameters to run pixel shifting deskew (default None)
-    """
-    group_attributes = ([] if group_attributes is None else group_attributes)
-    deskew_options = ({} if deskew_options is None else deskew_options)
-
-    joined_shapes = joined_mimg_shape_from_fns(
-        mimgfns, concurrency=concurrency,
-        interleaved_channels=interleaved_channels, channel=channel)
-    if deskew_options and deskew_options["deskew_method"] == "ps":
-        block_size = chunk_size[0]
-        slice_length = int(chunk_size[0]/deskew_options['deskew_stride'])
-        deskew_kwargs = psd.psdeskew_kwargs(skew_dims_zyx=(slice_length, joined_shapes[1], joined_shapes[2]),
-                                            **deskew_options
-                                            )
-        joined_shapes = psd.reshape_joined_shapes(
-            joined_shapes, deskew_options['deskew_stride'], **deskew_kwargs)
-    else:
-        block_size = chunk_size[0]
-        slice_length = block_size
-        deskew_kwargs = {}
-
-    # TODO DESKEW: does this generally work regardless of skew?
-
-    workers = concurrency // slice_concurrency
-
-    with z5py.File(output_n5) as f:
-        mip_ds = {}
-        # create groups with custom attributes
-        group_objs = []
-        for i, group_name in enumerate(group_names):
-            try:
-                g = group_objs[-1].create_group(f"{group_name}")
-            except IndexError:
-                try:
-                    g = f.create_group(f"{group_name}")
-                except KeyError:
-                    g = f[f"{group_name}"]
-            group_objs.append(g)
-            try:
-                attributes = group_attributes[i]
-            except IndexError:
-                continue
-            if deskew_options:
-                if "pixelResolution" in attributes:
-                    attributes["pixelResolution"]["dimensions"][0] /= deskew_options["deskew_stride"]
-            for k, v in attributes.items():
-                g.attrs[k] = v
-        scales = []
-        for mip_lvl in range(max_mip + 1):
-            ds_lvl = g.create_dataset(
-                f"s{mip_lvl}",
-                chunks=chunk_size,
-                shape=mip_level_shape(mip_lvl, joined_shapes),
-                compression=compression,
-                dtype=dtype,
-                n_threads=slice_concurrency)
-            dsfactors = [int(i)**mip_lvl for i in mip_dsfactor]
-            ds_lvl.attrs["downsamplingFactors"] = dsfactors
-            mip_ds[mip_lvl] = ds_lvl
-            scales.append(dsfactors)
-        g.attrs["scales"] = scales
-        group_objs[0].attrs["downsamplingFactors"] = scales
-        group_objs[0].attrs["dataType"] = dtype
-
-        with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as e:
-            futs = []
-            for miparr in iterate_mip_levels_from_mimgfns(
-                    mimgfns, max_mip, block_size, slice_length, mip_dsfactor,
-                    lvl_to_mip_kwargs=lvl_to_mip_kwargs,
-                    interleaved_channels=interleaved_channels,
-                    channel=channel, deskew_kwargs=deskew_kwargs):
-                futs.append(e.submit(
-                    dswrite_chunk, mip_ds[miparr.lvl],
-                    miparr.start, miparr.end, miparr.array))
-            for fut in concurrent.futures.as_completed(futs):
-                _ = fut.result()
-
-
-def omezarr_attrs(name, position_xyz, lvl0_xyz_res, max_lvl):
-    # TODO only single channel implemented
-    datasets = []
-    for i in range(max_lvl+1):
-        s = [(2**i)*r for r in lvl0_xyz_res]
-        d = {"coordinateTransformations": [
-            {"scale": [
-                1.0,
-                1.0,
-                s[2],
-                s[1],
-                s[0]
-            ],
-                "type": "scale"
-            }
-        ],
-            "path": str(i)
-        }
-        datasets.append(d)
-    attrs = {}
-    attrs["multiscales"] = [
-        {
-            "axes": [
-                {
-                    "name": "t",
-                    "type": "time",
-                    "unit": "millisecond"
-                },
-                {
-                    "name": "c",
-                    "type": "channel"
-                },
-                {
-                    "name": "z",
-                    "type": "space",
-                    "unit": "micrometer"
-                },
-                {
-                    "name": "y",
-                    "type": "space",
-                    "unit": "micrometer"
-                },
-                {
-                    "name": "x",
-                    "type": "space",
-                    "unit": "micrometer"
-                }
-            ],
-            "datasets": datasets,
-            "name": "/" + name,
-            "translation": [
-                0,
-                0,
-                0.0,
-                0.0,
-                0.0
-            ],
-            "version": "0.4",
-            "coordinateTransformations": [
-                {
-                    "translation": [
-                        0.0,
-                        0.0,
-                        position_xyz[2],
-                        position_xyz[1],
-                        position_xyz[0]
-                    ],
-                    "type": "translation"
-                }
-            ]
-        }
-    ]
-    attrs["omero"] = {
-        "channels": [
-            {
-                "active": True,
-                "coefficient": 1,
-                "color": "000000",
-                "family": "linear",
-                "inverted": False,
-                "label": "Channel:" + name + ":0",
-                "window": {
-                    "end": 1.0,
-                    "max": 1.0,
-                    "min": 0.0,
-                    "start": 0.0
-                }
-            }
-        ],
-        "id": 1,
-        "name": name,
-        "rdefs": {
-            "defaultT": 0,
-            "defaultZ": 288,
-            "model": "color"
-        },
-        "version": "0.4"
-    }
-    return attrs
 
 
 class TiffToNGFFException(Exception):
@@ -873,9 +655,6 @@ def tiffdir_to_ngff_group(tiffdir, output, *args, **kwargs):
     if output == 'zarr':
         print('converting to zarr')
         return write_mimgfns_to_zarr(mimgfns, *args, **kwargs)
-    else:
-        print('converting to n5')
-        return write_mimgfns_to_n5(mimgfns, *args, **kwargs)
 
 
 class DownsampleOptions(argschema.schemas.DefaultSchema):
@@ -931,9 +710,7 @@ class TiffDirToZarrInputParameters(argschema.ArgSchema,
     chunk_size = argschema.fields.Tuple((
         argschema.fields.Int(),
         argschema.fields.Int(),
-        argschema.fields.Int(),
-        argschema.fields.Int(),
-        argschema.fields.Int()), required=False, default=(1, 1, 64, 64, 64))
+        argschema.fields.Int()), required=False, default=(256, 256, 256))
 
 
 class TiffDirToN5LegacyParameters(argschema.ArgSchema,

From 98fd1b48a9854c0dc8124a9b8618c538538df62e Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Mon, 3 Jun 2024 21:13:35 -0700
Subject: [PATCH 011/115] streamlining

---
 .../convert_to_n5/tiff_to_visor.py            | 20 ++++---------------
 1 file changed, 4 insertions(+), 16 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
index 34ac700..1166740 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
@@ -676,9 +676,9 @@ class NGFFGenerationParameters(argschema.schemas.DefaultSchema):
     max_mip = argschema.fields.Int(required=False, default=0)
     concurrency = argschema.fields.Int(required=False, default=10)
     compression = argschema.fields.Str(required=False, default="raw")
-    lvl_to_mip_kwargs = argschema.fields.Dict(
-        keys=argschema.fields.Int(),
-        values=argschema.fields.Nested(DownsampleOptions))
+    # lvl_to_mip_kwargs = argschema.fields.Dict(
+    #     keys=argschema.fields.Int(),
+    #     values=argschema.fields.Nested(DownsampleOptions))
 
     # FIXME argschema supports lists and tuples,
     #   but has some version differences
@@ -713,14 +713,6 @@ class TiffDirToZarrInputParameters(argschema.ArgSchema,
         argschema.fields.Int()), required=False, default=(256, 256, 256))
 
 
-class TiffDirToN5LegacyParameters(argschema.ArgSchema,
-                                  TiffDirToNGFFParameters):
-    chunk_size = argschema.fields.Tuple((
-        argschema.fields.Int(),
-        argschema.fields.Int(),
-        argschema.fields.Int()), required=False, default=(64, 64, 64))
-
-
 class TiffDirToZarr(argschema.ArgSchemaParser):
     default_schema = TiffDirToZarrInputParameters
 
@@ -736,14 +728,10 @@ def run(self):
             self.args["chunk_size"],
             concurrency=self.args["concurrency"],
             compression=self.args["compression"],
-            lvl_to_mip_kwargs=self.args["lvl_to_mip_kwargs"],
+            #lvl_to_mip_kwargs=self.args["lvl_to_mip_kwargs"],
             deskew_options=deskew_options)
 
 
-class TiffDirToN5(TiffDirToZarr):
-    default_schema = TiffDirToN5LegacyParameters
-
-
 if __name__ == "__main__":
     mod = TiffDirToZarr()
     mod.run()

From 2fcf56bcc409ca271c935acad77d5b86e085364a Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Mon, 3 Jun 2024 21:26:45 -0700
Subject: [PATCH 012/115] write to same zarr

---
 .../stitching_modules/convert_to_n5/tiff_to_visor.py         | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
index 1166740..e542cdd 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
@@ -612,7 +612,10 @@ def write_mimgfns_to_zarr(
         # shuffle=Blosc.BITSHUFFLE)
         compression = Blosc(cname='zstd', clevel=1)
         for mip_lvl in range(max_mip + 1):
-            g = f.create_group(f"Resolution_Level_{mip_lvl}")
+            try:
+                g = f.create_group(f"Resolution_Level_{mip_lvl}")
+            except KeyError:
+                g = f[f"Resolution_Level_{mip_lvl}"]
             g = g.create_group(f"Stack_{stack_group}")
             mip_3dshape = mip_level_shape(mip_lvl, joined_shapes)
             ds_lvl = g.create_dataset(

From 6a5f3baf737e4f2e5f3ecc5feadac5286531a173 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Mon, 3 Jun 2024 21:30:49 -0700
Subject: [PATCH 013/115] fixing mip groups

---
 .../stitching_modules/convert_to_n5/tiff_to_visor.py     | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
index e542cdd..86bd3a5 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
@@ -612,10 +612,11 @@ def write_mimgfns_to_zarr(
         # shuffle=Blosc.BITSHUFFLE)
         compression = Blosc(cname='zstd', clevel=1)
         for mip_lvl in range(max_mip + 1):
-            try:
-                g = f.create_group(f"Resolution_Level_{mip_lvl}")
-            except KeyError:
-                g = f[f"Resolution_Level_{mip_lvl}"]
+            mip_group = f"Resolution_Level_{mip_lvl}"
+            if mip_group in f:
+                g = f[mip_group]
+            else:
+                g = f.create_group(mip_group)
             g = g.create_group(f"Stack_{stack_group}")
             mip_3dshape = mip_level_shape(mip_lvl, joined_shapes)
             ds_lvl = g.create_dataset(

From 5d6159990daf7f480b323bd1f27b6a58483d65fe Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Mon, 3 Jun 2024 21:34:27 -0700
Subject: [PATCH 014/115] fixing mip groups

---
 .../stitching_modules/convert_to_n5/tiff_to_visor.py            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
index 86bd3a5..e1904da 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
@@ -612,7 +612,7 @@ def write_mimgfns_to_zarr(
         # shuffle=Blosc.BITSHUFFLE)
         compression = Blosc(cname='zstd', clevel=1)
         for mip_lvl in range(max_mip + 1):
-            mip_group = f"Resolution_Level_{mip_lvl}"
+            mip_group = f"Resolution_Level_{mip_lvl+1}"
             if mip_group in f:
                 g = f[mip_group]
             else:

From 2cb0ef984c358a7be63251f9793a1d67378b0a2d Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 9 Jul 2024 16:02:23 -0700
Subject: [PATCH 015/115] refactoring shared methods

---
 .../convert_to_n5/ims_to_ngff.py              | 1244 +++++++++--------
 .../convert_to_n5/tiff_to_ngff.py             |  193 ++-
 2 files changed, 792 insertions(+), 645 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 02876a1..ee1447a 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -19,582 +19,582 @@
 
 import acpreprocessing.utils.convert
 import acpreprocessing.stitching_modules.convert_to_n5.psdeskew as psd
-
-
-def iterate_chunks(ds, slice_length, interleaved_channels=1, channel=0, interleaving_offset=0):
-    """given an iterator, iterate over tuples of a
-    given length from that iterator
-
-    Parameters
-    ----------
-    it : iterable
-        iterable object to iterate over in slices of slice_length
-    slice_length : int
-        number of items in iterator per chunk
-
-    Yields
-    ------
-    chunk : tuple
-        tuple of items from iterator of size slice_length. Yields
-        remainder of iterator when exhausted
-    """
-    start = 0
-    stop = slice_length
-    chunk = ds[start:stop]
-    while start<ds.shape[0]:
-        yield chunk
-        start = stop
-        stop = start + slice_length
-        if stop < ds.shape[0]:
-            chunk = ds[start:stop]
-        else:
-            chunk = ds[start:]
-
-
-def iter_arrays(r, interleaved_channels=1, channel=0, interleaving_offset=0):
-    """iterate arrays from an imageio tiff reader.  Allows the last image
-    of the array to be None, as is the case for data with 'dropped frames'.
-
-    Parameters
-    ----------
-    r : imageio.reader
-        imageio tiff reader from which to iterate multitiff arrays
-    interleaved_channels : int
-        number of channels interleaved in tiff when selecting which
-        arrays to read
-    channel : int
-        channel to iterate over in case of interleaved tiffs
-
-    Yields
-    -------
-    arr : numpy.ndarray
-        constituent page array of reader r
-    """
-    for i, p in enumerate(r._tf.pages):
-        page_channel = (i + interleaving_offset) % interleaved_channels
-        if page_channel != channel:
-            continue
-        arr = p.asarray()
-        if arr is not None:
-            yield arr
-        elif i == r.get_length() - 1:
-            continue
-        else:
-            raise ValueError
-
-
-def iterate_2d_arrays_from_mimgfns(mimgfns, interleaved_channels=1, channel=0):
-    """iterate constituent arrays from an iterator of image filenames
-    that can be opened as an imageio multi-image.
-
-    Parameters
-    ----------
-    mimgfns : list of str
-        imageio-compatible name inputs to be opened as multi-images
-
-    Yields
-    ------
-    array : numpy.ndarray
-        constituent array from ordered iterator of multi-image files
-    """
-    offset = 0
-    for mimgfn in mimgfns:
-        with imageio.get_reader(mimgfn, mode="I") as r:
-            yield from iter_arrays(r, interleaved_channels, channel, offset)
-            offset = (offset + r.get_length()) % interleaved_channels
-
-
-def iterate_numpy_chunks_from_dataset(
-        dataset, slice_length=None, pad=True, *args, **kwargs):
-    """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
-
-    Parameters
-    ----------
-    dataset : hdf5 dataset
-        imageio-compatible name inputs to be opened as multi-images
-    slice_length : int
-        number of 2d arrays from mimgfns included per chunk
-    pad : bool, optional
-        whether to extend final returned chunks with zeros
-
-    Yields
-    ------
-    arr : numpy.ndarray
-        3D numpy array representing a consecutive chunk of 2D arrays
-    """
-    #array_gen = iterate_2d_arrays_from_dataset(mimgfns, *args, **kwargs)
-    for chunk in iterate_chunks(dataset, slice_length,*args,**kwargs):
-        arr = numpy.asarray(chunk)
-        if pad:
-            if arr.shape[0] != slice_length:
-                newarr = numpy.zeros((slice_length, *arr.shape[1:]),
-                                     dtype=arr.dtype)
-                newarr[:arr.shape[0], :, :] = arr[:, :, :]
-                yield newarr
-            else:
-                yield arr
-        else:
-            yield arr
-
-
-def length_to_interleaved_length(length, interleaved_channels):
-    """get length of each interleaved channel that contributes to a
-    total array length
-
-    Parameters
-    ----------
-    length : int
-        length of interleaved array
-    interleaved_channels : int
-        number of constituent channels in interleaved array
-
-    Returns
-    -------
-    channel_lengths : list of int
-        length of each constituent channel in interleaved array
-    """
-    interleaved_length, r = divmod(length, interleaved_channels)
-    return [
-        interleaved_length + (1 if ic < r else 0)
-        for ic in range(interleaved_channels)
-    ]
-
-
-def mimg_shape_from_fn(mimg_fn, only_length_tup=False):
-    """get the shape of an imageio multi-image file without
-    reading it as a volume
-
-    Parameters
-    ----------
-    mimg_fn : str
-        imageio-compatible name input to be opened as multi-image
-    only_length_tup : bool, optional
-        whether to return only the number of frames or
-        include the dimensions of a representative frame
-
-    Returns
-    -------
-    shape : tuple of int
-        shape of array defined by mimg_fn
-    """
-    with imageio.get_reader(mimg_fn, mode="I") as r:
-        l = r.get_length()
-
-        if only_length_tup:
-            s = (l,)
-        else:
-            s = (l, *r.get_data(0).shape)
-    return s
-
-
-def array_downsample_chunks(arr, ds_factors, block_shape, **kwargs):
-    """downsample a numpy array by inputting subvolumes
-    to acpreprocessing.utils.convert.downsample_stack_volume
-
-    Parameters
-    ----------
-    arr : numpy.ndarray
-        array to downsample
-    ds_factors : tuple of int
-        integer downsample factor by which arr should be downsamples.
-        Length must match arr.ndim
-    block_shape : tuple of int
-        shape of individual blocks to process
-
-    Returns
-    -------
-    output_arr : numpy.ndarray
-        downsampled version of arr
-    """
-    blocks = skimage.util.view_as_blocks(arr, block_shape)
-
-    # construct output array and blocks
-    output_shape = tuple(
-        int(math.ceil(s / f)) for s, f in zip(arr.shape, ds_factors))
-    output_block_shape = tuple(
-        int(math.ceil(s / f)) for s, f in zip(block_shape, ds_factors))
-    output_arr = numpy.empty(output_shape, dtype=arr.dtype)
-    output_blocks = skimage.util.view_as_blocks(
-        output_arr, output_block_shape)
-
-    # process
-    for idxs in numpy.ndindex(blocks.shape[:arr.ndim]):
-        output_blocks[idxs] = (
-            acpreprocessing.utils.convert.downsample_stack_volume(
-                blocks[idxs], ds_factors, **kwargs))
-
-    return output_arr
-
-
-def array_downsample_threaded(arr, ds_factors, block_shape,
-                              n_threads=3, **kwargs):
-    """downsample a numpy array by concurrently inputting subvolumes
-    to acpreprocessing.utils.convert.downsample_stack_volume
-
-    Parameters
-    ----------
-    arr : numpy.ndarray
-        array to downsample
-    ds_factors : tuple of int
-        integer downsample factor by which arr should be downsamples.
-        Length must match arr.ndim
-    block_shape : tuple of int
-        shape of individual blocks to process
-    n_threads :  int
-        number of threads used to concurrently downsample chunks
-
-    Returns
-    -------
-    output_arr : numpy.ndarray
-        downsampled version of arr
-    """
-    blocks = skimage.util.view_as_blocks(arr, block_shape)
-
-    # construct output array and blocks
-    output_shape = tuple(
-        int(math.ceil(s / f)) for s, f in zip(arr.shape, ds_factors))
-    output_block_shape = tuple(
-        int(math.ceil(s / f)) for s, f in zip(block_shape, ds_factors))
-    output_arr = numpy.empty(output_shape, dtype=arr.dtype)
-    output_blocks = skimage.util.view_as_blocks(
-        output_arr, output_block_shape)
-
-    # process
-    with concurrent.futures.ThreadPoolExecutor(max_workers=n_threads) as e:
-        fut_to_idxs = {}
-        for idxs in numpy.ndindex(blocks.shape[:arr.ndim]):
-            fut_to_idxs[e.submit(
-                acpreprocessing.utils.convert.downsample_stack_volume,
-                blocks[idxs], ds_factors, **kwargs)] = idxs
-
-        for fut in concurrent.futures.as_completed(fut_to_idxs.keys()):
-            idxs = fut_to_idxs[fut]
-            output_blocks[idxs] = fut.result()
-    return output_arr
-
-
-class ArrayChunkingError(ValueError):
-    """ValueError caused by incorrect array chunking"""
-
-
-def downsample_array(arr, *args, n_threads=None, block_shape=None,
-                     block_divs=None, allow_failover=True, **kwargs):
-    """downsample array with optional subvolume chunking and threading
-
-    Parameters
-    ----------
-    arr : numpy.ndarray
-        array to downsample
-    ds_factors : tuple of int
-        integer downsample factor by which arr should be downsamples.
-        Length must match arr.ndim
-    block_shape : tuple of int
-        shape of individual blocks to process.
-        Supersedes block_divs if both are provided
-    block_divs : tuple of int
-        number of divisions for array chunking.
-    n_threads :  int
-        number of threads used to concurrently downsample chunks
-    allow_failover : bool, optional
-        whether to default to using
-        acpreprocessing.utils.convert.downsample_stack_volume when
-        block_shape or block_divs is not actionable.  (default: True)
-
-    Returns
-    -------
-    output_arr : numpy.ndarray
-        downsampled version of arr
-    """
-    try:
-        if block_shape is not None or block_divs is not None:
-            if block_shape is None:
-                block_shape = [a_s / d for a_s, d in zip(
-                    arr.shape, block_divs)]
-                if not all([s.is_integer() for s in block_shape]):
-                    raise ArrayChunkingError(
-                        "cannot divide array of shape {} by {}".format(
-                            arr.shape, block_divs))
-                block_shape = tuple(int(s) for s in block_shape)
-            if n_threads is not None:
-                return array_downsample_threaded(
-                    arr, *args, n_threads=n_threads,
-                    block_shape=block_shape, **kwargs)
-            return array_downsample_chunks(
-                arr, *args, block_shape=block_shape, **kwargs)
-    except ArrayChunkingError:
-        if not allow_failover:
-            raise
-    return acpreprocessing.utils.convert.downsample_stack_volume(
-        arr, *args, **kwargs)
-
-
-def mip_level_shape(lvl, lvl_0_shape):
-    """get the expected shape of a MIP level downsample for a given
-    full resolution shape.  Expects 2x downsampling on each dimension.
-
-    Parameters
-    ----------
-    lvl : int
-        mip level for which shape is calculated
-    lvl_0_shape : tuple of int
-        shape tuple of level 0
-
-    Returns
-    -------
-    lvl_shape : tuple of int
-        shape tuple for lvl
-    """
-    return tuple([*map(lambda x: math.ceil(x / (2**lvl)), lvl_0_shape)])
-
-
-def dswrite_chunk(ds, start, end, arr, silent_overflow=True):
-    """write array arr into array-like n5 dataset defined by
-    start and end points on 0 axis.
-
-    Parameters
-    ----------
-    ds : z5py.dataset.Dataset (n5 3D) or zarr.dataset (zarr 5D)
-        array-like dataset to fill in
-    start : int
-        start index along 0 axis of ds to fill in
-    end : int
-        end index along 0 axis of ds to fill in
-    arr : numpy.ndarray
-        array from which ds values will be taken
-    silent_overflow : bool, optional
-        whether to shrink the end index to match the
-        shape of ds (default: True)
-    """
-    if len(ds.shape) == 5:  # dataset dimensions should be 3 or 5
-        if end >= ds.shape[2] and silent_overflow:
-            end = ds.shape[2]
-        if end > start:
-            ds[0, 0, start:end, :, :] = arr[:(end - start), :, :]
-    elif len(ds.shape) == 3:
-        if end >= ds.shape[0] and silent_overflow:
-            end = ds.shape[0]
-        if end > start:
-            ds[start:end, :, :] = arr[:(end - start), :, :]
-
-
-@dataclasses.dataclass
-class MIPArray:
-    lvl: int
-    array: numpy.ndarray
-    start: int
-    end: int
-
-
-def iterate_mip_levels_from_dataset(
-        dataset, lvl, block_size, slice_length, downsample_factor,
-        downsample_method=None, lvl_to_mip_kwargs=None,
-        interleaved_channels=1, channel=0, deskew_kwargs={}):
-    """recursively generate MIPmap levels from an iterator of multi-image files
-
-    Parameters
-    ----------
-    dataset : hdf5 dataset
-        imageio-compatible name inputs to be opened as multi-images
-    lvl : int
-        integer mip level to generate
-    block_size : int
-        number of 2D arrays in chunk to process for a chunk at lvl
-    slice_length : int
-        number of 2D arrays to gather from tiff stacks
-    downsample_factor : tuple of int
-        integer downsampling factor for MIP levels
-    downsample_method : str, optional
-        downsampling method for
-        acpreprocessing.utils.convert.downsample_stack_volume
-    lvl_to_mip_kwargs :  dict, optional
-        mapping of MIP level to kwargs used in MIPmap generation
-    interleaved_channels : int
-        number of channels interleaved in the tiff files (default 1)
-    channel : int, optional
-        channel from which interleaved data should be read (default 0)
-    deskew_kwargs : dict, optional
-        parameters for pixel shifting deskew
-
-    Yields
-    ------
-    ma : acpreprocessing.stitching_modules.convert_to_n5.tiff_to_n5.MipArray
-        object describing chunked array, MIP level of origin, and chunk indices
-    """
-    lvl_to_mip_kwargs = ({} if lvl_to_mip_kwargs is None
-                         else lvl_to_mip_kwargs)
-    mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
-    start_index = 0
-    chunk_index = 0
-    if lvl > 0:
-        num_chunks = downsample_factor[0]
-
-        i = 0
-        for ma in iterate_mip_levels_from_dataset(
-                dataset, lvl-1, block_size, slice_length,
-                downsample_factor, downsample_method,
-                lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
-                channel=channel, deskew_kwargs=deskew_kwargs):
-            chunk = ma.array
-            # throw array up for further processing
-            yield ma
-            # ignore if not parent resolution
-            if ma.lvl != lvl-1:
-                continue
-
-            try:
-                temp_lminus1_arr
-            except NameError:
-                temp_lminus1_arr = numpy.empty(
-                    (num_chunks*chunk.shape[0], *chunk.shape[1:]),
-                    dtype=chunk.dtype)
-
-            # fill in temporary block according to index
-            chunk_size = chunk.shape[0]
-            block_offset = i*block_size
-
-            temp_lminus1_arr[
-                block_offset:block_offset+chunk_size, :, :] = chunk[:, :, :]
-
-            # copy op only for uneven final chunk
-            if chunk_size != block_size:
-                temp_lminus1_arr = temp_lminus1_arr[
-                    :block_offset+chunk_size, :, :]
-
-            if i == num_chunks - 1:
-                temp_arr = (
-                    downsample_array(
-                        temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
-                        method=downsample_method, **mip_kwargs))
-                end_index = start_index + temp_arr.shape[0]
-                yield MIPArray(lvl, temp_arr, start_index, end_index)
-                start_index += temp_arr.shape[0]
-                i = 0
-            else:
-                i += 1
-        # add any leftovers
-        if i > 0:
-            temp_arr = acpreprocessing.utils.convert.downsample_stack_volume(
-                temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
-                method=downsample_method)
-            end_index = start_index + temp_arr.shape[0]
-            yield MIPArray(lvl, temp_arr, start_index, end_index)
-    else:
-        # get level 0 chunks
-        # block_size is the number of slices to read from tiffs
-        for chunk in iterate_numpy_chunks_from_dataset(
-                dataset, slice_length, pad=False,
-                interleaved_channels=interleaved_channels,
-                channel=channel):
-            # deskew level 0 chunk
-            if deskew_kwargs:
-                chunk = numpy.transpose(psd.deskew_block(
-                    chunk, chunk_index, **deskew_kwargs), (2, 1, 0))
-            end_index = start_index + chunk.shape[0]
-            yield MIPArray(lvl, chunk, start_index, end_index)
-            start_index += chunk.shape[0]
-            chunk_index += 1
-
-
-
-def omezarr_attrs(name, position_xyz, lvl0_xyz_res, max_lvl):
-    # TODO only single channel implemented
-    datasets = []
-    for i in range(max_lvl+1):
-        s = [(2**i)*r for r in lvl0_xyz_res]
-        d = {"coordinateTransformations": [
-            {"scale": [
-                1.0,
-                1.0,
-                s[2],
-                s[1],
-                s[0]
-            ],
-                "type": "scale"
-            }
-        ],
-            "path": str(i)
-        }
-        datasets.append(d)
-    attrs = {}
-    attrs["multiscales"] = [
-        {
-            "axes": [
-                {
-                    "name": "t",
-                    "type": "time",
-                    "unit": "millisecond"
-                },
-                {
-                    "name": "c",
-                    "type": "channel"
-                },
-                {
-                    "name": "z",
-                    "type": "space",
-                    "unit": "micrometer"
-                },
-                {
-                    "name": "y",
-                    "type": "space",
-                    "unit": "micrometer"
-                },
-                {
-                    "name": "x",
-                    "type": "space",
-                    "unit": "micrometer"
-                }
-            ],
-            "datasets": datasets,
-            "name": "/" + name,
-            "translation": [
-                0,
-                0,
-                0.0,
-                0.0,
-                0.0
-            ],
-            "version": "0.4",
-            "coordinateTransformations": [
-                {
-                    "translation": [
-                        0.0,
-                        0.0,
-                        position_xyz[2],
-                        position_xyz[1],
-                        position_xyz[0]
-                    ],
-                    "type": "translation"
-                }
-            ]
-        }
-    ]
-    attrs["omero"] = {
-        "channels": [
-            {
-                "active": True,
-                "coefficient": 1,
-                "color": "000000",
-                "family": "linear",
-                "inverted": False,
-                "label": "Channel:" + name + ":0",
-                "window": {
-                    "end": 1.0,
-                    "max": 1.0,
-                    "min": 0.0,
-                    "start": 0.0
-                }
-            }
-        ],
-        "id": 1,
-        "name": name,
-        "rdefs": {
-            "defaultT": 0,
-            "defaultZ": 288,
-            "model": "color"
-        },
-        "version": "0.4"
-    }
-    return attrs
+from acpreprocessing.stitching_modules.convert_to_n5.tiff_to_ngff import iterate_mip_levels_from_dataset,mip_level_shape,dswrite_chunk,omezarr_attrs,TiffDirToZarrInputParameters
+
+# def iterate_chunks(ds, slice_length, interleaved_channels=1, channel=0, interleaving_offset=0):
+#     """given an iterator, iterate over tuples of a
+#     given length from that iterator
+
+#     Parameters
+#     ----------
+#     it : iterable
+#         iterable object to iterate over in slices of slice_length
+#     slice_length : int
+#         number of items in iterator per chunk
+
+#     Yields
+#     ------
+#     chunk : tuple
+#         tuple of items from iterator of size slice_length. Yields
+#         remainder of iterator when exhausted
+#     """
+#     start = 0
+#     stop = slice_length
+#     chunk = ds[start:stop]
+#     while start<ds.shape[0]:
+#         yield chunk
+#         start = stop
+#         stop = start + slice_length
+#         if stop < ds.shape[0]:
+#             chunk = ds[start:stop]
+#         else:
+#             chunk = ds[start:]
+
+
+# def iter_arrays(r, interleaved_channels=1, channel=0, interleaving_offset=0):
+#     """iterate arrays from an imageio tiff reader.  Allows the last image
+#     of the array to be None, as is the case for data with 'dropped frames'.
+
+#     Parameters
+#     ----------
+#     r : imageio.reader
+#         imageio tiff reader from which to iterate multitiff arrays
+#     interleaved_channels : int
+#         number of channels interleaved in tiff when selecting which
+#         arrays to read
+#     channel : int
+#         channel to iterate over in case of interleaved tiffs
+
+#     Yields
+#     -------
+#     arr : numpy.ndarray
+#         constituent page array of reader r
+#     """
+#     for i, p in enumerate(r._tf.pages):
+#         page_channel = (i + interleaving_offset) % interleaved_channels
+#         if page_channel != channel:
+#             continue
+#         arr = p.asarray()
+#         if arr is not None:
+#             yield arr
+#         elif i == r.get_length() - 1:
+#             continue
+#         else:
+#             raise ValueError
+
+
+# def iterate_2d_arrays_from_mimgfns(mimgfns, interleaved_channels=1, channel=0):
+#     """iterate constituent arrays from an iterator of image filenames
+#     that can be opened as an imageio multi-image.
+
+#     Parameters
+#     ----------
+#     mimgfns : list of str
+#         imageio-compatible name inputs to be opened as multi-images
+
+#     Yields
+#     ------
+#     array : numpy.ndarray
+#         constituent array from ordered iterator of multi-image files
+#     """
+#     offset = 0
+#     for mimgfn in mimgfns:
+#         with imageio.get_reader(mimgfn, mode="I") as r:
+#             yield from iter_arrays(r, interleaved_channels, channel, offset)
+#             offset = (offset + r.get_length()) % interleaved_channels
+
+
+# def iterate_numpy_chunks_from_dataset(
+#         dataset, slice_length=None, pad=True, *args, **kwargs):
+#     """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
+
+#     Parameters
+#     ----------
+#     dataset : hdf5 dataset
+#         imageio-compatible name inputs to be opened as multi-images
+#     slice_length : int
+#         number of 2d arrays from mimgfns included per chunk
+#     pad : bool, optional
+#         whether to extend final returned chunks with zeros
+
+#     Yields
+#     ------
+#     arr : numpy.ndarray
+#         3D numpy array representing a consecutive chunk of 2D arrays
+#     """
+#     #array_gen = iterate_2d_arrays_from_dataset(mimgfns, *args, **kwargs)
+#     for chunk in iterate_chunks(dataset, slice_length,*args,**kwargs):
+#         arr = numpy.asarray(chunk)
+#         if pad:
+#             if arr.shape[0] != slice_length:
+#                 newarr = numpy.zeros((slice_length, *arr.shape[1:]),
+#                                       dtype=arr.dtype)
+#                 newarr[:arr.shape[0], :, :] = arr[:, :, :]
+#                 yield newarr
+#             else:
+#                 yield arr
+#         else:
+#             yield arr
+
+
+# def length_to_interleaved_length(length, interleaved_channels):
+#     """get length of each interleaved channel that contributes to a
+#     total array length
+
+#     Parameters
+#     ----------
+#     length : int
+#         length of interleaved array
+#     interleaved_channels : int
+#         number of constituent channels in interleaved array
+
+#     Returns
+#     -------
+#     channel_lengths : list of int
+#         length of each constituent channel in interleaved array
+#     """
+#     interleaved_length, r = divmod(length, interleaved_channels)
+#     return [
+#         interleaved_length + (1 if ic < r else 0)
+#         for ic in range(interleaved_channels)
+#     ]
+
+
+# def mimg_shape_from_fn(mimg_fn, only_length_tup=False):
+#     """get the shape of an imageio multi-image file without
+#     reading it as a volume
+
+#     Parameters
+#     ----------
+#     mimg_fn : str
+#         imageio-compatible name input to be opened as multi-image
+#     only_length_tup : bool, optional
+#         whether to return only the number of frames or
+#         include the dimensions of a representative frame
+
+#     Returns
+#     -------
+#     shape : tuple of int
+#         shape of array defined by mimg_fn
+#     """
+#     with imageio.get_reader(mimg_fn, mode="I") as r:
+#         l = r.get_length()
+
+#         if only_length_tup:
+#             s = (l,)
+#         else:
+#             s = (l, *r.get_data(0).shape)
+#     return s
+
+
+# def array_downsample_chunks(arr, ds_factors, block_shape, **kwargs):
+#     """downsample a numpy array by inputting subvolumes
+#     to acpreprocessing.utils.convert.downsample_stack_volume
+
+#     Parameters
+#     ----------
+#     arr : numpy.ndarray
+#         array to downsample
+#     ds_factors : tuple of int
+#         integer downsample factor by which arr should be downsamples.
+#         Length must match arr.ndim
+#     block_shape : tuple of int
+#         shape of individual blocks to process
+
+#     Returns
+#     -------
+#     output_arr : numpy.ndarray
+#         downsampled version of arr
+#     """
+#     blocks = skimage.util.view_as_blocks(arr, block_shape)
+
+#     # construct output array and blocks
+#     output_shape = tuple(
+#         int(math.ceil(s / f)) for s, f in zip(arr.shape, ds_factors))
+#     output_block_shape = tuple(
+#         int(math.ceil(s / f)) for s, f in zip(block_shape, ds_factors))
+#     output_arr = numpy.empty(output_shape, dtype=arr.dtype)
+#     output_blocks = skimage.util.view_as_blocks(
+#         output_arr, output_block_shape)
+
+#     # process
+#     for idxs in numpy.ndindex(blocks.shape[:arr.ndim]):
+#         output_blocks[idxs] = (
+#             acpreprocessing.utils.convert.downsample_stack_volume(
+#                 blocks[idxs], ds_factors, **kwargs))
+
+#     return output_arr
+
+
+# def array_downsample_threaded(arr, ds_factors, block_shape,
+#                               n_threads=3, **kwargs):
+#     """downsample a numpy array by concurrently inputting subvolumes
+#     to acpreprocessing.utils.convert.downsample_stack_volume
+
+#     Parameters
+#     ----------
+#     arr : numpy.ndarray
+#         array to downsample
+#     ds_factors : tuple of int
+#         integer downsample factor by which arr should be downsamples.
+#         Length must match arr.ndim
+#     block_shape : tuple of int
+#         shape of individual blocks to process
+#     n_threads :  int
+#         number of threads used to concurrently downsample chunks
+
+#     Returns
+#     -------
+#     output_arr : numpy.ndarray
+#         downsampled version of arr
+#     """
+#     blocks = skimage.util.view_as_blocks(arr, block_shape)
+
+#     # construct output array and blocks
+#     output_shape = tuple(
+#         int(math.ceil(s / f)) for s, f in zip(arr.shape, ds_factors))
+#     output_block_shape = tuple(
+#         int(math.ceil(s / f)) for s, f in zip(block_shape, ds_factors))
+#     output_arr = numpy.empty(output_shape, dtype=arr.dtype)
+#     output_blocks = skimage.util.view_as_blocks(
+#         output_arr, output_block_shape)
+
+#     # process
+#     with concurrent.futures.ThreadPoolExecutor(max_workers=n_threads) as e:
+#         fut_to_idxs = {}
+#         for idxs in numpy.ndindex(blocks.shape[:arr.ndim]):
+#             fut_to_idxs[e.submit(
+#                 acpreprocessing.utils.convert.downsample_stack_volume,
+#                 blocks[idxs], ds_factors, **kwargs)] = idxs
+
+#         for fut in concurrent.futures.as_completed(fut_to_idxs.keys()):
+#             idxs = fut_to_idxs[fut]
+#             output_blocks[idxs] = fut.result()
+#     return output_arr
+
+
+# class ArrayChunkingError(ValueError):
+#     """ValueError caused by incorrect array chunking"""
+
+
+# def downsample_array(arr, *args, n_threads=None, block_shape=None,
+#                       block_divs=None, allow_failover=True, **kwargs):
+#     """downsample array with optional subvolume chunking and threading
+
+#     Parameters
+#     ----------
+#     arr : numpy.ndarray
+#         array to downsample
+#     ds_factors : tuple of int
+#         integer downsample factor by which arr should be downsamples.
+#         Length must match arr.ndim
+#     block_shape : tuple of int
+#         shape of individual blocks to process.
+#         Supersedes block_divs if both are provided
+#     block_divs : tuple of int
+#         number of divisions for array chunking.
+#     n_threads :  int
+#         number of threads used to concurrently downsample chunks
+#     allow_failover : bool, optional
+#         whether to default to using
+#         acpreprocessing.utils.convert.downsample_stack_volume when
+#         block_shape or block_divs is not actionable.  (default: True)
+
+#     Returns
+#     -------
+#     output_arr : numpy.ndarray
+#         downsampled version of arr
+#     """
+#     try:
+#         if block_shape is not None or block_divs is not None:
+#             if block_shape is None:
+#                 block_shape = [a_s / d for a_s, d in zip(
+#                     arr.shape, block_divs)]
+#                 if not all([s.is_integer() for s in block_shape]):
+#                     raise ArrayChunkingError(
+#                         "cannot divide array of shape {} by {}".format(
+#                             arr.shape, block_divs))
+#                 block_shape = tuple(int(s) for s in block_shape)
+#             if n_threads is not None:
+#                 return array_downsample_threaded(
+#                     arr, *args, n_threads=n_threads,
+#                     block_shape=block_shape, **kwargs)
+#             return array_downsample_chunks(
+#                 arr, *args, block_shape=block_shape, **kwargs)
+#     except ArrayChunkingError:
+#         if not allow_failover:
+#             raise
+#     return acpreprocessing.utils.convert.downsample_stack_volume(
+#         arr, *args, **kwargs)
+
+
+# def mip_level_shape(lvl, lvl_0_shape):
+#     """get the expected shape of a MIP level downsample for a given
+#     full resolution shape.  Expects 2x downsampling on each dimension.
+
+#     Parameters
+#     ----------
+#     lvl : int
+#         mip level for which shape is calculated
+#     lvl_0_shape : tuple of int
+#         shape tuple of level 0
+
+#     Returns
+#     -------
+#     lvl_shape : tuple of int
+#         shape tuple for lvl
+#     """
+#     return tuple([*map(lambda x: math.ceil(x / (2**lvl)), lvl_0_shape)])
+
+
+# def dswrite_chunk(ds, start, end, arr, silent_overflow=True):
+#     """write array arr into array-like n5 dataset defined by
+#     start and end points on 0 axis.
+
+#     Parameters
+#     ----------
+#     ds : z5py.dataset.Dataset (n5 3D) or zarr.dataset (zarr 5D)
+#         array-like dataset to fill in
+#     start : int
+#         start index along 0 axis of ds to fill in
+#     end : int
+#         end index along 0 axis of ds to fill in
+#     arr : numpy.ndarray
+#         array from which ds values will be taken
+#     silent_overflow : bool, optional
+#         whether to shrink the end index to match the
+#         shape of ds (default: True)
+#     """
+#     if len(ds.shape) == 5:  # dataset dimensions should be 3 or 5
+#         if end >= ds.shape[2] and silent_overflow:
+#             end = ds.shape[2]
+#         if end > start:
+#             ds[0, 0, start:end, :, :] = arr[:(end - start), :, :]
+#     elif len(ds.shape) == 3:
+#         if end >= ds.shape[0] and silent_overflow:
+#             end = ds.shape[0]
+#         if end > start:
+#             ds[start:end, :, :] = arr[:(end - start), :, :]
+
+
+# @dataclasses.dataclass
+# class MIPArray:
+#     lvl: int
+#     array: numpy.ndarray
+#     start: int
+#     end: int
+
+
+# def iterate_mip_levels_from_dataset(
+#         dataset, lvl, block_size, slice_length, downsample_factor,
+#         downsample_method=None, lvl_to_mip_kwargs=None,
+#         interleaved_channels=1, channel=0, deskew_kwargs={}):
+#     """recursively generate MIPmap levels from an iterator of multi-image files
+
+#     Parameters
+#     ----------
+#     dataset : hdf5 dataset
+#         imageio-compatible name inputs to be opened as multi-images
+#     lvl : int
+#         integer mip level to generate
+#     block_size : int
+#         number of 2D arrays in chunk to process for a chunk at lvl
+#     slice_length : int
+#         number of 2D arrays to gather from tiff stacks
+#     downsample_factor : tuple of int
+#         integer downsampling factor for MIP levels
+#     downsample_method : str, optional
+#         downsampling method for
+#         acpreprocessing.utils.convert.downsample_stack_volume
+#     lvl_to_mip_kwargs :  dict, optional
+#         mapping of MIP level to kwargs used in MIPmap generation
+#     interleaved_channels : int
+#         number of channels interleaved in the tiff files (default 1)
+#     channel : int, optional
+#         channel from which interleaved data should be read (default 0)
+#     deskew_kwargs : dict, optional
+#         parameters for pixel shifting deskew
+
+#     Yields
+#     ------
+#     ma : acpreprocessing.stitching_modules.convert_to_n5.tiff_to_n5.MipArray
+#         object describing chunked array, MIP level of origin, and chunk indices
+#     """
+#     lvl_to_mip_kwargs = ({} if lvl_to_mip_kwargs is None
+#                           else lvl_to_mip_kwargs)
+#     mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
+#     start_index = 0
+#     chunk_index = 0
+#     if lvl > 0:
+#         num_chunks = downsample_factor[0]
+
+#         i = 0
+#         for ma in iterate_mip_levels_from_dataset(
+#                 dataset, lvl-1, block_size, slice_length,
+#                 downsample_factor, downsample_method,
+#                 lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
+#                 channel=channel, deskew_kwargs=deskew_kwargs):
+#             chunk = ma.array
+#             # throw array up for further processing
+#             yield ma
+#             # ignore if not parent resolution
+#             if ma.lvl != lvl-1:
+#                 continue
+
+#             try:
+#                 temp_lminus1_arr
+#             except NameError:
+#                 temp_lminus1_arr = numpy.empty(
+#                     (num_chunks*chunk.shape[0], *chunk.shape[1:]),
+#                     dtype=chunk.dtype)
+
+#             # fill in temporary block according to index
+#             chunk_size = chunk.shape[0]
+#             block_offset = i*block_size
+
+#             temp_lminus1_arr[
+#                 block_offset:block_offset+chunk_size, :, :] = chunk[:, :, :]
+
+#             # copy op only for uneven final chunk
+#             if chunk_size != block_size:
+#                 temp_lminus1_arr = temp_lminus1_arr[
+#                     :block_offset+chunk_size, :, :]
+
+#             if i == num_chunks - 1:
+#                 temp_arr = (
+#                     downsample_array(
+#                         temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
+#                         method=downsample_method, **mip_kwargs))
+#                 end_index = start_index + temp_arr.shape[0]
+#                 yield MIPArray(lvl, temp_arr, start_index, end_index)
+#                 start_index += temp_arr.shape[0]
+#                 i = 0
+#             else:
+#                 i += 1
+#         # add any leftovers
+#         if i > 0:
+#             temp_arr = acpreprocessing.utils.convert.downsample_stack_volume(
+#                 temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
+#                 method=downsample_method)
+#             end_index = start_index + temp_arr.shape[0]
+#             yield MIPArray(lvl, temp_arr, start_index, end_index)
+#     else:
+#         # get level 0 chunks
+#         # block_size is the number of slices to read from tiffs
+#         for chunk in iterate_numpy_chunks_from_dataset(
+#                 dataset, slice_length, pad=False,
+#                 interleaved_channels=interleaved_channels,
+#                 channel=channel):
+#             # deskew level 0 chunk
+#             if deskew_kwargs:
+#                 chunk = numpy.transpose(psd.deskew_block(
+#                     chunk, chunk_index, **deskew_kwargs), (2, 1, 0))
+#             end_index = start_index + chunk.shape[0]
+#             yield MIPArray(lvl, chunk, start_index, end_index)
+#             start_index += chunk.shape[0]
+#             chunk_index += 1
+
+
+
+# def omezarr_attrs(name, position_xyz, lvl0_xyz_res, max_lvl):
+#     # TODO only single channel implemented
+#     datasets = []
+#     for i in range(max_lvl+1):
+#         s = [(2**i)*r for r in lvl0_xyz_res]
+#         d = {"coordinateTransformations": [
+#             {"scale": [
+#                 1.0,
+#                 1.0,
+#                 s[2],
+#                 s[1],
+#                 s[0]
+#             ],
+#                 "type": "scale"
+#             }
+#         ],
+#             "path": str(i)
+#         }
+#         datasets.append(d)
+#     attrs = {}
+#     attrs["multiscales"] = [
+#         {
+#             "axes": [
+#                 {
+#                     "name": "t",
+#                     "type": "time",
+#                     "unit": "millisecond"
+#                 },
+#                 {
+#                     "name": "c",
+#                     "type": "channel"
+#                 },
+#                 {
+#                     "name": "z",
+#                     "type": "space",
+#                     "unit": "micrometer"
+#                 },
+#                 {
+#                     "name": "y",
+#                     "type": "space",
+#                     "unit": "micrometer"
+#                 },
+#                 {
+#                     "name": "x",
+#                     "type": "space",
+#                     "unit": "micrometer"
+#                 }
+#             ],
+#             "datasets": datasets,
+#             "name": "/" + name,
+#             "translation": [
+#                 0,
+#                 0,
+#                 0.0,
+#                 0.0,
+#                 0.0
+#             ],
+#             "version": "0.4",
+#             "coordinateTransformations": [
+#                 {
+#                     "translation": [
+#                         0.0,
+#                         0.0,
+#                         position_xyz[2],
+#                         position_xyz[1],
+#                         position_xyz[0]
+#                     ],
+#                     "type": "translation"
+#                 }
+#             ]
+#         }
+#     ]
+#     attrs["omero"] = {
+#         "channels": [
+#             {
+#                 "active": True,
+#                 "coefficient": 1,
+#                 "color": "000000",
+#                 "family": "linear",
+#                 "inverted": False,
+#                 "label": "Channel:" + name + ":0",
+#                 "window": {
+#                     "end": 1.0,
+#                     "max": 1.0,
+#                     "min": 0.0,
+#                     "start": 0.0
+#                 }
+#             }
+#         ],
+#         "id": 1,
+#         "name": name,
+#         "rdefs": {
+#             "defaultT": 0,
+#             "defaultZ": 288,
+#             "model": "color"
+#         },
+#         "version": "0.4"
+#     }
+#     return attrs
 
 
 class TiffToNGFFException(Exception):
@@ -655,6 +655,8 @@ def write_ims_to_zarr(
     dataset = f['DataSet']['ResolutionLevel 0']['TimePoint 0']['Channel 0']['Data']
     
     joined_shapes = dataset.shape
+    print("ims_to_ngff dataset shape:" + str(joined_shapes))
+    return
 
     if deskew_options and deskew_options["deskew_method"] == "ps":
         block_size = chunk_size[2]
@@ -750,67 +752,67 @@ def ims_to_ngff_group(ims_fn, output, *args, **kwargs):
         print('unknown output format: ' + output)
 
 
-class DownsampleOptions(argschema.schemas.DefaultSchema):
-    block_divs = argschema.fields.List(
-        argschema.fields.Int, required=False, allow_none=True)
-    n_threads = argschema.fields.Int(required=False, allow_none=True)
+# class DownsampleOptions(argschema.schemas.DefaultSchema):
+#     block_divs = argschema.fields.List(
+#         argschema.fields.Int, required=False, allow_none=True)
+#     n_threads = argschema.fields.Int(required=False, allow_none=True)
 
 
-class DeskewOptions(argschema.schemas.DefaultSchema):
-    deskew_method = argschema.fields.Str(required=False, default='')
-    deskew_stride = argschema.fields.Int(required=False, default=None)
-    deskew_flip = argschema.fields.Bool(required=False, default=True)
-    deskew_crop = argschema.fields.Float(required=False, default=1.0)
+# class DeskewOptions(argschema.schemas.DefaultSchema):
+#     deskew_method = argschema.fields.Str(required=False, default='')
+#     deskew_stride = argschema.fields.Int(required=False, default=None)
+#     deskew_flip = argschema.fields.Bool(required=False, default=True)
+#     deskew_crop = argschema.fields.Float(required=False, default=1.0)
 
 
-class NGFFGenerationParameters(argschema.schemas.DefaultSchema):
-    output_format = argschema.fields.Str(required=True)
-    output_file = argschema.fields.Str(required=True)
-    max_mip = argschema.fields.Int(required=False, default=0)
-    concurrency = argschema.fields.Int(required=False, default=10)
-    compression = argschema.fields.Str(required=False, default="raw")
-    lvl_to_mip_kwargs = argschema.fields.Dict(
-        keys=argschema.fields.Int(),
-        values=argschema.fields.Nested(DownsampleOptions))
+# class NGFFGenerationParameters(argschema.schemas.DefaultSchema):
+#     output_format = argschema.fields.Str(required=True)
+#     output_file = argschema.fields.Str(required=True)
+#     max_mip = argschema.fields.Int(required=False, default=0)
+#     concurrency = argschema.fields.Int(required=False, default=10)
+#     compression = argschema.fields.Str(required=False, default="raw")
+#     lvl_to_mip_kwargs = argschema.fields.Dict(
+#         keys=argschema.fields.Int(),
+#         values=argschema.fields.Nested(DownsampleOptions))
 
-    # FIXME argschema supports lists and tuples,
-    #   but has some version differences
+#     # FIXME argschema supports lists and tuples,
+#     #   but has some version differences
 
-    mip_dsfactor = argschema.fields.Tuple((
-        argschema.fields.Int(),
-        argschema.fields.Int(),
-        argschema.fields.Int()), required=False, default=(2, 2, 2))
-    deskew_options = argschema.fields.Nested(
-        DeskewOptions, required=False)
+#     mip_dsfactor = argschema.fields.Tuple((
+#         argschema.fields.Int(),
+#         argschema.fields.Int(),
+#         argschema.fields.Int()), required=False, default=(2, 2, 2))
+#     deskew_options = argschema.fields.Nested(
+#         DeskewOptions, required=False)
 
 
-class NGFFGroupGenerationParameters(NGFFGenerationParameters):
-    group_names = argschema.fields.List(
-        argschema.fields.Str, required=True)
-    group_attributes = argschema.fields.List(
-        argschema.fields.Dict(required=False, default={}), default=[],
-        required=False)
+# class NGFFGroupGenerationParameters(NGFFGenerationParameters):
+#     group_names = argschema.fields.List(
+#         argschema.fields.Str, required=True)
+#     group_attributes = argschema.fields.List(
+#         argschema.fields.Dict(required=False, default={}), default=[],
+#         required=False)
 
 
-class IMSToNGFFParameters(NGFFGroupGenerationParameters):
-    input_file = argschema.fields.Str(required=True)
-    interleaved_channels = argschema.fields.Int(required=False, default=1)
-    channel = argschema.fields.Int(required=False, default=0)
+# class IMSToNGFFParameters(NGFFGroupGenerationParameters):
+#     input_file = argschema.fields.Str(required=True)
+#     interleaved_channels = argschema.fields.Int(required=False, default=1)
+#     channel = argschema.fields.Int(required=False, default=0)
 
 
-class IMSToZarrInputParameters(argschema.ArgSchema,
-                                   IMSToNGFFParameters):
-    chunk_size = argschema.fields.Tuple((
-        argschema.fields.Int(),
-        argschema.fields.Int(),
-        argschema.fields.Int(),
-        argschema.fields.Int(),
-        argschema.fields.Int()), required=False, default=(1, 1, 64, 64, 64))
+# class IMSToZarrInputParameters(argschema.ArgSchema,
+#                                    IMSToNGFFParameters):
+#     chunk_size = argschema.fields.Tuple((
+#         argschema.fields.Int(),
+#         argschema.fields.Int(),
+#         argschema.fields.Int(),
+#         argschema.fields.Int(),
+#         argschema.fields.Int()), required=False, default=(1, 1, 64, 64, 64))
 
 
 
 class IMSToZarr(argschema.ArgSchemaParser):
-    default_schema = IMSToZarrInputParameters
+    default_schema = TiffDirToZarrInputParameters #IMSToZarrInputParameters
 
     def run(self):
         deskew_options = (self.args["deskew_options"]
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py
index a14053e..05c59d3 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py
@@ -19,7 +19,6 @@
 import acpreprocessing.utils.convert
 import acpreprocessing.stitching_modules.convert_to_n5.psdeskew as psd
 
-
 def iterate_chunks(it, slice_length):
     """given an iterator, iterate over tuples of a
     given length from that iterator
@@ -97,14 +96,47 @@ def iterate_2d_arrays_from_mimgfns(mimgfns, interleaved_channels=1, channel=0):
             offset = (offset + r.get_length()) % interleaved_channels
 
 
-def iterate_numpy_chunks_from_mimgfns(
-        mimgfns, slice_length=None, pad=True, *args, **kwargs):
-    """iterate over a contiguous iterator of imageio multi-image files as
-    chunks of numpy arrays
+# def iterate_numpy_chunks_from_mimgfns(
+#         mimgfns, slice_length=None, pad=True, *args, **kwargs):
+#     """iterate over a contiguous iterator of imageio multi-image files as
+#     chunks of numpy arrays
+
+#     Parameters
+#     ----------
+#     mimgfns : list of str
+#         imageio-compatible name inputs to be opened as multi-images
+#     slice_length : int
+#         number of 2d arrays from mimgfns included per chunk
+#     pad : bool, optional
+#         whether to extend final returned chunks with zeros
+
+#     Yields
+#     ------
+#     arr : numpy.ndarray
+#         3D numpy array representing a consecutive chunk of 2D arrays
+#     """
+#     array_gen = iterate_2d_arrays_from_mimgfns(mimgfns, *args, **kwargs)
+#     for chunk in iterate_chunks(array_gen, slice_length):
+#         arr = numpy.array(chunk)
+#         if pad:
+#             if arr.shape[0] != slice_length:
+#                 newarr = numpy.zeros((slice_length, *arr.shape[1:]),
+#                                      dtype=arr.dtype)
+#                 newarr[:arr.shape[0], :, :] = arr[:, :, :]
+#                 yield newarr
+#             else:
+#                 yield arr
+#         else:
+#             yield arr
+
+
+def iterate_numpy_chunks_from_dataset(
+        dataset, slice_length=None, pad=True, *args, **kwargs):
+    """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
 
     Parameters
     ----------
-    mimgfns : list of str
+    dataset : hdf5 dataset
         imageio-compatible name inputs to be opened as multi-images
     slice_length : int
         number of 2d arrays from mimgfns included per chunk
@@ -116,13 +148,13 @@ def iterate_numpy_chunks_from_mimgfns(
     arr : numpy.ndarray
         3D numpy array representing a consecutive chunk of 2D arrays
     """
-    array_gen = iterate_2d_arrays_from_mimgfns(mimgfns, *args, **kwargs)
-    for chunk in iterate_chunks(array_gen, slice_length):
-        arr = numpy.array(chunk)
+    #array_gen = iterate_2d_arrays_from_dataset(mimgfns, *args, **kwargs)
+    for chunk in iterate_chunks(dataset, slice_length,*args,**kwargs):
+        arr = numpy.asarray(chunk)
         if pad:
             if arr.shape[0] != slice_length:
                 newarr = numpy.zeros((slice_length, *arr.shape[1:]),
-                                     dtype=arr.dtype)
+                                      dtype=arr.dtype)
                 newarr[:arr.shape[0], :, :] = arr[:, :, :]
                 yield newarr
             else:
@@ -415,15 +447,125 @@ class MIPArray:
     end: int
 
 
-def iterate_mip_levels_from_mimgfns(
-        mimgfns, lvl, block_size, slice_length, downsample_factor,
+# def iterate_mip_levels_from_mimgfns(
+#         mimgfns, lvl, block_size, slice_length, downsample_factor,
+#         downsample_method=None, lvl_to_mip_kwargs=None,
+#         interleaved_channels=1, channel=0, deskew_kwargs={}):
+#     """recursively generate MIPmap levels from an iterator of multi-image files
+
+#     Parameters
+#     ----------
+#     mimgfns : list of str
+#         imageio-compatible name inputs to be opened as multi-images
+#     lvl : int
+#         integer mip level to generate
+#     block_size : int
+#         number of 2D arrays in chunk to process for a chunk at lvl
+#     slice_length : int
+#         number of 2D arrays to gather from tiff stacks
+#     downsample_factor : tuple of int
+#         integer downsampling factor for MIP levels
+#     downsample_method : str, optional
+#         downsampling method for
+#         acpreprocessing.utils.convert.downsample_stack_volume
+#     lvl_to_mip_kwargs :  dict, optional
+#         mapping of MIP level to kwargs used in MIPmap generation
+#     interleaved_channels : int
+#         number of channels interleaved in the tiff files (default 1)
+#     channel : int, optional
+#         channel from which interleaved data should be read (default 0)
+#     deskew_kwargs : dict, optional
+#         parameters for pixel shifting deskew
+
+#     Yields
+#     ------
+#     ma : acpreprocessing.stitching_modules.convert_to_n5.tiff_to_n5.MipArray
+#         object describing chunked array, MIP level of origin, and chunk indices
+#     """
+#     lvl_to_mip_kwargs = ({} if lvl_to_mip_kwargs is None
+#                          else lvl_to_mip_kwargs)
+#     mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
+#     start_index = 0
+#     chunk_index = 0
+#     if lvl > 0:
+#         num_chunks = downsample_factor[0]
+
+#         i = 0
+#         for ma in iterate_mip_levels_from_mimgfns(
+#                 mimgfns, lvl-1, block_size, slice_length,
+#                 downsample_factor, downsample_method,
+#                 lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
+#                 channel=channel, deskew_kwargs=deskew_kwargs):
+#             chunk = ma.array
+#             # throw array up for further processing
+#             yield ma
+#             # ignore if not parent resolution
+#             if ma.lvl != lvl-1:
+#                 continue
+
+#             try:
+#                 temp_lminus1_arr
+#             except NameError:
+#                 temp_lminus1_arr = numpy.empty(
+#                     (num_chunks*chunk.shape[0], *chunk.shape[1:]),
+#                     dtype=chunk.dtype)
+
+#             # fill in temporary block according to index
+#             chunk_size = chunk.shape[0]
+#             block_offset = i*block_size
+
+#             temp_lminus1_arr[
+#                 block_offset:block_offset+chunk_size, :, :] = chunk[:, :, :]
+
+#             # copy op only for uneven final chunk
+#             if chunk_size != block_size:
+#                 temp_lminus1_arr = temp_lminus1_arr[
+#                     :block_offset+chunk_size, :, :]
+
+#             if i == num_chunks - 1:
+#                 temp_arr = (
+#                     downsample_array(
+#                         temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
+#                         method=downsample_method, **mip_kwargs))
+#                 end_index = start_index + temp_arr.shape[0]
+#                 yield MIPArray(lvl, temp_arr, start_index, end_index)
+#                 start_index += temp_arr.shape[0]
+#                 i = 0
+#             else:
+#                 i += 1
+#         # add any leftovers
+#         if i > 0:
+#             temp_arr = acpreprocessing.utils.convert.downsample_stack_volume(
+#                 temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
+#                 method=downsample_method)
+#             end_index = start_index + temp_arr.shape[0]
+#             yield MIPArray(lvl, temp_arr, start_index, end_index)
+#     else:
+#         # get level 0 chunks
+#         # block_size is the number of slices to read from tiffs
+#         for chunk in iterate_numpy_chunks_from_mimgfns(
+#                 mimgfns, slice_length, pad=False,
+#                 interleaved_channels=interleaved_channels,
+#                 channel=channel):
+#             # deskew level 0 chunk
+#             if deskew_kwargs:
+#                 chunk = numpy.transpose(psd.deskew_block(
+#                     chunk, chunk_index, **deskew_kwargs), (2, 1, 0))
+#             end_index = start_index + chunk.shape[0]
+#             yield MIPArray(lvl, chunk, start_index, end_index)
+#             start_index += chunk.shape[0]
+#             chunk_index += 1
+
+
+def iterate_mip_levels_from_dataset(
+        dataset, lvl, block_size, slice_length, downsample_factor,
         downsample_method=None, lvl_to_mip_kwargs=None,
         interleaved_channels=1, channel=0, deskew_kwargs={}):
     """recursively generate MIPmap levels from an iterator of multi-image files
 
     Parameters
     ----------
-    mimgfns : list of str
+    dataset : hdf5 dataset
         imageio-compatible name inputs to be opened as multi-images
     lvl : int
         integer mip level to generate
@@ -451,7 +593,7 @@ def iterate_mip_levels_from_mimgfns(
         object describing chunked array, MIP level of origin, and chunk indices
     """
     lvl_to_mip_kwargs = ({} if lvl_to_mip_kwargs is None
-                         else lvl_to_mip_kwargs)
+                          else lvl_to_mip_kwargs)
     mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
     start_index = 0
     chunk_index = 0
@@ -459,8 +601,8 @@ def iterate_mip_levels_from_mimgfns(
         num_chunks = downsample_factor[0]
 
         i = 0
-        for ma in iterate_mip_levels_from_mimgfns(
-                mimgfns, lvl-1, block_size, slice_length,
+        for ma in iterate_mip_levels_from_dataset(
+                dataset, lvl-1, block_size, slice_length,
                 downsample_factor, downsample_method,
                 lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
                 channel=channel, deskew_kwargs=deskew_kwargs):
@@ -511,8 +653,8 @@ def iterate_mip_levels_from_mimgfns(
     else:
         # get level 0 chunks
         # block_size is the number of slices to read from tiffs
-        for chunk in iterate_numpy_chunks_from_mimgfns(
-                mimgfns, slice_length, pad=False,
+        for chunk in iterate_numpy_chunks_from_dataset(
+                dataset, slice_length, pad=False,
                 interleaved_channels=interleaved_channels,
                 channel=channel):
             # deskew level 0 chunk
@@ -570,7 +712,8 @@ def write_mimgfns_to_n5(
     """
     group_attributes = ([] if group_attributes is None else group_attributes)
     deskew_options = ({} if deskew_options is None else deskew_options)
-
+    
+    array_gen = iterate_2d_arrays_from_mimgfns(mimgfns, interleaved_channels=interleaved_channels,channel=channel)
     joined_shapes = joined_mimg_shape_from_fns(
         mimgfns, concurrency=concurrency,
         interleaved_channels=interleaved_channels, channel=channel)
@@ -632,8 +775,8 @@ def write_mimgfns_to_n5(
 
         with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as e:
             futs = []
-            for miparr in iterate_mip_levels_from_mimgfns(
-                    mimgfns, max_mip, block_size, slice_length, mip_dsfactor,
+            for miparr in iterate_mip_levels_from_dataset(
+                    array_gen, max_mip, block_size, slice_length, mip_dsfactor,
                     lvl_to_mip_kwargs=lvl_to_mip_kwargs,
                     interleaved_channels=interleaved_channels,
                     channel=channel, deskew_kwargs=deskew_kwargs):
@@ -798,7 +941,8 @@ def write_mimgfns_to_zarr(
     """
     group_attributes = ([] if group_attributes is None else group_attributes)
     deskew_options = ({} if deskew_options is None else deskew_options)
-
+    
+    array_gen = iterate_2d_arrays_from_mimgfns(mimgfns, interleaved_channels=interleaved_channels,channel=channel)
     joined_shapes = joined_mimg_shape_from_fns(
         mimgfns, concurrency=concurrency,
         interleaved_channels=interleaved_channels, channel=channel)
@@ -863,8 +1007,8 @@ def write_mimgfns_to_zarr(
 
         with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as e:
             futs = []
-            for miparr in iterate_mip_levels_from_mimgfns(
-                    mimgfns, max_mip, block_size, slice_length, mip_dsfactor,
+            for miparr in iterate_mip_levels_from_dataset(
+                    array_gen, max_mip, block_size, slice_length, mip_dsfactor,
                     lvl_to_mip_kwargs=lvl_to_mip_kwargs,
                     interleaved_channels=interleaved_channels,
                     channel=channel, deskew_kwargs=deskew_kwargs):
@@ -905,6 +1049,7 @@ class DeskewOptions(argschema.schemas.DefaultSchema):
     deskew_method = argschema.fields.Str(required=False, default='')
     deskew_stride = argschema.fields.Int(required=False, default=None)
     deskew_flip = argschema.fields.Bool(required=False, default=True)
+    deskew_transpose = argschema.fields.Bool(required=False, default=False)
     deskew_crop = argschema.fields.Float(required=False, default=1.0)
 
 

From 225b4cf1c21b3e4d4f8eaa9150f65463a270bdc1 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 9 Jul 2024 16:10:19 -0700
Subject: [PATCH 016/115] input parameter fix

---
 .../convert_to_n5/ims_to_ngff.py              | 28 +++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index ee1447a..0aff4a9 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -19,7 +19,7 @@
 
 import acpreprocessing.utils.convert
 import acpreprocessing.stitching_modules.convert_to_n5.psdeskew as psd
-from acpreprocessing.stitching_modules.convert_to_n5.tiff_to_ngff import iterate_mip_levels_from_dataset,mip_level_shape,dswrite_chunk,omezarr_attrs,TiffDirToZarrInputParameters
+from acpreprocessing.stitching_modules.convert_to_n5.tiff_to_ngff import iterate_mip_levels_from_dataset,mip_level_shape,dswrite_chunk,omezarr_attrs,NGFFGroupGenerationParameters
 
 # def iterate_chunks(ds, slice_length, interleaved_channels=1, channel=0, interleaving_offset=0):
 #     """given an iterator, iterate over tuples of a
@@ -794,25 +794,25 @@ def ims_to_ngff_group(ims_fn, output, *args, **kwargs):
 #         required=False)
 
 
-# class IMSToNGFFParameters(NGFFGroupGenerationParameters):
-#     input_file = argschema.fields.Str(required=True)
-#     interleaved_channels = argschema.fields.Int(required=False, default=1)
-#     channel = argschema.fields.Int(required=False, default=0)
+class IMSToNGFFParameters(NGFFGroupGenerationParameters):
+    input_file = argschema.fields.Str(required=True)
+    interleaved_channels = argschema.fields.Int(required=False, default=1)
+    channel = argschema.fields.Int(required=False, default=0)
 
 
-# class IMSToZarrInputParameters(argschema.ArgSchema,
-#                                    IMSToNGFFParameters):
-#     chunk_size = argschema.fields.Tuple((
-#         argschema.fields.Int(),
-#         argschema.fields.Int(),
-#         argschema.fields.Int(),
-#         argschema.fields.Int(),
-#         argschema.fields.Int()), required=False, default=(1, 1, 64, 64, 64))
+class IMSToZarrInputParameters(argschema.ArgSchema,
+                                    IMSToNGFFParameters):
+    chunk_size = argschema.fields.Tuple((
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int()), required=False, default=(1, 1, 64, 64, 64))
 
 
 
 class IMSToZarr(argschema.ArgSchemaParser):
-    default_schema = TiffDirToZarrInputParameters #IMSToZarrInputParameters
+    default_schema = IMSToZarrInputParameters
 
     def run(self):
         deskew_options = (self.args["deskew_options"]

From 0d7b75f1e5840fd58da6706155caa89abb744f16 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 9 Jul 2024 16:33:38 -0700
Subject: [PATCH 017/115] adding transpose for exaspim deskew

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py    |  2 +-
 .../stitching_modules/convert_to_n5/psdeskew.py       | 11 +++++++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 0aff4a9..7693bba 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -656,7 +656,6 @@ def write_ims_to_zarr(
     
     joined_shapes = dataset.shape
     print("ims_to_ngff dataset shape:" + str(joined_shapes))
-    return
 
     if deskew_options and deskew_options["deskew_method"] == "ps":
         block_size = chunk_size[2]
@@ -666,6 +665,7 @@ def write_ims_to_zarr(
                                             )
         joined_shapes = psd.reshape_joined_shapes(
             joined_shapes, deskew_options['deskew_stride'], **deskew_kwargs)
+        print("deskewed shape:" + str(joined_shapes))
     else:
         block_size = chunk_size[2]
         slice_length = block_size
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 7e4cdf0..928f84d 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -11,7 +11,7 @@
 import numpy as np
 
 
-def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_crop=1, dtype='uint16', **kwargs):
+def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_transpose=False, deskew_crop=1, dtype='uint16', **kwargs):
     """get keyword arguments for deskew_block
 
     Parameters
@@ -22,6 +22,8 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_cr
         number of camera pixels per deskewed sampling plane (divides z resolution)
     deskewFlip : bool
         flip data blocks before deskewing
+    deskewTranspose : bool
+        transpose x,y axes before deskewing
     dtype : str
         datatype for deskew output
     crop_factor : float
@@ -32,6 +34,8 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_cr
     kwargs : dict
         parameters representing pixel deskew operation for deskew_block
     """
+    if deskew_transpose:
+        skew_dims_zyx = (skew_dims_zyx[0],skew_dims_zyx[2],skew_dims_zyx[1])
     sdims = skew_dims_zyx
     crop_factor = deskew_crop
     stride = deskew_stride
@@ -68,13 +72,14 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_cr
               'blockdims': blockdims,
               'subblocks': subblocks,
               'flip': deskew_flip,
+              'transpose': deskew_transpose,
               'dtype': dtype,
               'chunklength': blockx
               }
     return kwargs
 
 
-def deskew_block(blockData, n, dsi, si, slice1d, blockdims, subblocks, flip, dtype, chunklength, *args, **kwargs):
+def deskew_block(blockData, n, dsi, si, slice1d, blockdims, subblocks, flip, transpose, dtype, chunklength, *args, **kwargs):
     """deskew a data chunk in sequence with prior chunks
 
     Parameters
@@ -105,6 +110,8 @@ def deskew_block(blockData, n, dsi, si, slice1d, blockdims, subblocks, flip, dty
     block3d : numpy.ndarray
         pixel shifted deskewed data ordered (z,y,x) by sample axes 
     """
+    if transpose:
+        blockData = blockData.transpose((0,2,1))
     subb = subblocks
     block3d = np.zeros(blockdims, dtype=dtype)
     zdim = block3d.shape[0]

From 206cceb7ba8b50c5303be59093eb1d86687b6dde Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 9 Jul 2024 16:55:31 -0700
Subject: [PATCH 018/115] fixing

---
 acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py    | 3 ++-
 .../stitching_modules/convert_to_n5/tiff_to_ngff.py            | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 928f84d..e656a18 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -112,7 +112,8 @@ def deskew_block(blockData, n, dsi, si, slice1d, blockdims, subblocks, flip, tra
     """
     if transpose:
         blockData = blockData.transpose((0,2,1))
-    subb = subblocks
+    # subb = subblocks
+    subb = 5
     block3d = np.zeros(blockdims, dtype=dtype)
     zdim = block3d.shape[0]
     ydim = block3d.shape[1]
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py
index 05c59d3..69b81fb 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py
@@ -149,7 +149,7 @@ def iterate_numpy_chunks_from_dataset(
         3D numpy array representing a consecutive chunk of 2D arrays
     """
     #array_gen = iterate_2d_arrays_from_dataset(mimgfns, *args, **kwargs)
-    for chunk in iterate_chunks(dataset, slice_length,*args,**kwargs):
+    for chunk in iterate_chunks(dataset, slice_length):#,*args,**kwargs):
         arr = numpy.asarray(chunk)
         if pad:
             if arr.shape[0] != slice_length:

From 9afaca70ca68436f84d5bd909e161324d151c4bb Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 16 Jul 2024 12:12:47 -0700
Subject: [PATCH 019/115] redundant methods

---
 .../convert_to_n5/ims_to_ngff.py              |  12 +-
 .../convert_to_n5/tiff_to_visor.py            | 571 +-----------------
 2 files changed, 10 insertions(+), 573 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 7693bba..a155975 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -17,9 +17,9 @@
 from numcodecs import Blosc
 import argschema
 
-import acpreprocessing.utils.convert
+# import acpreprocessing.utils.convert
 import acpreprocessing.stitching_modules.convert_to_n5.psdeskew as psd
-from acpreprocessing.stitching_modules.convert_to_n5.tiff_to_ngff import iterate_mip_levels_from_dataset,mip_level_shape,dswrite_chunk,omezarr_attrs,NGFFGroupGenerationParameters
+from acpreprocessing.stitching_modules.convert_to_n5.tiff_to_ngff import iterate_mip_levels_from_dataset,mip_level_shape,dswrite_chunk,omezarr_attrs,NGFFGroupGenerationParameters,TiffToNGFFValueError
 
 # def iterate_chunks(ds, slice_length, interleaved_channels=1, channel=0, interleaving_offset=0):
 #     """given an iterator, iterate over tuples of a
@@ -597,12 +597,12 @@
 #     return attrs
 
 
-class TiffToNGFFException(Exception):
-    """class to describe exceptions with TiffToNGFF module"""
+# class TiffToNGFFException(Exception):
+#     """class to describe exceptions with TiffToNGFF module"""
 
 
-class TiffToNGFFValueError(TiffToNGFFException, ValueError):
-    """value error in TiffToNgff"""
+# class TiffToNGFFValueError(TiffToNGFFException, ValueError):
+#     """value error in TiffToNgff"""
 
 
 def write_ims_to_zarr(
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
index e1904da..d393f91 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
@@ -15,524 +15,8 @@
 from numcodecs import Blosc
 import argschema
 
-import acpreprocessing.utils.convert
 import acpreprocessing.stitching_modules.convert_to_n5.psdeskew as psd
-
-
-def iterate_chunks(it, slice_length):
-    """given an iterator, iterate over tuples of a
-    given length from that iterator
-
-    Parameters
-    ----------
-    it : iterable
-        iterable object to iterate over in slices of slice_length
-    slice_length : int
-        number of items in iterator per chunk
-
-    Yields
-    ------
-    chunk : tuple
-        tuple of items from iterator of size slice_length. Yields
-        remainder of iterator when exhausted
-    """
-    it = iter(it)
-    chunk = tuple(itertools.islice(it, slice_length))
-    while chunk:
-        yield chunk
-        chunk = tuple(itertools.islice(it, slice_length))
-
-
-def iter_arrays(r, interleaved_channels=1, channel=0, interleaving_offset=0):
-    """iterate arrays from an imageio tiff reader.  Allows the last image
-    of the array to be None, as is the case for data with 'dropped frames'.
-
-    Parameters
-    ----------
-    r : imageio.reader
-        imageio tiff reader from which to iterate multitiff arrays
-    interleaved_channels : int
-        number of channels interleaved in tiff when selecting which
-        arrays to read
-    channel : int
-        channel to iterate over in case of interleaved tiffs
-
-    Yields
-    -------
-    arr : numpy.ndarray
-        constituent page array of reader r
-    """
-    for i, p in enumerate(r._tf.pages):
-        page_channel = (i + interleaving_offset) % interleaved_channels
-        if page_channel != channel:
-            continue
-        arr = p.asarray()
-        if arr is not None:
-            yield arr
-        elif i == r.get_length() - 1:
-            continue
-        else:
-            raise ValueError
-
-
-def iterate_2d_arrays_from_mimgfns(mimgfns, interleaved_channels=1, channel=0):
-    """iterate constituent arrays from an iterator of image filenames
-    that can be opened as an imageio multi-image.
-
-    Parameters
-    ----------
-    mimgfns : list of str
-        imageio-compatible name inputs to be opened as multi-images
-
-    Yields
-    ------
-    array : numpy.ndarray
-        constituent array from ordered iterator of multi-image files
-    """
-    offset = 0
-    for mimgfn in mimgfns:
-        with imageio.get_reader(mimgfn, mode="I") as r:
-            yield from iter_arrays(r, interleaved_channels, channel, offset)
-            offset = (offset + r.get_length()) % interleaved_channels
-
-
-def iterate_numpy_chunks_from_mimgfns(
-        mimgfns, slice_length=None, pad=True, *args, **kwargs):
-    """iterate over a contiguous iterator of imageio multi-image files as
-    chunks of numpy arrays
-
-    Parameters
-    ----------
-    mimgfns : list of str
-        imageio-compatible name inputs to be opened as multi-images
-    slice_length : int
-        number of 2d arrays from mimgfns included per chunk
-    pad : bool, optional
-        whether to extend final returned chunks with zeros
-
-    Yields
-    ------
-    arr : numpy.ndarray
-        3D numpy array representing a consecutive chunk of 2D arrays
-    """
-    array_gen = iterate_2d_arrays_from_mimgfns(mimgfns, *args, **kwargs)
-    for chunk in iterate_chunks(array_gen, slice_length):
-        arr = numpy.array(chunk)
-        if pad:
-            if arr.shape[0] != slice_length:
-                newarr = numpy.zeros((slice_length, *arr.shape[1:]),
-                                     dtype=arr.dtype)
-                newarr[:arr.shape[0], :, :] = arr[:, :, :]
-                yield newarr
-            else:
-                yield arr
-        else:
-            yield arr
-
-
-def length_to_interleaved_length(length, interleaved_channels):
-    """get length of each interleaved channel that contributes to a
-    total array length
-
-    Parameters
-    ----------
-    length : int
-        length of interleaved array
-    interleaved_channels : int
-        number of constituent channels in interleaved array
-
-    Returns
-    -------
-    channel_lengths : list of int
-        length of each constituent channel in interleaved array
-    """
-    interleaved_length, r = divmod(length, interleaved_channels)
-    return [
-        interleaved_length + (1 if ic < r else 0)
-        for ic in range(interleaved_channels)
-    ]
-
-
-def mimg_shape_from_fn(mimg_fn, only_length_tup=False):
-    """get the shape of an imageio multi-image file without
-    reading it as a volume
-
-    Parameters
-    ----------
-    mimg_fn : str
-        imageio-compatible name input to be opened as multi-image
-    only_length_tup : bool, optional
-        whether to return only the number of frames or
-        include the dimensions of a representative frame
-
-    Returns
-    -------
-    shape : tuple of int
-        shape of array defined by mimg_fn
-    """
-    with imageio.get_reader(mimg_fn, mode="I") as r:
-        l = r.get_length()
-
-        if only_length_tup:
-            s = (l,)
-        else:
-            s = (l, *r.get_data(0).shape)
-    return s
-
-
-def joined_mimg_shape_from_fns(mimg_fns, concurrency=1,
-                               interleaved_channels=1, channel=0,
-                               *args, **kwargs):
-    """concurrently read image shapes from tiff files representing a
-    contiguous stack to get the shape of the combined stack.
-
-    Parameters
-    ----------
-    mimg_fns : list of str
-        imageio-compatible name inputs to be opened as multi-images
-    concurrency : int, optional
-        number of threads to use when getting image shape metadata
-    interleaved_channels : int, optional
-        number of channels interleaved in the tiff files (default 1)
-    channel : int, optional
-        channel from which interleaved data should be read (default 0)
-
-    Returns
-    -------
-    shape : tuple of int
-        shape of 3D array represented by concatenating mimg_fns
-    """
-    # join shapes while reading concurrently
-    with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as e:
-        futs = [e.submit(mimg_shape_from_fn, fn, *args, **kwargs)
-                for fn in mimg_fns]
-        shapes = [fut.result() for fut
-                  in concurrent.futures.as_completed(futs)]
-    return (
-        length_to_interleaved_length(
-            sum([s[0] for s in shapes]), interleaved_channels)[channel],
-        shapes[0][1],
-        shapes[0][2]
-    )
-
-
-def array_downsample_chunks(arr, ds_factors, block_shape, **kwargs):
-    """downsample a numpy array by inputting subvolumes
-    to acpreprocessing.utils.convert.downsample_stack_volume
-
-    Parameters
-    ----------
-    arr : numpy.ndarray
-        array to downsample
-    ds_factors : tuple of int
-        integer downsample factor by which arr should be downsamples.
-        Length must match arr.ndim
-    block_shape : tuple of int
-        shape of individual blocks to process
-
-    Returns
-    -------
-    output_arr : numpy.ndarray
-        downsampled version of arr
-    """
-    blocks = skimage.util.view_as_blocks(arr, block_shape)
-
-    # construct output array and blocks
-    output_shape = tuple(
-        int(math.ceil(s / f)) for s, f in zip(arr.shape, ds_factors))
-    output_block_shape = tuple(
-        int(math.ceil(s / f)) for s, f in zip(block_shape, ds_factors))
-    output_arr = numpy.empty(output_shape, dtype=arr.dtype)
-    output_blocks = skimage.util.view_as_blocks(
-        output_arr, output_block_shape)
-
-    # process
-    for idxs in numpy.ndindex(blocks.shape[:arr.ndim]):
-        output_blocks[idxs] = (
-            acpreprocessing.utils.convert.downsample_stack_volume(
-                blocks[idxs], ds_factors, **kwargs))
-
-    return output_arr
-
-
-def array_downsample_threaded(arr, ds_factors, block_shape,
-                              n_threads=3, **kwargs):
-    """downsample a numpy array by concurrently inputting subvolumes
-    to acpreprocessing.utils.convert.downsample_stack_volume
-
-    Parameters
-    ----------
-    arr : numpy.ndarray
-        array to downsample
-    ds_factors : tuple of int
-        integer downsample factor by which arr should be downsamples.
-        Length must match arr.ndim
-    block_shape : tuple of int
-        shape of individual blocks to process
-    n_threads :  int
-        number of threads used to concurrently downsample chunks
-
-    Returns
-    -------
-    output_arr : numpy.ndarray
-        downsampled version of arr
-    """
-    blocks = skimage.util.view_as_blocks(arr, block_shape)
-
-    # construct output array and blocks
-    output_shape = tuple(
-        int(math.ceil(s / f)) for s, f in zip(arr.shape, ds_factors))
-    output_block_shape = tuple(
-        int(math.ceil(s / f)) for s, f in zip(block_shape, ds_factors))
-    output_arr = numpy.empty(output_shape, dtype=arr.dtype)
-    output_blocks = skimage.util.view_as_blocks(
-        output_arr, output_block_shape)
-
-    # process
-    with concurrent.futures.ThreadPoolExecutor(max_workers=n_threads) as e:
-        fut_to_idxs = {}
-        for idxs in numpy.ndindex(blocks.shape[:arr.ndim]):
-            fut_to_idxs[e.submit(
-                acpreprocessing.utils.convert.downsample_stack_volume,
-                blocks[idxs], ds_factors, **kwargs)] = idxs
-
-        for fut in concurrent.futures.as_completed(fut_to_idxs.keys()):
-            idxs = fut_to_idxs[fut]
-            output_blocks[idxs] = fut.result()
-    return output_arr
-
-
-class ArrayChunkingError(ValueError):
-    """ValueError caused by incorrect array chunking"""
-
-
-def downsample_array(arr, *args, n_threads=None, block_shape=None,
-                     block_divs=None, allow_failover=True, **kwargs):
-    """downsample array with optional subvolume chunking and threading
-
-    Parameters
-    ----------
-    arr : numpy.ndarray
-        array to downsample
-    ds_factors : tuple of int
-        integer downsample factor by which arr should be downsamples.
-        Length must match arr.ndim
-    block_shape : tuple of int
-        shape of individual blocks to process.
-        Supersedes block_divs if both are provided
-    block_divs : tuple of int
-        number of divisions for array chunking.
-    n_threads :  int
-        number of threads used to concurrently downsample chunks
-    allow_failover : bool, optional
-        whether to default to using
-        acpreprocessing.utils.convert.downsample_stack_volume when
-        block_shape or block_divs is not actionable.  (default: True)
-
-    Returns
-    -------
-    output_arr : numpy.ndarray
-        downsampled version of arr
-    """
-    try:
-        if block_shape is not None or block_divs is not None:
-            if block_shape is None:
-                block_shape = [a_s / d for a_s, d in zip(
-                    arr.shape, block_divs)]
-                if not all([s.is_integer() for s in block_shape]):
-                    raise ArrayChunkingError(
-                        "cannot divide array of shape {} by {}".format(
-                            arr.shape, block_divs))
-                block_shape = tuple(int(s) for s in block_shape)
-            if n_threads is not None:
-                return array_downsample_threaded(
-                    arr, *args, n_threads=n_threads,
-                    block_shape=block_shape, **kwargs)
-            return array_downsample_chunks(
-                arr, *args, block_shape=block_shape, **kwargs)
-    except ArrayChunkingError:
-        if not allow_failover:
-            raise
-    return acpreprocessing.utils.convert.downsample_stack_volume(
-        arr, *args, **kwargs)
-
-
-def mip_level_shape(lvl, lvl_0_shape):
-    """get the expected shape of a MIP level downsample for a given
-    full resolution shape.  Expects 2x downsampling on each dimension.
-
-    Parameters
-    ----------
-    lvl : int
-        mip level for which shape is calculated
-    lvl_0_shape : tuple of int
-        shape tuple of level 0
-
-    Returns
-    -------
-    lvl_shape : tuple of int
-        shape tuple for lvl
-    """
-    return tuple([*map(lambda x: math.ceil(x / (2**lvl)), lvl_0_shape)])
-
-
-def dswrite_chunk(ds, start, end, arr, silent_overflow=True):
-    """write array arr into array-like n5 dataset defined by
-    start and end points on 0 axis.
-
-    Parameters
-    ----------
-    ds : z5py.dataset.Dataset (n5 3D) or zarr.dataset (zarr 5D)
-        array-like dataset to fill in
-    start : int
-        start index along 0 axis of ds to fill in
-    end : int
-        end index along 0 axis of ds to fill in
-    arr : numpy.ndarray
-        array from which ds values will be taken
-    silent_overflow : bool, optional
-        whether to shrink the end index to match the
-        shape of ds (default: True)
-    """
-    if len(ds.shape) == 5:  # dataset dimensions should be 3 or 5
-        if end >= ds.shape[2] and silent_overflow:
-            end = ds.shape[2]
-        if end > start:
-            ds[0, 0, start:end, :, :] = arr[:(end - start), :, :]
-    elif len(ds.shape) == 3:
-        if end >= ds.shape[0] and silent_overflow:
-            end = ds.shape[0]
-        if end > start:
-            ds[start:end, :, :] = arr[:(end - start), :, :]
-
-
-@dataclasses.dataclass
-class MIPArray:
-    lvl: int
-    array: numpy.ndarray
-    start: int
-    end: int
-
-
-def iterate_mip_levels_from_mimgfns(
-        mimgfns, lvl, block_size, slice_length, downsample_factor,
-        downsample_method=None, lvl_to_mip_kwargs=None,
-        interleaved_channels=1, channel=0, deskew_kwargs={}):
-    """recursively generate MIPmap levels from an iterator of multi-image files
-
-    Parameters
-    ----------
-    mimgfns : list of str
-        imageio-compatible name inputs to be opened as multi-images
-    lvl : int
-        integer mip level to generate
-    block_size : int
-        number of 2D arrays in chunk to process for a chunk at lvl
-    slice_length : int
-        number of 2D arrays to gather from tiff stacks
-    downsample_factor : tuple of int
-        integer downsampling factor for MIP levels
-    downsample_method : str, optional
-        downsampling method for
-        acpreprocessing.utils.convert.downsample_stack_volume
-    lvl_to_mip_kwargs :  dict, optional
-        mapping of MIP level to kwargs used in MIPmap generation
-    interleaved_channels : int
-        number of channels interleaved in the tiff files (default 1)
-    channel : int, optional
-        channel from which interleaved data should be read (default 0)
-    deskew_kwargs : dict, optional
-        parameters for pixel shifting deskew
-
-    Yields
-    ------
-    ma : acpreprocessing.stitching_modules.convert_to_n5.tiff_to_n5.MipArray
-        object describing chunked array, MIP level of origin, and chunk indices
-    """
-    lvl_to_mip_kwargs = ({} if lvl_to_mip_kwargs is None
-                         else lvl_to_mip_kwargs)
-    mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
-    start_index = 0
-    chunk_index = 0
-    if lvl > 0:
-        num_chunks = downsample_factor[0]
-
-        i = 0
-        for ma in iterate_mip_levels_from_mimgfns(
-                mimgfns, lvl-1, block_size, slice_length,
-                downsample_factor, downsample_method,
-                lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
-                channel=channel, deskew_kwargs=deskew_kwargs):
-            chunk = ma.array
-            # throw array up for further processing
-            yield ma
-            # ignore if not parent resolution
-            if ma.lvl != lvl-1:
-                continue
-
-            try:
-                temp_lminus1_arr
-            except NameError:
-                temp_lminus1_arr = numpy.empty(
-                    (num_chunks*chunk.shape[0], *chunk.shape[1:]),
-                    dtype=chunk.dtype)
-
-            # fill in temporary block according to index
-            chunk_size = chunk.shape[0]
-            block_offset = i*block_size
-
-            temp_lminus1_arr[
-                block_offset:block_offset+chunk_size, :, :] = chunk[:, :, :]
-
-            # copy op only for uneven final chunk
-            if chunk_size != block_size:
-                temp_lminus1_arr = temp_lminus1_arr[
-                    :block_offset+chunk_size, :, :]
-
-            if i == num_chunks - 1:
-                temp_arr = (
-                    downsample_array(
-                        temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
-                        method=downsample_method, **mip_kwargs))
-                end_index = start_index + temp_arr.shape[0]
-                yield MIPArray(lvl, temp_arr, start_index, end_index)
-                start_index += temp_arr.shape[0]
-                i = 0
-            else:
-                i += 1
-        # add any leftovers
-        if i > 0:
-            temp_arr = acpreprocessing.utils.convert.downsample_stack_volume(
-                temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
-                method=downsample_method)
-            end_index = start_index + temp_arr.shape[0]
-            yield MIPArray(lvl, temp_arr, start_index, end_index)
-    else:
-        # get level 0 chunks
-        # block_size is the number of slices to read from tiffs
-        for chunk in iterate_numpy_chunks_from_mimgfns(
-                mimgfns, slice_length, pad=False,
-                interleaved_channels=interleaved_channels,
-                channel=channel):
-            # deskew level 0 chunk
-            if deskew_kwargs:
-                chunk = numpy.transpose(psd.deskew_block(
-                    chunk, chunk_index, **deskew_kwargs), (2, 1, 0))
-            end_index = start_index + chunk.shape[0]
-            yield MIPArray(lvl, chunk, start_index, end_index)
-            start_index += chunk.shape[0]
-            chunk_index += 1
-
-
-
-
-
-class TiffToNGFFException(Exception):
-    """class to describe exceptions with TiffToNGFF module"""
-
-
-class TiffToNGFFValueError(TiffToNGFFException, ValueError):
-    """value error in TiffToNgff"""
+from acpreprocessing.stitching_modules.convert_to_n5.tiff_to_ngff import iterate_mip_levels_from_dataset,mip_level_shape,dswrite_chunk,iterate_2d_arrays_from_mimgfns,joined_mimg_shape_from_fns,TiffToNGFFValueError,TiffDirToNGFFParameters
 
 
 def write_mimgfns_to_zarr(
@@ -581,6 +65,7 @@ def write_mimgfns_to_zarr(
     group_attributes = ([] if group_attributes is None else group_attributes)
     deskew_options = ({} if deskew_options is None else deskew_options)
 
+    array_gen = iterate_2d_arrays_from_mimgfns(mimgfns, interleaved_channels=interleaved_channels,channel=channel)    
     joined_shapes = joined_mimg_shape_from_fns(
         mimgfns, concurrency=concurrency,
         interleaved_channels=interleaved_channels, channel=channel)
@@ -632,8 +117,8 @@ def write_mimgfns_to_zarr(
 
         with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as e:
             futs = []
-            for miparr in iterate_mip_levels_from_mimgfns(
-                    mimgfns, max_mip, block_size, slice_length, mip_dsfactor,
+            for miparr in iterate_mip_levels_from_dataset(
+                    array_gen, max_mip, block_size, slice_length, mip_dsfactor,
                     lvl_to_mip_kwargs=lvl_to_mip_kwargs,
                     interleaved_channels=interleaved_channels,
                     channel=channel, deskew_kwargs=deskew_kwargs):
@@ -661,54 +146,6 @@ def tiffdir_to_ngff_group(tiffdir, output, *args, **kwargs):
         return write_mimgfns_to_zarr(mimgfns, *args, **kwargs)
 
 
-class DownsampleOptions(argschema.schemas.DefaultSchema):
-    block_divs = argschema.fields.List(
-        argschema.fields.Int, required=False, allow_none=True)
-    n_threads = argschema.fields.Int(required=False, allow_none=True)
-
-
-class DeskewOptions(argschema.schemas.DefaultSchema):
-    deskew_method = argschema.fields.Str(required=False, default='')
-    deskew_stride = argschema.fields.Int(required=False, default=None)
-    deskew_flip = argschema.fields.Bool(required=False, default=True)
-    deskew_crop = argschema.fields.Float(required=False, default=1.0)
-
-
-class NGFFGenerationParameters(argschema.schemas.DefaultSchema):
-    output_format = argschema.fields.Str(required=True)
-    output_file = argschema.fields.Str(required=True)
-    max_mip = argschema.fields.Int(required=False, default=0)
-    concurrency = argschema.fields.Int(required=False, default=10)
-    compression = argschema.fields.Str(required=False, default="raw")
-    # lvl_to_mip_kwargs = argschema.fields.Dict(
-    #     keys=argschema.fields.Int(),
-    #     values=argschema.fields.Nested(DownsampleOptions))
-
-    # FIXME argschema supports lists and tuples,
-    #   but has some version differences
-
-    mip_dsfactor = argschema.fields.Tuple((
-        argschema.fields.Int(),
-        argschema.fields.Int(),
-        argschema.fields.Int()), required=False, default=(2, 2, 2))
-    deskew_options = argschema.fields.Nested(
-        DeskewOptions, required=False)
-
-
-class NGFFGroupGenerationParameters(NGFFGenerationParameters):
-    group_names = argschema.fields.List(
-        argschema.fields.Str, required=True)
-    group_attributes = argschema.fields.List(
-        argschema.fields.Dict(required=False, default={}), default=[],
-        required=False)
-
-
-class TiffDirToNGFFParameters(NGFFGroupGenerationParameters):
-    input_dir = argschema.fields.InputDir(required=True)
-    interleaved_channels = argschema.fields.Int(required=False, default=1)
-    channel = argschema.fields.Int(required=False, default=0)
-
-
 class TiffDirToZarrInputParameters(argschema.ArgSchema,
                                    TiffDirToNGFFParameters):
     chunk_size = argschema.fields.Tuple((

From c84c760127ade45a200679e3e5b2d97fed360e14 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 30 Jul 2024 11:45:18 -0700
Subject: [PATCH 020/115] clean up ims conversion module old code

---
 .../convert_to_n5/ims_to_ngff.py              | 624 ------------------
 .../convert_to_n5/psdeskew.py                 |  16 +
 2 files changed, 16 insertions(+), 624 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index a155975..c4d75dc 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -21,589 +21,6 @@
 import acpreprocessing.stitching_modules.convert_to_n5.psdeskew as psd
 from acpreprocessing.stitching_modules.convert_to_n5.tiff_to_ngff import iterate_mip_levels_from_dataset,mip_level_shape,dswrite_chunk,omezarr_attrs,NGFFGroupGenerationParameters,TiffToNGFFValueError
 
-# def iterate_chunks(ds, slice_length, interleaved_channels=1, channel=0, interleaving_offset=0):
-#     """given an iterator, iterate over tuples of a
-#     given length from that iterator
-
-#     Parameters
-#     ----------
-#     it : iterable
-#         iterable object to iterate over in slices of slice_length
-#     slice_length : int
-#         number of items in iterator per chunk
-
-#     Yields
-#     ------
-#     chunk : tuple
-#         tuple of items from iterator of size slice_length. Yields
-#         remainder of iterator when exhausted
-#     """
-#     start = 0
-#     stop = slice_length
-#     chunk = ds[start:stop]
-#     while start<ds.shape[0]:
-#         yield chunk
-#         start = stop
-#         stop = start + slice_length
-#         if stop < ds.shape[0]:
-#             chunk = ds[start:stop]
-#         else:
-#             chunk = ds[start:]
-
-
-# def iter_arrays(r, interleaved_channels=1, channel=0, interleaving_offset=0):
-#     """iterate arrays from an imageio tiff reader.  Allows the last image
-#     of the array to be None, as is the case for data with 'dropped frames'.
-
-#     Parameters
-#     ----------
-#     r : imageio.reader
-#         imageio tiff reader from which to iterate multitiff arrays
-#     interleaved_channels : int
-#         number of channels interleaved in tiff when selecting which
-#         arrays to read
-#     channel : int
-#         channel to iterate over in case of interleaved tiffs
-
-#     Yields
-#     -------
-#     arr : numpy.ndarray
-#         constituent page array of reader r
-#     """
-#     for i, p in enumerate(r._tf.pages):
-#         page_channel = (i + interleaving_offset) % interleaved_channels
-#         if page_channel != channel:
-#             continue
-#         arr = p.asarray()
-#         if arr is not None:
-#             yield arr
-#         elif i == r.get_length() - 1:
-#             continue
-#         else:
-#             raise ValueError
-
-
-# def iterate_2d_arrays_from_mimgfns(mimgfns, interleaved_channels=1, channel=0):
-#     """iterate constituent arrays from an iterator of image filenames
-#     that can be opened as an imageio multi-image.
-
-#     Parameters
-#     ----------
-#     mimgfns : list of str
-#         imageio-compatible name inputs to be opened as multi-images
-
-#     Yields
-#     ------
-#     array : numpy.ndarray
-#         constituent array from ordered iterator of multi-image files
-#     """
-#     offset = 0
-#     for mimgfn in mimgfns:
-#         with imageio.get_reader(mimgfn, mode="I") as r:
-#             yield from iter_arrays(r, interleaved_channels, channel, offset)
-#             offset = (offset + r.get_length()) % interleaved_channels
-
-
-# def iterate_numpy_chunks_from_dataset(
-#         dataset, slice_length=None, pad=True, *args, **kwargs):
-#     """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
-
-#     Parameters
-#     ----------
-#     dataset : hdf5 dataset
-#         imageio-compatible name inputs to be opened as multi-images
-#     slice_length : int
-#         number of 2d arrays from mimgfns included per chunk
-#     pad : bool, optional
-#         whether to extend final returned chunks with zeros
-
-#     Yields
-#     ------
-#     arr : numpy.ndarray
-#         3D numpy array representing a consecutive chunk of 2D arrays
-#     """
-#     #array_gen = iterate_2d_arrays_from_dataset(mimgfns, *args, **kwargs)
-#     for chunk in iterate_chunks(dataset, slice_length,*args,**kwargs):
-#         arr = numpy.asarray(chunk)
-#         if pad:
-#             if arr.shape[0] != slice_length:
-#                 newarr = numpy.zeros((slice_length, *arr.shape[1:]),
-#                                       dtype=arr.dtype)
-#                 newarr[:arr.shape[0], :, :] = arr[:, :, :]
-#                 yield newarr
-#             else:
-#                 yield arr
-#         else:
-#             yield arr
-
-
-# def length_to_interleaved_length(length, interleaved_channels):
-#     """get length of each interleaved channel that contributes to a
-#     total array length
-
-#     Parameters
-#     ----------
-#     length : int
-#         length of interleaved array
-#     interleaved_channels : int
-#         number of constituent channels in interleaved array
-
-#     Returns
-#     -------
-#     channel_lengths : list of int
-#         length of each constituent channel in interleaved array
-#     """
-#     interleaved_length, r = divmod(length, interleaved_channels)
-#     return [
-#         interleaved_length + (1 if ic < r else 0)
-#         for ic in range(interleaved_channels)
-#     ]
-
-
-# def mimg_shape_from_fn(mimg_fn, only_length_tup=False):
-#     """get the shape of an imageio multi-image file without
-#     reading it as a volume
-
-#     Parameters
-#     ----------
-#     mimg_fn : str
-#         imageio-compatible name input to be opened as multi-image
-#     only_length_tup : bool, optional
-#         whether to return only the number of frames or
-#         include the dimensions of a representative frame
-
-#     Returns
-#     -------
-#     shape : tuple of int
-#         shape of array defined by mimg_fn
-#     """
-#     with imageio.get_reader(mimg_fn, mode="I") as r:
-#         l = r.get_length()
-
-#         if only_length_tup:
-#             s = (l,)
-#         else:
-#             s = (l, *r.get_data(0).shape)
-#     return s
-
-
-# def array_downsample_chunks(arr, ds_factors, block_shape, **kwargs):
-#     """downsample a numpy array by inputting subvolumes
-#     to acpreprocessing.utils.convert.downsample_stack_volume
-
-#     Parameters
-#     ----------
-#     arr : numpy.ndarray
-#         array to downsample
-#     ds_factors : tuple of int
-#         integer downsample factor by which arr should be downsamples.
-#         Length must match arr.ndim
-#     block_shape : tuple of int
-#         shape of individual blocks to process
-
-#     Returns
-#     -------
-#     output_arr : numpy.ndarray
-#         downsampled version of arr
-#     """
-#     blocks = skimage.util.view_as_blocks(arr, block_shape)
-
-#     # construct output array and blocks
-#     output_shape = tuple(
-#         int(math.ceil(s / f)) for s, f in zip(arr.shape, ds_factors))
-#     output_block_shape = tuple(
-#         int(math.ceil(s / f)) for s, f in zip(block_shape, ds_factors))
-#     output_arr = numpy.empty(output_shape, dtype=arr.dtype)
-#     output_blocks = skimage.util.view_as_blocks(
-#         output_arr, output_block_shape)
-
-#     # process
-#     for idxs in numpy.ndindex(blocks.shape[:arr.ndim]):
-#         output_blocks[idxs] = (
-#             acpreprocessing.utils.convert.downsample_stack_volume(
-#                 blocks[idxs], ds_factors, **kwargs))
-
-#     return output_arr
-
-
-# def array_downsample_threaded(arr, ds_factors, block_shape,
-#                               n_threads=3, **kwargs):
-#     """downsample a numpy array by concurrently inputting subvolumes
-#     to acpreprocessing.utils.convert.downsample_stack_volume
-
-#     Parameters
-#     ----------
-#     arr : numpy.ndarray
-#         array to downsample
-#     ds_factors : tuple of int
-#         integer downsample factor by which arr should be downsamples.
-#         Length must match arr.ndim
-#     block_shape : tuple of int
-#         shape of individual blocks to process
-#     n_threads :  int
-#         number of threads used to concurrently downsample chunks
-
-#     Returns
-#     -------
-#     output_arr : numpy.ndarray
-#         downsampled version of arr
-#     """
-#     blocks = skimage.util.view_as_blocks(arr, block_shape)
-
-#     # construct output array and blocks
-#     output_shape = tuple(
-#         int(math.ceil(s / f)) for s, f in zip(arr.shape, ds_factors))
-#     output_block_shape = tuple(
-#         int(math.ceil(s / f)) for s, f in zip(block_shape, ds_factors))
-#     output_arr = numpy.empty(output_shape, dtype=arr.dtype)
-#     output_blocks = skimage.util.view_as_blocks(
-#         output_arr, output_block_shape)
-
-#     # process
-#     with concurrent.futures.ThreadPoolExecutor(max_workers=n_threads) as e:
-#         fut_to_idxs = {}
-#         for idxs in numpy.ndindex(blocks.shape[:arr.ndim]):
-#             fut_to_idxs[e.submit(
-#                 acpreprocessing.utils.convert.downsample_stack_volume,
-#                 blocks[idxs], ds_factors, **kwargs)] = idxs
-
-#         for fut in concurrent.futures.as_completed(fut_to_idxs.keys()):
-#             idxs = fut_to_idxs[fut]
-#             output_blocks[idxs] = fut.result()
-#     return output_arr
-
-
-# class ArrayChunkingError(ValueError):
-#     """ValueError caused by incorrect array chunking"""
-
-
-# def downsample_array(arr, *args, n_threads=None, block_shape=None,
-#                       block_divs=None, allow_failover=True, **kwargs):
-#     """downsample array with optional subvolume chunking and threading
-
-#     Parameters
-#     ----------
-#     arr : numpy.ndarray
-#         array to downsample
-#     ds_factors : tuple of int
-#         integer downsample factor by which arr should be downsamples.
-#         Length must match arr.ndim
-#     block_shape : tuple of int
-#         shape of individual blocks to process.
-#         Supersedes block_divs if both are provided
-#     block_divs : tuple of int
-#         number of divisions for array chunking.
-#     n_threads :  int
-#         number of threads used to concurrently downsample chunks
-#     allow_failover : bool, optional
-#         whether to default to using
-#         acpreprocessing.utils.convert.downsample_stack_volume when
-#         block_shape or block_divs is not actionable.  (default: True)
-
-#     Returns
-#     -------
-#     output_arr : numpy.ndarray
-#         downsampled version of arr
-#     """
-#     try:
-#         if block_shape is not None or block_divs is not None:
-#             if block_shape is None:
-#                 block_shape = [a_s / d for a_s, d in zip(
-#                     arr.shape, block_divs)]
-#                 if not all([s.is_integer() for s in block_shape]):
-#                     raise ArrayChunkingError(
-#                         "cannot divide array of shape {} by {}".format(
-#                             arr.shape, block_divs))
-#                 block_shape = tuple(int(s) for s in block_shape)
-#             if n_threads is not None:
-#                 return array_downsample_threaded(
-#                     arr, *args, n_threads=n_threads,
-#                     block_shape=block_shape, **kwargs)
-#             return array_downsample_chunks(
-#                 arr, *args, block_shape=block_shape, **kwargs)
-#     except ArrayChunkingError:
-#         if not allow_failover:
-#             raise
-#     return acpreprocessing.utils.convert.downsample_stack_volume(
-#         arr, *args, **kwargs)
-
-
-# def mip_level_shape(lvl, lvl_0_shape):
-#     """get the expected shape of a MIP level downsample for a given
-#     full resolution shape.  Expects 2x downsampling on each dimension.
-
-#     Parameters
-#     ----------
-#     lvl : int
-#         mip level for which shape is calculated
-#     lvl_0_shape : tuple of int
-#         shape tuple of level 0
-
-#     Returns
-#     -------
-#     lvl_shape : tuple of int
-#         shape tuple for lvl
-#     """
-#     return tuple([*map(lambda x: math.ceil(x / (2**lvl)), lvl_0_shape)])
-
-
-# def dswrite_chunk(ds, start, end, arr, silent_overflow=True):
-#     """write array arr into array-like n5 dataset defined by
-#     start and end points on 0 axis.
-
-#     Parameters
-#     ----------
-#     ds : z5py.dataset.Dataset (n5 3D) or zarr.dataset (zarr 5D)
-#         array-like dataset to fill in
-#     start : int
-#         start index along 0 axis of ds to fill in
-#     end : int
-#         end index along 0 axis of ds to fill in
-#     arr : numpy.ndarray
-#         array from which ds values will be taken
-#     silent_overflow : bool, optional
-#         whether to shrink the end index to match the
-#         shape of ds (default: True)
-#     """
-#     if len(ds.shape) == 5:  # dataset dimensions should be 3 or 5
-#         if end >= ds.shape[2] and silent_overflow:
-#             end = ds.shape[2]
-#         if end > start:
-#             ds[0, 0, start:end, :, :] = arr[:(end - start), :, :]
-#     elif len(ds.shape) == 3:
-#         if end >= ds.shape[0] and silent_overflow:
-#             end = ds.shape[0]
-#         if end > start:
-#             ds[start:end, :, :] = arr[:(end - start), :, :]
-
-
-# @dataclasses.dataclass
-# class MIPArray:
-#     lvl: int
-#     array: numpy.ndarray
-#     start: int
-#     end: int
-
-
-# def iterate_mip_levels_from_dataset(
-#         dataset, lvl, block_size, slice_length, downsample_factor,
-#         downsample_method=None, lvl_to_mip_kwargs=None,
-#         interleaved_channels=1, channel=0, deskew_kwargs={}):
-#     """recursively generate MIPmap levels from an iterator of multi-image files
-
-#     Parameters
-#     ----------
-#     dataset : hdf5 dataset
-#         imageio-compatible name inputs to be opened as multi-images
-#     lvl : int
-#         integer mip level to generate
-#     block_size : int
-#         number of 2D arrays in chunk to process for a chunk at lvl
-#     slice_length : int
-#         number of 2D arrays to gather from tiff stacks
-#     downsample_factor : tuple of int
-#         integer downsampling factor for MIP levels
-#     downsample_method : str, optional
-#         downsampling method for
-#         acpreprocessing.utils.convert.downsample_stack_volume
-#     lvl_to_mip_kwargs :  dict, optional
-#         mapping of MIP level to kwargs used in MIPmap generation
-#     interleaved_channels : int
-#         number of channels interleaved in the tiff files (default 1)
-#     channel : int, optional
-#         channel from which interleaved data should be read (default 0)
-#     deskew_kwargs : dict, optional
-#         parameters for pixel shifting deskew
-
-#     Yields
-#     ------
-#     ma : acpreprocessing.stitching_modules.convert_to_n5.tiff_to_n5.MipArray
-#         object describing chunked array, MIP level of origin, and chunk indices
-#     """
-#     lvl_to_mip_kwargs = ({} if lvl_to_mip_kwargs is None
-#                           else lvl_to_mip_kwargs)
-#     mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
-#     start_index = 0
-#     chunk_index = 0
-#     if lvl > 0:
-#         num_chunks = downsample_factor[0]
-
-#         i = 0
-#         for ma in iterate_mip_levels_from_dataset(
-#                 dataset, lvl-1, block_size, slice_length,
-#                 downsample_factor, downsample_method,
-#                 lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
-#                 channel=channel, deskew_kwargs=deskew_kwargs):
-#             chunk = ma.array
-#             # throw array up for further processing
-#             yield ma
-#             # ignore if not parent resolution
-#             if ma.lvl != lvl-1:
-#                 continue
-
-#             try:
-#                 temp_lminus1_arr
-#             except NameError:
-#                 temp_lminus1_arr = numpy.empty(
-#                     (num_chunks*chunk.shape[0], *chunk.shape[1:]),
-#                     dtype=chunk.dtype)
-
-#             # fill in temporary block according to index
-#             chunk_size = chunk.shape[0]
-#             block_offset = i*block_size
-
-#             temp_lminus1_arr[
-#                 block_offset:block_offset+chunk_size, :, :] = chunk[:, :, :]
-
-#             # copy op only for uneven final chunk
-#             if chunk_size != block_size:
-#                 temp_lminus1_arr = temp_lminus1_arr[
-#                     :block_offset+chunk_size, :, :]
-
-#             if i == num_chunks - 1:
-#                 temp_arr = (
-#                     downsample_array(
-#                         temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
-#                         method=downsample_method, **mip_kwargs))
-#                 end_index = start_index + temp_arr.shape[0]
-#                 yield MIPArray(lvl, temp_arr, start_index, end_index)
-#                 start_index += temp_arr.shape[0]
-#                 i = 0
-#             else:
-#                 i += 1
-#         # add any leftovers
-#         if i > 0:
-#             temp_arr = acpreprocessing.utils.convert.downsample_stack_volume(
-#                 temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
-#                 method=downsample_method)
-#             end_index = start_index + temp_arr.shape[0]
-#             yield MIPArray(lvl, temp_arr, start_index, end_index)
-#     else:
-#         # get level 0 chunks
-#         # block_size is the number of slices to read from tiffs
-#         for chunk in iterate_numpy_chunks_from_dataset(
-#                 dataset, slice_length, pad=False,
-#                 interleaved_channels=interleaved_channels,
-#                 channel=channel):
-#             # deskew level 0 chunk
-#             if deskew_kwargs:
-#                 chunk = numpy.transpose(psd.deskew_block(
-#                     chunk, chunk_index, **deskew_kwargs), (2, 1, 0))
-#             end_index = start_index + chunk.shape[0]
-#             yield MIPArray(lvl, chunk, start_index, end_index)
-#             start_index += chunk.shape[0]
-#             chunk_index += 1
-
-
-
-# def omezarr_attrs(name, position_xyz, lvl0_xyz_res, max_lvl):
-#     # TODO only single channel implemented
-#     datasets = []
-#     for i in range(max_lvl+1):
-#         s = [(2**i)*r for r in lvl0_xyz_res]
-#         d = {"coordinateTransformations": [
-#             {"scale": [
-#                 1.0,
-#                 1.0,
-#                 s[2],
-#                 s[1],
-#                 s[0]
-#             ],
-#                 "type": "scale"
-#             }
-#         ],
-#             "path": str(i)
-#         }
-#         datasets.append(d)
-#     attrs = {}
-#     attrs["multiscales"] = [
-#         {
-#             "axes": [
-#                 {
-#                     "name": "t",
-#                     "type": "time",
-#                     "unit": "millisecond"
-#                 },
-#                 {
-#                     "name": "c",
-#                     "type": "channel"
-#                 },
-#                 {
-#                     "name": "z",
-#                     "type": "space",
-#                     "unit": "micrometer"
-#                 },
-#                 {
-#                     "name": "y",
-#                     "type": "space",
-#                     "unit": "micrometer"
-#                 },
-#                 {
-#                     "name": "x",
-#                     "type": "space",
-#                     "unit": "micrometer"
-#                 }
-#             ],
-#             "datasets": datasets,
-#             "name": "/" + name,
-#             "translation": [
-#                 0,
-#                 0,
-#                 0.0,
-#                 0.0,
-#                 0.0
-#             ],
-#             "version": "0.4",
-#             "coordinateTransformations": [
-#                 {
-#                     "translation": [
-#                         0.0,
-#                         0.0,
-#                         position_xyz[2],
-#                         position_xyz[1],
-#                         position_xyz[0]
-#                     ],
-#                     "type": "translation"
-#                 }
-#             ]
-#         }
-#     ]
-#     attrs["omero"] = {
-#         "channels": [
-#             {
-#                 "active": True,
-#                 "coefficient": 1,
-#                 "color": "000000",
-#                 "family": "linear",
-#                 "inverted": False,
-#                 "label": "Channel:" + name + ":0",
-#                 "window": {
-#                     "end": 1.0,
-#                     "max": 1.0,
-#                     "min": 0.0,
-#                     "start": 0.0
-#                 }
-#             }
-#         ],
-#         "id": 1,
-#         "name": name,
-#         "rdefs": {
-#             "defaultT": 0,
-#             "defaultZ": 288,
-#             "model": "color"
-#         },
-#         "version": "0.4"
-#     }
-#     return attrs
-
-
-# class TiffToNGFFException(Exception):
-#     """class to describe exceptions with TiffToNGFF module"""
-
-
-# class TiffToNGFFValueError(TiffToNGFFException, ValueError):
-#     """value error in TiffToNgff"""
-
 
 def write_ims_to_zarr(
         ims_fn, output_n5, group_names, group_attributes=None, max_mip=0,
@@ -752,47 +169,6 @@ def ims_to_ngff_group(ims_fn, output, *args, **kwargs):
         print('unknown output format: ' + output)
 
 
-# class DownsampleOptions(argschema.schemas.DefaultSchema):
-#     block_divs = argschema.fields.List(
-#         argschema.fields.Int, required=False, allow_none=True)
-#     n_threads = argschema.fields.Int(required=False, allow_none=True)
-
-
-# class DeskewOptions(argschema.schemas.DefaultSchema):
-#     deskew_method = argschema.fields.Str(required=False, default='')
-#     deskew_stride = argschema.fields.Int(required=False, default=None)
-#     deskew_flip = argschema.fields.Bool(required=False, default=True)
-#     deskew_crop = argschema.fields.Float(required=False, default=1.0)
-
-
-# class NGFFGenerationParameters(argschema.schemas.DefaultSchema):
-#     output_format = argschema.fields.Str(required=True)
-#     output_file = argschema.fields.Str(required=True)
-#     max_mip = argschema.fields.Int(required=False, default=0)
-#     concurrency = argschema.fields.Int(required=False, default=10)
-#     compression = argschema.fields.Str(required=False, default="raw")
-#     lvl_to_mip_kwargs = argschema.fields.Dict(
-#         keys=argschema.fields.Int(),
-#         values=argschema.fields.Nested(DownsampleOptions))
-
-#     # FIXME argschema supports lists and tuples,
-#     #   but has some version differences
-
-#     mip_dsfactor = argschema.fields.Tuple((
-#         argschema.fields.Int(),
-#         argschema.fields.Int(),
-#         argschema.fields.Int()), required=False, default=(2, 2, 2))
-#     deskew_options = argschema.fields.Nested(
-#         DeskewOptions, required=False)
-
-
-# class NGFFGroupGenerationParameters(NGFFGenerationParameters):
-#     group_names = argschema.fields.List(
-#         argschema.fields.Str, required=True)
-#     group_attributes = argschema.fields.List(
-#         argschema.fields.Dict(required=False, default={}), default=[],
-#         required=False)
-
 
 class IMSToNGFFParameters(NGFFGroupGenerationParameters):
     input_file = argschema.fields.Str(required=True)
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index e656a18..dffbf1d 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -79,6 +79,22 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_tr
     return kwargs
 
 
+def calculate_skewed_indices(xi,yi,zi,s):
+    xs = s*zi + xi % s
+    ys = yi
+    zs = xi // s - zi
+    return xs,ys,zs
+
+
+def get_deskewed_block(dataset,xi,yi,zi,**kwargs):
+    sdims = dataset.shape
+    xs,ys,zs = calculate_skewed_indices(xi,yi,zi)
+    fli = np.ravel_multi_index((xs,ys,zs),sdims).flatten()
+    fldata = dataset[fli]
+    block = fldata.reshape((len(xi),len(yi),len(zi)))
+    return block
+
+
 def deskew_block(blockData, n, dsi, si, slice1d, blockdims, subblocks, flip, transpose, dtype, chunklength, *args, **kwargs):
     """deskew a data chunk in sequence with prior chunks
 

From 720b391ebf2b7628205da6aef73bef53885fb9a4 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 31 Jul 2024 10:32:06 -0700
Subject: [PATCH 021/115] attempting block conversion

---
 .../convert_to_n5/ims_to_ngff.py              | 196 +++++++++++++++++-
 1 file changed, 191 insertions(+), 5 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index c4d75dc..fcc3d6e 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -19,7 +19,190 @@
 
 # import acpreprocessing.utils.convert
 import acpreprocessing.stitching_modules.convert_to_n5.psdeskew as psd
-from acpreprocessing.stitching_modules.convert_to_n5.tiff_to_ngff import iterate_mip_levels_from_dataset,mip_level_shape,dswrite_chunk,omezarr_attrs,NGFFGroupGenerationParameters,TiffToNGFFValueError
+from acpreprocessing.stitching_modules.convert_to_n5.tiff_to_ngff import downsample_array,mip_level_shape,omezarr_attrs,NGFFGroupGenerationParameters,TiffToNGFFValueError
+
+
+@dataclasses.dataclass
+class MIPArray:
+    lvl: int
+    array: numpy.ndarray
+    start: tuple # tuple of indices (int)
+    end: tuple # tuple of indices (int)
+
+
+def dswrite_block(ds, start, end, arr, silent_overflow=True):
+    """write array arr into array-like n5 dataset defined by
+    start and end point 3-tuples.
+
+    Parameters
+    ----------
+    ds : z5py.dataset.Dataset (n5 3D) or zarr.dataset (zarr 5D)
+        array-like dataset to fill in
+    start : tuple[int]
+        start index along 0 axis of ds to fill in
+    end : tuple[int]
+        end index along 0 axis of ds to fill in
+    arr : numpy.ndarray
+        array from which ds values will be taken
+    silent_overflow : bool, optional
+        whether to shrink the end index to match the
+        shape of ds (default: True)
+    """
+    if len(ds.shape) == 5:  # dataset dimensions should be 3 or 5
+        for i in range(3):    
+            if end[i] >= ds.shape[2+i] and silent_overflow:
+                end[i] = ds.shape[2+i]
+        #if end > start:
+        ds[0, 0, start[0]:end[0], start[1]:end[1], start[2]:end[2]] = arr[:(end[0] - start[0]), :(end[1] - start[1]), :(end[2] - start[2])]
+    elif len(ds.shape) == 3:
+        for i in range(3):    
+            if end[i] >= ds.shape[i] and silent_overflow:
+                end[i] = ds.shape[i]
+        #if end > start:
+        ds[start[0]:end[0], start[1]:end[1], start[2]:end[2]] = arr[:(end[0] - start[0]), :(end[1] - start[1]), :(end[2] - start[2])]
+
+
+# def iterate_numpy_chunks_from_dataset(
+#         dataset, block_size=None, pad=True, *args, **kwargs):
+#     """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
+
+#     Parameters
+#     ----------
+#     dataset : hdf5 dataset
+#         imageio-compatible name inputs to be opened as multi-images
+#     slice_length : int
+#         number of 2d arrays from mimgfns included per chunk
+#     pad : bool, optional
+#         whether to extend final returned chunks with zeros
+
+#     Yields
+#     ------
+#     arr : numpy.ndarray
+#         3D numpy array representing a consecutive chunk of 2D arrays
+#     """
+#     for chunk in iterate_chunks(dataset, block_size):#,*args,**kwargs):
+#         arr = numpy.asarray(chunk)
+#         if pad:
+#             if any([arr.shape[k] != block_size[k] for k in range(3)]):
+#                 newarr = numpy.zeros(block_size,
+#                                       dtype=arr.dtype)
+#                 newarr[:arr.shape[0], :arr.shape[1], :arr.shape[2]] = arr[:, :, :]
+#                 yield newarr
+#             else:
+#                 yield arr
+#         else:
+#             yield arr
+
+
+def iterate_numpy_blocks_from_dataset(
+        dataset, maxlvl, nblocks, block_size=None, pad=True, *args, **kwargs):
+    """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
+
+    Parameters
+    ----------
+    dataset : hdf5 dataset
+        imageio-compatible name inputs to be opened as multi-images
+    nblocks : tuple[int]
+        number of 2d arrays from mimgfns included per chunk
+    pad : bool, optional
+        whether to extend final returned chunks with zeros
+
+    Yields
+    ------
+    arr : numpy.ndarray
+        3D numpy array representing a consecutive chunk of 2D arrays
+    """
+    for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
+        chunk_tuple = numpy.unravel_index(i,nblocks)
+        block_start = [chunk_tuple[k]*block_size[k] for k in range(3)]
+        block_end = [block_start[k] + block_size[k] for k in range(3)]
+        arr = dataset[block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]]
+        if pad:
+            if any([arr.shape[k] != block_size[k] for k in range(3)]):
+                newarr = numpy.zeros(block_size,
+                                      dtype=arr.dtype)
+                newarr[:arr.shape[0], :arr.shape[1], :arr.shape[2]] = arr[:, :, :]
+                yield newarr
+            else:
+                yield arr
+        else:
+            yield arr
+
+
+def iterate_mip_levels_from_dataset(
+        dataset, lvl, maxlvl, nblocks, block_size, downsample_factor,
+        downsample_method=None, lvl_to_mip_kwargs=None,
+        interleaved_channels=1, channel=0, deskew_kwargs={}):
+    """recursively generate MIPmap levels from an iterator of blocks
+
+    Parameters
+    ----------
+    dataset : hdf5 dataset
+        imageio-compatible name inputs to be opened as multi-images
+    lvl : int
+        integer mip level to generate
+    block_size : int
+        number of 2D arrays in chunk to process for a chunk at lvl
+    slice_length : int
+        number of 2D arrays to gather from tiff stacks
+    downsample_factor : tuple of int
+        integer downsampling factor for MIP levels
+    downsample_method : str, optional
+        downsampling method for
+        acpreprocessing.utils.convert.downsample_stack_volume
+    lvl_to_mip_kwargs :  dict, optional
+        mapping of MIP level to kwargs used in MIPmap generation
+    interleaved_channels : int
+        number of channels interleaved in the tiff files (default 1)
+    channel : int, optional
+        channel from which interleaved data should be read (default 0)
+    deskew_kwargs : dict, optional
+        parameters for pixel shifting deskew
+
+    Yields
+    ------
+    ma : acpreprocessing.stitching_modules.convert_to_n5.tiff_to_n5.MipArray
+        object describing chunked array, MIP level of origin, and chunk indices
+    """
+    lvl_to_mip_kwargs = ({} if lvl_to_mip_kwargs is None
+                          else lvl_to_mip_kwargs)
+    mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
+    block_index = 0
+    if lvl > 0:
+        for ma in iterate_mip_levels_from_dataset(
+                dataset, lvl-1, maxlvl, nblocks, block_size,
+                downsample_factor, downsample_method,
+                lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
+                channel=channel, deskew_kwargs=deskew_kwargs):
+            chunk = ma.array
+            # throw array up for further processing
+            yield ma
+            # ignore if not parent resolution
+            if ma.lvl != lvl-1:
+                continue
+            
+            temp_arr = (
+                downsample_array(
+                    chunk, downsample_factor, dtype=chunk.dtype,
+                    method=downsample_method, **mip_kwargs))
+            chunk_start = (ma.start[k]/(downsample_factor[k]**lvl) for k in range(3))
+            chunk_end = (chunk_start[k] + temp_arr.shape[k] for k in range(3))
+            yield MIPArray(lvl, temp_arr, chunk_start, chunk_end)
+    else:
+        # get level 0 chunks
+        # block_size is the number of slices to read from tiffs
+        for block in iterate_numpy_blocks_from_dataset(
+                dataset, maxlvl, nblocks, pad=False,
+                channel=channel):
+            # deskew level 0 chunk
+            # if deskew_kwargs:
+            #     chunk = numpy.transpose(psd.deskew_block(
+            #         chunk, chunk_index, **deskew_kwargs), (2, 1, 0))
+            block_tuple = numpy.unravel_index(block_index,nblocks)
+            block_start = (block_tuple[k]*block_size[k] for k in range(3))
+            block_end = (block_start[k] + block.shape[k] for k in range(3))
+            yield MIPArray(lvl, block, block_start, block_end)
+            block_index += 1
 
 
 def write_ims_to_zarr(
@@ -84,8 +267,7 @@ def write_ims_to_zarr(
             joined_shapes, deskew_options['deskew_stride'], **deskew_kwargs)
         print("deskewed shape:" + str(joined_shapes))
     else:
-        block_size = chunk_size[2]
-        slice_length = block_size
+        block_size = chunk_size[2:]
         deskew_kwargs = {}
 
     # TODO DESKEW: does this generally work regardless of skew?
@@ -136,16 +318,20 @@ def write_ims_to_zarr(
             dsfactors = [int(i)**mip_lvl for i in mip_dsfactor]
             mip_ds[mip_lvl] = ds_lvl
             scales.append(dsfactors)
+        
+        nblocks = (int(numpy.ceil(joined_shapes[k]/block_size[k])) for k in range(3))
+        print(str(nblocks) + "number of chunks per axis")
+        print(str(g[0].nchunks) + " chunk number sanity check")
 
         with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as e:
             futs = []
             for miparr in iterate_mip_levels_from_dataset(
-                    dataset, max_mip, block_size, slice_length, mip_dsfactor,
+                    dataset, max_mip, max_mip, nblocks, block_size, mip_dsfactor,
                     lvl_to_mip_kwargs=lvl_to_mip_kwargs,
                     interleaved_channels=interleaved_channels,
                     channel=channel, deskew_kwargs=deskew_kwargs):
                 futs.append(e.submit(
-                    dswrite_chunk, mip_ds[miparr.lvl],
+                    dswrite_block, mip_ds[miparr.lvl],
                     miparr.start, miparr.end, miparr.array))
             for fut in concurrent.futures.as_completed(futs):
                 _ = fut.result()

From 5995d636896f511a34e6dff6b6a2ea8fe317e9b1 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 31 Jul 2024 10:41:11 -0700
Subject: [PATCH 022/115] fixing block conversion

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py     | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index fcc3d6e..5e70262 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -185,8 +185,8 @@ def iterate_mip_levels_from_dataset(
                 downsample_array(
                     chunk, downsample_factor, dtype=chunk.dtype,
                     method=downsample_method, **mip_kwargs))
-            chunk_start = (ma.start[k]/(downsample_factor[k]**lvl) for k in range(3))
-            chunk_end = (chunk_start[k] + temp_arr.shape[k] for k in range(3))
+            chunk_start = [ma.start[k]/(downsample_factor[k]**lvl) for k in range(3)]
+            chunk_end = [chunk_start[k] + temp_arr.shape[k] for k in range(3)]
             yield MIPArray(lvl, temp_arr, chunk_start, chunk_end)
     else:
         # get level 0 chunks
@@ -199,8 +199,8 @@ def iterate_mip_levels_from_dataset(
             #     chunk = numpy.transpose(psd.deskew_block(
             #         chunk, chunk_index, **deskew_kwargs), (2, 1, 0))
             block_tuple = numpy.unravel_index(block_index,nblocks)
-            block_start = (block_tuple[k]*block_size[k] for k in range(3))
-            block_end = (block_start[k] + block.shape[k] for k in range(3))
+            block_start = [block_tuple[k]*block_size[k] for k in range(3)]
+            block_end = [block_start[k] + block.shape[k] for k in range(3)]
             yield MIPArray(lvl, block, block_start, block_end)
             block_index += 1
 
@@ -319,7 +319,7 @@ def write_ims_to_zarr(
             mip_ds[mip_lvl] = ds_lvl
             scales.append(dsfactors)
         
-        nblocks = (int(numpy.ceil(joined_shapes[k]/block_size[k])) for k in range(3))
+        nblocks = [int(numpy.ceil(joined_shapes[k]/block_size[k])) for k in range(3)]
         print(str(nblocks) + "number of chunks per axis")
         print(str(g[0].nchunks) + " chunk number sanity check")
 

From e93b426133f6c335fa83086a7847b4d987b9f3cc Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 31 Jul 2024 10:43:19 -0700
Subject: [PATCH 023/115] fixing block conversion

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 5e70262..d74e3d4 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -113,7 +113,8 @@ def iterate_numpy_blocks_from_dataset(
         3D numpy array representing a consecutive chunk of 2D arrays
     """
     for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
-        chunk_tuple = numpy.unravel_index(i,nblocks)
+        chunk_tuple = numpy.unravel_index(i,tuple(nblocks))
+        print(str(chunk_tuple))
         block_start = [chunk_tuple[k]*block_size[k] for k in range(3)]
         block_end = [block_start[k] + block_size[k] for k in range(3)]
         arr = dataset[block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]]

From 1568b62eea5eb612d76e8d68352d8d5b6e8bd576 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 31 Jul 2024 10:45:17 -0700
Subject: [PATCH 024/115] fixing block conversion

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index d74e3d4..813adff 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -193,7 +193,7 @@ def iterate_mip_levels_from_dataset(
         # get level 0 chunks
         # block_size is the number of slices to read from tiffs
         for block in iterate_numpy_blocks_from_dataset(
-                dataset, maxlvl, nblocks, pad=False,
+                dataset, maxlvl, nblocks, block_size=block_size, pad=False,
                 channel=channel):
             # deskew level 0 chunk
             # if deskew_kwargs:

From 24b56cdaac669c1f02ae5d5bf3e269afaa7215a1 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 31 Jul 2024 19:17:00 -0700
Subject: [PATCH 025/115] fixing block conversion

---
 .../convert_to_n5/ims_to_ngff.py              | 23 +++++++++++--------
 .../convert_to_n5/psdeskew.py                 |  4 ++--
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 813adff..73f6127 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -26,8 +26,8 @@
 class MIPArray:
     lvl: int
     array: numpy.ndarray
-    start: tuple # tuple of indices (int)
-    end: tuple # tuple of indices (int)
+    start: tuple # list of indices (int)
+    end: tuple # list of indices (int)
 
 
 def dswrite_block(ds, start, end, arr, silent_overflow=True):
@@ -95,7 +95,7 @@ def dswrite_block(ds, start, end, arr, silent_overflow=True):
 
 
 def iterate_numpy_blocks_from_dataset(
-        dataset, maxlvl, nblocks, block_size=None, pad=True, *args, **kwargs):
+        dataset, maxlvl, nblocks, block_size=None, pad=True, deskew_kwargs={}, *args, **kwargs):
     """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
 
     Parameters
@@ -114,10 +114,15 @@ def iterate_numpy_blocks_from_dataset(
     """
     for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nblocks))
-        print(str(chunk_tuple))
+        #print(str(chunk_tuple))
         block_start = [chunk_tuple[k]*block_size[k] for k in range(3)]
         block_end = [block_start[k] + block_size[k] for k in range(3)]
-        arr = dataset[block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]]
+        if deskew_kwargs:
+            if deskew_kwargs["deskew_method"] == "ps":
+                pass
+                #arr = psd.get_deskewed_block(dataset,xi,yi,zi,**deskew_kwargs)
+        else:
+            arr = dataset[block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]]
         if pad:
             if any([arr.shape[k] != block_size[k] for k in range(3)]):
                 newarr = numpy.zeros(block_size,
@@ -186,8 +191,8 @@ def iterate_mip_levels_from_dataset(
                 downsample_array(
                     chunk, downsample_factor, dtype=chunk.dtype,
                     method=downsample_method, **mip_kwargs))
-            chunk_start = [ma.start[k]/(downsample_factor[k]**lvl) for k in range(3)]
-            chunk_end = [chunk_start[k] + temp_arr.shape[k] for k in range(3)]
+            chunk_start = tuple([int(ma.start[k]/(downsample_factor[k]**lvl)) for k in range(3)])
+            chunk_end = tuple([chunk_start[k] + temp_arr.shape[k] for k in range(3)])
             yield MIPArray(lvl, temp_arr, chunk_start, chunk_end)
     else:
         # get level 0 chunks
@@ -200,8 +205,8 @@ def iterate_mip_levels_from_dataset(
             #     chunk = numpy.transpose(psd.deskew_block(
             #         chunk, chunk_index, **deskew_kwargs), (2, 1, 0))
             block_tuple = numpy.unravel_index(block_index,nblocks)
-            block_start = [block_tuple[k]*block_size[k] for k in range(3)]
-            block_end = [block_start[k] + block.shape[k] for k in range(3)]
+            block_start = tuple([block_tuple[k]*block_size[k] for k in range(3)])
+            block_end = tuple([block_start[k] + block.shape[k] for k in range(3)])
             yield MIPArray(lvl, block, block_start, block_end)
             block_index += 1
 
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index dffbf1d..5e347ba 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -86,9 +86,9 @@ def calculate_skewed_indices(xi,yi,zi,s):
     return xs,ys,zs
 
 
-def get_deskewed_block(dataset,xi,yi,zi,**kwargs):
+def get_deskewed_block(dataset,xi,yi,zi,stride,**kwargs):
     sdims = dataset.shape
-    xs,ys,zs = calculate_skewed_indices(xi,yi,zi)
+    xs,ys,zs = calculate_skewed_indices(xi,yi,zi,stride)
     fli = np.ravel_multi_index((xs,ys,zs),sdims).flatten()
     fldata = dataset[fli]
     block = fldata.reshape((len(xi),len(yi),len(zi)))

From 075520e694f6ce4e413a005b70fcca9a40ab4ecf Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 31 Jul 2024 19:52:05 -0700
Subject: [PATCH 026/115] fixing block conversion

---
 .../convert_to_n5/ims_to_ngff.py              | 45 ++++---------------
 .../convert_to_n5/psdeskew.py                 |  4 +-
 2 files changed, 10 insertions(+), 39 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 73f6127..0d1edee 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -62,38 +62,6 @@ def dswrite_block(ds, start, end, arr, silent_overflow=True):
         ds[start[0]:end[0], start[1]:end[1], start[2]:end[2]] = arr[:(end[0] - start[0]), :(end[1] - start[1]), :(end[2] - start[2])]
 
 
-# def iterate_numpy_chunks_from_dataset(
-#         dataset, block_size=None, pad=True, *args, **kwargs):
-#     """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
-
-#     Parameters
-#     ----------
-#     dataset : hdf5 dataset
-#         imageio-compatible name inputs to be opened as multi-images
-#     slice_length : int
-#         number of 2d arrays from mimgfns included per chunk
-#     pad : bool, optional
-#         whether to extend final returned chunks with zeros
-
-#     Yields
-#     ------
-#     arr : numpy.ndarray
-#         3D numpy array representing a consecutive chunk of 2D arrays
-#     """
-#     for chunk in iterate_chunks(dataset, block_size):#,*args,**kwargs):
-#         arr = numpy.asarray(chunk)
-#         if pad:
-#             if any([arr.shape[k] != block_size[k] for k in range(3)]):
-#                 newarr = numpy.zeros(block_size,
-#                                       dtype=arr.dtype)
-#                 newarr[:arr.shape[0], :arr.shape[1], :arr.shape[2]] = arr[:, :, :]
-#                 yield newarr
-#             else:
-#                 yield arr
-#         else:
-#             yield arr
-
-
 def iterate_numpy_blocks_from_dataset(
         dataset, maxlvl, nblocks, block_size=None, pad=True, deskew_kwargs={}, *args, **kwargs):
     """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
@@ -114,13 +82,16 @@ def iterate_numpy_blocks_from_dataset(
     """
     for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nblocks))
-        #print(str(chunk_tuple))
+        if chunk_tuple[2] == 0:
+            print(str(chunk_tuple))
         block_start = [chunk_tuple[k]*block_size[k] for k in range(3)]
         block_end = [block_start[k] + block_size[k] for k in range(3)]
         if deskew_kwargs:
             if deskew_kwargs["deskew_method"] == "ps":
-                pass
-                #arr = psd.get_deskewed_block(dataset,xi,yi,zi,**deskew_kwargs)
+                zi = numpy.arange(block_start[0],block_end[0])
+                yi = numpy.arange(block_start[1],block_end[1])
+                xi = numpy.arange(block_start[2],block_end[2])
+                arr = psd.get_deskewed_block(dataset,zi,yi,xi,**deskew_kwargs)
         else:
             arr = dataset[block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]]
         if pad:
@@ -273,7 +244,7 @@ def write_ims_to_zarr(
             joined_shapes, deskew_options['deskew_stride'], **deskew_kwargs)
         print("deskewed shape:" + str(joined_shapes))
     else:
-        block_size = chunk_size[2:]
+        block_size = [8*sz for sz in chunk_size[2:]]
         deskew_kwargs = {}
 
     # TODO DESKEW: does this generally work regardless of skew?
@@ -326,7 +297,7 @@ def write_ims_to_zarr(
             scales.append(dsfactors)
         
         nblocks = [int(numpy.ceil(joined_shapes[k]/block_size[k])) for k in range(3)]
-        print(str(nblocks) + "number of chunks per axis")
+        print(str(nblocks) + " number of chunks per axis")
         print(str(g[0].nchunks) + " chunk number sanity check")
 
         with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as e:
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 5e347ba..be200d2 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -79,14 +79,14 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_tr
     return kwargs
 
 
-def calculate_skewed_indices(xi,yi,zi,s):
+def calculate_skewed_indices(zi,yi,xi,s):
     xs = s*zi + xi % s
     ys = yi
     zs = xi // s - zi
     return xs,ys,zs
 
 
-def get_deskewed_block(dataset,xi,yi,zi,stride,**kwargs):
+def get_deskewed_block(dataset,zi,yi,xi,stride,**kwargs):
     sdims = dataset.shape
     xs,ys,zs = calculate_skewed_indices(xi,yi,zi,stride)
     fli = np.ravel_multi_index((xs,ys,zs),sdims).flatten()

From 9707af8a1172e2c688f60bcc945bbbfe7454d84a Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 31 Jul 2024 20:36:16 -0700
Subject: [PATCH 027/115] fixing block conversion

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 0d1edee..4d157d3 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -48,6 +48,8 @@ def dswrite_block(ds, start, end, arr, silent_overflow=True):
         whether to shrink the end index to match the
         shape of ds (default: True)
     """
+    start = list(start)
+    end = list(end)
     if len(ds.shape) == 5:  # dataset dimensions should be 3 or 5
         for i in range(3):    
             if end[i] >= ds.shape[2+i] and silent_overflow:

From 6ebddce6784b0a42f4b5a8bfaa2d35001a2f70bc Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Thu, 1 Aug 2024 22:19:54 -0700
Subject: [PATCH 028/115] fixing bugs

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py    | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 4d157d3..8bd8b2c 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -160,12 +160,11 @@ def iterate_mip_levels_from_dataset(
             if ma.lvl != lvl-1:
                 continue
             
-            temp_arr = (
-                downsample_array(
+            temp_arr = (downsample_array(
                     chunk, downsample_factor, dtype=chunk.dtype,
                     method=downsample_method, **mip_kwargs))
-            chunk_start = tuple([int(ma.start[k]/(downsample_factor[k]**lvl)) for k in range(3)])
-            chunk_end = tuple([chunk_start[k] + temp_arr.shape[k] for k in range(3)])
+            chunk_start = tuple(int(ma.start[k]/downsample_factor[k]) for k in range(3))
+            chunk_end = tuple(chunk_start[k] + temp_arr.shape[k] for k in range(3))
             yield MIPArray(lvl, temp_arr, chunk_start, chunk_end)
     else:
         # get level 0 chunks
@@ -178,8 +177,8 @@ def iterate_mip_levels_from_dataset(
             #     chunk = numpy.transpose(psd.deskew_block(
             #         chunk, chunk_index, **deskew_kwargs), (2, 1, 0))
             block_tuple = numpy.unravel_index(block_index,nblocks)
-            block_start = tuple([block_tuple[k]*block_size[k] for k in range(3)])
-            block_end = tuple([block_start[k] + block.shape[k] for k in range(3)])
+            block_start = tuple(block_tuple[k]*block_size[k] for k in range(3))
+            block_end = tuple(block_start[k] + block.shape[k] for k in range(3))
             yield MIPArray(lvl, block, block_start, block_end)
             block_index += 1
 

From 52cbae2cfddefb953d9d6a18a2522f7890c82bda Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Fri, 2 Aug 2024 11:25:54 -0700
Subject: [PATCH 029/115] testing

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 8bd8b2c..7637778 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -308,6 +308,7 @@ def write_ims_to_zarr(
                     lvl_to_mip_kwargs=lvl_to_mip_kwargs,
                     interleaved_channels=interleaved_channels,
                     channel=channel, deskew_kwargs=deskew_kwargs):
+                print(str(miparr.lvl))
                 futs.append(e.submit(
                     dswrite_block, mip_ds[miparr.lvl],
                     miparr.start, miparr.end, miparr.array))

From adc5dcec6376c3f72840585998e7c516de9aeef1 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Sun, 4 Aug 2024 21:27:04 -0700
Subject: [PATCH 030/115] threading multilevel block write

---
 .../convert_to_n5/ims_to_ngff.py               | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 7637778..05374cb 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -64,6 +64,11 @@ def dswrite_block(ds, start, end, arr, silent_overflow=True):
         ds[start[0]:end[0], start[1]:end[1], start[2]:end[2]] = arr[:(end[0] - start[0]), :(end[1] - start[1]), :(end[2] - start[2])]
 
 
+def write_mips(zgrp,miparrs):
+    for miparr in miparrs:
+        dswrite_block(ds=zgrp[miparr.lvl],start=miparr.start,end=miparr.end,arr=miparr.array)
+
+
 def iterate_numpy_blocks_from_dataset(
         dataset, maxlvl, nblocks, block_size=None, pad=True, deskew_kwargs={}, *args, **kwargs):
     """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
@@ -303,15 +308,20 @@ def write_ims_to_zarr(
 
         with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as e:
             futs = []
+            mips = []
             for miparr in iterate_mip_levels_from_dataset(
                     dataset, max_mip, max_mip, nblocks, block_size, mip_dsfactor,
                     lvl_to_mip_kwargs=lvl_to_mip_kwargs,
                     interleaved_channels=interleaved_channels,
                     channel=channel, deskew_kwargs=deskew_kwargs):
-                print(str(miparr.lvl))
-                futs.append(e.submit(
-                    dswrite_block, mip_ds[miparr.lvl],
-                    miparr.start, miparr.end, miparr.array))
+                mips.append(miparr)
+                if miparr.lvl == max_mip:
+                    # futs.append(e.submit(
+                    #     dswrite_block, mip_ds[miparr.lvl],
+                    #     miparr.start, miparr.end, miparr.array))
+                    futs.append(e.submit(
+                        write_mips, mip_ds, mips))
+                    mips = []
             for fut in concurrent.futures.as_completed(futs):
                 _ = fut.result()
     print("conversion complete, closing file")

From 291f3330a08c6d0bf3c1363975efa8aa83d02d45 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Mon, 5 Aug 2024 18:59:48 -0700
Subject: [PATCH 031/115] pixel shifting block deskew

---
 .../convert_to_n5/ims_to_ngff.py              | 35 ++++++++++-------
 .../convert_to_n5/psdeskew.py                 | 38 +++++++++++++------
 2 files changed, 48 insertions(+), 25 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 05374cb..d13711f 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -93,12 +93,15 @@ def iterate_numpy_blocks_from_dataset(
             print(str(chunk_tuple))
         block_start = [chunk_tuple[k]*block_size[k] for k in range(3)]
         block_end = [block_start[k] + block_size[k] for k in range(3)]
+        # deskew level 0 data blocks
         if deskew_kwargs:
             if deskew_kwargs["deskew_method"] == "ps":
-                zi = numpy.arange(block_start[0],block_end[0])
-                yi = numpy.arange(block_start[1],block_end[1])
-                xi = numpy.arange(block_start[2],block_end[2])
-                arr = psd.get_deskewed_block(dataset,zi,yi,xi,**deskew_kwargs)
+                stride = deskew_kwargs["deskew_stride"]
+                arr = psd.get_deskewed_block(blockdims=block_size,
+                                             dataset=dataset,
+                                             start=block_start,
+                                             end=block_end,
+                                             stride=stride)
         else:
             arr = dataset[block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]]
         if pad:
@@ -176,11 +179,8 @@ def iterate_mip_levels_from_dataset(
         # block_size is the number of slices to read from tiffs
         for block in iterate_numpy_blocks_from_dataset(
                 dataset, maxlvl, nblocks, block_size=block_size, pad=False,
+                deskew_kwargs=deskew_kwargs,
                 channel=channel):
-            # deskew level 0 chunk
-            # if deskew_kwargs:
-            #     chunk = numpy.transpose(psd.deskew_block(
-            #         chunk, chunk_index, **deskew_kwargs), (2, 1, 0))
             block_tuple = numpy.unravel_index(block_index,nblocks)
             block_start = tuple(block_tuple[k]*block_size[k] for k in range(3))
             block_end = tuple(block_start[k] + block.shape[k] for k in range(3))
@@ -241,14 +241,21 @@ def write_ims_to_zarr(
     print("ims_to_ngff dataset shape:" + str(joined_shapes))
 
     if deskew_options and deskew_options["deskew_method"] == "ps":
-        block_size = chunk_size[2]
-        slice_length = int(chunk_size[2]/deskew_options['deskew_stride'])
-        deskew_kwargs = psd.psdeskew_kwargs(skew_dims_zyx=(slice_length, joined_shapes[1], joined_shapes[2]),
-                                            **deskew_options
-                                            )
+        # block_size = chunk_size[2]
+        # slice_length = int(chunk_size[2]/deskew_options['deskew_stride'])
+        # deskew_kwargs = psd.psdeskew_kwargs(skew_dims_zyx=(slice_length, joined_shapes[1], joined_shapes[2]),
+        #                                     **deskew_options
+        #    )
+        print(deskew_options)
+        if not deskew_options["deskew_stride"] is None:
+            stride = deskew_options["deskew_stride"]
+        else:
+            stride = 1
+        block_size = [8*sz for sz in chunk_size[2:]]
         joined_shapes = psd.reshape_joined_shapes(
-            joined_shapes, deskew_options['deskew_stride'], **deskew_kwargs)
+            joined_shapes, stride, block_size, transpose=False)
         print("deskewed shape:" + str(joined_shapes))
+        deskew_kwargs = deskew_options
     else:
         block_size = [8*sz for sz in chunk_size[2:]]
         deskew_kwargs = {}
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index be200d2..2cffd02 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -80,19 +80,31 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_tr
 
 
 def calculate_skewed_indices(zi,yi,xi,s):
+    # convert input block voxel indices into skewed data space
+    # edge blocks may have negative or out-of-bounds skewed indices
     xs = s*zi + xi % s
     ys = yi
     zs = xi // s - zi
-    return xs,ys,zs
+    return zs,ys,xs
 
 
-def get_deskewed_block(dataset,zi,yi,xi,stride,**kwargs):
+def get_deskewed_block(blockdims,dataset,start,end,stride):
+    # create output block and get flattened indices
+    blockdata = np.zeros(blockdims,dtype=dataset.dtype)
+    zb,yb,xb = np.meshgrid(*[range(d) for d in blockdims],indexing="ij")
+    fb = np.ravel_multi_index((zb,yb,xb),blockdims)
+    # get indices of voxel data for input dataset
     sdims = dataset.shape
-    xs,ys,zs = calculate_skewed_indices(xi,yi,zi,stride)
-    fli = np.ravel_multi_index((xs,ys,zs),sdims).flatten()
-    fldata = dataset[fli]
-    block = fldata.reshape((len(xi),len(yi),len(zi)))
-    return block
+    zi,yi,xi = np.meshgrid(*[range(s,e) for s,e in zip(start,end)],indexing="ij")
+    zs,ys,xs = calculate_skewed_indices(zi,yi,xi,stride)
+    fi = np.ravel_multi_index((zs,ys,xs),sdims,mode='clip').flatten()
+    # filter out-of-bounds voxels
+    r = fi > 0 and fi < np.prod(sdims)-1
+    fb = fb[r]
+    fi = fi[r]
+    # assign input to output
+    blockdata[fb] = dataset[fi]
+    return blockdata
 
 
 def deskew_block(blockData, n, dsi, si, slice1d, blockdims, subblocks, flip, transpose, dtype, chunklength, *args, **kwargs):
@@ -155,7 +167,7 @@ def deskew_block(blockData, n, dsi, si, slice1d, blockdims, subblocks, flip, tra
     return block3d
 
 
-def reshape_joined_shapes(joined_shapes, stride, blockdims, *args, **kwargs):
+def reshape_joined_shapes(joined_shapes, stride, blockdims, transpose=True, **kwargs):
     """get dimensions of deskewed joined shapes from skewed joined shapes
 
     Parameters
@@ -172,7 +184,11 @@ def reshape_joined_shapes(joined_shapes, stride, blockdims, *args, **kwargs):
     deskewed_shape : tuple of int
         shape of deskewed 3D array represented by joined_shapes
     """
-    deskewed_shape = (int(np.ceil(joined_shapes[0]/(blockdims[2]/stride))*blockdims[2]),
-                      blockdims[1],
-                      blockdims[0])
+    if transpose:
+        axes = (2,1,0)
+    else:
+        axes = (0,1,2)
+    deskewed_shape = (int(np.ceil(joined_shapes[0]/(blockdims[axes[0]]/stride))*blockdims[axes[0]]),
+                      blockdims[axes[1]],
+                      blockdims[axes[2]])
     return deskewed_shape

From dde31f0223085d35604656bba64f559b957e6bf3 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Mon, 5 Aug 2024 19:06:08 -0700
Subject: [PATCH 032/115] bug fix

---
 acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 2cffd02..599442a 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -189,6 +189,6 @@ def reshape_joined_shapes(joined_shapes, stride, blockdims, transpose=True, **kw
     else:
         axes = (0,1,2)
     deskewed_shape = (int(np.ceil(joined_shapes[0]/(blockdims[axes[0]]/stride))*blockdims[axes[0]]),
-                      blockdims[axes[1]],
-                      blockdims[axes[2]])
+                      joined_shapes[axes[1]],
+                      joined_shapes[axes[2]])
     return deskewed_shape

From 34c7537f167b3b1ee2e9fb322b631ecc2844de40 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 6 Aug 2024 09:01:18 -0700
Subject: [PATCH 033/115] new ps deskew approach

---
 .../convert_to_n5/ims_to_ngff.py              | 58 ++++++++++++-------
 .../convert_to_n5/psdeskew.py                 |  5 +-
 2 files changed, 39 insertions(+), 24 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index d13711f..67e3c1b 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -51,7 +51,7 @@ def dswrite_block(ds, start, end, arr, silent_overflow=True):
     start = list(start)
     end = list(end)
     if len(ds.shape) == 5:  # dataset dimensions should be 3 or 5
-        for i in range(3):    
+        for i in range(3):
             if end[i] >= ds.shape[2+i] and silent_overflow:
                 end[i] = ds.shape[2+i]
         #if end > start:
@@ -68,9 +68,9 @@ def write_mips(zgrp,miparrs):
     for miparr in miparrs:
         dswrite_block(ds=zgrp[miparr.lvl],start=miparr.start,end=miparr.end,arr=miparr.array)
 
-
+# HERE : NEED TO UNTANGLE MISMATCH BETWEEN BLOCK (deskewed) and CHUNK (skewed) size
 def iterate_numpy_blocks_from_dataset(
-        dataset, maxlvl, nblocks, block_size=None, pad=True, deskew_kwargs={}, *args, **kwargs):
+        dataset, nblocks, block_size=None, pad=True, deskew_kwargs={}, *args, **kwargs):
     """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
 
     Parameters
@@ -87,22 +87,39 @@ def iterate_numpy_blocks_from_dataset(
     arr : numpy.ndarray
         3D numpy array representing a consecutive chunk of 2D arrays
     """
+    # if deskew_kwargs and deskew_kwargs["deskew_method"] == "ps":
+    #     # create output block and get flattened indices
+    #     zb,yb,xb = numpy.meshgrid(*[range(d) for d in block_size],indexing="ij")
+    #     fb = numpy.ravel_multi_index((zb,yb,xb),block_size)
+    if deskew_kwargs:
+        chunk_size = (deskew_kwargs["chunklength"],block_size[1],block_size[2]*deskew_kwargs["stride"])
     for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
-        chunk_tuple = numpy.unravel_index(i,tuple(nblocks))
-        if chunk_tuple[2] == 0:
+        chunk_tuple = numpy.unravel_index(i,tuple(nblocks),order='F')
+        if chunk_tuple[0] == 0:
             print(str(chunk_tuple))
-        block_start = [chunk_tuple[k]*block_size[k] for k in range(3)]
-        block_end = [block_start[k] + block_size[k] for k in range(3)]
         # deskew level 0 data blocks
         if deskew_kwargs:
-            if deskew_kwargs["deskew_method"] == "ps":
-                stride = deskew_kwargs["deskew_stride"]
-                arr = psd.get_deskewed_block(blockdims=block_size,
-                                             dataset=dataset,
-                                             start=block_start,
-                                             end=block_end,
-                                             stride=stride)
+            if chunk_tuple[0] == 0 and i > 0:
+                chunk_index = 0
+                deskew_kwargs["slice1d"] *= 0
+            chunk_start = [t*s for t,s in zip(chunk_tuple,chunk_size)]
+            chunk_end = [st+s for st,s in zip(chunk_start,chunk_size)]
+            arr = numpy.transpose(psd.deskew_block(
+                dataset[chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]],
+                chunk_index,
+                **deskew_kwargs), (2, 1, 0))
+            chunk_index += 1
+            # arr = numpy.zeros(block_size,dtype=dataset.dtype)
+            # if deskew_kwargs["deskew_method"] == "ps":
+            #     stride = deskew_kwargs["deskew_stride"]
+            #     arr = psd.get_deskewed_block(blockdims=block_size,
+            #                                  dataset=dataset,
+            #                                  start=block_start,
+            #                                  end=block_end,
+            #                                  stride=stride)
         else:
+            block_start = [chunk_tuple[k]*block_size[k] for k in range(3)]
+            block_end = [block_start[k] + block_size[k] for k in range(3)]
             arr = dataset[block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]]
         if pad:
             if any([arr.shape[k] != block_size[k] for k in range(3)]):
@@ -178,10 +195,10 @@ def iterate_mip_levels_from_dataset(
         # get level 0 chunks
         # block_size is the number of slices to read from tiffs
         for block in iterate_numpy_blocks_from_dataset(
-                dataset, maxlvl, nblocks, block_size=block_size, pad=False,
+                dataset, nblocks, block_size=block_size, pad=False,
                 deskew_kwargs=deskew_kwargs,
                 channel=channel):
-            block_tuple = numpy.unravel_index(block_index,nblocks)
+            block_tuple = numpy.unravel_index(block_index,nblocks,order='F')
             block_start = tuple(block_tuple[k]*block_size[k] for k in range(3))
             block_end = tuple(block_start[k] + block.shape[k] for k in range(3))
             yield MIPArray(lvl, block, block_start, block_end)
@@ -241,11 +258,6 @@ def write_ims_to_zarr(
     print("ims_to_ngff dataset shape:" + str(joined_shapes))
 
     if deskew_options and deskew_options["deskew_method"] == "ps":
-        # block_size = chunk_size[2]
-        # slice_length = int(chunk_size[2]/deskew_options['deskew_stride'])
-        # deskew_kwargs = psd.psdeskew_kwargs(skew_dims_zyx=(slice_length, joined_shapes[1], joined_shapes[2]),
-        #                                     **deskew_options
-        #    )
         print(deskew_options)
         if not deskew_options["deskew_stride"] is None:
             stride = deskew_options["deskew_stride"]
@@ -255,7 +267,9 @@ def write_ims_to_zarr(
         joined_shapes = psd.reshape_joined_shapes(
             joined_shapes, stride, block_size, transpose=False)
         print("deskewed shape:" + str(joined_shapes))
-        deskew_kwargs = deskew_options
+        slice_length = int(block_size[0]/stride)
+        deskew_kwargs = psd.psdeskew_kwargs(skew_dims_zyx=(slice_length, block_size[1], block_size[2]*stride),
+                                            **deskew_options)
     else:
         block_size = [8*sz for sz in chunk_size[2:]]
         deskew_kwargs = {}
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 599442a..5972a18 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -74,7 +74,8 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_tr
               'flip': deskew_flip,
               'transpose': deskew_transpose,
               'dtype': dtype,
-              'chunklength': blockx
+              'chunklength': blockx,
+              'stride': stride
               }
     return kwargs
 
@@ -99,7 +100,7 @@ def get_deskewed_block(blockdims,dataset,start,end,stride):
     zs,ys,xs = calculate_skewed_indices(zi,yi,xi,stride)
     fi = np.ravel_multi_index((zs,ys,xs),sdims,mode='clip').flatten()
     # filter out-of-bounds voxels
-    r = fi > 0 and fi < np.prod(sdims)-1
+    r = (fi > 0) & (fi < np.prod(sdims)-1)
     fb = fb[r]
     fi = fi[r]
     # assign input to output

From 1ebff06b357135ed763b0c642f6b594188a3047e Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 6 Aug 2024 09:22:27 -0700
Subject: [PATCH 034/115] bug fixing

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py         | 4 ++--
 .../stitching_modules/convert_to_n5/psdeskew.py            | 7 +++++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 67e3c1b..b5af3ec 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -99,7 +99,7 @@ def iterate_numpy_blocks_from_dataset(
             print(str(chunk_tuple))
         # deskew level 0 data blocks
         if deskew_kwargs:
-            if chunk_tuple[0] == 0 and i > 0:
+            if chunk_tuple[0] == 0:
                 chunk_index = 0
                 deskew_kwargs["slice1d"] *= 0
             chunk_start = [t*s for t,s in zip(chunk_tuple,chunk_size)]
@@ -265,7 +265,7 @@ def write_ims_to_zarr(
             stride = 1
         block_size = [8*sz for sz in chunk_size[2:]]
         joined_shapes = psd.reshape_joined_shapes(
-            joined_shapes, stride, block_size, transpose=False)
+            joined_shapes, stride, block_size, transpose=True)
         print("deskewed shape:" + str(joined_shapes))
         slice_length = int(block_size[0]/stride)
         deskew_kwargs = psd.psdeskew_kwargs(skew_dims_zyx=(slice_length, block_size[1], block_size[2]*stride),
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 5972a18..9f66708 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -189,7 +189,10 @@ def reshape_joined_shapes(joined_shapes, stride, blockdims, transpose=True, **kw
         axes = (2,1,0)
     else:
         axes = (0,1,2)
-    deskewed_shape = (int(np.ceil(joined_shapes[0]/(blockdims[axes[0]]/stride))*blockdims[axes[0]]),
+    # deskewed_shape = (int(np.ceil(joined_shapes[axes[0]]/(blockdims[axes[0]]/stride))*blockdims[axes[0]]),
+    #                   joined_shapes[axes[1]],
+    #                   joined_shapes[axes[2]])
+    deskewed_shape = (int(np.ceil(joined_shapes[axes[0]]/(blockdims[axes[0]]/stride))*blockdims[axes[0]]),
                       joined_shapes[axes[1]],
-                      joined_shapes[axes[2]])
+                      int(np.ceil((joined_shapes[axes[0]] + joined_shapes[axes[2]]*stride)/(blockdims[axes[0]]))*blockdims[axes[0]]))
     return deskewed_shape

From aa243a5fa661abd3241c54aea8d9e3f558628066 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 6 Aug 2024 12:13:46 -0700
Subject: [PATCH 035/115] bug fixing

---
 .../convert_to_n5/ims_to_ngff.py              | 35 ++++++++++++-------
 .../convert_to_n5/psdeskew.py                 |  5 +++
 2 files changed, 27 insertions(+), 13 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index b5af3ec..6e46e44 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -70,7 +70,7 @@ def write_mips(zgrp,miparrs):
 
 # HERE : NEED TO UNTANGLE MISMATCH BETWEEN BLOCK (deskewed) and CHUNK (skewed) size
 def iterate_numpy_blocks_from_dataset(
-        dataset, nblocks, block_size=None, pad=True, deskew_kwargs={}, *args, **kwargs):
+        dataset, nblocks, block_size=None, pad=True, deskew_kwargs={}, numberblocks=0, *args, **kwargs):
     """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
 
     Parameters
@@ -93,21 +93,28 @@ def iterate_numpy_blocks_from_dataset(
     #     fb = numpy.ravel_multi_index((zb,yb,xb),block_size)
     if deskew_kwargs:
         chunk_size = (deskew_kwargs["chunklength"],block_size[1],block_size[2]*deskew_kwargs["stride"])
-    for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
+    if numberblocks < 1:
+        numberblocks = numpy.prod(nblocks)
+    for i in range(numberblocks):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nblocks),order='F')
-        if chunk_tuple[0] == 0:
+        if True: #chunk_tuple[0] == 0:
             print(str(chunk_tuple))
         # deskew level 0 data blocks
         if deskew_kwargs:
             if chunk_tuple[0] == 0:
                 chunk_index = 0
                 deskew_kwargs["slice1d"] *= 0
-            chunk_start = [t*s for t,s in zip(chunk_tuple,chunk_size)]
-            chunk_end = [st+s for st,s in zip(chunk_start,chunk_size)]
-            arr = numpy.transpose(psd.deskew_block(
-                dataset[chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]],
-                chunk_index,
-                **deskew_kwargs), (2, 1, 0))
+                first_z,first_slice = psd.calculate_first_chunk(chunk_size=chunk_size,x_index=chunk_tuple[2],stride=deskew_kwargs["stride"])
+            if chunk_tuple[0] < first_z or chunk_tuple[0]*chunk_size[0] + first_slice >= dataset.shape[0]:
+                arr = numpy.zeros(block_size,dtype=dataset.dtype)
+            else:
+                chunk_start = [t*s for t,s in zip(chunk_tuple,chunk_size)]
+                chunk_start[0] += first_slice
+                chunk_end = [st+s for st,s in zip(chunk_start,chunk_size)]
+                arr = numpy.transpose(psd.deskew_block(
+                    dataset[chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]],
+                    chunk_index,
+                    **deskew_kwargs), (2, 1, 0))
             chunk_index += 1
             # arr = numpy.zeros(block_size,dtype=dataset.dtype)
             # if deskew_kwargs["deskew_method"] == "ps":
@@ -136,7 +143,7 @@ def iterate_numpy_blocks_from_dataset(
 def iterate_mip_levels_from_dataset(
         dataset, lvl, maxlvl, nblocks, block_size, downsample_factor,
         downsample_method=None, lvl_to_mip_kwargs=None,
-        interleaved_channels=1, channel=0, deskew_kwargs={}):
+        interleaved_channels=1, channel=0, numchunks=0, deskew_kwargs={}):
     """recursively generate MIPmap levels from an iterator of blocks
 
     Parameters
@@ -197,7 +204,7 @@ def iterate_mip_levels_from_dataset(
         for block in iterate_numpy_blocks_from_dataset(
                 dataset, nblocks, block_size=block_size, pad=False,
                 deskew_kwargs=deskew_kwargs,
-                channel=channel):
+                channel=channel,numberblocks=numchunks):
             block_tuple = numpy.unravel_index(block_index,nblocks,order='F')
             block_start = tuple(block_tuple[k]*block_size[k] for k in range(3))
             block_end = tuple(block_start[k] + block.shape[k] for k in range(3))
@@ -210,7 +217,7 @@ def write_ims_to_zarr(
         mip_dsfactor=(2, 2, 2), chunk_size=(1, 1, 64, 64, 64),
         concurrency=10, slice_concurrency=1,
         compression="raw", dtype="uint16", lvl_to_mip_kwargs=None,
-        interleaved_channels=1, channel=0, deskew_options=None, **kwargs):
+        interleaved_channels=1, channel=0, deskew_options=None, numchunks=0, **kwargs):
     """write a stack represented by an iterator of multi-image files as a zarr
     volume with ome-ngff metadata
 
@@ -334,7 +341,7 @@ def write_ims_to_zarr(
                     dataset, max_mip, max_mip, nblocks, block_size, mip_dsfactor,
                     lvl_to_mip_kwargs=lvl_to_mip_kwargs,
                     interleaved_channels=interleaved_channels,
-                    channel=channel, deskew_kwargs=deskew_kwargs):
+                    channel=channel, numchunks=numchunks, deskew_kwargs=deskew_kwargs):
                 mips.append(miparr)
                 if miparr.lvl == max_mip:
                     # futs.append(e.submit(
@@ -370,6 +377,7 @@ class IMSToNGFFParameters(NGFFGroupGenerationParameters):
     input_file = argschema.fields.Str(required=True)
     interleaved_channels = argschema.fields.Int(required=False, default=1)
     channel = argschema.fields.Int(required=False, default=0)
+    num_chunks = argschema.fields.Int(required=False, default=0)
 
 
 class IMSToZarrInputParameters(argschema.ArgSchema,
@@ -399,6 +407,7 @@ def run(self):
             concurrency=self.args["concurrency"],
             compression=self.args["compression"],
             #lvl_to_mip_kwargs=self.args["lvl_to_mip_kwargs"],
+            numchunks = self.args["num_chunks"],
             deskew_options=deskew_options)
 
 
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 9f66708..a5d6099 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -108,6 +108,11 @@ def get_deskewed_block(blockdims,dataset,start,end,stride):
     return blockdata
 
 
+def calculate_first_chunk(dataset_shape,chunk_size,x_index,stride):
+    first_chunk = x_index*stride // chunk_size[2]
+    first_slice = x_index*stride % chunk_size[2]
+
+
 def deskew_block(blockData, n, dsi, si, slice1d, blockdims, subblocks, flip, transpose, dtype, chunklength, *args, **kwargs):
     """deskew a data chunk in sequence with prior chunks
 

From d7026ea8ebadb6e0641affd6f06e85a8cbbca643 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 6 Aug 2024 12:17:12 -0700
Subject: [PATCH 036/115] bug fixing

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py    | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 6e46e44..810c240 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -104,7 +104,7 @@ def iterate_numpy_blocks_from_dataset(
             if chunk_tuple[0] == 0:
                 chunk_index = 0
                 deskew_kwargs["slice1d"] *= 0
-                first_z,first_slice = psd.calculate_first_chunk(chunk_size=chunk_size,x_index=chunk_tuple[2],stride=deskew_kwargs["stride"])
+                first_z,first_slice = psd.calculate_first_chunk(dataset_shape=dataset.shape,chunk_size=chunk_size,x_index=chunk_tuple[2],stride=deskew_kwargs["stride"])
             if chunk_tuple[0] < first_z or chunk_tuple[0]*chunk_size[0] + first_slice >= dataset.shape[0]:
                 arr = numpy.zeros(block_size,dtype=dataset.dtype)
             else:
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index a5d6099..b06fd0a 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -111,6 +111,7 @@ def get_deskewed_block(blockdims,dataset,start,end,stride):
 def calculate_first_chunk(dataset_shape,chunk_size,x_index,stride):
     first_chunk = x_index*stride // chunk_size[2]
     first_slice = x_index*stride % chunk_size[2]
+    return first_chunk,first_slice
 
 
 def deskew_block(blockData, n, dsi, si, slice1d, blockdims, subblocks, flip, transpose, dtype, chunklength, *args, **kwargs):

From d7663eeb7562aa4e38ccaec07ec109485ae9743c Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 6 Aug 2024 12:22:26 -0700
Subject: [PATCH 037/115] bug fixing

---
 acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index b06fd0a..d741210 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -147,8 +147,8 @@ def deskew_block(blockData, n, dsi, si, slice1d, blockdims, subblocks, flip, tra
     """
     if transpose:
         blockData = blockData.transpose((0,2,1))
-    # subb = subblocks
-    subb = 5
+    subb = subblocks
+    # subb = 5
     block3d = np.zeros(blockdims, dtype=dtype)
     zdim = block3d.shape[0]
     ydim = block3d.shape[1]

From 98ff6b72c9af5face57582bb87c73fd6df70aa4a Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 6 Aug 2024 12:48:16 -0700
Subject: [PATCH 038/115] bug fixing subchunk conversion

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py  | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 810c240..6c4a1b4 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -95,6 +95,8 @@ def iterate_numpy_blocks_from_dataset(
         chunk_size = (deskew_kwargs["chunklength"],block_size[1],block_size[2]*deskew_kwargs["stride"])
     if numberblocks < 1:
         numberblocks = numpy.prod(nblocks)
+    else:
+        numberblocks = numpy.ravel_multi_index((0,numberblocks,0),nblocks,order='F')
     for i in range(numberblocks):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nblocks),order='F')
         if True: #chunk_tuple[0] == 0:
@@ -184,7 +186,7 @@ def iterate_mip_levels_from_dataset(
                 dataset, lvl-1, maxlvl, nblocks, block_size,
                 downsample_factor, downsample_method,
                 lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
-                channel=channel, deskew_kwargs=deskew_kwargs):
+                channel=channel, numchunks=numchunks, deskew_kwargs=deskew_kwargs):
             chunk = ma.array
             # throw array up for further processing
             yield ma
@@ -261,7 +263,12 @@ def write_ims_to_zarr(
     f = h5py.File(ims_fn, 'r')
     dataset = f['DataSet']['ResolutionLevel 0']['TimePoint 0']['Channel 0']['Data']
     
-    joined_shapes = dataset.shape
+    block_size = [8*sz for sz in chunk_size[2:]]
+    
+    if numchunks < 1:
+        joined_shapes = dataset.shape
+    else:
+        joined_shapes = [dataset.shape[0],numchunks*block_size[1],block_size[2]]
     print("ims_to_ngff dataset shape:" + str(joined_shapes))
 
     if deskew_options and deskew_options["deskew_method"] == "ps":
@@ -270,7 +277,6 @@ def write_ims_to_zarr(
             stride = deskew_options["deskew_stride"]
         else:
             stride = 1
-        block_size = [8*sz for sz in chunk_size[2:]]
         joined_shapes = psd.reshape_joined_shapes(
             joined_shapes, stride, block_size, transpose=True)
         print("deskewed shape:" + str(joined_shapes))
@@ -278,7 +284,6 @@ def write_ims_to_zarr(
         deskew_kwargs = psd.psdeskew_kwargs(skew_dims_zyx=(slice_length, block_size[1], block_size[2]*stride),
                                             **deskew_options)
     else:
-        block_size = [8*sz for sz in chunk_size[2:]]
         deskew_kwargs = {}
 
     # TODO DESKEW: does this generally work regardless of skew?

From c329197372430ffef84ed2dbb558b277f56f5b9e Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 6 Aug 2024 12:57:42 -0700
Subject: [PATCH 039/115] bug fixing subchunk conversion

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 6c4a1b4..854700b 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -70,7 +70,7 @@ def write_mips(zgrp,miparrs):
 
 # HERE : NEED TO UNTANGLE MISMATCH BETWEEN BLOCK (deskewed) and CHUNK (skewed) size
 def iterate_numpy_blocks_from_dataset(
-        dataset, nblocks, block_size=None, pad=True, deskew_kwargs={}, numberblocks=0, *args, **kwargs):
+        dataset, nblocks, block_size=None, pad=True, deskew_kwargs={}, *args, **kwargs):
     """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
 
     Parameters
@@ -93,11 +93,7 @@ def iterate_numpy_blocks_from_dataset(
     #     fb = numpy.ravel_multi_index((zb,yb,xb),block_size)
     if deskew_kwargs:
         chunk_size = (deskew_kwargs["chunklength"],block_size[1],block_size[2]*deskew_kwargs["stride"])
-    if numberblocks < 1:
-        numberblocks = numpy.prod(nblocks)
-    else:
-        numberblocks = numpy.ravel_multi_index((0,numberblocks,0),nblocks,order='F')
-    for i in range(numberblocks):#,*args,**kwargs):
+    for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nblocks),order='F')
         if True: #chunk_tuple[0] == 0:
             print(str(chunk_tuple))
@@ -145,7 +141,7 @@ def iterate_numpy_blocks_from_dataset(
 def iterate_mip_levels_from_dataset(
         dataset, lvl, maxlvl, nblocks, block_size, downsample_factor,
         downsample_method=None, lvl_to_mip_kwargs=None,
-        interleaved_channels=1, channel=0, numchunks=0, deskew_kwargs={}):
+        interleaved_channels=1, channel=0, deskew_kwargs={}):
     """recursively generate MIPmap levels from an iterator of blocks
 
     Parameters
@@ -186,7 +182,7 @@ def iterate_mip_levels_from_dataset(
                 dataset, lvl-1, maxlvl, nblocks, block_size,
                 downsample_factor, downsample_method,
                 lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
-                channel=channel, numchunks=numchunks, deskew_kwargs=deskew_kwargs):
+                channel=channel, deskew_kwargs=deskew_kwargs):
             chunk = ma.array
             # throw array up for further processing
             yield ma
@@ -206,7 +202,7 @@ def iterate_mip_levels_from_dataset(
         for block in iterate_numpy_blocks_from_dataset(
                 dataset, nblocks, block_size=block_size, pad=False,
                 deskew_kwargs=deskew_kwargs,
-                channel=channel,numberblocks=numchunks):
+                channel=channel):
             block_tuple = numpy.unravel_index(block_index,nblocks,order='F')
             block_start = tuple(block_tuple[k]*block_size[k] for k in range(3))
             block_end = tuple(block_start[k] + block.shape[k] for k in range(3))

From 8296319470d340d7e2675e999089e11aef652e94 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 6 Aug 2024 12:58:55 -0700
Subject: [PATCH 040/115] bug fixing subchunk conversion

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 854700b..1ba521f 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -342,7 +342,7 @@ def write_ims_to_zarr(
                     dataset, max_mip, max_mip, nblocks, block_size, mip_dsfactor,
                     lvl_to_mip_kwargs=lvl_to_mip_kwargs,
                     interleaved_channels=interleaved_channels,
-                    channel=channel, numchunks=numchunks, deskew_kwargs=deskew_kwargs):
+                    channel=channel, deskew_kwargs=deskew_kwargs):
                 mips.append(miparr)
                 if miparr.lvl == max_mip:
                     # futs.append(e.submit(

From e9f49135c77c0a535d138faa34afade2df71c29e Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 6 Aug 2024 14:00:33 -0700
Subject: [PATCH 041/115] bug fixing subchunk conversion

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py            | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 1ba521f..df161d0 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -111,7 +111,7 @@ def iterate_numpy_blocks_from_dataset(
                 chunk_end = [st+s for st,s in zip(chunk_start,chunk_size)]
                 arr = numpy.transpose(psd.deskew_block(
                     dataset[chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]],
-                    chunk_index,
+                    chunk_index,transpose=True,
                     **deskew_kwargs), (2, 1, 0))
             chunk_index += 1
             # arr = numpy.zeros(block_size,dtype=dataset.dtype)
@@ -264,7 +264,7 @@ def write_ims_to_zarr(
     if numchunks < 1:
         joined_shapes = dataset.shape
     else:
-        joined_shapes = [dataset.shape[0],numchunks*block_size[1],block_size[2]]
+        joined_shapes = [dataset.shape[0],numchunks*block_size[1],numchunks*block_size[2]]
     print("ims_to_ngff dataset shape:" + str(joined_shapes))
 
     if deskew_options and deskew_options["deskew_method"] == "ps":

From 64779397af3f9450038c1c4e93b53989d7c8afc4 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 6 Aug 2024 14:03:32 -0700
Subject: [PATCH 042/115] bug fixing subchunk conversion

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index df161d0..ca8b989 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -111,7 +111,7 @@ def iterate_numpy_blocks_from_dataset(
                 chunk_end = [st+s for st,s in zip(chunk_start,chunk_size)]
                 arr = numpy.transpose(psd.deskew_block(
                     dataset[chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]],
-                    chunk_index,transpose=True,
+                    chunk_index,
                     **deskew_kwargs), (2, 1, 0))
             chunk_index += 1
             # arr = numpy.zeros(block_size,dtype=dataset.dtype)

From 13cc660180552ade85d75006669773e69bc0001e Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 6 Aug 2024 16:33:20 -0700
Subject: [PATCH 043/115] bug fixing subchunk conversion

---
 acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index d741210..18041d8 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -109,8 +109,8 @@ def get_deskewed_block(blockdims,dataset,start,end,stride):
 
 
 def calculate_first_chunk(dataset_shape,chunk_size,x_index,stride):
-    first_chunk = x_index*stride // chunk_size[2]
-    first_slice = x_index*stride % chunk_size[2]
+    first_chunk = x_index // stride
+    first_slice = int(x_index*chunk_size[0]/stride) % chunk_size[0]
     return first_chunk,first_slice
 
 

From 7322a73165ab7ccb1e8db2efad5e2c54c62d2a2b Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 6 Aug 2024 21:17:39 -0700
Subject: [PATCH 044/115] bug fixing subchunk conversion

---
 .../convert_to_n5/ims_to_ngff.py                | 17 ++++++++++++++---
 .../stitching_modules/convert_to_n5/psdeskew.py |  2 +-
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index ca8b989..adf89b2 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -103,14 +103,25 @@ def iterate_numpy_blocks_from_dataset(
                 chunk_index = 0
                 deskew_kwargs["slice1d"] *= 0
                 first_z,first_slice = psd.calculate_first_chunk(dataset_shape=dataset.shape,chunk_size=chunk_size,x_index=chunk_tuple[2],stride=deskew_kwargs["stride"])
-            if chunk_tuple[0] < first_z or chunk_tuple[0]*chunk_size[0] + first_slice >= dataset.shape[0]:
+            if chunk_tuple[0] < first_z or chunk_tuple[0]*chunk_size[0] - first_slice >= dataset.shape[0]:
                 arr = numpy.zeros(block_size,dtype=dataset.dtype)
             else:
                 chunk_start = [t*s for t,s in zip(chunk_tuple,chunk_size)]
-                chunk_start[0] += first_slice
                 chunk_end = [st+s for st,s in zip(chunk_start,chunk_size)]
+                if chunk_start[0] < first_slice:
+                    chunk_end[0] -= first_slice - chunk_start[0]
+                    chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
+                    chunk[first_slice-chunk_start[0]:] = dataset[chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
+                else:
+                    chunk_start[0] -= first_slice
+                    chunk_end[0] -= first_slice
+                    if chunk_end[0] >= dataset.shape[0]:
+                        chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
+                        chunk[:dataset.shape[0]-chunk_start[0]] = dataset[chunk_start[0]:,chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
+                    else:
+                        chunk = dataset[chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
                 arr = numpy.transpose(psd.deskew_block(
-                    dataset[chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]],
+                    chunk,
                     chunk_index,
                     **deskew_kwargs), (2, 1, 0))
             chunk_index += 1
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 18041d8..0620db5 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -110,7 +110,7 @@ def get_deskewed_block(blockdims,dataset,start,end,stride):
 
 def calculate_first_chunk(dataset_shape,chunk_size,x_index,stride):
     first_chunk = x_index // stride
-    first_slice = int(x_index*chunk_size[0]/stride) % chunk_size[0]
+    first_slice = int(x_index*chunk_size[0]/stride) #% chunk_size[0]
     return first_chunk,first_slice
 
 

From 83bb803baffa82d8a0462abce61cdb83aff66d22 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 6 Aug 2024 21:34:41 -0700
Subject: [PATCH 045/115] bug fixing subchunk conversion

---
 acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 0620db5..b8dc002 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -109,8 +109,8 @@ def get_deskewed_block(blockdims,dataset,start,end,stride):
 
 
 def calculate_first_chunk(dataset_shape,chunk_size,x_index,stride):
-    first_chunk = x_index // stride
-    first_slice = int(x_index*chunk_size[0]/stride) #% chunk_size[0]
+    first_chunk = x_index*stride
+    first_slice = int(x_index*chunk_size[0]*stride) #% chunk_size[0]
     return first_chunk,first_slice
 
 

From 1e4e626beac78e8ea0ed59ca77a9fd0adfffb245 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 6 Aug 2024 21:51:25 -0700
Subject: [PATCH 046/115] double block size

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index adf89b2..199e9a6 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -270,7 +270,7 @@ def write_ims_to_zarr(
     f = h5py.File(ims_fn, 'r')
     dataset = f['DataSet']['ResolutionLevel 0']['TimePoint 0']['Channel 0']['Data']
     
-    block_size = [8*sz for sz in chunk_size[2:]]
+    block_size = [16*sz for sz in chunk_size[2:]]
     
     if numchunks < 1:
         joined_shapes = dataset.shape

From 7f8a99f60fcf01135758c63f804eb84c75a41725 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 7 Aug 2024 08:39:33 -0700
Subject: [PATCH 047/115] debugging

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py  | 2 +-
 acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 199e9a6..afe96b7 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -275,7 +275,7 @@ def write_ims_to_zarr(
     if numchunks < 1:
         joined_shapes = dataset.shape
     else:
-        joined_shapes = [dataset.shape[0],numchunks*block_size[1],numchunks*block_size[2]]
+        joined_shapes = [dataset.shape[0],numchunks*block_size[1],block_size[2]]
     print("ims_to_ngff dataset shape:" + str(joined_shapes))
 
     if deskew_options and deskew_options["deskew_method"] == "ps":
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py
index 69b81fb..10beb7a 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py
@@ -1048,7 +1048,7 @@ class DownsampleOptions(argschema.schemas.DefaultSchema):
 class DeskewOptions(argschema.schemas.DefaultSchema):
     deskew_method = argschema.fields.Str(required=False, default='')
     deskew_stride = argschema.fields.Int(required=False, default=None)
-    deskew_flip = argschema.fields.Bool(required=False, default=True)
+    deskew_flip = argschema.fields.Bool(required=False, default=False)
     deskew_transpose = argschema.fields.Bool(required=False, default=False)
     deskew_crop = argschema.fields.Float(required=False, default=1.0)
 

From 709d3de92fbf05ca0d14dcffa11cea56316af5ed Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 7 Aug 2024 19:56:02 -0700
Subject: [PATCH 048/115] debugging

---
 .../convert_to_n5/ims_to_ngff.py                 | 16 ++++++++++------
 .../stitching_modules/convert_to_n5/psdeskew.py  | 14 +++++++-------
 2 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index afe96b7..a22f025 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -95,13 +95,12 @@ def iterate_numpy_blocks_from_dataset(
         chunk_size = (deskew_kwargs["chunklength"],block_size[1],block_size[2]*deskew_kwargs["stride"])
     for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nblocks),order='F')
-        if True: #chunk_tuple[0] == 0:
-            print(str(chunk_tuple))
         # deskew level 0 data blocks
         if deskew_kwargs:
+            print(str(chunk_tuple))
             if chunk_tuple[0] == 0:
                 chunk_index = 0
-                deskew_kwargs["slice1d"] *= 0
+                deskew_kwargs["slice1d"][...] = 0
                 first_z,first_slice = psd.calculate_first_chunk(dataset_shape=dataset.shape,chunk_size=chunk_size,x_index=chunk_tuple[2],stride=deskew_kwargs["stride"])
             if chunk_tuple[0] < first_z or chunk_tuple[0]*chunk_size[0] - first_slice >= dataset.shape[0]:
                 arr = numpy.zeros(block_size,dtype=dataset.dtype)
@@ -134,6 +133,8 @@ def iterate_numpy_blocks_from_dataset(
             #                                  end=block_end,
             #                                  stride=stride)
         else:
+            if chunk_tuple[0] == 0:
+                print(str(chunk_tuple))
             block_start = [chunk_tuple[k]*block_size[k] for k in range(3)]
             block_end = [block_start[k] + block_size[k] for k in range(3)]
             arr = dataset[block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]]
@@ -269,6 +270,9 @@ def write_ims_to_zarr(
     
     f = h5py.File(ims_fn, 'r')
     dataset = f['DataSet']['ResolutionLevel 0']['TimePoint 0']['Channel 0']['Data']
+    if deskew_options and deskew_options["deskew_transpose"]:
+        dataset = dataset.transpose((0,2,1))
+        print("transposed shape: " + str(dataset.shape))
     
     block_size = [16*sz for sz in chunk_size[2:]]
     
@@ -284,12 +288,12 @@ def write_ims_to_zarr(
             stride = deskew_options["deskew_stride"]
         else:
             stride = 1
-        joined_shapes = psd.reshape_joined_shapes(
-            joined_shapes, stride, block_size, transpose=True)
-        print("deskewed shape:" + str(joined_shapes))
         slice_length = int(block_size[0]/stride)
         deskew_kwargs = psd.psdeskew_kwargs(skew_dims_zyx=(slice_length, block_size[1], block_size[2]*stride),
                                             **deskew_options)
+        joined_shapes = psd.reshape_joined_shapes(
+            joined_shapes, stride, block_size)# transpose=(2,1,0))
+        print("deskewed shape:" + str(joined_shapes))
     else:
         deskew_kwargs = {}
 
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index b8dc002..b340df2 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -34,8 +34,8 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_tr
     kwargs : dict
         parameters representing pixel deskew operation for deskew_block
     """
-    if deskew_transpose:
-        skew_dims_zyx = (skew_dims_zyx[0],skew_dims_zyx[2],skew_dims_zyx[1])
+    # if deskew_transpose:
+    #     skew_dims_zyx = (skew_dims_zyx[0],skew_dims_zyx[2],skew_dims_zyx[1])
     sdims = skew_dims_zyx
     crop_factor = deskew_crop
     stride = deskew_stride
@@ -145,8 +145,8 @@ def deskew_block(blockData, n, dsi, si, slice1d, blockdims, subblocks, flip, tra
     block3d : numpy.ndarray
         pixel shifted deskewed data ordered (z,y,x) by sample axes 
     """
-    if transpose:
-        blockData = blockData.transpose((0,2,1))
+    # if transpose:
+    #     blockData = blockData.transpose((0,2,1))
     subb = subblocks
     # subb = 5
     block3d = np.zeros(blockdims, dtype=dtype)
@@ -174,7 +174,7 @@ def deskew_block(blockData, n, dsi, si, slice1d, blockdims, subblocks, flip, tra
     return block3d
 
 
-def reshape_joined_shapes(joined_shapes, stride, blockdims, transpose=True, **kwargs):
+def reshape_joined_shapes(joined_shapes, stride, blockdims, transpose=None, **kwargs):
     """get dimensions of deskewed joined shapes from skewed joined shapes
 
     Parameters
@@ -191,8 +191,8 @@ def reshape_joined_shapes(joined_shapes, stride, blockdims, transpose=True, **kw
     deskewed_shape : tuple of int
         shape of deskewed 3D array represented by joined_shapes
     """
-    if transpose:
-        axes = (2,1,0)
+    if not transpose is None:
+        axes = transpose
     else:
         axes = (0,1,2)
     # deskewed_shape = (int(np.ceil(joined_shapes[axes[0]]/(blockdims[axes[0]]/stride))*blockdims[axes[0]]),

From ad9484f498e5cf111d248d57ef25e8b511515176 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Thu, 8 Aug 2024 12:34:12 -0700
Subject: [PATCH 049/115] debugging transposed ims data

---
 .../convert_to_n5/ims_to_ngff.py              | 23 +++++++++++++------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index a22f025..321e00b 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -91,22 +91,28 @@ def iterate_numpy_blocks_from_dataset(
     #     # create output block and get flattened indices
     #     zb,yb,xb = numpy.meshgrid(*[range(d) for d in block_size],indexing="ij")
     #     fb = numpy.ravel_multi_index((zb,yb,xb),block_size)
+    dshape = dataset.shape
     if deskew_kwargs:
         chunk_size = (deskew_kwargs["chunklength"],block_size[1],block_size[2]*deskew_kwargs["stride"])
+        if deskew_kwargs["transpose"]:
+            chunk_size = (chunk_size[0],chunk_size[2],chunk_size[1])
+            dshape.transpose((0,2,1))
     for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nblocks),order='F')
         # deskew level 0 data blocks
         if deskew_kwargs:
+            if deskew_kwargs["transpose"]:
+                chunk_tuple = (chunk_tuple[0],chunk_tuple[2],chunk_tuple[1])
             print(str(chunk_tuple))
             if chunk_tuple[0] == 0:
                 chunk_index = 0
                 deskew_kwargs["slice1d"][...] = 0
-                first_z,first_slice = psd.calculate_first_chunk(dataset_shape=dataset.shape,chunk_size=chunk_size,x_index=chunk_tuple[2],stride=deskew_kwargs["stride"])
+                first_z,first_slice = psd.calculate_first_chunk(dataset_shape=dshape,chunk_size=chunk_size,x_index=chunk_tuple[2],stride=deskew_kwargs["stride"])
             if chunk_tuple[0] < first_z or chunk_tuple[0]*chunk_size[0] - first_slice >= dataset.shape[0]:
                 arr = numpy.zeros(block_size,dtype=dataset.dtype)
             else:
-                chunk_start = [t*s for t,s in zip(chunk_tuple,chunk_size)]
-                chunk_end = [st+s for st,s in zip(chunk_start,chunk_size)]
+                chunk_start = numpy.array([t*s for t,s in zip(chunk_tuple,chunk_size)])
+                chunk_end = chunk_start + numpy.array(chunk_size)
                 if chunk_start[0] < first_slice:
                     chunk_end[0] -= first_slice - chunk_start[0]
                     chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
@@ -270,14 +276,17 @@ def write_ims_to_zarr(
     
     f = h5py.File(ims_fn, 'r')
     dataset = f['DataSet']['ResolutionLevel 0']['TimePoint 0']['Channel 0']['Data']
-    if deskew_options and deskew_options["deskew_transpose"]:
-        dataset = dataset.transpose((0,2,1))
-        print("transposed shape: " + str(dataset.shape))
+    # if deskew_options and deskew_options["deskew_transpose"]:
+    #     dataset = dataset.transpose((0,2,1))
+    #     print("transposed shape: " + str(dataset.shape))
     
     block_size = [16*sz for sz in chunk_size[2:]]
     
     if numchunks < 1:
         joined_shapes = dataset.shape
+        if deskew_options and deskew_options["deskew_transpose"]:
+            # input dataset must be transposed
+            joined_shapes = joined_shapes.transpose((0,2,1))
     else:
         joined_shapes = [dataset.shape[0],numchunks*block_size[1],block_size[2]]
     print("ims_to_ngff dataset shape:" + str(joined_shapes))
@@ -289,7 +298,7 @@ def write_ims_to_zarr(
         else:
             stride = 1
         slice_length = int(block_size[0]/stride)
-        deskew_kwargs = psd.psdeskew_kwargs(skew_dims_zyx=(slice_length, block_size[1], block_size[2]*stride),
+        deskew_kwargs = psd.psdeskew_kwargs(skew_dims_zyx=(slice_length, block_size[1], block_size[2]*stride), # size of skewed chunk to iterate for deskewed block
                                             **deskew_options)
         joined_shapes = psd.reshape_joined_shapes(
             joined_shapes, stride, block_size)# transpose=(2,1,0))

From 17ff0a8844c4261511cc0f7766932264f9f626bd Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Thu, 8 Aug 2024 12:40:37 -0700
Subject: [PATCH 050/115] debugging

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py   | 12 ++++++------
 .../stitching_modules/convert_to_n5/psdeskew.py      |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 321e00b..ae386e3 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -96,7 +96,7 @@ def iterate_numpy_blocks_from_dataset(
         chunk_size = (deskew_kwargs["chunklength"],block_size[1],block_size[2]*deskew_kwargs["stride"])
         if deskew_kwargs["transpose"]:
             chunk_size = (chunk_size[0],chunk_size[2],chunk_size[1])
-            dshape.transpose((0,2,1))
+            dshape = (dshape[0],dshape[2],dshape[1])
     for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nblocks),order='F')
         # deskew level 0 data blocks
@@ -107,8 +107,8 @@ def iterate_numpy_blocks_from_dataset(
             if chunk_tuple[0] == 0:
                 chunk_index = 0
                 deskew_kwargs["slice1d"][...] = 0
-                first_z,first_slice = psd.calculate_first_chunk(dataset_shape=dshape,chunk_size=chunk_size,x_index=chunk_tuple[2],stride=deskew_kwargs["stride"])
-            if chunk_tuple[0] < first_z or chunk_tuple[0]*chunk_size[0] - first_slice >= dataset.shape[0]:
+                first_z,first_slice = psd.calculate_first_chunk(chunk_size=chunk_size,x_index=chunk_tuple[2],stride=deskew_kwargs["stride"])
+            if chunk_tuple[0] < first_z or chunk_tuple[0]*chunk_size[0] - first_slice >= dshape[0]:
                 arr = numpy.zeros(block_size,dtype=dataset.dtype)
             else:
                 chunk_start = numpy.array([t*s for t,s in zip(chunk_tuple,chunk_size)])
@@ -120,9 +120,9 @@ def iterate_numpy_blocks_from_dataset(
                 else:
                     chunk_start[0] -= first_slice
                     chunk_end[0] -= first_slice
-                    if chunk_end[0] >= dataset.shape[0]:
+                    if chunk_end[0] >= dshape[0]:
                         chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
-                        chunk[:dataset.shape[0]-chunk_start[0]] = dataset[chunk_start[0]:,chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
+                        chunk[:dshape[0]-chunk_start[0]] = dataset[chunk_start[0]:,chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
                     else:
                         chunk = dataset[chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
                 arr = numpy.transpose(psd.deskew_block(
@@ -286,7 +286,7 @@ def write_ims_to_zarr(
         joined_shapes = dataset.shape
         if deskew_options and deskew_options["deskew_transpose"]:
             # input dataset must be transposed
-            joined_shapes = joined_shapes.transpose((0,2,1))
+            joined_shapes = (joined_shapes[0],joined_shapes[2],joined_shapes[1])
     else:
         joined_shapes = [dataset.shape[0],numchunks*block_size[1],block_size[2]]
     print("ims_to_ngff dataset shape:" + str(joined_shapes))
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index b340df2..0685c5a 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -108,7 +108,7 @@ def get_deskewed_block(blockdims,dataset,start,end,stride):
     return blockdata
 
 
-def calculate_first_chunk(dataset_shape,chunk_size,x_index,stride):
+def calculate_first_chunk(chunk_size,x_index,stride):
     first_chunk = x_index*stride
     first_slice = int(x_index*chunk_size[0]*stride) #% chunk_size[0]
     return first_chunk,first_slice

From 88def9cc178c68c3813c633d1f5583be1c635df1 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Thu, 8 Aug 2024 13:12:04 -0700
Subject: [PATCH 051/115] debugging

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index ae386e3..c0dac37 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -125,6 +125,8 @@ def iterate_numpy_blocks_from_dataset(
                         chunk[:dshape[0]-chunk_start[0]] = dataset[chunk_start[0]:,chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
                     else:
                         chunk = dataset[chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
+                if deskew_kwargs["transpose"]:
+                    chunk = chunk.transpose((0,2,1))
                 arr = numpy.transpose(psd.deskew_block(
                     chunk,
                     chunk_index,

From fc82864196a8af7d68f089a45f6a650d570f488b Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Mon, 12 Aug 2024 19:34:57 -0700
Subject: [PATCH 052/115] vectorized resampling

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py  |  2 +-
 .../stitching_modules/convert_to_n5/psdeskew.py     | 13 +++++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index c0dac37..f82c2aa 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -107,7 +107,7 @@ def iterate_numpy_blocks_from_dataset(
             if chunk_tuple[0] == 0:
                 chunk_index = 0
                 deskew_kwargs["slice1d"][...] = 0
-                first_z,first_slice = psd.calculate_first_chunk(chunk_size=chunk_size,x_index=chunk_tuple[2],stride=deskew_kwargs["stride"])
+                first_z,first_slice = psd.calculate_first_chunk(chunk_size=chunk_size,x_index=chunk_tuple[1],stride=deskew_kwargs["stride"])
             if chunk_tuple[0] < first_z or chunk_tuple[0]*chunk_size[0] - first_slice >= dshape[0]:
                 arr = numpy.zeros(block_size,dtype=dataset.dtype)
             else:
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 0685c5a..7006725 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -59,13 +59,18 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_tr
             sx = np.arange(sxstart, sxend)
             sxv.append(sx)
             szv.append(sz*np.ones(sx.shape, dtype=sx.dtype))
-        sxv = np.concatenate(sxv)
-        szv = np.concatenate(szv)
+        sxv = np.tile(np.concatenate(sxv)[:,np.newaxis],[1,blockdims[1]])
+        szv = np.tile(np.concatenate(szv)[:,np.newaxis],[1,blockdims[1]])
+        syv = np.tile(np.arange(blockdims[1]),[sxv.shape[0],1])
+        sxv = sxv.flatten()
+        szv = szv.flatten()
+        syv = syv.flatten()
         dsx = sxv + stride*szv - i_block*stride*blockx
         dsz = np.floor(sxv/stride).astype(int)
+        dsy = syv
         dsi.append(np.ravel_multi_index(
-            (dsz, dsx), (blockdims[0], blockdims[2])))
-        si.append(np.ravel_multi_index((szv, sxv), (sdims[0], sdims[2])))
+            (dsz,dsy,dsx), blockdims))
+        si.append(np.ravel_multi_index((szv,syv,sxv), sdims))
     kwargs = {'dsi': dsi,
               'si': si,
               'slice1d': np.zeros((subblocks, blockdims[1], blockdims[2]*blockdims[0]), dtype=dtype),

From ae8b6afcd03e0e9a629045b514eda05b491a38a6 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Mon, 12 Aug 2024 19:36:09 -0700
Subject: [PATCH 053/115] smaller block for vectorized

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index f82c2aa..fb76fd7 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -282,7 +282,7 @@ def write_ims_to_zarr(
     #     dataset = dataset.transpose((0,2,1))
     #     print("transposed shape: " + str(dataset.shape))
     
-    block_size = [16*sz for sz in chunk_size[2:]]
+    block_size = [8*sz for sz in chunk_size[2:]]
     
     if numchunks < 1:
         joined_shapes = dataset.shape

From 17e5f66204f97a55e7a480f0d6c2b6b686951050 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Mon, 12 Aug 2024 19:41:36 -0700
Subject: [PATCH 054/115] bug fixing vectorized

---
 .../stitching_modules/convert_to_n5/psdeskew.py     | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 7006725..c5c890d 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -169,13 +169,12 @@ def deskew_block(blockData, n, dsi, si, slice1d, blockdims, subblocks, flip, tra
         blockData = np.concatenate((blockData, np.zeros(
             (int(chunklength-blockData.shape[0]), blockData.shape[1], blockData.shape[2]))))
     order = (np.arange(subb)+n) % subb
-    for y in range(ydim):
-        for i, o in enumerate(order):
-            # flip stack axis 2 for ispim2
-            s = -1 if flip else 1
-            slice1d[o, y, :][dsi[i]] = blockData[:, y, ::s].ravel()[si[i]]
-        block3d[:, y, :] = slice1d[n % subb, y, :].reshape((zdim, xdim))
-        slice1d[n % subb, y, :] = 0
+    for i, o in enumerate(order):
+        # flip stack axis 2 for ispim2
+        s = -1 if flip else 1
+        slice1d[o, :, :][dsi[i]] = blockData[:, :, ::s].ravel()[si[i]]
+    block3d[:, :, :] = slice1d[n % subb, :, :].reshape((zdim,ydim,xdim))
+    slice1d[n % subb, :, :] = 0
     return block3d
 
 

From 7d236e95aa25bf5e35713ad6521fbe8d66fc2880 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Mon, 12 Aug 2024 19:50:53 -0700
Subject: [PATCH 055/115] bug fixing vectorized

---
 .../stitching_modules/convert_to_n5/psdeskew.py           | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index c5c890d..e20c779 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -73,7 +73,7 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_tr
         si.append(np.ravel_multi_index((szv,syv,sxv), sdims))
     kwargs = {'dsi': dsi,
               'si': si,
-              'slice1d': np.zeros((subblocks, blockdims[1], blockdims[2]*blockdims[0]), dtype=dtype),
+              'slice1d': np.zeros((subblocks, blockdims[2]*blockdims[1]*blockdims[0]), dtype=dtype),
               'blockdims': blockdims,
               'subblocks': subblocks,
               'flip': deskew_flip,
@@ -172,9 +172,9 @@ def deskew_block(blockData, n, dsi, si, slice1d, blockdims, subblocks, flip, tra
     for i, o in enumerate(order):
         # flip stack axis 2 for ispim2
         s = -1 if flip else 1
-        slice1d[o, :, :][dsi[i]] = blockData[:, :, ::s].ravel()[si[i]]
-    block3d[:, :, :] = slice1d[n % subb, :, :].reshape((zdim,ydim,xdim))
-    slice1d[n % subb, :, :] = 0
+        slice1d[o, :][dsi[i]] = blockData[:, :, ::s].ravel()[si[i]]
+    block3d[:, :, :] = slice1d[n % subb, :].reshape((zdim,ydim,xdim))
+    slice1d[n % subb, :] = 0
     return block3d
 
 

From eadc6ce52d84292ae35c2d6755d5b2d6bc21e5d6 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 13 Aug 2024 09:44:50 -0700
Subject: [PATCH 056/115] debugging

---
 acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index e20c779..77ad333 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -202,7 +202,7 @@ def reshape_joined_shapes(joined_shapes, stride, blockdims, transpose=None, **kw
     # deskewed_shape = (int(np.ceil(joined_shapes[axes[0]]/(blockdims[axes[0]]/stride))*blockdims[axes[0]]),
     #                   joined_shapes[axes[1]],
     #                   joined_shapes[axes[2]])
-    deskewed_shape = (int(np.ceil(joined_shapes[axes[0]]/(blockdims[axes[0]]/stride))*blockdims[axes[0]]),
+    deskewed_shape = (joined_shapes[axes[0]]*stride + joined_shapes[axes[2]],
                       joined_shapes[axes[1]],
-                      int(np.ceil((joined_shapes[axes[0]] + joined_shapes[axes[2]]*stride)/(blockdims[axes[0]]))*blockdims[axes[0]]))
+                      int(joined_shapes[axes[2]]/stride))
     return deskewed_shape

From 82923cc20f1ba60068338026a1a3ef33d0dd24eb Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 13 Aug 2024 12:34:52 -0700
Subject: [PATCH 057/115] debugging

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py          | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index fb76fd7..b49790a 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -290,7 +290,11 @@ def write_ims_to_zarr(
             # input dataset must be transposed
             joined_shapes = (joined_shapes[0],joined_shapes[2],joined_shapes[1])
     else:
-        joined_shapes = [dataset.shape[0],numchunks*block_size[1],block_size[2]]
+        if deskew_options and deskew_options["deskew_transpose"]:
+            # input dataset must be transposed
+            joined_shapes = (dataset.shape[0],numchunks*block_size[1],dataset.shape[1])
+        else:
+            joined_shapes = (dataset.shape[0],numchunks*block_size[1],dataset.shape[2])
     print("ims_to_ngff dataset shape:" + str(joined_shapes))
 
     if deskew_options and deskew_options["deskew_method"] == "ps":

From 52ef7ee5894b74def5d71895a5173478db816964 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 13 Aug 2024 18:16:24 -0700
Subject: [PATCH 058/115] debugging vectorization

---
 .../convert_to_n5/ims_to_ngff.py                | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index b49790a..443a7ef 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -125,12 +125,21 @@ def iterate_numpy_blocks_from_dataset(
                         chunk[:dshape[0]-chunk_start[0]] = dataset[chunk_start[0]:,chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
                     else:
                         chunk = dataset[chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
+                if any([sh<sz for sh,sz in zip(chunk.shape,chunk_size)]):
+                    print(str(chunk_tuple) + " chunk is small: filling with zeros")
+                    temp_chunk = numpy.zeros(chunk_size,dtype=chunk.dtype)
+                    temp_chunk[:chunk.shape[0],:chunk.shape[1],:chunk.shape[2]] = chunk
+                    chunk = temp_chunk
                 if deskew_kwargs["transpose"]:
                     chunk = chunk.transpose((0,2,1))
-                arr = numpy.transpose(psd.deskew_block(
-                    chunk,
-                    chunk_index,
-                    **deskew_kwargs), (2, 1, 0))
+                arr = numpy.flip(
+                    numpy.transpose(
+                        psd.deskew_block(
+                            chunk,
+                            chunk_index,
+                            **deskew_kwargs), 
+                        (2, 1, 0)),
+                    axis=2)
             chunk_index += 1
             # arr = numpy.zeros(block_size,dtype=dataset.dtype)
             # if deskew_kwargs["deskew_method"] == "ps":

From f4c4b82d7832f3bcdeb925c6f5716bf8603d8680 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 13 Aug 2024 19:14:53 -0700
Subject: [PATCH 059/115] debugging

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 443a7ef..f9d1f31 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -107,7 +107,7 @@ def iterate_numpy_blocks_from_dataset(
             if chunk_tuple[0] == 0:
                 chunk_index = 0
                 deskew_kwargs["slice1d"][...] = 0
-                first_z,first_slice = psd.calculate_first_chunk(chunk_size=chunk_size,x_index=chunk_tuple[1],stride=deskew_kwargs["stride"])
+                first_z,first_slice = psd.calculate_first_chunk(chunk_size=chunk_size,x_index=(nblocks[2] - chunk_tuple[1] - 1),stride=deskew_kwargs["stride"])
             if chunk_tuple[0] < first_z or chunk_tuple[0]*chunk_size[0] - first_slice >= dshape[0]:
                 arr = numpy.zeros(block_size,dtype=dataset.dtype)
             else:

From e81a2a677a794ac1a4726717d19020fc2d324c33 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 14 Aug 2024 08:40:31 -0700
Subject: [PATCH 060/115] debugging

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py     | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index f9d1f31..eacb28e 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -120,11 +120,11 @@ def iterate_numpy_blocks_from_dataset(
                 else:
                     chunk_start[0] -= first_slice
                     chunk_end[0] -= first_slice
-                    if chunk_end[0] >= dshape[0]:
-                        chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
-                        chunk[:dshape[0]-chunk_start[0]] = dataset[chunk_start[0]:,chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
-                    else:
-                        chunk = dataset[chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
+                    # if chunk_end[0] >= dshape[0]:
+                    #     chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
+                    #     chunk[:dshape[0]-chunk_start[0]] = dataset[chunk_start[0]:,chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
+                    # else:
+                    chunk = dataset[chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
                 if any([sh<sz for sh,sz in zip(chunk.shape,chunk_size)]):
                     print(str(chunk_tuple) + " chunk is small: filling with zeros")
                     temp_chunk = numpy.zeros(chunk_size,dtype=chunk.dtype)

From 60b09cb5834ccf444c0089f44563c65532ae80cc Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 14 Aug 2024 13:16:22 -0700
Subject: [PATCH 061/115] commenting old code

---
 .../convert_to_n5/psdeskew.py                 | 55 ++++++++++---------
 1 file changed, 28 insertions(+), 27 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 77ad333..a36948f 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -84,33 +84,34 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_tr
               }
     return kwargs
 
-
-def calculate_skewed_indices(zi,yi,xi,s):
-    # convert input block voxel indices into skewed data space
-    # edge blocks may have negative or out-of-bounds skewed indices
-    xs = s*zi + xi % s
-    ys = yi
-    zs = xi // s - zi
-    return zs,ys,xs
-
-
-def get_deskewed_block(blockdims,dataset,start,end,stride):
-    # create output block and get flattened indices
-    blockdata = np.zeros(blockdims,dtype=dataset.dtype)
-    zb,yb,xb = np.meshgrid(*[range(d) for d in blockdims],indexing="ij")
-    fb = np.ravel_multi_index((zb,yb,xb),blockdims)
-    # get indices of voxel data for input dataset
-    sdims = dataset.shape
-    zi,yi,xi = np.meshgrid(*[range(s,e) for s,e in zip(start,end)],indexing="ij")
-    zs,ys,xs = calculate_skewed_indices(zi,yi,xi,stride)
-    fi = np.ravel_multi_index((zs,ys,xs),sdims,mode='clip').flatten()
-    # filter out-of-bounds voxels
-    r = (fi > 0) & (fi < np.prod(sdims)-1)
-    fb = fb[r]
-    fi = fi[r]
-    # assign input to output
-    blockdata[fb] = dataset[fi]
-    return blockdata
+# commented code no longer in use
+#
+# def calculate_skewed_indices(zi,yi,xi,s):
+#     # convert input block voxel indices into skewed data space
+#     # edge blocks may have negative or out-of-bounds skewed indices
+#     xs = s*zi + xi % s
+#     ys = yi
+#     zs = xi // s - zi
+#     return zs,ys,xs
+
+
+# def get_deskewed_block(blockdims,dataset,start,end,stride):
+#     # create output block and get flattened indices
+#     blockdata = np.zeros(blockdims,dtype=dataset.dtype)
+#     zb,yb,xb = np.meshgrid(*[range(d) for d in blockdims],indexing="ij")
+#     fb = np.ravel_multi_index((zb,yb,xb),blockdims)
+#     # get indices of voxel data for input dataset
+#     sdims = dataset.shape
+#     zi,yi,xi = np.meshgrid(*[range(s,e) for s,e in zip(start,end)],indexing="ij")
+#     zs,ys,xs = calculate_skewed_indices(zi,yi,xi,stride)
+#     fi = np.ravel_multi_index((zs,ys,xs),sdims,mode='clip').flatten()
+#     # filter out-of-bounds voxels
+#     r = (fi > 0) & (fi < np.prod(sdims)-1)
+#     fb = fb[r]
+#     fi = fi[r]
+#     # assign input to output
+#     blockdata[fb] = dataset[fi]
+#     return blockdata
 
 
 def calculate_first_chunk(chunk_size,x_index,stride):

From 6056d9d40a557edc0e6b28e7290fbed7c32930ba Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Thu, 15 Aug 2024 17:18:32 -0700
Subject: [PATCH 062/115] debugging

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py        | 8 +++++---
 .../stitching_modules/convert_to_n5/psdeskew.py           | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index eacb28e..9d7f7c0 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -97,6 +97,7 @@ def iterate_numpy_blocks_from_dataset(
         if deskew_kwargs["transpose"]:
             chunk_size = (chunk_size[0],chunk_size[2],chunk_size[1])
             dshape = (dshape[0],dshape[2],dshape[1])
+        print("chunk size: " + str(chunk_size))
     for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nblocks),order='F')
         # deskew level 0 data blocks
@@ -126,7 +127,7 @@ def iterate_numpy_blocks_from_dataset(
                     # else:
                     chunk = dataset[chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
                 if any([sh<sz for sh,sz in zip(chunk.shape,chunk_size)]):
-                    print(str(chunk_tuple) + " chunk is small: filling with zeros")
+                    print(str(chunk.shape) + " is small for" + str(chunk_size) + ": filling with zeros")
                     temp_chunk = numpy.zeros(chunk_size,dtype=chunk.dtype)
                     temp_chunk[:chunk.shape[0],:chunk.shape[1],:chunk.shape[2]] = chunk
                     chunk = temp_chunk
@@ -155,8 +156,9 @@ def iterate_numpy_blocks_from_dataset(
             block_start = [chunk_tuple[k]*block_size[k] for k in range(3)]
             block_end = [block_start[k] + block_size[k] for k in range(3)]
             arr = dataset[block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]]
-        if pad:
-            if any([arr.shape[k] != block_size[k] for k in range(3)]):
+        if any([arr.shape[k] != block_size[k] for k in range(3)]):
+            print(str(arr.shape) + "is small for " + str(block_size))
+            if pad:
                 newarr = numpy.zeros(block_size,
                                       dtype=arr.dtype)
                 newarr[:arr.shape[0], :arr.shape[1], :arr.shape[2]] = arr[:, :, :]
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index a36948f..3c02d83 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -166,7 +166,7 @@ def deskew_block(blockData, n, dsi, si, slice1d, blockdims, subblocks, flip, tra
         blockData = blockData[:, y0:y1, :]
     #print('deskewing block ' + str(n) + ' with shape ' + str(blockData.shape))
     if blockData.shape[0] < chunklength:
-        #print('block is short, filling with zeros')
+        print("deskew block is short, filling with zeros, this should never happen")
         blockData = np.concatenate((blockData, np.zeros(
             (int(chunklength-blockData.shape[0]), blockData.shape[1], blockData.shape[2]))))
     order = (np.arange(subb)+n) % subb

From dc648c4e59c989f155a48ee5b4460ce0d0df794a Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Thu, 15 Aug 2024 19:07:28 -0700
Subject: [PATCH 063/115] temp debugging

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py       | 9 +--------
 .../stitching_modules/convert_to_n5/psdeskew.py          | 4 ++--
 2 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 9d7f7c0..6b4c842 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -142,14 +142,6 @@ def iterate_numpy_blocks_from_dataset(
                         (2, 1, 0)),
                     axis=2)
             chunk_index += 1
-            # arr = numpy.zeros(block_size,dtype=dataset.dtype)
-            # if deskew_kwargs["deskew_method"] == "ps":
-            #     stride = deskew_kwargs["deskew_stride"]
-            #     arr = psd.get_deskewed_block(blockdims=block_size,
-            #                                  dataset=dataset,
-            #                                  start=block_start,
-            #                                  end=block_end,
-            #                                  stride=stride)
         else:
             if chunk_tuple[0] == 0:
                 print(str(chunk_tuple))
@@ -294,6 +286,7 @@ def write_ims_to_zarr(
     #     print("transposed shape: " + str(dataset.shape))
     
     block_size = [8*sz for sz in chunk_size[2:]]
+    print("deskewed block size: " + str(block_size))
     
     if numchunks < 1:
         joined_shapes = dataset.shape
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 3c02d83..20f9814 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -41,7 +41,7 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_tr
     stride = deskew_stride
     ydim = int(sdims[1]*crop_factor)
     blockdims = (int(sdims[2]/stride), ydim, stride*sdims[0])
-    subblocks = int(np.ceil((sdims[2]+stride*sdims[0])/(stride*sdims[0])))
+    subblocks = 2*int(np.ceil((sdims[2]+stride*sdims[0])/(stride*sdims[0])))
     # print(subblocks)
     blockx = sdims[0]
     dsi = []
@@ -51,7 +51,7 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_tr
         szv = []
         for sz in range(blockx):
             sxstart = i_block*stride*blockx-stride*sz
-            sxend = (i_block+1)*stride*blockx-stride*sz
+            sxend = sxstart + stride*blockx #(i_block+1)*stride*blockx-stride*sz
             if sxstart < 0:
                 sxstart = 0
             if sxend > sdims[2]:

From 38e03c9c0731d634a36bbb98e0ccb1335168e9ed Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Thu, 15 Aug 2024 20:43:20 -0700
Subject: [PATCH 064/115] test condition

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 6b4c842..9cd978b 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -285,7 +285,7 @@ def write_ims_to_zarr(
     #     dataset = dataset.transpose((0,2,1))
     #     print("transposed shape: " + str(dataset.shape))
     
-    block_size = [8*sz for sz in chunk_size[2:]]
+    block_size = [m*sz for m,sz in zip([16,8,8],chunk_size[2:])]
     print("deskewed block size: " + str(block_size))
     
     if numchunks < 1:
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 20f9814..7b29f7d 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -41,7 +41,7 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_tr
     stride = deskew_stride
     ydim = int(sdims[1]*crop_factor)
     blockdims = (int(sdims[2]/stride), ydim, stride*sdims[0])
-    subblocks = 2*int(np.ceil((sdims[2]+stride*sdims[0])/(stride*sdims[0])))
+    subblocks = int(np.ceil((sdims[2]+stride*sdims[0])/(stride*sdims[0])))
     # print(subblocks)
     blockx = sdims[0]
     dsi = []

From 756542c4a80d2c8b1d7d8e9eddc0b927d0a2915f Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Thu, 15 Aug 2024 20:45:42 -0700
Subject: [PATCH 065/115] test condition

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py         | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 9cd978b..6fa88b4 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -91,6 +91,11 @@ def iterate_numpy_blocks_from_dataset(
     #     # create output block and get flattened indices
     #     zb,yb,xb = numpy.meshgrid(*[range(d) for d in block_size],indexing="ij")
     #     fb = numpy.ravel_multi_index((zb,yb,xb),block_size)
+    if nblocks[1] == 1:
+        print("test condition: using test tuple")
+        test = True
+    else:
+        test = False
     dshape = dataset.shape
     if deskew_kwargs:
         chunk_size = (deskew_kwargs["chunklength"],block_size[1],block_size[2]*deskew_kwargs["stride"])
@@ -100,6 +105,8 @@ def iterate_numpy_blocks_from_dataset(
         print("chunk size: " + str(chunk_size))
     for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nblocks),order='F')
+        if test:
+            chunk_tuple[1] = 11
         # deskew level 0 data blocks
         if deskew_kwargs:
             if deskew_kwargs["transpose"]:

From a481cc9704f0c1abdeed1094289f57d7ff2ccd64 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Thu, 15 Aug 2024 20:47:24 -0700
Subject: [PATCH 066/115] test condition

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 6fa88b4..3b5dcba 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -106,7 +106,7 @@ def iterate_numpy_blocks_from_dataset(
     for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nblocks),order='F')
         if test:
-            chunk_tuple[1] = 11
+            chunk_tuple = (chunk_tuple[0], 11, chunk_tuple[2])
         # deskew level 0 data blocks
         if deskew_kwargs:
             if deskew_kwargs["transpose"]:

From 646fb23bbaf64b62e70ae853cf5fcdef8a4a36e0 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Thu, 15 Aug 2024 21:07:29 -0700
Subject: [PATCH 067/115] debugging

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 3b5dcba..84638f4 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -292,7 +292,7 @@ def write_ims_to_zarr(
     #     dataset = dataset.transpose((0,2,1))
     #     print("transposed shape: " + str(dataset.shape))
     
-    block_size = [m*sz for m,sz in zip([16,8,8],chunk_size[2:])]
+    block_size = [m*sz for m,sz in zip([4,4,4],chunk_size[2:])]
     print("deskewed block size: " + str(block_size))
     
     if numchunks < 1:

From f7917f41a2de2dbf6cb550b3702cc9768ffc0be2 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Fri, 16 Aug 2024 09:30:57 -0700
Subject: [PATCH 068/115] implementing zarr to ngff

---
 .../convert_to_n5/zarr_to_ngff.py             | 461 ++++++++++++++++++
 1 file changed, 461 insertions(+)
 create mode 100644 acpreprocessing/stitching_modules/convert_to_n5/zarr_to_ngff.py

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/zarr_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/zarr_to_ngff.py
new file mode 100644
index 0000000..9aebd61
--- /dev/null
+++ b/acpreprocessing/stitching_modules/convert_to_n5/zarr_to_ngff.py
@@ -0,0 +1,461 @@
+#!/usr/bin/env python
+
+import concurrent.futures
+import dataclasses
+#import itertools
+import math
+#import pathlib
+
+import imageio
+#from natsort import natsorted
+import numpy
+import skimage
+
+import h5py
+import hdf5plugin
+import zarr
+from numcodecs import Blosc
+import argschema
+
+# import acpreprocessing.utils.convert
+import acpreprocessing.stitching_modules.convert_to_n5.psdeskew as psd
+from acpreprocessing.stitching_modules.acstitch.zarrutils import get_zarr_group
+from acpreprocessing.stitching_modules.convert_to_n5.tiff_to_ngff import downsample_array,mip_level_shape,omezarr_attrs,NGFFGroupGenerationParameters,TiffToNGFFValueError
+
+
+@dataclasses.dataclass
+class MIPArray:
+    lvl: int
+    array: numpy.ndarray
+    start: tuple # list of indices (int)
+    end: tuple # list of indices (int)
+
+
+def dswrite_block(ds, start, end, arr, silent_overflow=True):
+    """write array arr into array-like n5 dataset defined by
+    start and end point 3-tuples.
+
+    Parameters
+    ----------
+    ds : z5py.dataset.Dataset (n5 3D) or zarr.dataset (zarr 5D)
+        array-like dataset to fill in
+    start : tuple[int]
+        start index along 0 axis of ds to fill in
+    end : tuple[int]
+        end index along 0 axis of ds to fill in
+    arr : numpy.ndarray
+        array from which ds values will be taken
+    silent_overflow : bool, optional
+        whether to shrink the end index to match the
+        shape of ds (default: True)
+    """
+    start = list(start)
+    end = list(end)
+    if len(ds.shape) == 5:  # dataset dimensions should be 3 or 5
+        for i in range(3):
+            if end[i] >= ds.shape[2+i] and silent_overflow:
+                end[i] = ds.shape[2+i]
+        #if end > start:
+        ds[0, 0, start[0]:end[0], start[1]:end[1], start[2]:end[2]] = arr[:(end[0] - start[0]), :(end[1] - start[1]), :(end[2] - start[2])]
+    elif len(ds.shape) == 3:
+        for i in range(3):    
+            if end[i] >= ds.shape[i] and silent_overflow:
+                end[i] = ds.shape[i]
+        #if end > start:
+        ds[start[0]:end[0], start[1]:end[1], start[2]:end[2]] = arr[:(end[0] - start[0]), :(end[1] - start[1]), :(end[2] - start[2])]
+
+
+def write_mips(zgrp,miparrs):
+    for miparr in miparrs:
+        dswrite_block(ds=zgrp[miparr.lvl],start=miparr.start,end=miparr.end,arr=miparr.array)
+
+# HERE : NEED TO UNTANGLE MISMATCH BETWEEN BLOCK (deskewed) and CHUNK (skewed) size
+def iterate_numpy_blocks_from_dataset(
+        dataset, nblocks, block_size=None, pad=True, deskew_kwargs={}, *args, **kwargs):
+    """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
+
+    Parameters
+    ----------
+    dataset : hdf5 dataset
+        imageio-compatible name inputs to be opened as multi-images
+    nblocks : tuple[int]
+        number of 2d arrays from mimgfns included per chunk
+    pad : bool, optional
+        whether to extend final returned chunks with zeros
+
+    Yields
+    ------
+    arr : numpy.ndarray
+        3D numpy array representing a consecutive chunk of 2D arrays
+    """
+    # if deskew_kwargs and deskew_kwargs["deskew_method"] == "ps":
+    #     # create output block and get flattened indices
+    #     zb,yb,xb = numpy.meshgrid(*[range(d) for d in block_size],indexing="ij")
+    #     fb = numpy.ravel_multi_index((zb,yb,xb),block_size)
+    if nblocks[1] == 1:
+        print("test condition: using test tuple")
+        test = True
+    else:
+        test = False
+    dshape = dataset.shape[2:]
+    if deskew_kwargs:
+        chunk_size = (deskew_kwargs["chunklength"],block_size[1],block_size[2]*deskew_kwargs["stride"])
+        if deskew_kwargs["transpose"]:
+            chunk_size = (chunk_size[0],chunk_size[2],chunk_size[1])
+            dshape = (dshape[0],dshape[2],dshape[1])
+        print("chunk size: " + str(chunk_size))
+    for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
+        chunk_tuple = numpy.unravel_index(i,tuple(nblocks),order='F')
+        if test:
+            chunk_tuple = (chunk_tuple[0], 11, chunk_tuple[2])
+        # deskew level 0 data blocks
+        if deskew_kwargs:
+            if deskew_kwargs["transpose"]:
+                chunk_tuple = (chunk_tuple[0],chunk_tuple[2],chunk_tuple[1])
+            print(str(chunk_tuple))
+            if chunk_tuple[0] == 0:
+                chunk_index = 0
+                deskew_kwargs["slice1d"][...] = 0
+                first_z,first_slice = psd.calculate_first_chunk(chunk_size=chunk_size,x_index=(nblocks[2] - chunk_tuple[1] - 1),stride=deskew_kwargs["stride"])
+            if chunk_tuple[0] < first_z or chunk_tuple[0]*chunk_size[0] - first_slice >= dshape[0]:
+                arr = numpy.zeros(block_size,dtype=dataset.dtype)
+            else:
+                chunk_start = numpy.array([t*s for t,s in zip(chunk_tuple,chunk_size)])
+                chunk_end = chunk_start + numpy.array(chunk_size)
+                if chunk_start[0] < first_slice:
+                    chunk_end[0] -= first_slice - chunk_start[0]
+                    chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
+                    chunk[first_slice-chunk_start[0]:] = dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
+                else:
+                    chunk_start[0] -= first_slice
+                    chunk_end[0] -= first_slice
+                    # if chunk_end[0] >= dshape[0]:
+                    #     chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
+                    #     chunk[:dshape[0]-chunk_start[0]] = dataset[chunk_start[0]:,chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
+                    # else:
+                    chunk = dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
+                if any([sh<sz for sh,sz in zip(chunk.shape,chunk_size)]):
+                    print(str(chunk.shape) + " is small for" + str(chunk_size) + ": filling with zeros")
+                    temp_chunk = numpy.zeros(chunk_size,dtype=chunk.dtype)
+                    temp_chunk[:chunk.shape[0],:chunk.shape[1],:chunk.shape[2]] = chunk
+                    chunk = temp_chunk
+                if deskew_kwargs["transpose"]:
+                    chunk = chunk.transpose((0,2,1))
+                arr = numpy.flip(
+                    numpy.transpose(
+                        psd.deskew_block(
+                            chunk,
+                            chunk_index,
+                            **deskew_kwargs), 
+                        (2, 1, 0)),
+                    axis=2)
+            chunk_index += 1
+        else:
+            if chunk_tuple[0] == 0:
+                print(str(chunk_tuple))
+            block_start = [chunk_tuple[k]*block_size[k] for k in range(3)]
+            block_end = [block_start[k] + block_size[k] for k in range(3)]
+            arr = dataset[0,0,block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]]
+        if any([arr.shape[k] != block_size[k] for k in range(3)]):
+            print(str(arr.shape) + "is small for " + str(block_size))
+            if pad:
+                newarr = numpy.zeros(block_size,
+                                      dtype=arr.dtype)
+                newarr[:arr.shape[0], :arr.shape[1], :arr.shape[2]] = arr[:, :, :]
+                yield newarr
+            else:
+                yield arr
+        else:
+            yield arr
+
+
+def iterate_mip_levels_from_dataset(
+        dataset, lvl, maxlvl, nblocks, block_size, downsample_factor,
+        downsample_method=None, lvl_to_mip_kwargs=None,
+        interleaved_channels=1, channel=0, deskew_kwargs={}):
+    """recursively generate MIPmap levels from an iterator of blocks
+
+    Parameters
+    ----------
+    dataset : hdf5 dataset
+        imageio-compatible name inputs to be opened as multi-images
+    lvl : int
+        integer mip level to generate
+    block_size : int
+        number of 2D arrays in chunk to process for a chunk at lvl
+    slice_length : int
+        number of 2D arrays to gather from tiff stacks
+    downsample_factor : tuple of int
+        integer downsampling factor for MIP levels
+    downsample_method : str, optional
+        downsampling method for
+        acpreprocessing.utils.convert.downsample_stack_volume
+    lvl_to_mip_kwargs :  dict, optional
+        mapping of MIP level to kwargs used in MIPmap generation
+    interleaved_channels : int
+        number of channels interleaved in the tiff files (default 1)
+    channel : int, optional
+        channel from which interleaved data should be read (default 0)
+    deskew_kwargs : dict, optional
+        parameters for pixel shifting deskew
+
+    Yields
+    ------
+    ma : acpreprocessing.stitching_modules.convert_to_n5.tiff_to_n5.MipArray
+        object describing chunked array, MIP level of origin, and chunk indices
+    """
+    lvl_to_mip_kwargs = ({} if lvl_to_mip_kwargs is None
+                          else lvl_to_mip_kwargs)
+    mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
+    block_index = 0
+    if lvl > 0:
+        for ma in iterate_mip_levels_from_dataset(
+                dataset, lvl-1, maxlvl, nblocks, block_size,
+                downsample_factor, downsample_method,
+                lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
+                channel=channel, deskew_kwargs=deskew_kwargs):
+            chunk = ma.array
+            # throw array up for further processing
+            yield ma
+            # ignore if not parent resolution
+            if ma.lvl != lvl-1:
+                continue
+            
+            temp_arr = (downsample_array(
+                    chunk, downsample_factor, dtype=chunk.dtype,
+                    method=downsample_method, **mip_kwargs))
+            chunk_start = tuple(int(ma.start[k]/downsample_factor[k]) for k in range(3))
+            chunk_end = tuple(chunk_start[k] + temp_arr.shape[k] for k in range(3))
+            yield MIPArray(lvl, temp_arr, chunk_start, chunk_end)
+    else:
+        # get level 0 chunks
+        # block_size is the number of slices to read from tiffs
+        for block in iterate_numpy_blocks_from_dataset(
+                dataset, nblocks, block_size=block_size, pad=False,
+                deskew_kwargs=deskew_kwargs,
+                channel=channel):
+            block_tuple = numpy.unravel_index(block_index,nblocks,order='F')
+            block_start = tuple(block_tuple[k]*block_size[k] for k in range(3))
+            block_end = tuple(block_start[k] + block.shape[k] for k in range(3))
+            yield MIPArray(lvl, block, block_start, block_end)
+            block_index += 1
+
+
+def write_zarr_to_zarr(
+        zarr_fn, zarr_grp, output_n5, group_names, group_attributes=None, max_mip=0,
+        mip_dsfactor=(2, 2, 2), chunk_size=(1, 1, 64, 64, 64),
+        concurrency=10, slice_concurrency=1,
+        compression="raw", dtype="uint16", lvl_to_mip_kwargs=None,
+        interleaved_channels=1, channel=0, deskew_options=None, numchunks=0, **kwargs):
+    """write a stack represented by an iterator of multi-image files as a zarr
+    volume with ome-ngff metadata
+
+    Parameters
+    ----------
+    ims_fn : str
+        imageio-compatible name inputs to be opened as multi-images
+    output_n5 : str
+        output zarr directory
+    group_names : list of str
+        names of groups to generate within n5
+    group_attributes : list of dict, optional
+        attribute dictionaries corresponding to group with matching index
+    max_mip : int
+        maximum MIP level to generate
+    mip_dsfactor : tuple of int
+        integer downsampling factor for MIP levels
+    chunk_size : tuple of int
+        chunk size for n5 datasets
+    concurrency : int
+        total concurrency used for writing arrays to n5
+        (python threads = concurrency // slice_concurrency)
+    slice_concurrency : int
+        threads used by z5py
+    compression : str, optional
+        compression for n5 (default: raw)
+    dtype : str, optional
+        dtype for n5 (default: uint16)
+    lvl_to_mip_kwargs :  dict, optional
+        mapping of MIP level to kwargs used in MIPmap generation
+    interleaved_channels : int
+        number of channels interleaved in the tiff files (default 1)
+    channel : int, optional
+        channel from which interleaved data should be read (default 0)
+    deskew_options : dict, optional
+        dictionary of parameters to run pixel shifting deskew (default None)
+    """
+    group_attributes = ([] if group_attributes is None else group_attributes)
+    deskew_options = ({} if deskew_options is None else deskew_options)
+    
+    zgrp = get_zarr_group(zpath=zarr_fn,grpname=zarr_grp)
+    dataset = zgrp[0]
+    # if deskew_options and deskew_options["deskew_transpose"]:
+    #     dataset = dataset.transpose((0,2,1))
+    #     print("transposed shape: " + str(dataset.shape))
+    
+    block_size = [m*sz for m,sz in zip([4,4,4],chunk_size[2:])]
+    print("deskewed block size: " + str(block_size))
+    dshape = dataset.shape[2:]
+    if numchunks < 1:
+        joined_shapes = dshape
+        if deskew_options and deskew_options["deskew_transpose"]:
+            # input dataset must be transposed
+            joined_shapes = (joined_shapes[0],joined_shapes[2],joined_shapes[1])
+    else:
+        if deskew_options and deskew_options["deskew_transpose"]:
+            # input dataset must be transposed
+            joined_shapes = (dshape[0],numchunks*block_size[1],dshape[1])
+        else:
+            joined_shapes = (dshape[0],numchunks*block_size[1],dshape[2])
+    print("ims_to_ngff dataset shape:" + str(joined_shapes))
+
+    if deskew_options and deskew_options["deskew_method"] == "ps":
+        print(deskew_options)
+        if not deskew_options["deskew_stride"] is None:
+            stride = deskew_options["deskew_stride"]
+        else:
+            stride = 1
+        slice_length = int(block_size[0]/stride)
+        deskew_kwargs = psd.psdeskew_kwargs(skew_dims_zyx=(slice_length, block_size[1], block_size[2]*stride), # size of skewed chunk to iterate for deskewed block
+                                            **deskew_options)
+        joined_shapes = psd.reshape_joined_shapes(
+            joined_shapes, stride, block_size)# transpose=(2,1,0))
+        print("deskewed shape:" + str(joined_shapes))
+    else:
+        deskew_kwargs = {}
+
+    # TODO DESKEW: does this generally work regardless of skew?
+
+    workers = concurrency // slice_concurrency
+
+    zstore = zarr.DirectoryStore(output_n5, dimension_separator='/')
+    with zarr.open(zstore, mode='a') as f:
+        mip_ds = {}
+        # create groups with attributes according to omezarr spec
+        if len(group_names) == 1:
+            group_name = group_names[0]
+            try:
+                g = f.create_group(f"{group_name}")
+            except KeyError:
+                g = f[f"{group_name}"]
+            if group_attributes:
+                try:
+                    attributes = group_attributes[0]
+                except IndexError:
+                    print('attributes error')
+            else:
+                attributes = {}
+
+            if "pixelResolution" in attributes:
+                if deskew_options:
+                    attributes["pixelResolution"]["dimensions"][2] /= deskew_options["deskew_stride"]
+                attributes = omezarr_attrs(
+                    group_name, attributes["position"], attributes["pixelResolution"]["dimensions"], max_mip)
+            if attributes:
+                for k, v in attributes.items():
+                    g.attrs[k] = v
+        else:
+            raise TiffToNGFFValueError("only one group name expected")
+        scales = []
+
+        # shuffle=Blosc.BITSHUFFLE)
+        compression = Blosc(cname='zstd', clevel=1)
+        for mip_lvl in range(max_mip + 1):
+            mip_3dshape = mip_level_shape(mip_lvl, joined_shapes)
+            ds_lvl = g.create_dataset(
+                f"{mip_lvl}",
+                chunks=chunk_size,
+                shape=(1, 1, mip_3dshape[0], mip_3dshape[1], mip_3dshape[2]),
+                compression=compression,
+                dtype=dtype
+            )
+            dsfactors = [int(i)**mip_lvl for i in mip_dsfactor]
+            mip_ds[mip_lvl] = ds_lvl
+            scales.append(dsfactors)
+        
+        nblocks = [int(numpy.ceil(joined_shapes[k]/block_size[k])) for k in range(3)]
+        print(str(nblocks) + " number of chunks per axis")
+        print(str(g[0].nchunks) + " chunk number sanity check")
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as e:
+            futs = []
+            mips = []
+            for miparr in iterate_mip_levels_from_dataset(
+                    dataset, max_mip, max_mip, nblocks, block_size, mip_dsfactor,
+                    lvl_to_mip_kwargs=lvl_to_mip_kwargs,
+                    interleaved_channels=interleaved_channels,
+                    channel=channel, deskew_kwargs=deskew_kwargs):
+                mips.append(miparr)
+                if miparr.lvl == max_mip:
+                    # futs.append(e.submit(
+                    #     dswrite_block, mip_ds[miparr.lvl],
+                    #     miparr.start, miparr.end, miparr.array))
+                    futs.append(e.submit(
+                        write_mips, mip_ds, mips))
+                    mips = []
+            for fut in concurrent.futures.as_completed(futs):
+                _ = fut.result()
+    print("conversion complete, closing file")
+    f.close()
+
+
+def zarr_to_ngff_group(zarr_fn, zarr_grp, output, *args, **kwargs):
+    """convert directory of natsort-consecutive multitiffs to an n5 or zarr pyramid
+
+    Parameters
+    ----------
+    tiffdir : str
+        directory of consecutive multitiffs to convert
+    """
+
+    if output == 'zarr':
+        print('converting to zarr')
+        return write_zarr_to_zarr(zarr_fn, zarr_grp, *args, **kwargs)
+    else:
+        print('unknown output format: ' + output)
+
+
+
+class ZarrToNGFFParameters(NGFFGroupGenerationParameters):
+    input_file = argschema.fields.Str(required=True)
+    input_group = argschema.fields.Str(required=True)
+    interleaved_channels = argschema.fields.Int(required=False, default=1)
+    channel = argschema.fields.Int(required=False, default=0)
+    num_chunks = argschema.fields.Int(required=False, default=0)
+
+
+class ZarrToZarrInputParameters(argschema.ArgSchema,
+                                    ZarrToNGFFParameters):
+    chunk_size = argschema.fields.Tuple((
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int()), required=False, default=(1, 1, 64, 64, 64))
+
+
+
+class ZarrToZarr(argschema.ArgSchemaParser):
+    default_schema = ZarrToZarrInputParameters
+
+    def run(self):
+        deskew_options = (self.args["deskew_options"]
+                          if "deskew_options" in self.args else {})
+        zarr_to_ngff_group(
+            self.args["input_file"], self.args["input_group"], self.args["output_format"],
+            self.args["output_file"], self.args["group_names"],
+            self.args["group_attributes"],
+            self.args["max_mip"],
+            self.args["mip_dsfactor"],
+            self.args["chunk_size"],
+            concurrency=self.args["concurrency"],
+            compression=self.args["compression"],
+            #lvl_to_mip_kwargs=self.args["lvl_to_mip_kwargs"],
+            numchunks = self.args["num_chunks"],
+            deskew_options=deskew_options)
+
+
+
+if __name__ == "__main__":
+    mod = ZarrToZarr()
+    mod.run()

From eba27ee092932ec5aaafebea5738376cd888e2c1 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 18 Sep 2024 20:36:05 -0700
Subject: [PATCH 069/115] testing block size change

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py          | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 84638f4..9e30dda 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -242,7 +242,7 @@ def iterate_mip_levels_from_dataset(
 
 def write_ims_to_zarr(
         ims_fn, output_n5, group_names, group_attributes=None, max_mip=0,
-        mip_dsfactor=(2, 2, 2), chunk_size=(1, 1, 64, 64, 64),
+        mip_dsfactor=(2, 2, 2), chunk_size=(1, 1, 32, 32, 32),
         concurrency=10, slice_concurrency=1,
         compression="raw", dtype="uint16", lvl_to_mip_kwargs=None,
         interleaved_channels=1, channel=0, deskew_options=None, numchunks=0, **kwargs):
@@ -292,7 +292,7 @@ def write_ims_to_zarr(
     #     dataset = dataset.transpose((0,2,1))
     #     print("transposed shape: " + str(dataset.shape))
     
-    block_size = [m*sz for m,sz in zip([4,4,4],chunk_size[2:])]
+    block_size = [m*sz for m,sz in zip([32,32,32],chunk_size[2:])]
     print("deskewed block size: " + str(block_size))
     
     if numchunks < 1:
@@ -429,7 +429,7 @@ class IMSToZarrInputParameters(argschema.ArgSchema,
         argschema.fields.Int(),
         argschema.fields.Int(),
         argschema.fields.Int(),
-        argschema.fields.Int()), required=False, default=(1, 1, 64, 64, 64))
+        argschema.fields.Int()), required=False, default=(1, 1, 32, 32, 32)) # default=(1, 1, 64, 64, 64)
 
 
 

From feec173b36d2045cdcf6faf8e605f97db3ab7bf4 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Sun, 22 Sep 2024 19:49:23 -0700
Subject: [PATCH 070/115] reviving pyramid downsample

---
 .../convert_to_n5/ims_to_ngff.py              | 81 ++++++++++++++-----
 1 file changed, 62 insertions(+), 19 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 9e30dda..f770813 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -207,7 +207,11 @@ def iterate_mip_levels_from_dataset(
                           else lvl_to_mip_kwargs)
     mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
     block_index = 0
+    start_index = 0
     if lvl > 0:
+        num_chunks = downsample_factor[0]
+        
+        i = 0
         for ma in iterate_mip_levels_from_dataset(
                 dataset, lvl-1, maxlvl, nblocks, block_size,
                 downsample_factor, downsample_method,
@@ -220,12 +224,46 @@ def iterate_mip_levels_from_dataset(
             if ma.lvl != lvl-1:
                 continue
             
-            temp_arr = (downsample_array(
-                    chunk, downsample_factor, dtype=chunk.dtype,
-                    method=downsample_method, **mip_kwargs))
-            chunk_start = tuple(int(ma.start[k]/downsample_factor[k]) for k in range(3))
-            chunk_end = tuple(chunk_start[k] + temp_arr.shape[k] for k in range(3))
-            yield MIPArray(lvl, temp_arr, chunk_start, chunk_end)
+            try:
+                temp_lminus1_arr
+            except NameError:
+                temp_lminus1_arr = numpy.empty(
+                    (num_chunks*chunk.shape[0], *chunk.shape[1:]),
+                    dtype=chunk.dtype)
+            
+            # fill in temporary block according to index
+            chunk_size = chunk.shape[0]
+            block_offset = i*block_size
+            
+            temp_lminus1_arr[
+                block_offset:block_offset+chunk_size, :, :] = chunk[:, :, :]
+            
+            # copy op only for uneven final chunk
+            if chunk_size != block_size:
+                temp_lminus1_arr = temp_lminus1_arr[
+                    :block_offset+chunk_size, :, :]
+            
+            if i == num_chunks - 1:
+                temp_arr = (
+                    downsample_array(
+                        temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
+                        method=downsample_method, **mip_kwargs))
+                # end_index = start_index + temp_arr.shape[0]
+                # yield MIPArray(lvl, temp_arr, start_index, end_index)
+                chunk_start = tuple(start_index, int(ma.start[1]/downsample_factor[1]), int(ma.start[2]/downsample_factor[2]))#tuple(int(ma.start[k]/downsample_factor[k]) for k in range(3))
+                chunk_end = tuple(chunk_start[k] + temp_arr.shape[k] for k in range(3))
+                yield MIPArray(lvl, temp_arr, chunk_start, chunk_end)
+                start_index += temp_arr.shape[0]
+                i = 0
+            else:
+                i += 1
+            
+            # temp_arr = (downsample_array(
+            #         chunk, downsample_factor, dtype=chunk.dtype,
+            #         method=downsample_method, **mip_kwargs))
+            # chunk_start = tuple(int(ma.start[k]/downsample_factor[k]) for k in range(3))
+            # chunk_end = tuple(chunk_start[k] + temp_arr.shape[k] for k in range(3))
+            # yield MIPArray(lvl, temp_arr, chunk_start, chunk_end)
     else:
         # get level 0 chunks
         # block_size is the number of slices to read from tiffs
@@ -242,7 +280,7 @@ def iterate_mip_levels_from_dataset(
 
 def write_ims_to_zarr(
         ims_fn, output_n5, group_names, group_attributes=None, max_mip=0,
-        mip_dsfactor=(2, 2, 2), chunk_size=(1, 1, 32, 32, 32),
+        mip_dsfactor=(2, 2, 2), chunk_size=(1, 1, 64, 64, 64),
         concurrency=10, slice_concurrency=1,
         compression="raw", dtype="uint16", lvl_to_mip_kwargs=None,
         interleaved_channels=1, channel=0, deskew_options=None, numchunks=0, **kwargs):
@@ -292,7 +330,7 @@ def write_ims_to_zarr(
     #     dataset = dataset.transpose((0,2,1))
     #     print("transposed shape: " + str(dataset.shape))
     
-    block_size = [m*sz for m,sz in zip([32,32,32],chunk_size[2:])]
+    block_size = [m*sz for m,sz in zip([1,2**max_mip,2**max_mip],chunk_size[2:])]
     print("deskewed block size: " + str(block_size))
     
     if numchunks < 1:
@@ -378,20 +416,25 @@ def write_ims_to_zarr(
 
         with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as e:
             futs = []
-            mips = []
+            # mips = []
+            # for miparr in iterate_mip_levels_from_dataset(
+            #         dataset, max_mip, max_mip, nblocks, block_size, mip_dsfactor,
+            #         lvl_to_mip_kwargs=lvl_to_mip_kwargs,
+            #         interleaved_channels=interleaved_channels,
+            #         channel=channel, deskew_kwargs=deskew_kwargs):
+            #     mips.append(miparr)
+            #     if miparr.lvl == max_mip:
+            #         futs.append(e.submit(
+            #             write_mips, mip_ds, mips))
+            #         mips = []
             for miparr in iterate_mip_levels_from_dataset(
-                    dataset, max_mip, max_mip, nblocks, block_size, mip_dsfactor,
+                    dataset, max_mip, nblocks, block_size, mip_dsfactor,
                     lvl_to_mip_kwargs=lvl_to_mip_kwargs,
                     interleaved_channels=interleaved_channels,
                     channel=channel, deskew_kwargs=deskew_kwargs):
-                mips.append(miparr)
-                if miparr.lvl == max_mip:
-                    # futs.append(e.submit(
-                    #     dswrite_block, mip_ds[miparr.lvl],
-                    #     miparr.start, miparr.end, miparr.array))
-                    futs.append(e.submit(
-                        write_mips, mip_ds, mips))
-                    mips = []
+                futs.append(e.submit(
+                    dswrite_block, mip_ds[miparr.lvl],
+                    miparr.start, miparr.end, miparr.array))
             for fut in concurrent.futures.as_completed(futs):
                 _ = fut.result()
     print("conversion complete, closing file")
@@ -429,7 +472,7 @@ class IMSToZarrInputParameters(argschema.ArgSchema,
         argschema.fields.Int(),
         argschema.fields.Int(),
         argschema.fields.Int(),
-        argschema.fields.Int()), required=False, default=(1, 1, 32, 32, 32)) # default=(1, 1, 64, 64, 64)
+        argschema.fields.Int()), required=False, default=(1, 1, 64, 64, 64))
 
 
 

From cb885bde9b19d35216ae6b200c270698f9c1eba8 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Sun, 22 Sep 2024 20:31:24 -0700
Subject: [PATCH 071/115] debugging

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index f770813..d077d89 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -428,7 +428,7 @@ def write_ims_to_zarr(
             #             write_mips, mip_ds, mips))
             #         mips = []
             for miparr in iterate_mip_levels_from_dataset(
-                    dataset, max_mip, nblocks, block_size, mip_dsfactor,
+                    dataset, max_mip, max_mip, nblocks, block_size, mip_dsfactor,
                     lvl_to_mip_kwargs=lvl_to_mip_kwargs,
                     interleaved_channels=interleaved_channels,
                     channel=channel, deskew_kwargs=deskew_kwargs):

From f8e7a8d9efa593258bbef0a720d8275653018f20 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Sun, 22 Sep 2024 20:58:58 -0700
Subject: [PATCH 072/115] debugging

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py            | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index d077d89..b8a8531 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -233,13 +233,13 @@ def iterate_mip_levels_from_dataset(
             
             # fill in temporary block according to index
             chunk_size = chunk.shape[0]
-            block_offset = i*block_size
+            block_offset = i*block_size[0]
             
             temp_lminus1_arr[
                 block_offset:block_offset+chunk_size, :, :] = chunk[:, :, :]
             
             # copy op only for uneven final chunk
-            if chunk_size != block_size:
+            if chunk_size != block_size[0]:
                 temp_lminus1_arr = temp_lminus1_arr[
                     :block_offset+chunk_size, :, :]
             

From 7666e2019bb063e581dd4479c4d18665dde5a0b4 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Sun, 22 Sep 2024 21:04:34 -0700
Subject: [PATCH 073/115] debugging

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index b8a8531..6aa5847 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -250,7 +250,7 @@ def iterate_mip_levels_from_dataset(
                         method=downsample_method, **mip_kwargs))
                 # end_index = start_index + temp_arr.shape[0]
                 # yield MIPArray(lvl, temp_arr, start_index, end_index)
-                chunk_start = tuple(start_index, int(ma.start[1]/downsample_factor[1]), int(ma.start[2]/downsample_factor[2]))#tuple(int(ma.start[k]/downsample_factor[k]) for k in range(3))
+                chunk_start = tuple((start_index, int(ma.start[1]/downsample_factor[1]), int(ma.start[2]/downsample_factor[2])))#tuple(int(ma.start[k]/downsample_factor[k]) for k in range(3))
                 chunk_end = tuple(chunk_start[k] + temp_arr.shape[k] for k in range(3))
                 yield MIPArray(lvl, temp_arr, chunk_start, chunk_end)
                 start_index += temp_arr.shape[0]

From ca9399aa3abeffba9055aff698e3cb6de16bb895 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Sun, 22 Sep 2024 21:19:46 -0700
Subject: [PATCH 074/115] debugging

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py            | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 6aa5847..4e051cf 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -105,8 +105,8 @@ def iterate_numpy_blocks_from_dataset(
         print("chunk size: " + str(chunk_size))
     for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nblocks),order='F')
-        if test:
-            chunk_tuple = (chunk_tuple[0], 11, chunk_tuple[2])
+        # if test:
+        #     chunk_tuple = (chunk_tuple[0], 11, chunk_tuple[2])
         # deskew level 0 data blocks
         if deskew_kwargs:
             if deskew_kwargs["transpose"]:

From c01fd11c0aabe4ac18e4e2aac128b2fe72e904b1 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 24 Sep 2024 10:20:05 -0700
Subject: [PATCH 075/115] hot fix for smart conversion

---
 .../convert_to_n5/acquisition_dir_to_ngff.py             | 9 ++++-----
 .../stitching_modules/convert_to_n5/psdeskew.py          | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/acquisition_dir_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/acquisition_dir_to_ngff.py
index c79fcc8..af8f8b8 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/acquisition_dir_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/acquisition_dir_to_ngff.py
@@ -90,14 +90,13 @@ def acquisition_to_ngff(acquisition_dir, output, out_dir, concurrency=5, acq_par
     interleaved_channels = get_number_interleaved_channels_from_rootdir(
         acquisition_path)
     positionList = get_strip_positions_from_rootdir(acquisition_path)
+    ori = (1,1,1)
+    axes = (0,1,2)
     if acq_parameters and "stage_axes" in acq_parameters:
         axesStr = acq_parameters["stage_axes"]
         if axesStr=="yxz":
             axes = (1,0,2)
-        else:
-            axes = (0,1,2)
-    else:
-        axes = (0,1,2)
+            ori = (-1,1,1)
 
     try:
         setup_group_attributes = [{
@@ -106,7 +105,7 @@ def acquisition_to_ngff(acquisition_dir, output, out_dir, concurrency=5, acq_par
                     acquisition_path),
                 "unit": "um"
             },
-            "position": tuple(p[i] for i in axes)
+            "position": tuple(p[i]*o for i,o in zip(axes,ori))
         } for p in positionList]
     except (KeyError, FileNotFoundError):
         setup_group_attributes = {}
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 7b29f7d..4ec9686 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -42,7 +42,7 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_tr
     ydim = int(sdims[1]*crop_factor)
     blockdims = (int(sdims[2]/stride), ydim, stride*sdims[0])
     subblocks = int(np.ceil((sdims[2]+stride*sdims[0])/(stride*sdims[0])))
-    # print(subblocks)
+    print("number of subblocks = " + str(subblocks))
     blockx = sdims[0]
     dsi = []
     si = []

From cc474cf0892482fad05c5171053ffdfa42e97464 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 24 Sep 2024 20:20:36 -0700
Subject: [PATCH 076/115] reverting deskew to loop

---
 .../convert_to_n5/psdeskew.py                 | 59 ++++++++++++++-----
 1 file changed, 43 insertions(+), 16 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 4ec9686..30d1213 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -46,12 +46,37 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_tr
     blockx = sdims[0]
     dsi = []
     si = []
+    # for i_block in range(subblocks):
+    #     sxv = []
+    #     szv = []
+    #     for sz in range(blockx):
+    #         sxstart = i_block*stride*blockx-stride*sz
+    #         sxend = sxstart + stride*blockx #(i_block+1)*stride*blockx-stride*sz
+    #         if sxstart < 0:
+    #             sxstart = 0
+    #         if sxend > sdims[2]:
+    #             sxend = sdims[2]
+    #         sx = np.arange(sxstart, sxend)
+    #         sxv.append(sx)
+    #         szv.append(sz*np.ones(sx.shape, dtype=sx.dtype))
+    #     sxv = np.tile(np.concatenate(sxv)[:,np.newaxis],[1,blockdims[1]])
+    #     szv = np.tile(np.concatenate(szv)[:,np.newaxis],[1,blockdims[1]])
+    #     syv = np.tile(np.arange(blockdims[1]),[sxv.shape[0],1])
+    #     sxv = sxv.flatten()
+    #     szv = szv.flatten()
+    #     syv = syv.flatten()
+    #     dsx = sxv + stride*szv - i_block*stride*blockx
+    #     dsz = np.floor(sxv/stride).astype(int)
+    #     dsy = syv
+    #     dsi.append(np.ravel_multi_index(
+    #         (dsz,dsy,dsx), blockdims))
+    #     si.append(np.ravel_multi_index((szv,syv,sxv), sdims))
     for i_block in range(subblocks):
         sxv = []
         szv = []
         for sz in range(blockx):
             sxstart = i_block*stride*blockx-stride*sz
-            sxend = sxstart + stride*blockx #(i_block+1)*stride*blockx-stride*sz
+            sxend = (i_block+1)*stride*blockx-stride*sz
             if sxstart < 0:
                 sxstart = 0
             if sxend > sdims[2]:
@@ -59,18 +84,13 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_tr
             sx = np.arange(sxstart, sxend)
             sxv.append(sx)
             szv.append(sz*np.ones(sx.shape, dtype=sx.dtype))
-        sxv = np.tile(np.concatenate(sxv)[:,np.newaxis],[1,blockdims[1]])
-        szv = np.tile(np.concatenate(szv)[:,np.newaxis],[1,blockdims[1]])
-        syv = np.tile(np.arange(blockdims[1]),[sxv.shape[0],1])
-        sxv = sxv.flatten()
-        szv = szv.flatten()
-        syv = syv.flatten()
+        sxv = np.concatenate(sxv)
+        szv = np.concatenate(szv)
         dsx = sxv + stride*szv - i_block*stride*blockx
         dsz = np.floor(sxv/stride).astype(int)
-        dsy = syv
         dsi.append(np.ravel_multi_index(
-            (dsz,dsy,dsx), blockdims))
-        si.append(np.ravel_multi_index((szv,syv,sxv), sdims))
+            (dsz, dsx), (blockdims[0], blockdims[2])))
+        si.append(np.ravel_multi_index((szv, sxv), (sdims[0], sdims[2])))
     kwargs = {'dsi': dsi,
               'si': si,
               'slice1d': np.zeros((subblocks, blockdims[2]*blockdims[1]*blockdims[0]), dtype=dtype),
@@ -170,12 +190,19 @@ def deskew_block(blockData, n, dsi, si, slice1d, blockdims, subblocks, flip, tra
         blockData = np.concatenate((blockData, np.zeros(
             (int(chunklength-blockData.shape[0]), blockData.shape[1], blockData.shape[2]))))
     order = (np.arange(subb)+n) % subb
-    for i, o in enumerate(order):
-        # flip stack axis 2 for ispim2
-        s = -1 if flip else 1
-        slice1d[o, :][dsi[i]] = blockData[:, :, ::s].ravel()[si[i]]
-    block3d[:, :, :] = slice1d[n % subb, :].reshape((zdim,ydim,xdim))
-    slice1d[n % subb, :] = 0
+    # for i, o in enumerate(order):
+    #     # flip stack axis 2 for ispim2
+    #     s = -1 if flip else 1
+    #     slice1d[o, :][dsi[i]] = blockData[:, :, ::s].ravel()[si[i]]
+    # block3d[:, :, :] = slice1d[n % subb, :].reshape((zdim,ydim,xdim))
+    # slice1d[n % subb, :] = 0
+    for y in range(ydim):
+        for i, o in enumerate(order):
+            # flip stack axis 2 for ispim2
+            s = -1 if flip else 1
+            slice1d[o, y, :][dsi[i]] = blockData[:, y, ::s].ravel()[si[i]]
+        block3d[:, y, :] = slice1d[n % subb, y, :].reshape((zdim, xdim))
+        slice1d[n % subb, y, :] = 0
     return block3d
 
 

From 4136d95b103a7d629a7de88f4a50d63c13f13ffd Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 24 Sep 2024 20:27:00 -0700
Subject: [PATCH 077/115] cleaning code

---
 .../convert_to_n5/psdeskew.py                 | 31 +------------------
 1 file changed, 1 insertion(+), 30 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
index 30d1213..21e5683 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/psdeskew.py
@@ -93,7 +93,7 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_tr
         si.append(np.ravel_multi_index((szv, sxv), (sdims[0], sdims[2])))
     kwargs = {'dsi': dsi,
               'si': si,
-              'slice1d': np.zeros((subblocks, blockdims[2]*blockdims[1]*blockdims[0]), dtype=dtype),
+              'slice1d': np.zeros((subblocks, blockdims[1], blockdims[2]*blockdims[0]), dtype=dtype), #np.zeros((subblocks, blockdims[2]*blockdims[1]*blockdims[0]), dtype=dtype),
               'blockdims': blockdims,
               'subblocks': subblocks,
               'flip': deskew_flip,
@@ -104,35 +104,6 @@ def psdeskew_kwargs(skew_dims_zyx, deskew_stride=1, deskew_flip=False, deskew_tr
               }
     return kwargs
 
-# commented code no longer in use
-#
-# def calculate_skewed_indices(zi,yi,xi,s):
-#     # convert input block voxel indices into skewed data space
-#     # edge blocks may have negative or out-of-bounds skewed indices
-#     xs = s*zi + xi % s
-#     ys = yi
-#     zs = xi // s - zi
-#     return zs,ys,xs
-
-
-# def get_deskewed_block(blockdims,dataset,start,end,stride):
-#     # create output block and get flattened indices
-#     blockdata = np.zeros(blockdims,dtype=dataset.dtype)
-#     zb,yb,xb = np.meshgrid(*[range(d) for d in blockdims],indexing="ij")
-#     fb = np.ravel_multi_index((zb,yb,xb),blockdims)
-#     # get indices of voxel data for input dataset
-#     sdims = dataset.shape
-#     zi,yi,xi = np.meshgrid(*[range(s,e) for s,e in zip(start,end)],indexing="ij")
-#     zs,ys,xs = calculate_skewed_indices(zi,yi,xi,stride)
-#     fi = np.ravel_multi_index((zs,ys,xs),sdims,mode='clip').flatten()
-#     # filter out-of-bounds voxels
-#     r = (fi > 0) & (fi < np.prod(sdims)-1)
-#     fb = fb[r]
-#     fi = fi[r]
-#     # assign input to output
-#     blockdata[fb] = dataset[fi]
-#     return blockdata
-
 
 def calculate_first_chunk(chunk_size,x_index,stride):
     first_chunk = x_index*stride

From d664791e15678b5328d1cd5b05a44869f38a18a3 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Wed, 25 Sep 2024 18:19:50 -0700
Subject: [PATCH 078/115] block size testing

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 4e051cf..c5d1345 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -330,7 +330,7 @@ def write_ims_to_zarr(
     #     dataset = dataset.transpose((0,2,1))
     #     print("transposed shape: " + str(dataset.shape))
     
-    block_size = [m*sz for m,sz in zip([1,2**max_mip,2**max_mip],chunk_size[2:])]
+    block_size = [128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
     print("deskewed block size: " + str(block_size))
     
     if numchunks < 1:

From b7813a782e57bdfcdea18234de49abfa1f36c397 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Thu, 26 Sep 2024 13:09:43 -0700
Subject: [PATCH 079/115] cleaning up code

---
 .../convert_to_n5/ims_to_ngff.py              | 29 +++++++------------
 1 file changed, 10 insertions(+), 19 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index c5d1345..a8999ee 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -3,13 +3,13 @@
 import concurrent.futures
 import dataclasses
 #import itertools
-import math
+# import math
 #import pathlib
 
-import imageio
+# import imageio
 #from natsort import natsorted
 import numpy
-import skimage
+# import skimage
 
 import h5py
 import hdf5plugin
@@ -64,11 +64,11 @@ def dswrite_block(ds, start, end, arr, silent_overflow=True):
         ds[start[0]:end[0], start[1]:end[1], start[2]:end[2]] = arr[:(end[0] - start[0]), :(end[1] - start[1]), :(end[2] - start[2])]
 
 
-def write_mips(zgrp,miparrs):
-    for miparr in miparrs:
-        dswrite_block(ds=zgrp[miparr.lvl],start=miparr.start,end=miparr.end,arr=miparr.array)
+# def write_mips(zgrp,miparrs):
+#     for miparr in miparrs:
+#         dswrite_block(ds=zgrp[miparr.lvl],start=miparr.start,end=miparr.end,arr=miparr.array)
+
 
-# HERE : NEED TO UNTANGLE MISMATCH BETWEEN BLOCK (deskewed) and CHUNK (skewed) size
 def iterate_numpy_blocks_from_dataset(
         dataset, nblocks, block_size=None, pad=True, deskew_kwargs={}, *args, **kwargs):
     """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
@@ -87,15 +87,7 @@ def iterate_numpy_blocks_from_dataset(
     arr : numpy.ndarray
         3D numpy array representing a consecutive chunk of 2D arrays
     """
-    # if deskew_kwargs and deskew_kwargs["deskew_method"] == "ps":
-    #     # create output block and get flattened indices
-    #     zb,yb,xb = numpy.meshgrid(*[range(d) for d in block_size],indexing="ij")
-    #     fb = numpy.ravel_multi_index((zb,yb,xb),block_size)
-    if nblocks[1] == 1:
-        print("test condition: using test tuple")
-        test = True
-    else:
-        test = False
+    
     dshape = dataset.shape
     if deskew_kwargs:
         chunk_size = (deskew_kwargs["chunklength"],block_size[1],block_size[2]*deskew_kwargs["stride"])
@@ -105,13 +97,12 @@ def iterate_numpy_blocks_from_dataset(
         print("chunk size: " + str(chunk_size))
     for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nblocks),order='F')
-        # if test:
-        #     chunk_tuple = (chunk_tuple[0], 11, chunk_tuple[2])
         # deskew level 0 data blocks
         if deskew_kwargs:
             if deskew_kwargs["transpose"]:
                 chunk_tuple = (chunk_tuple[0],chunk_tuple[2],chunk_tuple[1])
-            print(str(chunk_tuple))
+            if chunk_tuple[0] == 0:
+                print(str(chunk_tuple))
             if chunk_tuple[0] == 0:
                 chunk_index = 0
                 deskew_kwargs["slice1d"][...] = 0

From e25de26e78d47448ada1b33ed14a904bf4a56a09 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Thu, 26 Sep 2024 22:02:07 -0700
Subject: [PATCH 080/115] print chunks

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index a8999ee..eb0848b 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -317,6 +317,7 @@ def write_ims_to_zarr(
     
     f = h5py.File(ims_fn, 'r')
     dataset = f['DataSet']['ResolutionLevel 0']['TimePoint 0']['Channel 0']['Data']
+    print("ims chunks: " + str(dataset.chunks))
     # if deskew_options and deskew_options["deskew_transpose"]:
     #     dataset = dataset.transpose((0,2,1))
     #     print("transposed shape: " + str(dataset.shape))

From 279fd3f9b9a34bb7162b1d00220fde746649e64c Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Thu, 26 Sep 2024 22:07:17 -0700
Subject: [PATCH 081/115] block size from ims chunk size

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py           | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index eb0848b..093a66b 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -317,12 +317,13 @@ def write_ims_to_zarr(
     
     f = h5py.File(ims_fn, 'r')
     dataset = f['DataSet']['ResolutionLevel 0']['TimePoint 0']['Channel 0']['Data']
-    print("ims chunks: " + str(dataset.chunks))
+    ims_chunk_size = dataset.chunks
+    print("ims chunks: " + str(ims_chunk_size))
     # if deskew_options and deskew_options["deskew_transpose"]:
     #     dataset = dataset.transpose((0,2,1))
     #     print("transposed shape: " + str(dataset.shape))
     
-    block_size = [128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
+    block_size = [ims_chunk_size[0]*2,ims_chunk_size[1]*32,ims_chunk_size[2]*8] #[128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
     print("deskewed block size: " + str(block_size))
     
     if numchunks < 1:

From fe56df6f179befcad5e9ffe9b14ae42220c6b090 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Fri, 27 Sep 2024 07:07:35 -0700
Subject: [PATCH 082/115] trying increasing cache

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py            | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 093a66b..894fda9 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -315,7 +315,7 @@ def write_ims_to_zarr(
     group_attributes = ([] if group_attributes is None else group_attributes)
     deskew_options = ({} if deskew_options is None else deskew_options)
     
-    f = h5py.File(ims_fn, 'r')
+    f = h5py.File(ims_fn, 'r', rdcc_nbytes=1024**3)
     dataset = f['DataSet']['ResolutionLevel 0']['TimePoint 0']['Channel 0']['Data']
     ims_chunk_size = dataset.chunks
     print("ims chunks: " + str(ims_chunk_size))
@@ -323,7 +323,7 @@ def write_ims_to_zarr(
     #     dataset = dataset.transpose((0,2,1))
     #     print("transposed shape: " + str(dataset.shape))
     
-    block_size = [ims_chunk_size[0]*2,ims_chunk_size[1]*32,ims_chunk_size[2]*8] #[128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
+    block_size = [256,2048,512] #[ims_chunk_size[0]*2,ims_chunk_size[1]*32,ims_chunk_size[2]*8] #[128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
     print("deskewed block size: " + str(block_size))
     
     if numchunks < 1:

From ff255f761105badc71bdd670e7d3ca32b98570a3 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Fri, 27 Sep 2024 16:32:52 -0700
Subject: [PATCH 083/115] adding ims zarr wrapper functionality

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py    | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 894fda9..ccc5c98 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -11,8 +11,9 @@
 import numpy
 # import skimage
 
+import hdf5plugin # this needs to be imported
 import h5py
-import hdf5plugin
+from imaris_ims_file_reader.ims import ims
 import zarr
 from numcodecs import Blosc
 import argschema
@@ -315,9 +316,11 @@ def write_ims_to_zarr(
     group_attributes = ([] if group_attributes is None else group_attributes)
     deskew_options = ({} if deskew_options is None else deskew_options)
     
-    f = h5py.File(ims_fn, 'r', rdcc_nbytes=1024**3)
-    dataset = f['DataSet']['ResolutionLevel 0']['TimePoint 0']['Channel 0']['Data']
-    ims_chunk_size = dataset.chunks
+    #f = h5py.File(ims_fn, 'r', rdcc_nbytes=512*1024**2)
+    #dataset = f['DataSet']['ResolutionLevel 0']['TimePoint 0']['Channel 0']['Data']
+    store = ims(ims_fn,ResolutionLevelLock=0,aszarr=True)
+    dataset = zarr.open(store,mode="r")
+    ims_chunk_size = store.chunks
     print("ims chunks: " + str(ims_chunk_size))
     # if deskew_options and deskew_options["deskew_transpose"]:
     #     dataset = dataset.transpose((0,2,1))

From c60227dcd3ad7cb68d8f69cf87e2345ec23931e2 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Fri, 27 Sep 2024 20:52:57 -0700
Subject: [PATCH 084/115] 5d dataset as zarr

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index ccc5c98..0c6683f 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -116,7 +116,7 @@ def iterate_numpy_blocks_from_dataset(
                 if chunk_start[0] < first_slice:
                     chunk_end[0] -= first_slice - chunk_start[0]
                     chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
-                    chunk[first_slice-chunk_start[0]:] = dataset[chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
+                    chunk[first_slice-chunk_start[0]:] = dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
                 else:
                     chunk_start[0] -= first_slice
                     chunk_end[0] -= first_slice
@@ -124,7 +124,7 @@ def iterate_numpy_blocks_from_dataset(
                     #     chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
                     #     chunk[:dshape[0]-chunk_start[0]] = dataset[chunk_start[0]:,chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
                     # else:
-                    chunk = dataset[chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
+                    chunk = dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
                 if any([sh<sz for sh,sz in zip(chunk.shape,chunk_size)]):
                     print(str(chunk.shape) + " is small for" + str(chunk_size) + ": filling with zeros")
                     temp_chunk = numpy.zeros(chunk_size,dtype=chunk.dtype)
@@ -146,7 +146,7 @@ def iterate_numpy_blocks_from_dataset(
                 print(str(chunk_tuple))
             block_start = [chunk_tuple[k]*block_size[k] for k in range(3)]
             block_end = [block_start[k] + block_size[k] for k in range(3)]
-            arr = dataset[block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]]
+            arr = dataset[0,0,block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]]
         if any([arr.shape[k] != block_size[k] for k in range(3)]):
             print(str(arr.shape) + "is small for " + str(block_size))
             if pad:
@@ -330,16 +330,16 @@ def write_ims_to_zarr(
     print("deskewed block size: " + str(block_size))
     
     if numchunks < 1:
-        joined_shapes = dataset.shape
+        joined_shapes = dataset.shape[2:]
         if deskew_options and deskew_options["deskew_transpose"]:
             # input dataset must be transposed
             joined_shapes = (joined_shapes[0],joined_shapes[2],joined_shapes[1])
     else:
         if deskew_options and deskew_options["deskew_transpose"]:
             # input dataset must be transposed
-            joined_shapes = (dataset.shape[0],numchunks*block_size[1],dataset.shape[1])
+            joined_shapes = (dataset.shape[2],numchunks*block_size[1],dataset.shape[3])
         else:
-            joined_shapes = (dataset.shape[0],numchunks*block_size[1],dataset.shape[2])
+            joined_shapes = (dataset.shape[2],numchunks*block_size[1],dataset.shape[4])
     print("ims_to_ngff dataset shape:" + str(joined_shapes))
 
     if deskew_options and deskew_options["deskew_method"] == "ps":
@@ -434,7 +434,7 @@ def write_ims_to_zarr(
             for fut in concurrent.futures.as_completed(futs):
                 _ = fut.result()
     print("conversion complete, closing file")
-    f.close()
+    #f.close()
 
 
 def ims_to_ngff_group(ims_fn, output, *args, **kwargs):

From facf32a80185f0c8df94cf750c41e8a6a5b72b03 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Sat, 28 Sep 2024 12:59:11 -0700
Subject: [PATCH 085/115] debugging

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py         | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 0c6683f..939980d 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -18,7 +18,6 @@
 from numcodecs import Blosc
 import argschema
 
-# import acpreprocessing.utils.convert
 import acpreprocessing.stitching_modules.convert_to_n5.psdeskew as psd
 from acpreprocessing.stitching_modules.convert_to_n5.tiff_to_ngff import downsample_array,mip_level_shape,omezarr_attrs,NGFFGroupGenerationParameters,TiffToNGFFValueError
 
@@ -116,7 +115,7 @@ def iterate_numpy_blocks_from_dataset(
                 if chunk_start[0] < first_slice:
                     chunk_end[0] -= first_slice - chunk_start[0]
                     chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
-                    chunk[first_slice-chunk_start[0]:] = dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
+                    chunk[first_slice-chunk_start[0]:] = numpy.squeeze(dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]])
                 else:
                     chunk_start[0] -= first_slice
                     chunk_end[0] -= first_slice
@@ -124,7 +123,7 @@ def iterate_numpy_blocks_from_dataset(
                     #     chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
                     #     chunk[:dshape[0]-chunk_start[0]] = dataset[chunk_start[0]:,chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
                     # else:
-                    chunk = dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
+                    chunk = numpy.squeeze(dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]])
                 if any([sh<sz for sh,sz in zip(chunk.shape,chunk_size)]):
                     print(str(chunk.shape) + " is small for" + str(chunk_size) + ": filling with zeros")
                     temp_chunk = numpy.zeros(chunk_size,dtype=chunk.dtype)
@@ -146,7 +145,7 @@ def iterate_numpy_blocks_from_dataset(
                 print(str(chunk_tuple))
             block_start = [chunk_tuple[k]*block_size[k] for k in range(3)]
             block_end = [block_start[k] + block_size[k] for k in range(3)]
-            arr = dataset[0,0,block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]]
+            arr = numpy.squeeze(dataset[0,0,block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]])
         if any([arr.shape[k] != block_size[k] for k in range(3)]):
             print(str(arr.shape) + "is small for " + str(block_size))
             if pad:

From dd0fb61e34c4ae686dfb3f9ace79cad06c65aa2c Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Sun, 29 Sep 2024 09:33:33 -0700
Subject: [PATCH 086/115] debugging

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py     | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 939980d..4c883ae 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -115,7 +115,9 @@ def iterate_numpy_blocks_from_dataset(
                 if chunk_start[0] < first_slice:
                     chunk_end[0] -= first_slice - chunk_start[0]
                     chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
-                    chunk[first_slice-chunk_start[0]:] = numpy.squeeze(dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]])
+                    zdata = numpy.squeeze(numpy.asarray(dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]))
+                    print("data dimension is " + str(zdata.shape) + " max is " + str(numpy.max(zdata)))
+                    chunk[first_slice-chunk_start[0]:] = zdata
                 else:
                     chunk_start[0] -= first_slice
                     chunk_end[0] -= first_slice
@@ -123,7 +125,9 @@ def iterate_numpy_blocks_from_dataset(
                     #     chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
                     #     chunk[:dshape[0]-chunk_start[0]] = dataset[chunk_start[0]:,chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
                     # else:
-                    chunk = numpy.squeeze(dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]])
+                    zdata = numpy.squeeze(numpy.asarray(dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]))
+                    print("data dimension is " + str(zdata.shape) + " max is " + str(numpy.max(zdata)))
+                    chunk = zdata
                 if any([sh<sz for sh,sz in zip(chunk.shape,chunk_size)]):
                     print(str(chunk.shape) + " is small for" + str(chunk_size) + ": filling with zeros")
                     temp_chunk = numpy.zeros(chunk_size,dtype=chunk.dtype)
@@ -145,7 +149,7 @@ def iterate_numpy_blocks_from_dataset(
                 print(str(chunk_tuple))
             block_start = [chunk_tuple[k]*block_size[k] for k in range(3)]
             block_end = [block_start[k] + block_size[k] for k in range(3)]
-            arr = numpy.squeeze(dataset[0,0,block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]])
+            arr = numpy.squeeze(numpy.asarray(dataset[0,0,block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]]))
         if any([arr.shape[k] != block_size[k] for k in range(3)]):
             print(str(arr.shape) + "is small for " + str(block_size))
             if pad:

From e018a7b78b86fa20fbc68abf4d648007134030c1 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Sun, 29 Sep 2024 20:44:12 -0700
Subject: [PATCH 087/115] major debug

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py            | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 4c883ae..bd226fe 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -202,7 +202,6 @@ def iterate_mip_levels_from_dataset(
                           else lvl_to_mip_kwargs)
     mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
     block_index = 0
-    start_index = 0
     if lvl > 0:
         num_chunks = downsample_factor[0]
         
@@ -245,10 +244,9 @@ def iterate_mip_levels_from_dataset(
                         method=downsample_method, **mip_kwargs))
                 # end_index = start_index + temp_arr.shape[0]
                 # yield MIPArray(lvl, temp_arr, start_index, end_index)
-                chunk_start = tuple((start_index, int(ma.start[1]/downsample_factor[1]), int(ma.start[2]/downsample_factor[2])))#tuple(int(ma.start[k]/downsample_factor[k]) for k in range(3))
+                chunk_start = tuple(int(ma.start[k]/downsample_factor[k]) for k in range(3))
                 chunk_end = tuple(chunk_start[k] + temp_arr.shape[k] for k in range(3))
                 yield MIPArray(lvl, temp_arr, chunk_start, chunk_end)
-                start_index += temp_arr.shape[0]
                 i = 0
             else:
                 i += 1

From 1c75a71076f97370954b7b3faa3ec6a343382e67 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Sun, 29 Sep 2024 21:00:18 -0700
Subject: [PATCH 088/115] reverting chunking scheme

---
 .../convert_to_n5/ims_to_ngff.py              | 105 +++++++++++++++---
 1 file changed, 87 insertions(+), 18 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index bd226fe..4a27e5b 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -64,9 +64,9 @@ def dswrite_block(ds, start, end, arr, silent_overflow=True):
         ds[start[0]:end[0], start[1]:end[1], start[2]:end[2]] = arr[:(end[0] - start[0]), :(end[1] - start[1]), :(end[2] - start[2])]
 
 
-# def write_mips(zgrp,miparrs):
-#     for miparr in miparrs:
-#         dswrite_block(ds=zgrp[miparr.lvl],start=miparr.start,end=miparr.end,arr=miparr.array)
+def write_mips(zgrp,miparrs):
+    for miparr in miparrs:
+        dswrite_block(ds=zgrp[miparr.lvl],start=miparr.start,end=miparr.end,arr=miparr.array)
 
 
 def iterate_numpy_blocks_from_dataset(
@@ -169,6 +169,75 @@ def iterate_mip_levels_from_dataset(
         interleaved_channels=1, channel=0, deskew_kwargs={}):
     """recursively generate MIPmap levels from an iterator of blocks
 
+    Parameters
+    ----------
+    dataset : hdf5 dataset
+        imageio-compatible name inputs to be opened as multi-images
+    lvl : int
+        integer mip level to generate
+    block_size : int
+        number of 2D arrays in chunk to process for a chunk at lvl
+    slice_length : int
+        number of 2D arrays to gather from tiff stacks
+    downsample_factor : tuple of int
+        integer downsampling factor for MIP levels
+    downsample_method : str, optional
+        downsampling method for
+        acpreprocessing.utils.convert.downsample_stack_volume
+    lvl_to_mip_kwargs :  dict, optional
+        mapping of MIP level to kwargs used in MIPmap generation
+    interleaved_channels : int
+        number of channels interleaved in the tiff files (default 1)
+    channel : int, optional
+        channel from which interleaved data should be read (default 0)
+    deskew_kwargs : dict, optional
+        parameters for pixel shifting deskew
+
+    Yields
+    ------
+    ma : acpreprocessing.stitching_modules.convert_to_n5.tiff_to_n5.MipArray
+        object describing chunked array, MIP level of origin, and chunk indices
+    """
+    lvl_to_mip_kwargs = ({} if lvl_to_mip_kwargs is None
+                          else lvl_to_mip_kwargs)
+    mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
+    block_index = 0
+    if lvl > 0:
+        for ma in iterate_mip_levels_from_dataset(
+                dataset, lvl-1, maxlvl, nblocks, block_size,
+                downsample_factor, downsample_method,
+                lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
+                channel=channel, deskew_kwargs=deskew_kwargs):
+            chunk = ma.array
+            # throw array up for further processing
+            yield ma
+            # ignore if not parent resolution
+            if ma.lvl != lvl-1:
+                continue
+            
+            temp_arr = (downsample_array(
+                    chunk, downsample_factor, dtype=chunk.dtype,
+                    method=downsample_method, **mip_kwargs))
+            chunk_start = tuple(int(ma.start[k]/downsample_factor[k]) for k in range(3))
+            chunk_end = tuple(chunk_start[k] + temp_arr.shape[k] for k in range(3))
+            yield MIPArray(lvl, temp_arr, chunk_start, chunk_end)
+    else:
+        # get level 0 chunks
+        # block_size is the number of slices to read from tiffs
+        for block in iterate_numpy_blocks_from_dataset(
+                dataset, nblocks, block_size=block_size, pad=False,
+                deskew_kwargs=deskew_kwargs,
+                channel=channel):
+            block_tuple = numpy.unravel_index(block_index,nblocks,order='F')
+            block_start = tuple(block_tuple[k]*block_size[k] for k in range(3))
+            block_end = tuple(block_start[k] + block.shape[k] for k in range(3))
+            yield MIPArray(lvl, block, block_start, block_end)
+            block_index += 1
+        dataset, lvl, maxlvl, nblocks, block_size, downsample_factor,
+        downsample_method=None, lvl_to_mip_kwargs=None,
+        interleaved_channels=1, channel=0, deskew_kwargs={}):
+    """recursively generate MIPmap levels from an iterator of blocks
+
     Parameters
     ----------
     dataset : hdf5 dataset
@@ -327,7 +396,7 @@ def write_ims_to_zarr(
     #     dataset = dataset.transpose((0,2,1))
     #     print("transposed shape: " + str(dataset.shape))
     
-    block_size = [256,2048,512] #[ims_chunk_size[0]*2,ims_chunk_size[1]*32,ims_chunk_size[2]*8] #[128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
+    block_size = [512,1024,512] #[ims_chunk_size[0]*2,ims_chunk_size[1]*32,ims_chunk_size[2]*8] #[128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
     print("deskewed block size: " + str(block_size))
     
     if numchunks < 1:
@@ -413,25 +482,25 @@ def write_ims_to_zarr(
 
         with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as e:
             futs = []
-            # mips = []
-            # for miparr in iterate_mip_levels_from_dataset(
-            #         dataset, max_mip, max_mip, nblocks, block_size, mip_dsfactor,
-            #         lvl_to_mip_kwargs=lvl_to_mip_kwargs,
-            #         interleaved_channels=interleaved_channels,
-            #         channel=channel, deskew_kwargs=deskew_kwargs):
-            #     mips.append(miparr)
-            #     if miparr.lvl == max_mip:
-            #         futs.append(e.submit(
-            #             write_mips, mip_ds, mips))
-            #         mips = []
+            mips = []
             for miparr in iterate_mip_levels_from_dataset(
                     dataset, max_mip, max_mip, nblocks, block_size, mip_dsfactor,
                     lvl_to_mip_kwargs=lvl_to_mip_kwargs,
                     interleaved_channels=interleaved_channels,
                     channel=channel, deskew_kwargs=deskew_kwargs):
-                futs.append(e.submit(
-                    dswrite_block, mip_ds[miparr.lvl],
-                    miparr.start, miparr.end, miparr.array))
+                mips.append(miparr)
+                if miparr.lvl == max_mip:
+                    futs.append(e.submit(
+                        write_mips, mip_ds, mips))
+                    mips = []
+            # for miparr in iterate_mip_levels_from_dataset(
+            #         dataset, max_mip, max_mip, nblocks, block_size, mip_dsfactor,
+            #         lvl_to_mip_kwargs=lvl_to_mip_kwargs,
+            #         interleaved_channels=interleaved_channels,
+            #         channel=channel, deskew_kwargs=deskew_kwargs):
+            #     futs.append(e.submit(
+            #         dswrite_block, mip_ds[miparr.lvl],
+            #         miparr.start, miparr.end, miparr.array))
             for fut in concurrent.futures.as_completed(futs):
                 _ = fut.result()
     print("conversion complete, closing file")

From 88fa5ca719184215ca30fa80b0f1bed7deb4f0b2 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Sun, 29 Sep 2024 21:03:38 -0700
Subject: [PATCH 089/115] fixing deletion

---
 .../convert_to_n5/ims_to_ngff.py              | 105 ------------------
 1 file changed, 105 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 4a27e5b..2be5e89 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -233,111 +233,6 @@ def iterate_mip_levels_from_dataset(
             block_end = tuple(block_start[k] + block.shape[k] for k in range(3))
             yield MIPArray(lvl, block, block_start, block_end)
             block_index += 1
-        dataset, lvl, maxlvl, nblocks, block_size, downsample_factor,
-        downsample_method=None, lvl_to_mip_kwargs=None,
-        interleaved_channels=1, channel=0, deskew_kwargs={}):
-    """recursively generate MIPmap levels from an iterator of blocks
-
-    Parameters
-    ----------
-    dataset : hdf5 dataset
-        imageio-compatible name inputs to be opened as multi-images
-    lvl : int
-        integer mip level to generate
-    block_size : int
-        number of 2D arrays in chunk to process for a chunk at lvl
-    slice_length : int
-        number of 2D arrays to gather from tiff stacks
-    downsample_factor : tuple of int
-        integer downsampling factor for MIP levels
-    downsample_method : str, optional
-        downsampling method for
-        acpreprocessing.utils.convert.downsample_stack_volume
-    lvl_to_mip_kwargs :  dict, optional
-        mapping of MIP level to kwargs used in MIPmap generation
-    interleaved_channels : int
-        number of channels interleaved in the tiff files (default 1)
-    channel : int, optional
-        channel from which interleaved data should be read (default 0)
-    deskew_kwargs : dict, optional
-        parameters for pixel shifting deskew
-
-    Yields
-    ------
-    ma : acpreprocessing.stitching_modules.convert_to_n5.tiff_to_n5.MipArray
-        object describing chunked array, MIP level of origin, and chunk indices
-    """
-    lvl_to_mip_kwargs = ({} if lvl_to_mip_kwargs is None
-                          else lvl_to_mip_kwargs)
-    mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
-    block_index = 0
-    if lvl > 0:
-        num_chunks = downsample_factor[0]
-        
-        i = 0
-        for ma in iterate_mip_levels_from_dataset(
-                dataset, lvl-1, maxlvl, nblocks, block_size,
-                downsample_factor, downsample_method,
-                lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
-                channel=channel, deskew_kwargs=deskew_kwargs):
-            chunk = ma.array
-            # throw array up for further processing
-            yield ma
-            # ignore if not parent resolution
-            if ma.lvl != lvl-1:
-                continue
-            
-            try:
-                temp_lminus1_arr
-            except NameError:
-                temp_lminus1_arr = numpy.empty(
-                    (num_chunks*chunk.shape[0], *chunk.shape[1:]),
-                    dtype=chunk.dtype)
-            
-            # fill in temporary block according to index
-            chunk_size = chunk.shape[0]
-            block_offset = i*block_size[0]
-            
-            temp_lminus1_arr[
-                block_offset:block_offset+chunk_size, :, :] = chunk[:, :, :]
-            
-            # copy op only for uneven final chunk
-            if chunk_size != block_size[0]:
-                temp_lminus1_arr = temp_lminus1_arr[
-                    :block_offset+chunk_size, :, :]
-            
-            if i == num_chunks - 1:
-                temp_arr = (
-                    downsample_array(
-                        temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
-                        method=downsample_method, **mip_kwargs))
-                # end_index = start_index + temp_arr.shape[0]
-                # yield MIPArray(lvl, temp_arr, start_index, end_index)
-                chunk_start = tuple(int(ma.start[k]/downsample_factor[k]) for k in range(3))
-                chunk_end = tuple(chunk_start[k] + temp_arr.shape[k] for k in range(3))
-                yield MIPArray(lvl, temp_arr, chunk_start, chunk_end)
-                i = 0
-            else:
-                i += 1
-            
-            # temp_arr = (downsample_array(
-            #         chunk, downsample_factor, dtype=chunk.dtype,
-            #         method=downsample_method, **mip_kwargs))
-            # chunk_start = tuple(int(ma.start[k]/downsample_factor[k]) for k in range(3))
-            # chunk_end = tuple(chunk_start[k] + temp_arr.shape[k] for k in range(3))
-            # yield MIPArray(lvl, temp_arr, chunk_start, chunk_end)
-    else:
-        # get level 0 chunks
-        # block_size is the number of slices to read from tiffs
-        for block in iterate_numpy_blocks_from_dataset(
-                dataset, nblocks, block_size=block_size, pad=False,
-                deskew_kwargs=deskew_kwargs,
-                channel=channel):
-            block_tuple = numpy.unravel_index(block_index,nblocks,order='F')
-            block_start = tuple(block_tuple[k]*block_size[k] for k in range(3))
-            block_end = tuple(block_start[k] + block.shape[k] for k in range(3))
-            yield MIPArray(lvl, block, block_start, block_end)
-            block_index += 1
 
 
 def write_ims_to_zarr(

From 0d00d1d4f4dd578b574b9596118fb21a39a5df93 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Sun, 29 Sep 2024 21:41:07 -0700
Subject: [PATCH 090/115] debugging

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py          | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 2be5e89..8651c43 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -101,19 +101,19 @@ def iterate_numpy_blocks_from_dataset(
         if deskew_kwargs:
             if deskew_kwargs["transpose"]:
                 chunk_tuple = (chunk_tuple[0],chunk_tuple[2],chunk_tuple[1])
-            if chunk_tuple[0] == 0:
-                print(str(chunk_tuple))
+            print(str(chunk_tuple))
             if chunk_tuple[0] == 0:
                 chunk_index = 0
                 deskew_kwargs["slice1d"][...] = 0
                 first_z,first_slice = psd.calculate_first_chunk(chunk_size=chunk_size,x_index=(nblocks[2] - chunk_tuple[1] - 1),stride=deskew_kwargs["stride"])
+                print(str(first_z) + "," + str(first_slice))
             if chunk_tuple[0] < first_z or chunk_tuple[0]*chunk_size[0] - first_slice >= dshape[0]:
                 arr = numpy.zeros(block_size,dtype=dataset.dtype)
             else:
                 chunk_start = numpy.array([t*s for t,s in zip(chunk_tuple,chunk_size)])
                 chunk_end = chunk_start + numpy.array(chunk_size)
                 if chunk_start[0] < first_slice:
-                    chunk_end[0] -= first_slice - chunk_start[0]
+                    chunk_end[0] -= (first_slice - chunk_start[0])
                     chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
                     zdata = numpy.squeeze(numpy.asarray(dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]))
                     print("data dimension is " + str(zdata.shape) + " max is " + str(numpy.max(zdata)))

From 3b3da5bcc9822bc0d2169606cd19ac2db325074e Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Sun, 29 Sep 2024 22:09:25 -0700
Subject: [PATCH 091/115] debugging

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py           | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 8651c43..f8bea99 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -113,9 +113,9 @@ def iterate_numpy_blocks_from_dataset(
                 chunk_start = numpy.array([t*s for t,s in zip(chunk_tuple,chunk_size)])
                 chunk_end = chunk_start + numpy.array(chunk_size)
                 if chunk_start[0] < first_slice:
-                    chunk_end[0] -= (first_slice - chunk_start[0])
+                    chunk_end[0] = chunk_size[0] - (first_slice - chunk_start[0])
                     chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
-                    zdata = numpy.squeeze(numpy.asarray(dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]))
+                    zdata = numpy.squeeze(numpy.asarray(dataset[0,0,:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]))
                     print("data dimension is " + str(zdata.shape) + " max is " + str(numpy.max(zdata)))
                     chunk[first_slice-chunk_start[0]:] = zdata
                 else:
@@ -125,6 +125,7 @@ def iterate_numpy_blocks_from_dataset(
                     #     chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
                     #     chunk[:dshape[0]-chunk_start[0]] = dataset[chunk_start[0]:,chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
                     # else:
+                    print(str(chunk_start[0]))
                     zdata = numpy.squeeze(numpy.asarray(dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]))
                     print("data dimension is " + str(zdata.shape) + " max is " + str(numpy.max(zdata)))
                     chunk = zdata

From 4b7f84b41ccfc455dae57c41250b5a5e43365c17 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Sun, 29 Sep 2024 22:14:02 -0700
Subject: [PATCH 092/115] major bug fix

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index f8bea99..1f73bae 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -88,7 +88,7 @@ def iterate_numpy_blocks_from_dataset(
         3D numpy array representing a consecutive chunk of 2D arrays
     """
     
-    dshape = dataset.shape
+    dshape = dataset.shape[2:]
     if deskew_kwargs:
         chunk_size = (deskew_kwargs["chunklength"],block_size[1],block_size[2]*deskew_kwargs["stride"])
         if deskew_kwargs["transpose"]:

From a5a6ff20b2af7a846bc72166b320dd0086cdaa96 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Mon, 30 Sep 2024 06:45:47 -0700
Subject: [PATCH 093/115] readding test tuple

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py         | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 1f73bae..8a313eb 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -88,6 +88,11 @@ def iterate_numpy_blocks_from_dataset(
         3D numpy array representing a consecutive chunk of 2D arrays
     """
     
+    if nblocks[1] == 1:
+        print("test condition: using test tuple")
+        test = True
+    else:
+        test = False
     dshape = dataset.shape[2:]
     if deskew_kwargs:
         chunk_size = (deskew_kwargs["chunklength"],block_size[1],block_size[2]*deskew_kwargs["stride"])
@@ -97,6 +102,8 @@ def iterate_numpy_blocks_from_dataset(
         print("chunk size: " + str(chunk_size))
     for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nblocks),order='F')
+        if test:
+            chunk_tuple = (chunk_tuple[0], 6, chunk_tuple[2])
         # deskew level 0 data blocks
         if deskew_kwargs:
             if deskew_kwargs["transpose"]:

From 527ea31cf83bc011b32393e8a58a70ca97afaa2c Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 1 Oct 2024 06:47:12 -0700
Subject: [PATCH 094/115] trying parallel writing

---
 .../convert_to_n5/ims_to_ngff.py              | 23 +++++++++++--------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 8a313eb..5ece20f 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -70,7 +70,7 @@ def write_mips(zgrp,miparrs):
 
 
 def iterate_numpy_blocks_from_dataset(
-        dataset, nblocks, block_size=None, pad=True, deskew_kwargs={}, *args, **kwargs):
+        dataset, nblocks, chunknum=-1, block_size=None, pad=True, deskew_kwargs={}, *args, **kwargs):
     """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
 
     Parameters
@@ -88,8 +88,8 @@ def iterate_numpy_blocks_from_dataset(
         3D numpy array representing a consecutive chunk of 2D arrays
     """
     
-    if nblocks[1] == 1:
-        print("test condition: using test tuple")
+    if nblocks[2] == 1:
+        print("1 chunk: using chunk_num = " + str(chunknum))
         test = True
     else:
         test = False
@@ -103,7 +103,7 @@ def iterate_numpy_blocks_from_dataset(
     for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nblocks),order='F')
         if test:
-            chunk_tuple = (chunk_tuple[0], 6, chunk_tuple[2])
+            chunk_tuple = (chunk_tuple[0], chunk_tuple[1], chunknum)
         # deskew level 0 data blocks
         if deskew_kwargs:
             if deskew_kwargs["transpose"]:
@@ -173,7 +173,7 @@ def iterate_numpy_blocks_from_dataset(
 
 def iterate_mip_levels_from_dataset(
         dataset, lvl, maxlvl, nblocks, block_size, downsample_factor,
-        downsample_method=None, lvl_to_mip_kwargs=None,
+        chunknum=-1,downsample_method=None, lvl_to_mip_kwargs=None,
         interleaved_channels=1, channel=0, deskew_kwargs={}):
     """recursively generate MIPmap levels from an iterator of blocks
 
@@ -233,7 +233,7 @@ def iterate_mip_levels_from_dataset(
         # get level 0 chunks
         # block_size is the number of slices to read from tiffs
         for block in iterate_numpy_blocks_from_dataset(
-                dataset, nblocks, block_size=block_size, pad=False,
+                dataset, nblocks, chunknum=chunknum, block_size=block_size, pad=False,
                 deskew_kwargs=deskew_kwargs,
                 channel=channel):
             block_tuple = numpy.unravel_index(block_index,nblocks,order='F')
@@ -248,7 +248,7 @@ def write_ims_to_zarr(
         mip_dsfactor=(2, 2, 2), chunk_size=(1, 1, 64, 64, 64),
         concurrency=10, slice_concurrency=1,
         compression="raw", dtype="uint16", lvl_to_mip_kwargs=None,
-        interleaved_channels=1, channel=0, deskew_options=None, numchunks=0, **kwargs):
+        interleaved_channels=1, channel=0, deskew_options=None, numchunks=0, chunknum=-1, **kwargs):
     """write a stack represented by an iterator of multi-image files as a zarr
     volume with ome-ngff metadata
 
@@ -299,7 +299,7 @@ def write_ims_to_zarr(
     #     dataset = dataset.transpose((0,2,1))
     #     print("transposed shape: " + str(dataset.shape))
     
-    block_size = [512,1024,512] #[ims_chunk_size[0]*2,ims_chunk_size[1]*32,ims_chunk_size[2]*8] #[128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
+    block_size = [256,512,512] #[ims_chunk_size[0]*2,ims_chunk_size[1]*32,ims_chunk_size[2]*8] #[128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
     print("deskewed block size: " + str(block_size))
     
     if numchunks < 1:
@@ -310,9 +310,9 @@ def write_ims_to_zarr(
     else:
         if deskew_options and deskew_options["deskew_transpose"]:
             # input dataset must be transposed
-            joined_shapes = (dataset.shape[2],numchunks*block_size[1],dataset.shape[3])
+            joined_shapes = (dataset.shape[2],dataset.shape[4],numchunks*block_size[2])#(dataset.shape[2],numchunks*block_size[1],dataset.shape[3])
         else:
-            joined_shapes = (dataset.shape[2],numchunks*block_size[1],dataset.shape[4])
+            joined_shapes = (dataset.shape[2],dataset.shape[3],numchunks*block_size[2])#(dataset.shape[2],numchunks*block_size[1],dataset.shape[4])
     print("ims_to_ngff dataset shape:" + str(joined_shapes))
 
     if deskew_options and deskew_options["deskew_method"] == "ps":
@@ -388,6 +388,7 @@ def write_ims_to_zarr(
             mips = []
             for miparr in iterate_mip_levels_from_dataset(
                     dataset, max_mip, max_mip, nblocks, block_size, mip_dsfactor,
+                    chunknum=chunknum,
                     lvl_to_mip_kwargs=lvl_to_mip_kwargs,
                     interleaved_channels=interleaved_channels,
                     channel=channel, deskew_kwargs=deskew_kwargs):
@@ -432,6 +433,7 @@ class IMSToNGFFParameters(NGFFGroupGenerationParameters):
     interleaved_channels = argschema.fields.Int(required=False, default=1)
     channel = argschema.fields.Int(required=False, default=0)
     num_chunks = argschema.fields.Int(required=False, default=0)
+    chunk_num = argschema.fields.Int(required=False, default=-1)
 
 
 class IMSToZarrInputParameters(argschema.ArgSchema,
@@ -462,6 +464,7 @@ def run(self):
             compression=self.args["compression"],
             #lvl_to_mip_kwargs=self.args["lvl_to_mip_kwargs"],
             numchunks = self.args["num_chunks"],
+            chunknum = self.args["chunk_num"],
             deskew_options=deskew_options)
 
 

From 9f8da5f7104749af20169b883e0202979ce11da2 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 1 Oct 2024 06:54:27 -0700
Subject: [PATCH 095/115] debugging

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 5ece20f..d7f04ca 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -213,7 +213,7 @@ def iterate_mip_levels_from_dataset(
     if lvl > 0:
         for ma in iterate_mip_levels_from_dataset(
                 dataset, lvl-1, maxlvl, nblocks, block_size,
-                downsample_factor, downsample_method,
+                downsample_factor, chunknum, downsample_method,
                 lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
                 channel=channel, deskew_kwargs=deskew_kwargs):
             chunk = ma.array

From 92356ca56815b865e454354f78390b26bb85ffa5 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 1 Oct 2024 06:58:17 -0700
Subject: [PATCH 096/115] debugging

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py    | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index d7f04ca..d44e5f1 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -340,10 +340,15 @@ def write_ims_to_zarr(
         # create groups with attributes according to omezarr spec
         if len(group_names) == 1:
             group_name = group_names[0]
-            try:
-                g = f.create_group(f"{group_name}")
-            except KeyError:
+            if group_name in f:
                 g = f[f"{group_name}"]
+            else:
+                g = f.create_group(f"{group_name}")
+            # try:
+            #     g = f.create_group(f"{group_name}")
+            # except KeyError:
+            #     g = f[f"{group_name}"]
+                
             if group_attributes:
                 try:
                     attributes = group_attributes[0]

From 10cd03871833115db654ac0a8c755a04c1a969a2 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 1 Oct 2024 08:47:27 -0700
Subject: [PATCH 097/115] trying to parallelize slices

---
 .../convert_to_n5/ims_to_ngff.py              | 44 ++++++++++---------
 1 file changed, 24 insertions(+), 20 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index d44e5f1..1ba8a3c 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -70,7 +70,7 @@ def write_mips(zgrp,miparrs):
 
 
 def iterate_numpy_blocks_from_dataset(
-        dataset, nblocks, chunknum=-1, block_size=None, pad=True, deskew_kwargs={}, *args, **kwargs):
+        dataset, nblocks, numslice=0, chunknum=-1, block_size=None, pad=True, deskew_kwargs={}, *args, **kwargs):
     """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
 
     Parameters
@@ -88,10 +88,12 @@ def iterate_numpy_blocks_from_dataset(
         3D numpy array representing a consecutive chunk of 2D arrays
     """
     
-    if nblocks[2] == 1:
-        print("1 chunk: using chunk_num = " + str(chunknum))
+    if numslice == 1:
+        print("1 slice: using chunk x = " + str(chunknum))
+        nchunks = (nblocks[0],nblocks[1],1)
         test = True
     else:
+        nchunks = nblocks
         test = False
     dshape = dataset.shape[2:]
     if deskew_kwargs:
@@ -100,8 +102,8 @@ def iterate_numpy_blocks_from_dataset(
             chunk_size = (chunk_size[0],chunk_size[2],chunk_size[1])
             dshape = (dshape[0],dshape[2],dshape[1])
         print("chunk size: " + str(chunk_size))
-    for i in range(numpy.prod(nblocks)):#,*args,**kwargs):
-        chunk_tuple = numpy.unravel_index(i,tuple(nblocks),order='F')
+    for i in range(numpy.prod(nchunks)):#,*args,**kwargs):
+        chunk_tuple = numpy.unravel_index(i,tuple(nchunks),order='F')
         if test:
             chunk_tuple = (chunk_tuple[0], chunk_tuple[1], chunknum)
         # deskew level 0 data blocks
@@ -173,7 +175,7 @@ def iterate_numpy_blocks_from_dataset(
 
 def iterate_mip_levels_from_dataset(
         dataset, lvl, maxlvl, nblocks, block_size, downsample_factor,
-        chunknum=-1,downsample_method=None, lvl_to_mip_kwargs=None,
+        num_slice=0,chunknum=-1,downsample_method=None, lvl_to_mip_kwargs=None,
         interleaved_channels=1, channel=0, deskew_kwargs={}):
     """recursively generate MIPmap levels from an iterator of blocks
 
@@ -210,10 +212,12 @@ def iterate_mip_levels_from_dataset(
                           else lvl_to_mip_kwargs)
     mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
     block_index = 0
+    if num_slice == 1:
+        block_index = nblocks[0]*nblocks[1]*chunknum
     if lvl > 0:
         for ma in iterate_mip_levels_from_dataset(
                 dataset, lvl-1, maxlvl, nblocks, block_size,
-                downsample_factor, chunknum, downsample_method,
+                downsample_factor,num_slice,chunknum, downsample_method,
                 lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
                 channel=channel, deskew_kwargs=deskew_kwargs):
             chunk = ma.array
@@ -233,7 +237,7 @@ def iterate_mip_levels_from_dataset(
         # get level 0 chunks
         # block_size is the number of slices to read from tiffs
         for block in iterate_numpy_blocks_from_dataset(
-                dataset, nblocks, chunknum=chunknum, block_size=block_size, pad=False,
+                dataset, nblocks,num_slice=num_slice,chunknum=chunknum, block_size=block_size, pad=False,
                 deskew_kwargs=deskew_kwargs,
                 channel=channel):
             block_tuple = numpy.unravel_index(block_index,nblocks,order='F')
@@ -302,17 +306,17 @@ def write_ims_to_zarr(
     block_size = [256,512,512] #[ims_chunk_size[0]*2,ims_chunk_size[1]*32,ims_chunk_size[2]*8] #[128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
     print("deskewed block size: " + str(block_size))
     
-    if numchunks < 1:
-        joined_shapes = dataset.shape[2:]
-        if deskew_options and deskew_options["deskew_transpose"]:
-            # input dataset must be transposed
-            joined_shapes = (joined_shapes[0],joined_shapes[2],joined_shapes[1])
-    else:
-        if deskew_options and deskew_options["deskew_transpose"]:
-            # input dataset must be transposed
-            joined_shapes = (dataset.shape[2],dataset.shape[4],numchunks*block_size[2])#(dataset.shape[2],numchunks*block_size[1],dataset.shape[3])
-        else:
-            joined_shapes = (dataset.shape[2],dataset.shape[3],numchunks*block_size[2])#(dataset.shape[2],numchunks*block_size[1],dataset.shape[4])
+    # if numchunks < 1:
+    joined_shapes = dataset.shape[2:]
+    if deskew_options and deskew_options["deskew_transpose"]:
+        # input dataset must be transposed
+        joined_shapes = (joined_shapes[0],joined_shapes[2],joined_shapes[1])
+    # else:
+    #     if deskew_options and deskew_options["deskew_transpose"]:
+    #         # input dataset must be transposed
+    #         joined_shapes = (dataset.shape[2],dataset.shape[4],numchunks*block_size[2])#(dataset.shape[2],numchunks*block_size[1],dataset.shape[3])
+    #     else:
+    #         joined_shapes = (dataset.shape[2],dataset.shape[3],numchunks*block_size[2])#(dataset.shape[2],numchunks*block_size[1],dataset.shape[4])
     print("ims_to_ngff dataset shape:" + str(joined_shapes))
 
     if deskew_options and deskew_options["deskew_method"] == "ps":
@@ -393,7 +397,7 @@ def write_ims_to_zarr(
             mips = []
             for miparr in iterate_mip_levels_from_dataset(
                     dataset, max_mip, max_mip, nblocks, block_size, mip_dsfactor,
-                    chunknum=chunknum,
+                    num_slice=numchunks,chunknum=chunknum,
                     lvl_to_mip_kwargs=lvl_to_mip_kwargs,
                     interleaved_channels=interleaved_channels,
                     channel=channel, deskew_kwargs=deskew_kwargs):

From ab584f5ed7d30e07bf4c2f1d7f96e92e99b8e0d6 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 1 Oct 2024 08:58:48 -0700
Subject: [PATCH 098/115] trying parallel

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 1ba8a3c..dedeebe 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -382,6 +382,7 @@ def write_ims_to_zarr(
                 chunks=chunk_size,
                 shape=(1, 1, mip_3dshape[0], mip_3dshape[1], mip_3dshape[2]),
                 compression=compression,
+                synchronizer=zarr.ThreadSynchronizer(),
                 dtype=dtype
             )
             dsfactors = [int(i)**mip_lvl for i in mip_dsfactor]

From 2bbf6f0dada808761b277d50e996d776e922dba9 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 1 Oct 2024 09:05:22 -0700
Subject: [PATCH 099/115] debugging

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index dedeebe..1ba8a3c 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -382,7 +382,6 @@ def write_ims_to_zarr(
                 chunks=chunk_size,
                 shape=(1, 1, mip_3dshape[0], mip_3dshape[1], mip_3dshape[2]),
                 compression=compression,
-                synchronizer=zarr.ThreadSynchronizer(),
                 dtype=dtype
             )
             dsfactors = [int(i)**mip_lvl for i in mip_dsfactor]

From 186131b004c37298fa6156069a0358b8d7469e06 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 1 Oct 2024 09:20:07 -0700
Subject: [PATCH 100/115] debugging

---
 .../convert_to_n5/ims_to_ngff.py              | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 1ba8a3c..861bbed 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -376,14 +376,17 @@ def write_ims_to_zarr(
         # shuffle=Blosc.BITSHUFFLE)
         compression = Blosc(cname='zstd', clevel=1)
         for mip_lvl in range(max_mip + 1):
-            mip_3dshape = mip_level_shape(mip_lvl, joined_shapes)
-            ds_lvl = g.create_dataset(
-                f"{mip_lvl}",
-                chunks=chunk_size,
-                shape=(1, 1, mip_3dshape[0], mip_3dshape[1], mip_3dshape[2]),
-                compression=compression,
-                dtype=dtype
-            )
+            if not mip_lvl in g:
+                mip_3dshape = mip_level_shape(mip_lvl, joined_shapes)
+                ds_lvl = g.create_dataset(
+                    f"{mip_lvl}",
+                    chunks=chunk_size,
+                    shape=(1, 1, mip_3dshape[0], mip_3dshape[1], mip_3dshape[2]),
+                    compression=compression,
+                    dtype=dtype
+                )
+            else:
+                ds_lvl = g[mip_lvl]
             dsfactors = [int(i)**mip_lvl for i in mip_dsfactor]
             mip_ds[mip_lvl] = ds_lvl
             scales.append(dsfactors)

From 4dd1f43306b88bb9da3d4d9a6cb3a288cb080c1e Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 1 Oct 2024 09:35:17 -0700
Subject: [PATCH 101/115] debugging

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py        | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 861bbed..3d592b0 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -237,7 +237,7 @@ def iterate_mip_levels_from_dataset(
         # get level 0 chunks
         # block_size is the number of slices to read from tiffs
         for block in iterate_numpy_blocks_from_dataset(
-                dataset, nblocks,num_slice=num_slice,chunknum=chunknum, block_size=block_size, pad=False,
+                dataset, nblocks,numslice=num_slice,chunknum=chunknum, block_size=block_size, pad=False,
                 deskew_kwargs=deskew_kwargs,
                 channel=channel):
             block_tuple = numpy.unravel_index(block_index,nblocks,order='F')
@@ -249,10 +249,10 @@ def iterate_mip_levels_from_dataset(
 
 def write_ims_to_zarr(
         ims_fn, output_n5, group_names, group_attributes=None, max_mip=0,
-        mip_dsfactor=(2, 2, 2), chunk_size=(1, 1, 64, 64, 64),
+        mip_dsfactor=(2, 2, 2), chunk_size=(1, 1, 64, 64, 64), numchunks=0,
         concurrency=10, slice_concurrency=1,
         compression="raw", dtype="uint16", lvl_to_mip_kwargs=None,
-        interleaved_channels=1, channel=0, deskew_options=None, numchunks=0, chunknum=-1, **kwargs):
+        interleaved_channels=1, channel=0, deskew_options=None, chunknum=-1, **kwargs):
     """write a stack represented by an iterator of multi-image files as a zarr
     volume with ome-ngff metadata
 
@@ -472,10 +472,10 @@ def run(self):
             self.args["max_mip"],
             self.args["mip_dsfactor"],
             self.args["chunk_size"],
+            self.args["num_chunks"],
             concurrency=self.args["concurrency"],
             compression=self.args["compression"],
             #lvl_to_mip_kwargs=self.args["lvl_to_mip_kwargs"],
-            numchunks = self.args["num_chunks"],
             chunknum = self.args["chunk_num"],
             deskew_options=deskew_options)
 

From 03f6eda7f3696994809f569f309c8e8cb79b813b Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 1 Oct 2024 09:37:37 -0700
Subject: [PATCH 102/115] debugging

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py            | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 3d592b0..6ff9ca9 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -89,7 +89,7 @@ def iterate_numpy_blocks_from_dataset(
     """
     
     if numslice == 1:
-        print("1 slice: using chunk x = " + str(chunknum))
+        print("*****1 slice: using chunk x = " + str(chunknum) + "***")
         nchunks = (nblocks[0],nblocks[1],1)
         test = True
     else:
@@ -237,7 +237,7 @@ def iterate_mip_levels_from_dataset(
         # get level 0 chunks
         # block_size is the number of slices to read from tiffs
         for block in iterate_numpy_blocks_from_dataset(
-                dataset, nblocks,numslice=num_slice,chunknum=chunknum, block_size=block_size, pad=False,
+                dataset, nblocks,numslice = num_slice, chunknum=chunknum, block_size=block_size, pad=False,
                 deskew_kwargs=deskew_kwargs,
                 channel=channel):
             block_tuple = numpy.unravel_index(block_index,nblocks,order='F')

From 7c4ce1cbd1410e4cc59a6cb7c9df2f5ec946dbd9 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 1 Oct 2024 09:47:29 -0700
Subject: [PATCH 103/115] block size

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 6ff9ca9..7e4e87f 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -303,7 +303,7 @@ def write_ims_to_zarr(
     #     dataset = dataset.transpose((0,2,1))
     #     print("transposed shape: " + str(dataset.shape))
     
-    block_size = [256,512,512] #[ims_chunk_size[0]*2,ims_chunk_size[1]*32,ims_chunk_size[2]*8] #[128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
+    block_size = [256,256,512] #[ims_chunk_size[0]*2,ims_chunk_size[1]*32,ims_chunk_size[2]*8] #[128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
     print("deskewed block size: " + str(block_size))
     
     # if numchunks < 1:

From 4f470f1888b4fa58f5c9be105ad40fc0f6a6c3bf Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 1 Oct 2024 09:52:18 -0700
Subject: [PATCH 104/115] block size

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 7e4e87f..393103c 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -303,7 +303,7 @@ def write_ims_to_zarr(
     #     dataset = dataset.transpose((0,2,1))
     #     print("transposed shape: " + str(dataset.shape))
     
-    block_size = [256,256,512] #[ims_chunk_size[0]*2,ims_chunk_size[1]*32,ims_chunk_size[2]*8] #[128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
+    block_size = [512,512,512] #[ims_chunk_size[0]*2,ims_chunk_size[1]*32,ims_chunk_size[2]*8] #[128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
     print("deskewed block size: " + str(block_size))
     
     # if numchunks < 1:

From cad4b958405437613158e11c51a96a2fd00c704b Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 1 Oct 2024 10:29:52 -0700
Subject: [PATCH 105/115] single slice by single block

---
 .../convert_to_n5/ims_to_ngff.py              | 34 +++++++++++--------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 393103c..7ab5973 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -90,7 +90,7 @@ def iterate_numpy_blocks_from_dataset(
     
     if numslice == 1:
         print("*****1 slice: using chunk x = " + str(chunknum) + "***")
-        nchunks = (nblocks[0],nblocks[1],1)
+        nchunks = nblocks #(nblocks[0],nblocks[1],1)
         test = True
     else:
         nchunks = nblocks
@@ -114,7 +114,11 @@ def iterate_numpy_blocks_from_dataset(
             if chunk_tuple[0] == 0:
                 chunk_index = 0
                 deskew_kwargs["slice1d"][...] = 0
-                first_z,first_slice = psd.calculate_first_chunk(chunk_size=chunk_size,x_index=(nblocks[2] - chunk_tuple[1] - 1),stride=deskew_kwargs["stride"])
+                if test:
+                    first_z = 0
+                    first_slice = 0
+                else:
+                    first_z,first_slice = psd.calculate_first_chunk(chunk_size=chunk_size,x_index=(nblocks[2] - chunk_tuple[1] - 1),stride=deskew_kwargs["stride"])
                 print(str(first_z) + "," + str(first_slice))
             if chunk_tuple[0] < first_z or chunk_tuple[0]*chunk_size[0] - first_slice >= dshape[0]:
                 arr = numpy.zeros(block_size,dtype=dataset.dtype)
@@ -212,8 +216,8 @@ def iterate_mip_levels_from_dataset(
                           else lvl_to_mip_kwargs)
     mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
     block_index = 0
-    if num_slice == 1:
-        block_index = nblocks[0]*nblocks[1]*chunknum
+    # if num_slice == 1:
+    #     block_index = nblocks[0]*nblocks[1]*chunknum
     if lvl > 0:
         for ma in iterate_mip_levels_from_dataset(
                 dataset, lvl-1, maxlvl, nblocks, block_size,
@@ -306,17 +310,17 @@ def write_ims_to_zarr(
     block_size = [512,512,512] #[ims_chunk_size[0]*2,ims_chunk_size[1]*32,ims_chunk_size[2]*8] #[128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
     print("deskewed block size: " + str(block_size))
     
-    # if numchunks < 1:
-    joined_shapes = dataset.shape[2:]
-    if deskew_options and deskew_options["deskew_transpose"]:
-        # input dataset must be transposed
-        joined_shapes = (joined_shapes[0],joined_shapes[2],joined_shapes[1])
-    # else:
-    #     if deskew_options and deskew_options["deskew_transpose"]:
-    #         # input dataset must be transposed
-    #         joined_shapes = (dataset.shape[2],dataset.shape[4],numchunks*block_size[2])#(dataset.shape[2],numchunks*block_size[1],dataset.shape[3])
-    #     else:
-    #         joined_shapes = (dataset.shape[2],dataset.shape[3],numchunks*block_size[2])#(dataset.shape[2],numchunks*block_size[1],dataset.shape[4])
+    if numchunks < 1:
+        joined_shapes = dataset.shape[2:]
+        if deskew_options and deskew_options["deskew_transpose"]:
+            # input dataset must be transposed
+            joined_shapes = (joined_shapes[0],joined_shapes[2],joined_shapes[1])
+    else:
+        if deskew_options and deskew_options["deskew_transpose"]:
+            # input dataset must be transposed
+            joined_shapes = (dataset.shape[2],dataset.shape[4],numchunks*block_size[2])
+        else:
+            joined_shapes = (dataset.shape[2],dataset.shape[3],numchunks*block_size[2])
     print("ims_to_ngff dataset shape:" + str(joined_shapes))
 
     if deskew_options and deskew_options["deskew_method"] == "ps":

From 53bf56952e9e92dbd66be1f173c19dc1554afe3a Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Fri, 18 Oct 2024 21:56:44 -0700
Subject: [PATCH 106/115] dirty masking

---
 .../convert_to_n5/ims_to_ngff.py              | 168 +++++++++---------
 1 file changed, 87 insertions(+), 81 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 7ab5973..7fb45e0 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -70,7 +70,7 @@ def write_mips(zgrp,miparrs):
 
 
 def iterate_numpy_blocks_from_dataset(
-        dataset, nblocks, numslice=0, chunknum=-1, block_size=None, pad=True, deskew_kwargs={}, *args, **kwargs):
+        dataset, nblocks, chunknum=-1, block_size=None, pad=True, deskew_kwargs={}, *args, **kwargs):
     """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
 
     Parameters
@@ -88,13 +88,15 @@ def iterate_numpy_blocks_from_dataset(
         3D numpy array representing a consecutive chunk of 2D arrays
     """
     
-    if numslice == 1:
-        print("*****1 slice: using chunk x = " + str(chunknum) + "***")
-        nchunks = nblocks #(nblocks[0],nblocks[1],1)
-        test = True
-    else:
-        nchunks = nblocks
-        test = False
+    # if numslice == 1:
+    #     print("*****1 slice: using chunk x = " + str(chunknum) + "***")
+    #     nchunks = nblocks #(nblocks[0],nblocks[1],1)
+    #     test = True
+    # else:
+    #     nchunks = nblocks
+    #     test = False
+    nchunks = nblocks
+    test = False
     dshape = dataset.shape[2:]
     if deskew_kwargs:
         chunk_size = (deskew_kwargs["chunklength"],block_size[1],block_size[2]*deskew_kwargs["stride"])
@@ -104,82 +106,86 @@ def iterate_numpy_blocks_from_dataset(
         print("chunk size: " + str(chunk_size))
     for i in range(numpy.prod(nchunks)):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nchunks),order='F')
-        if test:
-            chunk_tuple = (chunk_tuple[0], chunk_tuple[1], chunknum)
-        # deskew level 0 data blocks
-        if deskew_kwargs:
-            if deskew_kwargs["transpose"]:
-                chunk_tuple = (chunk_tuple[0],chunk_tuple[2],chunk_tuple[1])
-            print(str(chunk_tuple))
-            if chunk_tuple[0] == 0:
-                chunk_index = 0
-                deskew_kwargs["slice1d"][...] = 0
-                if test:
-                    first_z = 0
-                    first_slice = 0
+        chunk_is_ok = chunk_tuple[2] > chunknum
+        if chunk_is_ok:
+            if test:
+                chunk_tuple = (chunk_tuple[0], chunk_tuple[1], chunknum)
+            # deskew level 0 data blocks
+            if deskew_kwargs:
+                if deskew_kwargs["transpose"]:
+                    chunk_tuple = (chunk_tuple[0],chunk_tuple[2],chunk_tuple[1])
+                print(str(chunk_tuple))
+                if chunk_tuple[0] == 0:
+                    chunk_index = 0
+                    deskew_kwargs["slice1d"][...] = 0
+                    if test:
+                        first_z = 0
+                        first_slice = 0
+                    else:
+                        first_z,first_slice = psd.calculate_first_chunk(chunk_size=chunk_size,x_index=(nblocks[2] - chunk_tuple[1] - 1),stride=deskew_kwargs["stride"])
+                    print(str(first_z) + "," + str(first_slice))
+                if chunk_tuple[0] < first_z or chunk_tuple[0]*chunk_size[0] - first_slice >= dshape[0]:
+                    arr = numpy.zeros(block_size,dtype=dataset.dtype)
                 else:
-                    first_z,first_slice = psd.calculate_first_chunk(chunk_size=chunk_size,x_index=(nblocks[2] - chunk_tuple[1] - 1),stride=deskew_kwargs["stride"])
-                print(str(first_z) + "," + str(first_slice))
-            if chunk_tuple[0] < first_z or chunk_tuple[0]*chunk_size[0] - first_slice >= dshape[0]:
-                arr = numpy.zeros(block_size,dtype=dataset.dtype)
+                    chunk_start = numpy.array([t*s for t,s in zip(chunk_tuple,chunk_size)])
+                    chunk_end = chunk_start + numpy.array(chunk_size)
+                    if chunk_start[0] < first_slice:
+                        chunk_end[0] = chunk_size[0] - (first_slice - chunk_start[0])
+                        chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
+                        zdata = numpy.squeeze(numpy.asarray(dataset[0,0,:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]))
+                        print("data dimension is " + str(zdata.shape) + " max is " + str(numpy.max(zdata)))
+                        chunk[first_slice-chunk_start[0]:] = zdata
+                    else:
+                        chunk_start[0] -= first_slice
+                        chunk_end[0] -= first_slice
+                        # if chunk_end[0] >= dshape[0]:
+                        #     chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
+                        #     chunk[:dshape[0]-chunk_start[0]] = dataset[chunk_start[0]:,chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
+                        # else:
+                        print(str(chunk_start[0]))
+                        zdata = numpy.squeeze(numpy.asarray(dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]))
+                        print("data dimension is " + str(zdata.shape) + " max is " + str(numpy.max(zdata)))
+                        chunk = zdata
+                    if any([sh<sz for sh,sz in zip(chunk.shape,chunk_size)]):
+                        print(str(chunk.shape) + " is small for" + str(chunk_size) + ": filling with zeros")
+                        temp_chunk = numpy.zeros(chunk_size,dtype=chunk.dtype)
+                        temp_chunk[:chunk.shape[0],:chunk.shape[1],:chunk.shape[2]] = chunk
+                        chunk = temp_chunk
+                    if deskew_kwargs["transpose"]:
+                        chunk = chunk.transpose((0,2,1))
+                    arr = numpy.flip(
+                        numpy.transpose(
+                            psd.deskew_block(
+                                chunk,
+                                chunk_index,
+                                **deskew_kwargs), 
+                            (2, 1, 0)),
+                        axis=2)
+                chunk_index += 1
             else:
-                chunk_start = numpy.array([t*s for t,s in zip(chunk_tuple,chunk_size)])
-                chunk_end = chunk_start + numpy.array(chunk_size)
-                if chunk_start[0] < first_slice:
-                    chunk_end[0] = chunk_size[0] - (first_slice - chunk_start[0])
-                    chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
-                    zdata = numpy.squeeze(numpy.asarray(dataset[0,0,:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]))
-                    print("data dimension is " + str(zdata.shape) + " max is " + str(numpy.max(zdata)))
-                    chunk[first_slice-chunk_start[0]:] = zdata
+                if chunk_tuple[0] == 0:
+                    print(str(chunk_tuple))
+                block_start = [chunk_tuple[k]*block_size[k] for k in range(3)]
+                block_end = [block_start[k] + block_size[k] for k in range(3)]
+                arr = numpy.squeeze(numpy.asarray(dataset[0,0,block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]]))
+            if any([arr.shape[k] != block_size[k] for k in range(3)]):
+                print(str(arr.shape) + "is small for " + str(block_size))
+                if pad:
+                    newarr = numpy.zeros(block_size,
+                                          dtype=arr.dtype)
+                    newarr[:arr.shape[0], :arr.shape[1], :arr.shape[2]] = arr[:, :, :]
+                    yield i,newarr
                 else:
-                    chunk_start[0] -= first_slice
-                    chunk_end[0] -= first_slice
-                    # if chunk_end[0] >= dshape[0]:
-                    #     chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
-                    #     chunk[:dshape[0]-chunk_start[0]] = dataset[chunk_start[0]:,chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
-                    # else:
-                    print(str(chunk_start[0]))
-                    zdata = numpy.squeeze(numpy.asarray(dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]))
-                    print("data dimension is " + str(zdata.shape) + " max is " + str(numpy.max(zdata)))
-                    chunk = zdata
-                if any([sh<sz for sh,sz in zip(chunk.shape,chunk_size)]):
-                    print(str(chunk.shape) + " is small for" + str(chunk_size) + ": filling with zeros")
-                    temp_chunk = numpy.zeros(chunk_size,dtype=chunk.dtype)
-                    temp_chunk[:chunk.shape[0],:chunk.shape[1],:chunk.shape[2]] = chunk
-                    chunk = temp_chunk
-                if deskew_kwargs["transpose"]:
-                    chunk = chunk.transpose((0,2,1))
-                arr = numpy.flip(
-                    numpy.transpose(
-                        psd.deskew_block(
-                            chunk,
-                            chunk_index,
-                            **deskew_kwargs), 
-                        (2, 1, 0)),
-                    axis=2)
-            chunk_index += 1
-        else:
-            if chunk_tuple[0] == 0:
-                print(str(chunk_tuple))
-            block_start = [chunk_tuple[k]*block_size[k] for k in range(3)]
-            block_end = [block_start[k] + block_size[k] for k in range(3)]
-            arr = numpy.squeeze(numpy.asarray(dataset[0,0,block_start[0]:block_end[0],block_start[1]:block_end[1],block_start[2]:block_end[2]]))
-        if any([arr.shape[k] != block_size[k] for k in range(3)]):
-            print(str(arr.shape) + "is small for " + str(block_size))
-            if pad:
-                newarr = numpy.zeros(block_size,
-                                      dtype=arr.dtype)
-                newarr[:arr.shape[0], :arr.shape[1], :arr.shape[2]] = arr[:, :, :]
-                yield newarr
+                    yield i,arr
             else:
-                yield arr
+                yield i,arr
         else:
-            yield arr
+            yield i,None
 
 
 def iterate_mip_levels_from_dataset(
         dataset, lvl, maxlvl, nblocks, block_size, downsample_factor,
-        num_slice=0,chunknum=-1,downsample_method=None, lvl_to_mip_kwargs=None,
+        chunknum=-1,downsample_method=None, lvl_to_mip_kwargs=None,
         interleaved_channels=1, channel=0, deskew_kwargs={}):
     """recursively generate MIPmap levels from an iterator of blocks
 
@@ -215,13 +221,13 @@ def iterate_mip_levels_from_dataset(
     lvl_to_mip_kwargs = ({} if lvl_to_mip_kwargs is None
                           else lvl_to_mip_kwargs)
     mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
-    block_index = 0
+    # block_index = 0
     # if num_slice == 1:
     #     block_index = nblocks[0]*nblocks[1]*chunknum
     if lvl > 0:
         for ma in iterate_mip_levels_from_dataset(
                 dataset, lvl-1, maxlvl, nblocks, block_size,
-                downsample_factor,num_slice,chunknum, downsample_method,
+                downsample_factor, chunknum, downsample_method,
                 lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
                 channel=channel, deskew_kwargs=deskew_kwargs):
             chunk = ma.array
@@ -240,15 +246,15 @@ def iterate_mip_levels_from_dataset(
     else:
         # get level 0 chunks
         # block_size is the number of slices to read from tiffs
-        for block in iterate_numpy_blocks_from_dataset(
-                dataset, nblocks,numslice = num_slice, chunknum=chunknum, block_size=block_size, pad=False,
+        for block_index,block in iterate_numpy_blocks_from_dataset(
+                dataset, nblocks, chunknum=chunknum, block_size=block_size, pad=False,
                 deskew_kwargs=deskew_kwargs,
                 channel=channel):
             block_tuple = numpy.unravel_index(block_index,nblocks,order='F')
             block_start = tuple(block_tuple[k]*block_size[k] for k in range(3))
             block_end = tuple(block_start[k] + block.shape[k] for k in range(3))
             yield MIPArray(lvl, block, block_start, block_end)
-            block_index += 1
+            # block_index += 1
 
 
 def write_ims_to_zarr(
@@ -404,7 +410,7 @@ def write_ims_to_zarr(
             mips = []
             for miparr in iterate_mip_levels_from_dataset(
                     dataset, max_mip, max_mip, nblocks, block_size, mip_dsfactor,
-                    num_slice=numchunks,chunknum=chunknum,
+                    chunknum=chunknum,
                     lvl_to_mip_kwargs=lvl_to_mip_kwargs,
                     interleaved_channels=interleaved_channels,
                     channel=channel, deskew_kwargs=deskew_kwargs):

From e1ec5f5688feb1c29fa896126b8302e12bf24aff Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Fri, 18 Oct 2024 22:02:34 -0700
Subject: [PATCH 107/115] hotfix

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 7fb45e0..f56e327 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -106,7 +106,7 @@ def iterate_numpy_blocks_from_dataset(
         print("chunk size: " + str(chunk_size))
     for i in range(numpy.prod(nchunks)):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nchunks),order='F')
-        chunk_is_ok = chunk_tuple[2] > chunknum
+        chunk_is_ok = chunk_tuple[2] >= chunknum
         if chunk_is_ok:
             if test:
                 chunk_tuple = (chunk_tuple[0], chunk_tuple[1], chunknum)

From bf1a0342cacd1d38f30d975410185ac89f9651d5 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Fri, 18 Oct 2024 22:06:17 -0700
Subject: [PATCH 108/115] hotfix

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py       | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index f56e327..fdd7dc1 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -250,10 +250,11 @@ def iterate_mip_levels_from_dataset(
                 dataset, nblocks, chunknum=chunknum, block_size=block_size, pad=False,
                 deskew_kwargs=deskew_kwargs,
                 channel=channel):
-            block_tuple = numpy.unravel_index(block_index,nblocks,order='F')
-            block_start = tuple(block_tuple[k]*block_size[k] for k in range(3))
-            block_end = tuple(block_start[k] + block.shape[k] for k in range(3))
-            yield MIPArray(lvl, block, block_start, block_end)
+            if not block is None:
+                block_tuple = numpy.unravel_index(block_index,nblocks,order='F')
+                block_start = tuple(block_tuple[k]*block_size[k] for k in range(3))
+                block_end = tuple(block_start[k] + block.shape[k] for k in range(3))
+                yield MIPArray(lvl, block, block_start, block_end)
             # block_index += 1
 
 

From 75c628edc32693286d33babfca23841cee9b3e2a Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Sat, 19 Oct 2024 09:45:06 -0700
Subject: [PATCH 109/115] chunk by slice

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py           | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index fdd7dc1..5fefb6c 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -106,7 +106,10 @@ def iterate_numpy_blocks_from_dataset(
         print("chunk size: " + str(chunk_size))
     for i in range(numpy.prod(nchunks)):#,*args,**kwargs):
         chunk_tuple = numpy.unravel_index(i,tuple(nchunks),order='F')
-        chunk_is_ok = chunk_tuple[2] >= chunknum
+        if chunknum >= 0:
+            chunk_is_ok = (chunk_tuple[2] == chunknum)
+        else:
+            chunk_is_ok = True
         if chunk_is_ok:
             if test:
                 chunk_tuple = (chunk_tuple[0], chunk_tuple[1], chunknum)

From b658b527e4d3b9f8be727bf72c564080e7dfbf6e Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 26 Nov 2024 16:36:30 -0800
Subject: [PATCH 110/115] fixing coordinates

---
 .../convert_to_n5/ims_to_ngff.py              | 28 ++++++++-----------
 1 file changed, 11 insertions(+), 17 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 5fefb6c..f51ffb5 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -88,23 +88,18 @@ def iterate_numpy_blocks_from_dataset(
         3D numpy array representing a consecutive chunk of 2D arrays
     """
     
-    # if numslice == 1:
-    #     print("*****1 slice: using chunk x = " + str(chunknum) + "***")
-    #     nchunks = nblocks #(nblocks[0],nblocks[1],1)
-    #     test = True
-    # else:
-    #     nchunks = nblocks
-    #     test = False
     nchunks = nblocks
     test = False
     dshape = dataset.shape[2:]
     if deskew_kwargs:
+        # calculate input data chunk dims based on output block dims
         chunk_size = (deskew_kwargs["chunklength"],block_size[1],block_size[2]*deskew_kwargs["stride"])
         if deskew_kwargs["transpose"]:
+            # transpose dims if needed
             chunk_size = (chunk_size[0],chunk_size[2],chunk_size[1])
             dshape = (dshape[0],dshape[2],dshape[1])
         print("chunk size: " + str(chunk_size))
-    for i in range(numpy.prod(nchunks)):#,*args,**kwargs):
+    for i in range(numpy.prod(nchunks)):
         chunk_tuple = numpy.unravel_index(i,tuple(nchunks),order='F')
         if chunknum >= 0:
             chunk_is_ok = (chunk_tuple[2] == chunknum)
@@ -125,7 +120,11 @@ def iterate_numpy_blocks_from_dataset(
                         first_z = 0
                         first_slice = 0
                     else:
-                        first_z,first_slice = psd.calculate_first_chunk(chunk_size=chunk_size,x_index=(nblocks[2] - chunk_tuple[1] - 1),stride=deskew_kwargs["stride"])
+                        if deskew_kwargs["flip"]:
+                            x_index = chunk_tuple[1]
+                        else:
+                            x_index = nblocks[2] - chunk_tuple[1] - 1
+                        first_z,first_slice = psd.calculate_first_chunk(chunk_size=chunk_size,x_index=x_index,stride=deskew_kwargs["stride"])
                     print(str(first_z) + "," + str(first_slice))
                 if chunk_tuple[0] < first_z or chunk_tuple[0]*chunk_size[0] - first_slice >= dshape[0]:
                     arr = numpy.zeros(block_size,dtype=dataset.dtype)
@@ -141,10 +140,6 @@ def iterate_numpy_blocks_from_dataset(
                     else:
                         chunk_start[0] -= first_slice
                         chunk_end[0] -= first_slice
-                        # if chunk_end[0] >= dshape[0]:
-                        #     chunk = numpy.zeros(chunk_size,dtype=dataset.dtype)
-                        #     chunk[:dshape[0]-chunk_start[0]] = dataset[chunk_start[0]:,chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]
-                        # else:
                         print(str(chunk_start[0]))
                         zdata = numpy.squeeze(numpy.asarray(dataset[0,0,chunk_start[0]:chunk_end[0],chunk_start[1]:chunk_end[1],chunk_start[2]:chunk_end[2]]))
                         print("data dimension is " + str(zdata.shape) + " max is " + str(numpy.max(zdata)))
@@ -255,6 +250,8 @@ def iterate_mip_levels_from_dataset(
                 channel=channel):
             if not block is None:
                 block_tuple = numpy.unravel_index(block_index,nblocks,order='F')
+                if deskew_kwargs["flip"]:
+                    block_tuple[2] = nblocks[2] - block_tuple[2] - 1
                 block_start = tuple(block_tuple[k]*block_size[k] for k in range(3))
                 block_end = tuple(block_start[k] + block.shape[k] for k in range(3))
                 yield MIPArray(lvl, block, block_start, block_end)
@@ -313,15 +310,12 @@ def write_ims_to_zarr(
     dataset = zarr.open(store,mode="r")
     ims_chunk_size = store.chunks
     print("ims chunks: " + str(ims_chunk_size))
-    # if deskew_options and deskew_options["deskew_transpose"]:
-    #     dataset = dataset.transpose((0,2,1))
-    #     print("transposed shape: " + str(dataset.shape))
     
     block_size = [512,512,512] #[ims_chunk_size[0]*2,ims_chunk_size[1]*32,ims_chunk_size[2]*8] #[128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
     print("deskewed block size: " + str(block_size))
     
+    joined_shapes = dataset.shape[2:]
     if numchunks < 1:
-        joined_shapes = dataset.shape[2:]
         if deskew_options and deskew_options["deskew_transpose"]:
             # input dataset must be transposed
             joined_shapes = (joined_shapes[0],joined_shapes[2],joined_shapes[1])

From 4f12a68a48052d7a38d169d4f33f277c4f37afa6 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 26 Nov 2024 16:41:04 -0800
Subject: [PATCH 111/115] fixing coordinates

---
 .../stitching_modules/convert_to_n5/ims_to_ngff.py            | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index f51ffb5..9ff6144 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -120,7 +120,7 @@ def iterate_numpy_blocks_from_dataset(
                         first_z = 0
                         first_slice = 0
                     else:
-                        if deskew_kwargs["flip"]:
+                        if not deskew_kwargs["flip"]:
                             x_index = chunk_tuple[1]
                         else:
                             x_index = nblocks[2] - chunk_tuple[1] - 1
@@ -250,7 +250,7 @@ def iterate_mip_levels_from_dataset(
                 channel=channel):
             if not block is None:
                 block_tuple = numpy.unravel_index(block_index,nblocks,order='F')
-                if deskew_kwargs["flip"]:
+                if not deskew_kwargs["flip"]:
                     block_tuple[2] = nblocks[2] - block_tuple[2] - 1
                 block_start = tuple(block_tuple[k]*block_size[k] for k in range(3))
                 block_end = tuple(block_start[k] + block.shape[k] for k in range(3))

From 7e6429ed416be43d1efda87d7eff8f4418b2386a Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 26 Nov 2024 18:19:18 -0800
Subject: [PATCH 112/115] bug fixing

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index 9ff6144..c94e86c 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -251,7 +251,7 @@ def iterate_mip_levels_from_dataset(
             if not block is None:
                 block_tuple = numpy.unravel_index(block_index,nblocks,order='F')
                 if not deskew_kwargs["flip"]:
-                    block_tuple[2] = nblocks[2] - block_tuple[2] - 1
+                    block_tuple = (block_tuple[0],block_tuple[1],nblocks[2] - block_tuple[2] - 1)
                 block_start = tuple(block_tuple[k]*block_size[k] for k in range(3))
                 block_end = tuple(block_start[k] + block.shape[k] for k in range(3))
                 yield MIPArray(lvl, block, block_start, block_end)

From 97a7287580daa61fd255c6f661c3f945ebac7d24 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 26 Nov 2024 20:21:28 -0800
Subject: [PATCH 113/115] process concurrency

---
 .../convert_to_n5/ims_to_ngff.py              | 55 +++++++++++++------
 1 file changed, 38 insertions(+), 17 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index c94e86c..eb7ffb1 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -260,8 +260,8 @@ def iterate_mip_levels_from_dataset(
 
 def write_ims_to_zarr(
         ims_fn, output_n5, group_names, group_attributes=None, max_mip=0,
-        mip_dsfactor=(2, 2, 2), chunk_size=(1, 1, 64, 64, 64), numchunks=0,
-        concurrency=10, slice_concurrency=1,
+        mip_dsfactor=(2, 2, 2), chunk_size=(1, 1, 64, 64, 64),
+        concurrency=10, slice_concurrency=1, block_size=(512,512,512),
         compression="raw", dtype="uint16", lvl_to_mip_kwargs=None,
         interleaved_channels=1, channel=0, deskew_options=None, chunknum=-1, **kwargs):
     """write a stack represented by an iterator of multi-image files as a zarr
@@ -311,20 +311,19 @@ def write_ims_to_zarr(
     ims_chunk_size = store.chunks
     print("ims chunks: " + str(ims_chunk_size))
     
-    block_size = [512,512,512] #[ims_chunk_size[0]*2,ims_chunk_size[1]*32,ims_chunk_size[2]*8] #[128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
+    # block_size = [512,512,512] #[ims_chunk_size[0]*2,ims_chunk_size[1]*32,ims_chunk_size[2]*8] #[128,2048,512] #[m*sz for m,sz in zip([2,2**max_mip,8],chunk_size[2:])]
     print("deskewed block size: " + str(block_size))
     
     joined_shapes = dataset.shape[2:]
-    if numchunks < 1:
-        if deskew_options and deskew_options["deskew_transpose"]:
-            # input dataset must be transposed
-            joined_shapes = (joined_shapes[0],joined_shapes[2],joined_shapes[1])
-    else:
-        if deskew_options and deskew_options["deskew_transpose"]:
-            # input dataset must be transposed
-            joined_shapes = (dataset.shape[2],dataset.shape[4],numchunks*block_size[2])
-        else:
-            joined_shapes = (dataset.shape[2],dataset.shape[3],numchunks*block_size[2])
+    if deskew_options and deskew_options["deskew_transpose"]:
+        # input dataset must be transposed
+        joined_shapes = (joined_shapes[0],joined_shapes[2],joined_shapes[1])
+    # else:
+    #     if deskew_options and deskew_options["deskew_transpose"]:
+    #         # input dataset must be transposed
+    #         joined_shapes = (dataset.shape[2],dataset.shape[4],numchunks*block_size[2])
+    #     else:
+    #         joined_shapes = (dataset.shape[2],dataset.shape[3],numchunks*block_size[2])
     print("ims_to_ngff dataset shape:" + str(joined_shapes))
 
     if deskew_options and deskew_options["deskew_method"] == "ps":
@@ -431,7 +430,15 @@ def write_ims_to_zarr(
     #f.close()
 
 
-def ims_to_ngff_group(ims_fn, output, *args, **kwargs):
+def calculate_blocks(block_size,deskew_options,**kwargs):
+    if deskew_options:
+        stride = deskew_options["stride"]
+    else:
+        stride = 1
+    return int(numpy.ceil(10640/(block_size[2]*stride)))
+
+
+def ims_to_ngff_group(ims_fn, output, *args, block_cc=1, chunknum=-1, **kwargs):
     """convert directory of natsort-consecutive multitiffs to an n5 or zarr pyramid
 
     Parameters
@@ -442,7 +449,16 @@ def ims_to_ngff_group(ims_fn, output, *args, **kwargs):
 
     if output == 'zarr':
         print('converting to zarr')
-        return write_ims_to_zarr(ims_fn, *args, **kwargs)
+        if chunknum > -1:
+            return write_ims_to_zarr(ims_fn, *args, slice_concurrency=1, chunknum=chunknum, **kwargs)
+        else:
+            numblocks = calculate_blocks(**kwargs)
+            with concurrent.futures.ProcessPoolExecutor(max_workers=block_cc) as e:
+                futs = []
+                for n in range(numblocks):
+                    futs.append(e.submit(write_ims_to_zarr,ims_fn,*args,slice_concurrency=block_cc, chunknum=n, **kwargs))
+                for fut in concurrent.futures.as_completed(futs):
+                    _ = fut.result()
     else:
         print('unknown output format: ' + output)
 
@@ -452,8 +468,12 @@ class IMSToNGFFParameters(NGFFGroupGenerationParameters):
     input_file = argschema.fields.Str(required=True)
     interleaved_channels = argschema.fields.Int(required=False, default=1)
     channel = argschema.fields.Int(required=False, default=0)
-    num_chunks = argschema.fields.Int(required=False, default=0)
     chunk_num = argschema.fields.Int(required=False, default=-1)
+    block_concurrency = argschema.fields.Int(required=False, default=1)
+    block_size = argschema.fields.Tuple((
+        argschema.fields.Int(),
+        argschema.fields.Int(),
+        argschema.fields.Int()), required=False, default=(512,512,512))
 
 
 class IMSToZarrInputParameters(argschema.ArgSchema,
@@ -480,11 +500,12 @@ def run(self):
             self.args["max_mip"],
             self.args["mip_dsfactor"],
             self.args["chunk_size"],
-            self.args["num_chunks"],
             concurrency=self.args["concurrency"],
             compression=self.args["compression"],
             #lvl_to_mip_kwargs=self.args["lvl_to_mip_kwargs"],
             chunknum = self.args["chunk_num"],
+            block_cc = self.args["block_concurrency"],
+            block_size = self.args["block_size"],
             deskew_options=deskew_options)
 
 

From ba040c3a7098786cf4372e4544f1a41077a50352 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Tue, 26 Nov 2024 20:28:30 -0800
Subject: [PATCH 114/115] bug fixing

---
 acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
index eb7ffb1..f10467b 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/ims_to_ngff.py
@@ -432,7 +432,7 @@ def write_ims_to_zarr(
 
 def calculate_blocks(block_size,deskew_options,**kwargs):
     if deskew_options:
-        stride = deskew_options["stride"]
+        stride = deskew_options["deskew_stride"]
     else:
         stride = 1
     return int(numpy.ceil(10640/(block_size[2]*stride)))

From 8e7ad85aa6bf96adf615684f9cc72032039cf750 Mon Sep 17 00:00:00 2001
From: Kevin Takasaki <kevint@alleninstitute.org>
Date: Thu, 16 Jan 2025 15:17:52 -0800
Subject: [PATCH 115/115] housecleaning and comments for merging

---
 .../convert_to_n5/tiff_to_ngff.py             | 152 +-----------------
 .../convert_to_n5/tiff_to_visor.py            |   3 +
 .../convert_to_n5/zarr_to_ngff.py             |   4 +
 3 files changed, 11 insertions(+), 148 deletions(-)

diff --git a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py
index 10beb7a..043181e 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_ngff.py
@@ -96,48 +96,14 @@ def iterate_2d_arrays_from_mimgfns(mimgfns, interleaved_channels=1, channel=0):
             offset = (offset + r.get_length()) % interleaved_channels
 
 
-# def iterate_numpy_chunks_from_mimgfns(
-#         mimgfns, slice_length=None, pad=True, *args, **kwargs):
-#     """iterate over a contiguous iterator of imageio multi-image files as
-#     chunks of numpy arrays
-
-#     Parameters
-#     ----------
-#     mimgfns : list of str
-#         imageio-compatible name inputs to be opened as multi-images
-#     slice_length : int
-#         number of 2d arrays from mimgfns included per chunk
-#     pad : bool, optional
-#         whether to extend final returned chunks with zeros
-
-#     Yields
-#     ------
-#     arr : numpy.ndarray
-#         3D numpy array representing a consecutive chunk of 2D arrays
-#     """
-#     array_gen = iterate_2d_arrays_from_mimgfns(mimgfns, *args, **kwargs)
-#     for chunk in iterate_chunks(array_gen, slice_length):
-#         arr = numpy.array(chunk)
-#         if pad:
-#             if arr.shape[0] != slice_length:
-#                 newarr = numpy.zeros((slice_length, *arr.shape[1:]),
-#                                      dtype=arr.dtype)
-#                 newarr[:arr.shape[0], :, :] = arr[:, :, :]
-#                 yield newarr
-#             else:
-#                 yield arr
-#         else:
-#             yield arr
-
-
 def iterate_numpy_chunks_from_dataset(
         dataset, slice_length=None, pad=True, *args, **kwargs):
     """iterate over a contiguous hdf5 daataset as chunks of numpy arrays
 
     Parameters
     ----------
-    dataset : hdf5 dataset
-        imageio-compatible name inputs to be opened as multi-images
+    dataset : sliceable array
+        iterable array generated from mimgfns or hdf5 dataset
     slice_length : int
         number of 2d arrays from mimgfns included per chunk
     pad : bool, optional
@@ -447,116 +413,6 @@ class MIPArray:
     end: int
 
 
-# def iterate_mip_levels_from_mimgfns(
-#         mimgfns, lvl, block_size, slice_length, downsample_factor,
-#         downsample_method=None, lvl_to_mip_kwargs=None,
-#         interleaved_channels=1, channel=0, deskew_kwargs={}):
-#     """recursively generate MIPmap levels from an iterator of multi-image files
-
-#     Parameters
-#     ----------
-#     mimgfns : list of str
-#         imageio-compatible name inputs to be opened as multi-images
-#     lvl : int
-#         integer mip level to generate
-#     block_size : int
-#         number of 2D arrays in chunk to process for a chunk at lvl
-#     slice_length : int
-#         number of 2D arrays to gather from tiff stacks
-#     downsample_factor : tuple of int
-#         integer downsampling factor for MIP levels
-#     downsample_method : str, optional
-#         downsampling method for
-#         acpreprocessing.utils.convert.downsample_stack_volume
-#     lvl_to_mip_kwargs :  dict, optional
-#         mapping of MIP level to kwargs used in MIPmap generation
-#     interleaved_channels : int
-#         number of channels interleaved in the tiff files (default 1)
-#     channel : int, optional
-#         channel from which interleaved data should be read (default 0)
-#     deskew_kwargs : dict, optional
-#         parameters for pixel shifting deskew
-
-#     Yields
-#     ------
-#     ma : acpreprocessing.stitching_modules.convert_to_n5.tiff_to_n5.MipArray
-#         object describing chunked array, MIP level of origin, and chunk indices
-#     """
-#     lvl_to_mip_kwargs = ({} if lvl_to_mip_kwargs is None
-#                          else lvl_to_mip_kwargs)
-#     mip_kwargs = lvl_to_mip_kwargs.get(lvl, {})
-#     start_index = 0
-#     chunk_index = 0
-#     if lvl > 0:
-#         num_chunks = downsample_factor[0]
-
-#         i = 0
-#         for ma in iterate_mip_levels_from_mimgfns(
-#                 mimgfns, lvl-1, block_size, slice_length,
-#                 downsample_factor, downsample_method,
-#                 lvl_to_mip_kwargs, interleaved_channels=interleaved_channels,
-#                 channel=channel, deskew_kwargs=deskew_kwargs):
-#             chunk = ma.array
-#             # throw array up for further processing
-#             yield ma
-#             # ignore if not parent resolution
-#             if ma.lvl != lvl-1:
-#                 continue
-
-#             try:
-#                 temp_lminus1_arr
-#             except NameError:
-#                 temp_lminus1_arr = numpy.empty(
-#                     (num_chunks*chunk.shape[0], *chunk.shape[1:]),
-#                     dtype=chunk.dtype)
-
-#             # fill in temporary block according to index
-#             chunk_size = chunk.shape[0]
-#             block_offset = i*block_size
-
-#             temp_lminus1_arr[
-#                 block_offset:block_offset+chunk_size, :, :] = chunk[:, :, :]
-
-#             # copy op only for uneven final chunk
-#             if chunk_size != block_size:
-#                 temp_lminus1_arr = temp_lminus1_arr[
-#                     :block_offset+chunk_size, :, :]
-
-#             if i == num_chunks - 1:
-#                 temp_arr = (
-#                     downsample_array(
-#                         temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
-#                         method=downsample_method, **mip_kwargs))
-#                 end_index = start_index + temp_arr.shape[0]
-#                 yield MIPArray(lvl, temp_arr, start_index, end_index)
-#                 start_index += temp_arr.shape[0]
-#                 i = 0
-#             else:
-#                 i += 1
-#         # add any leftovers
-#         if i > 0:
-#             temp_arr = acpreprocessing.utils.convert.downsample_stack_volume(
-#                 temp_lminus1_arr, downsample_factor, dtype=chunk.dtype,
-#                 method=downsample_method)
-#             end_index = start_index + temp_arr.shape[0]
-#             yield MIPArray(lvl, temp_arr, start_index, end_index)
-#     else:
-#         # get level 0 chunks
-#         # block_size is the number of slices to read from tiffs
-#         for chunk in iterate_numpy_chunks_from_mimgfns(
-#                 mimgfns, slice_length, pad=False,
-#                 interleaved_channels=interleaved_channels,
-#                 channel=channel):
-#             # deskew level 0 chunk
-#             if deskew_kwargs:
-#                 chunk = numpy.transpose(psd.deskew_block(
-#                     chunk, chunk_index, **deskew_kwargs), (2, 1, 0))
-#             end_index = start_index + chunk.shape[0]
-#             yield MIPArray(lvl, chunk, start_index, end_index)
-#             start_index += chunk.shape[0]
-#             chunk_index += 1
-
-
 def iterate_mip_levels_from_dataset(
         dataset, lvl, block_size, slice_length, downsample_factor,
         downsample_method=None, lvl_to_mip_kwargs=None,
@@ -565,8 +421,8 @@ def iterate_mip_levels_from_dataset(
 
     Parameters
     ----------
-    dataset : hdf5 dataset
-        imageio-compatible name inputs to be opened as multi-images
+    dataset : sliceable array
+        hdf dataset or iterable array generated from mimgfns
     lvl : int
         integer mip level to generate
     block_size : int
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
index d393f91..0226945 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/tiff_to_visor.py
@@ -1,5 +1,8 @@
 #!/usr/bin/env python
 
+"""conversion of tiff data to visor-zarr format for Visor software compatibility
+"""
+
 import concurrent.futures
 import dataclasses
 import itertools
diff --git a/acpreprocessing/stitching_modules/convert_to_n5/zarr_to_ngff.py b/acpreprocessing/stitching_modules/convert_to_n5/zarr_to_ngff.py
index 9aebd61..6bdce0f 100644
--- a/acpreprocessing/stitching_modules/convert_to_n5/zarr_to_ngff.py
+++ b/acpreprocessing/stitching_modules/convert_to_n5/zarr_to_ngff.py
@@ -1,5 +1,9 @@
 #!/usr/bin/env python
 
+"""auxiliary module to run deskew on data already converted to zarr
+"""
+
+
 import concurrent.futures
 import dataclasses
 #import itertools