From 16aedc05dffe92410c5348146ee558766364d55f Mon Sep 17 00:00:00 2001 From: Logan Walker Date: Tue, 19 Nov 2024 14:38:21 -0500 Subject: [PATCH] V0.2.0 Release (#36) * Bump version * Add repr for chunk debugging * Add sndif_util import for downsampling tools * Fix formatting * Fix formatting * Add downsampling inside of sisf_create function * Handle alternative channel counts more gracefully * Update iter variable to prevent scope conflict * Add error message for data input size during conversion --- src/pySISF/__init__.py | 2 +- src/pySISF/sisf.py | 92 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 84 insertions(+), 10 deletions(-) diff --git a/src/pySISF/__init__.py b/src/pySISF/__init__.py index dcc4534..57acdb1 100644 --- a/src/pySISF/__init__.py +++ b/src/pySISF/__init__.py @@ -5,7 +5,7 @@ """Import SISF components""" from __future__ import annotations -__version__ = "0.1.1" +__version__ = "0.2.0" from pySISF import sisf from pySISF import vidlib diff --git a/src/pySISF/sisf.py b/src/pySISF/sisf.py index 0a4cfcd..259c884 100644 --- a/src/pySISF/sisf.py +++ b/src/pySISF/sisf.py @@ -15,7 +15,7 @@ import zstd import numpy as np -from pySISF import vidlib +from pySISF import vidlib, sndif_utils METADATA_NAME = "metadata.bin" DEBUG = False @@ -162,7 +162,22 @@ def iter_chunks(executor): fmeta.write(bytes(towrite)) -def create_sisf(fname, data, mchunk_size, chunk_size, res, enable_status=True): +def create_sisf( + fname: str, data, mchunk_size, chunk_size, res, enable_status=True, downsampling=None, compression=1, thread_count=8 +) -> None: + """ + Function to create a SISF archive. + + Parameters: + fname (string): Name of the folder to place the SISF archive into, created if does not exist. + data (numpy array-like): Represents the data to be converted in CXYZ format. + mchunk_size (3-tuple): size of metachunks to create, e.g. (2000,2000,2000). + res (3-tuple): resolution of the dataset in nanometers (nm), e.g. (100,100,100). + enable_status (bool, default True): If true, print out a loading bar for creation using `tqdm`. + downsampling (int, default None): How many downsample tiers to generate. + compression (int, default 1->ZSTD): What compression codec to use. + thread_count (int, default 8): How many threads to use for data packing. + """ if fname.endswith("/"): fname = fname[:-1] @@ -173,12 +188,16 @@ def create_sisf(fname, data, mchunk_size, chunk_size, res, enable_status=True): pass # folder exists if len(data.shape) == 3: - data = np.expand_dims(data, 0) - - channel_count = data.shape[0] - size = data.shape[1:] + channel_count = 1 + size = data.shape + elif len(data.shape) == 4: + channel_count = data.shape[0] + size = data.shape[1:] + else: + raise ValueError(f"Invalid image dimension size {data.shape}!") dtype_code = get_dtype_code(data.dtype) + # TODO handle dtype errors print(channel_count, size) @@ -204,15 +223,67 @@ def create_sisf(fname, data, mchunk_size, chunk_size, res, enable_status=True): osizej = jend - jstart osizek = kend - kstart + # Generate file names chunk_name = f"chunk_{i}_{j}_{k}.{c}.1X" chunk_name_data = f"{fname}/data/{chunk_name}.data" chunk_name_meta = f"{fname}/meta/{chunk_name}.meta" - # make buffer + # Make buffer of only this metachunk chunk = np.zeros((osizei, osizej, osizek), dtype=np.uint16) - chunk[...] = data[c, istart:iend, jstart:jend, kstart:kend] - create_shard(chunk_name_data, chunk_name_meta, chunk, chunk_size, 1) + if channel_count == 1: + chunk[...] = data[istart:iend, jstart:jend, kstart:kend] + elif channel_count > 1: + chunk[...] = data[c, istart:iend, jstart:jend, kstart:kend] + else: + raise ValueError(f"Invalid channel count! ({channel_count})") + + # Save 1X image + create_shard( + chunk_name_data, chunk_name_meta, chunk, chunk_size, compression, thread_count=thread_count + ) + + # Perform downsampling + if downsampling is not None: + downsample_pyramid = [chunk] + + for scalei in range(downsampling): + # convert from 0, 1, 2, etc. -> 1X, 2X, 4X, etc. + scale = 2**scalei + + # Skip basecase, already handled above + if scale == 1: + continue + + # Generate downsampled file names + new_chunk_name_data = chunk_name_data.replace(".1X.", f".{scale}X.") + new_chunk_name_meta = chunk_name_meta.replace(".1X.", f".{scale}X.") + + chunk_down = np.zeros( + shape=( + downsample_pyramid[-1].shape[0] // 2, + downsample_pyramid[-1].shape[1] // 2, + downsample_pyramid[-1].shape[2] // 2, + ), + dtype=np.uint16, + ) + + # calculate downsampled image + sndif_utils.downsample(downsample_pyramid[-1], chunk_down) + downsample_pyramid.append(chunk_down) + + # Assign offsets, but not relevant here + + create_shard( + new_chunk_name_data, + new_chunk_name_meta, + downsample_pyramid[-1], + chunk_size, + compression, + thread_count=thread_count, + ) + + del downsample_pyramid if enable_status: status_bar.update(1) @@ -403,6 +474,9 @@ def __getitem__(self, key): return out + def __repr__(self): + return f"" + @staticmethod def iterate_chunks(rstart, rstop, cs): for cstart in range(cs * (rstart // cs), cs * ((rstop + cs - 1) // cs), cs):