diff --git a/docs/source/api.rst b/docs/source/api.rst index 6b2274d..58f5f88 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -172,19 +172,33 @@ Functions for reconstruction samplers: pymaid.get_sampler_domains pymaid.get_sampler_counts -Image data (tiles) +Image metadata +-------------- +Functions to fetch information about the image stacks CATMAID knows about. + +.. autosummary:: + :toctree: generated/ + + pymaid.get_stacks + pymaid.get_stack_info + pymaid.get_mirror_info + +Image data (tiles and N5 volumes) ------------------ Functions to fetch and process image data. Note that this is not imported at top level but has to be imported explicitly:: >>> from pymaid import tiles >>> help(tiles.crop_neuron) + >>> from pymaid.stack import Stack + >>> help(Stack) .. autosummary:: :toctree: generated/ pymaid.tiles.TileLoader pymaid.tiles.crop_neuron + pymaid.stack.Stack .. _api_misc: diff --git a/docs/source/whats_new.rst b/docs/source/whats_new.rst index cb6fc6d..8975c0d 100644 --- a/docs/source/whats_new.rst +++ b/docs/source/whats_new.rst @@ -15,6 +15,8 @@ What's new? - BREAKING: Drop python 3.7 support. - - :class:`pymaid.neuron_label.NeuronLabeller` added for labelling neurons like in the CATMAID frontend. + - :func:`pymaid.get_stacks`, :func:`pymaid.get_stack_info`, :func:`pymaid.get_mirror_info` functions for getting information about image data + - :class:`pymaid.stack.Stack` class for accessing N5 and JPEG tile image data as a :class:`xarray.DataArray` * - 2.4.0 - 27/05/23 - - :func:`pymaid.get_annotation_graph` deprecated in favour of the new diff --git a/pymaid/fetch/__init__.py b/pymaid/fetch/__init__.py index fc150b7..f01ae11 100644 --- a/pymaid/fetch/__init__.py +++ b/pymaid/fetch/__init__.py @@ -57,6 +57,7 @@ from .landmarks import get_landmarks, get_landmark_groups from .skeletons import get_skeleton_ids from .annotations import get_annotation_graph, get_entity_graph, get_annotation_id +from .stack import get_stacks, get_stack_info, get_mirror_info __all__ = ['get_annotation_details', 'get_annotation_id', @@ -93,6 +94,7 @@ 'get_landmarks', 'get_landmark_groups', 'get_skeleton_ids', + 'get_stacks', 'get_stack_info', 'get_mirror_info', ] # Set up logging diff --git a/pymaid/fetch/annotations.py b/pymaid/fetch/annotations.py index 1db874e..cba400a 100644 --- a/pymaid/fetch/annotations.py +++ b/pymaid/fetch/annotations.py @@ -391,7 +391,7 @@ def get_entity_graph( Neurons additionally have - - skeleton_ids: list[int] + - skeleton_ids: List[int] Skeletons additionally have diff --git a/pymaid/fetch/stack.py b/pymaid/fetch/stack.py new file mode 100644 index 0000000..0ac8997 --- /dev/null +++ b/pymaid/fetch/stack.py @@ -0,0 +1,251 @@ +from typing import Any, Optional, Union, Literal, Sequence, Tuple, Dict, List +import numpy as np +from ..utils import _eval_remote_instance +from ..client import CatmaidInstance +from enum import IntEnum +from dataclasses import dataclass, asdict + +Dimension = Literal["x", "y", "z"] + + +class Orientation(IntEnum): + XY = 0 + # todo: check these + XZ = 1 + ZY = 2 + + def __bool__(self) -> bool: + return True + + def full_orientation(self, reverse=False) -> Tuple[Dimension, Dimension, Dimension]: + out = [ + ("x", "y", "z"), + ("x", "z", "y"), + ("z", "y", "x"), + ][self.value] + if reverse: + out = out[::-1] + return out + + @classmethod + def from_dims(cls, dims: Sequence[Dimension]): + pair = (dims[0].lower(), dims[1].lower()) + out = { + ("x", "y"): cls.XY, + ("x", "z"): cls.XZ, + ("z", "y"): cls.ZY, + }.get(pair) + if out is None: + raise ValueError(f"Unknown dimensions: {dims}") + return out + + +@dataclass +class StackSummary: + id: int + pid: int + title: str + comment: str + + +def get_stacks(remote_instance: Optional[CatmaidInstance] = None) -> List[StackSummary]: + """Get summary of all stacks in the project. + + Parameters + ---------- + remote_instance : Optional[CatmaidInstance], optional + By default global instance. + + Returns + ------- + stacks + List of StackSummary objects. + """ + cm = _eval_remote_instance(remote_instance) + url = cm.make_url(cm.project_id, "stacks") + return [StackSummary(**r) for r in cm.fetch(url)] + + +@dataclass +class MirrorInfo: + id: int + title: str + image_base: str + tile_width: int + tile_height: int + tile_source_type: int + file_extension: str + position: int + + def to_jso(self): + return asdict(self) + + +@dataclass +class Color: + r: float + g: float + b: float + a: float + + +@dataclass +class StackInfo: + sid: int + pid: int + ptitle: str + stitle: str + downsample_factors: Optional[List[Dict[Dimension, float]]] + num_zoom_levels: int + translation: Dict[Dimension, float] + resolution: Dict[Dimension, float] + dimension: Dict[Dimension, int] + comment: str + description: str + metadata: Optional[str] + broken_slices: Dict[int, int] + mirrors: List[MirrorInfo] + orientation: Orientation + attribution: str + canary_location: Dict[Dimension, int] + placeholder_color: Color + + @classmethod + def from_jso(cls, sinfo: Dict[str, Any]): + sinfo["orientation"] = Orientation(sinfo["orientation"]) + sinfo["placeholder_color"] = Color(**sinfo["placeholder_color"]) + sinfo["mirrors"] = [MirrorInfo(**m) for m in sinfo["mirrors"]] + return StackInfo(**sinfo) + + def to_jso(self): + return asdict(self) + + def get_downsample(self, scale_level=0) -> Dict[Dimension, float]: + """Get the downsample factors for a given scale level. + + If the downsample factors are explicit in the stack info, + use that value. + Otherwise, use the CATMAID default: + scale by a factor of 2 per scale level in everything except the slicing dimension. + If number of scale levels is known, + ensure the scale level exists. + + Parameters + ---------- + scale_level : int, optional + + Returns + ------- + dict[Dimension, float] + + Raises + ------ + IndexError + If the scale level is known not to exist + """ + if self.downsample_factors is not None: + return self.downsample_factors[scale_level] + if self.num_zoom_levels > 0 and scale_level >= self.num_zoom_levels: + raise IndexError("list index out of range") + + first, second, slicing = self.orientation.full_orientation() + return {first: 2**scale_level, second: 2**scale_level, slicing: 1} + + def get_coords(self, scale_level: int = 0) -> Dict[Dimension, np.ndarray]: + dims = self.orientation.full_orientation() + dims = dims[::-1] + + downsamples = self.get_downsample(scale_level) + + out: Dict[Dimension, np.ndarray] = dict() + for d in dims: + c = np.arange(self.dimension[d], dtype=float) + c *= self.resolution[d] + c *= downsamples[d] + c += self.translation[d] + out[d] = c + return out + + +def get_stack_info( + stack: Union[int, str], remote_instance: Optional[CatmaidInstance] = None +) -> StackInfo: + """Get information about an image stack. + + Parameters + ---------- + stack : Union[int, str] + Integer ID or string title of the stack. + remote_instance : Optional[CatmaidInstance], optional + By default global. + + Returns + ------- + StackInfo + + Raises + ------ + ValueError + If an unknown stack title is given. + """ + cm = _eval_remote_instance(remote_instance) + if isinstance(stack, str): + stacks = get_stacks(cm) + for s in stacks: + if s.title == stack: + stack_id = s.id + break + else: + raise ValueError(f"No stack with title '{stack}'") + else: + stack_id = int(stack) + + url = cm.make_url(cm.project_id, "stack", stack_id, "info") + sinfo = cm.fetch(url) + return StackInfo.from_jso(sinfo) + + +def get_mirror_info( + stack: Union[int, str, StackInfo], + mirror: Union[int, str], + remote_instance: Optional[CatmaidInstance] = None, +) -> MirrorInfo: + """Get information about a stack mirror. + + Parameters + ---------- + stack : Union[int, str, StackInfo] + Integer stack ID, string stack title, + or an existing StackInfo object (avoids server request). + mirror : Union[int, str] + Integer mirror ID, or string mirror title. + remote_instance : Optional[CatmaidInstance] + By default, global. + + Returns + ------- + MirrorInfo + + Raises + ------ + ValueError + No mirror matching given ID/ title. + """ + if isinstance(stack, StackInfo): + stack_info = stack + else: + stack_info = get_stack_info(stack, remote_instance) + + if isinstance(mirror, str): + key = "title" + else: + key = "id" + mirror = int(mirror) + + for m in stack_info.mirrors: + if getattr(m, key) == mirror: + return m + + raise ValueError( + f"No mirror for stack '{stack_info.stitle}' with {key} {repr(mirror)}" + ) diff --git a/pymaid/neuron_label.py b/pymaid/neuron_label.py index 4f90844..ae46cbb 100644 --- a/pymaid/neuron_label.py +++ b/pymaid/neuron_label.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from functools import cache -from typing import Optional, Union +from typing import Optional, Union, List, Tuple import re import networkx as nx @@ -38,7 +38,7 @@ def __init__( self, skeleton_id: Optional[int] = None, name: Optional[str] = None, - annotations: Optional[list[str]] = None, + annotations: Optional[List[str]] = None, remote_instance: Optional[CatmaidInstance] = None, ) -> None: """ @@ -51,7 +51,7 @@ def __init__( If None, determined from name. name : Optional[str], optional If None, determined from skeleton ID. - annotations : Optional[list[str]], optional + annotations : Optional[List[str]], optional If None, determined from skeleton ID or name. remote_instance : Optional[CatmaidInstance], optional If None, uses global instance. @@ -90,7 +90,7 @@ def name(self) -> str: return self._name @property - def annotations(self) -> list[str]: + def annotations(self) -> List[str]: if self._annotations is None: skid = self.skeleton_id skid_to_anns = pymaid.get_annotations(skid) @@ -155,8 +155,8 @@ def __init__( super().__init__() def _filter_by_author( - self, annotations: list[str], remote_instance: CatmaidInstance - ) -> list[str]: + self, annotations: List[str], remote_instance: CatmaidInstance + ) -> List[str]: if self.annotator_name is None or not annotations: return annotations @@ -166,8 +166,8 @@ def _filter_by_author( return [a for a in annotations if a in allowed] def _filter_by_annotation( - self, annotations: list[str], remote_instance: CatmaidInstance - ) -> list[str]: + self, annotations: List[str], remote_instance: CatmaidInstance + ) -> List[str]: if self.annotated_with is None or not annotations: return annotations @@ -193,7 +193,7 @@ def dedup_whitespace(s: str): @cache def parse_components( fmt: str, -) -> tuple[list[str], list[tuple[str, int, Optional[str]]]]: +) -> Tuple[List[str], List[Tuple[str, int, Optional[str]]]]: joiners = [] components = [] last_end = 0 @@ -264,7 +264,7 @@ class NeuronLabeller: """Class for calculating neurons' labels, as used in the CATMAID frontend.""" def __init__( self, - components: Optional[list[LabelComponent]] = None, + components: Optional[List[LabelComponent]] = None, fmt="%0", trim_empty=True, remove_neighboring_duplicates=True, @@ -273,7 +273,7 @@ def __init__( Parameters ---------- - components : list[LabelComponent], optional + components : List[LabelComponent], optional The label components as used in CATMAID's user settings. See `SkeletonId`, `NeuronName`, and `Annotations`. First component should be ``SkeletonId()`` for compatibility with CATMAID. diff --git a/pymaid/stack.py b/pymaid/stack.py new file mode 100644 index 0000000..0c2b267 --- /dev/null +++ b/pymaid/stack.py @@ -0,0 +1,581 @@ +"""Access to image data as ``xarray.DataArray``s. + +CATMAID's image source conventions are documented here +https://catmaid.readthedocs.io/en/stable/tile_sources.html +""" +from __future__ import annotations +from functools import wraps +from io import BytesIO +from typing import Any, Literal, Optional, Sequence, Type, Union, Dict +from abc import ABC +import sys + +import numpy as np +from numpy.typing import DTypeLike, ArrayLike +import json +import requests + +from pymaid.client import CatmaidInstance + +from . import utils +from .fetch.stack import ( + StackInfo, + MirrorInfo, + get_stacks, + get_stack_info, + get_mirror_info, +) + +try: + import aiohttp + from dask import array as da + import imageio.v3 as iio + import xarray as xr + import zarr + from zarr.storage import BaseStore +except ImportError as e: + raise ImportError( + 'Optional dependencies for stack viewing are not available. ' + 'Make sure the appropriate extra is installed: `pip install pymaid[stack]`. ' + f'Original error: "{str(e)}"' + ) + +Dimension = Literal["x", "y", "z"] +# Orientation = Literal["xy", "xz", "zy"] +HALF_PX = 0.5 +ENDIAN = "<" if sys.byteorder == "little" else ">" + + +@wraps(print) +def eprint(*args, **kwargs): + """Thin wrapper around ``print`` which defaults to stderr""" + kwargs.setdefault("file", sys.stderr) + return print(*args, **kwargs) + + +def select_cli(prompt: str, options: Dict[int, str]) -> Optional[int]: + out = None + eprint(prompt) + for k, v in sorted(options.items()): + eprint(f"\t{k}.\t{v}") + p = "Type number and press enter (empty to cancel): " + while out is None: + result_str = input(p).strip() + if not result_str: + break + try: + result = int(result_str) + except ValueError: + eprint("Not an integer, try again") + continue + if result not in options: + eprint("Not a valid option, try again") + continue + out = result + return out + + +def to_array( + coord: Union[Dict[Dimension, Any], ArrayLike], + dtype: DTypeLike = np.float64, + order: Sequence[Dimension] = ("z", "y", "x"), +) -> np.ndarray: + if isinstance(coord, dict): + coord = [coord[d] for d in order] + return np.asarray(coord, dtype=dtype) + + +class ImageIoStore(BaseStore, ABC): + """ + Must include instance variable 'fmt', + which is a format string with variables: + image_base, zoom_level, file_extension, row, col, slice_idx + """ + + tile_source_type: int + fmt: str + _writeable = False + _erasable = False + _listable = False + + def __init__( + self, + stack_info: StackInfo, + mirror_info: MirrorInfo, + zoom_level: int, + session: Union[requests.Session, CatmaidInstance, None] = None, + ) -> None: + if mirror_info.tile_source_type != self.tile_source_type: + raise ValueError("Mismatched tile source type") + self.stack_info = stack_info + self.mirror_info = mirror_info + self.zoom_level = zoom_level + + if isinstance(session, CatmaidInstance): + self.session = session._session + elif isinstance(session, requests.Session): + self.session = session + elif session is None: + session = requests.Session() + + brok_sl = {int(k): int(k) + v for k, v in self.stack_info.broken_slices.items()} + self.broken_slices = dict() + for k, v in brok_sl.items(): + while v in brok_sl: + v = brok_sl[v] + self.broken_slices[k] = v + + order = self.stack_info.orientation.full_orientation(reverse=True) + self.metadata_bytes = json.dumps( + { + "zarr_format": 2, + "shape": to_array(stack_info.dimension, int, order).tolist(), + "chunks": [1, mirror_info.tile_height, mirror_info.tile_width], + "dtype": ENDIAN + "u1", + "compressor": None, + "fill_value": 0, + "order": "C", + "filters": None, + "dimension_separator": ".", + } + ).encode() + self.attrs_bytes = json.dumps( + { + "stack_info": self.stack_info.to_jso(), + "mirror_info": self.mirror_info.to_jso(), + "scale_level": self.zoom_level, + } + ).encode() + + self.empty = np.zeros( + ( + self.mirror_info.tile_width, + self.mirror_info.tile_height, + 1, + ), + "uint8", + ).tobytes() + + def _format_url(self, row: int, col: int, slice_idx: int) -> str: + return self.fmt.format( + image_base=self.mirror_info.image_base, + zoom_level=self.zoom_level, + slice_idx=slice_idx, + row=row, + col=col, + file_extension=self.mirror_info.file_extension, + ) + + def __delitem__(self, __key) -> None: + raise NotImplementedError() + + def __iter__(self): + raise NotImplementedError() + + def __len__(self) -> int: + raise NotImplementedError() + + def __setitem__(self, __key, __value) -> None: + raise NotImplementedError() + + def _resolve_broken_slices(self, slice_idx: int) -> int: + return self.broken_slices.get(slice_idx, slice_idx) + + def __getitem__(self, key): + last = key.split("/")[-1] + if last == ".zarray": + return self.metadata_bytes + elif last == ".zattrs": + return self.attrs_bytes + + # todo: check order + slice_idx, row, col = (int(i) for i in last.split(".")) + slice_idx = self._resolve_broken_slices(slice_idx) + + url = self._format_url(row, col, slice_idx) + response = self.session.get(url) + if response.status_code == 404: + return self.empty + response.raise_for_status() + ext = self.mirror_info.file_extension.split("?")[0] + if not ext.startswith("."): + ext = "." + ext + arr = iio.imread( + BytesIO(response.content), + extension=ext, + mode="L", + ) + return arr.tobytes() + + def to_zarr_array(self) -> zarr.Array: + return zarr.open_array(self, "r") + + def to_dask_array(self) -> xr.DataArray: + # todo: transpose? + as_zarr = self.to_zarr_array() + return da.from_zarr(as_zarr) + + def to_xarray(self) -> xr.DataArray: + as_dask = self.to_dask_array() + return xr.DataArray( + as_dask, + coords=self.stack_info.get_coords(self.zoom_level), + dims=self.stack_info.orientation.full_orientation(True), + ) + + +class TileStore1(ImageIoStore): + """File-based image stack.""" + tile_source_type = 1 + fmt = "{image_base}{slice_idx}/{row}_{col}_{zoom_level}.{file_extension}" + + +class TileStore4(ImageIoStore): + """File-based image stack with zoom level directories.""" + tile_source_type = 4 + fmt = "{image_base}{slice_idx}/{zoom_level}/{row}_{col}.{file_extension}" + + +class TileStore5(ImageIoStore): + """Directory-based image stack with zoom, z, and row directories.""" + tile_source_type = 5 + fmt = "{image_base}{zoom_level}/{slice_idx}/{row}/{col}.{file_extension}" + + +# class TileStore10(ImageIoStore): +# """H2N5 tile stack.""" +# tile_source_type = 10 +# fmt = "{image_base}.{file_extension}" + +# def _format_url(self, row: int, col: int, slice_idx: int) -> str: +# s = self.fmt.format( +# image_base=self.mirror_info.image_base, +# # todo: manually change quality? +# file_extension=self.mirror_info.file_extension, +# ) +# s = s.replace("%SCALE_DATASET%", f"s{self.zoom_level}") +# s = s.replace("%AXIS_0%", str(col * self.mirror_info.tile_width)) +# s = s.replace("%AXIS_1%", str(row * self.mirror_info.tile_height)) +# s = s.replace("%AXIS_2%", str(slice_idx)) +# return s + + +tile_stores: Dict[int, Type[ImageIoStore]] = { + t.tile_source_type: t + for t in [ + TileStore1, + TileStore4, + TileStore5, + # TileStore10 + ] +} +source_client_types = {k: (requests.Session,) for k in tile_stores} +source_client_types[11] = (aiohttp.ClientSession,) + +Client = Union[requests.Session, aiohttp.ClientSession] + + +def select_stack(remote_instance=None) -> Optional[int]: + """""" + stacks = get_stacks(remote_instance) + options = {s.id: s.title for s in stacks} + return select_cli("Select stack:", options) + + +class Stack: + """Class representing a CATMAID stack of images. + + Stacks are usually a scale pyramid. + This class can, for certain stack mirror types, + allow access to individual scale levels as arrays + which can be queried in voxel or world coordinates. + + HTTP requests to fetch stack data are often configured + differently for different stack mirrors and tile source types. + For most non-public mirrors, you will need to set the object to make these requests: + see the ``my_stack.set_mirror_session()`` method; + if you just need to set HTTP Basic authentication headers, + see the ``my_stack.set_mirror_auth()`` convenience method. + + See the ``my_stack.get_scale()`` method for getting an + `xarray.DataArray `_ + representing that scale level. + This can be queried in stack/ voxel or project/ world coordinates, + efficiently sliced and transposed etc.. + """ + def __init__( + self, + stack_info: StackInfo, + mirror: Optional[Union[int, str]] = None, + ): + """The :func:`Stack.from_catmaid` constructor may be more convenient. + + Parameters + ---------- + stack_info : StackInfo + mirror_id : Optional[int], optional + """ + self.stack_info = stack_info + self.mirror_info: Optional[MirrorInfo] = None + self.mirror_session: Dict[int, Any] = dict() + + if mirror is not None: + self.set_mirror(mirror) + + def set_mirror_auth(self, mirror: Union[int, str, None, MirrorInfo], http_user: str, http_password: str): + """Set the HTTP Basic credentials for a particular stack mirror. + + This will replace any other session configured for that mirror. + + For more fine-grained control (e.g. setting other headers), + or to re-use the session object from a ``CatmaidInstance``, + see ``my_stack.set_mirror_session()``. + + Parameters + ---------- + mirror : Union[int, str, None, MirrorInfo] + Mirror, as MirrorInfo, intger ID, string title, or None (use default) + http_user : str + HTTP Basic username + http_password : str + HTTP Basic password + + Raises + ------ + ValueError + If the given mirror is not supported. + """ + minfo = self._get_mirror_info(mirror) + if minfo.tile_source_type == 11: + s = aiohttp.ClientSession(auth=aiohttp.BasicAuth(http_user, http_password)) + return self.set_mirror_session(mirror, s) + elif minfo.tile_source_type in source_client_types: + s = requests.Session() + s.auth = (http_user, http_password) + return self.set_mirror_session(mirror, s) + else: + raise ValueError("Mirror's tile source type is unsupported: %s", minfo.tile_source_type) + + def set_mirror_session( + self, mirror: Union[int, str, None, MirrorInfo], session: Client, + ): + """Set functions which construct the session for fetching image data, per mirror. + + For most tile stacks, this is a + `requests.Session `_. + See ``get_remote_instance_session`` to use the session from a given + ``CatmaidInstance`` (including the global). + + For N5 (tile source 11), this is a + `aiohttp.ClientSession `_. + + Parameters + ---------- + mirror : Union[int, str, None] + Mirror, as integer ID, string name, or None to use the one defined on the class. + session : Union[requests.Session, aiohttp.ClientSession] + HTTP session of the appropriate type. + For example, to re-use the ``requests.Session`` from the + global ``CatmaidInstance`` for mirror with ID 1, use + ``my_stack.set_mirror_instance(1, get_remote_instance_session())``. + To use HTTP basic auth for an N5 stack mirror (tile source 11) with ID 2, use + ``my_stack.set_mirror_instance_factor(2, aiohttp.ClientSession(auth=aiohttp.BasicAuth("myusername", "mypassword")))``. + """ + + minfo = self._get_mirror_info(mirror) + self.mirror_session[minfo.id] = session + + @classmethod + def select_from_catmaid(cls, remote_instance=None): + """Interactively select a stack and mirror from those available. + + Parameters + ---------- + remote_instance : CatmaidInstance, optional + By default global. + """ + stacks = get_stacks(remote_instance) + options = {s.id: s.title for s in stacks} + sid = select_cli("Select stack:", options) + if not sid: + return None + out = cls.from_catmaid(sid, remote_instance=remote_instance) + out.select_mirror() + return out + + @classmethod + def from_catmaid( + cls, stack: Union[str, int], mirror: Optional[Union[int, str]] = None, remote_instance=None + ): + """Fetch relevant data from CATMAID and build a Stack. + + Parameters + ---------- + stack : Union[str, int] + Integer stack ID or string stack title. + mirror : Optional[int, str], optional + Integer mirror ID or string mirror title, by default None + remote_instance : CatmaidInstance, optional + By default global. + """ + sinfo = get_stack_info(stack, remote_instance) + return cls(sinfo, mirror) + + def _get_mirror_info(self, mirror: Union[int, str, None, MirrorInfo] = None) -> MirrorInfo: + if isinstance(mirror, MirrorInfo): + return mirror + + if mirror is None: + if self.mirror_info is None: + raise ValueError("No default mirror ID set") + return self.mirror_info + + return get_mirror_info(self.stack_info, mirror) + + def set_mirror(self, mirror: Union[int, str]): + """Set the mirror using its int ID or str title.""" + self.mirror_info = self._get_mirror_info(mirror) + + def select_mirror(self): + """Interactively select a mirror from those available. + """ + options = { + m.id: m.title + for m in self.stack_info.mirrors + if m.tile_source_type in source_client_types + } + if not options: + eprint("No mirrors with supported tile source type") + return + + result = select_cli( + f"Select mirror for stack '{self.stack_info.stitle}':", + options, + ) + if result is not None: + self.set_mirror(result) + + def _get_session(self, mirror_id: int, default: Optional[Any]=None): + try: + return self.mirror_session[mirror_id] + except KeyError: + if default is None: + raise + else: + return default + + def get_scale( + self, scale_level: int, mirror_id: Optional[int] = None + ) -> xr.DataArray: + """Get an xarray.DataArray representing the given scale level. + + Note that depending on the metadata available, + missing scale levels may throw different errors. + + Parameters + ---------- + scale_level : int + 0 for full resolution + mirror_id : Optional[int], optional + By default the one set on the class. + + Returns + ------- + xr.DataArray + Can be queried in voxel or world space. + + Raises + ------ + ValueError + Scale level does not exist, according to metadata + NotImplementedError + Unknown tile source type for this mirror + """ + mirror_info = self._get_mirror_info(mirror_id) + if ( + self.stack_info.num_zoom_levels > 0 + and scale_level > self.stack_info.num_zoom_levels + ): + raise ValueError( + f"Scale level {scale_level} does not exist " + f"for stack {self.stack_info.sid} " + f"with {self.stack_info.num_zoom_levels} stack levels" + ) + + if mirror_info.tile_source_type in tile_stores: + store_class = tile_stores[mirror_info.tile_source_type] + session = self._get_session( + mirror_info.id, + requests.Session(), + ) + check_session_type(session, mirror_info.tile_source_type) + store = store_class( + self.stack_info, mirror_info, scale_level, session + ) + return store.to_xarray() + elif mirror_info.tile_source_type == 11: + return self._get_n5(mirror_info, scale_level) + + raise NotImplementedError( + f"Tile source type {mirror_info.tile_source_type} not implemented" + ) + + def _get_n5( + self, + mirror_info: MirrorInfo, + scale_level: int, + ) -> xr.DataArray: + if mirror_info.tile_source_type != 11: + raise ValueError("Mirror info not from an N5 tile source") + formatted = mirror_info.image_base.replace( + "%SCALE_DATASET%", f"s{scale_level}" + ) + *components, transpose_str = formatted.split("/") + transpose = [int(t) for t in transpose_str.split("_")] + + container_comp = [] + arr_comp = [] + this = container_comp + for comp in components: + this.append(comp) + if comp.lower().endswith(".n5"): + this = arr_comp + + if not arr_comp: + raise ValueError("N5 container must have '.n5' suffix") + + kwargs = dict() + session = self._get_session(mirror_info.id, None) + if session is not None: + check_session_type(session, 11) + kwargs["get_client"] = lambda: session + + store = zarr.N5FSStore("/".join(container_comp), **kwargs) + + container = zarr.open(store, "r") + as_zarr = container["/".join(arr_comp)] + # todo: check this transpose + as_dask = da.from_zarr(as_zarr).transpose(transpose) + return xr.DataArray( + as_dask, + coords=self.stack_info.get_coords(scale_level), + dims=self.stack_info.orientation.full_orientation(True), + ) + + +def check_session_type(session, tile_source_type: int): + expected = source_client_types[tile_source_type] + if not isinstance(session, expected): + raise ValueError( + f"Incorrect HTTP client type for tile source {tile_source_type}. " + f"Got {type(session)} but expected one of {expected}." + ) + + +def get_remote_instance_session(remote_instance: Optional[CatmaidInstance] = None): + """Get the ``requests.Session`` from the given ``CatmaidInstance``. + + If ``None`` is given, use the global ``CatmaidInstance``. + """ + cm = utils._eval_remote_instance(remote_instance) + return cm._session diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a6433d5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "extreqs"] +build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index 7e1232d..fa52bee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,16 @@ scipy>=1.3.0 six>=1.11.0 tqdm>=4.50.0 psutil>=5.4.3 + +#extra: extras +ujson~=1.35 + +#extra: stack +zarr +fsspec[http] +xarray[parallel] +imageio + # diskcache>=4.0.0 # Below are optional dependencies diff --git a/setup.py b/setup.py index dabc62a..083fc11 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,9 @@ +import itertools from setuptools import setup, find_packages import re +from pathlib import Path + +from extreqs import parse_requirement_files VERSIONFILE = "pymaid/__init__.py" @@ -11,9 +15,8 @@ else: raise RuntimeError("Unable to find version string in %s." % (VERSIONFILE,)) -with open('requirements.txt') as f: - requirements = f.read().splitlines() - requirements = [l for l in requirements if not l.startswith('#')] +install_requires, extras_require = parse_requirement_files(Path("requirements.txt")) +extras_require["all"] = list(set(itertools.chain.from_iterable(extras_require.values()))) setup( name='python-catmaid', @@ -45,9 +48,8 @@ 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', ], - install_requires=requirements, - extras_require={'extras': ['fuzzywuzzy[speedup]~=0.17.0', - 'ujson~=1.35']}, + install_requires=install_requires, + extras_require=extras_require, python_requires='>=3.9', zip_safe=False )