Merge pull request #605 from bioimage-io/update_example

Improve pretty validation errors and update example notebook

.github/workflows/build.yml

@@ -66,11 +66,11 @@ jobs:
       - name: Generate developer docs
         run: |
           pdoc \
-          --logo https://bioimage.io/static/img/bioimage-io-logo.svg \
-          --logo-link https://bioimage.io/ \
-          --favicon https://bioimage.io/static/img/bioimage-io-icon-small.svg \
-          --footer-text 'bioimageio.spec ${{steps.get_version.outputs.version}}' \
-          -o ./dist bioimageio.spec
+          --logo "https://bioimage.io/static/img/bioimage-io-logo.svg" \
+          --logo-link "https://bioimage.io/" \
+          --favicon "https://bioimage.io/static/img/bioimage-io-icon-small.svg" \
+          --footer-text "bioimageio.spec ${{steps.get_version.outputs.version}}" \
+          -o ./dist bioimageio.spec bioimageio.spec._internal
       - name: copy legacy file until BioImage.IO-packager is updated  # TODO: remove if packager does not depend on it anymore
         run: cp weight_formats_spec.json ./dist/weight_formats_spec.json
       - name: Get branch name to deploy to

README.md

@@ -105,6 +105,13 @@ Made with [contrib.rocks](https://contrib.rocks).
 
 ### bioimageio.spec Python package
 
+#### bioimageio.spec 0.5.3post1
+
+* bump patch version during loading for model 0.5.x
+* improve validation error formatting
+* validate URLs first with a head request, if forbidden, follow up with a get request that is streamed and if that is also forbidden a regular get request.
+* `RelativePath.absolute()` is now a method (not a property) analog to `pathlib.Path`
+
 #### bioimageio.spec 0.5.3
 
 * remove collection description

bioimageio/spec/VERSION

@@ -1,3 +1,3 @@
 {
-    "version": "0.5.3"
+    "version": "0.5.3post1"
 }

bioimageio/spec/_internal/__init__.py

@@ -1 +1,3 @@
+"""internal helper modules; do not use outside of bioimageio.spec!"""
+
 from ._settings import settings as settings

bioimageio/spec/_internal/common_nodes.py

@@ -54,7 +54,7 @@
 from .io import BioimageioYamlContent
 from .node import Node as Node
 from .url import HttpUrl
-from .utils import assert_all_params_set_explicitly
+from .utils import assert_all_params_set_explicitly, get_format_version_tuple
 from .validation_context import (
     ValidationContext,
     validation_context_var,
@@ -282,34 +282,21 @@ class ResourceDescrBase(
     @model_validator(mode="before")
     @classmethod
     def _ignore_future_patch(cls, data: Union[Dict[Any, Any], Any], /) -> Any:
-        if not isinstance(data, dict) or "format_version" not in data:
+        if (
+            cls.implemented_format_version == "unknown"
+            or not isinstance(data, dict)
+            or "format_version" not in data
+        ):
             return data
 
         value = data["format_version"]
-
-        def get_maj(v: str):
-            parts = v.split(".")
-            if parts and (p := parts[0]).isdecimal():
-                return int(p)
-            else:
-                return 0
-
-        def get_min_patch(v: str):
-            parts = v.split(".")
-            if len(parts) == 3:
-                _, m, p = parts
-                if m.isdecimal() and p.isdecimal():
-                    return int(m), int(p)
-
-            return (0, 0)
+        fv = get_format_version_tuple(value)
+        if fv is None:
+            return data
 
         if (
-            cls.implemented_format_version != "unknown"
-            and value != cls.implemented_format_version
-            and isinstance(value, str)
-            and value.count(".") == 2
-            and get_maj(value) == cls.implemented_format_version_tuple[0]
-            and get_min_patch(value) > cls.implemented_format_version_tuple[1:]
+            fv[0] == cls.implemented_format_version_tuple[0]
+            and fv[1:] > cls.implemented_format_version_tuple[1:]
         ):
             issue_warning(
                 "future format_version '{value}' treated as '{implemented}'",
@@ -364,13 +351,11 @@ def __pydantic_init_subclass__(cls, **kwargs: Any):
             if "." not in cls.implemented_format_version:
                 cls.implemented_format_version_tuple = (0, 0, 0)
             else:
-                cls.implemented_format_version_tuple = cast(
-                    Tuple[int, int, int],
-                    tuple(int(x) for x in cls.implemented_format_version.split(".")),
-                )
-            assert (
-                len(cls.implemented_format_version_tuple) == 3
-            ), cls.implemented_format_version_tuple
+                fv_tuple = get_format_version_tuple(cls.implemented_format_version)
+                assert (
+                    fv_tuple is not None
+                ), f"failed to cast '{cls.implemented_format_version}' to tuple"
+                cls.implemented_format_version_tuple = fv_tuple
 
     @classmethod
     def load(

bioimageio/spec/_internal/io.py

@@ -8,6 +8,7 @@
 from datetime import date as _date
 from datetime import datetime as _datetime
 from functools import lru_cache
+from math import ceil
 from pathlib import Path, PurePath
 from typing import (
     Any,
@@ -42,6 +43,7 @@
     model_validator,
 )
 from pydantic_core import core_schema
+from tqdm import tqdm
 from typing_extensions import (
     Annotated,
     LiteralString,
@@ -87,9 +89,13 @@ class RelativePathBase(RootModel[PurePath], Generic[AbsolutePathT], frozen=True)
     def path(self) -> PurePath:
         return self.root
 
-    @property
-    def absolute(self) -> AbsolutePathT:
-        """the absolute path/url (resolved at time of initialization with the root of the ValidationContext)"""
+    def absolute(  # method not property analog to `pathlib.Path.absolute()`
+        self,
+    ) -> AbsolutePathT:
+        """get the absolute path/url
+
+        (resolved at time of initialization with the root of the ValidationContext)
+        """
         return self._absolute
 
     def model_post_init(self, __context: Any) -> None:
@@ -223,10 +229,10 @@ def get_absolute(
 
 
 FileSource = Annotated[
-    Union[HttpUrl, RelativeFilePath, pydantic.HttpUrl, FilePath],
+    Union[HttpUrl, RelativeFilePath, FilePath],
     Field(union_mode="left_to_right"),
 ]
-PermissiveFileSource = Union[FileSource, str]
+PermissiveFileSource = Union[FileSource, str, pydantic.HttpUrl]
 
 V_suffix = TypeVar("V_suffix", bound=FileSource)
 path_or_url_adapter = TypeAdapter(Union[FilePath, DirectoryPath, HttpUrl])
@@ -353,7 +359,7 @@ def _package(value: FileSource, info: SerializationInfo) -> Union[str, Path, Fil
     # package the file source:
     # add it to the current package's file sources and return its collision free file name
     if isinstance(value, RelativeFilePath):
-        src = value.absolute
+        src = value.absolute()
     elif isinstance(value, pydantic.AnyUrl):
         src = HttpUrl(str(value))
     elif isinstance(value, HttpUrl):
@@ -502,21 +508,18 @@ class HashKwargs(TypedDict):
     sha256: NotRequired[Optional[Sha256]]
 
 
-StrictFileSource = Annotated[
-    Union[HttpUrl, FilePath, RelativeFilePath], Field(union_mode="left_to_right")
-]
-_strict_file_source_adapter = TypeAdapter(StrictFileSource)
+_file_source_adapter = TypeAdapter(FileSource)
 
 
-def interprete_file_source(file_source: PermissiveFileSource) -> StrictFileSource:
+def interprete_file_source(file_source: PermissiveFileSource) -> FileSource:
     if isinstance(file_source, (HttpUrl, Path)):
         return file_source
 
     if isinstance(file_source, pydantic.AnyUrl):
         file_source = str(file_source)
 
     with validation_context_var.get().replace(perform_io_checks=False):
-        strict = _strict_file_source_adapter.validate_python(file_source)
+        strict = _file_source_adapter.validate_python(file_source)
 
     return strict
 
@@ -553,7 +556,7 @@ def download(
 
     strict_source = interprete_file_source(source)
     if isinstance(strict_source, RelativeFilePath):
-        strict_source = strict_source.absolute
+        strict_source = strict_source.absolute()
 
     if isinstance(strict_source, PurePath):
         if not strict_source.exists():
@@ -652,12 +655,17 @@ def extract_file_name(
 def get_sha256(path: Path) -> Sha256:
     """from https://stackoverflow.com/a/44873382"""
     h = hashlib.sha256()
-    b = bytearray(128 * 1024)
+    chunksize = 128 * 1024
+    b = bytearray(chunksize)
     mv = memoryview(b)
+    desc = f"computing SHA256 of {path.name}"
+    pbar = tqdm(desc=desc, total=ceil(path.stat().st_size / chunksize))
     with open(path, "rb", buffering=0) as f:
         for n in iter(lambda: f.readinto(mv), 0):
             h.update(mv[:n])
-
+            _ = pbar.update()
     sha = h.hexdigest()
+
+    pbar.set_description(desc=desc + f" (result: {sha})")
     assert len(sha) == 64
     return Sha256(sha)

bioimageio/spec/_internal/io_utils.py

@@ -1,15 +1,15 @@
 import io
 import warnings
 from contextlib import nullcontext
-from functools import lru_cache
+from dataclasses import dataclass
 from pathlib import Path
+from types import MappingProxyType
 from typing import (
     IO,
     Any,
     Dict,
     List,
     Mapping,
-    NamedTuple,
     Optional,
     TextIO,
     Union,
@@ -28,6 +28,7 @@
 from ._settings import settings
 from .io import (
     BIOIMAGEIO_YAML,
+    SLOTS,
     BioimageioYamlContent,
     FileDescr,
     HashKwargs,
@@ -38,6 +39,7 @@
 )
 from .io_basics import FileName, Sha256
 from .types import FileSource, PermissiveFileSource
+from .utils import cache
 
 yaml = YAML(typ="safe")
 
@@ -113,9 +115,9 @@ def open_bioimageio_yaml(
 
         entry = collection[source]
         logger.info(
-            "{} loading {} {} from {}",
+            "{} loading {}/{} from {}",
             entry.emoji,
-            f"{entry.id}/{entry.version}",
+            entry.id,
             entry.version,
             entry.url,
         )
@@ -136,17 +138,32 @@ def open_bioimageio_yaml(
     return OpenedBioimageioYaml(content, root, downloaded.original_file_name)
 
 
-class _CollectionEntry(NamedTuple):
+@dataclass(frozen=True, **SLOTS)
+class CollectionEntry:
+    """collection entry
+
+    note: The BioImage.IO collection is still under development;
+          this collection entry might change in the future!
+    """
+
     id: str
+    """concept id of the resource; to identify a resource version use <id>/<version>.
+    See `version` below."""
+    version: str
+    """version. To identify a resource version use <id>/<version> for reference"""
     emoji: str
+    """a Unicode emoji string symbolizing this resource"""
     url: str
+    """Resource Description File (RDF) URL"""
     sha256: Optional[Sha256]
-    version: str
+    """SHA256 hash value of RDF"""
     doi: Optional[str]
+    """DOI (regsitered through zenodo.org)
+    as alternative reference of this bioimage.io upload."""
 
 
 def _get_one_collection(url: str):
-    ret: Dict[str, _CollectionEntry] = {}
+    ret: Dict[str, CollectionEntry] = {}
     if not isinstance(url, str) or "/" not in url:
         logger.error("invalid collection url: {}", url)
     try:
@@ -162,9 +179,12 @@ def _get_one_collection(url: str):
         return ret
 
     for raw_entry in collection:
+        assert isinstance(raw_entry, dict), type(raw_entry)
+        v: Any
+        d: Any
         try:
             for i, (v, d) in enumerate(zip(raw_entry["versions"], raw_entry["dois"])):
-                entry = _CollectionEntry(
+                entry = CollectionEntry(
                     id=raw_entry["id"],
                     emoji=raw_entry.get("id_emoji", raw_entry.get("nickname_icon", "")),
                     url=raw_entry["rdf_source"],
@@ -199,20 +219,21 @@ def _get_one_collection(url: str):
     return ret
 
 
-@lru_cache
-def get_collection() -> Mapping[str, _CollectionEntry]:
+@cache
+def get_collection() -> Mapping[str, CollectionEntry]:
     try:
         if settings.resolve_draft:
             ret = _get_one_collection(settings.collection_draft)
         else:
             ret = {}
 
         ret.update(_get_one_collection(settings.collection))
-        return ret
 
     except Exception as e:
         logger.error("failed to get resource id mapping: {}", e)
-        return {}
+        ret = {}
+
+    return MappingProxyType(ret)
 
 
 def unzip(

bioimageio/spec/_internal/root_url.py

@@ -15,6 +15,10 @@ class RootHttpUrl(ValidatedString):
     root_model: ClassVar[Type[RootModel[Any]]] = RootModel[pydantic.HttpUrl]
     _validated: pydantic.HttpUrl
 
+    def absolute(self):
+        """analog to `absolute` method of pathlib."""
+        return self
+
     @property
     def scheme(self) -> str:
         return self._validated.scheme