Merge pull request #681 from bioimage-io/improve_docs

Improve documentation

.github/workflows/build.yml

@@ -54,6 +54,11 @@ jobs:
         env:
           BIOIMAGEIO_CACHE_PATH: bioimageio_cache
           RUN_EXPENSIVE_TESTS: ${{ matrix.run-expensive-tests && 'true' || 'false' }}
+      - run: pytest scripts  # also test docstrings in scripts for dev-version
+        if: ${{matrix.is-dev-version}}
+        env:
+          BIOIMAGEIO_CACHE_PATH: bioimageio_cache
+          RUN_EXPENSIVE_TESTS: ${{ matrix.run-expensive-tests && 'true' || 'false' }}
       - uses: actions/cache/save@v4
         # explicit restore/save instead of cache action to cache even if coverage fails
         with:

bioimageio/spec/_internal/__init__.py

@@ -1,5 +1,3 @@
 """internal helper modules; do not use outside of bioimageio.spec!"""
 
-from ._settings import settings
-
-__all__ = ["settings"]
+from ._settings import settings as settings

bioimageio/spec/_internal/common_nodes.py

@@ -2,7 +2,7 @@
 
 import collections.abc
 import traceback
-from abc import ABC, abstractmethod
+from abc import ABC
 from copy import deepcopy
 from io import BytesIO
 from pathlib import Path
@@ -12,16 +12,13 @@
     Any,
     ClassVar,
     Dict,
-    Final,
     FrozenSet,
-    Generic,
     List,
     Optional,
     Protocol,
     Tuple,
     Type,
     Union,
-    cast,
     get_type_hints,
 )
 from zipfile import ZipFile
@@ -37,14 +34,7 @@
     model_validator,
 )
 from pydantic_core import PydanticUndefined, core_schema
-from typing_extensions import (
-    Annotated,
-    LiteralString,
-    Self,
-    TypeVar,
-    TypeVarTuple,
-    Unpack,
-)
+from typing_extensions import Annotated, LiteralString, Self
 
 from ..summary import (
     WARNING_LEVEL_TO_NAME,
@@ -63,7 +53,6 @@
 from .root_url import RootHttpUrl
 from .url import HttpUrl
 from .utils import (
-    assert_all_params_set_explicitly,
     get_format_version_tuple,
 )
 from .validation_context import (
@@ -154,88 +143,6 @@ def _serialize(self) -> str:
         return self.data
 
 
-SRC = TypeVar("SRC", bound=Union[Node, StringNode])
-TGT = TypeVar("TGT", bound=Node)
-
-
-# converter without any additional args or kwargs:
-# class Converter(Generic[SRC, TGT], ABC):
-#     # src: ClassVar[Type[SRC]]
-#     # tgt: ClassVar[Type[TGT]]
-#     # note: the above is not yet possible, see https://github.com/python/typing/discussions/1424
-#     # we therefore use an instance
-#     def __init__(self, src: Type[SRC], tgt: Type[TGT], /):
-#         super().__init__()
-#         self.src: Final[Type[SRC]] = src
-#         self.tgt: Final[Type[TGT]] = tgt
-
-#     @abstractmethod
-#     def _convert(self, src: SRC, tgt: "type[TGT | dict[str, Any]] ", /) -> "TGT | dict[str, Any]":
-#         ...
-
-#     def convert(self, source: SRC, /) -> TGT:
-#         """convert `source` node
-
-#         Args:
-#             source: A bioimageio description node
-
-#         Raises:
-#             ValidationError: conversion failed
-#         """
-#         data = self.convert_as_dict(source)
-#         return assert_all_params_set_explicitly(self.tgt)(**data)
-
-#     def convert_as_dict(self, source: SRC) -> Dict[str, Any]:
-#         return cast(Dict[str, Any], self._convert(source, dict))
-
-
-# A TypeVar bound to a TypedDict seemed like a good way to add converter kwargs:
-# ```
-# class ConverterKwargs(TypedDict):
-#     pass
-# KW = TypeVar("KW", bound=ConverterKwargs, default=ConverterKwargs)
-# ```
-# sadly we cannot use a TypeVar bound to TypedDict and then unpack it in the Converter methods,
-# see https://github.com/python/typing/issues/1399
-# Therefore we use a TypeVarTuple and positional only args instead
-# (We are avoiding ParamSpec for its ambiguity 'args vs kwargs')
-CArgs = TypeVarTuple("CArgs")
-
-
-class Converter(Generic[SRC, TGT, Unpack[CArgs]], ABC):
-    # src: ClassVar[Type[SRC]]
-    # tgt: ClassVar[Type[TGT]]
-    # note: the above is not yet possible, see https://github.com/python/typing/discussions/1424
-    # we therefore use an instance
-    def __init__(self, src: Type[SRC], tgt: Type[TGT], /):
-        super().__init__()
-        self.src: Final[Type[SRC]] = src
-        self.tgt: Final[Type[TGT]] = tgt
-
-    @abstractmethod
-    def _convert(
-        self, src: SRC, tgt: "type[TGT | dict[str, Any]]", /, *args: Unpack[CArgs]
-    ) -> "TGT | dict[str, Any]": ...
-
-    # note: the following is not (yet) allowed, see https://github.com/python/typing/issues/1399
-    #       we therefore use `kwargs` (and not `**kwargs`)
-    # def convert(self, source: SRC, /, **kwargs: Unpack[KW]) -> TGT:
-    def convert(self, source: SRC, /, *args: Unpack[CArgs]) -> TGT:
-        """convert `source` node
-
-        Args:
-            source: A bioimageio description node
-
-        Raises:
-            ValidationError: conversion failed
-        """
-        data = self.convert_as_dict(source, *args)
-        return assert_all_params_set_explicitly(self.tgt)(**data)
-
-    def convert_as_dict(self, source: SRC, /, *args: Unpack[CArgs]) -> Dict[str, Any]:
-        return cast(Dict[str, Any], self._convert(source, dict, *args))
-
-
 class NodeWithExplicitlySetFields(Node):
     fields_to_set_explicitly: ClassVar[FrozenSet[LiteralString]] = frozenset()
     """set set these fields explicitly with their default value if they are not set,
@@ -350,6 +257,7 @@ def validation_summary(self) -> ValidationSummary:
 
     @property
     def root(self) -> Union[RootHttpUrl, DirectoryPath, ZipPath]:
+        """The URL/Path prefix to resolve any relative paths with."""
         return self._root
 
     @classmethod
@@ -379,6 +287,7 @@ def __pydantic_init_subclass__(cls, **kwargs: Any):
     def load(
         cls, data: BioimageioYamlContent, context: Optional[ValidationContext] = None
     ) -> Union[Self, InvalidDescr]:
+        """factory method to create a resource description object"""
         context = context or validation_context_var.get()
         assert isinstance(data, dict)
         with context.replace(log_warnings=False):  # don't log warnings to console

bioimageio/spec/_internal/node_converter.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import (
+    Any,
+    Dict,
+    Final,
+    Generic,
+    Type,
+    Union,
+    cast,
+)
+
+from typing_extensions import (
+    TypeVar,
+    TypeVarTuple,
+    Unpack,
+)
+
+from .common_nodes import StringNode
+from .node import Node
+from .utils import (
+    assert_all_params_set_explicitly,
+)
+
+SRC = TypeVar("SRC", bound=Union[Node, StringNode])
+TGT = TypeVar("TGT", bound=Node)
+
+
+# converter without any additional args or kwargs:
+# class Converter(Generic[SRC, TGT], ABC):
+#     # src: ClassVar[Type[SRC]]
+#     # tgt: ClassVar[Type[TGT]]
+#     # note: the above is not yet possible, see https://github.com/python/typing/discussions/1424
+#     # we therefore use an instance
+#     def __init__(self, src: Type[SRC], tgt: Type[TGT], /):
+#         super().__init__()
+#         self.src: Final[Type[SRC]] = src
+#         self.tgt: Final[Type[TGT]] = tgt
+
+#     @abstractmethod
+#     def _convert(self, src: SRC, tgt: "type[TGT | dict[str, Any]] ", /) -> "TGT | dict[str, Any]":
+#         ...
+
+#     def convert(self, source: SRC, /) -> TGT:
+#         """convert `source` node
+
+#         Args:
+#             source: A bioimageio description node
+
+#         Raises:
+#             ValidationError: conversion failed
+#         """
+#         data = self.convert_as_dict(source)
+#         return assert_all_params_set_explicitly(self.tgt)(**data)
+
+#     def convert_as_dict(self, source: SRC) -> Dict[str, Any]:
+#         return cast(Dict[str, Any], self._convert(source, dict))
+
+
+# A TypeVar bound to a TypedDict seemed like a good way to add converter kwargs:
+# ```
+# class ConverterKwargs(TypedDict):
+#     pass
+# KW = TypeVar("KW", bound=ConverterKwargs, default=ConverterKwargs)
+# ```
+# sadly we cannot use a TypeVar bound to TypedDict and then unpack it in the Converter methods,
+# see https://github.com/python/typing/issues/1399
+# Therefore we use a TypeVarTuple and positional only args instead
+# (We are avoiding ParamSpec for its ambiguity 'args vs kwargs')
+CArgs = TypeVarTuple("CArgs")
+
+
+class Converter(Generic[SRC, TGT, Unpack[CArgs]], ABC):
+    # src: ClassVar[Type[SRC]]
+    # tgt: ClassVar[Type[TGT]]
+    # note: the above is not yet possible, see https://github.com/python/typing/discussions/1424
+    # we therefore use an instance
+    def __init__(self, src: Type[SRC], tgt: Type[TGT], /):
+        super().__init__()
+        self.src: Final[Type[SRC]] = src
+        self.tgt: Final[Type[TGT]] = tgt
+
+    @abstractmethod
+    def _convert(
+        self, src: SRC, tgt: "type[TGT | dict[str, Any]]", /, *args: Unpack[CArgs]
+    ) -> "TGT | dict[str, Any]": ...
+
+    # note: the following is not (yet) allowed, see https://github.com/python/typing/issues/1399
+    #       we therefore use `kwargs` (and not `**kwargs`)
+    # def convert(self, source: SRC, /, **kwargs: Unpack[KW]) -> TGT:
+    def convert(self, source: SRC, /, *args: Unpack[CArgs]) -> TGT:
+        """convert `source` node
+
+        Args:
+            source: A bioimageio description node
+
+        Raises:
+            ValidationError: conversion failed
+        """
+        data = self.convert_as_dict(source, *args)
+        return assert_all_params_set_explicitly(self.tgt)(**data)
+
+    def convert_as_dict(self, source: SRC, /, *args: Unpack[CArgs]) -> Dict[str, Any]:
+        return cast(Dict[str, Any], self._convert(source, dict, *args))

bioimageio/spec/_internal/types.py

@@ -106,6 +106,8 @@ class Datetime(
 
 
 class Doi(ValidatedString):
+    """A digital object identifier, see https://www.doi.org/"""
+
     root_model: ClassVar[Type[RootModel[Any]]] = RootModel[
         Annotated[str, StringConstraints(pattern=DOI_REGEX)]
     ]
@@ -131,7 +133,7 @@ class LowerCaseIdentifier(ValidatedString):
 
 
 class OrcidId(ValidatedString):
-    """an ORCID identifier"""
+    """An ORCID identifier, see https://orcid.org/"""
 
     root_model: ClassVar[Type[RootModel[Any]]] = RootModel[
         Annotated[str, AfterValidator(_validate_orcid_id)]
@@ -146,7 +148,7 @@ def _normalize_multiplication(si_unit: Union[Any, str]) -> Union[Any, str]:
 
 
 class SiUnit(ValidatedString):
-    """Si unit"""
+    """An SI unit"""
 
     root_model: ClassVar[Type[RootModel[Any]]] = RootModel[
         Annotated[

bioimageio/spec/generic/v0_3.py

@@ -20,11 +20,7 @@
 from pydantic import Field, RootModel, ValidationInfo, field_validator, model_validator
 from typing_extensions import Annotated
 
-from .._internal.common_nodes import (
-    Converter,
-    Node,
-    ResourceDescrBase,
-)
+from .._internal.common_nodes import Node, ResourceDescrBase
 from .._internal.constants import TAG_CATEGORIES
 from .._internal.field_validation import validate_gh_user
 from .._internal.field_warning import as_warning, warn
@@ -38,6 +34,7 @@
 )
 from .._internal.io_basics import AbsoluteFilePath, Sha256
 from .._internal.license_id import DeprecatedLicenseId, LicenseId
+from .._internal.node_converter import Converter
 from .._internal.types import ImportantFileSource, NotEmpty, RelativeFilePath
 from .._internal.url import HttpUrl
 from .._internal.validated_string import ValidatedString

bioimageio/spec/model/v0_4.py

@@ -29,7 +29,7 @@
     field_validator,
     model_validator,
 )
-from typing_extensions import Annotated, LiteralString, Self, assert_never
+from typing_extensions import Annotated, LiteralString, Self, assert_never, get_args
 
 from .._internal.common_nodes import (
     KwargsNode,
@@ -211,14 +211,7 @@ def check_one_entry(self) -> Self:
 
     def __getitem__(
         self,
-        key: Literal[
-            "keras_hdf5",
-            "onnx",
-            "pytorch_state_dict",
-            "tensorflow_js",
-            "tensorflow_saved_model_bundle",
-            "torchscript",
-        ],
+        key: WeightsFormat,
     ):
         if key == "keras_hdf5":
             ret = self.keras_hdf5
@@ -240,6 +233,37 @@ def __getitem__(
 
         return ret
 
+    @property
+    def available_formats(self):
+        return {
+            **({} if self.keras_hdf5 is None else {"keras_hdf5": self.keras_hdf5}),
+            **({} if self.onnx is None else {"onnx": self.onnx}),
+            **(
+                {}
+                if self.pytorch_state_dict is None
+                else {"pytorch_state_dict": self.pytorch_state_dict}
+            ),
+            **(
+                {}
+                if self.tensorflow_js is None
+                else {"tensorflow_js": self.tensorflow_js}
+            ),
+            **(
+                {}
+                if self.tensorflow_saved_model_bundle is None
+                else {
+                    "tensorflow_saved_model_bundle": self.tensorflow_saved_model_bundle
+                }
+            ),
+            **({} if self.torchscript is None else {"torchscript": self.torchscript}),
+        }
+
+    @property
+    def missing_formats(self):
+        return {
+            wf for wf in get_args(WeightsFormat) if wf not in self.available_formats
+        }
+
 
 class WeightsEntryDescrBase(FileDescr):
     type: ClassVar[WeightsFormat]