Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add functional API for algorithm contributtions #876

Open
wants to merge 33 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
dd0bee5
initial implementation
Czaki Jan 1, 2023
423288f
add checks for proper methods and types
Czaki Jan 2, 2023
0799fef
add name detection base of function name
Czaki Jan 2, 2023
350a363
Iprove validation of function parameter
Czaki Jan 2, 2023
526d8f1
add __argument_class__ creation
Czaki Jan 3, 2023
42f7739
add calling function directly.
Czaki Jan 3, 2023
47b4d4f
add sample usage of functional api
Czaki Jan 3, 2023
2f8c470
fix tests
Czaki Jan 3, 2023
a858a74
improve tests
Czaki Jan 3, 2023
3f0f9dc
simplify code
Czaki Jan 3, 2023
2525ce4
try upload coverage
Czaki Jan 3, 2023
94bfd79
fix coverage
Czaki Jan 3, 2023
02e7763
a little cleanup
Czaki Jan 4, 2023
fb941dd
add test for decorator
Czaki Jan 4, 2023
077f13a
add test for function without params
Czaki Jan 4, 2023
77adef1
Merge branch 'develop' into feature/functional_plugin_api
Czaki Jan 4, 2023
bebab4c
try to fix warnings
Czaki Jan 4, 2023
2beb83c
check if warning is resolved
Czaki Jan 4, 2023
c0a3770
fix remaining names
Czaki Jan 4, 2023
e7480a5
use better names
Czaki Jan 4, 2023
413ead0
Merge branch 'develop' into feature/functional_plugin_api
Czaki Jan 11, 2023
ef72b97
allow kwargs function argument
Czaki Jan 5, 2023
05e2a86
improve type annotation
Czaki Jan 5, 2023
7749300
install libhdf
Czaki Jan 11, 2023
ac7e352
fix type name
Czaki Jan 11, 2023
2d75267
Merge branch 'develop' into feature/functional_plugin_api
Czaki Feb 1, 2023
be6bef0
add `kwargs_to_model` decorator for simply add backward compatybility
Czaki Feb 7, 2023
fb1a0db
Merge remote-tracking branch 'origin-public/develop' into feature/fun…
Czaki Feb 10, 2023
a97ceef
Merge remote-tracking branch 'origin-public/develop' into feature/fun…
Czaki May 18, 2023
d555168
Merge branch 'develop' into feature/functional_plugin_api
Czaki Dec 4, 2023
ad9946a
Merge branch 'develop' into feature/functional_plugin_api
Czaki Dec 4, 2023
a104055
Merge branch 'develop' into feature/functional_plugin_api
Czaki Jan 26, 2024
28b697a
fix tests
Czaki Jan 26, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
207 changes: 206 additions & 1 deletion package/PartSegCore/algorithm_describe_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@
from PartSegCore.utils import BaseModel
from PartSegImage import Channel

T = typing.TypeVar("T", bound="AlgorithmDescribeBase")

TypeT = typing.Type[T]


class AlgorithmDescribeNotFound(Exception):
"""
Expand Down Expand Up @@ -116,7 +120,7 @@ def _partial_abstractmethod(funcobj):


class AlgorithmDescribeBaseMeta(ABCMeta):
def __new__(cls, name, bases, attrs, **kwargs):
def __new__(cls, name, bases, attrs, method_from_fun=None, additional_parameters=None, **kwargs):
cls2 = super().__new__(cls, name, bases, attrs, **kwargs)
if (
not inspect.isabstract(cls2)
Expand All @@ -125,8 +129,181 @@ def __new__(cls, name, bases, attrs, **kwargs):
):
raise RuntimeError("class need to have __argument_class__ set or get_fields functions defined")
cls2.__new_style__ = getattr(cls2.get_fields, "__is_partial_abstractmethod__", False)
cls2.__from_function__ = getattr(cls2, "__from_function__", False)
cls2.__abstract_getters__ = {}
cls2.__method_name__ = method_from_fun or getattr(cls2, "__method_name__", None)
cls2.__additional_parameters_name__ = additional_parameters or getattr(
cls2, "__additional_parameters_name__", None
)
if cls2.__additional_parameters_name__ is None:
cls2.__additional_parameters_name__ = cls._get_calculation_method_params_name(cls2)

cls2.__support_from_function__ = (
cls2.__method_name__ is not None and cls2.__additional_parameters_name__ is not None
)

cls2.__abstract_getters__, cls2.__support_from_function__ = cls._get_abstract_getters(
cls2, cls2.__support_from_function__, method_from_fun
)
return cls2

@staticmethod
def _get_abstract_getters(
cls2, support_from_function, calculation_method
) -> typing.Tuple[typing.Dict[str, typing.Any], bool]:
abstract_getters = {}
if hasattr(cls2, "__abstractmethods__") and cls2.__abstractmethods__:
# get all abstract methods that starts with `get_`
for method_name in cls2.__abstractmethods__:
if method_name.startswith("get_"):
method = getattr(cls2, method_name)
if "return" not in method.__annotations__:
msg = f"Method {method_name} of {cls2.__qualname__} need to have return type defined"
try:
file_name = inspect.getsourcefile(method)
line = inspect.getsourcelines(method)[1]
msg += f" in {file_name}:{line}"
except TypeError:
pass
raise RuntimeError(msg)

abstract_getters[method_name[4:]] = getattr(cls2, method_name).__annotations__["return"]
elif method_name != calculation_method:
support_from_function = False
return abstract_getters, support_from_function
Comment on lines +150 to +173
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The _get_abstract_getters method has been modified to support the new functionality related to creating classes from functions. This method now checks for abstract methods starting with get_ and validates their return type annotations. While this approach ensures that abstract getters are properly defined, it may be too restrictive by excluding potentially valid abstract methods that do not follow this naming convention. Additionally, the error message generated when a return type is not defined could be enhanced to provide more specific guidance on how to correct the issue.

Consider allowing more flexibility in the naming convention of abstract methods and improve the error message for missing return type annotations to offer clearer guidance on resolving the issue.


@staticmethod
def _get_calculation_method_params_name(cls2) -> typing.Optional[str]:
if cls2.__method_name__ is None:
return None
signature = inspect.signature(getattr(cls2, cls2.__method_name__))
if "arguments" in signature.parameters:
return "arguments"
if "params" in signature.parameters:
return "params"
if "parameters" in signature.parameters:
return "parameters"
raise RuntimeError(f"Cannot determine arguments parameter name in {cls2.__method_name__}")
Comment on lines +175 to +186
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The _get_calculation_method_params_name method attempts to determine the name of the parameter that represents arguments in the calculation method. This method assumes specific parameter names (arguments, params, parameters) and raises a runtime error if none are found. This approach may not accommodate all valid parameter naming conventions used by developers, potentially limiting the flexibility of the API.

Enhance the method to support a wider range of parameter naming conventions or provide a mechanism for developers to specify the parameter name explicitly, improving the API's flexibility.


@staticmethod
def _validate_if_all_abstract_getters_are_defined(abstract_getters, kwargs):
abstract_getters_set = set(abstract_getters)
kwargs_set = set(kwargs.keys())

if abstract_getters_set != kwargs_set:
# Provide a nice error message with information about what is missing and is obsolete
missing_text = ", ".join(sorted(abstract_getters_set - kwargs_set))
if missing_text:
missing_text = f"Not all abstract methods are provided, missing: {missing_text}."
else:
missing_text = ""
extra_text = ", ".join(sorted(kwargs_set - abstract_getters_set))
if extra_text:
extra_text = f"There are extra attributes in call: {extra_text}."
else:
extra_text = ""

raise ValueError(f"{missing_text} {extra_text}")
Comment on lines +188 to +206
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The _validate_if_all_abstract_getters_are_defined method checks if all required abstract getters are provided in kwargs. While this validation is crucial for ensuring that all necessary information is provided, the error message generated when there are missing or extra attributes could be more actionable. Specifically, it could guide developers on how to correct the issue by providing examples or suggesting where to define the missing getters.

Improve the error message generated by _validate_if_all_abstract_getters_are_defined to include actionable guidance on how to resolve issues with missing or extra attributes.


@staticmethod
def _validate_function_parameters(func, method, method_name) -> set:
"""
Validate if all parameters without default values are defined in self.__calculation_method__

:param func: function to validate
:return: set of parameters that should be dropped
"""
signature = inspect.signature(func)
base_method_signature = inspect.signature(method)
take_all = False

for parameter in signature.parameters.values():
if parameter.kind in {inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.POSITIONAL_ONLY}:
raise ValueError(f"Function {func} should not have positional only parameters")
if (
parameter.default is inspect.Parameter.empty
and parameter.name not in base_method_signature.parameters
and parameter.kind != inspect.Parameter.VAR_KEYWORD
):
raise ValueError(f"Parameter {parameter.name} is not defined in {method_name} method")

if parameter.kind == inspect.Parameter.VAR_KEYWORD:
take_all = True

if take_all:
return set()

return {
parameters.name
for parameters in base_method_signature.parameters.values()
Comment on lines +208 to +238
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The _validate_function_parameters method validates if all parameters without default values are defined in the calculation method. This validation is essential for ensuring compatibility between the function and the class it is being converted into. However, the method currently raises a generic ValueError for missing parameters, which may not provide enough context for developers to understand the source of the error and how to fix it.

Enhance the error handling in _validate_function_parameters to provide more specific and actionable error messages, helping developers identify and correct issues with missing parameters more efficiently.

if parameters.name not in signature.parameters
}

@staticmethod
def _get_argument_class_from_signature(func, argument_name: str):
signature = inspect.signature(func)
if argument_name not in signature.parameters:
return BaseModel
return signature.parameters[argument_name].annotation

@staticmethod
def _get_parameters_from_signature(func):
signature = inspect.signature(func)
return [parameters.name for parameters in signature.parameters.values()]

def from_function(self, func=None, **kwargs):
"""generate new class from function"""

# Test if all abstract methods values are provided in kwargs

if not self.__support_from_function__:
raise RuntimeError("This class does not support from_function method")

self._validate_if_all_abstract_getters_are_defined(self.__abstract_getters__, kwargs)

# check if all values have correct type
for key, value in kwargs.items():
if not isinstance(value, self.__abstract_getters__[key]):
raise TypeError(f"Value for {key} should be {self.__abstract_getters__[key]}")

def _getter_by_name(name):
def _func():
return kwargs[name]

return _func

parameters_order = self._get_parameters_from_signature(getattr(self, self.__method_name__))

def _class_generator(func_):
drop_attr = self._validate_function_parameters(
func_, getattr(self, self.__method_name__), self.__method_name__
)

@wraps(func_)
def _calculate_method(*args, **kwargs_):
for attr, name in zip(args, parameters_order):
if name in kwargs_:
raise ValueError(f"Parameter {name} is defined twice")
kwargs_[name] = attr

for name in drop_attr:
kwargs_.pop(name, None)
return func_(**kwargs_)

class_dkt = {f"get_{name}": _getter_by_name(name) for name in self.__abstract_getters__}

class_dkt[self.__method_name__] = _calculate_method
class_dkt["__argument_class__"] = self._get_argument_class_from_signature(
func_, self.__additional_parameters_name__
)
class_dkt["__from_function__"] = True

return type(func_.__name__.replace("_", " ").title().replace(" ", ""), (self,), class_dkt)

if func is None:
return _class_generator
return _class_generator(func)
Comment on lines +254 to +305
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The from_function method in the AlgorithmDescribeBaseMeta class is a significant addition, enabling the generation of new classes from functions. This method includes comprehensive validation and error handling to ensure that the function and provided arguments meet the necessary criteria. However, the method's complexity and the multiple nested functions could impact readability and maintainability. Additionally, the method assumes that the function's name follows a specific naming convention (func_.__name__.replace("_", " ").title().replace(" ", "")), which may not always be desirable or applicable.

Consider refactoring the from_function method to simplify its structure and improve readability. Additionally, provide a way for developers to specify the class name explicitly when it cannot be appropriately inferred from the function's name.



class AlgorithmDescribeBase(ABC, metaclass=AlgorithmDescribeBaseMeta):
"""
Comment on lines 129 to 309
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📝 NOTE
This review was outside the diff hunks and was mapped to the diff hunk with the greatest overlap. Original lines [123-147]

The modifications to the AlgorithmDescribeBaseMeta metaclass introduce new parameters method_from_fun and additional_parameters to the __new__ method. These changes are designed to support the creation of classes from functions, enhancing the flexibility of algorithm definitions. However, there's a potential issue with the logic that determines whether a class supports the from_function method based on the presence of __method_name__ and __additional_parameters_name__. This logic may not accurately reflect all scenarios where from_function could be applicable or beneficial. Additionally, the error handling in the absence of __argument_class__ or get_fields functions could be more descriptive, providing guidance on how to resolve the issue.

Consider refining the logic that determines the support for from_function to cover more scenarios and improve the error message when __argument_class__ or get_fields functions are missing to guide developers on how to make their classes compatible.

Expand All @@ -138,6 +315,11 @@ class AlgorithmDescribeBase(ABC, metaclass=AlgorithmDescribeBaseMeta):
__argument_class__: typing.Optional[typing.Type[PydanticBaseModel]] = None
__new_style__: bool

def __new__(cls, *args, **kwargs):
if cls.__from_function__:
return getattr(cls, cls.__method_name__)(*args, **kwargs)
return super().__new__(cls)
Comment on lines +318 to +321
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The __new__ method in the AlgorithmDescribeBase class has been modified to support instances created from functions. This modification allows for dynamic behavior based on whether the class was generated from a function. While this approach adds flexibility, it also introduces complexity into the instantiation process, potentially affecting the predictability and debuggability of classes derived from AlgorithmDescribeBase.

Evaluate the impact of the modified __new__ method on the predictability and debuggability of classes derived from AlgorithmDescribeBase. Consider providing additional documentation or examples to help developers understand and effectively use this new functionality.


@classmethod
def get_doc_from_fields(cls):
resp = "{\n"
Expand All @@ -150,6 +332,29 @@ def get_doc_from_fields(cls):
resp += "}\n"
return resp

@classmethod
@typing.overload
def from_function(cls: TypeT, func: typing.Callable[..., typing.Any], **kwargs) -> TypeT:
...

@classmethod
@typing.overload
def from_function(cls: TypeT, **kwargs) -> typing.Callable[[typing.Callable[..., typing.Any]], TypeT]:
...

@classmethod
def from_function(
cls: TypeT, func=None, **kwargs
) -> typing.Union[TypeT, typing.Callable[[typing.Callable], TypeT]]:
def _from_function(func_) -> typing.Type["AlgorithmDescribeBase"]:
if "name" not in kwargs:
kwargs["name"] = func_.__name__.replace("_", " ").title()
return AlgorithmDescribeBaseMeta.from_function(cls, func_, **kwargs)

if func is None:
return _from_function
return _from_function(func)
Comment on lines +345 to +356
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The from_function class method in the AlgorithmDescribeBase class provides an overload mechanism to support different usage patterns (with or without a specified function). This method leverages the AlgorithmDescribeBaseMeta.from_function to generate new classes from functions, demonstrating a clear separation of concerns between the metaclass and the class itself. However, the documentation could be improved to better explain the method's overloads and provide examples of how to use each pattern effectively.

Enhance the documentation for the from_function class method in AlgorithmDescribeBase to include examples and explanations of the different usage patterns supported by the method's overloads.


@classmethod
@abstractmethod
def get_name(cls) -> str:
Expand Down
11 changes: 6 additions & 5 deletions package/PartSegCore/io_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ class SaveBase(AlgorithmDescribeBase, ABC):

@classmethod
@abstractmethod
def get_short_name(cls):
def get_short_name(cls) -> str:
raise NotImplementedError

@classmethod
Expand Down Expand Up @@ -102,10 +102,12 @@ def get_default_extension(cls):

@classmethod
def need_segmentation(cls):
"""If method requires segmentation (ROI) to work, or could work with image only"""
return True

@classmethod
def need_mask(cls):
"""If `mask` is required for perform save"""
return False

@classmethod
Expand All @@ -132,7 +134,7 @@ class LoadBase(AlgorithmDescribeBase, ABC):

@classmethod
@abstractmethod
def get_short_name(cls):
def get_short_name(cls) -> str:
raise NotImplementedError

@classmethod
Expand Down Expand Up @@ -161,8 +163,7 @@ def get_name_with_suffix(cls):

@classmethod
def get_extensions(cls) -> typing.List[str]:
match = re.match(r".*\((.*)\)", cls.get_name())
if match is None:
if (match := re.match(r".*\((.*)\)", cls.get_name())) is None:
raise ValueError(f"No extensions found in {cls.get_name()}")
extensions = match[1].split(" ")
if not all(x.startswith("*.") for x in extensions):
Expand Down Expand Up @@ -205,7 +206,7 @@ def load_metadata_base(data: typing.Union[str, Path]):
try:
decoded_data = json.loads(str(data), object_hook=partseg_object_hook)
except Exception: # pragma: no cover
raise e # noqa: B904
raise e from None
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Modifying exception handling in load_metadata_base to use raise e from None suppresses the context of the original exception, which might hinder debugging. Consider if this change aligns with the desired error handling behavior.

-            raise e from None
+            raise e

Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation.

Suggested change
raise e from None
raise e


return decoded_data

Expand Down
2 changes: 1 addition & 1 deletion package/PartSegCore/segmentation/algorithm_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ def calculation_run(self, report_fun: Callable[[str, int], None]) -> ROIExtracti
raise NotImplementedError

@abstractmethod
def get_info_text(self):
def get_info_text(self) -> str:
raise NotImplementedError

def get_channel(self, channel_idx):
Expand Down
2 changes: 1 addition & 1 deletion package/PartSegCore/segmentation/border_smoothing.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from PartSegCore.utils import BaseModel


class BaseSmoothing(AlgorithmDescribeBase, ABC):
class BaseSmoothing(AlgorithmDescribeBase, ABC, method_from_fun="smooth"):
__argument_class__ = BaseModel

@classmethod
Expand Down
5 changes: 3 additions & 2 deletions package/PartSegCore/segmentation/noise_filtering.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import typing
import warnings
from abc import ABC
from abc import ABC, abstractmethod
from enum import Enum

import numpy as np
Expand All @@ -22,10 +22,11 @@ def __str__(self):
return self.name.replace("_", " ")


class NoiseFilteringBase(AlgorithmDescribeBase, ABC):
class NoiseFilteringBase(AlgorithmDescribeBase, ABC, method_from_fun="noise_filter"):
Czaki marked this conversation as resolved.
Show resolved Hide resolved
"""Base class for noise filtering operations"""

@classmethod
@abstractmethod
def noise_filter(cls, channel: np.ndarray, spacing: typing.Iterable[float], arguments: dict) -> np.ndarray:
"""
This function need be overloaded in implementation
Expand Down
Loading
Loading