From c0dcda92842a11c976411e48ea4bc73465225f7f Mon Sep 17 00:00:00 2001 From: Roman <34752952+roman2git@users.noreply.github.com> Date: Fri, 22 Nov 2024 16:28:12 +0100 Subject: [PATCH] fix(#16): use Pydantic V2 models (#37) * using pydantic v2, tested, WIP * adjusted model config for V2, added models for V1 with wrapper to keep compability with V2 * Collecting coverage with tox to include both pydantic v1 and pydantic v2 * small fixes * fixes based on PR review, + ruff formatting --------- Co-authored-by: Charles <43383361+develop-cs@users.noreply.github.com> --- .pre-commit-config.yaml | 9 +- pyproject.toml | 18 ++++ src/arta/_engine.py | 16 +-- src/arta/models.py | 163 ++++++++++++++++++++++--------- tests/unit/test_engine_errors.py | 5 +- tox.ini | 9 +- 6 files changed, 156 insertions(+), 64 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d12ede3..0931412 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,7 +64,14 @@ repos: - id: cov-check name: Coverage language: system - entry: pytest -v --cov=arta --cov-fail-under=90 + entry: coverage report -m --fail-under=90 + types: [python] + pass_filenames: false + always_run: true + - id: cov-erase + name: Coverage - Erase + language: system + entry: coverage erase types: [python] pass_filenames: false always_run: true diff --git a/pyproject.toml b/pyproject.toml index cc6b949..ccfa73d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,3 +93,21 @@ exclude = ["tests/*"] [tool.ruff.format] docstring-code-format = true + +[tool.coverage.paths] +source = [ + "src/arta", + "*/.tox/*/lib/python*/site-packages/arta", + "*/.tox/pypy*/site-packages/arta", + "*/.tox\\*\\Lib\\site-packages\\arta", + "*/src/arta", + "*\\src\\arta"] + +[tool.coverage.run] +branch = true +parallel = false +source = ["arta"] +omit = ["*/tests/*"] + +[tool.coverage.report] +omit = ["tests"] diff --git a/src/arta/_engine.py b/src/arta/_engine.py index 21e8012..462e7a9 100644 --- a/src/arta/_engine.py +++ b/src/arta/_engine.py @@ -87,7 +87,8 @@ def __init__( # Initialize directly with a rules dict if rules_dict is not None: # Data validation - RulesDict.parse_obj(rules_dict) + # RulesDict.parse_obj(rules_dict) + RulesDict.model_validate(rules_dict) # Edge cases data validation if not isinstance(rules_dict, dict): @@ -104,9 +105,8 @@ def __init__( # Load config in attribute config_dict = load_config(config_path) - if config_dict is not None: - # Data validation - config: Configuration = Configuration(**config_dict) + # Data validation + config: Configuration = Configuration.model_validate(config_dict) if config.parsing_error_strategy is not None: # Set parsing error handling strategy from config @@ -125,7 +125,7 @@ def __init__( # Dictionary of condition instances (k: condition id, v: instance), built from config data if len(std_condition_functions) > 0: std_condition_instances = self._build_std_conditions( - config=config.dict(), condition_functions_dict=std_condition_functions + config=config.model_dump(), condition_functions_dict=std_condition_functions ) # User-defined/custom conditions @@ -150,7 +150,7 @@ def __init__( self.rules = self._build_rules( std_condition_instances=std_condition_instances, action_functions=action_functions, - config=config.dict(), + config=config.model_dump(), factory_mapping_classes=factory_mapping_classes, ) @@ -288,9 +288,9 @@ def _build_rules( Return a dictionary of Rule instances built from the configuration. Args: - rule_sets: Sets of rules to be loaded in the Rules Engine (as needed by further uses). + # rule_sets: Sets of rules to be loaded in the Rules Engine (as needed by further uses). std_condition_instances: Dictionary of condition instances (k: condition id, v: StandardCondition instance) - actions_dict: Dictionary of action functions (k: action name, v: Callable) + action_functions: Dictionary of action functions (k: action name, v: Callable) config: Dictionary of the imported configuration from yaml files. factory_mapping_classes: A mapping dictionary (k: condition conf. key, v: custom class object) diff --git a/src/arta/models.py b/src/arta/models.py index d3044a9..0a258a3 100644 --- a/src/arta/models.py +++ b/src/arta/models.py @@ -3,74 +3,143 @@ Note: Having no "from __future__ import annotations" here is wanted (pydantic compatibility). """ -from typing import Any, Callable, Dict, List, Optional +from typing import Annotated, Any, Callable, Dict, List, Optional -try: - from pydantic import v1 as pydantic -except ImportError: - import pydantic # type: ignore +import pydantic +from pydantic.version import VERSION from arta.utils import ParsingErrorStrategy +if not VERSION.startswith("1."): + # ---------------------------------- + # For instantiation using rules_dict + class RuleRaw(pydantic.BaseModel): + """Pydantic model for validating a rule.""" -# ---------------------------------- -# For instantiation using rules_dict -class RuleRaw(pydantic.BaseModel): - """Pydantic model for validating a rule.""" + condition: Optional[Callable] + condition_parameters: Optional[Dict[str, Any]] + action: Callable + action_parameters: Optional[Dict[str, Any]] - condition: Optional[Callable] - condition_parameters: Optional[Dict[str, Any]] - action: Callable - action_parameters: Optional[Dict[str, Any]] + model_config = pydantic.ConfigDict(extra="forbid") - class Config: - extra = "forbid" + class RulesGroup(pydantic.RootModel): # noqa + """Pydantic model for validating a rules group.""" + root: Dict[str, RuleRaw] -class RulesGroup(pydantic.BaseModel): - """Pydantic model for validating a rules group.""" + class RulesDict(pydantic.RootModel): # noqa + """Pydantic model for validating rules dict instanciation.""" - __root__: Dict[str, RuleRaw] + root: Dict[str, RulesGroup] + # ---------------------------------- + # For instantiation using config_path + class Condition(pydantic.BaseModel): + """Pydantic model for validating a condition.""" -class RulesDict(pydantic.BaseModel): - """Pydantic model for validating rules dict instanciation.""" + description: str + validation_function: str + condition_parameters: Optional[Dict[str, Any]] = None - __root__: Dict[str, RulesGroup] + class RulesConfig(pydantic.BaseModel): + """Pydantic model for validating a rule group from config file.""" + condition: Optional[str] = None + simple_condition: Optional[str] = None + action: Annotated[str, pydantic.StringConstraints(to_lower=True)] # type: ignore + action_parameters: Optional[Any] = None -# ---------------------------------- -# For instantiation using config_path -class Condition(pydantic.BaseModel): - """Pydantic model for validating a condition.""" + model_config = pydantic.ConfigDict(extra="allow") - description: str - validation_function: str - condition_parameters: Optional[Dict[str, Any]] + class Configuration(pydantic.BaseModel): + """Pydantic model for validating configuration files.""" + conditions: Optional[Dict[str, Condition]] = None + conditions_source_modules: Optional[List[str]] = None + actions_source_modules: List[str] + custom_classes_source_modules: Optional[List[str]] = None + condition_factory_mapping: Optional[Dict[str, str]] = None + rules: Dict[str, Dict[str, Dict[str, RulesConfig]]] + parsing_error_strategy: Optional[ParsingErrorStrategy] = None -class RulesConfig(pydantic.BaseModel): - """Pydantic model for validating a rule group from config file.""" + model_config = pydantic.ConfigDict(extra="ignore") - condition: Optional[str] - simple_condition: Optional[str] - action: pydantic.constr(to_lower=True) # type: ignore - action_parameters: Optional[Any] + @pydantic.field_validator("rules", mode="before") # noqa + def upper_key(cls, vl): # noqa + """Validate and uppercase keys for RulesConfig""" + for k, v in vl.items(): + for kk, vv in v.items(): + for key, rules in [*vv.items()]: + if key != str(key).upper(): + del vl[k][kk][key] + vl[k][kk][str(key).upper()] = rules + return vl - class Config: - extra = "allow" +else: + class BaseModelV2(pydantic.BaseModel): + """Wrapper to expose missed methods used elsewhere in the code""" -class Configuration(pydantic.BaseModel): - """Pydantic model for validating configuration files.""" + model_dump: Callable = pydantic.BaseModel.dict # noqa - conditions: Optional[Dict[str, Condition]] - conditions_source_modules: Optional[List[str]] - actions_source_modules: List[str] - custom_classes_source_modules: Optional[List[str]] - condition_factory_mapping: Optional[Dict[str, str]] - rules: Dict[str, Dict[str, Dict[pydantic.constr(to_upper=True), RulesConfig]]] # type: ignore - parsing_error_strategy: Optional[ParsingErrorStrategy] + @classmethod + def model_validate(cls, obj): # noqa + return cls.parse_obj(obj) # noqa - class Config: - extra = "ignore" + # ---------------------------------- + # For instantiation using rules_dict + class RuleRaw(BaseModelV2): # type: ignore[no-redef] + """Pydantic model for validating a rule.""" + + condition: Optional[Callable] + condition_parameters: Optional[Dict[str, Any]] + action: Callable + action_parameters: Optional[Dict[str, Any]] + + class Config: + extra = "forbid" + + class RulesGroup(pydantic.BaseModel): # type: ignore[no-redef] # noqa + """Pydantic model for validating a rules group.""" + + __root__: Dict[str, RuleRaw] # noqa + + class RulesDict(BaseModelV2): # type: ignore[no-redef] # noqa + """Pydantic model for validating rules dict instanciation.""" + + __root__: Dict[str, RulesGroup] # noqa + + # ---------------------------------- + # For instantiation using config_path + class Condition(BaseModelV2): # type: ignore[no-redef] + """Pydantic model for validating a condition.""" + + description: str + validation_function: str + condition_parameters: Optional[Dict[str, Any]] + + class RulesConfig(BaseModelV2): # type: ignore[no-redef] + """Pydantic model for validating a rule group from config file.""" + + condition: Optional[str] + simple_condition: Optional[str] + action: pydantic.constr(to_lower=True) # type: ignore + action_parameters: Optional[Any] + + class Config: + extra = "allow" + + class Configuration(BaseModelV2): # type: ignore[no-redef] + """Pydantic model for validating configuration files.""" + + conditions: Optional[Dict[str, Condition]] + conditions_source_modules: Optional[List[str]] + actions_source_modules: List[str] + custom_classes_source_modules: Optional[List[str]] + condition_factory_mapping: Optional[Dict[str, str]] + rules: Dict[str, Dict[str, Dict[pydantic.constr(to_upper=True), RulesConfig]]] # type: ignore + parsing_error_strategy: Optional[ParsingErrorStrategy] + + class Config: + extra = "ignore" diff --git a/tests/unit/test_engine_errors.py b/tests/unit/test_engine_errors.py index d8ce10f..9893cd4 100644 --- a/tests/unit/test_engine_errors.py +++ b/tests/unit/test_engine_errors.py @@ -6,10 +6,7 @@ from arta import RulesEngine from arta.exceptions import ConditionExecutionError, RuleExecutionError -try: - from pydantic import v1 as pydantic -except ImportError: - import pydantic # type: ignore +import pydantic @pytest.mark.parametrize( diff --git a/tox.ini b/tox.ini index 5f3ee84..e34568e 100644 --- a/tox.ini +++ b/tox.ini @@ -6,17 +6,18 @@ env_list = py310 py39 pydantic1-{py39,py310,py311,py312} - [testenv] description = run unit tests deps = pytest + pytest-cov pydantic>=2.0.0 -commands = pytest tests +commands = pytest tests --cov --cov-append -[testenv:pydantic1] +[testenv:pydantic1-{py39,py310,py311,py312}] description = check backward compatibility with pydantic < 2.0.0 deps = pytest + pytest-cov pydantic<2.0.0 -commands = pytest tests +commands = pytest tests --cov --cov-append