Merge pull request #121 from knopki/feat-warn-on-config

Print warnings when there are unknown keys in config

yascheduler/config/cloud.py

@@ -3,12 +3,12 @@
 
 from configparser import SectionProxy
 from functools import partial
-from typing import Optional, Union
+from typing import Optional, Sequence, Union
 
-from attrs import define, field, validators
+from attrs import define, field, fields, validators
 from typing_extensions import Self
 
-from .utils import _make_default_field, opt_str_val
+from .utils import _make_default_field, opt_str_val, warn_unknown_fields
 
 
 def _check_az_user(_: "ConfigCloudAzure", __, value: str):
@@ -66,12 +66,32 @@ class ConfigCloudAzure:
     jump_username: Optional[str] = field(default=None, validator=opt_str_val)
     jump_host: Optional[str] = field(default=None, validator=opt_str_val)
 
+    @classmethod
+    def get_valid_config_parser_fields(cls) -> Sequence[str]:
+        "Returns a list of valid config keys"
+        exclude_names = ["prefix", "username", "jump_username", "vm_image", "vm_size"]
+        include_names = ["user", "jump_user", "image", "size"]
+        return [
+            f"{cls.prefix}_{x}"
+            for x in [f.name for f in fields(cls) if f.name not in exclude_names]
+            + include_names
+        ]
+
     @classmethod
     def from_config_parser_section(cls, sec: SectionProxy) -> "ConfigCloudAzure":
         "Create config from config parser's section"
 
         fmt = partial(_fmt_key, cls.prefix)
 
+        warn_unknown_fields(
+            [
+                *cls.get_valid_config_parser_fields(),
+                *ConfigCloudHetzner.get_valid_config_parser_fields(),
+                *ConfigCloudUpcloud.get_valid_config_parser_fields(),
+            ],
+            sec,
+        )
+
         vm_image = sec.get(fmt("image"))
         image_ref = None
         if vm_image:
@@ -113,10 +133,31 @@ class ConfigCloudHetzner:
     jump_username: Optional[str] = field(default=None, validator=opt_str_val)
     jump_host: Optional[str] = field(default=None, validator=opt_str_val)
 
+    @classmethod
+    def get_valid_config_parser_fields(cls) -> Sequence[str]:
+        "Returns a list of valid config keys"
+        exclude_names = ["prefix", "username", "jump_username"]
+        include_names = ["user", "jump_user"]
+        return [
+            f"{cls.prefix}_{x}"
+            for x in [f.name for f in fields(cls) if f.name not in exclude_names]
+            + include_names
+        ]
+
     @classmethod
     def from_config_parser_section(cls, sec: SectionProxy) -> "ConfigCloudHetzner":
         "Create config from config parser's section"
         fmt = partial(_fmt_key, cls.prefix)
+
+        warn_unknown_fields(
+            [
+                *ConfigCloudAzure.get_valid_config_parser_fields(),
+                *cls.get_valid_config_parser_fields(),
+                *ConfigCloudUpcloud.get_valid_config_parser_fields(),
+            ],
+            sec,
+        )
+
         return cls(
             token=sec.get(fmt("token")),
             max_nodes=sec.getint(fmt("max_nodes")),
@@ -144,10 +185,31 @@ class ConfigCloudUpcloud:
     jump_username: Optional[str] = field(default=None, validator=opt_str_val)
     jump_host: Optional[str] = field(default=None, validator=opt_str_val)
 
+    @classmethod
+    def get_valid_config_parser_fields(cls) -> Sequence[str]:
+        "Returns a list of valid config keys"
+        exclude_names = ["prefix", "username", "jump_username"]
+        include_names = ["user", "jump_user"]
+        return [
+            f"{cls.prefix}_{x}"
+            for x in [f.name for f in fields(cls) if f.name not in exclude_names]
+            + include_names
+        ]
+
     @classmethod
     def from_config_parser_section(cls, sec: SectionProxy) -> "ConfigCloudUpcloud":
         "Create config from config parser's section"
         fmt = partial(_fmt_key, cls.prefix)
+
+        warn_unknown_fields(
+            [
+                *ConfigCloudAzure.get_valid_config_parser_fields(),
+                *ConfigCloudHetzner.get_valid_config_parser_fields(),
+                *cls.get_valid_config_parser_fields(),
+            ],
+            sec,
+        )
+
         return cls(
             login=sec.get(fmt("login")),
             password=sec.get(fmt("password")),

yascheduler/config/db.py

@@ -2,10 +2,11 @@
 """Database configuration"""
 
 from configparser import SectionProxy
+from typing import Sequence
 
-from attrs import define
+from attrs import define, fields
 
-from .utils import _make_default_field
+from .utils import _make_default_field, warn_unknown_fields
 
 
 @define(frozen=True)
@@ -18,9 +19,15 @@ class ConfigDb:
     host: str = _make_default_field("localhost")
     port: int = _make_default_field(5432)
 
+    @classmethod
+    def get_valid_config_parser_fields(cls) -> Sequence[str]:
+        "Returns a list of valid config keys"
+        return [f.name for f in fields(cls)]
+
     @classmethod
     def from_config_parser_section(cls, sec: SectionProxy) -> "ConfigDb":
         "Create config from config parser's section"
+        warn_unknown_fields(cls.get_valid_config_parser_fields(), sec)
         return cls(
             sec.get("user"),
             sec.get("password"),

yascheduler/config/engine.py

@@ -5,9 +5,9 @@
 from pathlib import PurePath
 from typing import Optional, Sequence, Tuple, Union
 
-from attrs import Attribute, define, field, validators
+from attrs import Attribute, define, field, fields, validators
 
-from .utils import _make_default_field
+from .utils import _make_default_field, warn_unknown_fields
 
 
 def _check_spawn(instance: "Engine", _, value: str):
@@ -108,12 +108,27 @@ class Engine:
     check_cmd_code: int = _make_default_field(0)
     sleep_interval: int = _make_default_field(10)
 
+    @classmethod
+    def get_valid_config_parser_fields(cls) -> Sequence[str]:
+        "Returns a list of valid config keys"
+        exclude_names = ["name", "deployable"]
+        include_names = [
+            "deploy_local_files",
+            "deploy_local_archive",
+            "deploy_remote_archive",
+        ]
+        return [
+            f.name for f in fields(cls) if f.name not in exclude_names
+        ] + include_names
+
     @classmethod
     def from_config_parser_section(
         cls, sec: SectionProxy, engines_dir: PurePath
     ) -> "Engine":
         "Create config from config parser's section"
 
+        warn_unknown_fields(cls.get_valid_config_parser_fields(), sec)
+
         def gettuple(key: str) -> Tuple[str]:
             return tuple(
                 x.strip() for x in filter(None, sec.get(key, fallback="").split())

yascheduler/config/local.py

@@ -5,9 +5,9 @@
 from pathlib import Path, PurePath
 from typing import Optional, Sequence
 
-from attrs import define, field, validators
+from attrs import define, field, fields, validators
 
-from .utils import _make_default_field
+from .utils import _make_default_field, warn_unknown_fields
 
 
 @define(frozen=True)
@@ -42,9 +42,17 @@ def get_private_keys(self) -> Sequence[PurePath]:
         filepaths = filter(lambda x: x.is_file(), Path(self.keys_dir).iterdir())
         return list(filepaths)
 
+    @classmethod
+    def get_valid_config_parser_fields(cls) -> Sequence[str]:
+        "Returns a list of valid config keys"
+        return [f.name for f in fields(cls)]
+
     @classmethod
     def from_config_parser_section(cls, sec: SectionProxy) -> "ConfigLocal":
         "Create config from config parser's section"
+
+        warn_unknown_fields(cls.get_valid_config_parser_fields(), sec)
+
         data_dir = Path(sec.get("data_dir", "./data")).resolve()
         return ConfigLocal(
             data_dir,

yascheduler/config/remote.py

@@ -3,11 +3,11 @@
 
 from configparser import SectionProxy
 from pathlib import PurePath
-from typing import Optional
+from typing import Optional, Sequence
 
-from attrs import define, field
+from attrs import define, field, fields
 
-from .utils import _make_default_field, opt_str_val
+from .utils import _make_default_field, opt_str_val, warn_unknown_fields
 
 
 @define(frozen=True)
@@ -21,9 +21,15 @@ class ConfigRemote:
     jump_username: Optional[str] = field(default=None, validator=opt_str_val)
     jump_host: Optional[str] = field(default=None, validator=opt_str_val)
 
+    @classmethod
+    def get_valid_config_parser_fields(cls) -> Sequence[str]:
+        "Returns a list of valid config keys"
+        return [f.name for f in fields(cls)]
+
     @classmethod
     def from_config_parser_section(cls, sec: SectionProxy) -> "ConfigRemote":
         "Create config from config parser's section"
+        warn_unknown_fields(cls.get_valid_config_parser_fields(), sec)
         data_dir = PurePath(sec.get("data_dir", "./data"))
         return cls(
             data_dir=data_dir,

yascheduler/config/utils.py

@@ -1,16 +1,31 @@
 #!/usr/bin/env python3
 """Config helper utilities"""
 
+import warnings
+from configparser import SectionProxy
 from typing import Optional, Sequence
 
 from attrs import converters, field, validators
 
 opt_str_val = validators.optional(validators.instance_of(str))
 
 
+class ConfigWarning(Warning):
+    "Warning about config"
+
+
 def _make_default_field(default, extra_validators: Optional[Sequence] = None):
     return field(
         default=default,
         converter=converters.default_if_none(default=default),
         validator=[validators.instance_of(type(default)), *(extra_validators or [])],
     )
+
+
+def warn_unknown_fields(known_fields: Sequence[str], sec: SectionProxy) -> None:
+    unknown_fields = list(set(sec.keys()) - set(known_fields))
+    if unknown_fields:
+        warnings.warn(
+            f"Config section {sec.name} unknown fields: {', '.join(unknown_fields)}",
+            ConfigWarning,
+        )

yascheduler/scheduler.py

@@ -46,6 +46,7 @@
 
 
 def get_logger(log_file, level: int = logging.INFO):
+    logging.captureWarnings(True)
     logger = logging.getLogger("yascheduler")
     logger.setLevel(level)
 

yascheduler/utils.py

@@ -30,8 +30,9 @@ def submit():
     if not script_file.exists():
         raise ValueError("Script parameter is not a file name")
 
+    logging.captureWarnings(True)
     log = logging.getLogger()
-    log.setLevel(logging.ERROR)
+    log.setLevel(logging.WARN)
     yac = Yascheduler(logger=log)
 
     script_params = {}