From 1598a8a70733d130ab95d788c3a5086da741dc2a Mon Sep 17 00:00:00 2001 From: Sergei Korolev Date: Wed, 26 Jul 2023 11:45:47 +0300 Subject: [PATCH] feat(config): warn on unknown config fields --- yascheduler/config/cloud.py | 68 ++++++++++++++++++++++++++++++++++-- yascheduler/config/db.py | 11 ++++-- yascheduler/config/engine.py | 19 ++++++++-- yascheduler/config/local.py | 12 +++++-- yascheduler/config/remote.py | 12 +++++-- yascheduler/config/utils.py | 15 ++++++++ yascheduler/scheduler.py | 1 + yascheduler/utils.py | 3 +- 8 files changed, 128 insertions(+), 13 deletions(-) diff --git a/yascheduler/config/cloud.py b/yascheduler/config/cloud.py index 0a94e95..67f7713 100644 --- a/yascheduler/config/cloud.py +++ b/yascheduler/config/cloud.py @@ -3,12 +3,12 @@ from configparser import SectionProxy from functools import partial -from typing import Optional, Union +from typing import Optional, Sequence, Union -from attrs import define, field, validators +from attrs import define, field, fields, validators from typing_extensions import Self -from .utils import _make_default_field, opt_str_val +from .utils import _make_default_field, opt_str_val, warn_unknown_fields def _check_az_user(_: "ConfigCloudAzure", __, value: str): @@ -66,12 +66,32 @@ class ConfigCloudAzure: jump_username: Optional[str] = field(default=None, validator=opt_str_val) jump_host: Optional[str] = field(default=None, validator=opt_str_val) + @classmethod + def get_valid_config_parser_fields(cls) -> Sequence[str]: + "Returns a list of valid config keys" + exclude_names = ["prefix", "username", "jump_username", "vm_image", "vm_size"] + include_names = ["user", "jump_user", "image", "size"] + return [ + f"{cls.prefix}_{x}" + for x in [f.name for f in fields(cls) if f.name not in exclude_names] + + include_names + ] + @classmethod def from_config_parser_section(cls, sec: SectionProxy) -> "ConfigCloudAzure": "Create config from config parser's section" fmt = partial(_fmt_key, cls.prefix) + warn_unknown_fields( + [ + *cls.get_valid_config_parser_fields(), + *ConfigCloudHetzner.get_valid_config_parser_fields(), + *ConfigCloudUpcloud.get_valid_config_parser_fields(), + ], + sec, + ) + vm_image = sec.get(fmt("image")) image_ref = None if vm_image: @@ -113,10 +133,31 @@ class ConfigCloudHetzner: jump_username: Optional[str] = field(default=None, validator=opt_str_val) jump_host: Optional[str] = field(default=None, validator=opt_str_val) + @classmethod + def get_valid_config_parser_fields(cls) -> Sequence[str]: + "Returns a list of valid config keys" + exclude_names = ["prefix", "username", "jump_username"] + include_names = ["user", "jump_user"] + return [ + f"{cls.prefix}_{x}" + for x in [f.name for f in fields(cls) if f.name not in exclude_names] + + include_names + ] + @classmethod def from_config_parser_section(cls, sec: SectionProxy) -> "ConfigCloudHetzner": "Create config from config parser's section" fmt = partial(_fmt_key, cls.prefix) + + warn_unknown_fields( + [ + *ConfigCloudAzure.get_valid_config_parser_fields(), + *cls.get_valid_config_parser_fields(), + *ConfigCloudUpcloud.get_valid_config_parser_fields(), + ], + sec, + ) + return cls( token=sec.get(fmt("token")), max_nodes=sec.getint(fmt("max_nodes")), @@ -144,10 +185,31 @@ class ConfigCloudUpcloud: jump_username: Optional[str] = field(default=None, validator=opt_str_val) jump_host: Optional[str] = field(default=None, validator=opt_str_val) + @classmethod + def get_valid_config_parser_fields(cls) -> Sequence[str]: + "Returns a list of valid config keys" + exclude_names = ["prefix", "username", "jump_username"] + include_names = ["user", "jump_user"] + return [ + f"{cls.prefix}_{x}" + for x in [f.name for f in fields(cls) if f.name not in exclude_names] + + include_names + ] + @classmethod def from_config_parser_section(cls, sec: SectionProxy) -> "ConfigCloudUpcloud": "Create config from config parser's section" fmt = partial(_fmt_key, cls.prefix) + + warn_unknown_fields( + [ + *ConfigCloudAzure.get_valid_config_parser_fields(), + *ConfigCloudHetzner.get_valid_config_parser_fields(), + *cls.get_valid_config_parser_fields(), + ], + sec, + ) + return cls( login=sec.get(fmt("login")), password=sec.get(fmt("password")), diff --git a/yascheduler/config/db.py b/yascheduler/config/db.py index a6d7fbe..97a809e 100644 --- a/yascheduler/config/db.py +++ b/yascheduler/config/db.py @@ -2,10 +2,11 @@ """Database configuration""" from configparser import SectionProxy +from typing import Sequence -from attrs import define +from attrs import define, fields -from .utils import _make_default_field +from .utils import _make_default_field, warn_unknown_fields @define(frozen=True) @@ -18,9 +19,15 @@ class ConfigDb: host: str = _make_default_field("localhost") port: int = _make_default_field(5432) + @classmethod + def get_valid_config_parser_fields(cls) -> Sequence[str]: + "Returns a list of valid config keys" + return [f.name for f in fields(cls)] + @classmethod def from_config_parser_section(cls, sec: SectionProxy) -> "ConfigDb": "Create config from config parser's section" + warn_unknown_fields(cls.get_valid_config_parser_fields(), sec) return cls( sec.get("user"), sec.get("password"), diff --git a/yascheduler/config/engine.py b/yascheduler/config/engine.py index 159e278..1339846 100644 --- a/yascheduler/config/engine.py +++ b/yascheduler/config/engine.py @@ -5,9 +5,9 @@ from pathlib import PurePath from typing import Optional, Sequence, Tuple, Union -from attrs import Attribute, define, field, validators +from attrs import Attribute, define, field, fields, validators -from .utils import _make_default_field +from .utils import _make_default_field, warn_unknown_fields def _check_spawn(instance: "Engine", _, value: str): @@ -108,12 +108,27 @@ class Engine: check_cmd_code: int = _make_default_field(0) sleep_interval: int = _make_default_field(10) + @classmethod + def get_valid_config_parser_fields(cls) -> Sequence[str]: + "Returns a list of valid config keys" + exclude_names = ["name", "deployable"] + include_names = [ + "deploy_local_files", + "deploy_local_archive", + "deploy_remote_archive", + ] + return [ + f.name for f in fields(cls) if f.name not in exclude_names + ] + include_names + @classmethod def from_config_parser_section( cls, sec: SectionProxy, engines_dir: PurePath ) -> "Engine": "Create config from config parser's section" + warn_unknown_fields(cls.get_valid_config_parser_fields(), sec) + def gettuple(key: str) -> Tuple[str]: return tuple( x.strip() for x in filter(None, sec.get(key, fallback="").split()) diff --git a/yascheduler/config/local.py b/yascheduler/config/local.py index 31803c4..d8ff57d 100644 --- a/yascheduler/config/local.py +++ b/yascheduler/config/local.py @@ -5,9 +5,9 @@ from pathlib import Path, PurePath from typing import Optional, Sequence -from attrs import define, field, validators +from attrs import define, field, fields, validators -from .utils import _make_default_field +from .utils import _make_default_field, warn_unknown_fields @define(frozen=True) @@ -42,9 +42,17 @@ def get_private_keys(self) -> Sequence[PurePath]: filepaths = filter(lambda x: x.is_file(), Path(self.keys_dir).iterdir()) return list(filepaths) + @classmethod + def get_valid_config_parser_fields(cls) -> Sequence[str]: + "Returns a list of valid config keys" + return [f.name for f in fields(cls)] + @classmethod def from_config_parser_section(cls, sec: SectionProxy) -> "ConfigLocal": "Create config from config parser's section" + + warn_unknown_fields(cls.get_valid_config_parser_fields(), sec) + data_dir = Path(sec.get("data_dir", "./data")).resolve() return ConfigLocal( data_dir, diff --git a/yascheduler/config/remote.py b/yascheduler/config/remote.py index 58fe403..a3df5bd 100644 --- a/yascheduler/config/remote.py +++ b/yascheduler/config/remote.py @@ -3,11 +3,11 @@ from configparser import SectionProxy from pathlib import PurePath -from typing import Optional +from typing import Optional, Sequence -from attrs import define, field +from attrs import define, field, fields -from .utils import _make_default_field, opt_str_val +from .utils import _make_default_field, opt_str_val, warn_unknown_fields @define(frozen=True) @@ -21,9 +21,15 @@ class ConfigRemote: jump_username: Optional[str] = field(default=None, validator=opt_str_val) jump_host: Optional[str] = field(default=None, validator=opt_str_val) + @classmethod + def get_valid_config_parser_fields(cls) -> Sequence[str]: + "Returns a list of valid config keys" + return [f.name for f in fields(cls)] + @classmethod def from_config_parser_section(cls, sec: SectionProxy) -> "ConfigRemote": "Create config from config parser's section" + warn_unknown_fields(cls.get_valid_config_parser_fields(), sec) data_dir = PurePath(sec.get("data_dir", "./data")) return cls( data_dir=data_dir, diff --git a/yascheduler/config/utils.py b/yascheduler/config/utils.py index 8748ea1..f86a47c 100644 --- a/yascheduler/config/utils.py +++ b/yascheduler/config/utils.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 """Config helper utilities""" +import warnings +from configparser import SectionProxy from typing import Optional, Sequence from attrs import converters, field, validators @@ -8,9 +10,22 @@ opt_str_val = validators.optional(validators.instance_of(str)) +class ConfigWarning(Warning): + "Warning about config" + + def _make_default_field(default, extra_validators: Optional[Sequence] = None): return field( default=default, converter=converters.default_if_none(default=default), validator=[validators.instance_of(type(default)), *(extra_validators or [])], ) + + +def warn_unknown_fields(known_fields: Sequence[str], sec: SectionProxy) -> None: + unknown_fields = list(set(sec.keys()) - set(known_fields)) + if unknown_fields: + warnings.warn( + f"Config section {sec.name} unknown fields: {', '.join(unknown_fields)}", + ConfigWarning, + ) diff --git a/yascheduler/scheduler.py b/yascheduler/scheduler.py index a5ea843..e2bb333 100755 --- a/yascheduler/scheduler.py +++ b/yascheduler/scheduler.py @@ -46,6 +46,7 @@ def get_logger(log_file, level: int = logging.INFO): + logging.captureWarnings(True) logger = logging.getLogger("yascheduler") logger.setLevel(level) diff --git a/yascheduler/utils.py b/yascheduler/utils.py index ee28f77..6f36113 100644 --- a/yascheduler/utils.py +++ b/yascheduler/utils.py @@ -30,8 +30,9 @@ def submit(): if not script_file.exists(): raise ValueError("Script parameter is not a file name") + logging.captureWarnings(True) log = logging.getLogger() - log.setLevel(logging.ERROR) + log.setLevel(logging.WARN) yac = Yascheduler(logger=log) script_params = {}