Merge pull request #37 from riotkit-org/issue_26

Temporary merge

docs/source/check-configuration-reference.rst

@@ -6,6 +6,7 @@ Check configuration reference
     {
         "type": "http",
         "description": "IWA-AIT check",
+        "results_cache_time": 300,
         "input": {
             "url": "http://iwa-ait.org",
             "expect_keyword": "iwa",
@@ -18,7 +19,10 @@ Check configuration reference
             "on_each_down": [
                 "echo \"Site under maintenance\" > /var/www/maintenance.html"
             ]
-        }
+        },
+        "quiet_periods": [
+            {"starts": "30 00 * * *", "duration": 60}
+        ]
     }
 
 
@@ -43,6 +47,12 @@ Optional text field, there can be left a note for other administrators to exchan
 of a failure.
 
 
+results_cache_time
+******************
+
+How long the check result should be kept in cache (in seconds)
+
+
 input
 *****
 
@@ -54,8 +64,14 @@ to UPPERCASE and passed as environment variables.
 hooks
 *****
 
-Execute shell commands on given events.
+(Optional) Execute shell commands on given events.
 
 - on_each_up: Everytime the check is OK
 - on_each_down: Everytime the check is FAILING
 
+
+quiet_periods
+*************
+
+(Optional) Defines time, when the check results should be ignored. For example setting "30 00 * * *" and 60m duration will
+result in ignoring check failure at 00:30 everyday for 60 minutes - till 01:30

infracheck/infracheck/bin.py

@@ -56,11 +56,6 @@ def main():
         help='Alternative project directory',
         default=''
     )
-    parser.add_argument(
-        '--server-port', '-p',
-        help='Server port, default is 7422',
-        default=7422
-    )
     parser.add_argument(
         '--db-path', '-b',
         help='Database path',
@@ -86,6 +81,11 @@ def main():
         help='Do not run the server, just run all the checks from CLI',
         action='store_true'
     )
+    parser.add_argument(
+        '--server-port', '-p',
+        help='Server port, default is 7422',
+        default=7422
+    )
     parser.add_argument(
         '--server-path-prefix',
         help='Optional path prefix to the routing, eg. /this-is-a-secret will make urls looking like: '

infracheck/infracheck/config.py

@@ -1,18 +1,28 @@
 
+"""
+Config
+======
+
+Configuration locator and loader, loads JSON objects from disk and returns ConfigurationCheck models.
+Searches through hierarchy of paths defined in controller.Controller._combine_project_dirs()
+"""
+
 import os
 import json
-
+from dataclasses import dataclass
+from typing import List
+from rkd.api.inputoutput import IO
 from .exceptions import ConfigurationException
+from .model import ConfiguredCheck
 from .rkd_support import is_rkd_check, rkd_module_exists
 
 
-class ConfigLoader:
-    paths = []
-
-    def __init__(self, paths):
-        self.paths = paths
+@dataclass
+class ConfigLoader(object):
+    paths: List[str]
+    io: IO
 
-    def load(self, config_name: str) -> dict:
+    def load(self, config_name: str) -> ConfiguredCheck:
         file_path = self._find_file_path('/configured/', config_name, '.json')
 
         if not file_path:
@@ -27,7 +37,7 @@ def load(self, config_name: str) -> dict:
         self._assert_valid_format(config_name, data)
         self._assert_has_valid_type(str(data['type']))
 
-        return data
+        return ConfiguredCheck.from_config(config_name, data, self.io)
 
     @staticmethod
     def _assert_valid_format(config_name: str, data):

infracheck/infracheck/controller.py

@@ -27,7 +27,7 @@ def __init__(self, project_dir: str, server_port: int, server_path_prefix: str,
         self.io = IO()
         self.io.set_log_level(log_level)
         self.project_dirs = self._combine_project_dirs(project_dir)
-        self.config_loader = ConfigLoader(self.project_dirs)
+        self.config_loader = ConfigLoader(self.project_dirs, self.io)
         self.repository = Repository(self.project_dirs, db_path)
 
         self.runner = Runner(dirs=self.project_dirs, config_loader=self.config_loader,

infracheck/infracheck/exceptions.py

@@ -45,7 +45,18 @@ def from_binary_not_found(cls, check_name: str, paths: list) -> 'ConfigurationEx
         )
 
     @classmethod
-    def from_rkd_check_url_error(cls, check_name):
+    def from_rkd_check_url_error(cls, check_name) -> 'ConfigurationException':
         return cls(
             'RiotKit-Do check syntax "{}" is invalid. Valid example: rkd://rkd.standardlib.shell:sh'.format(check_name)
         )
+
+    @classmethod
+    def from_quiet_periods_should_be_a_list_error(cls) -> 'ConfigurationException':
+        return cls('"quiet_periods" should be a list')
+
+    @classmethod
+    def from_quiet_periods_invalid_structure(cls) -> 'ConfigurationException':
+        return cls(
+            '"quiet_periods" contains invalid structure. Valid entry example: '
+            '{"starts": "30 00 * * *", "duration": 60}'
+        )

infracheck/infracheck/http.py

@@ -8,6 +8,7 @@
 
 import tornado.ioloop
 import tornado.web
+import sys
 import json
 from .controller import Controller
 from .model import ExecutedChecksResultList
@@ -63,4 +64,8 @@ def run(self):
         ])
 
         srv.listen(self.port)
-        tornado.ioloop.IOLoop.current().start()
+
+        try:
+            tornado.ioloop.IOLoop.current().start()
+        except KeyboardInterrupt:
+            sys.exit(0)

infracheck/infracheck/model.py

@@ -1,5 +1,108 @@
-from datetime import datetime
-from typing import Dict
+from datetime import datetime, timedelta
+from typing import Dict, List, Union, Optional
+from dataclasses import dataclass
+from croniter import croniter
+from rkd.api.inputoutput import IO
+
+from infracheck.infracheck.exceptions import ConfigurationException
+
+QUIET_PERIODS_DATA_STRUCT = List[Dict[str, Union[str, int]]]
+HOOKS_STRUCT = Dict[str, List[str]]
+INPUT_VARIABLES_STRUCT = Dict[str, Union[str, int]]
+
+
+@dataclass
+class ConfiguredCheck(object):
+    name: str
+    check_type: str  # type
+    description: str
+    input_variables: INPUT_VARIABLES_STRUCT
+    hooks: HOOKS_STRUCT
+    quiet_periods: QUIET_PERIODS_DATA_STRUCT
+    results_cache_time: Optional[int]
+    io: IO
+
+    @classmethod
+    def from_config(cls, name: str, config: dict, io: IO) -> 'ConfiguredCheck':
+        quiet_periods = config.get('quiet_periods', [])
+
+        if not isinstance(quiet_periods, list):
+            raise ConfigurationException.from_quiet_periods_should_be_a_list_error()
+
+        if quiet_periods:
+            for period in quiet_periods:
+                period: dict
+                if "starts" not in period or "duration" not in period:
+                    raise ConfigurationException.from_quiet_periods_invalid_structure()
+
+        # cache life time is disabled
+        if "results_cache_time" not in config or not config.get('results_cache_time'):
+            io.debug('results_cache_time not configured for {}'.format(name))
+
+        return cls(
+            name=name,
+            check_type=config.get('type'),
+            description=config.get('description', ''),
+            input_variables=config.get('input', {}),
+            hooks=config.get('hooks', {}),
+            quiet_periods=quiet_periods,
+            results_cache_time=int(config.get('results_cache_time')) if "results_cache_time" in config else None,
+            io=io
+        )
+
+    def should_status_be_ignored(self) -> bool:
+        """
+        Decides if health check failure status could be ignored in this time
+        :return:
+        """
+
+        if not self.quiet_periods:
+            self.io.debug('Quiet period not enabled')
+            return False
+
+        for period in self.quiet_periods:
+            period: dict
+            if "starts" not in period or "duration" not in period:
+                continue
+
+            schedule = croniter(period.get('starts'), start_time=self._time_now())
+            last_execution = self._strip_date(schedule.get_prev(ret_type=datetime))
+            next_execution = self._strip_date(schedule.get_next(ret_type=datetime))
+            duration = timedelta(minutes=int(period.get('duration')))
+            current_time = self._strip_date(self._time_now())
+
+            self.io.debug(f'Quiet period: last_execution={last_execution}, duration={duration}, now={current_time}')
+
+            # STARTED just now
+            if next_execution <= current_time:
+                return True
+
+            # ALREADY happening
+            if last_execution + duration >= current_time:
+                self.io.debug('Quiet period started')
+                return True
+
+        return False
+
+    @staticmethod
+    def _time_now() -> datetime:
+        return datetime.now()
+
+    @staticmethod
+    def _strip_date(date) -> datetime:
+        return date.replace(second=0, microsecond=0, tzinfo=None)
+
+    def should_check_run(self, last_cache_write_time: Optional[datetime]) -> bool:
+        if not self.results_cache_time:
+            return True
+
+        cache_lifetime_seconds = timedelta(seconds=self.results_cache_time)
+
+        if not last_cache_write_time:
+            self.io.debug('No last cache write time for {}'.format(self.name))
+            return True
+
+        return last_cache_write_time + cache_lifetime_seconds <= datetime.now()
 
 
 class ExecutedCheckResult(object):
@@ -13,14 +116,18 @@ class ExecutedCheckResult(object):
     configured_name: str
     refresh_time: datetime
     description: str
+    is_silenced: bool
+
+    def __init__(self, configured_name: str, output: str, exit_status: bool, hooks_output: str,
+                 description: str, is_silenced: bool = False):
 
-    def __init__(self, configured_name: str, output: str, exit_status: bool, hooks_output: str, description: str):
         self.configured_name = configured_name
         self.output = output
         self.exit_status = exit_status
         self.hooks_output = hooks_output
         self.refresh_time = datetime.now()
         self.description = description
+        self.is_silenced = is_silenced
 
     @classmethod
     def from_not_ready(cls, configured_name: str, description: str):
@@ -29,23 +136,29 @@ def from_not_ready(cls, configured_name: str, description: str):
             output='Check not ready',
             exit_status=False,
             hooks_output='',
-            description=description
+            description=description,
+            is_silenced=False
         )
 
         check.refresh_time = None
 
         return check
 
     def to_hash(self) -> dict:
+        exit_status = self.exit_status if not self.is_silenced else True
+
         return {
-            'status': self.exit_status,
+            'status': exit_status,
             'output': self.output,
             'description': self.description,
             'hooks_output': self.hooks_output,
-            'ident': self.configured_name + '=' + str(self.exit_status),
-            'checked_at': self.refresh_time.strftime('%Y-%m-%d %H-%M-%S') if self.refresh_time else ''
+            'ident': f'{self.configured_name}={exit_status}, silenced={self.is_silenced}',
+            'checked_at': self.refresh_time.strftime('%Y-%m-%d %H-%M-%S') if self.refresh_time else '',
         }
 
+    def enable_quiet_time_now(self) -> None:
+        self.is_silenced = True
+
 
 class ExecutedChecksResultList(object):
     checks: Dict[str, ExecutedCheckResult]

infracheck/infracheck/repository.py

@@ -118,7 +118,7 @@ def push_to_cache(self, check_name: str, data: ExecutedCheckResult):
 
         self.db.commit()
 
-    def _execute(self, query: str, parameters = None):
+    def _execute(self, query: str, parameters=None):
         if parameters is None:
             parameters = []