Improve startup sequence

clearml · Aug 24, 2020 · b93591e · b93591e
1 parent 0abfd8d
commit b93591e
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 20 deletions.
diff --git a/server/elastic/initialize.py b/server/elastic/initialize.py
@@ -49,16 +49,17 @@ def __init__(
         self.last_blocked = None
 
     def filter(self, record):
-        allow = (
-            (self.err_type is None or record.exc_info[0] != self.err_type)
-            and (self.level is None or record.levelno != self.level)
-            and (self.args is None or record.args[: len(self.args)] != self.args)
-        )
-
-        if not allow:
-            self.last_blocked = record
-
-        return int(allow)
+        try:
+            allow = (
+                (self.err_type is None or record.exc_info[0] != self.err_type)
+                and (self.level is None or record.levelno != self.level)
+                and (self.args is None or record.args[: len(self.args)] != self.args)
+            )
+            if not allow:
+                self.last_blocked = record
+            return allow
+        except Exception:
+            return True
 
 
 def check_elastic_empty() -> bool:

diff --git a/server/server.py b/server/server.py
@@ -57,21 +57,26 @@
         info.es_connection_error = True
 
     empty_db = check_mongo_empty()
-    if upgrade_monitoring:
-        if not empty_db and (info.es_connection_error or empty_es):
-            if get_last_server_version() < Version("0.16.0"):
-                log.info(f"ES database seems not migrated")
-                info.missed_es_upgrade = True
-        proceed_with_init = not (info.es_connection_error or info.missed_es_upgrade)
-    else:
-        proceed_with_init = True
+    if (
+        upgrade_monitoring
+        and not empty_db
+        and (info.es_connection_error or empty_es)
+        and get_last_server_version() < Version("0.16.0")
+    ):
+        log.info(f"ES database seems not migrated")
+        info.missed_es_upgrade = True
+
+    if info.es_connection_error and not info.missed_es_upgrade:
+        raise Exception(
+            "Error starting server: failed connecting to ElasticSearch service"
+        )
 
-    if proceed_with_init:
+    if not info.missed_es_upgrade:
         init_es_data()
         init_mongo_data()
 
 if (
-    proceed_with_init
+    not info.missed_es_upgrade
     and empty_db
     and config.get("apiserver.pre_populate.enabled", False)
 ):