change log to independent process (#798)

* change log to independent process * change LogServer to Singleton * recover daemon process * version
Meituan-Dianping · Nov 2, 2023 · c37d86d · c37d86d
1 parent 927ba0e
commit c37d86d
Show file tree

Hide file tree

Showing 8 changed files with 101 additions and 36 deletions.
diff --git a/lyrebird/application.py b/lyrebird/application.py
@@ -52,9 +52,9 @@ def make_fail_response(msg, **kwargs):
 encoders_decoders = None
 
 
-def start_server_without_mock():
+def start_server_without_mock_and_log():
     for name in server:
-        if name == 'mock':
+        if name == 'mock' or name == 'log':
             continue
         server[name].start()
 
@@ -63,6 +63,10 @@ def start_mock_server():
     server['mock'].start()
 
 
+def start_log_server():
+    server['log'].start()
+
+
 def start_server():
     for name in server:
         server[name].start()

diff --git a/lyrebird/base_server.py b/lyrebird/base_server.py
@@ -6,6 +6,7 @@
 from multiprocessing import Process, Queue
 from lyrebird import application
 
+
 service_msg_queue = Queue()
 
 
@@ -18,9 +19,9 @@ def __init__(self):
         self.args = []
         self.kwargs = {}
 
-    def run(self, queue, config, *args, **kwargs):
+    def run(self, msg_queue, config, log_queue, *args, **kwargs):
         '''
-        queue
+        msg_queue
         message queue for process server and main process
 
         #1. Send event to main process,
@@ -40,6 +41,9 @@ def run(self, queue, config, *args, **kwargs):
 
         config
         lyrebird config dict
+
+        log_queue
+        send log msg to logger process
         '''
         pass
 
@@ -49,8 +53,9 @@ def start(self):
 
         global service_msg_queue
         config = application.config.raw()
+        logger_queue = application.server['log'].queue
         self.server_process = Process(group=None, target=self.run,
-                                      args=[service_msg_queue, config, self.args],
+                                      args=[service_msg_queue, config, logger_queue, self.args],
                                       kwargs=self.kwargs,
                                       daemon=True)
         self.server_process.start()

diff --git a/lyrebird/log.py b/lyrebird/log.py
@@ -1,14 +1,14 @@
 import logging
+from .base_server import ProcessServer
+from multiprocessing import Queue, Lock
 from logging.handlers import TimedRotatingFileHandler
 from colorama import Fore, Style, Back
 from collections import namedtuple
 from pathlib import Path
-
+import os
 DEFAULT_LOG_PATH = '~/.lyrebird/lyrebird.log'
-
 LOGGER_INITED = False
 
-
 Color = namedtuple('Color', ['fore', 'style', 'back'])
 
 COLORS = dict(
@@ -20,6 +20,8 @@
     DEBUG=Color(fore=Fore.GREEN, style=Style.NORMAL, back=Back.RESET)
 )
 
+process = None
+
 
 def colorit(message, levelname):
     color = COLORS.get(levelname)
@@ -65,21 +67,20 @@ def make_file_handler(_log_path=None):
     return file_handler
 
 
-def init(config):
+def init(config, log_queue = None):
     global LOGGER_INITED
     if LOGGER_INITED:
         return
-
+    
     if not config:
         config = {}
+
+    if not log_queue:
+        log_server = LogServer()
+        log_queue = log_server.queue
 
     logging.addLevelName(60, 'NOTICE')
 
-    stream_handler = make_stream_handler()
-
-    log_path = config.get('log')
-    file_handler = make_file_handler(log_path)
-
     verbose = config.get('verbose', 0)
     if verbose == 0:
         logger_level = logging.ERROR
@@ -88,18 +89,68 @@ def init(config):
     elif verbose >= 2:
         logger_level = logging.DEBUG
 
-    lyrebird_logger = logging.getLogger('lyrebird')
-    lyrebird_logger.addHandler(stream_handler)
-    lyrebird_logger.addHandler(file_handler)
-    lyrebird_logger.setLevel(logger_level)
+    queue_handler = logging.handlers.QueueHandler(log_queue)
 
-    for _logger_name in ['socketio', 'engineio', 'mock', 'werkzeug', 'flask']:
+    for _logger_name in ['lyrebird', 'socketio', 'engineio', 'mock', 'werkzeug', 'flask']:
         _logger = logging.getLogger(_logger_name)
-        _logger.addHandler(file_handler)
+        _logger.addHandler(queue_handler)
         _logger.setLevel(logger_level)
-
+    
     LOGGER_INITED = True
 
 
-def get_logger() -> logging.Logger:
+class LogServer(ProcessServer):
+
+    _instance = None
+
+    def __init__(self):
+        super().__init__()
+        self.queue = Queue()
+        self.log_process_lock = Lock()
+
+    def __new__(cls, *args, **kwargs):
+        if not cls._instance:
+            cls._instance = super().__new__(cls, *args, **kwargs)
+        return cls._instance
+
+    def run(self, msg_queue, config, log_queue, *args, **kwargs):
+        if not self.log_process_lock.acquire(timeout=10):
+            return
+
+        logging.addLevelName(60, 'NOTICE')
+
+        stream_handler = make_stream_handler()
+
+        log_path = config.get('log')
+        file_handler = make_file_handler(log_path)
+
+        verbose = config.get('verbose', 0)
+        if verbose == 0:
+            logger_level = logging.ERROR
+        elif verbose == 1:
+            logger_level = logging.INFO
+        elif verbose >= 2:
+            logger_level = logging.DEBUG
+
+        lyrebird_logger = logging.getLogger('lyrebird')
+        lyrebird_logger.addHandler(stream_handler)
+        lyrebird_logger.addHandler(file_handler)
+        lyrebird_logger.setLevel(logger_level)
+
+        for _logger_name in ['socketio', 'engineio', 'mock', 'werkzeug', 'flask']:
+            _logger = logging.getLogger(_logger_name)
+            _logger.addHandler(file_handler)
+            _logger.setLevel(logger_level)
+
+        while True:
+            try:
+                log = log_queue.get()
+                logger = logging.getLogger(log.name)
+                logger.handle(log)
+            except KeyboardInterrupt:
+                self.log_process_lock.release()
+                break        
+
+
+def get_logger():
     return logging.getLogger('lyrebird')
diff --git a/lyrebird/manager.py b/lyrebird/manager.py
@@ -25,6 +25,7 @@
 from lyrebird.mitm.proxy_server import LyrebirdProxyServer
 from lyrebird.task import BackgroundTaskServer
 from lyrebird.base_server import MultiProcessServerMessageDispatcher
+from lyrebird.log import LogServer
 from lyrebird import utils
 
 
@@ -98,7 +99,9 @@ def main():
     # init logger for main process
     application._cm.config['verbose'] = args.verbose
     application._cm.config['log'] = args.log
-    log.init(application._cm.config)
+    application.server['log'] = LogServer()
+    application.start_log_server()
+    log.init(application._cm.config, application.server['log'].queue)
 
     # Add exception hook
     def process_excepthook(exc_type, exc_value, tb):
@@ -201,7 +204,7 @@ def run(args: argparse.Namespace):
     application.process_status_listener()
 
     # Start server without mock server, mock server must start after all blueprint is done
-    application.start_server_without_mock()
+    application.start_server_without_mock_and_log()
 
     # int statistics reporter
     application.reporter = reporter.Reporter()

diff --git a/lyrebird/mitm/proxy_server.py b/lyrebird/mitm/proxy_server.py
@@ -7,7 +7,6 @@
 import time
 from lyrebird.base_server import ProcessServer
 from lyrebird.mitm.mitm_installer import init_mitm
-
 """
 HTTP proxy server
 Default port 4272
@@ -104,12 +103,12 @@ def publish_init_status(self, queue, status):
             }
         })
 
-    def run(self, queue, config, *args, **kwargs):
+    def run(self, msg_queue, config, log_queue, *args, **kwargs):
         # Init logger
-        log.init(config)
+        log.init(config, log_queue)
         logger = log.get_logger()
         mitm_path = kwargs.get('mitm_path')
-        self.start_mitmdump(queue, config, logger, mitm_path)
+        self.start_mitmdump(msg_queue, config, logger, mitm_path)
 
 
 class UnsupportedPlatform(Exception):

diff --git a/lyrebird/mock/extra_mock_server/__init__.py b/lyrebird/mock/extra_mock_server/__init__.py
@@ -7,6 +7,6 @@
 
 class ExtraMockServer(ProcessServer):
 
-    def run(self, queue, config, *args, **kwargs):
-        publish_init_status(queue, 'READY')
-        serve(queue, config, *args, **kwargs)
+    def run(self, msg_queue, config, log_queue, *args, **kwargs):
+        publish_init_status(msg_queue, 'READY')
+        serve(msg_queue, config, log_queue, *args, **kwargs)
diff --git a/lyrebird/mock/extra_mock_server/server.py b/lyrebird/mock/extra_mock_server/server.py
@@ -13,6 +13,7 @@
 from lyrebird import log
 
 logger = None
+logger_queue = None
 lb_config = {}
 
 
@@ -116,7 +117,7 @@ def init_app(config):
     lb_config = config
 
     global logger
-    log.init(config)
+    log.init(config, logger_queue)
     logger = log.get_logger()
 
     app = web.Application()
@@ -193,15 +194,17 @@ def publish_init_status(queue, status):
     })
 
 
-def serve(queue, config, *args, **kwargs):
+def serve(msg_queue, config, log_queue, *args, **kwargs):
+    global logger_queue
+    logger_queue = log_queue
     loop = asyncio.new_event_loop()
     main_task = loop.create_task(_run_app(config))
 
     try:
         asyncio.set_event_loop(loop)
         loop.run_until_complete(main_task)
     except KeyboardInterrupt:
-        publish_init_status(queue, 'ERROR')
+        publish_init_status(msg_queue, 'ERROR')
     finally:
         _cancel_tasks({main_task}, loop)
         _cancel_tasks(asyncio.all_tasks(loop), loop)

diff --git a/lyrebird/version.py b/lyrebird/version.py
@@ -1,3 +1,3 @@
-IVERSION = (2, 20, 1)
+IVERSION = (2, 21, 0)
 VERSION = ".".join(str(i) for i in IVERSION)
 LYREBIRD = "Lyrebird " + VERSION