Allow usage of regex in LOG_FILTER setting

Fix getpelican#2893

RELEASE.md

@@ -0,0 +1,4 @@
+Release type: minor
+
+Allow regular expressions in `LOG_FILTER` setting
+

docs/settings.rst

@@ -128,11 +128,17 @@ Basic settings
 .. data:: LOG_FILTER = []
 
    A list of tuples containing the logging level (up to ``warning``) and the
-   message to be ignored.
+   message to be ignored or containing the type (either ``string`` or ``regex``)
+   , the logging level and a regular expression. If the latter format is used,
+   messages that match the regular expression will not be shown.
 
    Example::
 
-      LOG_FILTER = [(logging.WARN, 'TAG_SAVE_AS is set to False')]
+      LOG_FILTER = [
+        (logging.WARN, 'TAG_SAVE_AS is set to False'),
+        ('string', logging.WARN, 'Empty theme folder. Using `basic` theme.'),
+        ('regex', logging.WARN, r'Cannot get modification stamp for /foo/.*'),
+      ]
 
 .. data:: READERS = {}
 

pelican/log.py

@@ -5,7 +5,8 @@
 from rich.logging import RichHandler
 
 __all__ = [
-    'init'
+    'init',
+    'LimitFilter',
 ]
 
 console = Console()
@@ -23,7 +24,8 @@ class LimitFilter(logging.Filter):
 
     LOGS_DEDUP_MIN_LEVEL = logging.WARNING
 
-    _ignore = set()
+    ignore = set()
+    ignore_regexp = set()
     _raised_messages = set()
     _threshold = 5
     _group_count = defaultdict(int)
@@ -50,7 +52,11 @@ def filter(self, record):
         if logger_level > logging.DEBUG:
             template_key = (record.levelno, record.msg)
             message_key = (record.levelno, record.getMessage())
-            if (template_key in self._ignore or message_key in self._ignore):
+            if template_key in self.ignore or message_key in self.ignore:
+                return False
+            if any(regexp[1].match(record.getMessage())
+                   for regexp in self.ignore_regexp
+                   if regexp[0] == record.levelno):
                 return False
 
         # check if we went over threshold

pelican/settings.py

@@ -509,7 +509,19 @@ def configure_settings(settings):
 
     # specify the log messages to be ignored
     log_filter = settings.get('LOG_FILTER', DEFAULT_CONFIG['LOG_FILTER'])
-    LimitFilter._ignore.update(set(log_filter))
+    for item in log_filter:
+        if len(item) == 2:  # old-style string or template
+            LimitFilter.ignore.update({item})
+        elif len(item) == 3:  # new-style string/template or regexp
+            if item[0] == "string":
+                LimitFilter.ignore.update({(item[1:])})
+            elif item[0] == "regex":
+                regex = re.compile(item[2])
+                LimitFilter.ignore_regexp.update({(item[1], regex)})
+            else:
+                raise ValueError(f"Invalid LOG_FILTER type '{item[0]}'")
+        else:
+            raise ValueError(f"Invalid item '{str(item)}' in LOG_FILTER")
 
     # lookup the theme in "pelican/themes" if the given one doesn't exist
     if not os.path.isdir(settings['THEME']):

pelican/tests/test_log.py

@@ -1,4 +1,5 @@
 import logging
+import re
 import unittest
 from collections import defaultdict
 from contextlib import contextmanager
@@ -19,7 +20,8 @@ def tearDown(self):
         super().tearDown()
 
     def _reset_limit_filter(self):
-        log.LimitFilter._ignore = set()
+        log.LimitFilter.ignore = set()
+        log.LimitFilter.ignore_regexp = set()
         log.LimitFilter._raised_messages = set()
         log.LimitFilter._threshold = 5
         log.LimitFilter._group_count = defaultdict(int)
@@ -49,7 +51,7 @@ def do_logging():
 
         # filter by template
         with self.reset_logger():
-            log.LimitFilter._ignore.add((logging.WARNING, 'Log %s'))
+            log.LimitFilter.ignore.add((logging.WARNING, 'Log %s'))
             do_logging()
             self.assertEqual(
                 self.handler.count_logs('Log \\d', logging.WARNING),
@@ -60,7 +62,7 @@ def do_logging():
 
         # filter by exact message
         with self.reset_logger():
-            log.LimitFilter._ignore.add((logging.WARNING, 'Log 3'))
+            log.LimitFilter.ignore.add((logging.WARNING, 'Log 3'))
             do_logging()
             self.assertEqual(
                 self.handler.count_logs('Log \\d', logging.WARNING),
@@ -69,14 +71,30 @@ def do_logging():
                 self.handler.count_logs('Another log \\d', logging.WARNING),
                 5)
 
-        # filter by both
+        # filter by regular expression
         with self.reset_logger():
-            log.LimitFilter._ignore.add((logging.WARNING, 'Log 3'))
-            log.LimitFilter._ignore.add((logging.WARNING, 'Another log %s'))
+            log.LimitFilter.ignore_regexp.add((logging.WARNING,
+                                               re.compile(r'Log.*')))
+            log.LimitFilter.ignore_regexp.add((logging.WARNING,
+                                               re.compile(r'.*log 4')))
             do_logging()
             self.assertEqual(
                 self.handler.count_logs('Log \\d', logging.WARNING),
+                0)
+            self.assertEqual(
+                self.handler.count_logs('Another log \\d', logging.WARNING),
                 4)
+
+        # filter by all
+        with self.reset_logger():
+            log.LimitFilter.ignore.add((logging.WARNING, 'Log 3'))
+            log.LimitFilter.ignore.add((logging.WARNING, 'Another log %s'))
+            log.LimitFilter.ignore_regexp.add((logging.WARNING,
+                                               re.compile(r'Lo.*4$')))
+            do_logging()
+            self.assertEqual(
+                self.handler.count_logs('Log \\d', logging.WARNING),
+                3)
             self.assertEqual(
                 self.handler.count_logs('Another log \\d', logging.WARNING),
                 0)

pelican/tests/test_settings.py

@@ -1,9 +1,11 @@
 import copy
 import locale
+import logging
 import os
+import re
 from os.path import abspath, dirname, join
 
-
+from pelican.log import LimitFilter
 from pelican.settings import (DEFAULT_CONFIG, DEFAULT_THEME,
                               _printf_s_to_format_field,
                               configure_settings,
@@ -108,6 +110,41 @@ def test_configure_settings(self):
         configure_settings(settings)
         self.assertEqual(settings['FEED_DOMAIN'], 'http://feeds.example.com')
 
+    def test_configure_log_filter_settings(self):
+        # Various forms of filter settings should be applied correctly.
+        settings = {
+            'LOG_FILTER': [
+                (logging.WARNING, 'foo'),
+                ('string', logging.ERROR, 'bar'),
+                ('regex', logging.INFO, r'baz.*boo'),
+            ],
+            'PATH': os.curdir,
+            'THEME': DEFAULT_THEME,
+        }
+        configure_settings(settings)
+
+        self.assertEqual(LimitFilter.ignore, {
+            (logging.WARNING, 'foo'),
+            (logging.ERROR, 'bar'),
+        })
+        self.assertEqual(LimitFilter.ignore_regexp, {
+            (logging.INFO, re.compile(r'baz.*boo'))
+        })
+
+        settings['LOG_FILTER'] = [(1, 2, 3, 4)]
+        with self.assertRaisesRegex(
+            ValueError,
+            r"Invalid item '\(1, 2, 3, 4\)' in LOG_FILTER"
+        ):
+            configure_settings(settings)
+
+        settings['LOG_FILTER'] = [('foo', 'bar', 'baz')]
+        with self.assertRaisesRegex(
+            ValueError,
+            r"Invalid LOG_FILTER type 'foo'"
+        ):
+            configure_settings(settings)
+
     def test_theme_settings_exceptions(self):
         settings = self.settings