diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000000..0a251dd43f --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,4 @@ +Release type: minor + +Allow regular expressions in `LOG_FILTER` setting + diff --git a/docs/settings.rst b/docs/settings.rst index e51c6a1207..3589a64d2b 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -128,11 +128,17 @@ Basic settings .. data:: LOG_FILTER = [] A list of tuples containing the logging level (up to ``warning``) and the - message to be ignored. + message to be ignored or containing the type (either ``string`` or ``regex``) + , the logging level and a regular expression. If the latter format is used, + messages that match the regular expression will not be shown. Example:: - LOG_FILTER = [(logging.WARN, 'TAG_SAVE_AS is set to False')] + LOG_FILTER = [ + (logging.WARN, 'TAG_SAVE_AS is set to False'), + ('string', logging.WARN, 'Empty theme folder. Using `basic` theme.'), + ('regex', logging.WARN, r'Cannot get modification stamp for /foo/.*'), + ] .. data:: READERS = {} diff --git a/pelican/log.py b/pelican/log.py index be176ea89d..b7300b06a9 100644 --- a/pelican/log.py +++ b/pelican/log.py @@ -5,7 +5,8 @@ from rich.logging import RichHandler __all__ = [ - 'init' + 'init', + 'LimitFilter', ] console = Console() @@ -23,7 +24,8 @@ class LimitFilter(logging.Filter): LOGS_DEDUP_MIN_LEVEL = logging.WARNING - _ignore = set() + ignore = set() + ignore_regexp = set() _raised_messages = set() _threshold = 5 _group_count = defaultdict(int) @@ -50,7 +52,11 @@ def filter(self, record): if logger_level > logging.DEBUG: template_key = (record.levelno, record.msg) message_key = (record.levelno, record.getMessage()) - if (template_key in self._ignore or message_key in self._ignore): + if template_key in self.ignore or message_key in self.ignore: + return False + if any(regexp[1].match(record.getMessage()) + for regexp in self.ignore_regexp + if regexp[0] == record.levelno): return False # check if we went over threshold diff --git a/pelican/settings.py b/pelican/settings.py index 5b495e863a..66021ab60f 100644 --- a/pelican/settings.py +++ b/pelican/settings.py @@ -509,7 +509,19 @@ def configure_settings(settings): # specify the log messages to be ignored log_filter = settings.get('LOG_FILTER', DEFAULT_CONFIG['LOG_FILTER']) - LimitFilter._ignore.update(set(log_filter)) + for item in log_filter: + if len(item) == 2: # old-style string or template + LimitFilter.ignore.update({item}) + elif len(item) == 3: # new-style string/template or regexp + if item[0] == "string": + LimitFilter.ignore.update({(item[1:])}) + elif item[0] == "regex": + regex = re.compile(item[2]) + LimitFilter.ignore_regexp.update({(item[1], regex)}) + else: + raise ValueError(f"Invalid LOG_FILTER type '{item[0]}'") + else: + raise ValueError(f"Invalid item '{str(item)}' in LOG_FILTER") # lookup the theme in "pelican/themes" if the given one doesn't exist if not os.path.isdir(settings['THEME']): diff --git a/pelican/tests/test_log.py b/pelican/tests/test_log.py index 1f2fb83a84..0191fe3634 100644 --- a/pelican/tests/test_log.py +++ b/pelican/tests/test_log.py @@ -1,4 +1,5 @@ import logging +import re import unittest from collections import defaultdict from contextlib import contextmanager @@ -19,7 +20,8 @@ def tearDown(self): super().tearDown() def _reset_limit_filter(self): - log.LimitFilter._ignore = set() + log.LimitFilter.ignore = set() + log.LimitFilter.ignore_regexp = set() log.LimitFilter._raised_messages = set() log.LimitFilter._threshold = 5 log.LimitFilter._group_count = defaultdict(int) @@ -49,7 +51,7 @@ def do_logging(): # filter by template with self.reset_logger(): - log.LimitFilter._ignore.add((logging.WARNING, 'Log %s')) + log.LimitFilter.ignore.add((logging.WARNING, 'Log %s')) do_logging() self.assertEqual( self.handler.count_logs('Log \\d', logging.WARNING), @@ -60,7 +62,7 @@ def do_logging(): # filter by exact message with self.reset_logger(): - log.LimitFilter._ignore.add((logging.WARNING, 'Log 3')) + log.LimitFilter.ignore.add((logging.WARNING, 'Log 3')) do_logging() self.assertEqual( self.handler.count_logs('Log \\d', logging.WARNING), @@ -69,14 +71,30 @@ def do_logging(): self.handler.count_logs('Another log \\d', logging.WARNING), 5) - # filter by both + # filter by regular expression with self.reset_logger(): - log.LimitFilter._ignore.add((logging.WARNING, 'Log 3')) - log.LimitFilter._ignore.add((logging.WARNING, 'Another log %s')) + log.LimitFilter.ignore_regexp.add((logging.WARNING, + re.compile(r'Log.*'))) + log.LimitFilter.ignore_regexp.add((logging.WARNING, + re.compile(r'.*log 4'))) do_logging() self.assertEqual( self.handler.count_logs('Log \\d', logging.WARNING), + 0) + self.assertEqual( + self.handler.count_logs('Another log \\d', logging.WARNING), 4) + + # filter by all + with self.reset_logger(): + log.LimitFilter.ignore.add((logging.WARNING, 'Log 3')) + log.LimitFilter.ignore.add((logging.WARNING, 'Another log %s')) + log.LimitFilter.ignore_regexp.add((logging.WARNING, + re.compile(r'Lo.*4$'))) + do_logging() + self.assertEqual( + self.handler.count_logs('Log \\d', logging.WARNING), + 3) self.assertEqual( self.handler.count_logs('Another log \\d', logging.WARNING), 0) diff --git a/pelican/tests/test_settings.py b/pelican/tests/test_settings.py index 0f630ad55a..ab31c39d3e 100644 --- a/pelican/tests/test_settings.py +++ b/pelican/tests/test_settings.py @@ -1,9 +1,11 @@ import copy import locale +import logging import os +import re from os.path import abspath, dirname, join - +from pelican.log import LimitFilter from pelican.settings import (DEFAULT_CONFIG, DEFAULT_THEME, _printf_s_to_format_field, configure_settings, @@ -108,6 +110,41 @@ def test_configure_settings(self): configure_settings(settings) self.assertEqual(settings['FEED_DOMAIN'], 'http://feeds.example.com') + def test_configure_log_filter_settings(self): + # Various forms of filter settings should be applied correctly. + settings = { + 'LOG_FILTER': [ + (logging.WARNING, 'foo'), + ('string', logging.ERROR, 'bar'), + ('regex', logging.INFO, r'baz.*boo'), + ], + 'PATH': os.curdir, + 'THEME': DEFAULT_THEME, + } + configure_settings(settings) + + self.assertEqual(LimitFilter.ignore, { + (logging.WARNING, 'foo'), + (logging.ERROR, 'bar'), + }) + self.assertEqual(LimitFilter.ignore_regexp, { + (logging.INFO, re.compile(r'baz.*boo')) + }) + + settings['LOG_FILTER'] = [(1, 2, 3, 4)] + with self.assertRaisesRegex( + ValueError, + r"Invalid item '\(1, 2, 3, 4\)' in LOG_FILTER" + ): + configure_settings(settings) + + settings['LOG_FILTER'] = [('foo', 'bar', 'baz')] + with self.assertRaisesRegex( + ValueError, + r"Invalid LOG_FILTER type 'foo'" + ): + configure_settings(settings) + def test_theme_settings_exceptions(self): settings = self.settings