From dc601059265bfd0722b8d07c67dd1f39b091fbfa Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Mon, 4 Jan 2021 17:13:32 +0100 Subject: [PATCH] Stringify plugin definitions so they can be pickled during caching (#2835) --- RELEASE.md | 3 ++ pelican/__init__.py | 14 ++++--- pelican/plugins/_utils.py | 16 ++++++++ .../normal_plugin/normal_plugin/__init__.py | 2 - .../subpackage/subpackage.py | 3 -- .../normal_submodule_plugin/subplugin.py | 3 -- pelican/tests/test_plugins.py | 39 +++++++++++++------ 7 files changed, 56 insertions(+), 24 deletions(-) create mode 100644 RELEASE.md diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 000000000..9439717c7 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,3 @@ +Release type: patch + +Replace plugin definitions in settings with string representations after registering, so they can be cached correctly (#2828). diff --git a/pelican/__init__.py b/pelican/__init__.py index 0d7232204..1982f413b 100644 --- a/pelican/__init__.py +++ b/pelican/__init__.py @@ -20,7 +20,7 @@ PagesGenerator, SourceFileGenerator, StaticGenerator, TemplatePagesGenerator) from pelican.plugins import signals -from pelican.plugins._utils import load_plugins +from pelican.plugins._utils import get_plugin_name, load_plugins from pelican.readers import Readers from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer from pelican.settings import coerce_overrides, read_settings @@ -65,14 +65,18 @@ def init_path(self): sys.path.insert(0, '') def init_plugins(self): - self.plugins = load_plugins(self.settings) - for plugin in self.plugins: - logger.debug('Registering plugin `%s`', plugin.__name__) + self.plugins = [] + for plugin in load_plugins(self.settings): + name = get_plugin_name(plugin) + logger.debug('Registering plugin `%s`', name) try: plugin.register() + self.plugins.append(plugin) except Exception as e: logger.error('Cannot register plugin `%s`\n%s', - plugin.__name__, e) + name, e) + + self.settings['PLUGINS'] = [get_plugin_name(p) for p in self.plugins] def run(self): """Run the generators and return""" diff --git a/pelican/plugins/_utils.py b/pelican/plugins/_utils.py index ffe32799b..87877b08e 100644 --- a/pelican/plugins/_utils.py +++ b/pelican/plugins/_utils.py @@ -1,6 +1,7 @@ import importlib import importlib.machinery import importlib.util +import inspect import logging import pkgutil import sys @@ -107,3 +108,18 @@ def load_plugins(settings): plugins = list(namespace_plugins.values()) return plugins + + +def get_plugin_name(plugin): + """ + Plugins can be passed as module objects, however this breaks caching as + module objects cannot be pickled. To work around this, all plugins are + stringified post-initialization. + """ + if inspect.isclass(plugin): + return plugin.__qualname__ + + if inspect.ismodule(plugin): + return plugin.__name__ + + return type(plugin).__qualname__ diff --git a/pelican/tests/dummy_plugins/normal_plugin/normal_plugin/__init__.py b/pelican/tests/dummy_plugins/normal_plugin/normal_plugin/__init__.py index 5838a835f..e714c7a60 100644 --- a/pelican/tests/dummy_plugins/normal_plugin/normal_plugin/__init__.py +++ b/pelican/tests/dummy_plugins/normal_plugin/normal_plugin/__init__.py @@ -1,7 +1,5 @@ from .submodule import noop # noqa: F401 -NAME = 'normal plugin' - def register(): pass diff --git a/pelican/tests/dummy_plugins/normal_plugin/normal_submodule_plugin/subpackage/subpackage.py b/pelican/tests/dummy_plugins/normal_plugin/normal_submodule_plugin/subpackage/subpackage.py index ddb0eeca2..9e12b19f7 100644 --- a/pelican/tests/dummy_plugins/normal_plugin/normal_submodule_plugin/subpackage/subpackage.py +++ b/pelican/tests/dummy_plugins/normal_plugin/normal_submodule_plugin/subpackage/subpackage.py @@ -1,5 +1,2 @@ -NAME = 'normal subpackage plugin' - - def register(): pass diff --git a/pelican/tests/dummy_plugins/normal_plugin/normal_submodule_plugin/subplugin.py b/pelican/tests/dummy_plugins/normal_plugin/normal_submodule_plugin/subplugin.py index 377c788b5..9e12b19f7 100644 --- a/pelican/tests/dummy_plugins/normal_plugin/normal_submodule_plugin/subplugin.py +++ b/pelican/tests/dummy_plugins/normal_plugin/normal_submodule_plugin/subplugin.py @@ -1,5 +1,2 @@ -NAME = 'normal submodule plugin' - - def register(): pass diff --git a/pelican/tests/test_plugins.py b/pelican/tests/test_plugins.py index 29729539e..348c3e94b 100644 --- a/pelican/tests/test_plugins.py +++ b/pelican/tests/test_plugins.py @@ -1,7 +1,9 @@ import os from contextlib import contextmanager -from pelican.plugins._utils import get_namespace_plugins, load_plugins +import pelican.tests.dummy_plugins.normal_plugin.normal_plugin as normal_plugin +from pelican.plugins._utils import (get_namespace_plugins, get_plugin_name, + load_plugins) from pelican.tests.support import unittest @@ -81,9 +83,7 @@ def test_get_namespace_plugins(self): def test_load_plugins(self): def get_plugin_names(plugins): - return { - plugin.NAME if hasattr(plugin, 'NAME') else plugin.__name__ - for plugin in plugins} + return {get_plugin_name(p) for p in plugins} # existing namespace plugins existing_ns_plugins = load_plugins({}) @@ -93,7 +93,7 @@ def get_plugin_names(plugins): plugins = load_plugins({}) self.assertEqual(len(plugins), len(existing_ns_plugins)+1, plugins) self.assertEqual( - {'namespace plugin'} | get_plugin_names(existing_ns_plugins), + {'pelican.plugins.ns_plugin'} | get_plugin_names(existing_ns_plugins), get_plugin_names(plugins)) # disable namespace plugins with `PLUGINS = []` @@ -113,7 +113,7 @@ def get_plugin_names(plugins): plugins = load_plugins(SETTINGS) self.assertEqual(len(plugins), 1, plugins) self.assertEqual( - {'normal plugin'}, + {'normal_plugin'}, get_plugin_names(plugins)) # normal submodule/subpackage plugins @@ -127,8 +127,8 @@ def get_plugin_names(plugins): plugins = load_plugins(SETTINGS) self.assertEqual(len(plugins), 2, plugins) self.assertEqual( - {'normal submodule plugin', - 'normal subpackage plugin'}, + {'normal_submodule_plugin.subplugin', + 'normal_submodule_plugin.subpackage.subpackage'}, get_plugin_names(plugins)) # ensure normal plugins are loaded only once @@ -149,7 +149,7 @@ def get_plugin_names(plugins): plugins = load_plugins(SETTINGS) self.assertEqual(len(plugins), 1, plugins) self.assertEqual( - {'namespace plugin'}, + {'pelican.plugins.ns_plugin'}, get_plugin_names(plugins)) # namespace plugin long @@ -159,7 +159,7 @@ def get_plugin_names(plugins): plugins = load_plugins(SETTINGS) self.assertEqual(len(plugins), 1, plugins) self.assertEqual( - {'namespace plugin'}, + {'pelican.plugins.ns_plugin'}, get_plugin_names(plugins)) # normal and namespace plugin @@ -170,5 +170,22 @@ def get_plugin_names(plugins): plugins = load_plugins(SETTINGS) self.assertEqual(len(plugins), 2, plugins) self.assertEqual( - {'normal plugin', 'namespace plugin'}, + {'normal_plugin', 'pelican.plugins.ns_plugin'}, get_plugin_names(plugins)) + + def test_get_plugin_name(self): + self.assertEqual( + get_plugin_name(normal_plugin), + 'pelican.tests.dummy_plugins.normal_plugin.normal_plugin', + ) + + class NoopPlugin: + def register(self): + pass + + self.assertEqual( + get_plugin_name(NoopPlugin), + 'PluginTest.test_get_plugin_name..NoopPlugin') + self.assertEqual( + get_plugin_name(NoopPlugin()), + 'PluginTest.test_get_plugin_name..NoopPlugin')