Merge pull request #884 from sphinx-contrib/extended-publish-debugging

Extended publish debugging

doc/configuration.rst

@@ -1378,15 +1378,28 @@ Advanced publishing configuration
 .. confval:: confluence_publish_debug
 
     .. versionadded:: 1.8
+    .. versionchanged:: 2.5
 
-    A boolean value to whether or not to print debug requests made to a
-    Confluence instance. This can be helpful for users attempting to debug
-    their connection to a Confluence instance. By default, this option is
-    disabled with a value of ``False``.
+        Switched from boolean to string for setting new debugging options.
+
+    Configures the ability to enable certain debugging messages for requests
+    made to a Confluence instance. This can be helpful for users attempting
+    to debug their connection to a Confluence instance. By default, no
+    debugging is enabled.
+
+    Available options are as follows:
+
+    - ``all``: Enable all debugging options.
+    - ``deprecated``: Log warnings when a deprecated API call is used
+      (*for development purposes*).
+    - ``headers``: Log requests and responses, including their headers.
+    - ``urllib3``: Enable urllib3 library debugging messages.
+
+    An example debugging configuration is as follows:
 
     .. code-block:: python
 
-        confluence_publish_debug = True
+        confluence_publish_debug = 'urllib3'
 
 .. confval:: confluence_publish_delay
 

sphinxcontrib/confluencebuilder/__init__.py

@@ -187,8 +187,8 @@ def setup(app):
     cm.add_conf('confluence_proxy')
     # Subset of documents which are allowed to be published.
     cm.add_conf('confluence_publish_allowlist')
-    # Enable debugging for publish requests.
-    cm.add_conf_bool('confluence_publish_debug')
+    # Configure debugging for publish requests.
+    cm.add_conf('confluence_publish_debug')
     # Duration (in seconds) to delay each API request.
     cm.add_conf('confluence_publish_delay')
     # Subset of documents which are denied to be published.

sphinxcontrib/confluencebuilder/config/checks.py

@@ -4,6 +4,7 @@
 from sphinxcontrib.confluencebuilder.config.notifications import deprecated
 from sphinxcontrib.confluencebuilder.config.notifications import warnings
 from sphinxcontrib.confluencebuilder.config.validation import ConfigurationValidation
+from sphinxcontrib.confluencebuilder.debug import PublishDebug
 from sphinxcontrib.confluencebuilder.exceptions import ConfluenceConfigurationError
 from sphinxcontrib.confluencebuilder.std.confluence import EDITORS
 from sphinxcontrib.confluencebuilder.util import handle_cli_file_subset
@@ -554,6 +555,28 @@ def conf_translate(value):
 
     # ##################################################################
 
+    opts = PublishDebug._member_names_  # pylint: disable=no-member
+
+    # confluence_publish_debug
+    try:
+        validator.conf('confluence_publish_debug').bool()  # deprecated
+    except ConfluenceConfigurationError:
+        try:
+            validator.conf('confluence_publish_debug').enum(PublishDebug)
+        except ConfluenceConfigurationError as e:
+            opts = PublishDebug._member_names_  # pylint: disable=no-member
+            raise ConfluenceConfigurationError('''\
+{msg}
+
+The option 'confluence_publish_debug' has been configured to enable publish
+debugging. Accepted values include:
+
+ - all
+ - {opts}
+'''.format(msg=e, opts='\n - '.join(opts)))
+
+    # ##################################################################
+
     validator.conf('confluence_publish_delay') \
              .float_(positive=True)
 

sphinxcontrib/confluencebuilder/config/defaults.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: BSD-2-Clause
 # Copyright Sphinx Confluence Builder Contributors (AUTHORS)
 
+from sphinxcontrib.confluencebuilder.debug import PublishDebug
 from sphinxcontrib.confluencebuilder.util import str2bool
 import os
 
@@ -87,6 +88,19 @@ def apply_defaults(builder):
             conf.confluence_adv_permit_raw_html is not None:
         conf.confluence_permit_raw_html = conf.confluence_adv_permit_raw_html
 
+    # ensure confluence_publish_debug is set with its expected enum value
+    publish_debug = conf.confluence_publish_debug
+    if publish_debug is not None and publish_debug is not False:
+        # a boolean-provided publish debug is deprecated, but we will accept
+        # it as its original implementation as an indication to enable
+        # urllib3 logs
+        if publish_debug is True:
+            conf.confluence_publish_debug = PublishDebug.urllib3
+        elif not isinstance(publish_debug, PublishDebug):
+            conf.confluence_publish_debug = PublishDebug[publish_debug.lower()]
+    else:
+        conf.confluence_publish_debug = PublishDebug.none
+
     if conf.confluence_publish_intersphinx is None:
         conf.confluence_publish_intersphinx = True
 

sphinxcontrib/confluencebuilder/config/notifications.py

@@ -96,3 +96,7 @@ def warnings(validator):
     # confluence_editor assigned to an editor that is not supported
     if config.confluence_editor and config.confluence_editor not in EDITORS:
         logger.warn('confluence_editor configured with an unsupported editor')
+
+    # confluence_publish_debug should be using the new string values
+    if isinstance(config.confluence_publish_debug, bool):
+        logger.warn('confluence_publish_debug using deprecated bool value')

sphinxcontrib/confluencebuilder/config/validation.py

@@ -202,6 +202,33 @@ def docnames_from_file(self):
 
         return self
 
+    def enum(self, etype):
+        """
+        checks if a configuration is an enumeration type
+
+        After an instance has been set a configuration key (via `conf`), this
+        method can be used to check if the value (if any) configured with this
+        key is an enumeration of type `etype`. If not, a
+        `ConfluenceConfigurationError` exception will be thrown.
+
+        In the event that the configuration is not set (e.g. a value of `None`),
+        this method will have no effect.
+
+        Returns:
+            the validator instance
+        """
+
+        value = self._value()
+
+        if value is not None and not isinstance(value, etype):
+            try:
+                value = etype[value.lower()]
+            except KeyError:
+                raise ConfluenceConfigurationError(
+                    f'{self.key} is not an enumeration ({etype.__name__})')
+
+        return self
+
     def file(self):
         """
         checks if a configuration is a valid file

sphinxcontrib/confluencebuilder/debug.py

@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright Sphinx Confluence Builder Contributors (AUTHORS)
+
+from enum import Flag
+from enum import auto
+
+
+class PublishDebug(Flag):
+    """
+    publishing debugging enumeration
+
+    Defines a series of flags to track support debugging modes when enabling
+    publishing-specific debugging in this extension. Provides an explicit
+    list of supported options (that can be configuration checked), as well
+    as provides an "all" state, allowing easy implementation handling of
+    enabling specific debugging scenarios when all options are enabled.
+    """
+
+    # do not perform any logging
+    none = auto()
+    # logs warnings when confluence reports a deprecated api call
+    deprecated = auto()
+    # log raw requests/responses in stdout with header data
+    headers = auto()
+    # log urllib3-supported debug messages
+    urllib3 = auto()
+    # enable all logging
+    all = headers | urllib3
+    # enable all developer logging
+    developer = deprecated | all

sphinxcontrib/confluencebuilder/publisher.py

@@ -8,6 +8,7 @@
 """
 
 from sphinx.util.logging import skip_warningiserror
+from sphinxcontrib.confluencebuilder.debug import PublishDebug
 from sphinxcontrib.confluencebuilder.exceptions import ConfluenceBadApiError
 from sphinxcontrib.confluencebuilder.exceptions import ConfluenceBadServerUrlError
 from sphinxcontrib.confluencebuilder.exceptions import ConfluenceConfigurationError
@@ -41,7 +42,6 @@ def init(self, config, cloud=None):
         self.cloud = cloud
         self.config = config
         self.append_labels = config.confluence_append_labels
-        self.debug = config.confluence_publish_debug
         self.dryrun = config.confluence_publish_dryrun
         self.notify = not config.confluence_disable_notifications
         self.onlynew = config.confluence_publish_onlynew
@@ -56,7 +56,7 @@ def init(self, config, cloud=None):
             self.append_labels = True
 
         # if debugging, enable requests (urllib3) logging
-        if self.debug:
+        if PublishDebug.urllib3 in self.config.confluence_publish_debug:
             logging.basicConfig()
             logging.getLogger().setLevel(logging.DEBUG)
             rlog = logging.getLogger('requests.packages.urllib3')

sphinxcontrib/confluencebuilder/rest.py

@@ -4,6 +4,7 @@
 from functools import wraps
 from email.utils import mktime_tz
 from email.utils import parsedate_tz
+from sphinxcontrib.confluencebuilder.debug import PublishDebug
 from sphinxcontrib.confluencebuilder.exceptions import ConfluenceAuthenticationFailedUrlError
 from sphinxcontrib.confluencebuilder.exceptions import ConfluenceBadApiError
 from sphinxcontrib.confluencebuilder.exceptions import ConfluenceBadServerUrlError
@@ -235,14 +236,11 @@ def _setup_session(self, config):
 
     @rate_limited_retries()
     @requests_exception_wrappers()
-    def get(self, key, params=None):
-        rest_url = self.url + self.bind_path + '/' + key
-
-        rsp = self.session.get(rest_url, params=params, timeout=self.timeout)
-        self._handle_common_request(rsp)
+    def get(self, path, params=None):
+        rsp = self._process_request('GET', path, params=params)
 
         if not rsp.ok:
-            errdata = self._format_error(rsp, key)
+            errdata = self._format_error(rsp, path)
             raise ConfluenceBadApiError(rsp.status_code, errdata)
         if not rsp.text:
             raise ConfluenceSeraphAuthenticationFailedUrlError
@@ -258,15 +256,11 @@ def get(self, key, params=None):
 
     @rate_limited_retries()
     @requests_exception_wrappers()
-    def post(self, key, data, files=None):
-        rest_url = self.url + self.bind_path + '/' + key
-
-        rsp = self.session.post(
-            rest_url, json=data, files=files, timeout=self.timeout)
-        self._handle_common_request(rsp)
+    def post(self, path, data, files=None):
+        rsp = self._process_request('POST', path, json=data, files=files)
 
         if not rsp.ok:
-            errdata = self._format_error(rsp, key)
+            errdata = self._format_error(rsp, path)
             if self.verbosity > 0:
                 errdata += "\n"
                 errdata += json.dumps(data, indent=2)
@@ -285,14 +279,11 @@ def post(self, key, data, files=None):
 
     @rate_limited_retries()
     @requests_exception_wrappers()
-    def put(self, key, value, data):
-        rest_url = self.url + self.bind_path + '/' + key + '/' + str(value)
-
-        rsp = self.session.put(rest_url, json=data, timeout=self.timeout)
-        self._handle_common_request(rsp)
+    def put(self, path, value, data):
+        rsp = self._process_request('PUT', f'{path}/{value}', json=data)
 
         if not rsp.ok:
-            errdata = self._format_error(rsp, key)
+            errdata = self._format_error(rsp, path)
             if self.verbosity > 0:
                 errdata += "\n"
                 errdata += json.dumps(data, indent=2)
@@ -311,32 +302,52 @@ def put(self, key, value, data):
 
     @rate_limited_retries()
     @requests_exception_wrappers()
-    def delete(self, key, value):
-        rest_url = self.url + self.bind_path + '/' + key + '/' + str(value)
-
-        rsp = self.session.delete(rest_url, timeout=self.timeout)
-        self._handle_common_request(rsp)
+    def delete(self, path, value):
+        rsp = self._process_request('DELETE', f'{path}/{value}')
 
         if not rsp.ok:
-            errdata = self._format_error(rsp, key)
+            errdata = self._format_error(rsp, path)
             raise ConfluenceBadApiError(rsp.status_code, errdata)
 
     def close(self):
         self.session.close()
 
-    def _format_error(self, rsp, key):
+    def _format_error(self, rsp, path):
         err = ""
         err += f"REQ: {rsp.request.method}\n"
         err += "RSP: " + str(rsp.status_code) + "\n"
         err += "URL: " + self.url + self.bind_path + "\n"
-        err += "API: " + key + "\n"
+        err += "API: " + path + "\n"
         try:
             err += f'DATA: {json.dumps(rsp.json(), indent=2)}'
         except:  # noqa: E722
             err += 'DATA: <not-or-invalid-json>'
         return err
 
-    def _handle_common_request(self, rsp):
+    def _process_request(self, method, path, *args, **kwargs):
+        dump = PublishDebug.headers in self.config.confluence_publish_debug
+
+        rest_url = f'{self.url}{self.bind_path}/{path}'
+        base_req = requests.Request(method, rest_url, *args, **kwargs)
+        req = self.session.prepare_request(base_req)
+
+        # debug logging
+        if dump:
+            print('')  # leading newline, if debugging into active line
+            print('(debug) Request]')
+            print(f'{req.method} {req.url}')
+            print('\n'.join(f'{k}: {v}' for k, v in req.headers.items()))
+            print('')
+
+        # perform the rest request
+        rsp = self.session.send(req, timeout=self.timeout)
+
+        # debug logging
+        if dump:
+            print('(debug) Response]')
+            print(f'Code: {rsp.status_code}')
+            print('\n'.join(f'{k}: {v}' for k, v in rsp.headers.items()))
+            print('')
 
         # if confluence or a proxy reports a retry-after delay (to pace us),
         # track it to delay the next request made
@@ -366,6 +377,13 @@ def _handle_common_request(self, rsp):
                                 math.ceil(delay)))
                     self._reported_large_delay = True
 
+        # check if Confluence reports a `Deprecation` header in the response;
+        # if so, log a message is we have the debug message enabled to help
+        # inform developers that this api call may required updating
+        if PublishDebug.deprecated in self.config.confluence_publish_debug:
+            if rsp.headers.get('Deprecation'):
+                logger.warn(f'(warning) deprecated api call made: {path}')
+
         if rsp.status_code == 401:
             raise ConfluenceAuthenticationFailedUrlError
         if rsp.status_code == 403:
@@ -374,3 +392,5 @@ def _handle_common_request(self, rsp):
             raise ConfluenceProxyPermissionError
         if rsp.status_code == 429:
             raise ConfluenceRateLimitedError
+
+        return rsp

tests/lib/__init__.py

@@ -9,6 +9,7 @@
 from sphinx.util.console import nocolor
 from sphinx.util.docutils import docutils_namespace
 from sphinxcontrib.confluencebuilder import util
+from sphinxcontrib.confluencebuilder.debug import PublishDebug
 from threading import Event
 from threading import Lock
 from threading import Thread
@@ -480,8 +481,8 @@ def prepare_conf_publisher():
 
     config = prepare_conf()
 
-    # always enable debug prints from urllib
-    config.confluence_publish_debug = True
+    # always enable debug prints from urllib3
+    config.confluence_publish_debug = PublishDebug.urllib3
 
     # define a timeout to ensure publishing tests do not block
     config.confluence_timeout = 5