Sort methods in DocElement classes. Add a base class. (#1352)

This is the part of #1341 that I would like to include in the release.
Apart from a small tidy-up of the order of the methods and where are
stored some pattern constants, the proposed change allows to process the
documentation in a more similar fashion as Sphinx does, without limit of
"levels".
The rest of the code in #1341 can be moved to another repo to be worked
out independently.

mathics/doc/doc_entries.py

@@ -9,6 +9,7 @@
 
 import logging
 import re
+from abc import ABC
 from os import getenv
 from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Sequence, Tuple
 
@@ -68,6 +69,12 @@
 # Preserve space before and after in-line code variables.
 LATEX_RE = re.compile(r"(\s?)\$(\w+?)\$(\s?)")
 
+LATEX_DISPLAY_EQUATION_RE = re.compile(r"(?m)(?<!\\)\$\$([\s\S]+?)(?<!\\)\$\$")
+LATEX_INLINE_EQUATION_RE = re.compile(r"(?m)(?<!\\)\$([\s\S]+?)(?<!\\)\$")
+LATEX_HREF_RE = re.compile(r"(?s)\\href\{(?P<content>.*?)\}\{(?P<text>.*?)\}")
+LATEX_URL_RE = re.compile(r"(?s)\\url\{(?P<content>.*?)\}")
+
+
 LIST_ITEM_RE = re.compile(r"(?s)<li>(.*?)(?:</li>|(?=<li>)|$)")
 LIST_RE = re.compile(r"(?s)<(?P<tag>ul|ol)>(?P<content>.*?)</(?P=tag)>")
 MATHICS_RE = re.compile(r"(?<!\\)\'(.*?)(?<!\\)\'")
@@ -259,6 +266,37 @@ def parse_docstring_to_DocumentationEntry_items(
     return items
 
 
+class BaseDocElement(ABC):
+    """Base class for elements of the documentation system."""
+
+    def get_ancestors(self) -> list:
+        """
+        Get a list of the DocElements such that
+        each element is the parent of the following.
+        """
+        ancestors = []
+        parent = self.parent
+        while isinstance(parent, BaseDocElement):
+            ancestors.append(parent)
+            parent = parent.parent
+
+        ancestors = ancestors[::-1]
+        return ancestors
+
+    def get_children(self) -> list:
+        raise NotImplementedError
+
+    @property
+    def parent(self):
+        "the container where the element is"
+        raise NotImplementedError
+
+    @parent.setter
+    def parent(self, value):
+        "the container where the section is"
+        raise TypeError("parent is a read-only property")
+
+
 class DocTest:
     """
     Class to hold a single doctest.
@@ -277,6 +315,14 @@ class DocTest:
       `|`  Prints output.
     """
 
+    index: int
+    outs: List[_Out]
+    test: str
+    result: str
+    private: bool
+    ignore: bool
+    _key: Optional[tuple]
+
     def __init__(
         self,
         index: int,
@@ -302,7 +348,7 @@ def strip_sentinal(line: str):
 
         self.index = index
         self.outs: List[_Out] = []
-        self.result = None
+        result_value = None
 
         # Private test cases are executed, but NOT shown as part of the docs
         self.private = testcase[0] == "#"
@@ -317,7 +363,7 @@ def strip_sentinal(line: str):
             self.ignore = False
 
         self.test = strip_sentinal(testcase[1])
-        self._key: Optional[tuple] = key_prefix + (index,) if key_prefix else None
+        self._key = key_prefix + (index,) if key_prefix else None
 
         outs = testcase[2].splitlines()
         for line in outs:
@@ -328,8 +374,8 @@ def strip_sentinal(line: str):
                     if text.startswith(" "):
                         text = text[1:]
                     text = "\n" + text
-                    if self.result is not None:
-                        self.result += text
+                    if result_value is not None:
+                        result_value += text
                     elif self.outs:
                         self.outs[-1].text += text
                     continue
@@ -340,11 +386,12 @@ def strip_sentinal(line: str):
                 symbol, text = match.group(1), match.group(2)
                 text = text.strip()
                 if symbol == "=":
-                    self.result = text
+                    result_value = text
                 elif symbol == ":":
                     self.outs.append(Message("", "", text))
                 elif symbol == "|":
                     self.outs.append(Print(text))
+        self.result = result_value or ""
 
     def __str__(self) -> str:
         return self.test
@@ -356,31 +403,6 @@ def compare(self, result: Optional[str], out: tuple = tuple()) -> bool:
         """
         return self.compare_result(result) and self.compare_out(out)
 
-    def compare_result(self, result: Optional[str]):
-        """Compare a result with the expected result"""
-        wanted = self.result
-        # Check result
-        if wanted in ("...", result):
-            return True
-
-        if result is None or wanted is None:
-            return False
-        result_list = result.splitlines()
-        wanted_list = wanted.splitlines()
-        if result_list == [] and wanted_list == ["#<--#"]:
-            return True
-
-        if len(result_list) != len(wanted_list):
-            return False
-
-        for res, want in zip(result_list, wanted_list):
-            wanted_re = re.escape(want.strip())
-            wanted_re = wanted_re.replace("\\.\\.\\.", ".*?")
-            wanted_re = f"^{wanted_re}$"
-            if not re.match(wanted_re, res.strip()):
-                return False
-        return True
-
     def compare_out(self, outs: tuple = tuple()) -> bool:
         """Compare messages and warnings produced during the evaluation of
         the test with the expected messages and warnings."""
@@ -407,6 +429,32 @@ def tabs_to_spaces(val):
 
         return True
 
+    def compare_result(self, result: Optional[str]):
+        """Compare a result with the expected result"""
+        wanted = self.result
+        # Check result
+        if wanted in ("...", result):
+            return True
+
+        if result is None:
+            return wanted == ""
+
+        result_list = result.splitlines()
+        wanted_list = wanted.splitlines()
+        if result_list == [] and wanted_list == ["#<--#"]:
+            return True
+
+        if len(result_list) != len(wanted_list):
+            return False
+
+        for res, want in zip(result_list, wanted_list):
+            wanted_re = re.escape(want.strip())
+            wanted_re = wanted_re.replace("\\.\\.\\.", ".*?")
+            wanted_re = f"^{wanted_re}$"
+            if not re.match(wanted_re, res.strip()):
+                return False
+        return True
+
     @property
     def key(self):
         """key identifier of the test"""
@@ -429,6 +477,9 @@ def __init__(self):
         self.tests = []
         self.text = ""
 
+    def __str__(self) -> str:
+        return "\n".join(str(test) for test in self.tests)
+
     def get_tests(self) -> list:
         """
         Returns lists test objects.
@@ -439,9 +490,6 @@ def get_tests(self) -> list:
     #     """Returns True if this test is "private" not supposed to be visible as example documentation."""
     #     return all(test.private for test in self.tests)
 
-    def __str__(self) -> str:
-        return "\n".join(str(test) for test in self.tests)
-
     def test_indices(self) -> List[int]:
         """indices of the tests"""
         return [test.index for test in self.tests]
@@ -481,7 +529,7 @@ def test_indices(self) -> List[int]:
 
 
 # Former XMLDoc
-class DocumentationEntry:
+class DocumentationEntry(BaseDocElement):
     """
     A class to hold the content of a documentation entry,
     in our custom XML-like format.
@@ -540,6 +588,48 @@ def _set_classes(self):
     def __str__(self) -> str:
         return "\n\n".join(str(item) for item in self.items)
 
+    def get_children(self) -> list:
+        """Get children"""
+        return []
+
+    def get_tests(self) -> List["DocTest"]:
+        """retrieve a list of tests in the documentation entry"""
+        tests = []
+        for item in self.items:
+            tests.extend(item.get_tests())
+        return tests
+
+    @property
+    def parent(self):
+        "the container where the element is"
+        return self._parent
+
+    @parent.setter
+    def parent(self, value):
+        "the container where the section is"
+        raise TypeError("parent is a read-only property")
+
+    def set_parent_path(self, parent):
+        """Set the parent path"""
+        self._parent = parent
+        if parent is None:
+            return self
+
+        path_objs = parent.get_ancestors()[1:] + [parent]
+        path = [element.title for element in path_objs]
+        # Set the key on each test
+        for test in self.get_tests():
+            assert test.key is None
+            # For backward compatibility, we need
+            # to reduce this to three fields.
+            # TODO: remove me and ensure that this
+            # works here and in Mathics Django
+            if len(path) > 3:
+                path = path[:2] + [path[-1]]
+            test.key = tuple(path) + (test.index,)
+
+        return self
+
     def text(self) -> str:
         """text version of the documentation entry"""
         # used for introspection
@@ -567,39 +657,6 @@ def text(self) -> str:
         item = item.replace("_DOLARSIGN_", "$")
         return item
 
-    def get_tests(self) -> List["DocTest"]:
-        """retrieve a list of tests in the documentation entry"""
-        tests = []
-        for item in self.items:
-            tests.extend(item.get_tests())
-        return tests
-
-    def set_parent_path(self, parent):
-        """Set the parent path"""
-        self.path = None
-        path = []
-        while hasattr(parent, "parent"):
-            path = [parent.title] + path
-            parent = parent.parent
-
-        if hasattr(parent, "title"):
-            path = [parent.title] + path
-
-        if path:
-            self.path = path
-            # Set the key on each test
-            for test in self.get_tests():
-                assert test.key is None
-                # For backward compatibility, we need
-                # to reduce this to three fields.
-                # TODO: remove me and ensure that this
-                # works here and in Mathics Django
-                if len(path) > 3:
-                    path = path[:2] + [path[-1]]
-                test.key = tuple(path) + (test.index,)
-
-        return self
-
 
 class Tests:
     """

mathics/doc/latex_doc.py

@@ -14,7 +14,11 @@
     HYPERTEXT_RE,
     IMG_PNG_RE,
     IMG_RE,
+    LATEX_DISPLAY_EQUATION_RE,
+    LATEX_HREF_RE,
+    LATEX_INLINE_EQUATION_RE,
     LATEX_RE,
+    LATEX_URL_RE,
     LIST_ITEM_RE,
     LIST_RE,
     MATHICS_RE,
@@ -58,11 +62,6 @@
 LATEX_CONSOLE_RE = re.compile(r"\\console\{(.*?)\}")
 LATEX_INLINE_END_RE = re.compile(r"(?s)(?P<all>\\lstinline'[^']*?'\}?[.,;:])")
 
-LATEX_DISPLAY_EQUATION_RE = re.compile(r"(?m)(?<!\\)\$\$([\s\S]+?)(?<!\\)\$\$")
-LATEX_INLINE_EQUATION_RE = re.compile(r"(?m)(?<!\\)\$([\s\S]+?)(?<!\\)\$")
-LATEX_HREF_RE = re.compile(r"(?s)\\href\{(?P<content>.*?)\}\{(?P<text>.*?)\}")
-LATEX_URL_RE = re.compile(r"(?s)\\url\{(?P<content>.*?)\}")
-
 
 LATEX_TEXT_RE = re.compile(
     r"(?s)\\text\{([^{}]*?(?:[^{}]*?\{[^{}]*?(?:[^{}]*?\{[^{}]*?\}[^{}]*?)*?"
@@ -132,7 +131,7 @@ def escape_latex(text):
 
     * First the verbatim Python code is extracted.
     * Then,  URLs are and references are collected.
-    * After that, anything sorrounded by '$' or '$$'
+    * After that, anything surrounded by '$' or '$$'
       is processed.
     * Then, some LaTeX special characters, like brackets,
       are escaped.