Implement Unix Socket Controller (aio-libs#247)

* Decompose Controller
* Implement UnixSocketController
* Test newly-implemented TimeoutError
* Add tests for UnixSocketController
* Shorten some timeouts
* Replace DeprecationWarning for Session.login_data to log.warning
* Rename _test_server to _trigger_server
* Update documentation
* Change doctest runner to Sphinx.ext.doctest
* Fixes for controller.rst
* Also use sphinx-build doctest & html in GA
* Also install flake8-bugbear for the flake8 step in GA
* Update conf.py
* Add --color to sphinx-build
* Add fixture for "safe" socket path (aiohttp/3572)
* Wait until factory _is_ invoked
* Improve Controller.stop()
* Change _dynamic to _dump
* Activate "static" with win32 exclusion
* Move flake8 config to setup.cfg
* Add region..endregion fold marker
* Skip UnixSocketController test & doctest on cygwin
* Make "static" not run on cygwin as well
* Enforce different version in PR
* Update NEWS.rst
* Bump version to 1.4.0a5

.github/workflows/unit-testing-and-coverage.yml

@@ -40,15 +40,15 @@ jobs:
       - name: "flake8 Style Checking"
         # language=bash
         run: |
-          pip install colorama flake8
+          pip install colorama flake8 flake8-bugbear
           python -m flake8 aiosmtpd setup.py housekeep.py release.py
       - name: "Docs Checking"
         # language=bash
         run: |
           pip install colorama pytest pytest-mock sphinx sphinx-autofixture sphinx_rtd_theme
-          pytest -v aiosmtpd/docs
-          sphinx-build --color -b html -d build/.doctree aiosmtpd/docs build/html
-          sphinx-build --color -b man  -d build/.doctree aiosmtpd/docs build/man
+          sphinx-build --color -b doctest -d build/.doctree aiosmtpd/docs build/doctest
+          sphinx-build --color -b html    -d build/.doctree aiosmtpd/docs build/html
+          sphinx-build --color -b man     -d build/.doctree aiosmtpd/docs build/man
       - name: "Static Code Checking"
         # language=bash
         run: |
@@ -61,11 +61,25 @@ jobs:
         shell: bash
         # language=bash
         run: |
-          pip install pytest pytest-mock check-manifest
-          # Fetch master if needed because some test cases need its existence
+          # Strip the "__version__ = " part and the containing quotes (single or double), print,
+          # delete everything else
+          ver_sed='s/^__version__ = (["'"'"'])(.*)\1/\2/p;d'
+          verfile="aiosmtpd/__init__.py"
           if [[ $GITHUB_REF != refs/heads/master ]]; then
+            # We're not in master
+            # Fetch master because some test cases need its existence
             git fetch --no-tags --prune --no-recurse-submodules --depth=1 origin master:master
+            # Verify version in current commit different from master's
+            now_ver=$(sed -r -e "$ver_sed" $verfile)
+            master_ver=$(git show master:$verfile | sed -r -e "$ver_sed")
+            if [[ $now_ver == $master_ver ]]; then
+              echo "### FAIL : Version in PR must be different from version in master!"
+              exit 1
+            else
+              echo "### SUCCESS : Version in PR different from version in master."
+            fi
           fi
+          pip install pytest pytest-mock check-manifest
           pytest -v aiosmtpd/qa
           check-manifest -v
   testing:

.gitignore

@@ -26,6 +26,7 @@ docs/_build/
 nosetests.xml
 __pycache__/
 htmlcov/
+_dynamic/
 _dump/
 coverage.xml
 coverage-*.xml

aiosmtpd/__init__.py

@@ -1,4 +1,4 @@
 # Copyright 2014-2021 The aiosmtpd Developers
 # SPDX-License-Identifier: Apache-2.0
 
-__version__ = "1.4.0a4"
+__version__ = "1.4.0a5"

aiosmtpd/controller.py

@@ -7,11 +7,18 @@
 import ssl
 import threading
 import time
+from abc import ABCMeta, abstractmethod
 from contextlib import ExitStack
+from pathlib import Path
 from socket import AF_INET6, SOCK_STREAM, create_connection, has_ipv6
 from socket import socket as makesock
 from socket import timeout as socket_timeout
-from typing import Any, Coroutine, Dict, Optional
+
+try:
+    from socket import AF_UNIX
+except ImportError:  # pragma: on-not-win32
+    AF_UNIX = None
+from typing import Any, Coroutine, Dict, Optional, Union
 from warnings import warn
 
 from public import public
@@ -82,9 +89,13 @@ def _client_connected_cb(self, reader, writer):
 
 
 @public
-class Controller:
+class BaseThreadedController(metaclass=ABCMeta):
+    """
+    `Documentation can be found here
+    <https://aiosmtpd.readthedocs.io/en/latest/controller.html>`_.
+    """
     server: Optional[AsyncServer] = None
-    server_coro: Coroutine = None
+    server_coro: Optional[Coroutine] = None
     smtpd = None
     _factory_invoked: Optional[threading.Event] = None
     _thread: Optional[threading.Thread] = None
@@ -94,37 +105,31 @@ def __init__(
         self,
         handler,
         loop=None,
-        hostname=None,
-        port=8025,
         *,
-        ready_timeout=1.0,
-        ssl_context: ssl.SSLContext = None,
+        ready_timeout: float = 1.0,
+        ssl_context: Optional[ssl.SSLContext] = None,
         # SMTP parameters
-        server_hostname: str = None,
-        server_kwargs: Dict[str, Any] = None,
+        server_hostname: Optional[str] = None,
         **SMTP_parameters,
     ):
-        """
-        `Documentation can be found here
-        <http://aiosmtpd.readthedocs.io/en/latest/aiosmtpd\
-/docs/controller.html#controller-api>`_.
-        """
         self.handler = handler
-        self.hostname = get_localhost() if hostname is None else hostname
-        self.port = port
-        self.ssl_context = ssl_context
-        self.loop = asyncio.new_event_loop() if loop is None else loop
+        if loop is None:
+            self.loop = asyncio.new_event_loop()
+        else:
+            self.loop = loop
         self.ready_timeout = float(
             os.getenv("AIOSMTPD_CONTROLLER_TIMEOUT", ready_timeout)
         )
-        if server_kwargs:
+        self.ssl_context = ssl_context
+        self.SMTP_kwargs: Dict[str, Any] = {}
+        if "server_kwargs" in SMTP_parameters:
             warn(
                 "server_kwargs will be removed in version 2.0. "
                 "Just specify the keyword arguments to forward to SMTP "
                 "as kwargs to this __init__ method.",
                 DeprecationWarning,
             )
-        self.SMTP_kwargs: Dict[str, Any] = server_kwargs or {}
+            self.SMTP_kwargs = SMTP_parameters.pop("server_kwargs")
         self.SMTP_kwargs.update(SMTP_parameters)
         if server_hostname:
             self.SMTP_kwargs["hostname"] = server_hostname
@@ -150,21 +155,23 @@ def _factory_invoker(self):
         finally:
             self._factory_invoked.set()
 
+    @abstractmethod
+    def _create_server(self) -> Coroutine:
+        raise NotImplementedError  # pragma: nocover
+
+    @abstractmethod
+    def _trigger_server(self):
+        raise NotImplementedError  # pragma: nocover
+
     def _run(self, ready_event):
         asyncio.set_event_loop(self.loop)
         try:
             # Need to do two-step assignments here to ensure IDEs can properly
             # detect the types of the vars. Cannot use `assert isinstance`, because
             # Python 3.6 in asyncio debug mode has a bug wherein CoroWrapper is not
             # an instance of Coroutine
-            srv_coro: Coroutine = self.loop.create_server(
-                self._factory_invoker,
-                host=self.hostname,
-                port=self.port,
-                ssl=self.ssl_context,
-            )
-            self.server_coro = srv_coro
-            srv: AsyncServer = self.loop.run_until_complete(srv_coro)
+            self.server_coro = self._create_server()
+            srv: AsyncServer = self.loop.run_until_complete(self.server_coro)
             self.server = srv
         except Exception as error:  # pragma: on-wsl
             # Usually will enter this part only if create_server() cannot bind to the
@@ -183,24 +190,6 @@ def _run(self, ready_event):
         self.loop.close()
         self.server = None
 
-    def _testconn(self):
-        """
-        Opens a socket connection to the newly launched server, wrapping in an SSL
-        Context if necessary, and read some data from it to ensure that factory()
-        gets invoked.
-        """
-        # IMPORTANT: Windows does not need the next line; for some reasons,
-        # create_connection is happy with hostname="" on Windows, but screams murder
-        # in Linux.
-        # At this point, if self.hostname is Falsy, it most likely is "" (bind to all
-        # addresses). In such case, it should be safe to connect to localhost)
-        hostname = self.hostname or get_localhost()
-        with ExitStack() as stk:
-            s = stk.enter_context(create_connection((hostname, self.port), 1.0))
-            if self.ssl_context:
-                s = stk.enter_context(self.ssl_context.wrap_socket(s))
-            _ = s.recv(1024)
-
     def start(self):
         assert self._thread is None, "SMTP daemon already running"
         self._factory_invoked = threading.Event()
@@ -225,10 +214,9 @@ def start(self):
         # factory() go undetected. To trigger factory() invocation we need to open
         # a connection to the server and 'exchange' some traffic.
         try:
-            self._testconn()
+            self._trigger_server()
         except socket_timeout:
             # We totally don't care of timeout experienced by _testconn,
-            # which _will_ happen if factory() experienced problems.
             pass
         except Exception:
             # Raise other exceptions though
@@ -251,9 +239,111 @@ def _stop(self):
         for task in _all_tasks(self.loop):
             task.cancel()
 
-    def stop(self):
-        assert self._thread is not None, "SMTP daemon not running"
+    def stop(self, no_assert=False):
+        assert no_assert or self._thread is not None, "SMTP daemon not running"
         self.loop.call_soon_threadsafe(self._stop)
-        self._thread.join()
-        self._thread = None
+        if self._thread is not None:
+            self._thread.join()
+            self._thread = None
         self._thread_exception = None
+        self._factory_invoked = None
+        self.server_coro = None
+        self.server = None
+        self.smtpd = None
+
+
+@public
+class Controller(BaseThreadedController):
+    """
+    `Documentation can be found here
+    <https://aiosmtpd.readthedocs.io/en/latest/controller.html>`_.
+    """
+    def __init__(
+        self,
+        handler,
+        hostname: Optional[str] = None,
+        port: int = 8025,
+        loop=None,
+        *,
+        ready_timeout: float = 1.0,
+        ssl_context: ssl.SSLContext = None,
+        # SMTP parameters
+        server_hostname: Optional[str] = None,
+        **SMTP_parameters,
+    ):
+        super().__init__(
+            handler,
+            loop,
+            ready_timeout=ready_timeout,
+            server_hostname=server_hostname,
+            **SMTP_parameters
+        )
+        self.hostname = get_localhost() if hostname is None else hostname
+        self.port = port
+        self.ssl_context = ssl_context
+
+    def _create_server(self) -> Coroutine:
+        return self.loop.create_server(
+            self._factory_invoker,
+            host=self.hostname,
+            port=self.port,
+            ssl=self.ssl_context,
+        )
+
+    def _trigger_server(self):
+        """
+        Opens a socket connection to the newly launched server, wrapping in an SSL
+        Context if necessary, and read some data from it to ensure that factory()
+        gets invoked.
+        """
+        # At this point, if self.hostname is Falsy, it most likely is "" (bind to all
+        # addresses). In such case, it should be safe to connect to localhost)
+        hostname = self.hostname or get_localhost()
+        with ExitStack() as stk:
+            s = stk.enter_context(create_connection((hostname, self.port), 1.0))
+            if self.ssl_context:
+                s = stk.enter_context(self.ssl_context.wrap_socket(s))
+            _ = s.recv(1024)
+
+
+class UnixSocketController(BaseThreadedController):  # pragma: on-win32 on-cygwin
+    """
+    `Documentation can be found here
+    <https://aiosmtpd.readthedocs.io/en/latest/controller.html>`_.
+    """
+    def __init__(
+        self,
+        handler,
+        unix_socket: Optional[Union[str, Path]],
+        loop=None,
+        *,
+        ready_timeout=1.0,
+        ssl_context=None,
+        # SMTP parameters
+        server_hostname: str = None,
+        **SMTP_parameters,
+    ):
+        super().__init__(
+            handler,
+            loop,
+            ready_timeout=ready_timeout,
+            ssl_context=ssl_context,
+            server_hostname=server_hostname,
+            **SMTP_parameters
+        )
+        self.unix_socket = str(unix_socket)
+
+    def _create_server(self) -> Coroutine:
+        return self.loop.create_unix_server(
+            self._factory_invoker,
+            path=self.unix_socket,
+            ssl=self.ssl_context,
+        )
+
+    def _trigger_server(self):
+        with ExitStack() as stk:
+            s: makesock = stk.enter_context(makesock(AF_UNIX, SOCK_STREAM))
+            s.connect(self.unix_socket)
+            if self.ssl_context:
+                s = stk.enter_context(self.ssl_context.wrap_socket(s))
+            _ = s.recv(1024)

aiosmtpd/docs/NEWS.rst

@@ -11,6 +11,8 @@ Added
 * Support for |PROXY Protocol|_ (Closes #174)
 * Example for authentication
 * SSL Support for CLI. See :ref:`the man page <manpage>` for more info. (Closes #172)
+* New :class:`UnixSocketController` class to implement Unix socket-based SMTP server
+  (Closes #114)
 
 .. _`PROXY Protocol`: https://www.haproxy.com/blog/using-haproxy-with-the-proxy-protocol-to-better-secure-your-database/
 .. |PROXY Protocol| replace:: **PROXY Protocol**
@@ -19,6 +21,7 @@ Fixed/Improved
 --------------
 * ``pypy3`` testenv for tox can now run on Windows
 * ``static`` testenv now auto-skipped on Windows
+* Now uses Sphinx's Doctest facility, which is much more flexible than pytest's doctest
 
 
 1.3.2 (2021-02-20)

aiosmtpd/docs/conf.py

@@ -47,21 +47,23 @@ def syspath_insert(pth: Path):
 
 # autoprogramm needs Sphinx>=1.2.2
 # :classmethod: needs Sphinx>=2.1
-needs_sphinx = "2.1"
+# :noindex: needs Sphinx>=3.2
+needs_sphinx = "3.2"
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
     "sphinx.ext.intersphinx",
     "sphinx.ext.autodoc",
+    "sphinx.ext.doctest",
     "sphinx_autofixture",
     "autoprogramm",
     "sphinx_rtd_theme"
 ]
 # IMPORTANT: If you edit this, also edit the following:
 #   - aiosmtpd/docs/RTD-requirements.txt
-#   -
+#   - tox.ini
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
@@ -148,6 +150,12 @@ def syspath_insert(pth: Path):
     "python": ("https://docs.python.org/3", None),
 }
 
+doctest_global_setup = """
+import sys
+in_win32 = sys.platform == "win32"
+in_cygwin = sys.platform == "cygwin"
+"""
+
 # endregion