fixup! fixup! fixup! fixup! fixup! add support for mongodb Client Sid…

…e Field Level Encryption (CSFLE)

docs/api/ming.odm.rst

@@ -4,6 +4,8 @@
 
 .. automodule:: ming.odm.declarative
    :members:
+   :show-inheritance:
+   :inherited-members:
 
 .. automodule:: ming.odm.base
    :members:

docs/encryption.rst

@@ -13,12 +13,15 @@ This section describes how Ming can be used to automatically encrypt and decrypt
 .. _Client-Side Field Level Encryption (CSFLE): https://pymongo.readthedocs.io/en/stable/examples/encryption.html#client-side-field-level-encryption
 
 
-Declarative Field-Level Encryption
+Encryption at the Foundation Level
 ==================================
 
-When declaratively working with models by subclassing the :class:`ming.Document` in the :ref:`ming_baselevel` this is accomplished by pairing a :class:`~ming.encryption.DecryptedField` with a :class:`~ming.metadata.Field`.
+When declaratively working with models by subclassing :class:`~ming.declarative.Document` in the :ref:`ming_baselevel`, you can add field level encryption by pairing a :class:`~ming.encryption.DecryptedField` with a :class:`~ming.metadata.Field`.
 
-A simple example might look like the following.::
+
+A simple example might look like the following.
+
+.. code-block:: python
 
     class UserEmail(Document):
         class __mongometa__:
@@ -30,10 +33,57 @@ A simple example might look like the following.::
         email = DecryptedField(str, 'email_encrypted')
 
 
-Breaking it Down
-========================
+Breaking down DecryptedField
+----------------------------------
+
+This approach requires that you follow a few conventions:
+
+#. The field storing the encrypted data should be configured in the following way:
+
+   * It should be a :class:`~ming.metadata.Field`.
+   * The Field should be of type :class:`~ming.schema.Binary`.
+   * The Field's name should end with `_encrypted`.
+
+#. Next to this should be a corresponding :class:`~ming.encryption.DecryptedField` that will decrypt the data.
+
+   * Its first argument should be the type that you expect the decrypted data to be (`str`, `int`, etc.).
+   * The second argument should be the name of the encrypted field (e.g. `email_encrypted`).
+   * The DecryptedField's name should be the same as the encrypted :class:`~ming.metadata.Field`, but without the `_encrypted` suffix (e.g. `email`).
+
+
+Encryption at the Declarative Level
+========================================
+
+Similarly when working with the higher level of abstraction offered by :class:`~ming.odm.declarative.MappedClass`es, you can add field level encryption by pairing a :class:`~ming.odm.declarative.DecryptedProperty` with a :class:`~ming.odm.property.FieldProperty`
+
+
+A simple example might look like the following.
+
+.. code-block:: python
+    
+        class UserEmail(MappedClass):
+            class __mongometa__:
+                session = session
+                name = 'user_emails'
+            _id = FieldProperty(schema.ObjectId)
+    
+            email_encrypted = FieldProperty(S.Binary, if_missing=None)
+            email = DecryptedProperty(str, 'email_encrypted')
+
+
+Breaking down DecryptedProperty
+----------------------------------
+
+Similarly to the foundation level, this approach requires that you follow a few conventions:
+
+#. The field storing the encrypted data should be configured in the following way:
 
-This approach requires that you follow a few conventions in order to function correctly.
+   * It should be a :class:`~ming.odm.property.FieldProperty`.
+   * The FieldProperty should be of type :class:`~ming.schema.Binary`.
+   * The FieldProperty's name should end with `_encrypted`.
 
-.. 1. Fields  encrypted data must be named with the suffix `_encrypted`.
+#. Next to this should be a :class:`~ming.odm.declarative.DecryptedProperty` that will decrypt the data.
 
+   * Its first argument should be the type that you expect the decrypted data to be (`str`, `int`, etc.).
+   * The second argument should be the name of the encrypted field (e.g. `email_encrypted`).
+   * The DecryptedProperty's name should be the same as the encrypted :class:`~ming.odm.declarative.DecryptedProperty`, but without the `_encrypted` suffix (e.g. `email`).

ming/datastore.py

@@ -237,7 +237,7 @@ def encr(self, s: str | None, _first_attempt=True, provider='local') -> bytes |
         if s is None:
             return None
         try:
-            key_alt_name = self.encryption.get_key_alt_name(provider)
+            key_alt_name = self.encryption._get_key_alt_name(provider)
             return self.encryptor.encrypt(s, Algorithm.AEAD_AES_256_CBC_HMAC_SHA_512_Deterministic,
                                      key_alt_name=key_alt_name)
         except (EncryptionError, MongoCryptError) as e:

ming/encryption.py

@@ -1,12 +1,10 @@
 from __future__ import annotations
 
-from functools import lru_cache
 from typing import TYPE_CHECKING, TypeVar, Generic
 
 from ming.utils import classproperty
 
 if TYPE_CHECKING:
-    from pymongo import MongoClient
     import ming.datastore
 
 
@@ -17,8 +15,9 @@ class MingEncryptionError(Exception):
 class EncryptionConfig:
     """
     A class to hold the encryption configuration for a ming datastore.
-    :param encryption_config: a dictionary that closely resembles various features of the MongoDB
-    encryption that we support.
+
+    :param config: a dictionary that closely resembles various features of the MongoDB
+        encryption that we support.
     """
 
     def __init__(self, config: dict):
@@ -69,7 +68,7 @@ def provider_options(self) -> dict:
         """
         return self._encryption_config.get('provider_options')
 
-    def get_key_alt_name(self, provider='local') -> str:
+    def _get_key_alt_name(self, provider='local') -> str:
         return self.provider_options.get(provider)['key_alt_names'][0]
 
     @property
@@ -118,9 +117,12 @@ def __set__(self, instance: EncryptedMixin, value: T):
 
 
 class EncryptedMixin:
-    """A mixin intended to be used with ming.schema.Document classes to provide encryption.
+    """A mixin intended to be used with :class:`~ming.declarative.Document`
+    or :class:`~ming.odm.declarative.MappedClass` to provide encryption.
     All configuration is handled by an instance of a :class:`ming.encryption.EncryptionConfig`
-    that is passed to the :class:`ming.datastore.DataStore` instance that the Document is bound to.
+    that is passed to the :class:`ming.datastore.DataStore` instance that the Document/MappedClass is bound to.
+
+    Generally, don't use this directly, but instead call the methods on the Document/MappedClass you're working with.
     """
 
     @classproperty
@@ -191,7 +193,9 @@ def encrypt_some_fields(cls, data: dict) -> dict:
         return encrypted_data
 
     def decrypt_some_fields(self) -> dict:
-        # useful for json, removes encrypted fields and uses decrypted forms
+        """
+        Returns a `dict` with raw data. Removes encrypted fields and replaces them with decrypted data. Useful for json.
+        """
         decrypted_data = dict()
         for k in self._field_names:
             if k.endswith('_encrypted'):

ming/odm/declarative.py

@@ -1,9 +1,7 @@
 from ming.metadata import collection, Index
-from ming.datastore import DataStore
 from ming.encryption import EncryptedMixin
-from ming.utils import classproperty
-from .mapper import mapper, Mapper
-from .property import ORMProperty
+from ming.odm.mapper import mapper
+from ming.odm.property import ORMProperty
 
 
 class _MappedClassMeta(type):

ming/odm/mapper.py

@@ -8,7 +8,7 @@
 from ming.session import Session
 from ming.utils import wordwrap
 
-from .base import ObjectState, ObjectState, state, _with_hooks
+from .base import ObjectState, state, _with_hooks
 from .property import FieldProperty
 
 if typing.TYPE_CHECKING:
@@ -439,9 +439,11 @@ def __get__(self, self_, cls=None):
             return self.saving_init(self_)
         else:
             return self.nonsaving_init(self_)
+
+    __call__ = __get__
 
     @classmethod
-    def decorate(cls, mapped_class, mapper):
+    def decorate(cls, mapped_class: type[MappedClass], mapper: Mapper):
         old_init = mapped_class.__init__
         if isinstance(old_init, cls):
             mapped_class.__init__ = cls(mapper, old_init.func)

ming/odm/mapper.pyi

@@ -1,24 +1,87 @@
-from typing import type_check_only, TypeVar, Generic, Optional, Union, Any, Dict, overload
+from typing import Generator, List, type_check_only, TypeVar, Generic, Optional, Union, Any, overload, Type
 
 from bson import ObjectId
 from pymongo.command_cursor import CommandCursor
+import pymongo.results
 
-from ming.base import Cursor
 
-M = TypeVar('M')
+from ming import Document, Cursor, Session
+from ming.base import Object
+from ming.odm import MappedClass, FieldProperty, ODMSession
+from ming.odm.base import ObjectState
+
+TMappedClass = TypeVar('M', bound=MappedClass)
+TDocument = TypeVar('D', bound=Document)
+
+def mapper(cls: Type[TMappedClass], collection: TDocument = None, session: Session = None, **kwargs) -> Mapper: ...
+
+class Mapper(Generic[TMappedClass]):
+
+    properties: List[FieldProperty]
+    property_index: dict[str, FieldProperty]
+    collection: Type[TDocument]
+    mapped_class: Type[TMappedClass]
+    session: Session
+    extensions: list
+    options: Object
+
+    def __init__(self, mapped_class: Type[TMappedClass], collection: Type[Document], session: Session, **kwargs): ...
+
+    @classmethod
+    def replace_session(cls, session) -> None: ...
+
+    def insert(self, obj: MappedClass, state: ObjectState, session: ODMSession, **kwargs) -> pymongo.results.InsertOneResult: ...
+
+    def update(self, obj: MappedClass, state: ObjectState, session: ODMSession, **kwargs) -> ObjectId: ...
+
+    def delete(self, obj: MappedClass, state: ObjectState, session: ODMSession, **kwargs) -> pymongo.results.DeleteResult: ...
+
+    def remove(self, session: ODMSession, *args, **kwargs) -> pymongo.results.DeleteResult: ...
+
+    def create(self, doc, options, remake=True) -> TMappedClass: ...
+
+    def base_mappers(self) -> Generator[Mapper]: ...
+
+    def all_properties(self) -> Generator[FieldProperty]: ...
+
+    @classmethod
+    def by_collection(cls, collection_class: Type[TDocument]) -> Mapper[TMappedClass]: ...
+
+    @classmethod
+    def by_class(cls, mapped_class: Type[TMappedClass]) -> Mapper[TMappedClass]: ...
+
+    @classmethod
+    def by_classname(cls, name: str) -> Mapper[TMappedClass]: ...
+
+    @classmethod
+    def all_mappers(cls) -> List[Mapper[TMappedClass]]: ...
+
+    @classmethod
+    def compile_all(cls): ...
+
+    @classmethod
+    def clear_all(cls): ...
+
+    @classmethod
+    def ensure_all_indexes(cls): ...
+
+    def compile(self): ...
+
+    def update_partial(self, session: ODMSession, *args, **kwargs) -> pymongo.results.UpdateResult: ...
+
 
 MongoFilter = dict
 ChangeResult = dict
-class _ClassQuery(Generic[M]):
+class _ClassQuery(Generic[TMappedClass]):
     # proxies most of these from Session
-    def get(self, _id: Union[ObjectId|Any] = None, **kwargs) -> Optional[M]: ...
-    def find(self, filter: MongoFilter = None, *args, **kwargs) -> Cursor[M]: ...
-    def find_by(self, filter: MongoFilter = None, *args, **kwargs) -> Cursor[M]: ...
+    def get(self, _id: Union[ObjectId|Any] = None, **kwargs) -> Optional[TMappedClass]: ...
+    def find(self, filter: MongoFilter = None, *args, **kwargs) -> Cursor[TMappedClass]: ...
+    def find_by(self, filter: MongoFilter = None, *args, **kwargs) -> Cursor[TMappedClass]: ...
     def remove(self, spec_or_id: Union[MongoFilter, ObjectId] = None, **kwargs) -> ChangeResult: ...
     def count(self) -> int: ...
-    def find_one_and_update(self, **kwargs) -> M: ...
-    def find_one_and_replace(self, **kwargs) -> M: ...
-    def find_one_and_delete(self, **kwargs) -> M: ...
+    def find_one_and_update(self, **kwargs) -> TMappedClass: ...
+    def find_one_and_replace(self, **kwargs) -> TMappedClass: ...
+    def find_one_and_delete(self, **kwargs) -> TMappedClass: ...
     def update_partial(self, filter: MongoFilter, fields: dict, **kwargs) -> ChangeResult: ...
     def aggregate(self, pipeline: list, **kwargs) -> CommandCursor: ...
     def distinct(self, key: str, filter: MongoFilter | None = None, **kwargs) -> list: ...
@@ -30,7 +93,7 @@ class _InstQuery(object):
     def delete(self) -> ChangeResult: ...
 
 @type_check_only
-class Query(_ClassQuery[M], _InstQuery):
+class Query(_ClassQuery[TMappedClass], _InstQuery):
 
     @overload  # from _ClassQuery
     def update(self, spec: MongoFilter, fields: dict, **kwargs) -> ChangeResult: ...

ming/tests/test_datastore.py

@@ -1,8 +1,7 @@
-import os
 import sys
 from unittest import TestCase, main
 
-from unittest.mock import patch, Mock
+from unittest.mock import patch
 from pymongo.errors import ConnectionFailure
 
 import ming
@@ -11,7 +10,6 @@
 from ming import create_datastore, create_engine
 from ming.exc import MingConfigError
 from ming.datastore import Engine
-from ming.tests import EncryptionConfigHelper, make_encryption_key
 
 
 class DummyConnection: