Merge pull request #29 from brandicted/develop

release 0.2.1

docs/source/changelog.rst

@@ -1,6 +1,8 @@
 Changelog
 =========
 
+* :release:`0.2.1 <2015-05-27>`
+
 * :release:`0.2.0 <2015-04-07>`
 * :feature:`-` Relationship indexing
 

nefertari_mongodb/__init__.py

@@ -4,7 +4,7 @@
 
 from .documents import (
     BaseDocument, ESBaseDocument, BaseMixin,
-    get_document_cls)
+    get_document_cls, get_document_classes)
 from .serializers import JSONEncoder, ESJSONSerializer
 from .metaclasses import ESMetaclass
 from .utils import (

nefertari_mongodb/documents.py

@@ -12,7 +12,12 @@
 from .metaclasses import ESMetaclass, DocumentMetaclass
 from .fields import (
     DateTimeField, IntegerField, ForeignKeyField, RelationshipField,
-    DictField, ListField)
+    DictField, ListField, ChoiceField, ReferenceField, StringField,
+    TextField, UnicodeField, UnicodeTextField,
+    IdField, BooleanField, BinaryField, DecimalField, FloatField,
+    BigIntegerField, SmallIntegerField, IntervalField, DateField,
+    TimeField
+)
 
 
 log = logging.getLogger(__name__)
@@ -25,6 +30,22 @@ def get_document_cls(name):
         raise ValueError('`%s` does not exist in mongo db' % name)
 
 
+def get_document_classes():
+    """ Get all defined not abstract document classes
+
+    Class is assumed to be non-abstract if its `_meta['abstract']` is
+    defined and False.
+    """
+    document_classes = {}
+    registry = mongo.base.common._document_registry.copy()
+    for model_name, model_cls in registry.items():
+        _meta = getattr(model_cls, '_meta', {})
+        abstract = _meta.get('abstract', True)
+        if not abstract:
+            document_classes[model_name] = model_cls
+    return document_classes
+
+
 def process_lists(_dict):
     for k in _dict:
         new_k, _, _t = k.partition('__')
@@ -42,6 +63,33 @@ def process_bools(_dict):
     return _dict
 
 
+TYPES_MAP = {
+    StringField: {'type': 'string'},
+    TextField: {'type': 'string'},
+    UnicodeField: {'type': 'string'},
+    UnicodeTextField: {'type': 'string'},
+    mongo.fields.ObjectIdField: {'type': 'string'},
+    ForeignKeyField: {'type': 'string'},
+    IdField: {'type': 'string'},
+
+    BooleanField: {'type': 'boolean'},
+    BinaryField: {'type': 'object'},
+    DictField: {'type': 'object'},
+
+    DecimalField: {'type': 'double'},
+    FloatField: {'type': 'double'},
+
+    IntegerField: {'type': 'long'},
+    BigIntegerField: {'type': 'long'},
+    SmallIntegerField: {'type': 'long'},
+    IntervalField: {'type': 'long'},
+
+    DateTimeField: {'type': 'date', 'format': 'dateOptionalTime'},
+    DateField: {'type': 'date', 'format': 'dateOptionalTime'},
+    TimeField: {'type': 'date', 'format': 'HH:mm:ss'},
+}
+
+
 class BaseMixin(object):
     """ Represents mixin class for models.
 
@@ -64,6 +112,38 @@ class BaseMixin(object):
     _type = property(lambda self: self.__class__.__name__)
     Q = mongo.Q
 
+    @classmethod
+    def get_es_mapping(cls):
+        """ Generate ES mapping from model schema. """
+        from nefertari.elasticsearch import ES
+        ignored_types = set([
+            ReferenceField,
+            RelationshipField,
+        ])
+        properties = {}
+        mapping = {
+            ES.src2type(cls.__name__): {
+                'properties': properties
+            }
+        }
+        fields = cls._fields.copy()
+        fields['id'] = fields.get(cls.pk_field())
+
+        for name, field in fields.items():
+            if isinstance(field, ChoiceField):
+                field = field._real_field
+            field_type = type(field)
+            if field_type is ListField:
+                field_type = field.item_type
+            if field_type in ignored_types:
+                continue
+            if field_type not in TYPES_MAP:
+                continue
+            properties[name] = TYPES_MAP[field_type]
+
+        properties['_type'] = {'type': 'string'}
+        return mapping
+
     @classmethod
     def autogenerate_for(cls, model, set_to):
         """ Setup `post_save` event handler.
@@ -297,7 +377,6 @@ def _update(self, params, **kw):
                 self.update_iterables(value, key, unique=True, save=False)
             else:
                 setattr(self, key, value)
-
         return self.save(**kw)
 
     @classmethod
@@ -334,6 +413,21 @@ def get_by_ids(cls, ids, **params):
         })
         return cls.get_collection(**params)
 
+    @classmethod
+    def get_null_values(cls):
+        """ Get null values of :cls: fields. """
+        null_values = {}
+        field_names = cls._fields.keys()
+        for name in field_names:
+            field = getattr(cls, name)
+            if isinstance(field, RelationshipField):
+                value = []
+            else:
+                value = None
+            null_values[name] = value
+        null_values.pop('id', None)
+        return null_values
+
     def to_dict(self, **kwargs):
         def _process(key, val):
             is_doc = isinstance(val, mongo.Document)
@@ -383,27 +477,33 @@ def split_keys(keys):
                     pos_keys.append(key.strip())
             return pos_keys, neg_keys
 
-        def update_dict():
+        def update_dict(update_params):
             final_value = getattr(self, attr, {}) or {}
             final_value = final_value.copy()
-            positive, negative = split_keys(params.keys())
+            if update_params is None:
+                update_params = {
+                    '-' + key: val for key, val in final_value.items()}
+            positive, negative = split_keys(update_params.keys())
 
             # Pop negative keys
             for key in negative:
                 final_value.pop(key, None)
 
             # Set positive keys
             for key in positive:
-                final_value[unicode(key)] = params[key]
+                final_value[unicode(key)] = update_params[key]
 
             setattr(self, attr, final_value)
             if save:
                 self.save()
 
-        def update_list():
+        def update_list(update_params):
             final_value = getattr(self, attr, []) or []
             final_value = copy.deepcopy(final_value)
-            keys = params.keys() if isinstance(params, dict) else params
+            if update_params is None:
+                update_params = ['-' + val for val in final_value]
+            keys = (update_params.keys() if isinstance(update_params, dict)
+                    else update_params)
             positive, negative = split_keys(keys)
 
             if not (positive + negative):
@@ -422,10 +522,10 @@ def update_list():
                 self.save()
 
         if is_dict:
-            update_dict()
+            update_dict(params)
 
         elif is_list:
-            update_list()
+            update_list(params)
 
     @classmethod
     def expand_with(cls, with_cls, join_on=None, attr_name=None, params={},
@@ -453,6 +553,16 @@ def expand_with(cls, with_cls, join_on=None, attr_name=None, params={},
 
         return objs
 
+    def _is_modified(self):
+        """ Determine if instance is modified.
+
+        For instance to be marked as 'modified', it should:
+          * Have PK field set (not newly created)
+          * Have changed fields
+        """
+        modified = bool(self._get_changed_fields())
+        return modified
+
 
 class BaseDocument(BaseMixin, mongo.Document):
     __metaclass__ = DocumentMetaclass
@@ -464,6 +574,11 @@ class BaseDocument(BaseMixin, mongo.Document):
         'abstract': True,
     }
 
+    def _bump_version(self):
+        if self._is_modified():
+            self.updated_at = datetime.utcnow()
+            self._version += 1
+
     def save(self, *arg, **kw):
         """
         Force insert document in creation so that unique constraits are
@@ -474,9 +589,7 @@ def save(self, *arg, **kw):
         kw['force_insert'] = self._created
 
         sync_backref = kw.pop('sync_backref', True)
-        if self._get_changed_fields():
-            self.updated_at = datetime.utcnow()
-            self._version += 1
+        self._bump_version()
         try:
             super(BaseDocument, self).save(*arg, **kw)
         except (mongo.NotUniqueError, mongo.OperationError) as e:
@@ -524,15 +637,27 @@ def validate(self, *arg, **kw):
                 'Resource `%s`: %s' % (self.__class__.__name__, e),
                 extra={'data': e})
 
-    def clean(self):
-        """ Override `clean` method to apply each field's processors
-        before running validation.
+    def clean(self, force_all=False):
+        """ Override `clean` method to apply field processors to changed
+        fields before running validation.
+
+        Note that at this stage, field values are in the exact same state
+        you posted/set them. E.g. if you set time_field='11/22/2000',
+        self.time_field will be equal to '11/22/2000' here.
         """
-        for name, field in self._fields.items():
+        if self._created or force_all:  # New object
+            changed_fields = self._fields.keys()
+        else:
+            # Apply processors to updated fields only
+            changed_fields = self._get_changed_fields()
+
+        for name in changed_fields:
+            field = self._fields[name]
             if hasattr(field, 'apply_processors'):
-                value = getattr(self, name)
-                value = field.apply_processors(value)
-                setattr(self, name, value)
+                new_value = getattr(self, name)
+                processed_value = field.apply_processors(
+                    instance=self, new_value=new_value)
+                setattr(self, name, processed_value)
 
 
 class ESBaseDocument(BaseDocument):

nefertari_mongodb/fields.py

@@ -70,10 +70,10 @@ def __init__(self, *args, **kwargs):
         self.processors = kwargs.pop('processors', ())
         super(ProcessableMixin, self).__init__(*args, **kwargs)
 
-    def apply_processors(self, value):
+    def apply_processors(self, instance, new_value):
         for proc in self.processors:
-            value = proc(value)
-        return value
+            new_value = proc(instance=instance, new_value=new_value)
+        return new_value
 
 
 class IntegerField(ProcessableMixin, BaseFieldMixin, fields.IntField):
@@ -325,6 +325,7 @@ class ListField(ProcessableMixin, BaseFieldMixin, fields.ListField):
 
     def __init__(self, *args, **kwargs):
         self.list_choices = kwargs.pop('choices', None)
+        self.item_type = kwargs.pop('item_type')
         super(ListField, self).__init__(*args, **kwargs)
 
     def validate(self, value, **kwargs):
@@ -377,7 +378,7 @@ class ReferenceField(BaseFieldMixin, fields.ReferenceField):
     one-to-one relationship. It is also used to create backreferences.
 
     `reverse_rel_field`: string name of a field on the related document.
-        Used when generating backreferences so that fields on each side 
+        Used when generating backreferences so that fields on each side
         know the name of the field on the other side.
     """
     _valid_kwargs = ('document_type', 'dbref', 'reverse_delete_rule')
@@ -409,7 +410,7 @@ def _register_deletion_hook(self, old_object, instance):
         """ Register a backref hook to delete the `instance` from the `old_object`'s
         field to which the `instance` was related beforehand by the backref.
 
-        `instance` is either deleted from the `old_object` field's collection 
+        `instance` is either deleted from the `old_object` field's collection
         or `old_object`'s field, responsible for relationship is set to None.
         This depends on type of the field at `old_object`.
 
@@ -544,17 +545,18 @@ def translate_kwargs(self, kwargs):
         return kwargs
 
 
-class RelationshipField(ListField):
+class RelationshipField(ProcessableMixin, BaseFieldMixin, fields.ListField):
     """ Relationship field meant to be used to create one-to-many relationships.
 
     It is used in the `Relationship` function to generate one-to-many
     relationships. Under the hood it is just a ListField containing
     ReferenceFields.
 
     `reverse_rel_field`: string name of a field on the related document.
-        Used when generating backreferences so that fields on each side 
+        Used when generating backreferences so that fields on each side
         know the name of the field on the other side.
     """
+    _valid_kwargs = ('field',)
     _common_valid_kwargs = (
         'db_field', 'required', 'default', 'unique',
         'unique_with', 'primary_key', 'validation', 'choices',

nefertari_mongodb/metaclasses.py

@@ -73,8 +73,8 @@ def __init__(self, name, bases, attrs):
             # Add new field to `_fields_ordered`
             if (backref_name in target_cls._fields and
                     backref_name not in target_cls._fields_ordered):
-                target_cls._fields_ordered = sorted(
-                    target_cls._fields_ordered + (backref_name,))
+                fields = list(target_cls._fields_ordered) + [backref_name]
+                target_cls._fields_ordered = sorted(fields)
 
             # Set new field as an attribute of target class
             setattr(target_cls, backref_name, backref_field)