Feature/Sort fields and parent mapping

.gitignore

@@ -1,3 +1,6 @@
 *.pyc
 *~
 .tox
+.idea
+*.egg-info
+dist

django_elasticsearch/managers.py

@@ -14,6 +14,8 @@
 
 from django_elasticsearch.query import EsQueryset
 from django_elasticsearch.client import es_client
+import inspect
+
 
 # Note: we use long/double because different db backends
 # could store different sizes of numerics ?
@@ -65,7 +67,7 @@ def __init__(self, k):
             self.model = k
 
         self.serializer = None
-        self._mapping = None
+        self._full_mapping = None
 
     def get_index(self):
         return self.model.Elasticsearch.index
@@ -121,11 +123,26 @@ def deserialize(self, source):
 
     @needs_instance
     def do_index(self):
-        body = self.serialize()
-        es_client.index(index=self.index,
-                        doc_type=self.doc_type,
-                        id=self.instance.id,
-                        body=body)
+        kwargs = {
+            'index': self.index,
+            'doc_type': self.doc_type,
+            'id': self.instance.id,
+            'body': self.serialize()
+        }
+
+        parent = self.model.Elasticsearch.parent_model
+        if parent:
+            parent.es.create_index()
+            parent_instance = None
+            for member in inspect.getmembers(self.instance):
+                value = member[1]
+                if isinstance(value, parent):
+                    parent_instance = value
+                    break
+            parent_instance.es.do_index()
+            kwargs.update({'parent': parent_instance.id})
+
+        es_client.index(**kwargs)
 
     @needs_instance
     def delete(self):
@@ -253,6 +270,8 @@ def make_mapping(self):
         """
         mappings = {}
 
+        sort_fields = self.model.Elasticsearch.sort_fields
+
         for field_name in self.get_fields():
             try:
                 field = self.model._meta.get_field(field_name)
@@ -274,6 +293,16 @@ def make_mapping(self):
                 mapping.update(self.model.Elasticsearch.mappings[field_name])
             except (AttributeError, KeyError, TypeError):
                 pass
+
+            if sort_fields is not None and field_name in sort_fields:
+                if 'type' in mapping and mapping.get('type') == 'string':
+                    mapping['fields'] = {
+                        'raw': {
+                            'type': 'string',
+                            'index': 'not_analyzed'
+                        }
+                    }
+
             mappings[field_name] = mapping
 
         # add a completion mapping for every auto completable field
@@ -282,20 +311,29 @@ def make_mapping(self):
             complete_name = "{0}_complete".format(field_name)
             mappings[complete_name] = {"type": "completion"}
 
-        return {
+        es_mapping = {
             self.doc_type: {
                 "properties": mappings
             }
         }
 
-    def get_mapping(self):
-        if self._mapping is None:
-            # TODO: could be done once for every index/doc_type ?
-            full_mapping = es_client.indices.get_mapping(index=self.index,
-                                                         doc_type=self.doc_type)
-            self._mapping = full_mapping[self.index]['mappings'][self.doc_type]['properties']
+        parent = self.model.Elasticsearch.parent_model
+        if parent:
+            parent.es.create_index()
+            es_mapping[self.doc_type]['_parent'] = {
+                'type': parent.es.doc_type
+            }
+
+        return es_mapping
+
+    def get_full_mapping(self):
+        if self._full_mapping is None:
+            self._full_mapping = es_client.indices.get_mapping(index=self.index, doc_type=self.doc_type)
 
-        return self._mapping
+        return self._full_mapping
+
+    def get_mapping(self):
+        return self.get_full_mapping()[self.index]['mappings'][self.doc_type]['properties']
 
     def get_settings(self):
         """
@@ -332,7 +370,6 @@ def create_index(self, ignore=True):
         body = {}
         if hasattr(settings, 'ELASTICSEARCH_SETTINGS'):
             body['settings'] = settings.ELASTICSEARCH_SETTINGS
-
         es_client.indices.create(self.index,
                                  body=body,
                                  ignore=ignore and 400)

django_elasticsearch/models.py

@@ -31,6 +31,8 @@ class Elasticsearch:
         mapping = None
         serializer_class = EsJsonSerializer
         fields = None
+        sort_fields = None
+        parent_model = None
         facets_limit = 10
         facets_fields = None
         # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-suggesters-term.html

django_elasticsearch/tests/test_indexable.py

@@ -1,16 +1,13 @@
 # -*- coding: utf-8 -*-
-from elasticsearch import NotFoundError
-
+from django import get_version
 from django.test import TestCase
 from django.test.utils import override_settings
+from elasticsearch import NotFoundError
+from test_app.models import TestModel, Test2Model
 
 from django_elasticsearch.managers import es_client
 from django_elasticsearch.tests.utils import withattrs
 
-from test_app.models import TestModel
-
-from django import get_version
-
 
 class EsIndexableTestCase(TestCase):
     def setUp(self):
@@ -95,8 +92,8 @@ def test_fuzziness(self):
             "default": "test_analyzer",
             "analyzer": {
                 "test_analyzer": {
-                "type": "custom",
-                "tokenizer": "standard",
+                    "type": "custom",
+                    "tokenizer": "standard",
                 }
             }
         }
@@ -131,7 +128,7 @@ def test_auto_completion(self):
 
     @withattrs(TestModel.Elasticsearch, 'fields', ['username', 'date_joined'])
     def test_get_mapping(self):
-        TestModel.es._mapping = None
+        TestModel.es._full_mapping = None
         TestModel.es.flush()
         TestModel.es.do_update()
 
@@ -140,7 +137,51 @@ def test_get_mapping(self):
 
         # Reset the eventual cache on the Model mapping
         mapping = TestModel.es.get_mapping()
-        TestModel.es._mapping = None
+        TestModel.es._full_mapping = None
+        self.assertEqual(expected, mapping)
+
+    @withattrs(TestModel.Elasticsearch, 'fields', ['username', 'date_joined'])
+    def test_get_full_mapping(self):
+        TestModel.es._full_mapping = None
+        TestModel.es.flush()
+        TestModel.es.do_update()
+
+        expected = {u'django-test': {u'mappings': {u'test-doc-type': {u'properties': {
+            u'date_joined': {u'format': u'dateOptionalTime', u'type': u'date'},
+            u'username': {u'index': u'not_analyzed', u'type': u'string'}
+        }}}}}
+
+        # Reset the eventual cache on the Model mapping
+        mapping = TestModel.es.get_full_mapping()
+        TestModel.es._full_mapping = None
+        self.assertEqual(expected, mapping)
+
+    @withattrs(TestModel.Elasticsearch, 'fields', ['username', 'date_joined'])
+    @withattrs(Test2Model.Elasticsearch, 'doc_type', 'test-2-doc-type')
+    @withattrs(Test2Model.Elasticsearch, 'fields', ['text', 'email'])
+    @withattrs(Test2Model.Elasticsearch, 'parent_model', TestModel)
+    def test_get_parent_mapping(self):
+        self.maxDiff = None
+        Test2Model.es._full_mapping = None
+        Test2Model.es.flush()
+        Test2Model.es.do_update()
+
+        expected = {u'django-test': {u'mappings': {u'test-2-doc-type': {
+            u'properties': {
+                u'text': {u'type': u'string'},
+                u'email': {u'type': u'string'}
+            },
+            u'_routing': {
+                u'required': True
+            },
+            u'_parent': {
+                u'type': u'test-doc-type'
+            }
+        }}}}
+
+        # Reset the eventual cache on the Model mapping
+        mapping = Test2Model.es.get_full_mapping()
+        Test2Model.es._full_mapping = None
         self.assertEqual(expected, mapping)
 
     def test_get_settings(self):
@@ -171,8 +212,8 @@ def test_diff(self):
 
         expected = {
             u'first_name': {
-            'es': u'woot',
-            'db': u'pouet'
+                'es': u'woot',
+                'db': u'pouet'
             }
         }
 
@@ -209,7 +250,7 @@ def setUp(self):
         post_save.connect(es_save_callback)
         post_delete.connect(es_delete_callback)
         post_migrate.connect(es_syncdb_callback)
-        
+
         if int(get_version()[2]) >= 6:
             sender = app
         else:

django_elasticsearch/tests/test_qs.py

@@ -113,7 +113,7 @@ def test_suggestions(self):
             u'last_name': [
                 {u'length': 5,
                  u'offset': 0,
-                 u'options': [{u'freq': 3,
+                 u'options': [{u'freq': 6,
                                u'score': 0.8,
                                u'text': u'smith'}],
                  u'text': u'smath'}]}
@@ -220,7 +220,7 @@ def test_isnull_lookup(self):
     @withattrs(TestModel.Elasticsearch, 'fields', ['id', 'date_joined_exp'])
     def test_sub_object_lookup(self):
         TestModel.es._fields = None
-        TestModel.es._mapping = None
+        TestModel.es._full_mapping = None
         TestModel.es.flush()  # update the mapping
         time.sleep(2)
 
@@ -232,14 +232,14 @@ def test_sub_object_lookup(self):
         self.assertEqual(qs.count(), 4)
 
     def test_nested_filter(self):
-        TestModel.es._mapping = None
+        TestModel.es._full_mapping = None
         qs = TestModel.es.filter(groups=self.group)
         self.assertEqual(qs.count(), 1)
 
     @withattrs(TestModel.Elasticsearch, 'fields', ['id', 'date_joined_exp'])
     def test_filter_date_range(self):
         TestModel.es._fields = None
-        TestModel.es._mapping = None
+        TestModel.es._full_mapping = None
         TestModel.es.flush()  # update the mapping
         time.sleep(2)
 
@@ -303,7 +303,7 @@ def test_chain_filter_exclude(self):
     @withattrs(TestModel.Elasticsearch, 'mappings', {})
     def test_contains(self):
         TestModel.es._fields = None
-        TestModel.es._mapping = None
+        TestModel.es._full_mapping = None
         TestModel.es.flush()  # update the mapping, username is now analyzed
         time.sleep(2)  # TODO: flushing is not immediate, find a better way
         contents = TestModel.es.filter(username__contains='woot').deserialize()

readme.md

@@ -154,6 +154,10 @@ Each EsIndexable model receive an Elasticsearch class that contains its options
     Defaults to None  
     The fields on which to activate auto-completion (needs a specific mapping).
 
+* **sort_fields**  
+    Defaults to None  
+    A list of fields that will receive a specific mapping for sorting purposes. (See [this](https://www.elastic.co/guide/en/elasticsearch/guide/1.x/multi-fields.html))
+
 API
 ===
 

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name="django-elasticsearch",
-    version="0.5",
+    version="0.5.2",
     description="Simple wrapper around py-elasticsearch to index/search a django Model.",
     author="Robin Tissot",
     url="https://github.com/liberation/django_elasticsearch",

test_project/tox.ini

@@ -15,7 +15,7 @@ deps =
     django16: django>=1.6, <1.7
     django17: django>=1.7, <1.8
     django18: django>=1.8, <1.9
-    django19: django>=1.9, <2.0
+    django19: django>=1.9, <1.10
     django{14,16,17}: djangorestframework>=2.4, <3.0
     django{18,19}: djangorestframework>3.0, <3.2
     -r../requirements.txt