Rework the serializer to allow easy nested serialization,

also give it a dedicated test case,
see liberation#15.

django_elasticsearch/managers.py

@@ -19,12 +19,16 @@
     # both defaults to 'dateOptionalTime'
     u'DateField': 'date',
     u'DateTimeField': 'date',
+    # u'TimeField': 'string',
     u'FloatField': 'double',
     u'IntegerField': 'long',
     u'PositiveIntegerField': 'long',
     u'PositiveSmallIntegerField': 'short',
     u'SmallIntegerField': 'short',
-    u'ForeignKey': 'object'
+
+    u'ForeignKey': 'object',
+    u'OneToOneField': 'object',
+    u'ManyToManyField': 'object'
 }
 
 
@@ -76,15 +80,14 @@ def doc_type(self):
     def check_cluster(self):
         return es_client.ping()
 
-    def get_serializer(self):
-        if not self.serializer:
-            if isinstance(self.model.Elasticsearch.serializer_class, basestring):
-                module, kls = self.model.Elasticsearch.serializer_class.rsplit(".", 1)
-                mod = importlib.import_module(module)
-                self.serializer = getattr(mod, kls)(self.model)
-            else:
-                self.serializer = self.model.Elasticsearch.serializer_class(self.model)
-        return self.serializer
+    def get_serializer(self, **kwargs):
+        serializer = self.model.Elasticsearch.serializer_class
+        if isinstance(serializer, basestring):
+            module, kls = self.model.Elasticsearch.serializer_class.rsplit(".", 1)
+            mod = importlib.import_module(module)
+            return getattr(mod, kls)(self.model, **kwargs)
+        else:
+            return serializer(self.model, **kwargs)
 
     @needs_instance
     def serialize(self):
@@ -238,7 +241,9 @@ def do_update(self):
         es_client.indices.refresh(index=self.index)
 
     def get_fields(self):
-        model_fields = [f.name for f in self.model._meta.fields]
+        model_fields = [f.name for f in self.model._meta.fields +
+                        self.model._meta.many_to_many]
+
         return self.model.Elasticsearch.fields or model_fields
 
     def make_mapping(self):

django_elasticsearch/serializers.py

@@ -10,8 +10,11 @@ class ModelJsonSerializer(object):
     Default elasticsearch serializer for a django model
     """
 
-    def __init__(self, model):
+    def __init__(self, model, max_depth=2, cur_depth=1):
         self.model = model
+        # used in case of related field on 'self' tu avoid infinite loop
+        self.cur_depth = cur_depth
+        self.max_depth = max_depth
 
     def serialize_field(self, instance, field_name):
         """
@@ -28,22 +31,38 @@ def serialize_field(self, instance, field_name):
         try:
             field = self.model._meta.get_field(field_name)
         except FieldDoesNotExist:
-            # abstract field
-            raise TypeError("The serializer doesn't know how to serialize {0}, "
-                            "please provide it a {1} method."
-                            "".format(field_name, method_name))
+            # abstract field: check for a model property/attribute
+            if hasattr(instance, field_name):
+                return getattr(instance, field_name)
+
+            raise AttributeError("The serializer doesn't know how to serialize {0}, "
+                                 "please provide it a {1} method."
+                                 "".format(field_name, method_name))
 
         field_type_method_name = 'serialize_type_{0}'.format(
             field.__class__.__name__.lower())
         if hasattr(self, field_type_method_name):
             return getattr(self, field_type_method_name)(instance, field_name)
 
         if field.rel:
+            # m2m
             if isinstance(field, ManyToManyField):
                 return [dict(id=r.pk, value=unicode(r))
                         for r in getattr(instance, field.name).all()]
             rel = getattr(instance, field.name)
-            if rel:
+            # fk, oto
+            if rel:  # should be a model instance
+                # check for Elasticsearch.serializer on the related model
+                if self.cur_depth >= self.max_depth:
+                    return
+
+                if hasattr(rel, 'Elasticsearch'):
+                    serializer = rel.es.get_serializer(max_depth=self.max_depth,
+                                                       cur_depth=self.cur_depth + 1)
+
+                    obj = serializer.format(rel)
+                    return obj
+
                 # Use the __unicode__ value of the related model instance.
                 if not hasattr(rel, '__unicode__'):
                     raise AttributeError(
@@ -67,10 +86,8 @@ def deserialize_field(self, source, field_name):
                 pass
         return source.get(field_name)
 
-    def serialize(self, instance):
-        model_fields = [f.name for f in instance._meta.fields]
-        fields = instance.Elasticsearch.fields or model_fields
-
+    def format(self, instance):
+        fields = self.model.es.get_fields()
         obj = dict([(field, self.serialize_field(instance, field))
                     for field in fields])
 
@@ -82,7 +99,10 @@ def serialize(self, instance):
             # heavy processing or db requests.
             obj[suggest_name] = self.serialize_field(instance, field_name)
 
-        return json.dumps(obj,
+        return obj
+
+    def serialize(self, instance):
+        return json.dumps(self.format(instance),
                           default=lambda d: (
                               d.isoformat() if isinstance(d, datetime.datetime)
                               or isinstance(d, datetime.date) else None))

django_elasticsearch/tests/__init__.py

@@ -1,6 +1,11 @@
 from django_elasticsearch.tests.test_views import EsViewTestCase
 from django_elasticsearch.tests.test_qs import EsQuerysetTestCase
 from django_elasticsearch.tests.test_indexable import EsIndexableTestCase
+from django_elasticsearch.tests.test_serializer import EsJsonSerializerTestCase
+from django_elasticsearch.tests.test_restframework import EsRestFrameworkTestCase
 
-__all__ = ['EsQuerysetTestCase', 'EsViewTestCase', 'EsIndexableTestCase', 'EsRestFrameworkTestCase']
-
+__all__ = ['EsQuerysetTestCase',
+           'EsViewTestCase',
+           'EsIndexableTestCase',
+           'EsJsonSerializerTestCase',
+           'EsRestFrameworkTestCase']

django_elasticsearch/tests/test_indexable.py

@@ -1,21 +1,13 @@
 # -*- coding: utf-8 -*-
-import json as json_serializer
-
 from elasticsearch import NotFoundError
 
 from django.test import TestCase
 from django.test.utils import override_settings
 
 from django_elasticsearch.managers import es_client
 from django_elasticsearch.tests.utils import withattrs
-from django_elasticsearch.tests.models import TestModel
-from django_elasticsearch.tests.models import TestModelESSerializer
-from django_elasticsearch.serializers import ModelJsonSerializer
-
 
-class CustomSerializer(ModelJsonSerializer):
-    def serialize_first_name(self, instance, field_name):
-        return u'pedro'
+from test_app.models import TestModel
 
 
 class EsIndexableTestCase(TestCase):
@@ -32,33 +24,6 @@ def tearDown(self):
         super(EsIndexableTestCase, self).tearDown()
         es_client.indices.delete(index=TestModel.es.get_index())
 
-    def _serialize(self):
-        json = self.instance.es.serialize()
-        # Checking one by one, different types of fields
-        self.assertIn('"id": %d' % self.instance.id, json)
-        self.assertIn('"first_name": "woot"', json)
-        self.assertIn('"last_name": "foo"', json)
-
-        return json
-
-    def test_serializer(self):
-        json = self._serialize()
-        s = self.instance.es.serializer.serialize_date_joined(self.instance, 'date_joined')
-        self.assertIn('"date_joined": %s' % json_serializer.dumps(s), json)
-
-    @withattrs(TestModel.Elasticsearch, 'serializer_class',
-               'django_elasticsearch.serializers.ModelJsonSerializer')
-    def test_dynamic_serializer_import(self):
-        self.instance.es.serializer = None  # reset cached property
-        json = self._serialize()
-        self.instance.es.serializer = None  # reset cached property
-        self.assertIn('"date_joined": "%s"' % self.instance.date_joined.isoformat(), json)
-
-    def test_deserialize(self):
-        instance = TestModel.es.deserialize({'username':'test'})
-        self.assertEqual(instance.username, 'test')
-        self.assertRaises(ValueError, instance.save)
-
     def test_do_index(self):
         self.instance.es.do_index()
         r = TestModel.es.deserialize(self.instance.es.get())
@@ -154,14 +119,8 @@ def test_get_mapping(self):
         TestModel.es.flush()
         TestModel.es.do_update()
 
-        expected = {
-            u'username': {u'index': u'not_analyzed', u'type': u'string'},
-            u'date_joined': {u'properties': {
-                u'iso': {u'format': u'dateOptionalTime', u'type': u'date'},
-                u'date': {u'type': u'string'},
-                u'time': {u'type': u'string'}
-            }}
-        }
+        expected = {u'date_joined': {u'format': u'dateOptionalTime', u'type': u'date'},
+                    u'username': {u'index': u'not_analyzed', u'type': u'string'}}
 
         # Reset the eventual cache on the Model mapping
         mapping = TestModel.es.get_mapping()
@@ -180,13 +139,6 @@ def test_custom_fields(self):
         expected = '{"first_name": "woot", "last_name": "foo"}'
         self.assertEqual(json, expected)
 
-    def test_custom_serializer(self):
-        old_serializer = self.instance.es.serializer
-        self.instance.es.serializer = CustomSerializer(TestModel)
-        json = self.instance.es.serialize()
-        self.assertIn('"first_name": "pedro"', json)
-        self.instance.es.serializer = old_serializer
-
     def test_custom_index(self):
         es_client.indices.exists(TestModel.Elasticsearch.index)
 

django_elasticsearch/tests/test_qs.py

@@ -1,3 +1,4 @@
+import time
 import mock
 from datetime import datetime, timedelta
 
@@ -7,7 +8,8 @@
 from django_elasticsearch.client import es_client
 from django_elasticsearch.managers import EsQueryset
 from django_elasticsearch.tests.utils import withattrs
-from django_elasticsearch.tests.models import TestModel
+
+from test_app.models import TestModel
 
 
 class EsQuerysetTestCase(TestCase):
@@ -188,21 +190,28 @@ def test_isnull_lookup(self):
         self.assertEqual(qs.count(), 3)
         self.assertFalse(self.t1 in qs)
 
+    @withattrs(TestModel.Elasticsearch, 'fields', ['id', 'date_joined_exp'])
     def test_sub_object_lookup(self):
-        qs = TestModel.es.filter(date_joined__iso=self.t1.date_joined).deserialize()
+        TestModel.es._fields = None
+        TestModel.es._mapping = None
+        TestModel.es.flush()  # update the mapping
+        time.sleep(2)
+
+        qs = TestModel.es.filter(date_joined_exp__iso=self.t1.date_joined).deserialize()
         self.assertEqual(qs.count(), 1)
         self.assertTrue(self.t1 in qs)
 
-        qs = TestModel.es.filter(date_joined__iso__isnull=False)
+        qs = TestModel.es.filter(date_joined_exp__iso__isnull=False)
         self.assertEqual(qs.count(), 4)
 
-    def test_sub_object_nested_lookup(self):
-        qs = TestModel.es.filter(date_joined__iso=self.t1.date_joined).deserialize()
-        self.assertTrue(qs.count(), 1)
-        self.assertTrue(self.t1 in qs)
-
+    @withattrs(TestModel.Elasticsearch, 'fields', ['id', 'date_joined_exp'])
     def test_filter_date_range(self):
-        contents = TestModel.es.queryset.filter(date_joined__iso__gte=self.t2.date_joined.isoformat()).deserialize()
+        TestModel.es._fields = None
+        TestModel.es._mapping = None
+        TestModel.es.flush()  # update the mapping
+        time.sleep(2)
+
+        contents = TestModel.es.filter(date_joined_exp__iso__gte=self.t2.date_joined.isoformat()).deserialize()
         self.assertTrue(self.t1 not in contents)
         self.assertTrue(self.t2 in contents)
         self.assertTrue(self.t3 in contents)
@@ -242,7 +251,7 @@ def test_excluding_lookups(self):
 
     def test_chain_filter_exclude(self):
         contents = TestModel.es.filter(last_name=u"Smith").exclude(username=u"woot").deserialize()
-        self.assertTrue(self.t1 in contents)
+        self.assertTrue(self.t1 in contents)  # note: it works because username is "not analyzed"
         self.assertTrue(self.t2 not in contents)  # excluded
         self.assertTrue(self.t3 in contents)
         self.assertTrue(self.t4 not in contents)  # not a Smith
@@ -253,8 +262,7 @@ def test_contains(self):
         TestModel.es._fields = None
         TestModel.es._mapping = None
         TestModel.es.flush()  # update the mapping, username is now analyzed
-        import time
-        time.sleep(2)  # flushing is not immediate :(
+        time.sleep(2)  # TODO: flushing is not immediate, find a better way
         contents = TestModel.es.filter(username__contains='woot').deserialize()
         self.assertTrue(self.t1 in contents)
         self.assertTrue(self.t2 in contents)