From 6f9b2c2c05612996f8a56da357566f7ca45ab679 Mon Sep 17 00:00:00 2001 From: "Xuye (Chris) Qin" Date: Sat, 6 Jul 2019 09:08:49 +0800 Subject: [PATCH] [BACKPORT] Add serialize support for complex type (#519) * Add serialize support for complex type (#488) * sync more serialize part from master --- mars/serialize/core.pxd | 20 ++- mars/serialize/core.pyx | 99 +++++++++++-- mars/serialize/jsonserializer.pyx | 39 ++++- mars/serialize/pbserializer.pyx | 70 ++++++++- mars/serialize/protos/value.proto | 8 +- mars/serialize/protos/value_pb2.py | 142 ++++++++++++------ mars/serialize/tests/test_serialize.py | 197 ++++++++++++++++++++----- mars/serialize/tests/testser.proto | 24 ++- mars/serialize/tests/testser_pb2.py | 189 ++++++++++++++++++------ 9 files changed, 638 insertions(+), 150 deletions(-) diff --git a/mars/serialize/core.pxd b/mars/serialize/core.pxd index 21e5ce993a..540c717466 100644 --- a/mars/serialize/core.pxd +++ b/mars/serialize/core.pxd @@ -30,6 +30,8 @@ cpdef enum PrimitiveType: float64 = 12 bytes = 13 unicode = 14 + complex64 = 24 + complex128 = 25 cpdef enum ExtendType: @@ -75,14 +77,18 @@ cdef class ValueType: pass +cdef class SelfReferenceOverwritten(Exception): + pass + + cdef class Field: cdef object tag cdef object default_val cdef str _tag_name cdef object _type + cdef object _model_cls cdef public bint weak_ref - cdef public object model cdef public str attr cdef public object on_serialize cdef public object on_deserialize @@ -148,6 +154,14 @@ cdef class Float64Field(Field): pass +cdef class Complex64Field(Field): + pass + + +cdef class Complex128Field(Field): + pass + + cdef class StringField(Field): pass @@ -181,7 +195,7 @@ cdef class DataTypeField(Field): cdef class ListField(Field): - cdef object _nest_ref + cdef public object _nest_ref cdef class TupleField(Field): @@ -193,7 +207,7 @@ cdef class DictField(Field): cdef class ReferenceField(Field): - cdef object _model + cdef public object _model cdef class OneOfField(Field): diff --git a/mars/serialize/core.pyx b/mars/serialize/core.pyx index 6016036e48..53a4241a12 100644 --- a/mars/serialize/core.pyx +++ b/mars/serialize/core.pyx @@ -16,6 +16,7 @@ import importlib import inspect +import copy from collections import Iterable from ..compat import six, OrderedDict @@ -100,6 +101,8 @@ cdef class ValueType: bytes = PrimitiveType.bytes unicode = PrimitiveType.unicode string = PrimitiveType.unicode if PY3 else PrimitiveType.bytes + complex64 = PrimitiveType.complex64 + complex128 = PrimitiveType.complex128 slice = ExtendType.slice arr = ExtendType.arr dtype = ExtendType.dtype @@ -119,7 +122,7 @@ cdef class ValueType: cdef class Field: def __init__(self, tag, default=None, bint weak_ref=False, on_serialize=None, on_deserialize=None): - self.model = None + self._model_cls = None self.attr = None self.tag = tag @@ -131,6 +134,14 @@ cdef class Field: self.on_serialize = on_serialize self.on_deserialize = on_deserialize + @property + def model(self): + return self._model_cls + + @model.setter + def model(self, model_cls): + self._model_cls = model_cls + cpdef str tag_name(self, Provider provider): if self._tag_name is None: return self.tag(provider) @@ -267,6 +278,22 @@ cdef class Float64Field(Field): self._type = ValueType.float64 +cdef class Complex64Field(Field): + def __init__(self, tag, default=None, bint weak_ref=False, on_serialize=None, on_deserialize=None): + super(Complex64Field, self).__init__( + tag, default=default, weak_ref=weak_ref, + on_serialize=on_serialize, on_deserialize=on_deserialize) + self._type = ValueType.complex64 + + +cdef class Complex128Field(Field): + def __init__(self, tag, default=None, bint weak_ref=False, on_serialize=None, on_deserialize=None): + super(Complex128Field, self).__init__( + tag, default=default, weak_ref=weak_ref, + on_serialize=on_serialize, on_deserialize=on_deserialize) + self._type = ValueType.complex128 + + cdef class StringField(Field): def __init__(self, tag, default=None, bint weak_ref=False, on_serialize=None, on_deserialize=None): super(StringField, self).__init__( @@ -354,6 +381,18 @@ cdef class ListField(Field): else: self._type = ValueType.list(tp) + @property + def model(self): + return self._model_cls + + @model.setter + def model(self, new_model_cls): + if getattr(self, '_nest_ref', None) is not None and \ + self._nest_ref.model == 'self' and self._model_cls is not None and \ + new_model_cls is not None: + raise SelfReferenceOverwritten('self reference is overwritten') + self._model_cls = new_model_cls + @property def type(self): if self._type is None: @@ -397,6 +436,17 @@ cdef class ReferenceField(Field): self._type = None self._model = model + @property + def model(self): + return self._model_cls + + @model.setter + def model(self, new_model_cls): + if getattr(self, '_model', None) == 'self' and \ + self._model_cls is not None and new_model_cls is not None: + raise SelfReferenceOverwritten('self reference is overwritten') + self._model_cls = new_model_cls + @property def type(self): if not self._type: @@ -432,6 +482,42 @@ cdef class OneOfField(Field): self._type = ValueType.oneof(*[f.type for f in self.fields]) return self._type + @property + def attrs(self): + return [f.attr for f in self.fields] + + +cdef inline set_model(dict fields, cls): + cdef str slot + cdef bint modified + + for slot, field in fields.items(): + if not isinstance(field, OneOfField): + try: + field.model = cls + except SelfReferenceOverwritten: + field = copy.copy(field) + # reset old model after copy + field.model = None + field.model = cls + cls._FIELDS[slot] = field + else: + one_field_fields = [] + modified = False + for f in field.fields: + try: + f.model = cls + except SelfReferenceOverwritten: + f = copy.copy(f) + # reset old model after copy + f.model = None + f.model = cls + modified = True + f.attr = field.attr + one_field_fields.append(f) + if modified: + field.fields = one_field_fields + class SerializableMetaclass(type): def __new__(mcs, str name, tuple bases, dict kv): @@ -477,13 +563,7 @@ class SerializableMetaclass(type): kv['__slots__'] = tuple(slots) cls = type.__new__(mcs, name, bases, kv) - for field in fields.values(): - if not isinstance(field, OneOfField): - field.model = cls - else: - for f in field.fields: - f.model = cls - f.attr = field.attr + set_model(fields, cls) return cls @@ -495,7 +575,8 @@ class Serializable(six.with_metaclass(SerializableMetaclass, Base)): if provider.type == ProviderType.json: return dict - raise TypeError('Unknown provider type: {0}'.format(provider.type.value)) + raise TypeError('Unknown provider type `{0}` for class `{1}`'.format( + ProviderType(provider.type).name, cls.__name__)) def serialize(self, Provider provider, obj=None): return provider.serialize_model(self, obj=obj) diff --git a/mars/serialize/jsonserializer.pyx b/mars/serialize/jsonserializer.pyx index 8bbc07cb20..e37f24287d 100644 --- a/mars/serialize/jsonserializer.pyx +++ b/mars/serialize/jsonserializer.pyx @@ -54,6 +54,8 @@ cdef dict EXTEND_TYPE_TO_NAME = { ValueType.key: 'key', ValueType.datetime64: 'datetime64', ValueType.timedelta64: 'timedelta64', + ValueType.complex64: 'complex64', + ValueType.complex128: 'complex128', } @@ -260,6 +262,18 @@ cdef class JsonSerializeProvider(Provider): cdef inline _deserialize_timedelta64(self, obj, list callbacks): return self._deserialize_datetime64_timedelta64(obj, callbacks) + cdef inline dict _serialize_complex(self, value, tp): + return { + 'type': _get_name(tp), + 'value': (value.real, value.imag) + } + + cdef inline _deserialize_complex(self, obj, list callbacks): + cdef list v + + v = obj['value'] + return complex(*v) + cdef inline object _serialize_typed_value(self, value, tp, bint weak_ref=False): if type(tp) not in (List, Tuple, Dict) and weak_ref: # not iterable, and is weak ref @@ -276,6 +290,8 @@ cdef class JsonSerializeProvider(Provider): return value elif type(tp) is Identity: return self._serialize_typed_value(value, tp.type, weak_ref=weak_ref) + elif tp in {ValueType.complex64, ValueType.complex128}: + return self._serialize_complex(value, tp) elif tp is ValueType.slice: return self._serialize_slice(value) elif tp is ValueType.arr: @@ -319,6 +335,8 @@ cdef class JsonSerializeProvider(Provider): return value elif isinstance(value, float): return value + elif isinstance(value, complex): + return self._serialize_complex(value, ValueType.complex128) elif isinstance(value, slice): return self._serialize_slice(value) elif isinstance(value, np.ndarray): @@ -337,6 +355,8 @@ cdef class JsonSerializeProvider(Provider): return self._serialize_datetime64(value) elif isinstance(value, np.timedelta64): return self._serialize_timedelta64(value) + elif isinstance(value, np.number): + return self._serialize_untyped_value(value.item()) else: raise TypeError('Unknown type to serialize: {0}'.format(type(value))) @@ -364,8 +384,12 @@ cdef class JsonSerializeProvider(Provider): field_val = getattr(model_instance, field.attr) if field.weak_ref: field_val = field_val() - value = self._on_serial(field, field_val) - value.serialize(self, new_obj) + if field_val is not None: + if not isinstance(field_val, field.type.model): + raise TypeError('Does not match type for reference field {0}: ' + 'expect {1}, got {2}'.format(tag, field.type.model, type(field_val))) + value = self._on_serial(field, field_val) + value.serialize(self, new_obj) elif isinstance(field, OneOfField): has_val = False field_val = getattr(model_instance, field.attr, None) @@ -384,6 +408,9 @@ cdef class JsonSerializeProvider(Provider): new_obj = obj[tag] = dict() value.serialize(self, new_obj) return + if not has_val and value is not None: + raise ValueError('Value {0} cannot match any type for OneOfField `{1}`'.format( + value, field.tag_name(self))) elif isinstance(field, ListField) and type(field.type.type) == Reference: tag = field.tag_name(self) value = self._on_serial(field, getattr(model_instance, field.attr, None)) @@ -394,7 +421,11 @@ cdef class JsonSerializeProvider(Provider): if field.weak_ref: val = val() if val is not None: - new_obj.append(val.serialize(self, dict())) + if isinstance(val, field.type.type.model): + new_obj.append(val.serialize(self, dict())) + else: + raise TypeError('Does not match type for reference in list field {0}: ' + 'expect {1}, got {2}'.format(tag, field.type.type.model, type(val))) else: new_obj.append(None) else: @@ -417,6 +448,8 @@ cdef class JsonSerializeProvider(Provider): if tp is ValueType.bytes: return ref(base64.b64decode(obj['value'])) + elif tp in {ValueType.complex64, ValueType.complex128}: + return ref(self._deserialize_complex(obj, callbacks)) elif tp is ValueType.slice: return ref(self._deserialize_slice(obj, callbacks)) elif tp is ValueType.arr: diff --git a/mars/serialize/pbserializer.pyx b/mars/serialize/pbserializer.pyx index 2a9bf4fc7d..4afcb49168 100644 --- a/mars/serialize/pbserializer.pyx +++ b/mars/serialize/pbserializer.pyx @@ -138,6 +138,13 @@ cdef class ProtobufSerializeProvider(Provider): x = obj.timedelta64 return np.load(six.BytesIO(x)) if x is not None and len(x) > 0 else None + cdef inline void _set_complex(self, value, obj, tp=None): + obj.c.real = value.real + obj.c.imag = value.imag + + cdef inline object _get_complex(self, obj): + return complex(obj.c.real, obj.c.imag) + @cython.boundscheck(False) @cython.wraparound(False) @cython.nonecheck(False) @@ -326,6 +333,8 @@ cdef class ProtobufSerializeProvider(Provider): self._set_datetime64(value, obj, tp) elif tp is ValueType.timedelta64: self._set_timedelta64(value, obj, tp) + elif tp in {ValueType.complex64, ValueType.complex128}: + self._set_complex(value, obj, tp) elif isinstance(tp, Identity): value_field = PRIMITIVE_TYPE_TO_VALUE_FIELD[tp.type] setattr(obj, value_field, value) @@ -362,6 +371,8 @@ cdef class ProtobufSerializeProvider(Provider): obj.i = value elif isinstance(value, float): obj.f = value + elif isinstance(value, complex): + self._set_complex(value, obj) elif isinstance(value, slice): self._set_slice(value, obj) elif isinstance(value, np.ndarray): @@ -401,15 +412,21 @@ cdef class ProtobufSerializeProvider(Provider): cdef object field_obj cdef object add cdef object it_obj + cdef bint matched cdef OneOfField oneoffield if isinstance(field, OneOfField): oneoffield = field value = getattr(model_instance, oneoffield.attr, None) + matched = False for f in oneoffield.fields: if isinstance(value, f.type.model): f.serialize(self, model_instance, obj) + matched = True return + if not matched and value is not None: + raise ValueError('Value {0} cannot match any type for OneOfField `{1}`'.format( + value, field.tag_name(self))) return value = getattr(model_instance, field.attr, field.default) @@ -424,6 +441,9 @@ cdef class ProtobufSerializeProvider(Provider): if isinstance(field, ReferenceField): if field.weak_ref: field_obj = field_obj() + if not isinstance(value, field.type.model): + raise TypeError('Does not match type for reference field {0}: ' + 'expect {1}, got {2}'.format(tag, field.type.model, type(value))) value.serialize(self, obj=field_obj) elif isinstance(field, ListField): if type(field.type.type) == Reference: @@ -433,7 +453,11 @@ cdef class ProtobufSerializeProvider(Provider): val = val() it_obj = add() if val is not None: - val.serialize(self, obj=it_obj) + if isinstance(val, field.type.type.model): + val.serialize(self, obj=it_obj) + else: + raise TypeError('Does not match type for reference in list field {0}: ' + 'expect {1}, got {2}'.format(tag, field.type.type.model, type(val))) elif isinstance(it_obj, Value): it_obj.is_null = True else: @@ -476,11 +500,34 @@ cdef class ProtobufSerializeProvider(Provider): value = getattr(model_instance, name, None) if value is None: continue + tag = field.tag_name(self) k = d_obj.dict.keys.value.add() self._set_value(field.tag_name(self), k, tp=ValueType.string) v = d_obj.dict.values.value.add() - self._set_value(value, v, tp=field.type, weak_ref=field.weak_ref) + if isinstance(field, ReferenceField): + if field.weak_ref: + value = value() + if isinstance(value, field.type.model): + value.serialize(self, obj=v) + else: + raise TypeError('Does not match type for reference field {0}: ' + 'expect {1}, got {2}'.format(tag, field.type.model, type(value))) + elif isinstance(field, ListField) and type(field.type.type) == Reference: + for val in value: + if field.weak_ref: + val = val() + it_obj = v.list.value.add() + if val is not None: + if isinstance(val, field.type.type.model): + val.serialize(self, obj=it_obj) + else: + raise TypeError('Does not match type for reference in list field {0}: ' + 'expect {1}, got {2}'.format(tag, field.type.type.model, type(val))) + elif isinstance(it_obj, Value): + it_obj.is_null = True + else: + self._set_value(value, v, tp=field.type, weak_ref=field.weak_ref) else: fields = model_instance._FIELDS for name, field in fields.items(): @@ -504,6 +551,8 @@ cdef class ProtobufSerializeProvider(Provider): if tp in PRIMITIVE_TYPE_TO_VALUE_FIELD: value_field = PRIMITIVE_TYPE_TO_VALUE_FIELD[tp] return ref(getattr(obj, value_field)) + elif tp in {ValueType.complex64, ValueType.complex128}: + return ref(self._get_complex(obj)) elif tp is ValueType.slice: return ref(self._get_slice(obj)) elif tp is ValueType.arr: @@ -547,6 +596,8 @@ cdef class ProtobufSerializeProvider(Provider): elif field in 'bif': # primitive type return ref(getattr(obj, field)) + elif field == 'c': + return ref(self._get_complex(obj)) elif field == 's': # bytes b = obj.s @@ -702,8 +753,19 @@ cdef class ProtobufSerializeProvider(Provider): for k, v in zip(d_obj.dict.keys.value, d_obj.dict.values.value): tag = self._get_value(k, ValueType.string, callbacks, False) it_field = tag_to_fields[tag] - setattr(model_instance, it_field.attr, - self._get_value(v, it_field.type, callbacks, it_field.weak_ref)) + if isinstance(it_field, ReferenceField): + setattr(model_instance, it_field.attr, + self._on_deserial( + it_field, it_field.type.model.deserialize(self, v, callbacks, key_to_instance))) + elif isinstance(it_field, ListField) and type(it_field.type.type) == Reference: + setattr(model_instance, it_field.attr, + self._on_deserial( + it_field, + [it_field.type.type.model.deserialize(self, it_obj, callbacks, key_to_instance) + for it_obj in v.list.value])) + else: + setattr(model_instance, it_field.attr, + self._get_value(v, it_field.type, callbacks, it_field.weak_ref)) else: for o_tag in d_obj: it_field = tag_to_fields[o_tag] diff --git a/mars/serialize/protos/value.proto b/mars/serialize/protos/value.proto index 65f6bc3499..1c4023112e 100644 --- a/mars/serialize/protos/value.proto +++ b/mars/serialize/protos/value.proto @@ -19,6 +19,11 @@ message Value { List values = 2; } + message Complex { + double real = 1; + double imag = 2; + } + message Key { string key = 1; string id = 2; // since different object may have the same key, we need an identity @@ -28,7 +33,8 @@ message Value { bool is_null = 1; // NULL bool b = 2; // bool int64 i = 3; // int - float f = 4; // float + double f = 4; // double + Complex c = 19; // complex bytes s = 5; // bytes string u = 6; // unicode List list = 7; diff --git a/mars/serialize/protos/value_pb2.py b/mars/serialize/protos/value_pb2.py index dcaa3f2056..0244e73a5d 100644 --- a/mars/serialize/protos/value_pb2.py +++ b/mars/serialize/protos/value_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: mars/serialize/protos/value.proto @@ -19,7 +20,7 @@ package='', syntax='proto3', serialized_options=None, - serialized_pb=_b('\n!mars/serialize/protos/value.proto\"\xa6\x04\n\x05Value\x12\x11\n\x07is_null\x18\x01 \x01(\x08H\x00\x12\x0b\n\x01\x62\x18\x02 \x01(\x08H\x00\x12\x0b\n\x01i\x18\x03 \x01(\x03H\x00\x12\x0b\n\x01\x66\x18\x04 \x01(\x02H\x00\x12\x0b\n\x01s\x18\x05 \x01(\x0cH\x00\x12\x0b\n\x01u\x18\x06 \x01(\tH\x00\x12\x1b\n\x04list\x18\x07 \x01(\x0b\x32\x0b.Value.ListH\x00\x12\x1b\n\x04\x64ict\x18\x08 \x01(\x0b\x32\x0b.Value.DictH\x00\x12\x1d\n\x05slice\x18\t \x01(\x0b\x32\x0c.Value.SliceH\x00\x12\r\n\x03\x61rr\x18\n \x01(\x0cH\x00\x12\x0f\n\x05\x64type\x18\x0b \x01(\x0cH\x00\x12\x19\n\x03key\x18\x0c \x01(\x0b\x32\n.Value.KeyH\x00\x12\x14\n\ndatetime64\x18\r \x01(\x0cH\x00\x12\x15\n\x0btimedelta64\x18\x0e \x01(\x0cH\x00\x1a/\n\x04List\x12\x10\n\x08is_tuple\x18\x01 \x01(\x08\x12\x15\n\x05value\x18\x02 \x03(\x0b\x32\x06.Value\x1an\n\x05Slice\x12\x0f\n\x07is_null\x18\x04 \x01(\x08\x12\x13\n\tstart_val\x18\x01 \x01(\x03H\x00\x12\x12\n\x08stop_val\x18\x02 \x01(\x03H\x01\x12\x12\n\x08step_val\x18\x03 \x01(\x03H\x02\x42\x07\n\x05startB\x06\n\x04stopB\x06\n\x04step\x1a>\n\x04\x44ict\x12\x19\n\x04keys\x18\x01 \x01(\x0b\x32\x0b.Value.List\x12\x1b\n\x06values\x18\x02 \x01(\x0b\x32\x0b.Value.List\x1a\x1e\n\x03Key\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\tB\x07\n\x05valueb\x06proto3') + serialized_pb=_b('\n!mars/serialize/protos/value.proto\"\xea\x04\n\x05Value\x12\x11\n\x07is_null\x18\x01 \x01(\x08H\x00\x12\x0b\n\x01\x62\x18\x02 \x01(\x08H\x00\x12\x0b\n\x01i\x18\x03 \x01(\x03H\x00\x12\x0b\n\x01\x66\x18\x04 \x01(\x01H\x00\x12\x1b\n\x01\x63\x18\x13 \x01(\x0b\x32\x0e.Value.ComplexH\x00\x12\x0b\n\x01s\x18\x05 \x01(\x0cH\x00\x12\x0b\n\x01u\x18\x06 \x01(\tH\x00\x12\x1b\n\x04list\x18\x07 \x01(\x0b\x32\x0b.Value.ListH\x00\x12\x1b\n\x04\x64ict\x18\x08 \x01(\x0b\x32\x0b.Value.DictH\x00\x12\x1d\n\x05slice\x18\t \x01(\x0b\x32\x0c.Value.SliceH\x00\x12\r\n\x03\x61rr\x18\n \x01(\x0cH\x00\x12\x0f\n\x05\x64type\x18\x0b \x01(\x0cH\x00\x12\x19\n\x03key\x18\x0c \x01(\x0b\x32\n.Value.KeyH\x00\x12\x14\n\ndatetime64\x18\r \x01(\x0cH\x00\x12\x15\n\x0btimedelta64\x18\x0e \x01(\x0cH\x00\x1a/\n\x04List\x12\x10\n\x08is_tuple\x18\x01 \x01(\x08\x12\x15\n\x05value\x18\x02 \x03(\x0b\x32\x06.Value\x1an\n\x05Slice\x12\x0f\n\x07is_null\x18\x04 \x01(\x08\x12\x13\n\tstart_val\x18\x01 \x01(\x03H\x00\x12\x12\n\x08stop_val\x18\x02 \x01(\x03H\x01\x12\x12\n\x08step_val\x18\x03 \x01(\x03H\x02\x42\x07\n\x05startB\x06\n\x04stopB\x06\n\x04step\x1a>\n\x04\x44ict\x12\x19\n\x04keys\x18\x01 \x01(\x0b\x32\x0b.Value.List\x12\x1b\n\x06values\x18\x02 \x01(\x0b\x32\x0b.Value.List\x1a%\n\x07\x43omplex\x12\x0c\n\x04real\x18\x01 \x01(\x01\x12\x0c\n\x04imag\x18\x02 \x01(\x01\x1a\x1e\n\x03Key\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\tB\x07\n\x05valueb\x06proto3') ) @@ -58,8 +59,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=324, - serialized_end=371, + serialized_start=353, + serialized_end=400, ) _VALUE_SLICE = _descriptor.Descriptor( @@ -118,8 +119,8 @@ name='step', full_name='Value.Slice.step', index=2, containing_type=None, fields=[]), ], - serialized_start=373, - serialized_end=483, + serialized_start=402, + serialized_end=512, ) _VALUE_DICT = _descriptor.Descriptor( @@ -155,8 +156,45 @@ extension_ranges=[], oneofs=[ ], - serialized_start=485, - serialized_end=547, + serialized_start=514, + serialized_end=576, +) + +_VALUE_COMPLEX = _descriptor.Descriptor( + name='Complex', + full_name='Value.Complex', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='real', full_name='Value.Complex.real', index=0, + number=1, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='imag', full_name='Value.Complex.imag', index=1, + number=2, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=578, + serialized_end=615, ) _VALUE_KEY = _descriptor.Descriptor( @@ -192,8 +230,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=549, - serialized_end=579, + serialized_start=617, + serialized_end=647, ) _VALUE = _descriptor.Descriptor( @@ -226,76 +264,83 @@ serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='f', full_name='Value.f', index=3, - number=4, type=2, cpp_type=6, label=1, + number=4, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='s', full_name='Value.s', index=4, + name='c', full_name='Value.c', index=4, + number=19, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='s', full_name='Value.s', index=5, number=5, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='u', full_name='Value.u', index=5, + name='u', full_name='Value.u', index=6, number=6, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='list', full_name='Value.list', index=6, + name='list', full_name='Value.list', index=7, number=7, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='dict', full_name='Value.dict', index=7, + name='dict', full_name='Value.dict', index=8, number=8, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='slice', full_name='Value.slice', index=8, + name='slice', full_name='Value.slice', index=9, number=9, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='arr', full_name='Value.arr', index=9, + name='arr', full_name='Value.arr', index=10, number=10, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='dtype', full_name='Value.dtype', index=10, + name='dtype', full_name='Value.dtype', index=11, number=11, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='key', full_name='Value.key', index=11, + name='key', full_name='Value.key', index=12, number=12, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='datetime64', full_name='Value.datetime64', index=12, + name='datetime64', full_name='Value.datetime64', index=13, number=13, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='timedelta64', full_name='Value.timedelta64', index=13, + name='timedelta64', full_name='Value.timedelta64', index=14, number=14, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, @@ -304,7 +349,7 @@ ], extensions=[ ], - nested_types=[_VALUE_LIST, _VALUE_SLICE, _VALUE_DICT, _VALUE_KEY, ], + nested_types=[_VALUE_LIST, _VALUE_SLICE, _VALUE_DICT, _VALUE_COMPLEX, _VALUE_KEY, ], enum_types=[ ], serialized_options=None, @@ -317,7 +362,7 @@ index=0, containing_type=None, fields=[]), ], serialized_start=38, - serialized_end=588, + serialized_end=656, ) _VALUE_LIST.fields_by_name['value'].message_type = _VALUE @@ -335,7 +380,9 @@ _VALUE_DICT.fields_by_name['keys'].message_type = _VALUE_LIST _VALUE_DICT.fields_by_name['values'].message_type = _VALUE_LIST _VALUE_DICT.containing_type = _VALUE +_VALUE_COMPLEX.containing_type = _VALUE _VALUE_KEY.containing_type = _VALUE +_VALUE.fields_by_name['c'].message_type = _VALUE_COMPLEX _VALUE.fields_by_name['list'].message_type = _VALUE_LIST _VALUE.fields_by_name['dict'].message_type = _VALUE_DICT _VALUE.fields_by_name['slice'].message_type = _VALUE_SLICE @@ -352,6 +399,9 @@ _VALUE.oneofs_by_name['value'].fields.append( _VALUE.fields_by_name['f']) _VALUE.fields_by_name['f'].containing_oneof = _VALUE.oneofs_by_name['value'] +_VALUE.oneofs_by_name['value'].fields.append( + _VALUE.fields_by_name['c']) +_VALUE.fields_by_name['c'].containing_oneof = _VALUE.oneofs_by_name['value'] _VALUE.oneofs_by_name['value'].fields.append( _VALUE.fields_by_name['s']) _VALUE.fields_by_name['s'].containing_oneof = _VALUE.oneofs_by_name['value'] @@ -385,43 +435,51 @@ DESCRIPTOR.message_types_by_name['Value'] = _VALUE _sym_db.RegisterFileDescriptor(DESCRIPTOR) -Value = _reflection.GeneratedProtocolMessageType('Value', (_message.Message,), dict( +Value = _reflection.GeneratedProtocolMessageType('Value', (_message.Message,), { - List = _reflection.GeneratedProtocolMessageType('List', (_message.Message,), dict( - DESCRIPTOR = _VALUE_LIST, - __module__ = 'mars.serialize.protos.value_pb2' + 'List' : _reflection.GeneratedProtocolMessageType('List', (_message.Message,), { + 'DESCRIPTOR' : _VALUE_LIST, + '__module__' : 'mars.serialize.protos.value_pb2' # @@protoc_insertion_point(class_scope:Value.List) - )) + }) , - Slice = _reflection.GeneratedProtocolMessageType('Slice', (_message.Message,), dict( - DESCRIPTOR = _VALUE_SLICE, - __module__ = 'mars.serialize.protos.value_pb2' + 'Slice' : _reflection.GeneratedProtocolMessageType('Slice', (_message.Message,), { + 'DESCRIPTOR' : _VALUE_SLICE, + '__module__' : 'mars.serialize.protos.value_pb2' # @@protoc_insertion_point(class_scope:Value.Slice) - )) + }) , - Dict = _reflection.GeneratedProtocolMessageType('Dict', (_message.Message,), dict( - DESCRIPTOR = _VALUE_DICT, - __module__ = 'mars.serialize.protos.value_pb2' + 'Dict' : _reflection.GeneratedProtocolMessageType('Dict', (_message.Message,), { + 'DESCRIPTOR' : _VALUE_DICT, + '__module__' : 'mars.serialize.protos.value_pb2' # @@protoc_insertion_point(class_scope:Value.Dict) - )) + }) + , + + 'Complex' : _reflection.GeneratedProtocolMessageType('Complex', (_message.Message,), { + 'DESCRIPTOR' : _VALUE_COMPLEX, + '__module__' : 'mars.serialize.protos.value_pb2' + # @@protoc_insertion_point(class_scope:Value.Complex) + }) , - Key = _reflection.GeneratedProtocolMessageType('Key', (_message.Message,), dict( - DESCRIPTOR = _VALUE_KEY, - __module__ = 'mars.serialize.protos.value_pb2' + 'Key' : _reflection.GeneratedProtocolMessageType('Key', (_message.Message,), { + 'DESCRIPTOR' : _VALUE_KEY, + '__module__' : 'mars.serialize.protos.value_pb2' # @@protoc_insertion_point(class_scope:Value.Key) - )) + }) , - DESCRIPTOR = _VALUE, - __module__ = 'mars.serialize.protos.value_pb2' + 'DESCRIPTOR' : _VALUE, + '__module__' : 'mars.serialize.protos.value_pb2' # @@protoc_insertion_point(class_scope:Value) - )) + }) _sym_db.RegisterMessage(Value) _sym_db.RegisterMessage(Value.List) _sym_db.RegisterMessage(Value.Slice) _sym_db.RegisterMessage(Value.Dict) +_sym_db.RegisterMessage(Value.Complex) _sym_db.RegisterMessage(Value.Key) diff --git a/mars/serialize/tests/test_serialize.py b/mars/serialize/tests/test_serialize.py index eaa4f21ad7..885c59bc37 100644 --- a/mars/serialize/tests/test_serialize.py +++ b/mars/serialize/tests/test_serialize.py @@ -28,9 +28,11 @@ from mars.compat import six, OrderedDict, BytesIO from mars.lib import sparse -from mars.serialize.core import Serializable, IdentityField, StringField, Int32Field, BytesField, \ - KeyField, ReferenceField, OneOfField, ListField, NDArrayField, DictField, TupleField, \ - ValueType, serializes, deserializes, ProviderType, AttributeAsDict +from mars.serialize.core import Serializable, IdentityField, StringField, \ + BytesField, Int8Field, Int16Field, Int32Field, Int64Field, UInt8Field, UInt16Field, \ + UInt32Field, UInt64Field, Float16Field, Float32Field, Float64Field, BoolField, \ + KeyField, ReferenceField, OneOfField, ListField, NDArrayField, DictField, TupleField, ValueType, \ + serializes, deserializes, Complex64Field, Complex128Field, AnyField, ProviderType, AttributeAsDict from mars.serialize import dataserializer from mars.serialize.pbserializer import ProtobufSerializeProvider from mars.serialize.jsonserializer import JsonSerializeProvider @@ -40,11 +42,24 @@ class Node1(Serializable): a = IdentityField('a', ValueType.string) - b = Int32Field('b') - c = KeyField('c') - d = ReferenceField('d', 'Node2') - e = ListField('e') - f = ListField('f', ValueType.reference('self')) + b1 = Int8Field('b1') + b2 = Int16Field('b2') + b3 = Int32Field('b3') + b4 = Int64Field('b4') + c1 = UInt8Field('c1') + c2 = UInt16Field('c2') + c3 = UInt32Field('c3') + c4 = UInt64Field('c4') + d1 = Float16Field('d1') + d2 = Float32Field('d2') + d3 = Float64Field('d3') + cl1 = Complex64Field('cl1') + cl2 = Complex128Field('cl2') + e = BoolField('e') + f = KeyField('f') + g = ReferenceField('g', 'Node2') + h = ListField('h') + i = ListField('i', ValueType.reference('self')) def __new__(cls, *args, **kwargs): if 'a' in kwargs and kwargs['a'] == 'test1': @@ -63,6 +78,10 @@ class Node8(Node1): pass +class Node9(Node1): + f = AnyField('f') + + class Node2(Serializable, BaseWithKey): a = ListField('a', ValueType.list(ValueType.string)) _key = StringField('key') @@ -133,10 +152,17 @@ def testPBSerialize(self): provider = ProtobufSerializeProvider() node2 = Node2(a=[['ss'], ['dd']], data=[3, 7, 212]) - node1 = Node1(a='test1', b=2, d=Node2(a=[['1', '2'], ['3', '4']]), - c=node2, - e=[[2, 3], node2, True, {1: node2}, np.datetime64('1066-10-13'), np.timedelta64(1, 'D')], - f=[Node1(b=111), Node1(b=222)]) + node1 = Node1(a='test1', + b1=-2, b2=2000, b3=-5000, b4=500000, + c1=2, c2=2000, c3=5000, c4=500000, + d1=2.5, d2=7.37, d3=5.976321, + cl1=1+2j, cl2=2.5+3.1j, + e=False, + f=node2, + g=Node2(a=[['1', '2'], ['3', '4']]), + h=[[2, 3], node2, True, {1: node2}, np.datetime64('1066-10-13'), + np.timedelta64(1, 'D'), np.complex64(1+2j), np.complex128(2+3j)], + i=[Node8(b1=111), Node8(b1=222)]) node3 = Node3(value=node1) serials = serializes(provider, [node2, node3]) @@ -150,26 +176,51 @@ def testPBSerialize(self): self.assertIsInstance(d_node3.value, Node8) self.assertIsNot(node3.value, d_node3.value) self.assertEqual(node3.value.a, d_node3.value.a) - self.assertEqual(node3.value.b, d_node3.value.b) - self.assertIsNot(node3.value.c, d_node3.value.c) - self.assertEqual(node3.value.c.a, d_node3.value.c.a) - self.assertIsNot(node3.value.d, d_node3.value.d) - self.assertEqual(node3.value.d.a, d_node3.value.d.a) - self.assertEqual(node3.value.e[0], d_node3.value.e[0]) - self.assertNotIsInstance(d_node3.value.e[1], six.string_types) - self.assertIs(d_node3.value.e[1], d_node3.value.c) - self.assertEqual(node3.value.e[2], True) - self.assertEqual([n.b for n in node3.value.f], [n.b for n in d_node3.value.f]) - self.assertNotIsInstance(node3.value.f[0], Node8) + self.assertEqual(node3.value.b1, d_node3.value.b1) + self.assertEqual(node3.value.b2, d_node3.value.b2) + self.assertEqual(node3.value.b3, d_node3.value.b3) + self.assertEqual(node3.value.b4, d_node3.value.b4) + self.assertEqual(node3.value.c1, d_node3.value.c1) + self.assertEqual(node3.value.c2, d_node3.value.c2) + self.assertEqual(node3.value.c3, d_node3.value.c3) + self.assertEqual(node3.value.c4, d_node3.value.c4) + self.assertAlmostEqual(node3.value.d1, d_node3.value.d1, places=2) + self.assertAlmostEqual(node3.value.d2, d_node3.value.d2, places=4) + self.assertAlmostEqual(node3.value.d3, d_node3.value.d3) + self.assertAlmostEqual(node3.value.cl1, d_node3.value.cl1) + self.assertAlmostEqual(node3.value.cl2, d_node3.value.cl2) + self.assertEqual(node3.value.e, d_node3.value.e) + self.assertIsNot(node3.value.f, d_node3.value.f) + self.assertEqual(node3.value.f.a, d_node3.value.f.a) + self.assertIsNot(node3.value.g, d_node3.value.g) + self.assertEqual(node3.value.g.a, d_node3.value.g.a) + self.assertEqual(node3.value.h[0], d_node3.value.h[0]) + self.assertNotIsInstance(d_node3.value.h[1], six.string_types) + self.assertIs(d_node3.value.h[1], d_node3.value.f) + self.assertEqual(node3.value.h[2], True) + self.assertAlmostEqual(node3.value.h[6], d_node3.value.h[6]) + self.assertAlmostEqual(node3.value.h[7], d_node3.value.h[7]) + self.assertEqual([n.b1 for n in node3.value.i], [n.b1 for n in d_node3.value.i]) + self.assertIsInstance(d_node3.value.i[0], Node8) + + with self.assertRaises(ValueError): + serializes(provider, [Node3(value='sth else')]) def testJSONSerialize(self): provider = JsonSerializeProvider() node2 = Node2(a=[['ss'], ['dd']], data=[3, 7, 212]) - node1 = Node1(a='test1', b=2, d=Node2(a=[['1', '2'], ['3', '4']]), - c=node2, - e=[[2, 3], node2, True, {1: node2}, np.datetime64('1066-10-13'), np.timedelta64(1, 'D')], - f=[Node1(b=111), Node1(b=222)]) + node1 = Node1(a='test1', + b1=2, b2=2000, b3=5000, b4=500000, + c1=2, c2=2000, c3=5000, c4=500000, + d1=2.5, d2=7.37, d3=5.976321, + cl1=1+2j, cl2=2.5+3.1j, + e=False, + f=node2, + g=Node2(a=[['1', '2'], ['3', '4']]), + h=[[2, 3], node2, True, {1: node2}, np.datetime64('1066-10-13'), + np.timedelta64(1, 'D'), np.complex64(1+2j), np.complex128(2+3j)], + i=[Node8(b1=111), Node8(b1=222)]) node3 = Node3(value=node1) serials = serializes(provider, [node2, node3]) @@ -184,17 +235,85 @@ def testJSONSerialize(self): self.assertIsInstance(d_node3.value, Node8) self.assertIsNot(node3.value, d_node3.value) self.assertEqual(node3.value.a, d_node3.value.a) - self.assertEqual(node3.value.b, d_node3.value.b) - self.assertIsNot(node3.value.c, d_node3.value.c) - self.assertEqual(node3.value.c.a, d_node3.value.c.a) - self.assertIsNot(node3.value.d, d_node3.value.d) - self.assertEqual(node3.value.d.a, d_node3.value.d.a) - self.assertEqual(node3.value.e[0], d_node3.value.e[0]) - self.assertNotIsInstance(d_node3.value.e[1], six.string_types) - self.assertIs(d_node3.value.e[1], d_node3.value.c) - self.assertEqual(node3.value.e[2], True) - self.assertEqual([n.b for n in node3.value.f], [n.b for n in d_node3.value.f]) - self.assertNotIsInstance(node3.value.f[0], Node8) + self.assertEqual(node3.value.b1, d_node3.value.b1) + self.assertEqual(node3.value.b2, d_node3.value.b2) + self.assertEqual(node3.value.b3, d_node3.value.b3) + self.assertEqual(node3.value.b4, d_node3.value.b4) + self.assertEqual(node3.value.c1, d_node3.value.c1) + self.assertEqual(node3.value.c2, d_node3.value.c2) + self.assertEqual(node3.value.c3, d_node3.value.c3) + self.assertEqual(node3.value.c4, d_node3.value.c4) + self.assertAlmostEqual(node3.value.d1, d_node3.value.d1, places=2) + self.assertAlmostEqual(node3.value.d2, d_node3.value.d2, places=4) + self.assertAlmostEqual(node3.value.d3, d_node3.value.d3) + self.assertAlmostEqual(node3.value.cl1, d_node3.value.cl1) + self.assertAlmostEqual(node3.value.cl2, d_node3.value.cl2) + self.assertEqual(node3.value.e, d_node3.value.e) + self.assertIsNot(node3.value.f, d_node3.value.f) + self.assertEqual(node3.value.f.a, d_node3.value.f.a) + self.assertIsNot(node3.value.g, d_node3.value.g) + self.assertEqual(node3.value.g.a, d_node3.value.g.a) + self.assertEqual(node3.value.h[0], d_node3.value.h[0]) + self.assertNotIsInstance(d_node3.value.h[1], six.string_types) + self.assertIs(d_node3.value.h[1], d_node3.value.f) + self.assertEqual(node3.value.h[2], True) + self.assertAlmostEqual(node3.value.h[6], d_node3.value.h[6]) + self.assertAlmostEqual(node3.value.h[7], d_node3.value.h[7]) + self.assertEqual([n.b1 for n in node3.value.i], [n.b1 for n in d_node3.value.i]) + self.assertIsInstance(d_node3.value.i[0], Node8) + + with self.assertRaises(ValueError): + serializes(provider, [Node3(value='sth else')]) + + def testNumpyDtypePBSerialize(self): + provider = ProtobufSerializeProvider() + + node9 = Node9(b1=np.int8(-2), b2=np.int16(2000), b3=np.int32(-5000), b4=np.int64(500000), + c1=np.uint8(2), c2=np.uint16(2000), c3=np.uint32(5000), c4=np.uint64(500000), + d1=np.float16(2.5), d2=np.float32(7.37), d3=np.float64(5.976321), + f=np.int8(3)) + + serials = serializes(provider, [node9]) + d_node9, = deserializes(provider, [Node9], serials) + + self.assertIsNot(node9, d_node9) + self.assertEqual(node9.b1, d_node9.b1) + self.assertEqual(node9.b2, d_node9.b2) + self.assertEqual(node9.b3, d_node9.b3) + self.assertEqual(node9.b4, d_node9.b4) + self.assertEqual(node9.c1, d_node9.c1) + self.assertEqual(node9.c2, d_node9.c2) + self.assertEqual(node9.c3, d_node9.c3) + self.assertEqual(node9.c4, d_node9.c4) + self.assertAlmostEqual(node9.d1, d_node9.d1, places=2) + self.assertAlmostEqual(node9.d2, d_node9.d2, places=4) + self.assertAlmostEqual(node9.d3, d_node9.d3) + self.assertEqual(node9.f, d_node9.f) + + def testNumpyDtypeJSONSerialize(self): + provider = JsonSerializeProvider() + + node9 = Node9(b1=np.int8(-2), b2=np.int16(2000), b3=np.int32(-5000), b4=np.int64(500000), + c1=np.uint8(2), c2=np.uint16(2000), c3=np.uint32(5000), c4=np.uint64(500000), + d1=np.float16(2.5), d2=np.float32(7.37), d3=np.float64(5.976321), + f=np.int8(3)) + + serials = serializes(provider, [node9]) + d_node9, = deserializes(provider, [Node9], serials) + + self.assertIsNot(node9, d_node9) + self.assertEqual(node9.b1, d_node9.b1) + self.assertEqual(node9.b2, d_node9.b2) + self.assertEqual(node9.b3, d_node9.b3) + self.assertEqual(node9.b4, d_node9.b4) + self.assertEqual(node9.c1, d_node9.c1) + self.assertEqual(node9.c2, d_node9.c2) + self.assertEqual(node9.c3, d_node9.c3) + self.assertEqual(node9.c4, d_node9.c4) + self.assertAlmostEqual(node9.d1, d_node9.d1, places=2) + self.assertAlmostEqual(node9.d2, d_node9.d2, places=4) + self.assertAlmostEqual(node9.d3, d_node9.d3) + self.assertEqual(node9.f, d_node9.f) def testAttributeAsDict(self): node4 = Node4(a=to_binary('中文'), b=np.random.randint(4, size=(3, 4)), @@ -242,7 +361,7 @@ def testAttributeAsDict(self): self.assertEqual(d_node4.i.b, 3) def testException(self): - node1 = Node1(e=[object()]) + node1 = Node1(h=[object()]) pbs = ProtobufSerializeProvider() diff --git a/mars/serialize/tests/testser.proto b/mars/serialize/tests/testser.proto index 964ff38ed9..470970a870 100644 --- a/mars/serialize/tests/testser.proto +++ b/mars/serialize/tests/testser.proto @@ -5,11 +5,24 @@ import "mars/serialize/protos/value.proto"; message Node1Def { string a = 1; - int32 b = 2; - Value c = 3; - Node2Def d = 4; - repeated Value e = 5; - repeated Node1Def f = 6; + int32 b1 = 2; + int32 b2 = 3; + int32 b3 = 4; + int64 b4 = 5; + uint32 c1 = 6; + uint32 c2 = 7; + uint32 c3 = 8; + uint64 c4 = 9; + float d1 = 10; + float d2 = 11; + double d3 = 12; + Value cl1 = 18; + Value cl2 = 19; + bool e = 13; + Value f = 14; + Node2Def g = 15; + repeated Value h = 16; + repeated Node1Def i = 17; } @@ -18,6 +31,7 @@ message Node2Def { string key = 2; repeated int32 data = 3; string id = 4; + string name = 5; } diff --git a/mars/serialize/tests/testser_pb2.py b/mars/serialize/tests/testser_pb2.py index f156606919..c04c04f5d4 100644 --- a/mars/serialize/tests/testser_pb2.py +++ b/mars/serialize/tests/testser_pb2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: mars/serialize/tests/testser.proto @@ -20,7 +21,7 @@ package='', syntax='proto3', serialized_options=None, - serialized_pb=_b('\n\"mars/serialize/tests/testser.proto\x1a!mars/serialize/protos/value.proto\"r\n\x08Node1Def\x12\t\n\x01\x61\x18\x01 \x01(\t\x12\t\n\x01\x62\x18\x02 \x01(\x05\x12\x11\n\x01\x63\x18\x03 \x01(\x0b\x32\x06.Value\x12\x14\n\x01\x64\x18\x04 \x01(\x0b\x32\t.Node2Def\x12\x11\n\x01\x65\x18\x05 \x03(\x0b\x32\x06.Value\x12\x14\n\x01\x66\x18\x06 \x03(\x0b\x32\t.Node1Def\"D\n\x08Node2Def\x12\x11\n\x01\x61\x18\x01 \x01(\x0b\x32\x06.Value\x12\x0b\n\x03key\x18\x02 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x03 \x03(\x05\x12\n\n\x02id\x18\x04 \x01(\t\"E\n\x08Node3Def\x12\x17\n\x02n1\x18\x01 \x01(\x0b\x32\t.Node1DefH\x00\x12\x17\n\x02n2\x18\x02 \x01(\x0b\x32\t.Node2DefH\x00\x42\x07\n\x05value\"b\n\x08Node4Def\x12!\n\x04\x61ttr\x18\x01 \x03(\x0b\x32\x13.Node4Def.AttrEntry\x1a\x33\n\tAttrEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x15\n\x05value\x18\x02 \x01(\x0b\x32\x06.Value:\x02\x38\x01\x62\x06proto3') + serialized_pb=_b('\n\"mars/serialize/tests/testser.proto\x1a!mars/serialize/protos/value.proto\"\xa0\x02\n\x08Node1Def\x12\t\n\x01\x61\x18\x01 \x01(\t\x12\n\n\x02\x62\x31\x18\x02 \x01(\x05\x12\n\n\x02\x62\x32\x18\x03 \x01(\x05\x12\n\n\x02\x62\x33\x18\x04 \x01(\x05\x12\n\n\x02\x62\x34\x18\x05 \x01(\x03\x12\n\n\x02\x63\x31\x18\x06 \x01(\r\x12\n\n\x02\x63\x32\x18\x07 \x01(\r\x12\n\n\x02\x63\x33\x18\x08 \x01(\r\x12\n\n\x02\x63\x34\x18\t \x01(\x04\x12\n\n\x02\x64\x31\x18\n \x01(\x02\x12\n\n\x02\x64\x32\x18\x0b \x01(\x02\x12\n\n\x02\x64\x33\x18\x0c \x01(\x01\x12\x13\n\x03\x63l1\x18\x12 \x01(\x0b\x32\x06.Value\x12\x13\n\x03\x63l2\x18\x13 \x01(\x0b\x32\x06.Value\x12\t\n\x01\x65\x18\r \x01(\x08\x12\x11\n\x01\x66\x18\x0e \x01(\x0b\x32\x06.Value\x12\x14\n\x01g\x18\x0f \x01(\x0b\x32\t.Node2Def\x12\x11\n\x01h\x18\x10 \x03(\x0b\x32\x06.Value\x12\x14\n\x01i\x18\x11 \x03(\x0b\x32\t.Node1Def\"R\n\x08Node2Def\x12\x11\n\x01\x61\x18\x01 \x01(\x0b\x32\x06.Value\x12\x0b\n\x03key\x18\x02 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x03 \x03(\x05\x12\n\n\x02id\x18\x04 \x01(\t\x12\x0c\n\x04name\x18\x05 \x01(\t\"E\n\x08Node3Def\x12\x17\n\x02n1\x18\x01 \x01(\x0b\x32\t.Node1DefH\x00\x12\x17\n\x02n2\x18\x02 \x01(\x0b\x32\t.Node2DefH\x00\x42\x07\n\x05value\"b\n\x08Node4Def\x12!\n\x04\x61ttr\x18\x01 \x03(\x0b\x32\x13.Node4Def.AttrEntry\x1a\x33\n\tAttrEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x15\n\x05value\x18\x02 \x01(\x0b\x32\x06.Value:\x02\x38\x01\x62\x06proto3') , dependencies=[mars_dot_serialize_dot_protos_dot_value__pb2.DESCRIPTOR,]) @@ -42,36 +43,127 @@ is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='b', full_name='Node1Def.b', index=1, + name='b1', full_name='Node1Def.b1', index=1, number=2, type=5, cpp_type=1, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='c', full_name='Node1Def.c', index=2, - number=3, type=11, cpp_type=10, label=1, + name='b2', full_name='Node1Def.b2', index=2, + number=3, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='b3', full_name='Node1Def.b3', index=3, + number=4, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='b4', full_name='Node1Def.b4', index=4, + number=5, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='c1', full_name='Node1Def.c1', index=5, + number=6, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='c2', full_name='Node1Def.c2', index=6, + number=7, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='c3', full_name='Node1Def.c3', index=7, + number=8, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='c4', full_name='Node1Def.c4', index=8, + number=9, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='d1', full_name='Node1Def.d1', index=9, + number=10, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='d2', full_name='Node1Def.d2', index=10, + number=11, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='d3', full_name='Node1Def.d3', index=11, + number=12, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='cl1', full_name='Node1Def.cl1', index=12, + number=18, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='d', full_name='Node1Def.d', index=3, - number=4, type=11, cpp_type=10, label=1, + name='cl2', full_name='Node1Def.cl2', index=13, + number=19, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='e', full_name='Node1Def.e', index=4, - number=5, type=11, cpp_type=10, label=3, + name='e', full_name='Node1Def.e', index=14, + number=13, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='f', full_name='Node1Def.f', index=15, + number=14, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='g', full_name='Node1Def.g', index=16, + number=15, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='h', full_name='Node1Def.h', index=17, + number=16, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='f', full_name='Node1Def.f', index=5, - number=6, type=11, cpp_type=10, label=3, + name='i', full_name='Node1Def.i', index=18, + number=17, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -88,8 +180,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=73, - serialized_end=187, + serialized_start=74, + serialized_end=362, ) @@ -128,6 +220,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='name', full_name='Node2Def.name', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -140,8 +239,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=189, - serialized_end=257, + serialized_start=364, + serialized_end=446, ) @@ -181,8 +280,8 @@ name='value', full_name='Node3Def.value', index=0, containing_type=None, fields=[]), ], - serialized_start=259, - serialized_end=328, + serialized_start=448, + serialized_end=517, ) @@ -219,8 +318,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=377, - serialized_end=428, + serialized_start=566, + serialized_end=617, ) _NODE4DEF = _descriptor.Descriptor( @@ -249,14 +348,16 @@ extension_ranges=[], oneofs=[ ], - serialized_start=330, - serialized_end=428, + serialized_start=519, + serialized_end=617, ) -_NODE1DEF.fields_by_name['c'].message_type = mars_dot_serialize_dot_protos_dot_value__pb2._VALUE -_NODE1DEF.fields_by_name['d'].message_type = _NODE2DEF -_NODE1DEF.fields_by_name['e'].message_type = mars_dot_serialize_dot_protos_dot_value__pb2._VALUE -_NODE1DEF.fields_by_name['f'].message_type = _NODE1DEF +_NODE1DEF.fields_by_name['cl1'].message_type = mars_dot_serialize_dot_protos_dot_value__pb2._VALUE +_NODE1DEF.fields_by_name['cl2'].message_type = mars_dot_serialize_dot_protos_dot_value__pb2._VALUE +_NODE1DEF.fields_by_name['f'].message_type = mars_dot_serialize_dot_protos_dot_value__pb2._VALUE +_NODE1DEF.fields_by_name['g'].message_type = _NODE2DEF +_NODE1DEF.fields_by_name['h'].message_type = mars_dot_serialize_dot_protos_dot_value__pb2._VALUE +_NODE1DEF.fields_by_name['i'].message_type = _NODE1DEF _NODE2DEF.fields_by_name['a'].message_type = mars_dot_serialize_dot_protos_dot_value__pb2._VALUE _NODE3DEF.fields_by_name['n1'].message_type = _NODE1DEF _NODE3DEF.fields_by_name['n2'].message_type = _NODE2DEF @@ -275,39 +376,39 @@ DESCRIPTOR.message_types_by_name['Node4Def'] = _NODE4DEF _sym_db.RegisterFileDescriptor(DESCRIPTOR) -Node1Def = _reflection.GeneratedProtocolMessageType('Node1Def', (_message.Message,), dict( - DESCRIPTOR = _NODE1DEF, - __module__ = 'mars.serialize.tests.testser_pb2' +Node1Def = _reflection.GeneratedProtocolMessageType('Node1Def', (_message.Message,), { + 'DESCRIPTOR' : _NODE1DEF, + '__module__' : 'mars.serialize.tests.testser_pb2' # @@protoc_insertion_point(class_scope:Node1Def) - )) + }) _sym_db.RegisterMessage(Node1Def) -Node2Def = _reflection.GeneratedProtocolMessageType('Node2Def', (_message.Message,), dict( - DESCRIPTOR = _NODE2DEF, - __module__ = 'mars.serialize.tests.testser_pb2' +Node2Def = _reflection.GeneratedProtocolMessageType('Node2Def', (_message.Message,), { + 'DESCRIPTOR' : _NODE2DEF, + '__module__' : 'mars.serialize.tests.testser_pb2' # @@protoc_insertion_point(class_scope:Node2Def) - )) + }) _sym_db.RegisterMessage(Node2Def) -Node3Def = _reflection.GeneratedProtocolMessageType('Node3Def', (_message.Message,), dict( - DESCRIPTOR = _NODE3DEF, - __module__ = 'mars.serialize.tests.testser_pb2' +Node3Def = _reflection.GeneratedProtocolMessageType('Node3Def', (_message.Message,), { + 'DESCRIPTOR' : _NODE3DEF, + '__module__' : 'mars.serialize.tests.testser_pb2' # @@protoc_insertion_point(class_scope:Node3Def) - )) + }) _sym_db.RegisterMessage(Node3Def) -Node4Def = _reflection.GeneratedProtocolMessageType('Node4Def', (_message.Message,), dict( +Node4Def = _reflection.GeneratedProtocolMessageType('Node4Def', (_message.Message,), { - AttrEntry = _reflection.GeneratedProtocolMessageType('AttrEntry', (_message.Message,), dict( - DESCRIPTOR = _NODE4DEF_ATTRENTRY, - __module__ = 'mars.serialize.tests.testser_pb2' + 'AttrEntry' : _reflection.GeneratedProtocolMessageType('AttrEntry', (_message.Message,), { + 'DESCRIPTOR' : _NODE4DEF_ATTRENTRY, + '__module__' : 'mars.serialize.tests.testser_pb2' # @@protoc_insertion_point(class_scope:Node4Def.AttrEntry) - )) + }) , - DESCRIPTOR = _NODE4DEF, - __module__ = 'mars.serialize.tests.testser_pb2' + 'DESCRIPTOR' : _NODE4DEF, + '__module__' : 'mars.serialize.tests.testser_pb2' # @@protoc_insertion_point(class_scope:Node4Def) - )) + }) _sym_db.RegisterMessage(Node4Def) _sym_db.RegisterMessage(Node4Def.AttrEntry)