draft implementation of decimal encode/decode

asynctnt/_testbase.py

@@ -147,7 +147,8 @@ def _make_instance(cls, *args, **kwargs):
             tnt = TarantoolSyncDockerInstance(
                 applua=cls.read_applua(),
                 docker_image=tarantool_docker_image,
-                docker_tag=tarantool_docker_tag
+                docker_tag=tarantool_docker_tag,
+                timeout=4*60
             )
             in_docker = True
         else:

asynctnt/iproto/buffer.pxd

@@ -37,6 +37,7 @@ cdef class WriteBuffer:
                              const char *str, uint32_t len) except NULL
     cdef char *mp_encode_bin(self, char *p,
                              const char *data, uint32_t len) except NULL
+    cdef char *mp_encode_decimal(self, char *p, object value) except NULL
     cdef char *mp_encode_array(self, char *p, uint32_t len) except NULL
     cdef char *mp_encode_map(self, char *p, uint32_t len) except NULL
     cdef char *mp_encode_list(self, char *p, list arr) except NULL

asynctnt/iproto/buffer.pyx

@@ -10,7 +10,10 @@ from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free
 from cpython.ref cimport PyObject
 
 from libc.string cimport memcpy
-from libc.stdint cimport uint32_t, uint64_t, int64_t
+from libc.stdint cimport uint32_t, uint64_t, int64_t, uint8_t
+from libc.stdio cimport printf
+
+from decimal import Decimal
 
 # noinspection PyUnresolvedReferences
 # noinspection PyAttributeOutsideInit
@@ -208,6 +211,36 @@ cdef class WriteBuffer:
         self._length += (p - begin)
         return p
 
+    cdef char *mp_encode_decimal(self, char *p, object value) except NULL:
+        cdef:
+            char *begin
+            char *svp
+            uint8_t sign
+            tuple digits
+            int exponent
+            uint32_t length
+
+        decimal_tuple = value.as_tuple()
+        sign = <uint8_t> decimal_tuple.sign
+        digits = <tuple> decimal_tuple.digits
+        exponent = <int> decimal_tuple.exponent
+
+        length = decimal_len(exponent, digits)
+
+        p = begin = self._ensure_allocated(p, mp_sizeof_ext(length))
+
+        # encode header
+        p = mp_encode_extl(p, tarantool.MP_DECIMAL, length)
+
+        # encode decimal
+        p = decimal_encode(p, sign, digits, exponent)
+
+        self._length += (p - begin)
+
+        printf("%s\n", xd(begin, (p-begin)))
+
+        return p
+
     cdef char *mp_encode_array(self, char *p, uint32_t len) except NULL:
         cdef char *begin
         p = begin = self._ensure_allocated(p, mp_sizeof_array(len))
@@ -292,12 +325,12 @@ cdef class WriteBuffer:
         if o is None:
             return self.mp_encode_nil(p)
 
-        elif isinstance(o, bool):
-            return self.mp_encode_bool(p, <bint> o)
-
         elif isinstance(o, float):
             return self.mp_encode_double(p, <double> o)
 
+        elif isinstance(o, bool):
+            return self.mp_encode_bool(p, <bint> o)
+
         elif isinstance(o, int):
             if o >= 0:
                 return self.mp_encode_uint(p, <uint64_t> o)
@@ -332,6 +365,9 @@ cdef class WriteBuffer:
         elif isinstance(o, dict):
             return self.mp_encode_dict(p, <dict> o)
 
+        elif isinstance(o, Decimal):
+            return self.mp_encode_decimal(p, o)
+
         else:
             raise TypeError(
                 'Type `{}` is not supported for encoding'.format(type(o)))

asynctnt/iproto/cmsgpuck.pxd

@@ -1,4 +1,4 @@
-from libc.stdint cimport uint32_t, uint64_t, int64_t, uint8_t, uint16_t
+from libc.stdint cimport uint32_t, uint64_t, int64_t, uint8_t, uint16_t, int8_t
 from libc.stdio cimport FILE
 
 cdef extern from "../../third_party/msgpuck/msgpuck.h":
@@ -64,6 +64,13 @@ cdef extern from "../../third_party/msgpuck/msgpuck.h":
     cdef char *mp_encode_bin(char *data, const char *str, uint32_t len)
     cdef const char *mp_decode_bin(const char **data, uint32_t *len)
 
+    cdef uint32_t mp_sizeof_extl(uint32_t len)
+    cdef uint32_t mp_sizeof_ext(uint32_t len)
+    cdef char *mp_encode_extl(char *data, int8_t type, uint32_t len)
+    cdef char *mp_encode_ext(char *data, int8_t type, char *str, uint32_t len)
+    cdef uint32_t mp_decode_extl(const char **data, int8_t *type)
+    cdef const char *mp_decode_ext(const char **data, int8_t *type, uint32_t *len)
+
     cdef uint32_t mp_decode_strbinl(const char **data)
     cdef const char *mp_decode_strbin(const char **data, uint32_t *len)
 

asynctnt/iproto/ext.pxd

@@ -0,0 +1,5 @@
+from libc.stdint cimport uint32_t, uint8_t
+
+cdef uint32_t decimal_len(int exponent, tuple digits)
+cdef char *decimal_encode(char *p, uint8_t sign, tuple digits, int exponent)
+cdef object decimal_decode(const char **p, uint32_t length)

asynctnt/iproto/ext.pyx

@@ -0,0 +1,161 @@
+from libc cimport math
+
+from decimal import Decimal
+
+cdef uint32_t decimal_len(int exponent, tuple digits):
+    cdef:
+        uint32_t length
+        uint32_t digits_len
+        bint is_odd
+
+    length = 0
+    if exponent > 0:
+        length += mp_sizeof_int(-exponent)
+    else:
+        length += mp_sizeof_uint(-exponent)
+
+    digits_len = <uint32_t> len(digits)
+    is_odd = (digits_len % 2 == 1)
+    length += 1 + <uint32_t> math.ceil((is_odd + digits_len) / 2)
+    return length
+
+cdef char *decimal_encode(char *p, uint8_t sign, tuple digits, int exponent):
+    cdef:
+        uint8_t digit
+        int i
+        uint8_t byte
+        uint32_t n
+        uint32_t digits_len
+        bint is_even
+        uint32_t delta
+
+    # encode exponent
+    if exponent > 0:
+        p = mp_encode_int(p, -exponent)
+    else:
+        p = mp_encode_uint(p, -exponent)
+
+    digits_len = <uint32_t> len(digits)
+    is_odd = (digits_len % 2 == 1)
+
+    print('encode', exponent, digits, sign)
+
+    # encode 1st digit
+    if is_odd:
+        p[0] = 0
+        p += 1
+        delta = 0
+    else:
+        byte = <uint8_t> digits[0]
+        p[0] = <char> byte
+        p += 1
+        delta = 1
+
+    n = <uint32_t> math.ceil((digits_len - (delta + 1)) / 2)
+    for i in range(n):
+        byte = 0
+        byte = <uint8_t> digits[delta + i]
+        byte <<= 4
+        byte |= <uint8_t> digits[delta + i + 1]
+        p[0] = <char> byte
+        p += 1
+
+    # encode last digit
+    byte = 0
+    byte = digits[digits_len - 1]
+    byte <<= 4
+
+    # encode nibble
+    if sign == 1:
+        byte |= 0x0d
+    else:
+        byte |= 0x0c
+
+    p[0] = <char> byte
+    p += 1
+    return p
+
+cdef object decimal_decode(const char ** p, uint32_t length):
+    cdef:
+        int exponent
+        uint8_t sign
+        mp_type obj_type
+        const char *svp
+        uint32_t digits_count
+        uint32_t i, total
+        uint8_t dig1, dig2
+
+    sign = 0
+    svp = &p[0][0]
+
+    # decode exponent
+    obj_type = mp_typeof(p[0][0])
+    if obj_type == MP_UINT:
+        exponent = -<int> mp_decode_uint(p)
+    elif obj_type == MP_INT:
+        exponent = -<int> mp_decode_int(p)
+    else:
+        raise TypeError('unexpected exponent type: {}'.format(obj_type))
+
+    length -= (&p[0][0] - svp)
+    svp = &p[0][0]
+
+    # decode digits
+    digits_count = 2 + 2 * (length - 2)
+    print('digits count:', digits_count)
+    digits = cpython.tuple.PyTuple_New(digits_count)
+
+    # decode 1st digit
+    dig1 = <uint8_t> svp[0]
+    item = <object> <int> dig1
+    cpython.Py_INCREF(item)
+    cpython.tuple.PyTuple_SET_ITEM(digits, 0, item)
+    print(item)
+
+    svp += 1
+
+    # decode digits
+    total = 1
+    i = 1
+    for i in range(1, length - 1):
+        dig2 = <uint8_t> svp[0]
+        dig1 = dig2 >> 4
+        dig2 &= 0x0f
+        svp += 1
+
+        item = <object> <int> dig1
+        cpython.Py_INCREF(item)
+        cpython.tuple.PyTuple_SET_ITEM(digits, total, item)
+        total += 1
+        print(item)
+
+        item = <object> <int> dig2
+        cpython.Py_INCREF(item)
+        cpython.tuple.PyTuple_SET_ITEM(digits, total, item)
+        total += 1
+        print(item)
+
+    # decode last digit and nibble
+    dig2 = <uint8_t> svp[0]
+    dig1 = dig2 >> 4
+    dig2 &= 0x0f
+    svp += 1
+
+    item = <object> <int> dig1
+    cpython.Py_INCREF(item)
+    cpython.tuple.PyTuple_SET_ITEM(digits, total, item)
+    total += 1
+
+    sign = dig2
+    if sign == 0x0a or sign == 0x0c or sign == 0x0e or sign == 0x0f:
+        sign = 0
+    else:
+        sign = 1
+
+    print(item, sign)
+
+    print('length: ', length)
+    p[0] += length
+    print('digits', digits)
+
+    return Decimal((<object> <int> sign, digits, <object> exponent))

asynctnt/iproto/protocol.pxd

@@ -4,10 +4,12 @@ cimport asynctnt.iproto.tarantool as tarantool
 include "const.pxi"
 
 include "cmsgpuck.pxd"
+include "xd.pxd"
 include "python.pxd"
 
 include "unicodeutil.pxd"
 include "schema.pxd"
+include "ext.pxd"
 include "buffer.pxd"
 include "rbuffer.pxd"
 

asynctnt/iproto/protocol.pyx

@@ -11,6 +11,7 @@ include "const.pxi"
 
 include "unicodeutil.pyx"
 include "schema.pyx"
+include "ext.pyx"
 include "buffer.pyx"
 include "rbuffer.pyx"
 

asynctnt/iproto/response.pyx

@@ -198,6 +198,8 @@ cdef object _decode_obj(const char **p, bytes encoding):
         uint32_t map_key_len
         object map_key
 
+        int8_t ext_type
+
     obj_type = mp_typeof(p[0][0])
     if obj_type == MP_UINT:
         return mp_decode_uint(p)
@@ -262,6 +264,14 @@ cdef object _decode_obj(const char **p, bytes encoding):
     elif obj_type == MP_NIL:
         mp_next(p)
         return None
+    elif obj_type == MP_EXT:
+        ext_type = 0
+        s_len = mp_decode_extl(p, &ext_type)
+        if ext_type == tarantool.MP_DECIMAL:
+            return decimal_decode(p, s_len)
+        else:
+            logger.warning('Unexpected ext type: %d', ext_type)
+            return None
     else:  # pragma: nocover
         mp_next(p)
         logger.warning('Unexpected obj type: %s', obj_type)

asynctnt/iproto/tarantool.pxd

@@ -76,3 +76,8 @@ cdef enum iproto_update_operation:
     IPROTO_OP_INSERT = b'!'
     IPROTO_OP_ASSIGN = b'='
     IPROTO_OP_SPLICE = b':'
+
+
+cdef enum mp_extension_type:
+    MP_UNKNOWN_EXTENSION = 0
+    MP_DECIMAL = 1

asynctnt/iproto/xd.pxd

@@ -0,0 +1,2 @@
+cdef extern from "../../third_party/xd.h":
+    char * xd(char *data, size_t size)

tests/files/app.lua

@@ -60,6 +60,7 @@ local function bootstrap()
         b.types.string = 'string'
         b.types.unsigned = 'unsigned'
         b.types.integer = 'integer'
+        b.types.decimal = 'decimal'
     else
         b.types.string = 'str'
         b.types.unsigned = 'num'
@@ -135,6 +136,15 @@ if B:check_version({2, 0}) then
             )
         ]])
     end)
+
+    box.once('v2.1', function()
+        local s = box.schema.create_space('tester_ext')
+        s:format({
+            {type=B.types.unsigned, name='f1'},
+            {type=B.types.decimal, name='f2'},
+        })
+        s:create_index('primary')
+    end)
 end