diff --git a/docs/release.rst b/docs/release.rst index 59ed6d28..e865a9e0 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -37,6 +37,13 @@ Fixes ~~~~~ * Remove redundant ``id`` from codec metadata serialization in Zarr3 codecs. By :user:`Norman Rzepka `, :issue:`685` +* Preallocate output buffers and resize directly as needed. + By :user:`John Kirkham `, :issue:`656` + +Maintenance +~~~~~~~~~~~ +* Replace internal ``Buffer`` usage with ``memoryview``\ s. + By :user:`John Kirkham `, :issue:`656` .. _release_0.15.0: diff --git a/numcodecs/blosc.pyx b/numcodecs/blosc.pyx index 3082aad4..17e6b4d8 100644 --- a/numcodecs/blosc.pyx +++ b/numcodecs/blosc.pyx @@ -9,12 +9,11 @@ import os from deprecated import deprecated -from cpython.buffer cimport PyBUF_ANY_CONTIGUOUS, PyBUF_WRITEABLE -from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AS_STRING +from cpython.bytes cimport PyBytes_AS_STRING, PyBytes_FromStringAndSize +from cpython.memoryview cimport PyMemoryView_GET_BUFFER +from .compat_ext cimport PyBytes_RESIZE, ensure_continguous_memoryview -from .compat_ext cimport Buffer -from .compat_ext import Buffer from .compat import ensure_contiguous_ndarray from .abc import Codec @@ -154,17 +153,16 @@ def _cbuffer_sizes(source): """ cdef: - Buffer buffer + memoryview source_mv + const Py_buffer* source_pb size_t nbytes, cbytes, blocksize - # obtain buffer - buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS) + # obtain source memoryview + source_mv = ensure_continguous_memoryview(source) + source_pb = PyMemoryView_GET_BUFFER(source_mv) # determine buffer size - blosc_cbuffer_sizes(buffer.ptr, &nbytes, &cbytes, &blocksize) - - # release buffers - buffer.release() + blosc_cbuffer_sizes(source_pb.buf, &nbytes, &cbytes, &blocksize) return nbytes, cbytes, blocksize @@ -173,16 +171,15 @@ cbuffer_sizes = deprecated(_cbuffer_sizes) def cbuffer_complib(source): """Return the name of the compression library used to compress `source`.""" cdef: - Buffer buffer + memoryview source_mv + const Py_buffer* source_pb - # obtain buffer - buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS) + # obtain source memoryview + source_mv = ensure_continguous_memoryview(source) + source_pb = PyMemoryView_GET_BUFFER(source_mv) # determine buffer size - complib = blosc_cbuffer_complib(buffer.ptr) - - # release buffers - buffer.release() + complib = blosc_cbuffer_complib(source_pb.buf) complib = complib.decode('ascii') @@ -202,18 +199,17 @@ def _cbuffer_metainfo(source): """ cdef: - Buffer buffer + memoryview source_mv + const Py_buffer* source_pb size_t typesize int flags - # obtain buffer - buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS) + # obtain source memoryview + source_mv = ensure_continguous_memoryview(source) + source_pb = PyMemoryView_GET_BUFFER(source_mv) # determine buffer size - blosc_cbuffer_metainfo(buffer.ptr, &typesize, &flags) - - # release buffers - buffer.release() + blosc_cbuffer_metainfo(source_pb.buf, &typesize, &flags) # decompose flags if flags & BLOSC_DOSHUFFLE: @@ -263,28 +259,34 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE, """ cdef: - char *source_ptr - char *dest_ptr - Buffer source_buffer + memoryview source_mv + const Py_buffer* source_pb + const char* source_ptr size_t nbytes, itemsize int cbytes bytes dest + char* dest_ptr # check valid cname early cname_str = cname.decode('ascii') if cname_str not in list_compressors(): _err_bad_cname(cname_str) - # setup source buffer - source_buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS) - source_ptr = source_buffer.ptr - nbytes = source_buffer.nbytes + # obtain source memoryview + source_mv = ensure_continguous_memoryview(source) + source_pb = PyMemoryView_GET_BUFFER(source_mv) + + # extract metadata + source_ptr = source_pb.buf + nbytes = source_pb.len + + # validate typesize if isinstance(typesize, int): if typesize < 1: raise ValueError(f"Cannot use typesize {typesize} less than 1.") itemsize = typesize else: - itemsize = source_buffer.itemsize + itemsize = source_pb.itemsize # determine shuffle if shuffle == AUTOSHUFFLE: @@ -333,16 +335,14 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE, cname, blocksize, 1) finally: - - # release buffers - source_buffer.release() + pass # check compression was successful if cbytes <= 0: raise RuntimeError('error during blosc compression: %d' % cbytes) # resize after compression - dest = dest[:cbytes] + PyBytes_RESIZE(dest, cbytes) return dest @@ -366,15 +366,20 @@ def decompress(source, dest=None): """ cdef: int ret - char *source_ptr - char *dest_ptr - Buffer source_buffer - Buffer dest_buffer = None + memoryview source_mv + const Py_buffer* source_pb + const char* source_ptr + memoryview dest_mv + Py_buffer* dest_pb + char* dest_ptr size_t nbytes, cbytes, blocksize - # setup source buffer - source_buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS) - source_ptr = source_buffer.ptr + # obtain source memoryview + source_mv = ensure_continguous_memoryview(source) + source_pb = PyMemoryView_GET_BUFFER(source_mv) + + # get source pointer + source_ptr = source_pb.buf # determine buffer size blosc_cbuffer_sizes(source_ptr, &nbytes, &cbytes, &blocksize) @@ -382,14 +387,15 @@ def decompress(source, dest=None): # setup destination buffer if dest is None: # allocate memory - dest = PyBytes_FromStringAndSize(NULL, nbytes) - dest_ptr = PyBytes_AS_STRING(dest) - dest_nbytes = nbytes + dest_1d = dest = PyBytes_FromStringAndSize(NULL, nbytes) else: - arr = ensure_contiguous_ndarray(dest) - dest_buffer = Buffer(arr, PyBUF_ANY_CONTIGUOUS | PyBUF_WRITEABLE) - dest_ptr = dest_buffer.ptr - dest_nbytes = dest_buffer.nbytes + dest_1d = ensure_contiguous_ndarray(dest) + + # obtain dest memoryview + dest_mv = memoryview(dest_1d) + dest_pb = PyMemoryView_GET_BUFFER(dest_mv) + dest_ptr = dest_pb.buf + dest_nbytes = dest_pb.len try: @@ -408,11 +414,7 @@ def decompress(source, dest=None): ret = blosc_decompress_ctx(source_ptr, dest_ptr, nbytes, 1) finally: - - # release buffers - source_buffer.release() - if dest_buffer is not None: - dest_buffer.release() + pass # handle errors if ret <= 0: @@ -449,14 +451,20 @@ def _decompress_partial(source, start, nitems, dest=None): int encoding_size int nitems_bytes int start_bytes - char *source_ptr - char *dest_ptr - Buffer source_buffer - Buffer dest_buffer = None + memoryview source_mv + const Py_buffer* source_pb + const char* source_ptr + memoryview dest_mv + Py_buffer* dest_pb + char* dest_ptr + size_t dest_nbytes - # setup source buffer - source_buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS) - source_ptr = source_buffer.ptr + # obtain source memoryview + source_mv = ensure_continguous_memoryview(source) + source_pb = PyMemoryView_GET_BUFFER(source_mv) + + # setup source pointer + source_ptr = source_pb.buf # get encoding size from source buffer header encoding_size = source[3] @@ -467,14 +475,16 @@ def _decompress_partial(source, start, nitems, dest=None): # setup destination buffer if dest is None: - dest = PyBytes_FromStringAndSize(NULL, nitems_bytes) - dest_ptr = PyBytes_AS_STRING(dest) - dest_nbytes = nitems_bytes + # allocate memory + dest_1d = dest = PyBytes_FromStringAndSize(NULL, nitems_bytes) else: - arr = ensure_contiguous_ndarray(dest) - dest_buffer = Buffer(arr, PyBUF_ANY_CONTIGUOUS | PyBUF_WRITEABLE) - dest_ptr = dest_buffer.ptr - dest_nbytes = dest_buffer.nbytes + dest_1d = ensure_contiguous_ndarray(dest) + + # obtain dest memoryview + dest_mv = memoryview(dest_1d) + dest_pb = PyMemoryView_GET_BUFFER(dest_mv) + dest_ptr = dest_pb.buf + dest_nbytes = dest_pb.len # try decompression try: @@ -482,11 +492,8 @@ def _decompress_partial(source, start, nitems, dest=None): raise ValueError('destination buffer too small; expected at least %s, ' 'got %s' % (nitems_bytes, dest_nbytes)) ret = blosc_getitem(source_ptr, start, nitems, dest_ptr) - finally: - source_buffer.release() - if dest_buffer is not None: - dest_buffer.release() + pass # ret refers to the number of bytes returned from blosc_getitem. if ret <= 0: diff --git a/numcodecs/compat_ext.pxd b/numcodecs/compat_ext.pxd index dfcaee0f..436c23fb 100644 --- a/numcodecs/compat_ext.pxd +++ b/numcodecs/compat_ext.pxd @@ -1,12 +1,11 @@ # cython: language_level=3 -cdef class Buffer: - cdef: - char *ptr - Py_buffer buffer - size_t nbytes - size_t itemsize - bint acquired - - cpdef release(self) +cdef extern from *: + """ + #define PyBytes_RESIZE(b, n) _PyBytes_Resize(&b, n) + """ + int PyBytes_RESIZE(object b, Py_ssize_t n) except -1 + + +cpdef memoryview ensure_continguous_memoryview(obj) diff --git a/numcodecs/compat_ext.pyx b/numcodecs/compat_ext.pyx index f57e3cfd..ae6a8c44 100644 --- a/numcodecs/compat_ext.pyx +++ b/numcodecs/compat_ext.pyx @@ -3,26 +3,17 @@ # cython: linetrace=False # cython: binding=False # cython: language_level=3 -from cpython.buffer cimport PyObject_GetBuffer, PyBuffer_Release +from cpython.buffer cimport PyBuffer_IsContiguous +from cpython.memoryview cimport PyMemoryView_GET_BUFFER -from .compat import ensure_contiguous_ndarray - -cdef class Buffer: - """Convenience class for buffer interface.""" - - def __cinit__(self, obj, int flags): - PyObject_GetBuffer(obj, &(self.buffer), flags) - self.acquired = True - self.ptr = self.buffer.buf - self.itemsize = self.buffer.itemsize - self.nbytes = self.buffer.len - - cpdef release(self): - if self.acquired: - PyBuffer_Release(&(self.buffer)) - self.acquired = False - - def __dealloc__(self): - self.release() +cpdef memoryview ensure_continguous_memoryview(obj): + cdef memoryview mv + if type(obj) is memoryview: + mv = obj + else: + mv = memoryview(obj) + if not PyBuffer_IsContiguous(PyMemoryView_GET_BUFFER(mv), b'A'): + raise BufferError("Expected contiguous memory") + return mv diff --git a/numcodecs/fletcher32.pyx b/numcodecs/fletcher32.pyx index 7c7b159f..ed549331 100644 --- a/numcodecs/fletcher32.pyx +++ b/numcodecs/fletcher32.pyx @@ -1,12 +1,17 @@ # cython: language_level=3 # cython: overflowcheck=False # cython: cdivision=True -import struct -from numcodecs.abc import Codec -from numcodecs.compat import ensure_contiguous_ndarray from libc.stdint cimport uint8_t, uint16_t, uint32_t +from libc.string cimport memcpy + +from cpython.bytes cimport PyBytes_FromStringAndSize + +from ._utils cimport store_le32, load_le32 + +from numcodecs.abc import Codec +from numcodecs.compat import ensure_contiguous_ndarray cdef uint32_t _fletcher32(const uint8_t[::1] _data): @@ -63,23 +68,41 @@ class Fletcher32(Codec): def encode(self, buf): """Return buffer plus 4-byte fletcher checksum""" buf = ensure_contiguous_ndarray(buf).ravel().view('uint8') - cdef const uint8_t[::1] b_ptr = buf - val = _fletcher32(b_ptr) - return buf.tobytes() + struct.pack("out + + memcpy(out_ptr, b_ptr, b_len) + store_le32(out_ptr + b_len, _fletcher32(b_mv)) + + return out def decode(self, buf, out=None): """Check fletcher checksum, and return buffer without it""" b = ensure_contiguous_ndarray(buf).view('uint8') - cdef const uint8_t[::1] b_ptr = b[:-4] - val = _fletcher32(b_ptr) - found = b[-4:].view("source_pb.buf + source_size = source_pb.len try: @@ -97,9 +101,7 @@ def compress(source, int acceleration=DEFAULT_ACCELERATION): acceleration) finally: - - # release buffers - source_buffer.release() + pass # check compression was successful if compressed_size <= 0: @@ -107,7 +109,7 @@ def compress(source, int acceleration=DEFAULT_ACCELERATION): # resize after compression compressed_size += sizeof(uint32_t) - dest = dest[:compressed_size] + PyBytes_RESIZE(dest, compressed_size) return dest @@ -129,17 +131,22 @@ def decompress(source, dest=None): """ cdef: - char *source_ptr - char *source_start - char *dest_ptr - Buffer source_buffer - Buffer dest_buffer = None + memoryview source_mv + const Py_buffer* source_pb + const char* source_ptr + const char* source_start + memoryview dest_mv + Py_buffer* dest_pb + char* dest_ptr int source_size, dest_size, decompressed_size # setup source buffer - source_buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS) - source_ptr = source_buffer.ptr - source_size = source_buffer.nbytes + source_mv = ensure_continguous_memoryview(source) + source_pb = PyMemoryView_GET_BUFFER(source_mv) + + # extract source metadata + source_ptr = source_pb.buf + source_size = source_pb.len try: @@ -155,26 +162,26 @@ def decompress(source, dest=None): # setup destination buffer if dest is None: # allocate memory - dest = PyBytes_FromStringAndSize(NULL, dest_size) - dest_ptr = PyBytes_AS_STRING(dest) + dest_1d = dest = PyBytes_FromStringAndSize(NULL, dest_size) else: - arr = ensure_contiguous_ndarray(dest) - dest_buffer = Buffer(arr, PyBUF_ANY_CONTIGUOUS | PyBUF_WRITEABLE) - dest_ptr = dest_buffer.ptr - if dest_buffer.nbytes < dest_size: - raise ValueError('destination buffer too small; expected at least %s, ' - 'got %s' % (dest_size, dest_buffer.nbytes)) + dest_1d = ensure_contiguous_ndarray(dest) + + # obtain dest memoryview + dest_mv = memoryview(dest_1d) + dest_pb = PyMemoryView_GET_BUFFER(dest_mv) + dest_ptr = dest_pb.buf + dest_nbytes = dest_pb.len + + if dest_nbytes < dest_size: + raise ValueError('destination buffer too small; expected at least %s, ' + 'got %s' % (dest_size, dest_nbytes)) # perform decompression with nogil: decompressed_size = LZ4_decompress_safe(source_start, dest_ptr, source_size, dest_size) finally: - - # release buffers - source_buffer.release() - if dest_buffer is not None: - dest_buffer.release() + pass # check decompression was successful if decompressed_size <= 0: diff --git a/numcodecs/vlen.pyx b/numcodecs/vlen.pyx index e1e149ee..f564a66e 100644 --- a/numcodecs/vlen.pyx +++ b/numcodecs/vlen.pyx @@ -5,27 +5,38 @@ # cython: language_level=3 -import cython cimport cython -from numpy cimport ndarray -import numpy as np -from .abc import Codec -from .compat_ext cimport Buffer -from .compat_ext import Buffer -from .compat import ensure_contiguous_ndarray -from cpython cimport (PyBytes_GET_SIZE, PyBytes_AS_STRING, PyBytes_Check, - PyBytes_FromStringAndSize, PyUnicode_AsUTF8String) -from cpython.buffer cimport PyBUF_ANY_CONTIGUOUS + from libc.stdint cimport uint8_t from libc.string cimport memcpy + +from cpython.buffer cimport PyBuffer_IsContiguous +from cpython.bytearray cimport ( + PyByteArray_AS_STRING, + PyByteArray_FromStringAndSize, +) +from cpython.bytes cimport ( + PyBytes_AS_STRING, + PyBytes_GET_SIZE, + PyBytes_Check, + PyBytes_FromStringAndSize, +) +from cpython.memoryview cimport PyMemoryView_GET_BUFFER +from cpython.unicode cimport ( + PyUnicode_AsUTF8String, + PyUnicode_Check, + PyUnicode_FromStringAndSize, +) + +from numpy cimport ndarray + +from .compat_ext cimport ensure_continguous_memoryview from ._utils cimport store_le32, load_le32 +import numpy as np -cdef extern from "Python.h": - bytearray PyByteArray_FromStringAndSize(char *v, Py_ssize_t l) - char* PyByteArray_AS_STRING(object string) - object PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) - int PyUnicode_Check(object text) +from .abc import Codec +from .compat import ensure_contiguous_ndarray # 4 bytes to store number of items @@ -132,23 +143,26 @@ class VLenUTF8(Codec): @cython.boundscheck(False) def decode(self, buf, out=None): cdef: - Buffer input_buffer - char* data - char* data_end - Py_ssize_t i, l, n_items, data_length, input_length + memoryview buf_mv + const Py_buffer* buf_pb + const char* data + const char* data_end + Py_ssize_t i, l, n_items, data_length - # accept any buffer + # obtain memoryview buf = ensure_contiguous_ndarray(buf) - input_buffer = Buffer(buf, PyBUF_ANY_CONTIGUOUS) - input_length = input_buffer.nbytes + buf_mv = memoryview(buf) + buf_pb = PyMemoryView_GET_BUFFER(buf_mv) # sanity checks - if input_length < HEADER_LENGTH: + if not PyBuffer_IsContiguous(buf_pb, b'A'): + raise BufferError("`buf` must contain contiguous memory") + if buf_pb.len < HEADER_LENGTH: raise ValueError('corrupt buffer, missing or truncated header') # obtain input data pointer - data = input_buffer.ptr - data_end = data + input_length + data = buf_pb.buf + data_end = data + buf_pb.len # load number of items n_items = load_le32(data) @@ -260,23 +274,26 @@ class VLenBytes(Codec): @cython.boundscheck(False) def decode(self, buf, out=None): cdef: - Buffer input_buffer - char* data - char* data_end - Py_ssize_t i, l, n_items, data_length, input_length + memoryview buf_mv + const Py_buffer* buf_pb + const char* data + const char* data_end + Py_ssize_t i, l, n_items, data_length - # accept any buffer + # obtain memoryview buf = ensure_contiguous_ndarray(buf) - input_buffer = Buffer(buf, PyBUF_ANY_CONTIGUOUS) - input_length = input_buffer.nbytes + buf_mv = memoryview(buf) + buf_pb = PyMemoryView_GET_BUFFER(buf_mv) # sanity checks - if input_length < HEADER_LENGTH: + if not PyBuffer_IsContiguous(buf_pb, b'A'): + raise BufferError("`buf` must contain contiguous memory") + if buf_pb.len < HEADER_LENGTH: raise ValueError('corrupt buffer, missing or truncated header') # obtain input data pointer - data = input_buffer.ptr - data_end = data + input_length + data = buf_pb.buf + data_end = data + buf_pb.len # load number of items n_items = load_le32(data) @@ -352,11 +369,12 @@ class VLenArray(Codec): object[:] values object[:] normed_values int[:] lengths - char* encv + const char* encv bytes b bytearray out char* data - Buffer value_buffer + memoryview value_mv + const Py_buffer* value_pb object v # normalise input @@ -398,11 +416,13 @@ class VLenArray(Codec): l = lengths[i] store_le32(data, l) data += 4 - value_buffer = Buffer(normed_values[i], PyBUF_ANY_CONTIGUOUS) - encv = value_buffer.ptr + + value_mv = ensure_continguous_memoryview(normed_values[i]) + value_pb = PyMemoryView_GET_BUFFER(value_mv) + encv = value_pb.buf + memcpy(data, encv, l) data += l - value_buffer.release() return out @@ -410,23 +430,29 @@ class VLenArray(Codec): @cython.boundscheck(False) def decode(self, buf, out=None): cdef: - Buffer input_buffer - char* data - char* data_end - Py_ssize_t i, l, n_items, data_length, input_length + memoryview buf_mv + const Py_buffer* buf_pb + const char* data + const char* data_end + object v + memoryview v_mv + Py_buffer* v_pb + Py_ssize_t i, l, n_items, data_length - # accept any buffer + # obtain memoryview buf = ensure_contiguous_ndarray(buf) - input_buffer = Buffer(buf, PyBUF_ANY_CONTIGUOUS) - input_length = input_buffer.nbytes + buf_mv = memoryview(buf) + buf_pb = PyMemoryView_GET_BUFFER(buf_mv) # sanity checks - if input_length < HEADER_LENGTH: + if not PyBuffer_IsContiguous(buf_pb, b'A'): + raise BufferError("`buf` must contain contiguous memory") + if buf_pb.len < HEADER_LENGTH: raise ValueError('corrupt buffer, missing or truncated header') # obtain input data pointer - data = input_buffer.ptr - data_end = data + input_length + data = buf_pb.buf + data_end = data + buf_pb.len # load number of items n_items = load_le32(data) @@ -448,7 +474,14 @@ class VLenArray(Codec): data += 4 if data + l > data_end: raise ValueError('corrupt buffer, data seem truncated') - out[i] = np.frombuffer(data[:l], dtype=self.dtype) + + # Create & fill array value + v = np.empty((l,), dtype="uint8").view(self.dtype) + v_mv = memoryview(v) + v_pb = PyMemoryView_GET_BUFFER(v_mv) + memcpy(v_pb.buf, data, l) + + out[i] = v data += l return out diff --git a/numcodecs/zstd.pyx b/numcodecs/zstd.pyx index efd12fa2..82f2844a 100644 --- a/numcodecs/zstd.pyx +++ b/numcodecs/zstd.pyx @@ -5,12 +5,11 @@ # cython: language_level=3 -from cpython.buffer cimport PyBUF_ANY_CONTIGUOUS, PyBUF_WRITEABLE -from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AS_STRING +from cpython.bytes cimport PyBytes_AS_STRING, PyBytes_FromStringAndSize +from cpython.memoryview cimport PyMemoryView_GET_BUFFER +from .compat_ext cimport PyBytes_RESIZE, ensure_continguous_memoryview -from .compat_ext cimport Buffer -from .compat_ext import Buffer from .compat import ensure_contiguous_ndarray from .abc import Codec @@ -92,20 +91,24 @@ def compress(source, int level=DEFAULT_CLEVEL, bint checksum=False): """ cdef: - char *source_ptr - char *dest_ptr - Buffer source_buffer + memoryview source_mv + const Py_buffer* source_pb + const char* source_ptr size_t source_size, dest_size, compressed_size bytes dest + char* dest_ptr # check level if level > MAX_CLEVEL: level = MAX_CLEVEL + # obtain source memoryview + source_mv = ensure_continguous_memoryview(source) + source_pb = PyMemoryView_GET_BUFFER(source_mv) + # setup source buffer - source_buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS) - source_ptr = source_buffer.ptr - source_size = source_buffer.nbytes + source_ptr = source_pb.buf + source_size = source_pb.len cctx = ZSTD_createCCtx() param_set_result = ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) @@ -134,8 +137,6 @@ def compress(source, int level=DEFAULT_CLEVEL, bint checksum=False): finally: if cctx: ZSTD_freeCCtx(cctx) - # release buffers - source_buffer.release() # check compression was successful if ZSTD_isError(compressed_size): @@ -143,7 +144,7 @@ def compress(source, int level=DEFAULT_CLEVEL, bint checksum=False): raise RuntimeError('Zstd compression error: %s' % error) # resize after compression - dest = dest[:compressed_size] + PyBytes_RESIZE(dest, compressed_size) return dest @@ -165,16 +166,22 @@ def decompress(source, dest=None): """ cdef: - char *source_ptr - char *dest_ptr - Buffer source_buffer - Buffer dest_buffer = None + memoryview source_mv + const Py_buffer* source_pb + const char* source_ptr + memoryview dest_mv + Py_buffer* dest_pb + char* dest_ptr size_t source_size, dest_size, decompressed_size + size_t nbytes, cbytes, blocksize - # setup source buffer - source_buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS) - source_ptr = source_buffer.ptr - source_size = source_buffer.nbytes + # obtain source memoryview + source_mv = ensure_continguous_memoryview(source) + source_pb = PyMemoryView_GET_BUFFER(source_mv) + + # get source pointer + source_ptr = source_pb.buf + source_size = source_pb.len try: @@ -186,26 +193,27 @@ def decompress(source, dest=None): # setup destination buffer if dest is None: # allocate memory - dest = PyBytes_FromStringAndSize(NULL, dest_size) - dest_ptr = PyBytes_AS_STRING(dest) + dest_1d = dest = PyBytes_FromStringAndSize(NULL, dest_size) else: - arr = ensure_contiguous_ndarray(dest) - dest_buffer = Buffer(arr, PyBUF_ANY_CONTIGUOUS | PyBUF_WRITEABLE) - dest_ptr = dest_buffer.ptr - if dest_buffer.nbytes < dest_size: - raise ValueError('destination buffer too small; expected at least %s, ' - 'got %s' % (dest_size, dest_buffer.nbytes)) + dest_1d = ensure_contiguous_ndarray(dest) + + # obtain dest memoryview + dest_mv = memoryview(dest_1d) + dest_pb = PyMemoryView_GET_BUFFER(dest_mv) + dest_ptr = dest_pb.buf + dest_nbytes = dest_pb.len + + # validate output buffer + if dest_nbytes < dest_size: + raise ValueError('destination buffer too small; expected at least %s, ' + 'got %s' % (dest_size, dest_nbytes)) # perform decompression with nogil: decompressed_size = ZSTD_decompress(dest_ptr, dest_size, source_ptr, source_size) finally: - - # release buffers - source_buffer.release() - if dest_buffer is not None: - dest_buffer.release() + pass # check decompression was successful if ZSTD_isError(decompressed_size):