From 383ff46254510b70bc1e1eb06d207c2f5693eb31 Mon Sep 17 00:00:00 2001 From: Jan Krems Date: Fri, 15 Mar 2024 14:54:30 -0700 Subject: [PATCH] zlib: add zstd support --- benchmark/zlib/creation.js | 2 +- benchmark/zlib/pipe.js | 15 +- lib/internal/errors.js | 1 + lib/zlib.js | 95 ++++- src/node_zlib.cc | 358 +++++++++++++++++- test/parallel/test-zlib-bytes-read.js | 1 + .../parallel/test-zlib-convenience-methods.js | 2 + test/parallel/test-zlib-empty-buffer.js | 2 + test/parallel/test-zlib-invalid-input.js | 1 + test/parallel/test-zlib-random-byte-pipes.js | 1 + test/parallel/test-zlib-write-after-flush.js | 1 + test/parallel/test-zlib-zero-byte.js | 9 +- test/parallel/test-zlib.js | 1 + 13 files changed, 476 insertions(+), 13 deletions(-) diff --git a/benchmark/zlib/creation.js b/benchmark/zlib/creation.js index 90b22780d2d312..41b1e4917a67bb 100644 --- a/benchmark/zlib/creation.js +++ b/benchmark/zlib/creation.js @@ -5,7 +5,7 @@ const zlib = require('zlib'); const bench = common.createBenchmark(main, { type: [ 'Deflate', 'DeflateRaw', 'Inflate', 'InflateRaw', 'Gzip', 'Gunzip', 'Unzip', - 'BrotliCompress', 'BrotliDecompress', + 'BrotliCompress', 'BrotliDecompress', 'ZstdCompress', 'ZstdDecompress', ], options: ['true', 'false'], n: [5e5], diff --git a/benchmark/zlib/pipe.js b/benchmark/zlib/pipe.js index a9c86e3de660a9..5a21c3ff417084 100644 --- a/benchmark/zlib/pipe.js +++ b/benchmark/zlib/pipe.js @@ -7,7 +7,7 @@ const bench = common.createBenchmark(main, { inputLen: [1024], duration: [5], type: ['string', 'buffer'], - algorithm: ['gzip', 'brotli'], + algorithm: ['gzip', 'brotli', 'zstd'], }, { test: { inputLen: 1024, @@ -15,14 +15,19 @@ const bench = common.createBenchmark(main, { }, }); +const algorithms = { + 'gzip': [zlib.createGzip, zlib.createGunzip], + 'brotli': [zlib.createBrotliCompress, zlib.createBrotliDecompress], + 'zstd': [zlib.createZstdCompress, zlib.createZstdDecompress], +}; + function main({ inputLen, duration, type, algorithm }) { const buffer = Buffer.alloc(inputLen, fs.readFileSync(__filename)); const chunk = type === 'buffer' ? buffer : buffer.toString('utf8'); - const input = algorithm === 'gzip' ? - zlib.createGzip() : zlib.createBrotliCompress(); - const output = algorithm === 'gzip' ? - zlib.createGunzip() : zlib.createBrotliDecompress(); + const [createCompress, createUncompress] = algorithms[algorithm]; + const input = createCompress(); + const output = createUncompress(); let readFromOutput = 0; input.pipe(output); diff --git a/lib/internal/errors.js b/lib/internal/errors.js index ef4295f9eaaaef..fc0309dc42ce06 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -1917,3 +1917,4 @@ E('ERR_WORKER_UNSERIALIZABLE_ERROR', E('ERR_WORKER_UNSUPPORTED_OPERATION', '%s is not supported in workers', TypeError); E('ERR_ZLIB_INITIALIZATION_FAILED', 'Initialization failed', Error); +E('ERR_ZSTD_INVALID_PARAM', '%s is not a valid zstd parameter', RangeError); diff --git a/lib/zlib.js b/lib/zlib.js index 4eaf1976e7d09f..621b36bb02342e 100644 --- a/lib/zlib.js +++ b/lib/zlib.js @@ -49,6 +49,7 @@ const { ERR_INVALID_ARG_TYPE, ERR_OUT_OF_RANGE, ERR_ZLIB_INITIALIZATION_FAILED, + ERR_ZSTD_INVALID_PARAM, }, genericNodeError, hideStackFrames, @@ -88,9 +89,12 @@ const { // Node's compression stream modes (node_zlib_mode) DEFLATE, DEFLATERAW, INFLATE, INFLATERAW, GZIP, GUNZIP, UNZIP, BROTLI_DECODE, BROTLI_ENCODE, + ZSTD_COMPRESS, ZSTD_DECOMPRESS, // Brotli operations (~flush levels) BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_FLUSH, BROTLI_OPERATION_FINISH, BROTLI_OPERATION_EMIT_METADATA, + // Zstd end directives (~flush levels) + ZSTD_e_continue, ZSTD_e_flush, ZSTD_e_end, } = constants; // Translation table for return codes. @@ -237,9 +241,11 @@ const checkRangesOrGetDefault = hideStackFrames( const FLUSH_BOUND = [ [ Z_NO_FLUSH, Z_BLOCK ], [ BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_EMIT_METADATA ], + [ ZSTD_e_continue, ZSTD_e_end ], ]; const FLUSH_BOUND_IDX_NORMAL = 0; const FLUSH_BOUND_IDX_BROTLI = 1; +const FLUSH_BOUND_IDX_ZSTD = 2; // The base class for all Zlib-style streams. function ZlibBase(opts, mode, handle, { flush, finishFlush, fullFlush }) { @@ -248,13 +254,15 @@ function ZlibBase(opts, mode, handle, { flush, finishFlush, fullFlush }) { // The ZlibBase class is not exported to user land, the mode should only be // passed in by us. assert(typeof mode === 'number'); - assert(mode >= DEFLATE && mode <= BROTLI_ENCODE); + assert(mode >= DEFLATE && mode <= ZSTD_DECOMPRESS); let flushBoundIdx; - if (mode !== BROTLI_ENCODE && mode !== BROTLI_DECODE) { - flushBoundIdx = FLUSH_BOUND_IDX_NORMAL; - } else { + if (mode === BROTLI_ENCODE || mode === BROTLI_DECODE) { flushBoundIdx = FLUSH_BOUND_IDX_BROTLI; + } else if (mode === ZSTD_COMPRESS || mode === ZSTD_DECOMPRESS) { + flushBoundIdx = FLUSH_BOUND_IDX_ZSTD; + } else { + flushBoundIdx = FLUSH_BOUND_IDX_NORMAL; } if (opts) { @@ -888,6 +896,77 @@ ObjectSetPrototypeOf(BrotliDecompress.prototype, Brotli.prototype); ObjectSetPrototypeOf(BrotliDecompress, Brotli); +const kMaxZstdParam = MathMaxApply(ArrayPrototypeMap( + ObjectKeys(constants), + (key) => (StringPrototypeStartsWith(key, 'ZSTD_c_') ? + constants[key] : + 0), +)); + +const zstdInitParamsArray = new Uint32Array(kMaxZstdParam + 1); + +const zstdDefaultOpts = { + flush: ZSTD_e_continue, // BROTLI_OPERATION_PROCESS, + finishFlush: ZSTD_e_end, // BROTLI_OPERATION_FINISH, + fullFlush: ZSTD_e_flush, // BROTLI_OPERATION_FLUSH, +}; +function Zstd(opts, mode) { + assert(mode === ZSTD_COMPRESS || mode === ZSTD_DECOMPRESS); + + TypedArrayPrototypeFill(zstdInitParamsArray, -1); + if (opts?.params) { + ArrayPrototypeForEach(ObjectKeys(opts.params), (origKey) => { + const key = +origKey; + if (NumberIsNaN(key) || key < 0 || key > kMaxZstdParam || + (zstdInitParamsArray[key] | 0) !== -1) { + throw new ERR_ZSTD_INVALID_PARAM(origKey); + } + + const value = opts.params[origKey]; + if (typeof value !== 'number' && typeof value !== 'boolean') { + throw new ERR_INVALID_ARG_TYPE('options.params[key]', + 'number', opts.params[origKey]); + } + zstdInitParamsArray[key] = value; + }); + } + + const handle = mode === ZSTD_COMPRESS ? + new binding.ZstdCompress() : new binding.ZstdDecompress(); + + this._writeState = new Uint32Array(2); + if (!handle.init(zstdInitParamsArray, + this._writeState, + processCallback)) { + throw new ERR_ZLIB_INITIALIZATION_FAILED(); + } + + ReflectApply(ZlibBase, this, [opts, mode, handle, zstdDefaultOpts]); +} +ObjectSetPrototypeOf(Zstd.prototype, ZlibBase.prototype); +ObjectSetPrototypeOf(Zstd, ZlibBase); + + +function ZstdCompress(opts) { + if (!(this instanceof ZstdCompress)) + return new ZstdCompress(opts); + + ReflectApply(Zstd, this, [opts, ZSTD_COMPRESS]); +} +ObjectSetPrototypeOf(ZstdCompress.prototype, Zstd.prototype); +ObjectSetPrototypeOf(ZstdCompress, Zstd); + + +function ZstdDecompress(opts) { + if (!(this instanceof ZstdDecompress)) + return new ZstdDecompress(opts); + + ReflectApply(Zstd, this, [opts, ZSTD_DECOMPRESS]); +} +ObjectSetPrototypeOf(ZstdDecompress.prototype, Zstd.prototype); +ObjectSetPrototypeOf(ZstdDecompress, Zstd); + + function createProperty(ctor) { return { __proto__: null, @@ -917,6 +996,8 @@ module.exports = { Unzip, BrotliCompress, BrotliDecompress, + ZstdCompress, + ZstdDecompress, // Convenience methods. // compress/decompress a string or buffer in one step. @@ -938,6 +1019,10 @@ module.exports = { brotliCompressSync: createConvenienceMethod(BrotliCompress, true), brotliDecompress: createConvenienceMethod(BrotliDecompress, false), brotliDecompressSync: createConvenienceMethod(BrotliDecompress, true), + zstdCompress: createConvenienceMethod(ZstdCompress, false), + zstdCompressSync: createConvenienceMethod(ZstdCompress, true), + zstdDecompress: createConvenienceMethod(ZstdDecompress, false), + zstdDecompressSync: createConvenienceMethod(ZstdDecompress, true), }; ObjectDefineProperties(module.exports, { @@ -950,6 +1035,8 @@ ObjectDefineProperties(module.exports, { createUnzip: createProperty(Unzip), createBrotliCompress: createProperty(BrotliCompress), createBrotliDecompress: createProperty(BrotliDecompress), + createZstdCompress: createProperty(ZstdCompress), + createZstdDecompress: createProperty(ZstdDecompress), constants: { __proto__: null, configurable: false, diff --git a/src/node_zlib.cc b/src/node_zlib.cc index 0c4ae0fc794347..ba536673182f6d 100644 --- a/src/node_zlib.cc +++ b/src/node_zlib.cc @@ -34,6 +34,8 @@ #include "brotli/encode.h" #include "brotli/decode.h" #include "zlib.h" +#include "zstd.h" +#include "zstd_errors.h" #include @@ -94,6 +96,44 @@ inline const char* ZlibStrerror(int err) { return "Z_UNKNOWN_ERROR"; } +#define ZSTD_ERROR_CODES(V) \ + V(ZSTD_error_no_error) \ + V(ZSTD_error_GENERIC) \ + V(ZSTD_error_prefix_unknown) \ + V(ZSTD_error_version_unsupported) \ + V(ZSTD_error_frameParameter_unsupported) \ + V(ZSTD_error_frameParameter_windowTooLarge) \ + V(ZSTD_error_corruption_detected) \ + V(ZSTD_error_checksum_wrong) \ + V(ZSTD_error_literals_headerWrong) \ + V(ZSTD_error_dictionary_corrupted) \ + V(ZSTD_error_dictionary_wrong) \ + V(ZSTD_error_dictionaryCreation_failed) \ + V(ZSTD_error_parameter_unsupported) \ + V(ZSTD_error_parameter_combination_unsupported) \ + V(ZSTD_error_parameter_outOfBound) \ + V(ZSTD_error_tableLog_tooLarge) \ + V(ZSTD_error_maxSymbolValue_tooLarge) \ + V(ZSTD_error_maxSymbolValue_tooSmall) \ + V(ZSTD_error_stabilityCondition_notRespected) \ + V(ZSTD_error_stage_wrong) \ + V(ZSTD_error_init_missing) \ + V(ZSTD_error_memory_allocation) \ + V(ZSTD_error_workSpace_tooSmall) \ + V(ZSTD_error_dstSize_tooSmall) \ + V(ZSTD_error_srcSize_wrong) \ + V(ZSTD_error_dstBuffer_null) \ + V(ZSTD_error_noForwardProgress_destFull) \ + V(ZSTD_error_noForwardProgress_inputEmpty) + +inline const char* ZstdStrerror(int err) { +#define V(code) \ + if (err == code) return #code; + ZSTD_ERROR_CODES(V) +#undef V + return "ZSTD_error_GENERIC"; +} + enum node_zlib_mode { NONE, DEFLATE, @@ -104,7 +144,9 @@ enum node_zlib_mode { INFLATERAW, UNZIP, BROTLI_DECODE, - BROTLI_ENCODE + BROTLI_ENCODE, + ZSTD_COMPRESS, + ZSTD_DECOMPRESS }; constexpr uint8_t GZIP_HEADER_ID1 = 0x1f; @@ -248,6 +290,80 @@ class BrotliDecoderContext final : public BrotliContext { DeleteFnPtr state_; }; +class ZstdContext : public MemoryRetainer { + public: + ZstdContext() = default; + + // Streaming-related, should be available for all compression libraries: + void Close(); + void SetBuffers(const char* in, uint32_t in_len, char* out, uint32_t out_len); + void SetFlush(int flush); + void GetAfterWriteOffsets(uint32_t* avail_in, uint32_t* avail_out) const; + CompressionError GetErrorInfo() const; + + ZstdContext(const ZstdContext&) = delete; + ZstdContext& operator=(const ZstdContext&) = delete; + + // Wrap ZSTD_freeCCtx to remove the return type. + static void FreeCCtx(ZSTD_CCtx* cctx) { ZSTD_freeCCtx(cctx); } + + protected: + ZSTD_EndDirective flush_ = ZSTD_e_continue; + + ZSTD_inBuffer input_ = {nullptr, 0, 0}; + ZSTD_outBuffer output_ = {nullptr, 0, 0}; + + ZSTD_ErrorCode error_ = ZSTD_error_no_error; + std::string error_string_; + std::string error_code_string_; +}; + +class ZstdCompressContext final : public ZstdContext { + public: + ZstdCompressContext() = default; + + // Streaming-related, should be available for all compression libraries: + void DoThreadPoolWork(); + CompressionError ResetStream(); + + // Zstd specific: + CompressionError Init(); + CompressionError SetParameter(int key, uint32_t value); + + // Wrap ZSTD_freeCCtx to remove the return type. + static void FreeCCtx(ZSTD_CCtx* cctx) { ZSTD_freeCCtx(cctx); } + + SET_MEMORY_INFO_NAME(ZstdCompressContext) + SET_SELF_SIZE(ZstdCompressContext) + SET_NO_MEMORY_INFO() + + private: + DeleteFnPtr cctx_; +}; + +class ZstdDecompressContext final : public ZstdContext { + public: + ZstdDecompressContext() = default; + + // Streaming-related, should be available for all compression libraries: + void DoThreadPoolWork(); + CompressionError ResetStream(); + + // Zstd specific: + CompressionError Init(); + CompressionError SetParameter(int key, uint32_t value); + + // Wrap ZSTD_freeDCtx to remove the return type. + static void FreeDCtx(ZSTD_DCtx* dctx) { ZSTD_freeDCtx(dctx); } + + SET_MEMORY_INFO_NAME(ZstdDecompressContext) + SET_SELF_SIZE(ZstdDecompressContext) + SET_NO_MEMORY_INFO() + + private: + DeleteFnPtr dctx_; +}; + template class CompressionStream : public AsyncWrap, public ThreadPoolWork { public: @@ -730,6 +846,74 @@ class BrotliCompressionStream final : using BrotliEncoderStream = BrotliCompressionStream; using BrotliDecoderStream = BrotliCompressionStream; +template +class ZstdStream final : public CompressionStream { + public: + ZstdStream(Environment* env, Local wrap) + : CompressionStream(env, wrap) {} + + inline CompressionContext* context() { + return this->CompressionStream::context(); + } + typedef typename CompressionStream::AllocScope AllocScope; + + static void New(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + new ZstdStream(env, args.This()); + } + + static void Init(const FunctionCallbackInfo& args) { + CHECK(args.Length() == 3 && "init(params, writeResult, writeCallback)"); + ZstdStream* wrap; + ASSIGN_OR_RETURN_UNWRAP(&wrap, args.Holder()); + + CHECK(args[1]->IsUint32Array()); + uint32_t* write_result = reinterpret_cast(Buffer::Data(args[1])); + + CHECK(args[2]->IsFunction()); + Local write_js_callback = args[2].As(); + wrap->InitStream(write_result, write_js_callback); + + AllocScope alloc_scope(wrap); + CompressionError err = wrap->context()->Init(); + if (err.IsError()) { + wrap->EmitError(err); + args.GetReturnValue().Set(false); + return; + } + + CHECK(args[0]->IsUint32Array()); + const uint32_t* data = reinterpret_cast(Buffer::Data(args[0])); + size_t len = args[0].As()->Length(); + + for (int i = 0; static_cast(i) < len; i++) { + if (data[i] == static_cast(-1)) continue; + + CompressionError err = wrap->context()->SetParameter(i, data[i]); + if (err.IsError()) { + wrap->EmitError(err); + args.GetReturnValue().Set(false); + return; + } + } + + args.GetReturnValue().Set(true); + } + + static void Params(const FunctionCallbackInfo& args) { + // Currently a no-op, and not accessed from JS land. + // At some point zstd may support changing parameters on the fly, + // in which case we can implement this and a JS equivalent similar to + // the zlib Params() function. + } + + SET_MEMORY_INFO_NAME(ZstdStream) + SET_SELF_SIZE(ZstdStream) +}; + +using ZstdCompressStream = ZstdStream; +using ZstdDecompressStream = ZstdStream; + void ZlibContext::Close() { { Mutex::ScopedLock lock(mutex_); @@ -1253,6 +1437,109 @@ CompressionError BrotliDecoderContext::GetErrorInfo() const { } } +void ZstdContext::Close() {} + +void ZstdContext::SetBuffers(const char* in, + uint32_t in_len, + char* out, + uint32_t out_len) { + input_.src = reinterpret_cast(in); + input_.size = in_len; + input_.pos = 0; + + output_.dst = reinterpret_cast(out); + output_.size = out_len; + output_.pos = 0; +} + +void ZstdContext::SetFlush(int flush) { + flush_ = static_cast(flush); +} + +void ZstdContext::GetAfterWriteOffsets(uint32_t* avail_in, + uint32_t* avail_out) const { + *avail_in = input_.size - input_.pos; + *avail_out = output_.size - output_.pos; +} + +CompressionError ZstdContext::GetErrorInfo() const { + if (error_ != ZSTD_error_no_error) { + return CompressionError(error_string_.c_str(), + error_code_string_.c_str(), + static_cast(error_)); + } else { + return CompressionError{}; + } +} + +CompressionError ZstdCompressContext::SetParameter(int key, uint32_t value) { + size_t result = ZSTD_CCtx_setParameter( + cctx_.get(), static_cast(key), value); + if (ZSTD_isError(result)) { + return CompressionError( + "Setting parameter failed", "ERR_ZSTD_PARAM_SET_FAILED", -1); + } + return CompressionError{}; +} + +CompressionError ZstdCompressContext::Init() { + cctx_.reset(ZSTD_createCCtx()); + if (!cctx_) { + return CompressionError("Could not initialize zstd instance", + "ERR_ZLIB_INITIALIZATION_FAILED", + -1); + } + return CompressionError{}; +} + +CompressionError ZstdCompressContext::ResetStream() { + return Init(); +} + +void ZstdCompressContext::DoThreadPoolWork() { + size_t const remaining = + ZSTD_compressStream2(cctx_.get(), &output_, &input_, flush_); + if (ZSTD_isError(remaining)) { + error_ = ZSTD_getErrorCode(remaining); + error_code_string_ = ZstdStrerror(error_); + error_string_ = ZSTD_getErrorString(error_); + return; + } +} + +CompressionError ZstdDecompressContext::SetParameter(int key, uint32_t value) { + size_t result = ZSTD_DCtx_setParameter( + dctx_.get(), static_cast(key), value); + if (ZSTD_isError(result)) { + return CompressionError( + "Setting parameter failed", "ERR_ZSTD_PARAM_SET_FAILED", -1); + } + return CompressionError{}; +} + +CompressionError ZstdDecompressContext::Init() { + dctx_.reset(ZSTD_createDCtx()); + if (!dctx_) { + return CompressionError("Could not initialize zstd instance", + "ERR_ZLIB_INITIALIZATION_FAILED", + -1); + } + return CompressionError{}; +} + +CompressionError ZstdDecompressContext::ResetStream() { + return Init(); +} + +void ZstdDecompressContext::DoThreadPoolWork() { + size_t const ret = ZSTD_decompressStream(dctx_.get(), &output_, &input_); + if (ZSTD_isError(ret)) { + error_ = ZSTD_getErrorCode(ret); + error_code_string_ = ZstdStrerror(error_); + error_string_ = ZSTD_getErrorString(error_); + return; + } +} template struct MakeClass { @@ -1295,6 +1582,8 @@ void Initialize(Local target, MakeClass::Make(env, target, "Zlib"); MakeClass::Make(env, target, "BrotliEncoder"); MakeClass::Make(env, target, "BrotliDecoder"); + MakeClass::Make(env, target, "ZstdCompress"); + MakeClass::Make(env, target, "ZstdDecompress"); target->Set(env->context(), FIXED_ONE_BYTE_STRING(env->isolate(), "ZLIB_VERSION"), @@ -1305,6 +1594,8 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) { MakeClass::Make(registry); MakeClass::Make(registry); MakeClass::Make(registry); + MakeClass::Make(registry); + MakeClass::Make(registry); } } // anonymous namespace @@ -1348,6 +1639,8 @@ void DefineZlibConstants(Local target) { NODE_DEFINE_CONSTANT(target, UNZIP); NODE_DEFINE_CONSTANT(target, BROTLI_DECODE); NODE_DEFINE_CONSTANT(target, BROTLI_ENCODE); + NODE_DEFINE_CONSTANT(target, ZSTD_DECOMPRESS); + NODE_DEFINE_CONSTANT(target, ZSTD_COMPRESS); NODE_DEFINE_CONSTANT(target, Z_MIN_WINDOWBITS); NODE_DEFINE_CONSTANT(target, Z_MAX_WINDOWBITS); @@ -1427,6 +1720,69 @@ void DefineZlibConstants(Local target) { NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2); NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES); NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_UNREACHABLE); + + // Zstd constants + NODE_DEFINE_CONSTANT(target, ZSTD_e_continue); + NODE_DEFINE_CONSTANT(target, ZSTD_e_flush); + NODE_DEFINE_CONSTANT(target, ZSTD_e_end); + NODE_DEFINE_CONSTANT(target, ZSTD_fast); + NODE_DEFINE_CONSTANT(target, ZSTD_dfast); + NODE_DEFINE_CONSTANT(target, ZSTD_greedy); + NODE_DEFINE_CONSTANT(target, ZSTD_lazy); + NODE_DEFINE_CONSTANT(target, ZSTD_lazy2); + NODE_DEFINE_CONSTANT(target, ZSTD_btlazy2); + NODE_DEFINE_CONSTANT(target, ZSTD_btopt); + NODE_DEFINE_CONSTANT(target, ZSTD_btultra); + NODE_DEFINE_CONSTANT(target, ZSTD_btultra2); + NODE_DEFINE_CONSTANT(target, ZSTD_c_compressionLevel); + NODE_DEFINE_CONSTANT(target, ZSTD_c_windowLog); + NODE_DEFINE_CONSTANT(target, ZSTD_c_hashLog); + NODE_DEFINE_CONSTANT(target, ZSTD_c_chainLog); + NODE_DEFINE_CONSTANT(target, ZSTD_c_searchLog); + NODE_DEFINE_CONSTANT(target, ZSTD_c_minMatch); + NODE_DEFINE_CONSTANT(target, ZSTD_c_targetLength); + NODE_DEFINE_CONSTANT(target, ZSTD_c_strategy); + NODE_DEFINE_CONSTANT(target, ZSTD_c_enableLongDistanceMatching); + NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmHashLog); + NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmMinMatch); + NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmBucketSizeLog); + NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmHashRateLog); + NODE_DEFINE_CONSTANT(target, ZSTD_c_contentSizeFlag); + NODE_DEFINE_CONSTANT(target, ZSTD_c_checksumFlag); + NODE_DEFINE_CONSTANT(target, ZSTD_c_dictIDFlag); + NODE_DEFINE_CONSTANT(target, ZSTD_c_nbWorkers); + NODE_DEFINE_CONSTANT(target, ZSTD_c_jobSize); + NODE_DEFINE_CONSTANT(target, ZSTD_c_overlapLog); + NODE_DEFINE_CONSTANT(target, ZSTD_d_windowLogMax); + // Error codes + NODE_DEFINE_CONSTANT(target, ZSTD_error_no_error); + NODE_DEFINE_CONSTANT(target, ZSTD_error_GENERIC); + NODE_DEFINE_CONSTANT(target, ZSTD_error_prefix_unknown); + NODE_DEFINE_CONSTANT(target, ZSTD_error_version_unsupported); + NODE_DEFINE_CONSTANT(target, ZSTD_error_frameParameter_unsupported); + NODE_DEFINE_CONSTANT(target, ZSTD_error_frameParameter_windowTooLarge); + NODE_DEFINE_CONSTANT(target, ZSTD_error_corruption_detected); + NODE_DEFINE_CONSTANT(target, ZSTD_error_checksum_wrong); + NODE_DEFINE_CONSTANT(target, ZSTD_error_literals_headerWrong); + NODE_DEFINE_CONSTANT(target, ZSTD_error_dictionary_corrupted); + NODE_DEFINE_CONSTANT(target, ZSTD_error_dictionary_wrong); + NODE_DEFINE_CONSTANT(target, ZSTD_error_dictionaryCreation_failed); + NODE_DEFINE_CONSTANT(target, ZSTD_error_parameter_unsupported); + NODE_DEFINE_CONSTANT(target, ZSTD_error_parameter_combination_unsupported); + NODE_DEFINE_CONSTANT(target, ZSTD_error_parameter_outOfBound); + NODE_DEFINE_CONSTANT(target, ZSTD_error_tableLog_tooLarge); + NODE_DEFINE_CONSTANT(target, ZSTD_error_maxSymbolValue_tooLarge); + NODE_DEFINE_CONSTANT(target, ZSTD_error_maxSymbolValue_tooSmall); + NODE_DEFINE_CONSTANT(target, ZSTD_error_stabilityCondition_notRespected); + NODE_DEFINE_CONSTANT(target, ZSTD_error_stage_wrong); + NODE_DEFINE_CONSTANT(target, ZSTD_error_init_missing); + NODE_DEFINE_CONSTANT(target, ZSTD_error_memory_allocation); + NODE_DEFINE_CONSTANT(target, ZSTD_error_workSpace_tooSmall); + NODE_DEFINE_CONSTANT(target, ZSTD_error_dstSize_tooSmall); + NODE_DEFINE_CONSTANT(target, ZSTD_error_srcSize_wrong); + NODE_DEFINE_CONSTANT(target, ZSTD_error_dstBuffer_null); + NODE_DEFINE_CONSTANT(target, ZSTD_error_noForwardProgress_destFull); + NODE_DEFINE_CONSTANT(target, ZSTD_error_noForwardProgress_inputEmpty); } } // namespace node diff --git a/test/parallel/test-zlib-bytes-read.js b/test/parallel/test-zlib-bytes-read.js index 605281cd3bd7e6..09b515cc7b70cf 100644 --- a/test/parallel/test-zlib-bytes-read.js +++ b/test/parallel/test-zlib-bytes-read.js @@ -33,6 +33,7 @@ for (const method of [ ['createDeflate', 'createInflate', true], ['createDeflateRaw', 'createInflateRaw', true], ['createBrotliCompress', 'createBrotliDecompress', true], + ['createZstdCompress', 'createZstdDecompress', false], ]) { let compWriter; let compData = Buffer.alloc(0); diff --git a/test/parallel/test-zlib-convenience-methods.js b/test/parallel/test-zlib-convenience-methods.js index 01ec7e211bd5aa..f938a1f6a756ca 100644 --- a/test/parallel/test-zlib-convenience-methods.js +++ b/test/parallel/test-zlib-convenience-methods.js @@ -54,6 +54,8 @@ for (const [type, expect] of [ ['deflateRaw', 'inflateRaw', 'DeflateRaw', 'InflateRaw'], ['brotliCompress', 'brotliDecompress', 'BrotliCompress', 'BrotliDecompress'], + ['zstdCompress', 'zstdDecompress', + 'ZstdCompress', 'ZstdDecompress'], ]) { zlib[method[0]](expect, opts, common.mustCall((err, result) => { zlib[method[1]](result, opts, common.mustCall((err, result) => { diff --git a/test/parallel/test-zlib-empty-buffer.js b/test/parallel/test-zlib-empty-buffer.js index 27fd1340fd1eb4..af53b3013ee82f 100644 --- a/test/parallel/test-zlib-empty-buffer.js +++ b/test/parallel/test-zlib-empty-buffer.js @@ -11,10 +11,12 @@ const emptyBuffer = Buffer.alloc(0); [ zlib.deflateSync, zlib.inflateSync, 'deflate sync' ], [ zlib.gzipSync, zlib.gunzipSync, 'gzip sync' ], [ zlib.brotliCompressSync, zlib.brotliDecompressSync, 'br sync' ], + [ zlib.zstdCompressSync, zlib.zstdDecompressSync, 'zstd sync' ], [ promisify(zlib.deflateRaw), promisify(zlib.inflateRaw), 'raw' ], [ promisify(zlib.deflate), promisify(zlib.inflate), 'deflate' ], [ promisify(zlib.gzip), promisify(zlib.gunzip), 'gzip' ], [ promisify(zlib.brotliCompress), promisify(zlib.brotliDecompress), 'br' ], + [ promisify(zlib.zstdCompress), promisify(zlib.zstdDecompress), 'zstd' ], ]) { const compressed = await compress(emptyBuffer); const decompressed = await decompress(compressed); diff --git a/test/parallel/test-zlib-invalid-input.js b/test/parallel/test-zlib-invalid-input.js index 7aa44dfe7090a1..7aea0efa06dbcf 100644 --- a/test/parallel/test-zlib-invalid-input.js +++ b/test/parallel/test-zlib-invalid-input.js @@ -40,6 +40,7 @@ const unzips = [ zlib.Inflate(), zlib.InflateRaw(), zlib.BrotliDecompress(), + zlib.ZstdDecompress(), ]; nonStringInputs.forEach(common.mustCall((input) => { diff --git a/test/parallel/test-zlib-random-byte-pipes.js b/test/parallel/test-zlib-random-byte-pipes.js index d8d039a6d65c4f..6a7d7c505e5c25 100644 --- a/test/parallel/test-zlib-random-byte-pipes.js +++ b/test/parallel/test-zlib-random-byte-pipes.js @@ -144,6 +144,7 @@ class HashStream extends Stream { for (const [ createCompress, createDecompress ] of [ [ zlib.createGzip, zlib.createGunzip ], [ zlib.createBrotliCompress, zlib.createBrotliDecompress ], + [ zlib.createZstdCompress, zlib.createZstdDecompress ], ]) { const inp = new RandomReadStream({ total: 1024, block: 256, jitter: 16 }); const out = new HashStream(); diff --git a/test/parallel/test-zlib-write-after-flush.js b/test/parallel/test-zlib-write-after-flush.js index 6edcae2e2f18bf..57988191f84c57 100644 --- a/test/parallel/test-zlib-write-after-flush.js +++ b/test/parallel/test-zlib-write-after-flush.js @@ -27,6 +27,7 @@ const zlib = require('zlib'); for (const [ createCompress, createDecompress ] of [ [ zlib.createGzip, zlib.createGunzip ], [ zlib.createBrotliCompress, zlib.createBrotliDecompress ], + [ zlib.createZstdCompress, zlib.createZstdDecompress ], ]) { const gzip = createCompress(); const gunz = createDecompress(); diff --git a/test/parallel/test-zlib-zero-byte.js b/test/parallel/test-zlib-zero-byte.js index fc57960f1e56cb..6b3ecdcc0224e4 100644 --- a/test/parallel/test-zlib-zero-byte.js +++ b/test/parallel/test-zlib-zero-byte.js @@ -24,7 +24,13 @@ const common = require('../common'); const assert = require('assert'); const zlib = require('zlib'); -for (const Compressor of [ zlib.Gzip, zlib.BrotliCompress ]) { +const compressors = [ + [zlib.Gzip, 20], + [zlib.BrotliCompress, 1], + [zlib.ZstdCompress, 9], +]; + +for (const [Compressor, expected] of compressors) { const gz = Compressor(); const emptyBuffer = Buffer.alloc(0); let received = 0; @@ -33,7 +39,6 @@ for (const Compressor of [ zlib.Gzip, zlib.BrotliCompress ]) { }); gz.on('end', common.mustCall(function() { - const expected = Compressor === zlib.Gzip ? 20 : 1; assert.strictEqual(received, expected, `${received}, ${expected}, ${Compressor.name}`); })); diff --git a/test/parallel/test-zlib.js b/test/parallel/test-zlib.js index 65050b85a036cf..3b8878d915c097 100644 --- a/test/parallel/test-zlib.js +++ b/test/parallel/test-zlib.js @@ -42,6 +42,7 @@ let zlibPairs = [ [zlib.Gzip, zlib.Unzip], [zlib.DeflateRaw, zlib.InflateRaw], [zlib.BrotliCompress, zlib.BrotliDecompress], + [zlib.ZstdCompress, zlib.ZstdDecompress], ]; // How fast to trickle through the slowstream