diff --git a/Makefile b/Makefile deleted file mode 100644 index b55ad9d..0000000 --- a/Makefile +++ /dev/null @@ -1,40 +0,0 @@ - -CC = gcc -#CC = clang - -CFLAGS = -W -pthread -Wall -pipe -#CFLAGS = -fomit-frame-pointer -LDFLAGS = -lpthread - -#CFLAGS += -DDEBUGME -#CFLAGS += -g -CFLAGS += -O3 -#CFLAGS += -march=native -#CFLAGS += -Wno-unused-but-set-variable -#CFLAGS += -Wno-unused-variable -#CFLAGS += -Wno-unused-function - -PRGS = zstdmt lz4mt lz5mt - -OBJ = threading.o util.o -OBJLZ4 = lz4mt_common.o lz4mt_compress.o lz4mt_decompress.o lz4mt.o -OBJLZ5 = lz5mt_common.o lz5mt_compress.o lz5mt_decompress.o lz5mt.o -OBJZSTD = zstdmt_common.o zstdmt_compress.o zstdmt_decompress.o zstdmt.o - -all: $(PRGS) -again: clean $(PRGS) - -lz4mt: $(OBJ) $(OBJLZ4) - $(CC) $(CFLAGS) $(LDFLAGS) -llz4 -o $@ $(OBJLZ4) $(OBJ) - -lz5mt: $(OBJ) $(OBJLZ5) - $(CC) $(CFLAGS) $(LDFLAGS) -llz5 -o $@ $(OBJLZ5) $(OBJ) - -zstdmt: $(OBJ) $(OBJZSTD) - $(CC) $(CFLAGS) $(LDFLAGS) -lzstd -o $@ $(OBJZSTD) $(OBJ) - -clean: - rm -rf $(PRGS) $(OBJLZ4) $(OBJLZ5) $(OBJZSTD) $(OBJ) - -.c.o: - $(CC) $(CFLAGS) -c $< -o $@ diff --git a/README.md b/README.md index 5e9b285..7fed3d5 100644 --- a/README.md +++ b/README.md @@ -9,114 +9,12 @@ ## Usage of the Testutils -### [LZ4] -``` -Usage: lz4mt [options] infile outfile - -Otions: - -l N set level (1..16) of compression (default: 1) - -t N set number of (de)compression threads (default: 2) - -i N set number of iterations for testing (default: 1) - -b N set input chunksize to N MiB (default: auto) - -c compress (default mode) - -d use decompress mode - -H print headline for the testing values - -h show usage - -v show version -``` - -### [LZ5] -``` -Usage: lz5mt [options] infile outfile - -Otions: - -l N set level (1..16) of compression (default: 1) - -t N set number of (de)compression threads (default: 2) - -i N set number of iterations for testing (default: 1) - -b N set input chunksize to N MiB (default: auto) - -c compress (default mode) - -d use decompress mode - -H print headline for the testing values - -h show usage - -v show version -``` - -### [ZStandard] -``` -Usage: zstdmt [options] infile outfile - -Otions: - -l N set level of compression (default: 3) - -t N set number of (de)compression threads (default: 2) - -i N set number of iterations for testing (default: 1) - -b N set input chunksize to N MiB (default: auto) - -c compress (default mode) - -d use decompress mode (XXX, not done) - -H print headline for the testing values - -h show usage - -v show version -``` +- see unix +- see windows ## Usage of the Library -### Compression -``` -typedef struct { - void *buf; /* ptr to data */ - size_t size; /* current filled in buf */ - size_t allocated; /* length of buf */ -} LZ4MT_Buffer; - -/** - * reading and writing functions - * - you can use stdio functions or plain read/write ... - * - a sample is given in 7-Zip ZS or lz4mt.c - */ -typedef int (fn_read) (void *args, LZ4MT_Buffer * in); -typedef int (fn_write) (void *args, LZ4MT_Buffer * out); - -typedef struct { - fn_read *fn_read; - void *arg_read; - fn_write *fn_write; - void *arg_write; -} LZ4MT_RdWr_t; - -typedef struct LZ4MT_CCtx_s LZ4MT_CCtx; - -/* 1) allocate new cctx */ -LZ4MT_CCtx *LZ4MT_createCCtx(int threads, int level, int inputsize); - -/* 2) threaded compression */ -size_t LZ4MT_CompressCCtx(LZ4MT_CCtx * ctx, LZ4MT_RdWr_t * rdwr); - -/* 3) get some statistic */ -size_t LZ4MT_GetFramesCCtx(LZ4MT_CCtx * ctx); -size_t LZ4MT_GetInsizeCCtx(LZ4MT_CCtx * ctx); -size_t LZ4MT_GetOutsizeCCtx(LZ4MT_CCtx * ctx); - -/* 4) free cctx */ -void LZ4MT_freeCCtx(LZ4MT_CCtx * ctx); -``` - -### Decompression -``` -typedef struct LZ4MT_DCtx_s LZ4MT_DCtx; - -/* 1) allocate new cctx */ -LZ4MT_DCtx *LZ4MT_createDCtx(int threads, int inputsize); - -/* 2) threaded compression */ -size_t LZ4MT_DecompressDCtx(LZ4MT_DCtx * ctx, LZ4MT_RdWr_t * rdwr); - -/* 3) get some statistic */ -size_t LZ4MT_GetFramesDCtx(LZ4MT_DCtx * ctx); -size_t LZ4MT_GetInsizeDCtx(LZ4MT_DCtx * ctx); -size_t LZ4MT_GetOutsizeDCtx(LZ4MT_DCtx * ctx); - -/* 4) free cctx */ -void LZ4MT_freeDCtx(LZ4MT_DCtx * ctx); -``` +- see lib ## Benchmarks @@ -283,9 +181,5 @@ Level|Threads|InSize|OutSize|Frames|Real|User|Sys|MaxMem [LZ5]:https://github.com/inikep/lz5 [ZStandard]:http://facebook.github.io/zstd/ -# TODO - -- threaded decompression for zstd not ready -- document the header files correctly ... just some first start currently -/TR 2016-09-25 +/TR 2016-09-28 diff --git a/lib/README.md b/lib/README.md new file mode 100644 index 0000000..32ad7cd --- /dev/null +++ b/lib/README.md @@ -0,0 +1,66 @@ + +# Multithreading Library for [LZ4], [LZ5] and [ZStandard] + + +### Compression +``` +typedef struct { + void *buf; /* ptr to data */ + size_t size; /* current filled in buf */ + size_t allocated; /* length of buf */ +} LZ4MT_Buffer; + +/** + * reading and writing functions + * - you can use stdio functions or plain read/write ... + * - a sample is given in 7-Zip ZS or lz4mt.c + */ +typedef int (fn_read) (void *args, LZ4MT_Buffer * in); +typedef int (fn_write) (void *args, LZ4MT_Buffer * out); + +typedef struct { + fn_read *fn_read; + void *arg_read; + fn_write *fn_write; + void *arg_write; +} LZ4MT_RdWr_t; + +typedef struct LZ4MT_CCtx_s LZ4MT_CCtx; + +/* 1) allocate new cctx */ +LZ4MT_CCtx *LZ4MT_createCCtx(int threads, int level, int inputsize); + +/* 2) threaded compression */ +size_t LZ4MT_CompressCCtx(LZ4MT_CCtx * ctx, LZ4MT_RdWr_t * rdwr); + +/* 3) get some statistic */ +size_t LZ4MT_GetFramesCCtx(LZ4MT_CCtx * ctx); +size_t LZ4MT_GetInsizeCCtx(LZ4MT_CCtx * ctx); +size_t LZ4MT_GetOutsizeCCtx(LZ4MT_CCtx * ctx); + +/* 4) free cctx */ +void LZ4MT_freeCCtx(LZ4MT_CCtx * ctx); +``` + +### Decompression +``` +typedef struct LZ4MT_DCtx_s LZ4MT_DCtx; + +/* 1) allocate new cctx */ +LZ4MT_DCtx *LZ4MT_createDCtx(int threads, int inputsize); + +/* 2) threaded compression */ +size_t LZ4MT_DecompressDCtx(LZ4MT_DCtx * ctx, LZ4MT_RdWr_t * rdwr); + +/* 3) get some statistic */ +size_t LZ4MT_GetFramesDCtx(LZ4MT_DCtx * ctx); +size_t LZ4MT_GetInsizeDCtx(LZ4MT_DCtx * ctx); +size_t LZ4MT_GetOutsizeDCtx(LZ4MT_DCtx * ctx); + +/* 4) free cctx */ +void LZ4MT_freeDCtx(LZ4MT_DCtx * ctx); +``` + +## Todo + +- add Makefile diff --git a/list.h b/lib/list.h similarity index 100% rename from list.h rename to lib/list.h diff --git a/lz4mt.h b/lib/lz4mt.h similarity index 100% rename from lz4mt.h rename to lib/lz4mt.h diff --git a/lz4mt_common.c b/lib/lz4mt_common.c similarity index 80% rename from lz4mt_common.c rename to lib/lz4mt_common.c index a0fcf0a..2dedd62 100644 --- a/lz4mt_common.c +++ b/lib/lz4mt_common.c @@ -24,14 +24,6 @@ size_t lz4mt_errcode; */ unsigned LZ4MT_isError(size_t code) { -#if 0 - printf("no_error = %zu\n", ERROR(no_error)); - printf("memory_allocation = %zu\n", ERROR(memory_allocation)); - printf("compression_library = %zu\n", ERROR(compression_library)); - printf("maxCode = %zu\n", ERROR(maxCode)); - printf("result = %zu\n", code); - printf("ok ZSTD_decompressStream() result=%zu >=%d max=%zu\n", code, code > ERROR(maxCode), ERROR(maxCode)); -#endif return (code > ERROR(maxCode)); } diff --git a/lz4mt_compress.c b/lib/lz4mt_compress.c similarity index 100% rename from lz4mt_compress.c rename to lib/lz4mt_compress.c diff --git a/lz4mt_decompress.c b/lib/lz4mt_decompress.c similarity index 100% rename from lz4mt_decompress.c rename to lib/lz4mt_decompress.c diff --git a/lz5mt.h b/lib/lz5mt.h similarity index 100% rename from lz5mt.h rename to lib/lz5mt.h diff --git a/lz5mt_common.c b/lib/lz5mt_common.c similarity index 100% rename from lz5mt_common.c rename to lib/lz5mt_common.c diff --git a/lz5mt_compress.c b/lib/lz5mt_compress.c similarity index 100% rename from lz5mt_compress.c rename to lib/lz5mt_compress.c diff --git a/lz5mt_decompress.c b/lib/lz5mt_decompress.c similarity index 100% rename from lz5mt_decompress.c rename to lib/lz5mt_decompress.c diff --git a/mem.h b/lib/mem.h similarity index 100% rename from mem.h rename to lib/mem.h diff --git a/threading.c b/lib/threading.c similarity index 100% rename from threading.c rename to lib/threading.c diff --git a/threading.h b/lib/threading.h similarity index 100% rename from threading.h rename to lib/threading.h diff --git a/zstdmt.h b/lib/zstdmt.h similarity index 100% rename from zstdmt.h rename to lib/zstdmt.h diff --git a/zstdmt_common.c b/lib/zstdmt_common.c similarity index 100% rename from zstdmt_common.c rename to lib/zstdmt_common.c diff --git a/zstdmt_compress.c b/lib/zstdmt_compress.c similarity index 100% rename from zstdmt_compress.c rename to lib/zstdmt_compress.c diff --git a/zstdmt_decompress.c b/lib/zstdmt_decompress.c similarity index 100% rename from zstdmt_decompress.c rename to lib/zstdmt_decompress.c diff --git a/multistream/README.md b/multistream/README.md index c751e5f..05af8f8 100644 --- a/multistream/README.md +++ b/multistream/README.md @@ -1,20 +1,6 @@ # zstdmt multistream testing, obsolete -zstdmt depends currently on posix threads for multi-threading and is -only tested on linux - -I will add support for windows and CreateThread() together with -WaitForMultipleObjects() / WaitForSingleObject() later... - -## Building for Linux with gcc - - - Run `make`. - -## Building for Linux with Clang - - - Run `make CC=clang` - ## Overview - `./zstd-zstd` - single threaded single stream mode (normal zstd stream) @@ -24,8 +10,7 @@ WaitForMultipleObjects() / WaitForSingleObject() later... ## Todo - - add support for windows - - fix and document return codes in own mini libraries + - nothing, this stuff is obsolte, just kept for history ## See also diff --git a/unix/Makefile b/unix/Makefile new file mode 100644 index 0000000..401f58e --- /dev/null +++ b/unix/Makefile @@ -0,0 +1,46 @@ + +CC = gcc +STRIP = strip +#CC = clang + +CFLAGS = -W -pthread -Wall -pipe +CFLAGS += -fomit-frame-pointer +CFLAGS += -I ../lib +LDFLAGS = -lpthread +# -static + +#CFLAGS += -DDEBUGME +#CFLAGS += -g +CFLAGS += -O3 +#CFLAGS += -march=native +#CFLAGS += -Wno-unused-but-set-variable +#CFLAGS += -Wno-unused-variable +#CFLAGS += -Wno-unused-function + +PRGS = zstdmt lz4mt lz5mt + +COMMON = util.c +LIBLZ4 = ../lib/threading.c ../lib/lz4mt_common.c ../lib/lz4mt_compress.c ../lib/lz4mt_decompress.c +LIBLZ5 = ../lib/threading.c ../lib/lz5mt_common.c ../lib/lz5mt_compress.c ../lib/lz5mt_decompress.c +LIBZSTD = ../lib/threading.c ../lib/zstdmt_common.c ../lib/zstdmt_compress.c ../lib/zstdmt_decompress.c + +all: $(PRGS) +again: clean $(PRGS) + +lz4mt: + $(CC) $(CFLAGS) $(LDFLAGS) -llz4 -o $@ lz4mt.c $(LIBLZ4) $(COMMON) + $(STRIP) $@ + +lz5mt: + $(CC) $(CFLAGS) $(LDFLAGS) -llz5 -o $@ lz5mt.c $(LIBLZ5) $(COMMON) + $(STRIP) $@ + +zstdmt: + $(CC) $(CFLAGS) $(LDFLAGS) -lzstd -o $@ zstdmt.c $(LIBZSTD) $(COMMON) + $(STRIP) $@ + +clean: + rm -rf $(PRGS) + +.c.c: + $(CC) $(CFLAGS) -c $< -o $@ diff --git a/unix/README.md b/unix/README.md new file mode 100644 index 0000000..4a4c95e --- /dev/null +++ b/unix/README.md @@ -0,0 +1,216 @@ + +## Usage of the Testutils @ UNIX / Linux + +### [LZ4] +``` +Usage: lz4mt [options] infile outfile + +Otions: + -l N set level (1..16) of compression (default: 1) + -t N set number of (de)compression threads (default: 2) + -i N set number of iterations for testing (default: 1) + -b N set input chunksize to N MiB (default: auto) + -c compress (default mode) + -d use decompress mode + -H print headline for the testing values + -h show usage + -v show version +``` + +### [LZ5] +``` +Usage: lz5mt [options] infile outfile + +Otions: + -l N set level (1..16) of compression (default: 1) + -t N set number of (de)compression threads (default: 2) + -i N set number of iterations for testing (default: 1) + -b N set input chunksize to N MiB (default: auto) + -c compress (default mode) + -d use decompress mode + -H print headline for the testing values + -h show usage + -v show version +``` + +### [ZStandard] +``` +Usage: zstdmt [options] infile outfile + +Otions: + -l N set level of compression (default: 3) + -t N set number of (de)compression threads (default: 2) + -i N set number of iterations for testing (default: 1) + -b N set input chunksize to N MiB (default: auto) + -c compress (default mode) + -d use decompress mode (XXX, not done) + -H print headline for the testing values + -h show usage + -v show version +``` + + +## Benchmarks + +### [LZ4] +Level|Threads|InSize|OutSize|Frames|Real|User|Sys|MaxMem +-----|-------|------|-------|------|----|----|---|------ +1|1|211957760|101069667|3235|1.393|1.347|0.42|2096 +1|2|211957760|101069667|3235|0.622|1.200|0.39|2096 +1|4|211957760|101069667|3235|0.397|1.541|0.30|2096 +2|1|211957760|101069667|3235|1.312|1.265|0.43|2096 +2|2|211957760|101069667|3235|0.734|1.415|0.44|2096 +2|4|211957760|101069667|3235|0.338|1.297|0.41|2248 +3|1|211957760|88923937|3235|6.945|6.854|0.61|2096 +3|2|211957760|88923937|3235|3.381|6.687|0.47|2096 +3|4|211957760|88923937|3235|1.999|7.891|0.64|2712 +4|1|211957760|87542131|3235|7.940|7.847|0.58|2096 +4|2|211957760|87542131|3235|4.161|8.224|0.62|2096 +4|4|211957760|87542131|3235|2.207|8.718|0.61|4796 +5|1|211957760|86701884|3235|8.994|8.907|0.51|2096 +5|2|211957760|86701884|3235|4.822|9.527|0.72|2096 +5|4|211957760|86701884|3235|2.771|10.967|0.61|2844 +6|1|211957760|86228097|3235|10.425|10.344|0.42|2096 +6|2|211957760|86228097|3235|5.388|10.677|0.57|2096 +6|4|211957760|86228097|3235|3.288|13.18|0.63|2720 +7|1|211957760|85963374|3235|11.856|11.749|0.61|2096 +7|2|211957760|85963374|3235|6.39|11.972|0.58|2096 +7|4|211957760|85963374|3235|3.603|14.282|0.61|2760 +8|1|211957760|85824153|3235|13.551|13.444|0.56|2096 +8|2|211957760|85824153|3235|6.897|13.695|0.43|2096 +8|4|211957760|85824153|3235|4.56|16.79|0.65|2976 +9|1|211957760|85762795|3235|14.623|14.515|0.54|2096 +9|2|211957760|85762795|3235|7.564|15.15|0.55|2096 +9|4|211957760|85762795|3235|4.593|18.215|0.68|3000 +10|1|211957760|85736453|3235|16.81|15.977|0.44|2096 +10|2|211957760|85736453|3235|8.225|16.328|0.54|2096 +10|4|211957760|85736453|3235|4.730|18.745|0.79|3144 +11|1|211957760|85725904|3235|17.6|16.894|0.48|2096 +11|2|211957760|85725904|3235|8.902|17.672|0.62|2096 +11|4|211957760|85725904|3235|5.73|20.123|0.58|3700 +12|1|211957760|85720684|3235|18.53|17.935|0.54|2096 +12|2|211957760|85720684|3235|9.186|18.250|0.54|2096 +12|4|211957760|85720684|3235|5.715|22.706|0.52|3536 +13|1|211957760|85717239|3235|19.752|19.622|0.60|2096 +13|2|211957760|85717239|3235|10.300|20.459|0.61|2164 +13|4|211957760|85717239|3235|6.69|24.71|0.91|3672 +14|1|211957760|85714803|3235|22.436|22.292|0.66|2096 +14|2|211957760|85714803|3235|11.744|23.339|0.57|2448 +14|4|211957760|85714803|3235|6.593|26.194|0.61|4104 +15|1|211957760|85713782|3235|24.788|24.633|0.72|2096 +15|2|211957760|85713782|3235|12.364|24.591|0.50|2968 +15|4|211957760|85713782|3235|7.336|29.152|0.59|7400 +16|1|211957760|85713635|3235|25.313|25.177|0.50|2096 +16|2|211957760|85713635|3235|12.791|25.426|0.62|3744 +16|4|211957760|85713635|3235|7.650|30.419|0.46|6840 + +### [LZ5] +Level|Threads|InSize|OutSize|Frames|Real|User|Sys|MaxMem +-----|-------|------|-------|------|----|----|---|------ +1|1|211957760|123117368|3235|1.523|1.466|0.50|2096 +1|2|211957760|123118976|3235|0.804|1.554|0.45|2096 +1|4|211957760|123118409|3235|0.442|1.707|0.46|2096 +2|1|211957760|110989793|3235|2.121|2.71|0.42|2096 +2|2|211957760|110990294|3235|1.36|2.22|0.41|2096 +2|4|211957760|110990676|3235|0.549|2.136|0.43|2120 +3|1|211957760|100263984|3235|2.632|2.589|0.33|2100 +3|2|211957760|100264075|3235|1.359|2.663|0.43|2100 +3|4|211957760|100263827|3235|0.841|3.305|0.38|2224 +4|1|211957760|91372757|3235|4.62|3.982|0.64|2100 +4|2|211957760|91372725|3235|2.200|4.331|0.51|2100 +4|4|211957760|91372698|3235|1.245|4.897|0.54|2848 +5|1|211957760|90471735|3235|4.528|4.454|0.54|2100 +5|2|211957760|90471659|3235|2.262|4.452|0.51|2484 +5|4|211957760|90471655|3235|1.377|5.404|0.61|4260 +6|1|211957760|86823397|3235|13.271|13.164|0.55|2100 +6|2|211957760|86823397|3235|6.844|13.570|0.60|2124 +6|4|211957760|86823397|3235|4.305|17.59|0.74|9732 +7|1|211957760|86367683|3235|13.664|13.551|0.59|2100 +7|2|211957760|86367683|3235|6.964|13.821|0.42|4224 +7|4|211957760|86367683|3235|4.310|17.75|0.71|5808 +8|1|211957760|86228271|3235|14.230|14.103|0.66|3528 +8|2|211957760|86228271|3235|7.698|15.258|0.65|9540 +8|4|211957760|86228271|3235|5.2|19.785|0.93|12604 +9|1|211957760|84279832|3235|32.157|17.532|14.468|36128 +9|2|211957760|84279832|3235|18.114|20.801|14.419|67788 +9|4|211957760|84279832|3235|15.102|30.357|21.389|134544 +10|1|211957760|83448337|3235|33.661|19.69|14.431|34252 +10|2|211957760|83448337|3235|21.210|23.979|16.901|70376 +10|4|211957760|83448337|3235|13.872|31.216|18.327|139448 +11|1|211957760|82198886|3235|46.662|28.395|18.34|34280 +11|2|211957760|82198886|3235|23.75|30.545|14.642|67912 +11|4|211957760|82198886|3235|15.690|39.594|17.732|136308 +12|1|211957760|81284779|3235|59.607|41.422|17.894|35904 +12|2|211957760|81284779|3235|29.520|43.461|14.636|67772 +12|4|211957760|81284779|3235|20.186|55.754|19.54|134728 +13|1|211957760|81940180|3235|81.349|64.895|16.94|36352 +13|2|211957760|81940180|3235|44.839|69.904|18.34|71612 +13|4|211957760|81940180|3235|28.12|84.279|19.784|142868 +14|1|211957760|81560383|3235|89.732|73.370|15.982|36352 +14|2|211957760|81560383|3235|48.969|78.345|17.997|73772 +14|4|211957760|81560383|3235|29.402|93.96|18.868|144112 +15|1|211957760|81490962|3235|598.712|135.637|460.587|1117296 +15|2|211957760|81490962|3235|348.964|152.81|528.464|2230196 +15|4|211957760|81490962|3235|264.598|191.203|816.42|4399220 +16|1|211957760|81490962|3235|595.970|133.238|460.31|1117236 +16|2|211957760|81490962|3235|384.303|161.587|588.589|2232060 +16|4|211957760|81490962|3235|295.830|206.416|922.797|4419620 + +### [ZStandard] +Level|Threads|InSize|OutSize|Frames|Real|User|Sys|MaxMem +-----|-------|------|-------|------|----|----|---|------ +1|1|211957760|73853773|102|2.694|2.636|0.47|6212 +1|2|211957760|73853773|102|1.329|2.579|0.55|11968 +1|4|211957760|73853773|102|0.898|3.448|0.84|20412 +2|1|211957760|70521776|102|3.104|3.45|0.46|6236 +2|2|211957760|70521776|102|1.536|2.995|0.56|11536 +2|4|211957760|70521776|102|1.76|4.127|0.96|21444 +3|1|211957760|67263553|51|4.265|4.185|0.62|11096 +3|2|211957760|67263553|51|2.138|4.171|0.68|22728 +3|4|211957760|67263553|51|1.626|6.61|0.101|47928 +4|1|211957760|66324953|51|4.642|4.560|0.62|13988 +4|2|211957760|66324953|51|2.697|5.266|0.83|30332 +4|4|211957760|66324953|51|2.8|7.556|0.120|57040 +5|1|211957760|65188167|34|6.892|6.804|0.60|15100 +5|2|211957760|65188167|34|3.835|7.403|0.71|29996 +5|4|211957760|65188167|34|2.543|9.372|0.76|60748 +6|1|211957760|63279533|34|9.109|9.9|0.62|19364 +6|2|211957760|63279533|34|5.7|9.793|0.75|38620 +6|4|211957760|63279533|34|3.613|13.559|0.98|75180 +7|1|211957760|62305767|34|9.966|9.868|0.56|23924 +7|2|211957760|62305767|34|6.59|11.905|0.77|47140 +7|4|211957760|62305767|34|3.924|14.428|0.106|94540 +8|1|211957760|61248190|26|14.264|14.132|0.72|28380 +8|2|211957760|61248190|26|7.927|15.427|0.71|55348 +8|4|211957760|61248190|26|5.350|19.541|0.129|105360 +9|1|211957760|60694170|26|16.305|16.158|0.76|33568 +9|2|211957760|60694170|26|10.784|20.648|0.104|66752 +9|4|211957760|60694170|26|7.110|26.91|0.162|134808 +10|1|211957760|60429156|26|17.495|17.348|0.71|37652 +10|2|211957760|60429156|26|10.216|19.839|0.81|76244 +10|4|211957760|60429156|26|7.16|26.528|0.146|153452 +11|1|211957760|59939304|26|19.225|19.77|0.67|60724 +11|2|211957760|59939304|26|11.967|23.53|0.103|122344 +11|4|211957760|59939304|26|7.677|29.2|0.150|243604 +12|1|211957760|59543345|26|25.706|25.522|0.71|60268 +12|2|211957760|59543345|26|15.107|29.754|0.84|122516 +12|4|211957760|59543345|26|10.435|39.818|0.162|248232 +13|1|211957760|59289863|26|29.759|29.566|0.76|68568 +13|2|211957760|59289863|26|18.462|36.283|0.100|141032 +13|4|211957760|59289863|26|12.230|46.21|0.154|283432 +14|1|211957760|58948339|26|43.408|43.162|0.82|68892 +14|2|211957760|58948339|26|27.227|54.62|0.103|145440 +14|4|211957760|58948339|26|18.10|69.415|0.162|287508 +15|1|211957760|58249793|13|78.811|78.428|0.77|62744 +15|2|211957760|58249793|13|47.213|92.540|0.137|123976 +15|4|211957760|58249793|13|27.105|100.529|0.144|245424 +16|1|211957760|57640281|13|85.943|85.423|0.156|100200 +16|2|211957760|57640281|13|48.152|93.87|0.200|199128 +16|4|211957760|57640281|13|29.156|106.315|0.237|396972 +17|1|211957760|56962464|13|106.187|105.615|0.96|82804 +17|2|211957760|56962464|13|56.340|110.251|0.114|164984 +17|4|211957760|56962464|13|34.900|127.95|0.160|329420 + +[LZ4]:https://cyan4973.github.io/lz4/ +[LZ5]:https://github.com/inikep/lz5 +[ZStandard]:http://facebook.github.io/zstd/ diff --git a/lz4mt.c b/unix/lz4mt.c similarity index 100% rename from lz4mt.c rename to unix/lz4mt.c index 19c7f05..9298170 100644 --- a/lz4mt.c +++ b/unix/lz4mt.c @@ -19,8 +19,8 @@ #include #include -#include "lz4mt.h" #include "util.h" +#include "lz4mt.h" /** * program for testing threaded stuff on zstd diff --git a/lz5mt.c b/unix/lz5mt.c similarity index 100% rename from lz5mt.c rename to unix/lz5mt.c index 3339069..a8c28d4 100644 --- a/lz5mt.c +++ b/unix/lz5mt.c @@ -19,8 +19,8 @@ #include #include -#include "lz5mt.h" #include "util.h" +#include "lz5mt.h" /** * program for testing threaded stuff on zstd diff --git a/util.c b/unix/util.c similarity index 100% rename from util.c rename to unix/util.c diff --git a/util.h b/unix/util.h similarity index 100% rename from util.h rename to unix/util.h diff --git a/zstdmt.c b/unix/zstdmt.c similarity index 99% rename from zstdmt.c rename to unix/zstdmt.c index ca62ba8..c7ce960 100644 --- a/zstdmt.c +++ b/unix/zstdmt.c @@ -20,7 +20,6 @@ #include #include "zstdmt.h" -#include "zstd.h" #include "util.h" /** diff --git a/windows/Makefile b/windows/Makefile new file mode 100644 index 0000000..c23e772 --- /dev/null +++ b/windows/Makefile @@ -0,0 +1,69 @@ + +CROSS = i686-w64-mingw32- +CC = $(CROSS)gcc +STRIP = $(CROSS)strip +RANLIB = $(CROSS)ranlib +#CC = clang + +CFLAGS = -W -pthread -Wall -pipe +CFLAGS = -fomit-frame-pointer +LDFLAGS = -lpthread + +#CFLAGS += -DDEBUGME +#CFLAGS += -g +CFLAGS += -O3 +#CFLAGS += -march=native +#CFLAGS += -Wno-unused-but-set-variable +#CFLAGS += -Wno-unused-variable +#CFLAGS += -Wno-unused-function + +PRGS = lz4mt lz5mt zstdmt + +all: loadsource $(PRGS) +again: clean $(PRGS) + +ZSTDMTDIR = ../lib +CFLAGS += -I. -I$(ZSTDMTDIR) + +LIBLZ4 = $(ZSTDMTDIR)/threading.c $(ZSTDMTDIR)/lz4mt_common.c $(ZSTDMTDIR)/lz4mt_compress.c $(ZSTDMTDIR)/lz4mt_decompress.c +LIBLZ5 = $(ZSTDMTDIR)/threading.c $(ZSTDMTDIR)/lz5mt_common.c $(ZSTDMTDIR)/lz5mt_compress.c $(ZSTDMTDIR)/lz5mt_decompress.c +LIBZSTD = $(ZSTDMTDIR)/threading.c $(ZSTDMTDIR)/zstdmt_common.c $(ZSTDMTDIR)/zstdmt_compress.c $(ZSTDMTDIR)/zstdmt_decompress.c + +LZ4DIR = lz4/lib +LIBLZ4 += $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/xxhash.c +CF_LZ4 += $(CFLAGS) -Ilz4/lib + +LZ5DIR = lz5/lib +LIBLZ5 += $(LZ5DIR)/lz5.c $(LZ5DIR)/lz5frame.c $(LZ5DIR)/lz5hc.c $(LZ5DIR)/xxhash.c +CF_LZ5 += $(CFLAGS) -Ilz5/lib + +ZSTDDIR = zstd/lib +ZSTDCOMMON = $(ZSTDDIR)/common +LIBZSTD += $(ZSTDCOMMON)/entropy_common.c $(ZSTDCOMMON)/fse_decompress.c $(ZSTDCOMMON)/xxhash.c $(ZSTDCOMMON)/zstd_common.c +ZSTDCOMP = $(ZSTDDIR)/compress +LIBZSTD += $(ZSTDCOMP)/fse_compress.c $(ZSTDCOMP)/huf_compress.c $(ZSTDCOMP)/zbuff_compress.c $(ZSTDCOMP)/zstd_compress.c +ZSTDDECOMP = $(ZSTDDIR)/decompress +LIBZSTD += $(ZSTDDECOMP)/huf_decompress.c $(ZSTDDECOMP)/zstd_decompress.c +ZSTDLEGACY = $(ZSTDDIR)/legacy +LIBZSTD += $(ZSTDLEGACY)/zstd_v05.c $(ZSTDLEGACY)/zstd_v06.c $(ZSTDLEGACY)/zstd_v07.c +CF_ZSTD += $(CFLAGS) -I$(ZSTDDIR) -I$(ZSTDCOMMON) -I$(ZSTDCOMP) -I$(ZSTDDECOMP) -I$(ZSTDLEGACY) + +loadsource: + test -d lz4 || git clone https://github.com/Cyan4973/lz4 -b r131 --depth=1 lz4 + test -d lz5 || git clone https://github.com/inikep/lz5 --depth=1 lz5 + test -d zstd || git clone https://github.com/facebook/zstd --depth=1 zstd + +lz4mt: lz4mt.o + $(CC) $(CF_LZ4) $(LDFLAGS) -o $@ $(LIBLZ4) lz4mt.o + $(STRIP) $@ + +lz5mt: lz5mt.o + $(CC) $(CF_LZ5) $(LDFLAGS) -o $@ $(LIBLZ5) lz5mt.o + $(STRIP) $@ + +zstdmt: zstdmt.o + $(CC) $(CF_ZSTD) $(LDFLAGS) -o $@ $(LIBZSTD) zstdmt.o + $(STRIP) $@ + +clean: + rm -f $(PRGS) *.o *.a diff --git a/windows/README.md b/windows/README.md new file mode 100644 index 0000000..27cfa8a --- /dev/null +++ b/windows/README.md @@ -0,0 +1,47 @@ + +## Usage of the Testutils @ Windows + +### [LZ4] +``` +Usage: lz4mt [options] infile outfile + +Otions: + -l N set level (1..16) of compression (default: 1) + -t N set number of (de)compression threads (default: 2) + -i N set number of iterations for testing (default: 1) + -b N set input chunksize to N MiB (default: auto) + -c compress (default mode) + -d use decompress mode + -h show usage + -v show version +``` + +### [LZ5] +``` +Usage: lz5mt [options] infile outfile + +Otions: + -l N set level (1..16) of compression (default: 1) + -t N set number of (de)compression threads (default: 2) + -i N set number of iterations for testing (default: 1) + -b N set input chunksize to N MiB (default: auto) + -c compress (default mode) + -d use decompress mode + -h show usage + -v show version +``` + +### [ZStandard] +``` +Usage: zstdmt [options] infile outfile + +Otions: + -l N set level of compression (default: 3) + -t N set number of (de)compression threads (default: 2) + -i N set number of iterations for testing (default: 1) + -b N set input chunksize to N MiB (default: auto) + -c compress (default mode) + -d use decompress mode + -h show usage + -v show version +``` diff --git a/windows/README.mdc b/windows/README.mdc new file mode 100644 index 0000000..5ceccf6 --- /dev/null +++ b/windows/README.mdc @@ -0,0 +1,237 @@ + +# compiling via cross compiler @ linux +- i686-w64-mingw32-ranlib libzstd.a + +# Multithreading Library for [LZ4], [LZ5] and [ZStandard] + + +### Compression +``` +typedef struct { + void *buf; /* ptr to data */ + size_t size; /* current filled in buf */ + size_t allocated; /* length of buf */ +} LZ4MT_Buffer; + +/** + * reading and writing functions + * - you can use stdio functions or plain read/write ... + * - a sample is given in 7-Zip ZS or lz4mt.c + */ +typedef int (fn_read) (void *args, LZ4MT_Buffer * in); +typedef int (fn_write) (void *args, LZ4MT_Buffer * out); + +typedef struct { + fn_read *fn_read; + void *arg_read; + fn_write *fn_write; + void *arg_write; +} LZ4MT_RdWr_t; + +typedef struct LZ4MT_CCtx_s LZ4MT_CCtx; + +/* 1) allocate new cctx */ +LZ4MT_CCtx *LZ4MT_createCCtx(int threads, int level, int inputsize); + +/* 2) threaded compression */ +size_t LZ4MT_CompressCCtx(LZ4MT_CCtx * ctx, LZ4MT_RdWr_t * rdwr); + +/* 3) get some statistic */ +size_t LZ4MT_GetFramesCCtx(LZ4MT_CCtx * ctx); +size_t LZ4MT_GetInsizeCCtx(LZ4MT_CCtx * ctx); +size_t LZ4MT_GetOutsizeCCtx(LZ4MT_CCtx * ctx); + +/* 4) free cctx */ +void LZ4MT_freeCCtx(LZ4MT_CCtx * ctx); +``` + +### Decompression +``` +typedef struct LZ4MT_DCtx_s LZ4MT_DCtx; + +/* 1) allocate new cctx */ +LZ4MT_DCtx *LZ4MT_createDCtx(int threads, int inputsize); + +/* 2) threaded compression */ +size_t LZ4MT_DecompressDCtx(LZ4MT_DCtx * ctx, LZ4MT_RdWr_t * rdwr); + +/* 3) get some statistic */ +size_t LZ4MT_GetFramesDCtx(LZ4MT_DCtx * ctx); +size_t LZ4MT_GetInsizeDCtx(LZ4MT_DCtx * ctx); +size_t LZ4MT_GetOutsizeDCtx(LZ4MT_DCtx * ctx); + +/* 4) free cctx */ +void LZ4MT_freeDCtx(LZ4MT_DCtx * ctx); +``` + +## Benchmarks + +### [LZ4] +Level|Threads|InSize|OutSize|Frames|Real|User|Sys|MaxMem +-----|-------|------|-------|------|----|----|---|------ +1|1|211957760|101069667|3235|1.393|1.347|0.42|2096 +1|2|211957760|101069667|3235|0.622|1.200|0.39|2096 +1|4|211957760|101069667|3235|0.397|1.541|0.30|2096 +2|1|211957760|101069667|3235|1.312|1.265|0.43|2096 +2|2|211957760|101069667|3235|0.734|1.415|0.44|2096 +2|4|211957760|101069667|3235|0.338|1.297|0.41|2248 +3|1|211957760|88923937|3235|6.945|6.854|0.61|2096 +3|2|211957760|88923937|3235|3.381|6.687|0.47|2096 +3|4|211957760|88923937|3235|1.999|7.891|0.64|2712 +4|1|211957760|87542131|3235|7.940|7.847|0.58|2096 +4|2|211957760|87542131|3235|4.161|8.224|0.62|2096 +4|4|211957760|87542131|3235|2.207|8.718|0.61|4796 +5|1|211957760|86701884|3235|8.994|8.907|0.51|2096 +5|2|211957760|86701884|3235|4.822|9.527|0.72|2096 +5|4|211957760|86701884|3235|2.771|10.967|0.61|2844 +6|1|211957760|86228097|3235|10.425|10.344|0.42|2096 +6|2|211957760|86228097|3235|5.388|10.677|0.57|2096 +6|4|211957760|86228097|3235|3.288|13.18|0.63|2720 +7|1|211957760|85963374|3235|11.856|11.749|0.61|2096 +7|2|211957760|85963374|3235|6.39|11.972|0.58|2096 +7|4|211957760|85963374|3235|3.603|14.282|0.61|2760 +8|1|211957760|85824153|3235|13.551|13.444|0.56|2096 +8|2|211957760|85824153|3235|6.897|13.695|0.43|2096 +8|4|211957760|85824153|3235|4.56|16.79|0.65|2976 +9|1|211957760|85762795|3235|14.623|14.515|0.54|2096 +9|2|211957760|85762795|3235|7.564|15.15|0.55|2096 +9|4|211957760|85762795|3235|4.593|18.215|0.68|3000 +10|1|211957760|85736453|3235|16.81|15.977|0.44|2096 +10|2|211957760|85736453|3235|8.225|16.328|0.54|2096 +10|4|211957760|85736453|3235|4.730|18.745|0.79|3144 +11|1|211957760|85725904|3235|17.6|16.894|0.48|2096 +11|2|211957760|85725904|3235|8.902|17.672|0.62|2096 +11|4|211957760|85725904|3235|5.73|20.123|0.58|3700 +12|1|211957760|85720684|3235|18.53|17.935|0.54|2096 +12|2|211957760|85720684|3235|9.186|18.250|0.54|2096 +12|4|211957760|85720684|3235|5.715|22.706|0.52|3536 +13|1|211957760|85717239|3235|19.752|19.622|0.60|2096 +13|2|211957760|85717239|3235|10.300|20.459|0.61|2164 +13|4|211957760|85717239|3235|6.69|24.71|0.91|3672 +14|1|211957760|85714803|3235|22.436|22.292|0.66|2096 +14|2|211957760|85714803|3235|11.744|23.339|0.57|2448 +14|4|211957760|85714803|3235|6.593|26.194|0.61|4104 +15|1|211957760|85713782|3235|24.788|24.633|0.72|2096 +15|2|211957760|85713782|3235|12.364|24.591|0.50|2968 +15|4|211957760|85713782|3235|7.336|29.152|0.59|7400 +16|1|211957760|85713635|3235|25.313|25.177|0.50|2096 +16|2|211957760|85713635|3235|12.791|25.426|0.62|3744 +16|4|211957760|85713635|3235|7.650|30.419|0.46|6840 + +### [LZ5] +Level|Threads|InSize|OutSize|Frames|Real|User|Sys|MaxMem +-----|-------|------|-------|------|----|----|---|------ +1|1|211957760|123117368|3235|1.523|1.466|0.50|2096 +1|2|211957760|123118976|3235|0.804|1.554|0.45|2096 +1|4|211957760|123118409|3235|0.442|1.707|0.46|2096 +2|1|211957760|110989793|3235|2.121|2.71|0.42|2096 +2|2|211957760|110990294|3235|1.36|2.22|0.41|2096 +2|4|211957760|110990676|3235|0.549|2.136|0.43|2120 +3|1|211957760|100263984|3235|2.632|2.589|0.33|2100 +3|2|211957760|100264075|3235|1.359|2.663|0.43|2100 +3|4|211957760|100263827|3235|0.841|3.305|0.38|2224 +4|1|211957760|91372757|3235|4.62|3.982|0.64|2100 +4|2|211957760|91372725|3235|2.200|4.331|0.51|2100 +4|4|211957760|91372698|3235|1.245|4.897|0.54|2848 +5|1|211957760|90471735|3235|4.528|4.454|0.54|2100 +5|2|211957760|90471659|3235|2.262|4.452|0.51|2484 +5|4|211957760|90471655|3235|1.377|5.404|0.61|4260 +6|1|211957760|86823397|3235|13.271|13.164|0.55|2100 +6|2|211957760|86823397|3235|6.844|13.570|0.60|2124 +6|4|211957760|86823397|3235|4.305|17.59|0.74|9732 +7|1|211957760|86367683|3235|13.664|13.551|0.59|2100 +7|2|211957760|86367683|3235|6.964|13.821|0.42|4224 +7|4|211957760|86367683|3235|4.310|17.75|0.71|5808 +8|1|211957760|86228271|3235|14.230|14.103|0.66|3528 +8|2|211957760|86228271|3235|7.698|15.258|0.65|9540 +8|4|211957760|86228271|3235|5.2|19.785|0.93|12604 +9|1|211957760|84279832|3235|32.157|17.532|14.468|36128 +9|2|211957760|84279832|3235|18.114|20.801|14.419|67788 +9|4|211957760|84279832|3235|15.102|30.357|21.389|134544 +10|1|211957760|83448337|3235|33.661|19.69|14.431|34252 +10|2|211957760|83448337|3235|21.210|23.979|16.901|70376 +10|4|211957760|83448337|3235|13.872|31.216|18.327|139448 +11|1|211957760|82198886|3235|46.662|28.395|18.34|34280 +11|2|211957760|82198886|3235|23.75|30.545|14.642|67912 +11|4|211957760|82198886|3235|15.690|39.594|17.732|136308 +12|1|211957760|81284779|3235|59.607|41.422|17.894|35904 +12|2|211957760|81284779|3235|29.520|43.461|14.636|67772 +12|4|211957760|81284779|3235|20.186|55.754|19.54|134728 +13|1|211957760|81940180|3235|81.349|64.895|16.94|36352 +13|2|211957760|81940180|3235|44.839|69.904|18.34|71612 +13|4|211957760|81940180|3235|28.12|84.279|19.784|142868 +14|1|211957760|81560383|3235|89.732|73.370|15.982|36352 +14|2|211957760|81560383|3235|48.969|78.345|17.997|73772 +14|4|211957760|81560383|3235|29.402|93.96|18.868|144112 +15|1|211957760|81490962|3235|598.712|135.637|460.587|1117296 +15|2|211957760|81490962|3235|348.964|152.81|528.464|2230196 +15|4|211957760|81490962|3235|264.598|191.203|816.42|4399220 +16|1|211957760|81490962|3235|595.970|133.238|460.31|1117236 +16|2|211957760|81490962|3235|384.303|161.587|588.589|2232060 +16|4|211957760|81490962|3235|295.830|206.416|922.797|4419620 + +### [ZStandard] +Level|Threads|InSize|OutSize|Frames|Real|User|Sys|MaxMem +-----|-------|------|-------|------|----|----|---|------ +1|1|211957760|73853773|102|2.694|2.636|0.47|6212 +1|2|211957760|73853773|102|1.329|2.579|0.55|11968 +1|4|211957760|73853773|102|0.898|3.448|0.84|20412 +2|1|211957760|70521776|102|3.104|3.45|0.46|6236 +2|2|211957760|70521776|102|1.536|2.995|0.56|11536 +2|4|211957760|70521776|102|1.76|4.127|0.96|21444 +3|1|211957760|67263553|51|4.265|4.185|0.62|11096 +3|2|211957760|67263553|51|2.138|4.171|0.68|22728 +3|4|211957760|67263553|51|1.626|6.61|0.101|47928 +4|1|211957760|66324953|51|4.642|4.560|0.62|13988 +4|2|211957760|66324953|51|2.697|5.266|0.83|30332 +4|4|211957760|66324953|51|2.8|7.556|0.120|57040 +5|1|211957760|65188167|34|6.892|6.804|0.60|15100 +5|2|211957760|65188167|34|3.835|7.403|0.71|29996 +5|4|211957760|65188167|34|2.543|9.372|0.76|60748 +6|1|211957760|63279533|34|9.109|9.9|0.62|19364 +6|2|211957760|63279533|34|5.7|9.793|0.75|38620 +6|4|211957760|63279533|34|3.613|13.559|0.98|75180 +7|1|211957760|62305767|34|9.966|9.868|0.56|23924 +7|2|211957760|62305767|34|6.59|11.905|0.77|47140 +7|4|211957760|62305767|34|3.924|14.428|0.106|94540 +8|1|211957760|61248190|26|14.264|14.132|0.72|28380 +8|2|211957760|61248190|26|7.927|15.427|0.71|55348 +8|4|211957760|61248190|26|5.350|19.541|0.129|105360 +9|1|211957760|60694170|26|16.305|16.158|0.76|33568 +9|2|211957760|60694170|26|10.784|20.648|0.104|66752 +9|4|211957760|60694170|26|7.110|26.91|0.162|134808 +10|1|211957760|60429156|26|17.495|17.348|0.71|37652 +10|2|211957760|60429156|26|10.216|19.839|0.81|76244 +10|4|211957760|60429156|26|7.16|26.528|0.146|153452 +11|1|211957760|59939304|26|19.225|19.77|0.67|60724 +11|2|211957760|59939304|26|11.967|23.53|0.103|122344 +11|4|211957760|59939304|26|7.677|29.2|0.150|243604 +12|1|211957760|59543345|26|25.706|25.522|0.71|60268 +12|2|211957760|59543345|26|15.107|29.754|0.84|122516 +12|4|211957760|59543345|26|10.435|39.818|0.162|248232 +13|1|211957760|59289863|26|29.759|29.566|0.76|68568 +13|2|211957760|59289863|26|18.462|36.283|0.100|141032 +13|4|211957760|59289863|26|12.230|46.21|0.154|283432 +14|1|211957760|58948339|26|43.408|43.162|0.82|68892 +14|2|211957760|58948339|26|27.227|54.62|0.103|145440 +14|4|211957760|58948339|26|18.10|69.415|0.162|287508 +15|1|211957760|58249793|13|78.811|78.428|0.77|62744 +15|2|211957760|58249793|13|47.213|92.540|0.137|123976 +15|4|211957760|58249793|13|27.105|100.529|0.144|245424 +16|1|211957760|57640281|13|85.943|85.423|0.156|100200 +16|2|211957760|57640281|13|48.152|93.87|0.200|199128 +16|4|211957760|57640281|13|29.156|106.315|0.237|396972 +17|1|211957760|56962464|13|106.187|105.615|0.96|82804 +17|2|211957760|56962464|13|56.340|110.251|0.114|164984 +17|4|211957760|56962464|13|34.900|127.95|0.160|329420 + +[LZ4]:https://cyan4973.github.io/lz4/ +[LZ5]:https://github.com/inikep/lz5 +[ZStandard]:http://facebook.github.io/zstd/ + +# TODO + +- threaded decompression for zstd not ready +- document the header files correctly ... just some first start currently + +/TR 2016-09-25 diff --git a/windows/lz4mt.c b/windows/lz4mt.c new file mode 100644 index 0000000..ebc416d --- /dev/null +++ b/windows/lz4mt.c @@ -0,0 +1,233 @@ + +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * You can contact the author at: + * - lz5mt source repository: https://github.com/mcmilk/zstdmt + */ + +#include +#include +#include + +#include "lz4mt.h" + +/** + * program for testing threaded stuff + */ + +static void perror_exit(const char *msg) +{ + printf("%s\n", msg); + fflush(stdout); + exit(1); +} + +static void version(void) +{ + printf("lz4mt version 0.1\n"); + + exit(0); +} + +static void usage(void) +{ + printf("Usage: lz4mt [options] infile outfile\n\n"); + printf("Otions:\n"); + printf(" -l N set level of compression (default: 1)\n"); + printf(" -t N set number of (de)compression threads (default: 2)\n"); + printf(" -i N set number of iterations for testing (default: 1)\n"); + printf(" -b N set input chunksize to N KiB (default: auto)\n"); + printf(" -c compress (default mode)\n"); + printf(" -d use decompress mode\n"); + printf(" -h show usage\n"); + printf(" -v show version\n"); + + exit(0); +} + +#define MODE_COMPRESS 1 +#define MODE_DECOMPRESS 2 + +/* for the -i option */ +#define MAX_ITERATIONS 1000 + +int my_read_loop(void *arg, LZ4MT_Buffer * in) +{ + FILE *fd = (FILE *) arg; + ssize_t done = fread(in->buf, 1, in->size, fd); + +#if 0 + printf("fread(), todo=%u done=%u\n", in->size, done); + fflush(stdout); +#endif + + in->size = done; + return done; +} + +int my_write_loop(void *arg, LZ4MT_Buffer * out) +{ + FILE *fd = (FILE *) arg; + ssize_t done = fwrite(out->buf, 1, out->size, fd); + +#if 0 + printf("fwrite(), todo=%u done=%u\n", out->size, done); + fflush(stdout); +#endif + + out->size = done; + return done; +} + +static void +do_compress(int threads, int level, int bufsize, FILE * fin, FILE * fout) +{ + LZ4MT_RdWr_t rdwr; + size_t ret; + + /* 1) setup read/write functions */ + rdwr.fn_read = my_read_loop; + rdwr.fn_write = my_write_loop; + rdwr.arg_read = (void *)fin; + rdwr.arg_write = (void *)fout; + + /* 2) create compression context */ + LZ4MT_CCtx *ctx = LZ4MT_createCCtx(threads, level, bufsize); + if (!ctx) + perror_exit("Allocating ctx failed!"); + + /* 3) compress */ + ret = LZ4MT_CompressCCtx(ctx, &rdwr); + if (LZ4MT_isError(ret)) + perror_exit(LZ4MT_getErrorString(ret)); + + /* 5) free resources */ + LZ4MT_freeCCtx(ctx); +} + +static void do_decompress(int threads, int bufsize, FILE * fin, FILE * fout) +{ + LZ4MT_RdWr_t rdwr; + size_t ret; + + /* 1) setup read/write functions */ + rdwr.fn_read = my_read_loop; + rdwr.fn_write = my_write_loop; + rdwr.arg_read = (void *)fin; + rdwr.arg_write = (void *)fout; + + /* 2) create compression context */ + LZ4MT_DCtx *ctx = LZ4MT_createDCtx(threads, bufsize); + if (!ctx) + perror_exit("Allocating ctx failed!"); + + /* 3) compress */ + ret = LZ4MT_DecompressDCtx(ctx, &rdwr); + if (LZ4MT_isError(ret)) + perror_exit(LZ4MT_getErrorString(ret)); + + /* 4) free resources */ + LZ4MT_freeDCtx(ctx); +} + +int main(int argc, char **argv) +{ + /* default options: */ + int opt, opt_threads = 2, opt_level = 1; + int opt_mode = MODE_COMPRESS; + int opt_iterations = 1, opt_bufsize = 0; + FILE *fin, *fout; + + while ((opt = getopt(argc, argv, "vhHl:t:i:dcb:")) != -1) { + switch (opt) { + case 'v': /* version */ + version(); + case 'h': /* help */ + usage(); + case 'l': /* level */ + opt_level = atoi(optarg); + break; + case 't': /* threads */ + opt_threads = atoi(optarg); + break; + case 'i': /* iterations */ + opt_iterations = atoi(optarg); + break; + case 'd': /* mode = decompress */ + opt_mode = MODE_DECOMPRESS; + break; + case 'c': /* mode = compress */ + opt_mode = MODE_COMPRESS; + break; + case 'b': /* input buffer in MB */ + opt_bufsize = atoi(optarg); + break; + default: + usage(); + } + } + + /* prog [options] infile outfile */ + if (argc != optind + 2) + usage(); + + /** + * check parameters + */ + + /* opt_level = 1..22 */ + if (opt_level < 1) + opt_level = 1; + else if (opt_level > LZ4MT_LEVEL_MAX) + opt_level = LZ4MT_LEVEL_MAX; + + /* opt_threads = 1..LZ4MT_THREAD_MAX */ + if (opt_threads < 1) + opt_threads = 1; + else if (opt_threads > LZ4MT_THREAD_MAX) + opt_threads = LZ4MT_THREAD_MAX; + + /* opt_iterations = 1..MAX_ITERATIONS */ + if (opt_iterations < 1) + opt_iterations = 1; + else if (opt_iterations > MAX_ITERATIONS) + opt_iterations = MAX_ITERATIONS; + + /* opt_bufsize is in KiB */ + if (opt_bufsize > 0) + opt_bufsize *= 1024; + + /* file names */ + fin = fopen(argv[optind], "rb"); + if (fin == NULL) + perror_exit("Opening infile failed"); + + fout = fopen(argv[optind + 1], "wb"); + if (fout == NULL) + perror_exit("Opening outfile failed"); + + for (;;) { + if (opt_mode == MODE_COMPRESS) { + do_compress(opt_threads, opt_level, opt_bufsize, fin, + fout); + } else { + do_decompress(opt_threads, opt_bufsize, fin, fout); + } + + opt_iterations--; + if (opt_iterations == 0) + break; + + fseek(fin, 0, SEEK_SET); + fseek(fout, 0, SEEK_SET); + } + + /* exit should flush stdout */ + exit(0); +} diff --git a/windows/lz5mt.c b/windows/lz5mt.c new file mode 100644 index 0000000..7cc9ae2 --- /dev/null +++ b/windows/lz5mt.c @@ -0,0 +1,233 @@ + +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * You can contact the author at: + * - lz5mt source repository: https://github.com/mcmilk/zstdmt + */ + +#include +#include +#include + +#include "lz5mt.h" + +/** + * program for testing threaded stuff + */ + +static void perror_exit(const char *msg) +{ + printf("%s\n", msg); + fflush(stdout); + exit(1); +} + +static void version(void) +{ + printf("lz5mt version 0.1\n"); + + exit(0); +} + +static void usage(void) +{ + printf("Usage: lz5mt [options] infile outfile\n\n"); + printf("Otions:\n"); + printf(" -l N set level of compression (default: 1)\n"); + printf(" -t N set number of (de)compression threads (default: 2)\n"); + printf(" -i N set number of iterations for testing (default: 1)\n"); + printf(" -b N set input chunksize to N KiB (default: auto)\n"); + printf(" -c compress (default mode)\n"); + printf(" -d use decompress mode\n"); + printf(" -h show usage\n"); + printf(" -v show version\n"); + + exit(0); +} + +#define MODE_COMPRESS 1 +#define MODE_DECOMPRESS 2 + +/* for the -i option */ +#define MAX_ITERATIONS 1000 + +int my_read_loop(void *arg, LZ5MT_Buffer * in) +{ + FILE *fd = (FILE *) arg; + ssize_t done = fread(in->buf, 1, in->size, fd); + +#if 0 + printf("fread(), todo=%u done=%u\n", in->size, done); + fflush(stdout); +#endif + + in->size = done; + return done; +} + +int my_write_loop(void *arg, LZ5MT_Buffer * out) +{ + FILE *fd = (FILE *) arg; + ssize_t done = fwrite(out->buf, 1, out->size, fd); + +#if 0 + printf("fwrite(), todo=%u done=%u\n", out->size, done); + fflush(stdout); +#endif + + out->size = done; + return done; +} + +static void +do_compress(int threads, int level, int bufsize, FILE * fin, FILE * fout) +{ + LZ5MT_RdWr_t rdwr; + size_t ret; + + /* 1) setup read/write functions */ + rdwr.fn_read = my_read_loop; + rdwr.fn_write = my_write_loop; + rdwr.arg_read = (void *)fin; + rdwr.arg_write = (void *)fout; + + /* 2) create compression context */ + LZ5MT_CCtx *ctx = LZ5MT_createCCtx(threads, level, bufsize); + if (!ctx) + perror_exit("Allocating ctx failed!"); + + /* 3) compress */ + ret = LZ5MT_CompressCCtx(ctx, &rdwr); + if (LZ5MT_isError(ret)) + perror_exit(LZ5MT_getErrorString(ret)); + + /* 5) free resources */ + LZ5MT_freeCCtx(ctx); +} + +static void do_decompress(int threads, int bufsize, FILE * fin, FILE * fout) +{ + LZ5MT_RdWr_t rdwr; + size_t ret; + + /* 1) setup read/write functions */ + rdwr.fn_read = my_read_loop; + rdwr.fn_write = my_write_loop; + rdwr.arg_read = (void *)fin; + rdwr.arg_write = (void *)fout; + + /* 2) create compression context */ + LZ5MT_DCtx *ctx = LZ5MT_createDCtx(threads, bufsize); + if (!ctx) + perror_exit("Allocating ctx failed!"); + + /* 3) compress */ + ret = LZ5MT_DecompressDCtx(ctx, &rdwr); + if (LZ5MT_isError(ret)) + perror_exit(LZ5MT_getErrorString(ret)); + + /* 4) free resources */ + LZ5MT_freeDCtx(ctx); +} + +int main(int argc, char **argv) +{ + /* default options: */ + int opt, opt_threads = 2, opt_level = 1; + int opt_mode = MODE_COMPRESS; + int opt_iterations = 1, opt_bufsize = 0; + FILE *fin, *fout; + + while ((opt = getopt(argc, argv, "vhHl:t:i:dcb:")) != -1) { + switch (opt) { + case 'v': /* version */ + version(); + case 'h': /* help */ + usage(); + case 'l': /* level */ + opt_level = atoi(optarg); + break; + case 't': /* threads */ + opt_threads = atoi(optarg); + break; + case 'i': /* iterations */ + opt_iterations = atoi(optarg); + break; + case 'd': /* mode = decompress */ + opt_mode = MODE_DECOMPRESS; + break; + case 'c': /* mode = compress */ + opt_mode = MODE_COMPRESS; + break; + case 'b': /* input buffer in MB */ + opt_bufsize = atoi(optarg); + break; + default: + usage(); + } + } + + /* prog [options] infile outfile */ + if (argc != optind + 2) + usage(); + + /** + * check parameters + */ + + /* opt_level = 1..22 */ + if (opt_level < 1) + opt_level = 1; + else if (opt_level > LZ5MT_LEVEL_MAX) + opt_level = LZ5MT_LEVEL_MAX; + + /* opt_threads = 1..LZ5MT_THREAD_MAX */ + if (opt_threads < 1) + opt_threads = 1; + else if (opt_threads > LZ5MT_THREAD_MAX) + opt_threads = LZ5MT_THREAD_MAX; + + /* opt_iterations = 1..MAX_ITERATIONS */ + if (opt_iterations < 1) + opt_iterations = 1; + else if (opt_iterations > MAX_ITERATIONS) + opt_iterations = MAX_ITERATIONS; + + /* opt_bufsize is in KiB */ + if (opt_bufsize > 0) + opt_bufsize *= 1024; + + /* file names */ + fin = fopen(argv[optind], "rb"); + if (fin == NULL) + perror_exit("Opening infile failed"); + + fout = fopen(argv[optind + 1], "wb"); + if (fout == NULL) + perror_exit("Opening outfile failed"); + + for (;;) { + if (opt_mode == MODE_COMPRESS) { + do_compress(opt_threads, opt_level, opt_bufsize, fin, + fout); + } else { + do_decompress(opt_threads, opt_bufsize, fin, fout); + } + + opt_iterations--; + if (opt_iterations == 0) + break; + + fseek(fin, 0, SEEK_SET); + fseek(fout, 0, SEEK_SET); + } + + /* exit should flush stdout */ + exit(0); +} diff --git a/windows/zstdmt.c b/windows/zstdmt.c new file mode 100644 index 0000000..013ac18 --- /dev/null +++ b/windows/zstdmt.c @@ -0,0 +1,233 @@ + +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + */ + +#include +#include +#include + +#include "zstdmt.h" + +/** + * program for testing threaded stuff on zstd + */ + +static void perror_exit(const char *msg) +{ + printf("%s\n", msg); + fflush(stdout); + exit(1); +} + +static void version(void) +{ + printf("zstdmt version 0.1\n"); + + exit(0); +} + +static void usage(void) +{ + printf("Usage: zstdmt [options] infile outfile\n\n"); + printf("Otions:\n"); + printf(" -l N set level of compression (default: 3)\n"); + printf(" -t N set number of (de)compression threads (default: 2)\n"); + printf(" -i N set number of iterations for testing (default: 1)\n"); + printf(" -b N set input chunksize to N KiB (default: auto)\n"); + printf(" -c compress (default mode)\n"); + printf(" -d use decompress mode\n"); + printf(" -h show usage\n"); + printf(" -v show version\n"); + + exit(0); +} + +#define MODE_COMPRESS 1 +#define MODE_DECOMPRESS 2 + +/* for the -i option */ +#define MAX_ITERATIONS 1000 + +int my_read_loop(void *arg, ZSTDMT_Buffer * in) +{ + FILE *fd = (FILE *) arg; + ssize_t done = fread(in->buf, 1, in->size, fd); + +#if 0 + printf("fread(), todo=%u done=%u\n", in->size, done); + fflush(stdout); +#endif + + in->size = done; + return done; +} + +int my_write_loop(void *arg, ZSTDMT_Buffer * out) +{ + FILE *fd = (FILE *) arg; + ssize_t done = fwrite(out->buf, 1, out->size, fd); + +#if 0 + printf("fwrite(), todo=%u done=%u\n", out->size, done); + fflush(stdout); +#endif + + out->size = done; + return done; +} + +static void +do_compress(int threads, int level, int bufsize, FILE * fin, FILE * fout) +{ + ZSTDMT_RdWr_t rdwr; + size_t ret; + + /* 1) setup read/write functions */ + rdwr.fn_read = my_read_loop; + rdwr.fn_write = my_write_loop; + rdwr.arg_read = (void *)fin; + rdwr.arg_write = (void *)fout; + + /* 2) create compression context */ + ZSTDMT_CCtx *ctx = ZSTDMT_createCCtx(threads, level, bufsize); + if (!ctx) + perror_exit("Allocating ctx failed!"); + + /* 3) compress */ + ret = ZSTDMT_CompressCCtx(ctx, &rdwr); + if (ZSTDMT_isError(ret)) + perror_exit(ZSTDMT_getErrorString(ret)); + + /* 5) free resources */ + ZSTDMT_freeCCtx(ctx); +} + +static void do_decompress(int threads, int bufsize, FILE * fin, FILE * fout) +{ + ZSTDMT_RdWr_t rdwr; + size_t ret; + + /* 1) setup read/write functions */ + rdwr.fn_read = my_read_loop; + rdwr.fn_write = my_write_loop; + rdwr.arg_read = (void *)fin; + rdwr.arg_write = (void *)fout; + + /* 2) create compression context */ + ZSTDMT_DCtx *ctx = ZSTDMT_createDCtx(threads, bufsize); + if (!ctx) + perror_exit("Allocating ctx failed!"); + + /* 3) compress */ + ret = ZSTDMT_DecompressDCtx(ctx, &rdwr); + if (ZSTDMT_isError(ret)) + perror_exit(ZSTDMT_getErrorString(ret)); + + /* 4) free resources */ + ZSTDMT_freeDCtx(ctx); +} + +int main(int argc, char **argv) +{ + /* default options: */ + int opt, opt_threads = 2, opt_level = 3; + int opt_mode = MODE_COMPRESS; + int opt_iterations = 1, opt_bufsize = 0; + FILE *fin, *fout; + + while ((opt = getopt(argc, argv, "vhHl:t:i:dcb:")) != -1) { + switch (opt) { + case 'v': /* version */ + version(); + case 'h': /* help */ + usage(); + case 'l': /* level */ + opt_level = atoi(optarg); + break; + case 't': /* threads */ + opt_threads = atoi(optarg); + break; + case 'i': /* iterations */ + opt_iterations = atoi(optarg); + break; + case 'd': /* mode = decompress */ + opt_mode = MODE_DECOMPRESS; + break; + case 'c': /* mode = compress */ + opt_mode = MODE_COMPRESS; + break; + case 'b': /* input buffer in MB */ + opt_bufsize = atoi(optarg); + break; + default: + usage(); + } + } + + /* prog [options] infile outfile */ + if (argc != optind + 2) + usage(); + + /** + * check parameters + */ + + /* opt_level = 1..22 */ + if (opt_level < 1) + opt_level = 1; + else if (opt_level > ZSTDMT_LEVEL_MAX) + opt_level = ZSTDMT_LEVEL_MAX; + + /* opt_threads = 1..ZSTDMT_THREAD_MAX */ + if (opt_threads < 1) + opt_threads = 1; + else if (opt_threads > ZSTDMT_THREAD_MAX) + opt_threads = ZSTDMT_THREAD_MAX; + + /* opt_iterations = 1..MAX_ITERATIONS */ + if (opt_iterations < 1) + opt_iterations = 1; + else if (opt_iterations > MAX_ITERATIONS) + opt_iterations = MAX_ITERATIONS; + + /* opt_bufsize is in KiB */ + if (opt_bufsize > 0) + opt_bufsize *= 1024; + + /* file names */ + fin = fopen(argv[optind], "rb"); + if (fin == NULL) + perror_exit("Opening infile failed"); + + fout = fopen(argv[optind + 1], "wb"); + if (fout == NULL) + perror_exit("Opening outfile failed"); + + for (;;) { + if (opt_mode == MODE_COMPRESS) { + do_compress(opt_threads, opt_level, opt_bufsize, fin, + fout); + } else { + do_decompress(opt_threads, opt_bufsize, fin, fout); + } + + opt_iterations--; + if (opt_iterations == 0) + break; + + fseek(fin, 0, SEEK_SET); + fseek(fout, 0, SEEK_SET); + } + + /* exit should flush stdout */ + exit(0); +}