From c9b5c4f60a5b838a893e854dd72f0cb79b765cec Mon Sep 17 00:00:00 2001 From: Phil Pearl Date: Mon, 29 Apr 2024 19:09:30 +0100 Subject: [PATCH] nsqd: Switch to Klaus Post's flate compression library This is a dropin replacement for the standard library flate compression. --- apps/nsq_to_file/file_logger.go | 6 +++--- go.mod | 1 + go.sum | 6 ++---- internal/http_api/compress.go | 5 +++-- nsqd/client_v2.go | 2 +- nsqd/protocol_v2_test.go | 6 +++--- nsqd/protocol_v2_unixsocket_test.go | 16 ++++++++++++++-- 7 files changed, 27 insertions(+), 15 deletions(-) diff --git a/apps/nsq_to_file/file_logger.go b/apps/nsq_to_file/file_logger.go index ae16b4082..d516d03a9 100644 --- a/apps/nsq_to_file/file_logger.go +++ b/apps/nsq_to_file/file_logger.go @@ -1,7 +1,6 @@ package main import ( - "compress/gzip" "errors" "fmt" "io" @@ -11,6 +10,7 @@ import ( "strings" "time" + "github.com/klauspost/compress/gzip" "github.com/nsqio/go-nsq" "github.com/nsqio/nsq/internal/lg" ) @@ -331,7 +331,7 @@ func (f *FileLogger) updateFile() { } else { openFlag |= os.O_APPEND } - f.out, err = os.OpenFile(absFilename, openFlag, 0666) + f.out, err = os.OpenFile(absFilename, openFlag, 0o666) if err != nil { if os.IsExist(err) { f.logf(lg.WARN, "[%s/%s] working file already exists: %s", f.topic, f.opts.Channel, absFilename) @@ -369,7 +369,7 @@ func (f *FileLogger) updateFile() { func makeDirFromPath(logf lg.AppLogFunc, path string) error { dir, _ := filepath.Split(path) if dir != "" { - return os.MkdirAll(dir, 0770) + return os.MkdirAll(dir, 0o770) } return nil } diff --git a/go.mod b/go.mod index 247d4c928..dbcbd9704 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/golang/snappy v0.0.4 github.com/judwhite/go-svc v1.2.1 github.com/julienschmidt/httprouter v1.3.0 + github.com/klauspost/compress v1.17.8 github.com/mreiferson/go-options v1.0.0 github.com/nsqio/go-diskqueue v1.1.0 github.com/nsqio/go-nsq v1.1.0 diff --git a/go.sum b/go.sum index eb4b44846..c53facc12 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= -github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0= @@ -18,6 +16,8 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/mreiferson/go-options v1.0.0 h1:RMLidydGlDWpL+lQTXo0bVIf/XT2CTq7AEJMoz5/VWs= github.com/mreiferson/go-options v1.0.0/go.mod h1:zHtCks/HQvOt8ATyfwVe3JJq2PPuImzXINPRTC03+9w= github.com/mreiferson/go-svc v1.2.2-0.20210815184239-7a96e00010f6 h1:NbuBXARvEXrYZ1SzN53ZpObeuwGhl1zvs/C+kzCggrQ= @@ -32,8 +32,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20211023085530-d6a326fbbf70 h1:SeSEfdIxyvwGJliREIJhRPPXvW6sDlLT+UQ3B0hD0NA= -golang.org/x/sys v0.0.0-20211023085530-d6a326fbbf70/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/http_api/compress.go b/internal/http_api/compress.go index ebecad9f1..651a6012c 100644 --- a/internal/http_api/compress.go +++ b/internal/http_api/compress.go @@ -7,11 +7,12 @@ package http_api import ( - "compress/flate" - "compress/gzip" "io" "net/http" "strings" + + "github.com/klauspost/compress/flate" + "github.com/klauspost/compress/gzip" ) type compressResponseWriter struct { diff --git a/nsqd/client_v2.go b/nsqd/client_v2.go index 32250e72c..2eabef269 100644 --- a/nsqd/client_v2.go +++ b/nsqd/client_v2.go @@ -2,7 +2,6 @@ package nsqd import ( "bufio" - "compress/flate" "crypto/tls" "fmt" "net" @@ -12,6 +11,7 @@ import ( "time" "github.com/golang/snappy" + "github.com/klauspost/compress/flate" "github.com/nsqio/nsq/internal/auth" ) diff --git a/nsqd/protocol_v2_test.go b/nsqd/protocol_v2_test.go index de7c0f8c4..bf881d836 100644 --- a/nsqd/protocol_v2_test.go +++ b/nsqd/protocol_v2_test.go @@ -3,7 +3,6 @@ package nsqd import ( "bufio" "bytes" - "compress/flate" "crypto/tls" _ "embed" "encoding/json" @@ -25,6 +24,7 @@ import ( "time" "github.com/golang/snappy" + "github.com/klauspost/compress/flate" "github.com/nsqio/go-nsq" "github.com/nsqio/nsq/internal/protocol" "github.com/nsqio/nsq/internal/test" @@ -968,7 +968,6 @@ func TestTLSAuthRequire(t *testing.T) { t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) - } func TestTLSAuthRequireVerify(t *testing.T) { @@ -1494,7 +1493,8 @@ func TestClientAuth(t *testing.T) { } func runAuthTest(t *testing.T, authResponse string, authSecret string, authError string, - authSuccess string, tlsEnabled bool, commonName string) { + authSuccess string, tlsEnabled bool, commonName string, +) { var err error var expectedRemoteIP string expectedTLS := "false" diff --git a/nsqd/protocol_v2_unixsocket_test.go b/nsqd/protocol_v2_unixsocket_test.go index 89b54ff20..6c3f8485d 100644 --- a/nsqd/protocol_v2_unixsocket_test.go +++ b/nsqd/protocol_v2_unixsocket_test.go @@ -3,7 +3,6 @@ package nsqd import ( "bufio" "bytes" - "compress/flate" "crypto/tls" "encoding/json" "errors" @@ -21,6 +20,7 @@ import ( "time" "github.com/golang/snappy" + "github.com/klauspost/compress/flate" "github.com/nsqio/go-nsq" "github.com/nsqio/nsq/internal/protocol" "github.com/nsqio/nsq/internal/test" @@ -903,7 +903,6 @@ func TestUnixSocketTLSAuthRequire(t *testing.T) { t.Logf("frameType: %d, data: %s", frameType, data) test.Equal(t, frameTypeResponse, frameType) test.Equal(t, []byte("OK"), data) - } func TestUnixSocketTLSAuthRequireVerify(t *testing.T) { @@ -1527,18 +1526,23 @@ func benchmarkUnixSocketProtocolV2PubMultiTopic(b *testing.B, numTopics int) { func BenchmarkUnixSocketProtocolV2PubMultiTopic1(b *testing.B) { benchmarkUnixSocketProtocolV2PubMultiTopic(b, 1) } + func BenchmarkUnixSocketkProtocolV2PubMultiTopic2(b *testing.B) { benchmarkUnixSocketProtocolV2PubMultiTopic(b, 2) } + func BenchmarkUnixSocketProtocolV2PubMultiTopic4(b *testing.B) { benchmarkUnixSocketProtocolV2PubMultiTopic(b, 4) } + func BenchmarkUnixSocketProtocolV2PubMultiTopic8(b *testing.B) { benchmarkUnixSocketProtocolV2PubMultiTopic(b, 8) } + func BenchmarkUnixSocketProtocolV2PubMultiTopic16(b *testing.B) { benchmarkUnixSocketProtocolV2PubMultiTopic(b, 16) } + func BenchmarkUnixSocketProtocolV2PubMultiTopic32(b *testing.B) { benchmarkUnixSocketProtocolV2PubMultiTopic(b, 32) } @@ -1624,9 +1628,11 @@ func BenchmarkUnixSocketProtocolV2Pub64k(b *testing.B) { benchmarkUnixSocketProt func BenchmarkUnixSocketProtocolV2Pub128k(b *testing.B) { benchmarkUnixSocketProtocolV2Pub(b, 128*1024) } + func BenchmarkUnixSocketProtocolV2Pub256k(b *testing.B) { benchmarkUnixSocketProtocolV2Pub(b, 256*1024) } + func BenchmarkUnixSocketProtocolV2Pub512k(b *testing.B) { benchmarkUnixSocketProtocolV2Pub(b, 512*1024) } @@ -1723,9 +1729,11 @@ func BenchmarkUnixSocketProtocolV2Sub64k(b *testing.B) { benchmarkUnixSocketProt func BenchmarkUnixSocketProtocolV2Sub128k(b *testing.B) { benchmarkUnixSocketProtocolV2Sub(b, 128*1024) } + func BenchmarkUnixSocketProtocolV2Sub256k(b *testing.B) { benchmarkUnixSocketProtocolV2Sub(b, 256*1024) } + func BenchmarkUnixSocketProtocolV2Sub512k(b *testing.B) { benchmarkUnixSocketProtocolV2Sub(b, 512*1024) } @@ -1775,15 +1783,19 @@ func benchmarkUnixSocketProtocolV2MultiSub(b *testing.B, num int) { func BenchmarkUnixSocketProtocolV2MultiSub2(b *testing.B) { benchmarkUnixSocketProtocolV2MultiSub(b, 2) } + func BenchmarkUnixSocketProtocolV2MultiSub1(b *testing.B) { benchmarkUnixSocketProtocolV2MultiSub(b, 1) } + func BenchmarkUnixSocketProtocolV2MultiSub4(b *testing.B) { benchmarkUnixSocketProtocolV2MultiSub(b, 4) } + func BenchmarkUnixSocketProtocolV2MultiSub8(b *testing.B) { benchmarkUnixSocketProtocolV2MultiSub(b, 8) } + func BenchmarkUnixSocketProtocolV2MultiSub16(b *testing.B) { benchmarkUnixSocketProtocolV2MultiSub(b, 16) }