-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix: add compress mechanism + fix the trace length + support New Reli…
…c APM (#17) * support New Relic APM * add compress mechanism for Marshal() and Unmarshal() function * fix the trace length is ~10x times of the cache size Co-authored-by: viney-shih <[email protected]>
- Loading branch information
1 parent
8d7a414
commit 49fc89b
Showing
9 changed files
with
241 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,6 @@ | ||
| language: go | ||
|
|
||
| go: | ||
| - "1.16" | ||
| - "1.17" | ||
| - "1.18" | ||
| - "1.19" | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| package cache | ||
|
|
||
| import ( | ||
| "fmt" | ||
|
|
||
| "github.com/klauspost/compress/s2" | ||
| "github.com/vmihailenco/msgpack/v5" | ||
| ) | ||
|
|
||
| // ref: https://github.com/go-redis/cache/blob/v8/cache.go | ||
|
|
||
| const ( | ||
| compressionThreshold = 64 | ||
| timeLen = 4 | ||
| ) | ||
|
|
||
| const ( | ||
| noCompression = 0x0 | ||
| s2Compression = 0x1 | ||
| ) | ||
|
|
||
| // Marshal marshals value by msgpack + compress | ||
| func Marshal(value interface{}) ([]byte, error) { | ||
| switch value := value.(type) { | ||
| case nil: | ||
| return nil, nil | ||
| case []byte: | ||
| return value, nil | ||
| case string: | ||
| return []byte(value), nil | ||
| } | ||
|
|
||
| b, err := msgpack.Marshal(value) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return compress(b), nil | ||
| } | ||
|
|
||
| func compress(data []byte) []byte { | ||
| if len(data) < compressionThreshold { | ||
| n := len(data) + 1 | ||
| b := make([]byte, n, n+timeLen) | ||
| copy(b, data) | ||
| b[len(b)-1] = noCompression | ||
| return b | ||
| } | ||
|
|
||
| n := s2.MaxEncodedLen(len(data)) + 1 | ||
| b := make([]byte, n, n+timeLen) | ||
| b = s2.Encode(b, data) | ||
| b = append(b, s2Compression) | ||
| return b | ||
| } | ||
|
|
||
| // Unmarshal unmarshals binary with the compress + msgpack | ||
| func Unmarshal(b []byte, value interface{}) error { | ||
| if len(b) == 0 { | ||
| return nil | ||
| } | ||
|
|
||
| switch value := value.(type) { | ||
| case nil: | ||
| return nil | ||
| case *[]byte: | ||
| clone := make([]byte, len(b)) | ||
| copy(clone, b) | ||
| *value = clone | ||
| return nil | ||
| case *string: | ||
| *value = string(b) | ||
| return nil | ||
| } | ||
|
|
||
| switch c := b[len(b)-1]; c { | ||
| case noCompression: | ||
| b = b[:len(b)-1] | ||
| case s2Compression: | ||
| b = b[:len(b)-1] | ||
|
|
||
| var err error | ||
| b, err = s2.Decode(nil, b) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| default: | ||
| return fmt.Errorf("unknown compression method: %x", c) | ||
| } | ||
|
|
||
| return msgpack.Unmarshal(b, value) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,121 @@ | ||
| package cache | ||
|
|
||
| import ( | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/stretchr/testify/suite" | ||
| ) | ||
|
|
||
| var ( | ||
| mockTimeNow = time.Date(2022, 11, 23, 0, 0, 0, 0, time.Local) | ||
| ) | ||
|
|
||
| type marshalerSuite struct { | ||
| suite.Suite | ||
| } | ||
|
|
||
| func (s *marshalerSuite) SetupSuite() {} | ||
|
|
||
| func (s *marshalerSuite) TearDownSuite() {} | ||
|
|
||
| func (s *marshalerSuite) SetupTest() {} | ||
|
|
||
| func (s *marshalerSuite) TearDownTest() {} | ||
|
|
||
| func TestMarshalerSuite(t *testing.T) { | ||
| suite.Run(t, new(marshalerSuite)) | ||
| } | ||
|
|
||
| type mockStruct struct { | ||
| ID int64 | ||
| Key string | ||
| CreatedAt time.Time | ||
| child *mockStruct | ||
| } | ||
|
|
||
| func (s *marshalerSuite) TestMarshaler() { | ||
| var bs []byte | ||
| var err error | ||
| marshal := Marshal | ||
| unmarshal := Unmarshal | ||
|
|
||
| // nil | ||
| var null error | ||
| bs, err = marshal(null) | ||
| s.Require().NoError(err) | ||
|
|
||
| var retNull error | ||
| s.Require().NoError(unmarshal(bs, &retNull)) | ||
| s.Require().Equal(null, retNull) | ||
|
|
||
| // bytes | ||
| bytes := []byte("strings to bytes") | ||
| bs, err = marshal(bytes) | ||
| s.Require().NoError(err) | ||
|
|
||
| var retBytes []byte | ||
| s.Require().NoError(unmarshal(bs, &retBytes)) | ||
| s.Require().Equal(bytes, retBytes) | ||
|
|
||
| // string | ||
| str := "this is a string" | ||
| bs, err = marshal(str) | ||
| s.Require().NoError(err) | ||
|
|
||
| var retStr string | ||
| s.Require().NoError(unmarshal(bs, &retStr)) | ||
| s.Require().Equal(str, retStr) | ||
|
|
||
| // pointer | ||
| num := 100 | ||
| intPtr := &num | ||
| bs, err = marshal(intPtr) | ||
| s.Require().NoError(err) | ||
|
|
||
| var retIntPtr *int | ||
| s.Require().NoError(unmarshal(bs, &retIntPtr)) | ||
| s.Require().Equal(intPtr, retIntPtr) | ||
|
|
||
| // struct | ||
| st := mockStruct{ | ||
| ID: 28825252, | ||
| Key: "I am rich", | ||
| CreatedAt: mockTimeNow, | ||
| } | ||
| bs, err = marshal(st) | ||
| s.Require().NoError(err) | ||
|
|
||
| retSt := mockStruct{} | ||
| s.Require().NoError(unmarshal(bs, &retSt)) | ||
| s.Require().Equal(st, retSt) | ||
|
|
||
| // struct without nil pointer | ||
| st2 := mockStruct{ | ||
| ID: 28825252, | ||
| Key: "I am rich", | ||
| CreatedAt: mockTimeNow, | ||
| child: &mockStruct{ | ||
| ID: 2266, | ||
| }, | ||
| } | ||
| bs, err = marshal(st2) | ||
| s.Require().NoError(err) | ||
|
|
||
| var retSt2 mockStruct | ||
| s.Require().NoError(unmarshal(bs, &retSt2)) | ||
| s.Require().Equal(st, retSt2) | ||
|
|
||
| // compress | ||
| st3 := mockStruct{ | ||
| ID: 1234567890, | ||
| Key: `1234567890123456789012345678901234567890123456789012345678901234567890`, // 70 chars | ||
| CreatedAt: mockTimeNow, | ||
| } | ||
| bs, err = marshal(st3) | ||
| s.Require().NoError(err) | ||
|
|
||
| var retSt3 mockStruct | ||
| s.Require().NoError(unmarshal(bs, &retSt3)) | ||
| s.Require().Equal(st3, retSt3) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters