From 8743209786edaeea4f684b78c908f8068dc4496b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Segura=20Lucas?= Date: Thu, 11 Jul 2024 13:00:08 +0200 Subject: [PATCH] Adding basic code for implementing a benchmark over Redis storage --- docs/becnhmark_redis_cluster_info_array.md | 64 ++++++++++++++++++++++ services/redis.go | 58 ++++++++++++++++++++ services/redis_test.go | 59 ++++++++++++++++++++ types/types.go | 14 +++-- 4 files changed, 191 insertions(+), 4 deletions(-) create mode 100644 docs/becnhmark_redis_cluster_info_array.md diff --git a/docs/becnhmark_redis_cluster_info_array.md b/docs/becnhmark_redis_cluster_info_array.md new file mode 100644 index 00000000..5e19e7d5 --- /dev/null +++ b/docs/becnhmark_redis_cluster_info_array.md @@ -0,0 +1,64 @@ +# Benchmark for storing ClusterInfo array in Redis with different data types + +## Using `String` type with 30000 clusters in one organization + +Key used: `organization:%d:clusters_info` + +``` +❯ go test -benchmem -run=^$ -bench ^Benchmark github.com/RedHatInsights/insights-results-smart-proxy/services -count 10 +goos: linux +goarch: amd64 +pkg: github.com/RedHatInsights/insights-results-smart-proxy/services +cpu: 11th Gen Intel(R) Core(TM) i7-1185G7 @ 3.00GHz +BenchmarkStoreClustersInfoForOrg-8 126 8816162 ns/op 9768702 B/op 2166 allocs/op +BenchmarkStoreClustersInfoForOrg-8 124 8480939 ns/op 9308795 B/op 2200 allocs/op +BenchmarkStoreClustersInfoForOrg-8 139 8447130 ns/op 9659379 B/op 1965 allocs/op +BenchmarkStoreClustersInfoForOrg-8 115 9224393 ns/op 9083709 B/op 2370 allocs/op +BenchmarkStoreClustersInfoForOrg-8 120 8928589 ns/op 8902301 B/op 2272 allocs/op +BenchmarkStoreClustersInfoForOrg-8 135 8229917 ns/op 9059775 B/op 2022 allocs/op +BenchmarkStoreClustersInfoForOrg-8 116 8957135 ns/op 8612202 B/op 2348 allocs/op +BenchmarkStoreClustersInfoForOrg-8 134 8338725 ns/op 8716508 B/op 2036 allocs/op +BenchmarkStoreClustersInfoForOrg-8 114 10978793 ns/op 9710755 B/op 2393 allocs/op +BenchmarkStoreClustersInfoForOrg-8 93 11711479 ns/op 8851276 B/op 2926 allocs/op +BenchmarkGetClustersInfoForOrg-8 19 63335464 ns/op 23237472 B/op 109004 allocs/op +BenchmarkGetClustersInfoForOrg-8 18 56157604 ns/op 23388576 B/op 110058 allocs/op +BenchmarkGetClustersInfoForOrg-8 16 73779307 ns/op 23747417 B/op 112561 allocs/op +BenchmarkGetClustersInfoForOrg-8 18 63152410 ns/op 23388618 B/op 110058 allocs/op +BenchmarkGetClustersInfoForOrg-8 15 76072282 ns/op 23962585 B/op 114061 allocs/op +BenchmarkGetClustersInfoForOrg-8 19 59169575 ns/op 23237498 B/op 109004 allocs/op +BenchmarkGetClustersInfoForOrg-8 18 57547196 ns/op 23388561 B/op 110058 allocs/op +BenchmarkGetClustersInfoForOrg-8 18 61328365 ns/op 23388518 B/op 110057 allocs/op +BenchmarkGetClustersInfoForOrg-8 21 54260694 ns/op 22978555 B/op 107199 allocs/op +BenchmarkGetClustersInfoForOrg-8 21 58311056 ns/op 22978575 B/op 107199 allocs/op +PASS +ok github.com/RedHatInsights/insights-results-smart-proxy/services 39.266s +``` + +Average writing time: 9.2113262 ms +Average reading time: 62.3113953 ms + +## Using `HSet` type with 30000 clusters in one organization + +Key used: `organization:%d:cluster:%s:info` +Stored struct: simplification of `ClusterInfo`, removing the `ClusterName` as used in the key + +``` +❯ go test -benchmem -run=^$ -bench ^BenchmarkStore github.com/RedHatInsights/insights-results-smart-proxy/services -count 10 +goos: linux +goarch: amd64 +pkg: github.com/RedHatInsights/insights-results-smart-proxy/services +cpu: 11th Gen Intel(R) Core(TM) i7-1185G7 @ 3.00GHz +BenchmarkStoreClustersInfoForOrg-8 9 130180591 ns/op 33897600 B/op 630078 allocs/op +BenchmarkStoreClustersInfoForOrg-8 7 216593692 ns/op 35141674 B/op 655802 allocs/op +BenchmarkStoreClustersInfoForOrg-8 5 298012948 ns/op 37380931 B/op 702104 allocs/op +BenchmarkStoreClustersInfoForOrg-8 5 277542327 ns/op 37380110 B/op 702097 allocs/op +BenchmarkStoreClustersInfoForOrg-8 4 255210232 ns/op 39339910 B/op 742614 allocs/op +BenchmarkStoreClustersInfoForOrg-8 6 176066511 ns/op 36074120 B/op 675087 allocs/op +BenchmarkStoreClustersInfoForOrg-8 6 195984145 ns/op 36074932 B/op 675098 allocs/op +BenchmarkStoreClustersInfoForOrg-8 5 231129258 ns/op 37381185 B/op 702103 allocs/op +BenchmarkStoreClustersInfoForOrg-8 4 252233715 ns/op 39339188 B/op 742608 allocs/op +BenchmarkStoreClustersInfoForOrg-8 4 279100256 ns/op 39339580 B/op 742614 allocs/op +PASS +ok github.com/RedHatInsights/insights-results-smart-proxy/services 38.118s + +Average writing time: 231.2053675 ms diff --git a/services/redis.go b/services/redis.go index e0565308..82a36f41 100644 --- a/services/redis.go +++ b/services/redis.go @@ -18,9 +18,11 @@ package services import ( "context" + "encoding/json" "fmt" "regexp" "strings" + "time" "github.com/RedHatInsights/insights-operator-utils/redis" utypes "github.com/RedHatInsights/insights-operator-utils/types" @@ -73,6 +75,13 @@ type RedisInterface interface { types.ClusterName, types.RequestID, ) ([]types.RuleID, error) + StoreClustersInfo( + types.OrgID, + []types.ClusterInfo, + ) error + GetClustersInfoForOrgID( + types.OrgID, + ) ([]types.ClusterInfo, error) } // RedisClient is a local type which embeds the imported redis.Client to include its own functionality @@ -249,3 +258,52 @@ func (redisClient *RedisClient) GetRuleHitsForRequest( return } + +// StoreClustersInfo is used to store a set of ClusterInfo elements into the Redis storage +func (redisClient *RedisClient) StoreClustersInfo( + orgID types.OrgID, + clustersInfo []types.ClusterInfo, +) error { + ctx := context.Background() + key := fmt.Sprintf("organization:%d:clusters_info", orgID) + + marshaledArray, err := json.Marshal(clustersInfo) + if err != nil { + return err + } + + cmd := redisClient.Client.Connection.Set(ctx, key, marshaledArray, 300*time.Second) + + if err := cmd.Err(); err != nil { + return err + } + + return nil +} + +// GetClustersInfoForOrgID is ised to get the clusters info for all the clusters in an organization +func (redisClient *RedisClient) GetClustersInfoForOrgID( + orgID types.OrgID, +) (clustersInfo []types.ClusterInfo, err error) { + ctx := context.Background() + key := fmt.Sprintf("organization:%d:clusters_info", orgID) + + cmd := redisClient.Client.Connection.Get(ctx, key) + if err = cmd.Err(); err != nil { + log.Error().Err(err).Msg(redisCmdExecutionFailedMsg) + return + } + + data, err := cmd.Result() + if err != nil { + log.Error().Err(err).Msg("Error getting result") + return + } + err = json.Unmarshal([]byte(data), &clustersInfo) + if err != nil { + log.Error().Err(err).Msg("failed to unmarshal result into an array") + return + } + + return +} diff --git a/services/redis_test.go b/services/redis_test.go index b5b75b82..4d37cfd2 100644 --- a/services/redis_test.go +++ b/services/redis_test.go @@ -19,6 +19,7 @@ package services_test import ( "errors" "fmt" + "math/rand" "testing" "time" @@ -28,6 +29,7 @@ import ( "github.com/RedHatInsights/insights-results-smart-proxy/tests/helpers" "github.com/RedHatInsights/insights-results-smart-proxy/tests/testdata" "github.com/RedHatInsights/insights-results-smart-proxy/types" + "github.com/hashicorp/go-uuid" "github.com/stretchr/testify/assert" ) @@ -450,3 +452,60 @@ func TestGetRuleHitsForRequest_OKScanError(t *testing.T) { helpers.RedisExpectationsMet(t, server) } + +const clustersInfoSize = 30000 + +func BenchmarkStoreClustersInfoForOrg(b *testing.B) { + params := prepareClustersInfoData(clustersInfoSize) + conf := services.RedisConfiguration{ + RedisEndpoint: "localhost:6379", + RedisDatabase: 0, + RedisTimeoutSeconds: 30, + } + client, err := services.NewRedisClient(conf) + if err != nil { + fmt.Printf("Error: %v", err) + b.Fail() + } + + for i := 0; i <= b.N; i++ { + client.StoreClustersInfo(types.OrgID(1), params) + } +} + +func BenchmarkGetClustersInfoForOrg(b *testing.B) { + conf := services.RedisConfiguration{ + RedisEndpoint: "localhost:6379", + RedisDatabase: 0, + RedisTimeoutSeconds: 30, + } + client, err := services.NewRedisClient(conf) + if err != nil { + fmt.Printf("Error: %v", err) + b.Fail() + } + params := prepareClustersInfoData(clustersInfoSize) + client.StoreClustersInfo(types.OrgID(1), params) + + for i := 0; i <= b.N; i++ { + client.GetClustersInfoForOrgID(types.OrgID(1)) + } +} + +func prepareClustersInfoData(n int) []types.ClusterInfo { + var clustersInfo []types.ClusterInfo = make([]types.ClusterInfo, n) + var src = rand.NewSource(time.Now().UnixNano()) + var r = rand.New(src) + + for index := 0; index < n; index++ { + uuid, _ := uuid.GenerateUUID() + clustersInfo[index] = types.ClusterInfo{ + ID: types.ClusterName(uuid), + DisplayName: fmt.Sprintf("The cluster name is %s", uuid), + Managed: r.Intn(2) != 0, + Status: "ok", + } + } + + return clustersInfo +} diff --git a/types/types.go b/types/types.go index a1dfe94e..5d31bc31 100644 --- a/types/types.go +++ b/types/types.go @@ -252,10 +252,16 @@ type InfoResponse struct { // ClusterInfo is a data structure containing some relevant cluster information type ClusterInfo struct { - ID ClusterName `json:"cluster_id"` - DisplayName string `json:"display_name"` - Managed bool `json:"managed"` - Status string `json:"status"` + ID ClusterName `json:"cluster_id" redis:"cluster_id"` + DisplayName string `json:"display_name" redis:"display_name"` + Managed bool `json:"managed" redis:"managed"` + Status string `json:"status" redis:"status"` +} + +type ReducedClusterInfo struct { + DisplayName string `redis:"display_name"` + Managed bool `redis:"managed"` + Status string `redis:"status"` } // ClustersDetailData is the inner data structure for /clusters_detail