Merge pull request #62 from cxl123156/br_release_sdk_v3

commit 24.3 code

README.md

@@ -74,6 +74,16 @@ auth:
 
 ```
 
+## Example of endpoint file(endpoints.yml)
+European site users should configure the domain names of the rms and eps services as follows.
+```
+"rms":
+  "https://rms.myhuaweicloud.eu"
+"eps":
+  "https://eps.eu-west-101.myhuaweicloud.eu"
+```
+Other site users should not configure anything in this file.
+
 ## Prometheus Configuration
 The huaweicloud exporter needs to be passed the address as a parameter, this can be done with relabelling.
 

README_cn.md

@@ -92,7 +92,18 @@ auth:
 ```
 注：默认的监控端口为8087.
 
-3. 启动cloudeye-exporter，默认读取当前目录下的clouds.yml文件，也可使用-config参数指定clouds.yml文件路径
+3. 欧洲站注意事项
+
+   欧洲站用户需要为rms、eps服务重新指定域名，如下所示
+```yaml
+"rms":
+  "https://rms.myhuaweicloud.eu"
+"eps":
+  "https://eps.eu-west-101.myhuaweicloud.eu"
+```
+注：其他站点用户无需在此文件配置域名信息
+
+4. 启动cloudeye-exporter，默认读取当前目录下的clouds.yml文件，也可使用-config参数指定clouds.yml文件路径
 ```
 ./cloudeye-exporter -config=clouds.yml
 ```

collector/cc.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"time"
 
+	http_client "github.com/huaweicloud/huaweicloud-sdk-go-v3/core"
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/global"
 	cc "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/cc/v3"
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/cc/v3/model"
@@ -157,8 +158,17 @@ func listBandwidthPackages() (map[string]model.BandwidthPackage, error) {
 }
 
 func getCCClient() *cc.CcClient {
-	return cc.NewCcClient(cc.CcClientBuilder().WithRegion(region.ValueOf("cn-north-4")).
-		WithCredential(global.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).Build()).Build())
+	return cc.NewCcClient(getCCClientBuilder().Build())
+}
+
+func getCCClientBuilder() *http_client.HcHttpClientBuilder {
+	builder := cc.CcClientBuilder().WithCredential(global.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).Build())
+	if endpoint, ok := endpointConfig["cc"]; ok {
+		builder.WithEndpoint(endpoint)
+	} else {
+		builder.WithRegion(region.ValueOf("cn-north-4"))
+	}
+	return builder
 }
 
 func listInterRegionBandwidths() ([]model.InterRegionBandwidth, error) {

collector/cc_test.go

@@ -68,3 +68,11 @@ func TestCCGetResourceInfo(t *testing.T) {
 	assert.Equal(t, 1, len(labels))
 	assert.Equal(t, 1, len(metrics))
 }
+
+func TestGetCCClient(t *testing.T) {
+	endpointConfig = map[string]string{
+		"cc": "https://cc.myhuaweicloud.com",
+	}
+	ccClient := getCCClient()
+	assert.NotNil(t, ccClient)
+}

collector/cfw.go

@@ -38,7 +38,7 @@ func (cfw CFWInfo) GetResourceInfo() (map[string]labelInfo, []cesmodel.MetricInf
 		servers, err := getResourcesBaseInfoFromRMS("cfw", "cfw_instance")
 		if err != nil {
 			logs.Logger.Errorf("Get resource base info from RMS Server error:", err.Error())
-			return ecsInfo.LabelInfo, ecsInfo.FilterMetrics
+			return cfwServerInfo.LabelInfo, cfwServerInfo.FilterMetrics
 		}
 
 		for _, server := range servers {

collector/collector_test.go

@@ -1,13 +1,12 @@
 package collector
 
 import (
-	"sync"
-	"testing"
-	"time"
-
 	"github.com/agiledragon/gomonkey/v2"
 	"github.com/huaweicloud/cloudeye-exporter/logs"
 	"go.uber.org/zap/zapcore"
+	"sync"
+	"testing"
+	"time"
 
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ces/v1/model"
 	"github.com/stretchr/testify/assert"
@@ -68,3 +67,87 @@ func TestGetLatestData(t *testing.T) {
 	assert.Equal(t, true, err == nil)
 	assert.Equal(t, avg2, value)
 }
+
+func TestIsMetricLabelConflict(t *testing.T) {
+	var label = labelInfo{
+		Name:  []string{"disk", "instance_id", "unit"},
+		Value: []string{"0eb113c0-0111-4f51-9d44-f2157fcc138c", "2e84018fc8b4484b94e89aae212fe615", "%"},
+	}
+	proMap := PrometheusMetricMap{
+		RWMutex:   sync.RWMutex{},
+		MetricMap: make(map[string]bool),
+	}
+	proMap.MetricMap["huaweicloud_agt_ecs_cpu_usage{instance_id=9234ad9f-87a5-49a9-b6ec-11ecde1d943b,mount_point=2e84018fc8b4484b94e89aae212fe615,unit=%}"] = true
+	result1 := isMetricLabelConflict("huaweicloud_agt_ecs_cpu_usage", label, &proMap)
+	assert.False(t, result1)
+
+	proMap.MetricMap["huaweicloud_agt_ecs_cpu_usage{instance_id=9234ad9f-87a5-49a9-b6ec-11ecde1d943b,disk=2e84018fc8b4484b94e89aae212fe615,unit=%}"] = true
+	result2 := isMetricLabelConflict("huaweicloud_agt_ecs_cpu_usage", label, &proMap)
+	assert.True(t, result2)
+}
+
+func TestIsAgentMetric(t *testing.T) {
+	resultECS := isAgentMetric("AGT.ECS")
+	assert.True(t, resultECS)
+
+	resultBMS := isAgentMetric("SERVICE.BMS")
+	assert.True(t, resultBMS)
+}
+
+func TestSetProData1(t *testing.T) {
+	proMap := PrometheusMetricMap{
+		RWMutex:   sync.RWMutex{},
+		MetricMap: make(map[string]bool),
+	}
+	proMap.MetricMap["agt_ecs_cpu_usage{instance_id=9234ad9f-87a5-49a9-b6ec-11ecde1d943b,mount_point=2e84018fc8b4484b94e89aae212fe615,unit=%}"] = true
+
+	unit := "%"
+	namespace := "AGT.ECS"
+	avg := 0.33
+	data := model.BatchMetricData{
+		Unit: &unit,
+		Datapoints: []model.DatapointForBatchMetric{
+			{
+				Average: &avg,
+			},
+		},
+		Namespace:  &namespace,
+		MetricName: "cpu_usage",
+		Dimensions: &[]model.MetricsDimension{
+			{
+				Name:  "instance_id",
+				Value: "9234ad9f-87a5-49a9-b6ec-11ecde1d943b",
+			},
+			{
+				Name:  "mount_point",
+				Value: "2e84018fc8b4484b94e89aae212fe615",
+			},
+		},
+	}
+	metricDataArray := []model.BatchMetricData{data}
+	var exporter BaseHuaweiCloudExporter
+	resourceInfo := make(map[string]labelInfo, 0)
+	var label = labelInfo{
+		Name:  []string{"mount_point", "instance_id", "unit"},
+		Value: []string{"2e84018fc8b4484b94e89aae212fe615", "0eb113c0-0111-4f51-9d44-f2157fcc138c", "%"},
+	}
+	resourceInfo["instance_id"] = label
+	confLoader := logs.ConfLoader{}
+	patches := gomonkey.ApplyMethodFunc(confLoader, "LoadFile", func(fPath string, cfg interface{}) error {
+		cfgTmp, ok := cfg.(*map[string][]logs.Config)
+		assert.True(t, ok)
+		cfgPointer := make(map[string][]logs.Config)
+		cfgPointer["business"] = []logs.Config{
+			{
+				Level: zapcore.InfoLevel,
+			},
+		}
+		*cfgTmp = cfgPointer
+		return nil
+	})
+	defer patches.Reset()
+	logs.InitLog()
+	exporter.Collect(nil)
+	exporter.setProData(nil, nil, metricDataArray, resourceInfo, &proMap)
+	assert.Equal(t, 3, len(label.Name))
+}

collector/config.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/global"
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/config"
-	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/iam/v3"
+	v3 "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/iam/v3"
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/iam/v3/model"
 	"gopkg.in/yaml.v3"
 
@@ -63,6 +63,7 @@ func InitCloudConf(file string) error {
 		return err
 	}
 
+	initEndpointConfig()
 	return err
 }
 
@@ -200,3 +201,17 @@ func getProjectInfo() (*model.KeystoneListProjectsResponse, error) {
 			Build())
 	return iamclient.KeystoneListProjects(&model.KeystoneListProjectsRequest{Name: &conf.ProjectName})
 }
+
+var endpointConfig map[string]string
+
+func initEndpointConfig() {
+	context, err := ioutil.ReadFile("endpoints.yml")
+	if err != nil {
+		logs.Logger.Errorf("Init endpoint config error: %s", err.Error())
+		return
+	}
+	err = yaml.Unmarshal(context, &endpointConfig)
+	if err != nil {
+		logs.Logger.Errorf("Init endpoint config error: %s", err.Error())
+	}
+}

collector/config_test.go

@@ -1,9 +1,15 @@
 package collector
 
 import (
+	"errors"
+	"os"
 	"testing"
 
+	"github.com/agiledragon/gomonkey/v2"
 	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v3"
+
+	"github.com/huaweicloud/cloudeye-exporter/logs"
 )
 
 func TestName(t *testing.T) {
@@ -53,3 +59,56 @@ func TestInitConfigSecurityModIsFalse(t *testing.T) {
 	assert.Equal(t, CloudConf.Auth.AccessKey, conf.AccessKey)
 	assert.Equal(t, CloudConf.Auth.SecretKey, conf.SecretKey)
 }
+
+func TestInitEndpointConfig(t *testing.T) {
+	testCases := []struct {
+		name    string
+		patches func() *gomonkey.Patches
+		expect  func(t *testing.T)
+	}{
+		{
+			"ReadConfigFileError",
+			func() *gomonkey.Patches {
+				patches := gomonkey.NewPatches()
+				patches.ApplyFuncReturn(os.ReadFile, nil, errors.New("read file error"))
+				return patches
+			},
+			func(t *testing.T) {
+				assert.NotNil(t, &endpointConfig)
+			},
+		},
+		{
+			"UnMarshalConfigError",
+			func() *gomonkey.Patches {
+				patches := gomonkey.NewPatches()
+				patches.ApplyFuncReturn(os.ReadFile, []byte("\"rms\":\r\n  \"https://rms.xxx.xxx.com\"\r\n\"eps\":\r\n  \"https://eps.xxx.xxx.com\""), nil)
+				patches.ApplyFuncReturn(yaml.Unmarshal, errors.New("unmarshal yaml error"))
+				return patches
+			},
+			func(t *testing.T) {
+				assert.NotNil(t, &endpointConfig)
+			},
+		},
+		{
+			"Success",
+			func() *gomonkey.Patches {
+				patches := gomonkey.NewPatches()
+				patches.ApplyFuncReturn(os.ReadFile, []byte("\"rms\":\r\n  \"https://rms.xxx.xxx.com\"\r\n\"eps\":\r\n  \"https://eps.xxx.xxx.com\""), nil)
+				return patches
+			},
+			func(t *testing.T) {
+				assert.NotNil(t, &endpointConfig)
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			patches := testCase.patches()
+			patches.ApplyMethod(&logs.Logger, "Errorf", func(logger *logs.LoggerConstructor, template string, args ...interface{}) {})
+			defer patches.Reset()
+			initEndpointConfig()
+			testCase.expect(t)
+		})
+	}
+}

collector/dms.go

@@ -77,7 +77,7 @@ func getDMSInstanceFromRMS() []ResourceBaseInfo {
 		instances = append(instances, rabbitResp...)
 	}
 
-	rocketMqs, err := getRocketMQInstances()
+	rocketMqs, err := getResourcesBaseInfoFromRMS("dms", "reliabilitys")
 	if err != nil {
 		logs.Logger.Errorf("Get all dms rocketmqs : %s", err.Error())
 	} else {

collector/dms_test.go

@@ -0,0 +1,21 @@
+package collector
+
+import (
+	"testing"
+
+	"github.com/agiledragon/gomonkey/v2"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDmsGetResourceInfo(t *testing.T) {
+	sysConfig := map[string][]string{"instance_id": {"cpu_util"}}
+	patches := getPatches()
+	patches = gomonkey.ApplyFuncReturn(getMetricConfigMap, sysConfig)
+	patches.ApplyFuncReturn(listResources, mockRmsResource(), nil)
+	defer patches.Reset()
+
+	var dmsInfo DMSInfo
+	labels, metrics := dmsInfo.GetResourceInfo()
+	assert.Equal(t, 1, len(labels))
+	assert.Equal(t, 0, len(metrics))
+}

collector/eps.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"time"
 
+	http_client "github.com/huaweicloud/huaweicloud-sdk-go-v3/core"
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/global"
 	eps "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/eps/v1"
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/eps/v1/model"
@@ -24,8 +25,17 @@ const HelpInfo = `# HELP huaweicloud_epinfo huaweicloud_epinfo
 `
 
 func getEPSClient() *eps.EpsClient {
-	return eps.NewEpsClient(eps.EpsClientBuilder().WithRegion(region.ValueOf("cn-north-4")).
-		WithCredential(global.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).Build()).Build())
+	return eps.NewEpsClient(getEPSClientBuilder().Build())
+}
+
+func getEPSClientBuilder() *http_client.HcHttpClientBuilder {
+	builder := eps.EpsClientBuilder().WithCredential(global.NewCredentialsBuilder().WithAk(conf.AccessKey).WithSk(conf.SecretKey).WithDomainId(conf.DomainID).Build())
+	if endpoint, ok := endpointConfig["eps"]; ok {
+		builder.WithEndpoint(endpoint)
+	} else {
+		builder.WithRegion(region.ValueOf("cn-north-4"))
+	}
+	return builder
 }
 
 func GetEPSInfo() (string, error) {

collector/eps_test.go

@@ -0,0 +1,15 @@
+package collector
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetEPSClient(t *testing.T) {
+	endpointConfig = map[string]string{
+		"eps": "https://eps.myhuaweicloud.com",
+	}
+	epsClient := getEPSClient()
+	assert.NotNil(t, epsClient)
+}