koordlet: support change pod CPUQOS by annotations (#2206)

Signed-off-by: j4ckstraw <[email protected]>
koordinator-sh · Dec 3, 2024 · b36d230 · b36d230
1 parent 46c7604
commit b36d230
Show file tree

Hide file tree

Showing 3 changed files with 94 additions and 0 deletions.
diff --git a/apis/slo/v1alpha1/pod.go b/apis/slo/v1alpha1/pod.go
@@ -18,6 +18,7 @@ package v1alpha1
 
 import (
 	"encoding/json"
+	"fmt"
 
 	corev1 "k8s.io/api/core/v1"
 
@@ -27,6 +28,8 @@ import (
 const (
 	AnnotationPodCPUBurst = apiext.DomainPrefix + "cpuBurst"
 
+	AnnotationPodCPUQoS = apiext.DomainPrefix + "cpuQOS"
+
 	AnnotationPodMemoryQoS = apiext.DomainPrefix + "memoryQOS"
 
 	AnnotationPodBlkioQoS = apiext.DomainPrefix + "blkioQOS"
@@ -65,6 +68,28 @@ func GetPodMemoryQoSConfig(pod *corev1.Pod) (*PodMemoryQOSConfig, error) {
 	return &cfg, nil
 }
 
+func GetPodCPUQoSConfigByAttr(labels, annotations map[string]string) (*CPUQOSCfg, error) {
+	value, exist := annotations[AnnotationPodCPUQoS]
+	if !exist {
+		return nil, nil
+	}
+	cfg := CPUQOSCfg{}
+	err := json.Unmarshal([]byte(value), &cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	// check before return
+	if cfg.GroupIdentity != nil {
+		bvt := *cfg.GroupIdentity
+		// bvt value allowed [-1, 2], see https://help.aliyun.com/zh/alinux/user-guide/group-identity-feature
+		if bvt < -1 || bvt > 2 {
+			return nil, fmt.Errorf("bad group identity value: %v", bvt)
+		}
+	}
+	return &cfg, nil
+}
+
 const (
 	// LabelCoreSchedGroupID is the label key of the group ID of the Linux Core Scheduling.
 	// Value can be a valid UUID or empty. If it is empty, the pod is considered to belong to a core sched group "".

diff --git a/pkg/koordlet/runtimehooks/hooks/groupidentity/interceptor.go b/pkg/koordlet/runtimehooks/hooks/groupidentity/interceptor.go
@@ -21,6 +21,7 @@ import (
 	"k8s.io/utils/pointer"
 
 	ext "github.com/koordinator-sh/koordinator/apis/extension"
+	slov1alpha1 "github.com/koordinator-sh/koordinator/apis/slo/v1alpha1"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/protocol"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/util"
 )
@@ -30,11 +31,33 @@ func (b *bvtPlugin) SetPodBvtValue(p protocol.HooksProtocol) error {
 	if r == nil {
 		return nil
 	}
+
 	podCtx := p.(*protocol.PodContext)
 	req := podCtx.Request
 	podQOS := ext.GetQoSClassByAttrs(req.Labels, req.Annotations)
 	podKubeQOS := util.GetKubeQoSByCgroupParent(req.CgroupParent)
 	podBvt := r.getPodBvtValue(podQOS, podKubeQOS)
+
+	// pod annotatations take precedence to the default CPUQoS which retrieve from getPodBvtValue
+	cfg, err := slov1alpha1.GetPodCPUQoSConfigByAttr(req.Labels, req.Annotations)
+	if err != nil {
+		return err
+	}
+	// we can change group identity by pod annotations only when QoS=LS
+	// because LS=2 and BE=-1 by default
+	// we only allow LS change to BE
+	// and not allow BE change to LS
+	// and do not change LSR/LSE
+	if cfg != nil && podQOS == ext.QoSLS {
+		if cfg.GroupIdentity != nil {
+			podBvt = *cfg.GroupIdentity
+		}
+
+		// if disabled, set bvt to zero
+		if cfg.Enable == nil || (cfg.Enable != nil && !(*cfg.Enable)) {
+			podBvt = 0
+		}
+	}
 	podCtx.Response.Resources.CPUBvt = pointer.Int64(podBvt)
 	return nil
 }

diff --git a/pkg/koordlet/runtimehooks/hooks/groupidentity/interceptor_test.go b/pkg/koordlet/runtimehooks/hooks/groupidentity/interceptor_test.go
@@ -25,6 +25,7 @@ import (
 
 	ext "github.com/koordinator-sh/koordinator/apis/extension"
 	runtimeapi "github.com/koordinator-sh/koordinator/apis/runtime/v1alpha1"
+	slov1alpha1 "github.com/koordinator-sh/koordinator/apis/slo/v1alpha1"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/resourceexecutor"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/protocol"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/util"
@@ -125,6 +126,51 @@ func Test_bvtPlugin_SetPodBvtValue_Proxy(t *testing.T) {
 				bvtValue: pointer.Int64(2),
 			},
 		},
+		{
+			name: "set ls pod bvt with annoation override succeed",
+			fields: fields{
+				rule:            defaultRule,
+				systemSupported: pointer.Bool(true),
+			},
+			args: args{
+				request: &runtimeapi.PodSandboxHookRequest{
+					Labels: map[string]string{
+						ext.LabelPodQoS: string(ext.QoSLS),
+					},
+					Annotations: map[string]string{
+						slov1alpha1.AnnotationPodCPUQoS: `{"enable":true,"groupIdentity":-1}`,
+					},
+					CgroupParent: "kubepods/pod-guaranteed-test-uid/",
+				},
+				response: &runtimeapi.PodSandboxHookResponse{},
+			},
+			want: want{
+				bvtValue: pointer.Int64(-1),
+			},
+		},
+		{
+			name: "set ls pod bvt with annoation override failed",
+			fields: fields{
+				rule:            defaultRule,
+				systemSupported: pointer.Bool(true),
+			},
+			args: args{
+				request: &runtimeapi.PodSandboxHookRequest{
+					Labels: map[string]string{
+						ext.LabelPodQoS: string(ext.QoSLS),
+					},
+					Annotations: map[string]string{
+						slov1alpha1.AnnotationPodCPUQoS: `{"enable":false}`,
+					},
+					CgroupParent: "kubepods/pod-guaranteed-test-uid/",
+				},
+				response: &runtimeapi.PodSandboxHookResponse{},
+			},
+			want: want{
+				// default value
+				bvtValue: pointer.Int64(0),
+			},
+		},
 		{
 			name: "set be pod bvt",
 			fields: fields{