Skip to content

Commit

Permalink
Refactor logic about ignore-selector--keys
Browse files Browse the repository at this point in the history
  • Loading branch information
@cwdsuzhou
cwdsuzhou committed Jul 8, 2020
1 parent e850173 commit f2ad1b8
Show file tree
Hide file tree
Showing 5 changed files with 141 additions and 43 deletions.
14 changes: 11 additions & 3 deletions pkg/scheduler/multischeduler/scheduler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"encoding/json"
"fmt"

"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/clientcmd"
Expand Down Expand Up @@ -103,12 +104,19 @@ func (m MultiSchedulingPlugin) Filter(pc *framework.PluginContext, pod *v1.Pod,
}

podCopy := pod.DeepCopy()
if podCopy.Spec.NodeSelector != nil {

cns := util.ConvertAnnotations(podCopy.Annotations)
// remove selector
if cns != nil {
podCopy.Spec.NodeSelector = cns.NodeSelector
podCopy.Spec.Affinity = cns.Affinity
podCopy.Spec.Tolerations = cns.Tolerations
} else {
podCopy.Spec.NodeSelector = nil
}
if podCopy.Spec.Affinity != nil {
podCopy.Spec.Affinity = nil
podCopy.Spec.Tolerations = nil
}

result, err := scheduler.Algorithm.Schedule(podCopy, pc)
klog.V(5).Infof("%v Nodes, Node %s can be scheduled to run pod", result.FeasibleNodes, result.SuggestedHost)
if err != nil {
Expand Down
14 changes: 14 additions & 0 deletions pkg/testbase/test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,13 @@ func PodForTestWithNodeSelectorAndAffinityClusterID() *v1.Pod {
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.
NodeSelectorTerm{{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "test0",
Operator: v1.NodeSelectorOpIn,
Values: []string{
"aa",
},
},
{
Key: "clusterID",
Operator: v1.NodeSelectorOpIn,
Expand All @@ -204,6 +211,13 @@ func PodForTestWithNodeSelectorAndAffinityClusterID() *v1.Pod {
"aa",
},
},
{
Key: "test1",
Operator: v1.NodeSelectorOpIn,
Values: []string{
"aa",
},
},
},
}}},
PreferredDuringSchedulingIgnoredDuringExecution: nil,
Expand Down
10 changes: 6 additions & 4 deletions pkg/util/conversions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func TrimPod(pod *corev1.Pod, ignoreLabels []string) *corev1.Pod {
podCopy.Annotations = make(map[string]string)
}
podCopy.Labels[VirtualPodLabel] = "true"
cns := convertAnnotations(pod.Annotations)
cns := ConvertAnnotations(pod.Annotations)
// remove selector
if cns != nil {
podCopy.Spec.NodeSelector = cns.NodeSelector
Expand Down Expand Up @@ -80,7 +80,7 @@ func trimContainers(containers []corev1.Container) []corev1.Container {
var newContainers []corev1.Container

for _, c := range containers {
volMounts := []corev1.VolumeMount{}
var volMounts []corev1.VolumeMount
for _, v := range c.VolumeMounts {
if strings.HasPrefix(v.Name, "default-token") {
continue
Expand Down Expand Up @@ -108,7 +108,7 @@ func GetUpdatedPod(orig, update *corev1.Pod, ignoreLabels []string) {
update.Annotations = make(map[string]string)
}
if orig.Annotations[SelectorKey] != update.Annotations[SelectorKey] {
if cns := convertAnnotations(update.Annotations); cns != nil {
if cns := ConvertAnnotations(update.Annotations); cns != nil {
// we assume tolerations would only add not remove
orig.Spec.Tolerations = cns.Tolerations
}
Expand Down Expand Up @@ -145,6 +145,7 @@ func RecoverLabels(labels map[string]string, annotations map[string]string) {
}
}

// trimLabels removes label from labels according to ignoreLabels
func trimLabels(labels map[string]string, ignoreLabels []string) map[string]string {
if ignoreLabels == nil {
return nil
Expand All @@ -160,7 +161,8 @@ func trimLabels(labels map[string]string, ignoreLabels []string) map[string]stri
return trippedLabels
}

func convertAnnotations(annotation map[string]string) *ClustersNodeSelection {
// ConvertAnnotations converts annotations to ClustersNodeSelection
func ConvertAnnotations(annotation map[string]string) *ClustersNodeSelection {
if annotation == nil {
return nil
}
Expand Down
84 changes: 48 additions & 36 deletions pkg/webhook/hook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"fmt"
"io/ioutil"
"net/http"
"strings"

"k8s.io/api/admission/v1beta1"
admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1"
Expand Down Expand Up @@ -242,8 +241,8 @@ func inject(pod *corev1.Pod, ignoreKeys []string) {
}

cns := util.ClustersNodeSelection{
NodeSelector: pod.Spec.NodeSelector,
Affinity: pod.Spec.Affinity,
NodeSelector: nodeSelector,
Affinity: affinity,
Tolerations: pod.Spec.Tolerations,
}
cnsByte, err := json.Marshal(cns)
Expand All @@ -259,8 +258,6 @@ func inject(pod *corev1.Pod, ignoreKeys []string) {
}
pod.Annotations[util.SelectorKey] = string(cnsByte)

pod.Spec.NodeSelector = nodeSelector
pod.Spec.Affinity = affinity
pod.Spec.Tolerations = getPodTolerations(pod)
}

Expand Down Expand Up @@ -290,21 +287,31 @@ func getPodTolerations(pod *corev1.Pod) []corev1.Toleration {
return tolerations
}

// injectNodeSelector reserve ignoreLabels in nodeSelector, others would be removed
func injectNodeSelector(nodeSelector map[string]string, ignoreLabels []string) map[string]string {
nodeSelectorBackup := make(map[string]string)
finalNodeSelector := make(map[string]string)
for _, key := range ignoreLabels {
if len(strings.TrimSpace(key)) == 0 {
continue
}
if v := nodeSelector[key]; v != "" {
finalNodeSelector[key] = v
delete(nodeSelector, key)
labelMap := make(map[string]string)
for _, v := range ignoreLabels {
labelMap[v] = v
}
for k, v := range nodeSelector {
// not found in label, delete
if labelMap[k] != "" {
nodeSelectorBackup[k] = v
} else {
finalNodeSelector[k] = v
}
}
nodeSelector = nodeSelectorBackup
return finalNodeSelector
}

func injectAffinity(affinity *corev1.Affinity, ignoreLabels []string) *corev1.Affinity {
labelMap := make(map[string]string)
for _, v := range ignoreLabels {
labelMap[v] = v
}
if affinity.NodeAffinity == nil {
return nil
}
Expand All @@ -314,33 +321,38 @@ func injectAffinity(affinity *corev1.Affinity, ignoreLabels []string) *corev1.Af
required := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution
requiredCopy := affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.DeepCopy()
var nodeSelectorTerm []corev1.NodeSelectorTerm
for _, key := range ignoreLabels {
for termIdx, term := range requiredCopy.NodeSelectorTerms {
var mes, mfs []corev1.NodeSelectorRequirement
for meIdx, me := range term.MatchExpressions {
if me.Key != key {
continue
}
mes = append(mes, *me.DeepCopy())
required.
NodeSelectorTerms[termIdx].MatchExpressions = append(required.
NodeSelectorTerms[termIdx].MatchExpressions[:meIdx], required.
NodeSelectorTerms[termIdx].MatchExpressions[meIdx+1:]...)
for termIdx, term := range requiredCopy.NodeSelectorTerms {
var mes, mfs []corev1.NodeSelectorRequirement
var mesDeleteCount, mfsDeleteCount int
for meIdx, me := range term.MatchExpressions {
if labelMap[me.Key] != "" {
// found key, do not delete
continue
}
mes = append(mes, *me.DeepCopy())

for mfIdx, mf := range term.MatchFields {
if mf.Key != key {
continue
}
mfs = append(mfs, *mf.DeepCopy())
required.
NodeSelectorTerms[termIdx].MatchFields = append(required.
NodeSelectorTerms[termIdx].MatchFields[:mfIdx],
required.NodeSelectorTerms[termIdx].MatchFields[mfIdx+1:]...)
}
if len(mfs) != 0 || len(mes) != 0 {
nodeSelectorTerm = append(nodeSelectorTerm, corev1.NodeSelectorTerm{MatchFields: mfs, MatchExpressions: mes})
required.
NodeSelectorTerms[termIdx].MatchExpressions = append(required.
NodeSelectorTerms[termIdx].MatchExpressions[:meIdx-mesDeleteCount], required.
NodeSelectorTerms[termIdx].MatchExpressions[meIdx-mesDeleteCount+1:]...)
mesDeleteCount++
}

for mfIdx, mf := range term.MatchFields {
if labelMap[mf.Key] != "" {
// found key, do not delete
continue
}

mfs = append(mfs, *mf.DeepCopy())
required.
NodeSelectorTerms[termIdx].MatchFields = append(required.
NodeSelectorTerms[termIdx].MatchFields[:mfIdx-mesDeleteCount],
required.NodeSelectorTerms[termIdx].MatchFields[mfIdx-mfsDeleteCount+1:]...)
mfsDeleteCount++
}
if len(mfs) != 0 || len(mes) != 0 {
nodeSelectorTerm = append(nodeSelectorTerm, corev1.NodeSelectorTerm{MatchFields: mfs, MatchExpressions: mes})
}

}
Expand Down
62 changes: 62 additions & 0 deletions pkg/webhook/hook_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,13 +109,27 @@ func TestInject(t *testing.T) {
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.
NodeSelectorTerm{{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "test0",
Operator: v1.NodeSelectorOpIn,
Values: []string{
"aa",
},
},
{
Key: "test",
Operator: v1.NodeSelectorOpIn,
Values: []string{
"aa",
},
},
{
Key: "test1",
Operator: v1.NodeSelectorOpIn,
Values: []string{
"aa",
},
},
},
}}},
PreferredDuringSchedulingIgnoredDuringExecution: nil,
Expand All @@ -134,6 +148,13 @@ func TestInject(t *testing.T) {
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.
NodeSelectorTerm{{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "test0",
Operator: v1.NodeSelectorOpIn,
Values: []string{
"aa",
},
},
{
Key: "clusterID",
Operator: v1.NodeSelectorOpIn,
Expand All @@ -148,12 +169,51 @@ func TestInject(t *testing.T) {
"aa",
},
},
{
Key: "test1",
Operator: v1.NodeSelectorOpIn,
Values: []string{
"aa",
},
},
},
}}},
PreferredDuringSchedulingIgnoredDuringExecution: nil,
},
},
},
},
{
name: "Pod For Test With Node Selector and Affinity with multi match labels",
pod: test.PodForTestWithNodeSelectorAndAffinityClusterID(),
desireCNS: util.ClustersNodeSelection{
NodeSelector: map[string]string{"testbase": "testbase"},
Affinity: &v1.Affinity{
NodeAffinity: &v1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.
NodeSelectorTerm{{
MatchExpressions: []v1.NodeSelectorRequirement{
{
Key: "test",
Operator: v1.NodeSelectorOpIn,
Values: []string{
"aa",
},
},
{
Key: "test1",
Operator: v1.NodeSelectorOpIn,
Values: []string{
"aa",
},
},
},
}}},
PreferredDuringSchedulingIgnoredDuringExecution: nil,
},
},
},
keys: []string{"clusterID", "test0"},
},
{
name: "Pod For Test With Affinity",
Expand Down Expand Up @@ -189,8 +249,10 @@ func TestInject(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Logf("ann: %v", str)
if !reflect.DeepEqual(cns, c.desireCNS) {
t.Fatalf("Desire: %v, Get: %v", c.desireCNS, cns)
}
t.Logf("Desire: %v, Get: %v", c.desireCNS, cns)
}
}

0 comments on commit f2ad1b8

Please sign in to comment.