From 9af3264ad47ceb41b170f68545210f4dd37bb1d8 Mon Sep 17 00:00:00 2001 From: "liheng.zms" Date: Tue, 19 Nov 2024 11:09:00 +0800 Subject: [PATCH] fix cloneSet controller block caused by scale expectation leakage Signed-off-by: liheng.zms --- .../cloneset/cloneset_controller.go | 38 +++++++++++++++++++ pkg/features/kruise_features.go | 16 +++++--- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/pkg/controller/cloneset/cloneset_controller.go b/pkg/controller/cloneset/cloneset_controller.go index 046c1ce461..9c5244f7dc 100644 --- a/pkg/controller/cloneset/cloneset_controller.go +++ b/pkg/controller/cloneset/cloneset_controller.go @@ -41,6 +41,7 @@ import ( "github.com/openkruise/kruise/pkg/util/refmanager" "github.com/openkruise/kruise/pkg/util/volumeclaimtemplate" + "github.com/prometheus/client_golang/prometheus" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -59,6 +60,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" @@ -66,6 +68,8 @@ import ( func init() { flag.IntVar(&concurrentReconciles, "cloneset-workers", concurrentReconciles, "Max concurrent workers for CloneSet controller.") + // register prometheus + metrics.Registry.MustRegister(CloneSetScaleExpectationLeakageMetrics) } var ( @@ -75,6 +79,16 @@ var ( minimumReplicasToPreDownloadImage int32 = 3 ) +var ( + CloneSetScaleExpectationLeakageMetrics = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cloneset_scale_expectation_leakage", + Help: "CloneSet Scale Expectation Leakage Metrics", + // cloneSet namespace, name + }, []string{"namespace", "name"}, + ) +) + // Add creates a new CloneSet Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(mgr manager.Manager) error { @@ -229,7 +243,31 @@ func (r *ReconcileCloneSet) doReconcile(request reconcile.Request) (res reconcil // If scaling expectations have not satisfied yet, just skip this reconcile. if scaleSatisfied, unsatisfiedDuration, scaleDirtyPods := clonesetutils.ScaleExpectations.SatisfiedExpectations(request.String()); !scaleSatisfied { if unsatisfiedDuration >= expectations.ExpectationTimeout { + // In some extreme scenarios, if the Pod is created and then quickly deleted, there may be event loss. + // Therefore, a touting mechanism is needed to ensure that clonesets can continue to work. klog.InfoS("Expectation unsatisfied overtime", "cloneSet", request, "scaleDirtyPods", scaleDirtyPods, "overTime", unsatisfiedDuration) + CloneSetScaleExpectationLeakageMetrics.WithLabelValues(request.Namespace, request.Name).Add(1) + // TODO: check the existence of resource in apiserver using client-go directly + /*for _, pods := range scaleDirtyPods { + for _, name := range pods { + _, err = kubeClient.GetGenericClient().KubeClient.CoreV1().Pods(request.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err == nil { + klog.Warningf("CloneSet(%s/%s) ScaleExpectations leakage, but Pod(%s) already exist", request.Namespace, request.Name, name) + return reconcile.Result{RequeueAfter: 30 * time.Second}, nil + } else if !errors.IsNotFound(err) { + klog.ErrorS(err, "Failed to get Pod", "cloneSet", request, "pod", name) + return reconcile.Result{RequeueAfter: 3 * time.Second}, nil + } + } + } + klog.InfoS("CloneSet ScaleExpectation DirtyPods no longer exists, and delete ScaleExpectation", "cloneSet", request)*/ + if utilfeature.DefaultFeatureGate.Enabled(features.ForceDeleteTimeoutExpectationFeatureGate) { + klog.InfoS("Expectation unsatisfied overtime, and force delete the timeout Expectation", "cloneSet", request, "scaleDirtyPods", scaleDirtyPods, "overTime", unsatisfiedDuration) + clonesetutils.ScaleExpectations.DeleteExpectations(request.String()) + // In order to avoid the scale expectation timeout, + // there is no subsequent Pod, CloneSet event causing CloneSet not to be scheduled + return reconcile.Result{RequeueAfter: 10 * time.Second}, nil + } return reconcile.Result{}, nil } klog.V(4).InfoS("Not satisfied scale", "cloneSet", request, "scaleDirtyPods", scaleDirtyPods) diff --git a/pkg/features/kruise_features.go b/pkg/features/kruise_features.go index 78eab56276..fb0de51b9c 100644 --- a/pkg/features/kruise_features.go +++ b/pkg/features/kruise_features.go @@ -131,6 +131,9 @@ const ( // Enables policies auto resizing PVCs created by a StatefulSet when user expands volumeClaimTemplates. StatefulSetAutoResizePVCGate featuregate.Feature = "StatefulSetAutoResizePVCGate" + + // ForceDeleteTimeoutExpectationFeatureGate enable delete timeout expectation, for example: cloneSet ScaleExpectation + ForceDeleteTimeoutExpectationFeatureGate = "ForceDeleteTimeoutExpectationGate" ) var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ @@ -160,12 +163,13 @@ var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ ResourceDistributionGate: {Default: false, PreRelease: featuregate.Alpha}, DeletionProtectionForCRDCascadingGate: {Default: false, PreRelease: featuregate.Alpha}, - EnhancedLivenessProbeGate: {Default: false, PreRelease: featuregate.Alpha}, - RecreatePodWhenChangeVCTInCloneSetGate: {Default: false, PreRelease: featuregate.Alpha}, - StatefulSetStartOrdinal: {Default: false, PreRelease: featuregate.Alpha}, - PodIndexLabel: {Default: true, PreRelease: featuregate.Beta}, - EnableExternalCerts: {Default: false, PreRelease: featuregate.Alpha}, - StatefulSetAutoResizePVCGate: {Default: false, PreRelease: featuregate.Alpha}, + EnhancedLivenessProbeGate: {Default: false, PreRelease: featuregate.Alpha}, + RecreatePodWhenChangeVCTInCloneSetGate: {Default: false, PreRelease: featuregate.Alpha}, + StatefulSetStartOrdinal: {Default: false, PreRelease: featuregate.Alpha}, + PodIndexLabel: {Default: true, PreRelease: featuregate.Beta}, + EnableExternalCerts: {Default: false, PreRelease: featuregate.Alpha}, + StatefulSetAutoResizePVCGate: {Default: false, PreRelease: featuregate.Alpha}, + ForceDeleteTimeoutExpectationFeatureGate: {Default: false, PreRelease: featuregate.Alpha}, } func init() {