Skip to content

Commit

Permalink
Cleanup stale objects (#3538)
Browse files Browse the repository at this point in the history
* Cleanup stale objects

Signed-off-by: Artiom Diomin <[email protected]>

* Post-review changes

Signed-off-by: Artiom Diomin <[email protected]>

---------

Signed-off-by: Artiom Diomin <[email protected]>
  • Loading branch information
kron4eg authored Feb 5, 2025
1 parent 487feee commit 2d66472
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 3 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ require (
k8s.io/kubectl v0.31.1
k8s.io/kubelet v0.31.1
k8s.io/utils v0.0.0-20240921022957-49e7df575cb6
sigs.k8s.io/controller-runtime v0.19.0
sigs.k8s.io/controller-runtime v0.19.4
sigs.k8s.io/yaml v1.4.0
)

Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -603,8 +603,8 @@ k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 h1:MDF6h2H/h4tbzmtIKTuctcwZmY0tY
k8s.io/utils v0.0.0-20240921022957-49e7df575cb6/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo=
oras.land/oras-go v1.2.5/go.mod h1:PuAwRShRZCsZb7g8Ar3jKKQR/2A/qN+pkYxIOd/FAoo=
sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q=
sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4=
sigs.k8s.io/controller-runtime v0.19.4 h1:SUmheabttt0nx8uJtoII4oIP27BVVvAKFvdvGFwV/Qo=
sigs.k8s.io/controller-runtime v0.19.4/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
sigs.k8s.io/kustomize/api v0.17.2 h1:E7/Fjk7V5fboiuijoZHgs4aHuexi5Y2loXlVOAVAG5g=
Expand Down
130 changes: 130 additions & 0 deletions pkg/tasks/cleanup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
/*
Copyright 2025 The KubeOne Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tasks

import (
"maps"

"k8c.io/kubeone/pkg/fail"
"k8c.io/kubeone/pkg/state"

rbacv1 "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
metav1unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
crclient "sigs.k8s.io/controller-runtime/pkg/client"
)

func cleanupStaleObjects(st *state.State) error {
st.Logger.Infoln("Cleanup stale objects...")

var cleanupObjects []crclient.Object
cleanupObjects = append(cleanupObjects, kuredObjects()...)

NextObject:
for _, obj := range cleanupObjects {
originalLabels := maps.Clone(obj.GetLabels())
obj.SetLabels(map[string]string{})

err := st.DynamicClient.Get(st.Context, crclient.ObjectKeyFromObject(obj), obj)
switch {
case apierrors.IsNotFound(err):
continue NextObject
case err != nil:
return fail.KubeClient(err, "checking stale object %s %q", obj.GetObjectKind().GroupVersionKind().String(), crclient.ObjectKeyFromObject(obj))
}

realLabels := obj.GetLabels()

// compare requested labels to the real of, if not match -> let the object live
for cleanupKey, cleanupValue := range originalLabels {
if val, ok := realLabels[cleanupKey]; !ok || val != cleanupValue {
st.Logger.Debugf("skip deleting object as labels are different: %s %q", obj.GetObjectKind().GroupVersionKind().String(), crclient.ObjectKeyFromObject(obj))

continue NextObject
}
}

if err := st.DynamicClient.Delete(st.Context, obj); crclient.IgnoreNotFound(err) != nil {
return fail.KubeClient(err, "deleting stale object %s %q", obj.GetObjectKind().GroupVersionKind().String(), crclient.ObjectKeyFromObject(obj))
}

st.Logger.Debugf("deleted stale object %s %q", obj.GetObjectKind().GroupVersionKind().String(), crclient.ObjectKeyFromObject(obj))
}

return nil
}

func kuredObjects() []crclient.Object {
labels := map[string]string{"kubeone.io/addon": "unattended-upgrades"}
unstructuredLabels := withLabels(labels)

cleanupObjects := []crclient.Object{
&rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: "kured",
Labels: labels,
},
},
newUnstructured(
"rbac.authorization.k8s.io/v1",
"ClusterRoleBinding",
crclient.ObjectKey{Name: "kured"},
unstructuredLabels,
),
newUnstructured(
"rbac.authorization.k8s.io/v1",
"Role",
crclient.ObjectKey{Name: "kured", Namespace: "kube-system"},
unstructuredLabels,
),
newUnstructured(
"rbac.authorization.k8s.io/v1",
"RoleBinding",
crclient.ObjectKey{Name: "kured", Namespace: "kube-system"},
unstructuredLabels,
),
newUnstructured(
"v1",
"ServiceAccount",
crclient.ObjectKey{Name: "kured", Namespace: "kube-system"},
unstructuredLabels,
),
}

return cleanupObjects
}

func newUnstructured(apiVersion string, kind string, identity crclient.ObjectKey, opts ...func(*metav1unstructured.Unstructured)) crclient.Object {
obj := &metav1unstructured.Unstructured{}
obj.SetAPIVersion(apiVersion)
obj.SetKind(kind)
obj.SetName(identity.Name)
obj.SetNamespace(identity.Namespace)

for _, opt := range opts {
opt(obj)
}

return obj
}

func withLabels(labels map[string]string) func(*metav1unstructured.Unstructured) {
return func(u *metav1unstructured.Unstructured) {
u.SetLabels(labels)
}
}
5 changes: 5 additions & 0 deletions pkg/tasks/tasks.go
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,11 @@ func WithResources(t Tasks) Tasks {
Operation: "labeling control-plane nodes",
Description: "labeling control-plane nodes",
},
{
Fn: cleanupStaleObjects,
Operation: "cleaning up any leftovers from addons",
Description: "clean up any leftovers from addons",
},
{
Fn: addons.Ensure,
Operation: "applying addons",
Expand Down

0 comments on commit 2d66472

Please sign in to comment.