Skip to content

Commit

Permalink
updating operator to check rollout instead of rs for ignoreExtraneous
Browse files Browse the repository at this point in the history
  • Loading branch information
yossig-runai committed Dec 5, 2024
1 parent 1563f2f commit e658ca7
Show file tree
Hide file tree
Showing 9 changed files with 455 additions and 262 deletions.
3 changes: 3 additions & 0 deletions .github/workflows/ci-pr-to-master.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ jobs:
- name: Create Kind cluster
run: make create-kind-cluster

- name: Install argo-rollout
run: make install-argo-rollout

- name: Make test
env:
USE_EXISTING_CLUSTER: "true"
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/ci-push-to-master.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ jobs:
- name: Create Kind cluster
run: make create-kind-cluster

- name: Install argo-rollout
run: make install-argo-rollout

- name: Make test
env:
USE_EXISTING_CLUSTER: "true"
Expand Down
10 changes: 9 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -310,4 +310,12 @@ catalog-push: ## Push a catalog image.

.PHONY: create-kind-cluster
create-kind-cluster: ## Create a kind cluster.
kind create cluster --name=argo-rollout-config-keeper
kind create cluster --name=argo-rollout-config-keeper

.PHONY: install-argo-rollout
install-argo-rollout: ## Install Argo Rollout.
{ \
kubectl create namespace argo-rollouts ;\
kubectl apply -n argo-rollouts -f https://github.com/argoproj/argo-rollouts/releases/latest/download/install.yaml ;\
kubectl wait --for=condition=available --timeout=600s deployment.apps/argo-rollouts -n argo-rollouts ;\
}
3 changes: 1 addition & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ module github.com/run-ai/argo-rollout-config-keeper
go 1.22.4

require (
github.com/argoproj/argo-rollouts v1.7.2
github.com/go-logr/logr v1.3.0
github.com/hashicorp/go-version v1.7.0
github.com/onsi/ginkgo/v2 v2.13.0
github.com/onsi/gomega v1.29.0
github.com/prometheus/client_golang v1.18.0
Expand Down Expand Up @@ -46,7 +46,6 @@ require (
github.com/prometheus/common v0.47.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.9.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.25.0 // indirect
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
github.com/argoproj/argo-rollouts v1.7.2 h1:faDUH/qePerYRwsrHfVzNQkhjGBgXIiVYdVK8824kMo=
github.com/argoproj/argo-rollouts v1.7.2/go.mod h1:Te4HrUELxKiBpK8lgk77o4gTa3mv8pXCd8xdPprKrbs=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
Expand Down Expand Up @@ -54,8 +56,6 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
Expand Down
118 changes: 92 additions & 26 deletions internal/common/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

"github.com/run-ai/argo-rollout-config-keeper/internal/tools"

v1alpha1 "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
"github.com/go-logr/logr"
keeperv1alpha1 "github.com/run-ai/argo-rollout-config-keeper/api/v1alpha1"
"github.com/run-ai/argo-rollout-config-keeper/internal/metrics"
Expand Down Expand Up @@ -248,28 +249,42 @@ func (r *ArgoRolloutConfigKeeperCommon) ignoreExtraneousOperation(ctx context.Co
r.Labels.AppLabel: appLabelValue,
r.Labels.AppVersionLabel: t.Labels[r.Labels.AppVersionLabel],
}
replicaSets, err := r.getFilteredReplicaSets(ctx, t.Namespace, labelSelector)

rolloutLabelSelector := map[string]string{
r.Labels.AppLabel: appLabelValue,
}

rollout, err := r.checkIfRolloutActive(ctx, t.Namespace, rolloutLabelSelector)

if err != nil {
r.Logger.Error(err, "unable to get filtered replicasets")
r.Logger.Error(err, "unable to check if rollout is active")
return err
}

rsAnnotation := replicaSets.Items[0].Annotations["rollout.argoproj.io/ephemeral-metadata"]
// if replicaSet Annotation is not nil or empty, and having the following value: '{"labels":{"role":"preview"}}' or '{"labels":{"role":"canary"}}' it should add IgnoreExtraneous annotation
if rsAnnotation != "" && (rsAnnotation == `{"labels":{"role":"preview"}}` || rsAnnotation == `{"labels":{"role":"canary"}}`) {
r.Logger.Info(fmt.Sprintf("adding IgnoreExtraneous annotation to %s configmap, reason: replica has rollout.argoproj.io/ephemeral-metadata annotation", t.Name))
if t.Annotations != nil {
t.Annotations["argocd.argoproj.io/compare-options"] = "IgnoreExtraneous"
} else {
t.Annotations = map[string]string{"argocd.argoproj.io/compare-options": "IgnoreExtraneous"}
}
if rollout {
replicaSets, err := r.getFilteredReplicaSets(ctx, t.Namespace, labelSelector)

err = r.Update(ctx, t)
if err != nil {
r.Logger.Error(err, "unable to update configmap")
r.Logger.Error(err, "unable to get filtered replicasets")
return err
}

rsAnnotation := replicaSets.Items[0].Annotations["rollout.argoproj.io/ephemeral-metadata"]
// if replicaSet Annotation is not nil or empty, and having the following value: '{"labels":{"role":"preview"}}' or '{"labels":{"role":"canary"}}' it should add IgnoreExtraneous annotation
if rsAnnotation != "" && (rsAnnotation == `{"labels":{"role":"active"}}` || rsAnnotation == `{"labels":{"role":"stable"}}`) {
r.Logger.Info(fmt.Sprintf("adding IgnoreExtraneous annotation to %s configmap, reason: replica has rollout.argoproj.io/ephemeral-metadata annotation", t.Name))
if t.Annotations != nil {
t.Annotations["argocd.argoproj.io/compare-options"] = "IgnoreExtraneous"
} else {
t.Annotations = map[string]string{"argocd.argoproj.io/compare-options": "IgnoreExtraneous"}
}

err = r.Update(ctx, t)
if err != nil {
r.Logger.Error(err, "unable to update configmap")
return err
}
}
}

return nil
Expand All @@ -278,28 +293,42 @@ func (r *ArgoRolloutConfigKeeperCommon) ignoreExtraneousOperation(ctx context.Co
r.Labels.AppLabel: appLabelValue,
r.Labels.AppVersionLabel: t.Labels[r.Labels.AppVersionLabel],
}
replicaSets, err := r.getFilteredReplicaSets(ctx, t.Namespace, labelSelector)

rolloutLabelSelector := map[string]string{
r.Labels.AppLabel: appLabelValue,
}

rollout, err := r.checkIfRolloutActive(ctx, t.Namespace, rolloutLabelSelector)

if err != nil {
r.Logger.Error(err, "unable to get filtered replicasets")
r.Logger.Error(err, "unable to check if rollout is active")
return err
}

rsAnnotation := replicaSets.Items[0].Annotations["rollout.argoproj.io/ephemeral-metadata"]
// if replicaSet Annotation is not nil or empty, and having the following value: '{"labels":{"role":"preview"}}' or '{"labels":{"role":"canary"}}' it should add IgnoreExtraneous annotation
if rsAnnotation != "" && (rsAnnotation == `{"labels":{"role":"preview"}}` || rsAnnotation == `{"labels":{"role":"canary"}}`) {
r.Logger.Info(fmt.Sprintf("adding IgnoreExtraneous annotation to %s secret, reason: replica has rollout.argoproj.io/ephemeral-metadata annotation", t.Name))
if t.Annotations != nil {
t.Annotations["argocd.argoproj.io/compare-options"] = "IgnoreExtraneous"
} else {
t.Annotations = map[string]string{"argocd.argoproj.io/compare-options": "IgnoreExtraneous"}
}
if rollout {
replicaSets, err := r.getFilteredReplicaSets(ctx, t.Namespace, labelSelector)

err = r.Update(ctx, t)
if err != nil {
r.Logger.Error(err, "unable to update configmap")
r.Logger.Error(err, "unable to get filtered replicasets")
return err
}

rsAnnotation := replicaSets.Items[0].Annotations["rollout.argoproj.io/ephemeral-metadata"]
// if replicaSet Annotation is not nil or empty, and having the following value: '{"labels":{"role":"preview"}}' or '{"labels":{"role":"canary"}}' it should add IgnoreExtraneous annotation
if rsAnnotation != "" && (rsAnnotation == `{"labels":{"role":"active"}}` || rsAnnotation == `{"labels":{"role":"stable"}}`) {
r.Logger.Info(fmt.Sprintf("adding IgnoreExtraneous annotation to %s secret, reason: replica has rollout.argoproj.io/ephemeral-metadata annotation", t.Name))
if t.Annotations != nil {
t.Annotations["argocd.argoproj.io/compare-options"] = "IgnoreExtraneous"
} else {
t.Annotations = map[string]string{"argocd.argoproj.io/compare-options": "IgnoreExtraneous"}
}

err = r.Update(ctx, t)
if err != nil {
r.Logger.Error(err, "unable to update configmap")
return err
}
}
}

return nil
Expand Down Expand Up @@ -391,3 +420,40 @@ func (r *ArgoRolloutConfigKeeperCommon) UpdateCondition(ctx context.Context, T i
return fmt.Errorf("unsupported type: %T", T)
}
}

func (r *ArgoRolloutConfigKeeperCommon) checkIfRolloutActive(ctx context.Context, namespace string, labelSelector map[string]string) (bool, error) {
rollouts := &v1alpha1.RolloutList{}
blueGreenEmptyStatus := v1alpha1.BlueGreenStatus{}
canaryEmptyStatus := v1alpha1.CanaryStatus{}

if err := r.List(ctx, rollouts, client.InNamespace(namespace), client.MatchingLabels(labelSelector)); err != nil {
r.Logger.Error(err, fmt.Sprintf("unable to list rollouts in %s namespace", namespace))
return false, client.IgnoreNotFound(err)
}

if len(rollouts.Items) == 0 {
return false, nil
}

rollout := rollouts.Items[0]

if rollout.Status.BlueGreen != blueGreenEmptyStatus {
if rollout.Status.BlueGreen.ActiveSelector != "" && rollout.Status.BlueGreen.ActiveSelector != rollout.Status.BlueGreen.PreviewSelector {
r.Logger.Info("blue-green rollout is active")
return true, nil
} else {
return false, nil
}
} else if rollout.Status.Canary != canaryEmptyStatus {
if rollout.Status.Canary.CurrentExperiment != "" ||
rollout.Status.Canary.CurrentStepAnalysisRunStatus != nil ||
rollout.Status.Canary.CurrentBackgroundAnalysisRunStatus != nil {
r.Logger.Info("canary rollout is active")
return true, nil
} else {
return false, nil
}
} else {
return false, nil
}
}
Loading

0 comments on commit e658ca7

Please sign in to comment.