From 8871b171b42048045135ce308a4c2421f7cb5e6f Mon Sep 17 00:00:00 2001 From: Abhishek Dubey Date: Sat, 1 May 2021 23:15:19 +0530 Subject: [PATCH] [Development][Feature] Added toleration support for operator (#61) * Added toleration support Signed-off-by: iamabhishek-dubey * Updated CHANGELOG according to v0.5.0 Signed-off-by: iamabhishek-dubey --- CHANGELOG.md | 13 +++ api/v1beta1/redis_types.go | 1 + api/v1beta1/zz_generated.deepcopy.go | 11 +++ .../redis.redis.opstreelabs.in_redis.yaml | 79 +++++++++++++++++++ controllers/redis_controller.go | 2 +- example/redis-cluster-example.yaml | 5 ++ k8sutils/redis.go | 19 ++--- k8sutils/statefulset.go | 3 + 8 files changed, 123 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6061fd581..95ae1d5d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +### v0.5.0 +##### May 1, 2021 + +#### :tada: Features + +- Added support for recovering redis nodes from failover +- Added toleration support for redis statefuls +- Added capability to use existing secret created inside K8s + +#### :beetle: Bug Fixes + +- Fixed logic for service and statefulset comparison in K8s + ### v0.4.0 ##### February 6, 2021 diff --git a/api/v1beta1/redis_types.go b/api/v1beta1/redis_types.go index bcdcf66bb..f76c83f17 100644 --- a/api/v1beta1/redis_types.go +++ b/api/v1beta1/redis_types.go @@ -40,6 +40,7 @@ type RedisSpec struct { SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"` PriorityClassName string `json:"priorityClassName,omitempty"` Affinity *corev1.Affinity `json:"affinity,omitempty"` + Tolerations *[]corev1.Toleration `json:"tolerations,omitempty"` } // RedisStatus defines the observed state of Redis diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index e230a2a36..e99d29a3f 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -258,6 +258,17 @@ func (in *RedisSpec) DeepCopyInto(out *RedisSpec) { *out = new(v1.Affinity) (*in).DeepCopyInto(*out) } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = new([]v1.Toleration) + if **in != nil { + in, out := *in, *out + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RedisSpec. diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml index e97a508f1..ecfcc5bb0 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml @@ -1230,6 +1230,45 @@ spec: type: object type: object type: object + tolerations: + items: + description: The pod this Toleration is attached to tolerates any + taint that matches the triple using the matching + operator . + properties: + effect: + description: Effect indicates the taint effect to match. Empty + means match all taint effects. When specified, allowed values + are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies + to. Empty means match all taint keys. If the key is empty, + operator must be Exists; this combination means to match all + values and all keys. + type: string + operator: + description: Operator represents a key's relationship to the + value. Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod + can tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of time + the toleration (which must be of effect NoExecute, otherwise + this field is ignored) tolerates the taint. By default, it + is not set, which means tolerate the taint forever (do not + evict). Zero and negative values will be treated as 0 (evict + immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches + to. If the operator is Exists, the value should be empty, + otherwise just a regular string. + type: string + type: object + type: array required: - global - mode @@ -2476,6 +2515,46 @@ spec: type: object type: object type: object + tolerations: + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple using + the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. When specified, allowed + values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies + to. Empty means match all taint keys. If the key is empty, + operator must be Exists; this combination means to match + all values and all keys. + type: string + operator: + description: Operator represents a key's relationship to + the value. Valid operators are Exists and Equal. Defaults + to Equal. Exists is equivalent to wildcard for value, + so that a pod can tolerate all taints of a particular + category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of + time the toleration (which must be of effect NoExecute, + otherwise this field is ignored) tolerates the taint. + By default, it is not set, which means tolerate the taint + forever (do not evict). Zero and negative values will + be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches + to. If the operator is Exists, the value should be empty, + otherwise just a regular string. + type: string + type: object + type: array required: - global - mode diff --git a/controllers/redis_controller.go b/controllers/redis_controller.go index 3fc9c55de..dd071e9b7 100644 --- a/controllers/redis_controller.go +++ b/controllers/redis_controller.go @@ -103,7 +103,7 @@ func (r *RedisReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl k8sutils.ExecuteRedisReplicationCommand(instance) } else { reqLogger.Info("Redis master count is desired") - if k8sutils.CheckRedisClusterState(instance) { + if k8sutils.CheckRedisClusterState(instance) >= int(*instance.Spec.Size)*2-1 { k8sutils.ExecuteFaioverOperation(instance) } return ctrl.Result{RequeueAfter: time.Second * 120}, nil diff --git a/example/redis-cluster-example.yaml b/example/redis-cluster-example.yaml index 3516450b3..429ce8e50 100644 --- a/example/redis-cluster-example.yaml +++ b/example/redis-cluster-example.yaml @@ -47,3 +47,8 @@ spec: resources: requests: storage: 1Gi + tolerations: + - key: "key1" + operator: "Equal" + value: "value1" + effect: "NoSchedule" diff --git a/k8sutils/redis.go b/k8sutils/redis.go index e2037a2f0..428c2bbdf 100644 --- a/k8sutils/redis.go +++ b/k8sutils/redis.go @@ -140,6 +140,11 @@ func executeFailoverCommand(cr *redisv1beta1.Redis, role string) { err := client.Process(cmd) if err != nil { reqLogger.Error(err, "Redis command failed with this error") + flushcommand := redis.NewStringCmd("flushall") + err := client.Process(flushcommand) + if err != nil { + reqLogger.Error(err, "Redis flush command failed with this error") + } } output, err := cmd.Result() @@ -165,17 +170,13 @@ func CheckRedisNodeCount(cr *redisv1beta1.Redis) int { } // CheckRedisClusterState will check the redis cluster state -func CheckRedisClusterState(cr *redisv1beta1.Redis) bool { +func CheckRedisClusterState(cr *redisv1beta1.Redis) int { reqLogger := log.WithValues("Request.Namespace", cr.Namespace, "Request.Name", cr.ObjectMeta.Name) output := checkRedisCluster(cr) - match, err := regexp.MatchString("fail", output) - if err != nil { - reqLogger.Error(err, "Error in compiling regex") - } - if match { - reqLogger.Info("Found cluster in failed state") - } - return match + pattern := regexp.MustCompile("fail") + match := pattern.FindAllStringIndex(output, -1) + reqLogger.Info("Number of failed nodes in cluster", "Failed Node Count", len(match)) + return len(match) } // configureRedisClient will configure the Redis Client diff --git a/k8sutils/statefulset.go b/k8sutils/statefulset.go index 3c0074817..48bc67d26 100644 --- a/k8sutils/statefulset.go +++ b/k8sutils/statefulset.go @@ -47,6 +47,9 @@ func GenerateStateFulSetsDef(cr *redisv1beta1.Redis, labels map[string]string, r }, }, } + if cr.Spec.Tolerations != nil { + statefulset.Spec.Template.Spec.Tolerations = *cr.Spec.Tolerations + } AddOwnerRefToObject(statefulset, AsOwner(cr)) return statefulset }