[Development][Feature] Added toleration support for operator (#61)

* Added toleration support Signed-off-by: iamabhishek-dubey <[email protected]> * Updated CHANGELOG according to v0.5.0 Signed-off-by: iamabhishek-dubey <[email protected]>
OT-CONTAINER-KIT · May 1, 2021 · 8871b17 · 8871b17
1 parent 0a26d7b
commit 8871b17
Show file tree

Hide file tree

Showing 8 changed files with 123 additions and 10 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,16 @@
+### v0.5.0
+##### May 1, 2021
+
+#### :tada: Features
+
+- Added support for recovering redis nodes from failover
+- Added toleration support for redis statefuls
+- Added capability to use existing secret created inside K8s
+
+#### :beetle: Bug Fixes
+
+- Fixed logic for service and statefulset comparison in K8s
+
 ### v0.4.0
 ##### February 6, 2021
 

diff --git a/api/v1beta1/redis_types.go b/api/v1beta1/redis_types.go
@@ -40,6 +40,7 @@ type RedisSpec struct {
 	SecurityContext   *corev1.PodSecurityContext `json:"securityContext,omitempty"`
 	PriorityClassName string                     `json:"priorityClassName,omitempty"`
 	Affinity          *corev1.Affinity           `json:"affinity,omitempty"`
+	Tolerations       *[]corev1.Toleration       `json:"tolerations,omitempty"`
 }
 
 // RedisStatus defines the observed state of Redis

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml
@@ -1230,6 +1230,45 @@ spec:
                         type: object
                     type: object
                 type: object
+              tolerations:
+                items:
+                  description: The pod this Toleration is attached to tolerates any
+                    taint that matches the triple <key,value,effect> using the matching
+                    operator <operator>.
+                  properties:
+                    effect:
+                      description: Effect indicates the taint effect to match. Empty
+                        means match all taint effects. When specified, allowed values
+                        are NoSchedule, PreferNoSchedule and NoExecute.
+                      type: string
+                    key:
+                      description: Key is the taint key that the toleration applies
+                        to. Empty means match all taint keys. If the key is empty,
+                        operator must be Exists; this combination means to match all
+                        values and all keys.
+                      type: string
+                    operator:
+                      description: Operator represents a key's relationship to the
+                        value. Valid operators are Exists and Equal. Defaults to Equal.
+                        Exists is equivalent to wildcard for value, so that a pod
+                        can tolerate all taints of a particular category.
+                      type: string
+                    tolerationSeconds:
+                      description: TolerationSeconds represents the period of time
+                        the toleration (which must be of effect NoExecute, otherwise
+                        this field is ignored) tolerates the taint. By default, it
+                        is not set, which means tolerate the taint forever (do not
+                        evict). Zero and negative values will be treated as 0 (evict
+                        immediately) by the system.
+                      format: int64
+                      type: integer
+                    value:
+                      description: Value is the taint value the toleration matches
+                        to. If the operator is Exists, the value should be empty,
+                        otherwise just a regular string.
+                      type: string
+                  type: object
+                type: array
             required:
             - global
             - mode
@@ -2476,6 +2515,46 @@ spec:
                             type: object
                         type: object
                     type: object
+                  tolerations:
+                    items:
+                      description: The pod this Toleration is attached to tolerates
+                        any taint that matches the triple <key,value,effect> using
+                        the matching operator <operator>.
+                      properties:
+                        effect:
+                          description: Effect indicates the taint effect to match.
+                            Empty means match all taint effects. When specified, allowed
+                            values are NoSchedule, PreferNoSchedule and NoExecute.
+                          type: string
+                        key:
+                          description: Key is the taint key that the toleration applies
+                            to. Empty means match all taint keys. If the key is empty,
+                            operator must be Exists; this combination means to match
+                            all values and all keys.
+                          type: string
+                        operator:
+                          description: Operator represents a key's relationship to
+                            the value. Valid operators are Exists and Equal. Defaults
+                            to Equal. Exists is equivalent to wildcard for value,
+                            so that a pod can tolerate all taints of a particular
+                            category.
+                          type: string
+                        tolerationSeconds:
+                          description: TolerationSeconds represents the period of
+                            time the toleration (which must be of effect NoExecute,
+                            otherwise this field is ignored) tolerates the taint.
+                            By default, it is not set, which means tolerate the taint
+                            forever (do not evict). Zero and negative values will
+                            be treated as 0 (evict immediately) by the system.
+                          format: int64
+                          type: integer
+                        value:
+                          description: Value is the taint value the toleration matches
+                            to. If the operator is Exists, the value should be empty,
+                            otherwise just a regular string.
+                          type: string
+                      type: object
+                    type: array
                 required:
                 - global
                 - mode

diff --git a/controllers/redis_controller.go b/controllers/redis_controller.go
@@ -103,7 +103,7 @@ func (r *RedisReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl
 				k8sutils.ExecuteRedisReplicationCommand(instance)
 			} else {
 				reqLogger.Info("Redis master count is desired")
-				if k8sutils.CheckRedisClusterState(instance) {
+				if k8sutils.CheckRedisClusterState(instance) >= int(*instance.Spec.Size)*2-1 {
 					k8sutils.ExecuteFaioverOperation(instance)
 				}
 				return ctrl.Result{RequeueAfter: time.Second * 120}, nil

diff --git a/example/redis-cluster-example.yaml b/example/redis-cluster-example.yaml
@@ -47,3 +47,8 @@ spec:
         resources:
           requests:
             storage: 1Gi
+  tolerations:
+  - key: "key1"
+    operator: "Equal"
+    value: "value1"
+    effect: "NoSchedule"
diff --git a/k8sutils/redis.go b/k8sutils/redis.go
@@ -140,6 +140,11 @@ func executeFailoverCommand(cr *redisv1beta1.Redis, role string) {
 		err := client.Process(cmd)
 		if err != nil {
 			reqLogger.Error(err, "Redis command failed with this error")
+			flushcommand := redis.NewStringCmd("flushall")
+			err := client.Process(flushcommand)
+			if err != nil {
+				reqLogger.Error(err, "Redis flush command failed with this error")
+			}
 		}
 
 		output, err := cmd.Result()
@@ -165,17 +170,13 @@ func CheckRedisNodeCount(cr *redisv1beta1.Redis) int {
 }
 
 // CheckRedisClusterState will check the redis cluster state
-func CheckRedisClusterState(cr *redisv1beta1.Redis) bool {
+func CheckRedisClusterState(cr *redisv1beta1.Redis) int {
 	reqLogger := log.WithValues("Request.Namespace", cr.Namespace, "Request.Name", cr.ObjectMeta.Name)
 	output := checkRedisCluster(cr)
-	match, err := regexp.MatchString("fail", output)
-	if err != nil {
-		reqLogger.Error(err, "Error in compiling regex")
-	}
-	if match {
-		reqLogger.Info("Found cluster in failed state")
-	}
-	return match
+	pattern := regexp.MustCompile("fail")
+	match := pattern.FindAllStringIndex(output, -1)
+	reqLogger.Info("Number of failed nodes in cluster", "Failed Node Count", len(match))
+	return len(match)
 }
 
 // configureRedisClient will configure the Redis Client

diff --git a/k8sutils/statefulset.go b/k8sutils/statefulset.go
@@ -47,6 +47,9 @@ func GenerateStateFulSetsDef(cr *redisv1beta1.Redis, labels map[string]string, r
 			},
 		},
 	}
+	if cr.Spec.Tolerations != nil {
+		statefulset.Spec.Template.Spec.Tolerations = *cr.Spec.Tolerations
+	}
 	AddOwnerRefToObject(statefulset, AsOwner(cr))
 	return statefulset
 }