From bcbb403cc4755a6788224a9dcedd07e3717871ee Mon Sep 17 00:00:00 2001 From: MyonKeminta Date: Mon, 12 Apr 2021 19:32:17 +0800 Subject: [PATCH] Try to add support for injecting network partition between two nodes --- pkg/core/nemesis.go | 4 +- pkg/nemesis/nemesis.go | 1 + pkg/nemesis/network_partition.go | 6 +- pkg/nemesis/network_partition_single_route.go | 110 ++++++++++++++++++ 4 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 pkg/nemesis/network_partition_single_route.go diff --git a/pkg/core/nemesis.go b/pkg/core/nemesis.go index b9def6b54..3b27a7d3a 100644 --- a/pkg/core/nemesis.go +++ b/pkg/core/nemesis.go @@ -20,8 +20,10 @@ const ( PodKill ChaosKind = "pod-kill" // ContainerKill will random kill the specified container of pod, but retain the pod ContainerKill ChaosKind = "container-kill" - // NetworkPartition partitions network between nodes + // NetworkPartition partitions network between two groups of nodes NetworkPartition ChaosKind = "network-partition" + // NetworkPartitionRoute partitions network between two pods without affection traffics between other pairs of pods. + NetworkPartitionSingleRoute ChaosKind = "network-partition-single-route" // NetemChaos adds corrupt or other chaos. NetemChaos ChaosKind = "netem-chaos" // TimeChaos means diff --git a/pkg/nemesis/nemesis.go b/pkg/nemesis/nemesis.go index 8a25f8f42..3dcd4baa8 100644 --- a/pkg/nemesis/nemesis.go +++ b/pkg/nemesis/nemesis.go @@ -46,6 +46,7 @@ func init() { core.RegisterNemesis(podKill{client}) core.RegisterNemesis(containerKill{client}) core.RegisterNemesis(networkPartition{client}) + core.RegisterNemesis(networkPartitionSingleRoute{client}) core.RegisterNemesis(netem{client}) core.RegisterNemesis(timeChaos{client}) core.RegisterNemesis(scaling{client}) diff --git a/pkg/nemesis/network_partition.go b/pkg/nemesis/network_partition.go index 1bca10099..b6cf3863a 100644 --- a/pkg/nemesis/network_partition.go +++ b/pkg/nemesis/network_partition.go @@ -94,7 +94,7 @@ func networkChaosSpecTemplate(partOneNs, partTwoNS string, partOne, partTwo []cl } func (n networkPartition) Invoke(ctx context.Context, _ *cluster.Node, args ...interface{}) error { - name, onePart, anotherPart := extractArgs(args...) + name, onePart, anotherPart := n.extractArgs(args...) log.Infof("apply nemesis %s %s between %+v and %+v", core.NetworkPartition, name, onePart, anotherPart) return n.cli.ApplyNetChaos(&chaosv1alpha1.NetworkChaos{ ObjectMeta: metav1.ObjectMeta{ @@ -107,7 +107,7 @@ func (n networkPartition) Invoke(ctx context.Context, _ *cluster.Node, args ...i } func (n networkPartition) Recover(ctx context.Context, _ *cluster.Node, args ...interface{}) error { - name, onePart, anotherPart := extractArgs(args...) + name, onePart, anotherPart := n.extractArgs(args...) log.Infof("unapply nemesis %s %s between %+v and %+v", core.NetworkPartition, name, onePart, anotherPart) return n.cli.CancelNetChaos(&chaosv1alpha1.NetworkChaos{ ObjectMeta: metav1.ObjectMeta{ @@ -123,7 +123,7 @@ func (n networkPartition) Name() string { return string(core.NetworkPartition) } -func extractArgs(args ...interface{}) (string, []cluster.Node, []cluster.Node) { +func (networkPartition) extractArgs(args ...interface{}) (string, []cluster.Node, []cluster.Node) { var name = args[0].(string) var networkParts [][]cluster.Node var onePart []cluster.Node diff --git a/pkg/nemesis/network_partition_single_route.go b/pkg/nemesis/network_partition_single_route.go new file mode 100644 index 000000000..cdf146a5b --- /dev/null +++ b/pkg/nemesis/network_partition_single_route.go @@ -0,0 +1,110 @@ +package nemesis + +import ( + "context" + "fmt" + "time" + + chaosv1alpha1 "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/ngaut/log" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/pingcap/tipocket/pkg/cluster" + "github.com/pingcap/tipocket/pkg/core" +) + +type networkPartitionSingleRouteGenerator struct { + sourceNS string + sourcePod string + targetNS string + targetPod string + direction chaosv1alpha1.Direction +} + +func NewNetworkPartitionSingleRoute( + sourceNS string, sourcePod string, targetNS string, targetPod string, direction chaosv1alpha1.Direction, +) core.NemesisGenerator { + return &networkPartitionSingleRouteGenerator{ + sourceNS: sourceNS, + sourcePod: sourcePod, + targetNS: targetNS, + targetPod: targetPod, + direction: direction, + } +} + +func (g *networkPartitionSingleRouteGenerator) Generate(nodes []cluster.Node) []*core.NemesisOperation { + name := fmt.Sprintf("%s-%s-%s", g.sourceNS, core.NetworkPartitionSingleRoute, randK8sObjectName()) + return []*core.NemesisOperation{{ + Type: core.NetworkPartitionSingleRoute, + InvokeArgs: []interface{}{name, g.sourceNS, g.sourcePod, g.targetNS, g.targetPod, g.direction}, + RecoverArgs: []interface{}{name, g.sourceNS, g.sourcePod, g.targetNS, g.targetPod, g.direction}, + RunTime: time.Hour * 10000, // Never recover before recovering manually + }} +} + +func (g *networkPartitionSingleRouteGenerator) Name() string { + return "partition_route" +} + +type networkPartitionSingleRoute struct { + k8sNemesisClient +} + +func (n networkPartitionSingleRoute) Invoke(ctx context.Context, _ *cluster.Node, args ...interface{}) error { + name, sourceNS, sourcePod, targetNS, targetPod, direction := n.extractArgs(args...) + log.Infof("apply nemesis %s %s between %v/%v and %v/%v (direction: %v)", + core.NetworkPartitionSingleRoute, name, sourceNS, sourcePod, targetNS, targetPod, direction) + return n.cli.ApplyNetChaos(&chaosv1alpha1.NetworkChaos{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: sourceNS, + }, + Spec: n.chaosSpec(sourceNS, sourcePod, targetNS, targetPod, direction), + }) +} + +func (n networkPartitionSingleRoute) Recover(ctx context.Context, _ *cluster.Node, args ...interface{}) error { + name, sourceNS, sourcePod, targetNS, targetPod, direction := n.extractArgs(args...) + log.Infof("unapply nemesis %s %s between %v/%v and %v/%v (direction: %v)", + core.NetworkPartitionSingleRoute, name, sourceNS, sourcePod, targetNS, targetPod, direction) + return n.cli.CancelNetChaos(&chaosv1alpha1.NetworkChaos{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: sourceNS, + }, + Spec: n.chaosSpec(sourceNS, sourcePod, targetNS, targetPod, direction), + }) +} + +func (networkPartitionSingleRoute) Name() string { + return string(core.NetworkPartitionSingleRoute) +} + +func (networkPartitionSingleRoute) extractArgs(args ...interface{}) (name, sourceNS, sourcePod, targetNS, targetPod string, direction chaosv1alpha1.Direction) { + if len(args) != 6 { + log.Fatalf("wrong number of arguments for %v, got %+v", core.NetworkPartitionSingleRoute, args) + } + return args[0].(string), args[1].(string), args[2].(string), args[3].(string), args[4].(string), args[5].(chaosv1alpha1.Direction) +} + +func (networkPartitionSingleRoute) chaosSpec(sourceNS, sourcePod, targetNS, targetPod string, direction chaosv1alpha1.Direction) chaosv1alpha1.NetworkChaosSpec { + return chaosv1alpha1.NetworkChaosSpec{ + Action: chaosv1alpha1.PartitionAction, + Mode: chaosv1alpha1.AllPodMode, + Selector: chaosv1alpha1.SelectorSpec{ + Pods: map[string][]string{ + sourceNS: {sourcePod}, + }, + }, + Direction: direction, + Target: &chaosv1alpha1.Target{ + TargetSelector: chaosv1alpha1.SelectorSpec{ + Pods: map[string][]string{ + targetNS: {targetPod}, + }, + }, + TargetMode: chaosv1alpha1.AllPodMode, + }, + } +}