(WIP)feat: long term stability test

Makefile

@@ -92,6 +92,15 @@ e2e-test:
 	@kind get kubeconfig --name $(KIND_NAME) > $$KUBECONFIG
 	DASHBOARD_VERSION=$(DASHBOARD_VERSION) go test ./test/e2e/ -test.timeout=$(TEST_TIMEOUT) -v -ginkgo.v -ginkgo.focus="$(TEST_FOCUS)"
 
+.PHONY: kind-lts-test
+kind-lts-test: kind-up build-image kind-load-images lts-test
+
+# lts-test is long-term-stability test
+.PHONY: lts-test
+lts-test:
+	@kind get kubeconfig --name $(KIND_NAME) > $$KUBECONFIG
+	DASHBOARD_VERSION=$(DASHBOARD_VERSION) go test ./test/long_term_stability/ -test.timeout=$(TEST_TIMEOUT) -v -ginkgo.v -ginkgo.focus="$(TEST_FOCUS)"
+
 .PHONY: conformance-test
 conformance-test:
 	DASHBOARD_VERSION=$(DASHBOARD_VERSION) go test -v ./test/conformance -tags=conformance

test/e2e/gatewayapi/httproute.go

@@ -16,7 +16,7 @@ import (
 var _ = Describe("Test HTTPRoute", func() {
 	s := scaffold.NewDefaultScaffold()
 
-	var defautlGatewayClass = `
+	var defaultGatewayClass = `
 apiVersion: gateway.networking.k8s.io/v1
 kind: GatewayClass
 metadata:
@@ -25,7 +25,7 @@ spec:
   controllerName: %s
 `
 
-	var defautlGateway = `
+	var defaultGateway = `
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata:
@@ -37,7 +37,7 @@ spec:
       protocol: HTTP
       port: 80
 `
-	var defautlGatewayHTTPS = `
+	var defaultGatewayHTTPS = `
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata:
@@ -78,7 +78,7 @@ spec:
 	var beforeEachHTTP = func() {
 		By("create GatewayClass")
 		gatewayClassName := fmt.Sprintf("api7-%d", time.Now().Unix())
-		err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName()), "")
+		err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGatewayClass, gatewayClassName, s.GetControllerName()), "")
 		Expect(err).NotTo(HaveOccurred(), "creating GatewayClass")
 		time.Sleep(5 * time.Second)
 
@@ -89,7 +89,7 @@ spec:
 		Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message")
 
 		By("create Gateway")
-		err = s.CreateResourceFromString(fmt.Sprintf(defautlGateway, gatewayClassName))
+		err = s.CreateResourceFromString(fmt.Sprintf(defaultGateway, gatewayClassName))
 		Expect(err).NotTo(HaveOccurred(), "creating Gateway")
 		time.Sleep(5 * time.Second)
 
@@ -105,7 +105,7 @@ spec:
 		createSecret(s, secretName)
 		By("create GatewayClass")
 		gatewayClassName := fmt.Sprintf("api7-%d", time.Now().Unix())
-		err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName()), "")
+		err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGatewayClass, gatewayClassName, s.GetControllerName()), "")
 		Expect(err).NotTo(HaveOccurred(), "creating GatewayClass")
 		time.Sleep(5 * time.Second)
 
@@ -116,7 +116,7 @@ spec:
 		Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message")
 
 		By("create Gateway")
-		err = s.CreateResourceFromString(fmt.Sprintf(defautlGatewayHTTPS, gatewayClassName))
+		err = s.CreateResourceFromString(fmt.Sprintf(defaultGatewayHTTPS, gatewayClassName))
 		Expect(err).NotTo(HaveOccurred(), "creating Gateway")
 		time.Sleep(5 * time.Second)
 

test/e2e/scaffold/httpbin.go

@@ -19,7 +19,7 @@ import (
 	"time"
 
 	"github.com/gruntwork-io/terratest/modules/k8s"
-	ginkgo "github.com/onsi/ginkgo/v2"
+	"github.com/onsi/ginkgo/v2"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -70,6 +70,10 @@ spec:
             - containerPort: 80
               name: "http"
               protocol: "TCP"
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "sleep 5"]              
 `
 	_httpService = `
 apiVersion: v1
@@ -112,7 +116,7 @@ func (s *Scaffold) NewHTTPBINWithNamespace(namespace string) (*corev1.Service, e
 	return s.newHTTPBIN()
 }
 
-// ScaleHTTPBIN scales the number of HTTPBIN pods to desired.
+// ScaleHTTPBIN scales the number of HTTPBIN pods to desire.
 func (s *Scaffold) ScaleHTTPBIN(desired int) error {
 	httpbinDeployment := fmt.Sprintf(s.FormatRegistry(_httpbinDeploymentTemplate), desired)
 	if err := s.CreateResourceFromString(httpbinDeployment); err != nil {

test/e2e/scaffold/k8s.go

@@ -110,6 +110,14 @@ func (s *Scaffold) ListPodsByLabels(labels string) ([]corev1.Pod, error) {
 	})
 }
 
+func (s *Scaffold) DeleteResourcesByLabels(resourceType, label string) error {
+	return k8s.RunKubectlE(s.t, s.kubectlOptions, "delete", resourceType, "-l", label)
+}
+
+func (s *Scaffold) GetResourcesByLabelsOutput(resourceType, label string) (string, error) {
+	return k8s.RunKubectlAndGetOutputE(s.t, s.kubectlOptions, "get", resourceType, "-l", label)
+}
+
 // CreateResourceFromStringWithNamespace creates resource from a loaded yaml string
 // and sets its namespace to the specified one.
 func (s *Scaffold) CreateResourceFromStringWithNamespace(yaml, namespace string) error {

test/e2e/scaffold/locust.go

@@ -0,0 +1,172 @@
+package scaffold
+
+import (
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/gruntwork-io/terratest/modules/k8s"
+	. "github.com/onsi/gomega"
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+)
+
+const (
+	_locustConfigMapTemplate = `
+apiVersion: v1 
+kind: ConfigMap 
+metadata: 
+  name: locust-config 
+data: 
+  locustfile.py: |- 
+    from locust import HttpUser, task, between 
+
+    class HttpbinRequester(HttpUser):
+        @task 
+        def headers(self): 
+            self.client.get("/headers", headers={"Host": "httpbin.example"})
+
+        @task
+        def get(self):
+            self.client.get("/get", headers={"Host": "httpbin.example"})
+
+        @task
+        def post(self):
+            self.client.post("/post", headers={"Host": "httpbin.example"})
+
+        @task
+        def image(self):
+            self.client.image("/image", headers={"Host": "httpbin.example"})
+
+  LOCUST_HOST: http://api7ee3-apisix-gateway-mtls:9080
+  LOCUST_SPAWN_RATE: "50"
+  LOCUST_USERS: "500"
+  LOCUST_AUTOSTART: "true"
+`
+	_locustDeploymentTemplate = `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: locust
+spec:
+  selector:
+    matchLabels:
+      app: locust
+  template:
+    metadata:
+      labels:
+        app: locust
+    spec:
+      containers:
+        - name: locust
+          image: locustio/locust
+          ports:
+            - containerPort: 8089
+          env:
+            - name: LOCUST_HOST
+              valueFrom:
+                configMapKeyRef:
+                  name: locust-config
+                  key: LOCUST_HOST
+            - name: LOCUST_SPAWN_RATE
+              valueFrom:
+                configMapKeyRef:
+                  name: locust-config
+                  key: LOCUST_SPAWN_RATE
+            - name: LOCUST_USERS
+              valueFrom:
+                configMapKeyRef:
+                  name: locust-config
+                  key: LOCUST_USERS
+            - name: LOCUST_AUTOSTART
+              valueFrom:
+                configMapKeyRef:
+                  name: locust-config
+                  key: LOCUST_AUTOSTART
+          volumeMounts:
+            - mountPath: /home/locust
+              name: locust-config
+      volumes:
+        - name: locust-config
+          configMap:
+            name: locust-config
+`
+	_locustServiceTemplate = `
+apiVersion: v1 
+kind: Service 
+metadata: 
+  name: locust 
+spec: 
+  selector: 
+    app: locust 
+  ports: 
+    - name: web
+      port: 8089 
+      targetPort: 8089 
+      protocol: TCP
+  type: ClusterIP `
+)
+
+func (s *Scaffold) DeployLocust() *corev1.Service {
+	// create ConfigMap, Deployment, Service
+	for _, yaml_ := range []string{_locustConfigMapTemplate, _locustDeploymentTemplate, _locustServiceTemplate} {
+		err := s.CreateResourceFromString(yaml_)
+		Expect(err).NotTo(HaveOccurred(), "create resource: %s", yaml_)
+	}
+
+	service, err := k8s.GetServiceE(s.t, s.kubectlOptions, "locust")
+	Expect(err).NotTo(HaveOccurred(), "get service: locust")
+
+	s.EnsureNumEndpointsReady(s.t, service.Name, 1)
+	s.locustTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "locust", 8089, 8089)
+	s.addFinalizers(s.locustTunnel.Close)
+
+	err = s.locustTunnel.ForwardPortE(s.t)
+	Expect(err).NotTo(HaveOccurred(), "port-forward service: locust")
+
+	return service
+}
+
+func (s *Scaffold) ResetLocust() error {
+	if s.locustTunnel == nil {
+		return errors.New("locust is not deployed")
+	}
+	resp, err := http.Get("http://" + s.locustTunnel.Endpoint() + "/stats/reset")
+	if err != nil {
+		return errors.Wrap(err, "failed to request reset locust")
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	if resp.StatusCode != http.StatusOK {
+		return errors.Errorf("request reset locust not OK, status: %s", resp.Status)
+	}
+	return nil
+}
+
+func (s *Scaffold) DownloadLocustReport(filename string) error {
+	if s.locustTunnel == nil {
+		return errors.New("locust is not deployed")
+	}
+	if !strings.EqualFold(filepath.Ext(filename), ".html") {
+		filename += ".html"
+	}
+	_ = os.Remove(filename)
+	resp, err := http.Get("http://" + s.locustTunnel.Endpoint() + "/stats/report?download=1&theme=light")
+	if err != nil {
+		return errors.Wrap(err, "failed to request download report")
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	if resp.StatusCode != http.StatusOK {
+		return errors.Errorf("request download report not OK, status: %s", resp.Status)
+	}
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return errors.Wrap(err, "failed to read report")
+	}
+	return os.WriteFile(filename, data, 0644)
+}

test/e2e/scaffold/scaffold.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/gavv/httpexpect/v2"
 	"github.com/gruntwork-io/terratest/modules/k8s"
+	"github.com/gruntwork-io/terratest/modules/logger"
 	"github.com/gruntwork-io/terratest/modules/testing"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -71,6 +72,11 @@ type Options struct {
 	NamespaceSelectorLabel   map[string][]string
 	DisableNamespaceSelector bool
 	DisableNamespaceLabel    bool
+
+	GinkgoBeforeCallback func(...any) bool
+	GinkgoAfterCallback  func(...any) bool
+
+	KubectlLogger *logger.Logger
 }
 
 type Scaffold struct {
@@ -96,6 +102,7 @@ type Scaffold struct {
 	apisixTCPTunnel        *k8s.Tunnel
 	apisixTLSOverTCPTunnel *k8s.Tunnel
 	apisixUDPTunnel        *k8s.Tunnel
+	locustTunnel           *k8s.Tunnel
 	// apisixControlTunnel    *k8s.Tunnel
 
 }
@@ -158,8 +165,14 @@ func NewScaffold(o *Options) *Scaffold {
 		t:         GinkgoT(),
 	}
 
-	BeforeEach(s.beforeEach)
-	AfterEach(s.afterEach)
+	if o.GinkgoBeforeCallback == nil {
+		o.GinkgoBeforeCallback = BeforeEach
+	}
+	if o.GinkgoAfterCallback == nil {
+		o.GinkgoAfterCallback = AfterEach
+	}
+	o.GinkgoBeforeCallback(s.beforeEach)
+	o.GinkgoAfterCallback(s.afterEach)
 
 	return s
 }
@@ -355,6 +368,9 @@ func (s *Scaffold) beforeEach() {
 		ConfigPath: s.opts.Kubeconfig,
 		Namespace:  s.namespace,
 	}
+	if s.opts.KubectlLogger != nil {
+		s.kubectlOptions.Logger = s.opts.KubectlLogger
+	}
 	if s.opts.ControllerName == "" {
 		s.opts.ControllerName = fmt.Sprintf("%s/%d", DefaultControllerName, time.Now().Nanosecond())
 	}

test/long_term_stability/lts_test.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2024.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package long_term_stability
+
+import (
+	"fmt"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/api7/api7-ingress-controller/test/e2e/framework"
+	_ "github.com/api7/api7-ingress-controller/test/long_term_stability/spec_subjects"
+)
+
+// Run long-term-stability tests using Ginkgo runner.
+func TestLongTermStability(t *testing.T) {
+	RegisterFailHandler(Fail)
+	var f = framework.NewFramework()
+
+	BeforeSuite(f.BeforeSuite)
+	AfterSuite(f.AfterSuite)
+
+	_, _ = fmt.Fprintf(GinkgoWriter, "Starting long-term-stability suite\n")
+	RunSpecs(t, "long-term-stability suite")
+}