diff --git a/test/e2e/framework/assertion.go b/test/e2e/framework/assertion.go index 8f7bfe8a..023bccc2 100644 --- a/test/e2e/framework/assertion.go +++ b/test/e2e/framework/assertion.go @@ -33,6 +33,52 @@ import ( "github.com/apache/apisix-ingress-controller/api/v1alpha1" ) +func GatewayClassMustHaveCondition(t testing.TestingT, cli client.Client, timeout time.Duration, gcNN types.NamespacedName, condition metav1.Condition) { + err := PollUntilGatewayClassMustHaveStatus(cli, timeout, gcNN, func(gc *gatewayv1.GatewayClass) bool { + if err := kubernetes.ConditionsHaveLatestObservedGeneration(gc, gc.Status.Conditions); err != nil { + log.Printf("GatewayClass %s %v", gcNN, err) + return false + } + if findConditionInList(gc.Status.Conditions, condition) { + return true + } + log.Printf("NOT FOUND condition %v in %v", condition, gc.Status.Conditions) + return false + }) + require.NoError(t, err, "waiting for GatewayClass to have condition %+v", condition) +} + +func PollUntilGatewayClassMustHaveStatus(cli client.Client, timeout time.Duration, gcNN types.NamespacedName, f func(gc *gatewayv1.GatewayClass) bool) error { + if err := gatewayv1.Install(cli.Scheme()); err != nil { + return err + } + return genericPollResource(new(gatewayv1.GatewayClass), cli, timeout, gcNN, f) +} + +func GatewayMustHaveCondition(t testing.TestingT, cli client.Client, timeout time.Duration, gwNN types.NamespacedName, condition metav1.Condition) { + err := PollUntilGatewayHaveStatus(cli, timeout, gwNN, func(gw *gatewayv1.Gateway) bool { + if err := kubernetes.ConditionsHaveLatestObservedGeneration(gw, gw.Status.Conditions); err != nil { + log.Printf("Gateway %s %v", gwNN, err) + return false + } + if findConditionInList(gw.Status.Conditions, condition) { + log.Printf("found condition %v in list %v", condition, gw.Status.Conditions) + return true + } else { + log.Printf("NOT FOUND condition %v in %v", condition, gw.Status.Conditions) + return false + } + }) + require.NoError(t, err, "waiting for Gateway to have condition %+v", condition) +} + +func PollUntilGatewayHaveStatus(cli client.Client, timeout time.Duration, gwNN types.NamespacedName, f func(gateway *gatewayv1.Gateway) bool) error { + if err := gatewayv1.Install(cli.Scheme()); err != nil { + return err + } + return genericPollResource(new(gatewayv1.Gateway), cli, timeout, gwNN, f) +} + func HTTPRouteMustHaveCondition(t testing.TestingT, cli client.Client, timeout time.Duration, refNN, hrNN types.NamespacedName, condition metav1.Condition) { err := PollUntilHTTPRouteHaveStatus(cli, timeout, hrNN, func(hr *gatewayv1.HTTPRoute) bool { for _, parent := range hr.Status.Parents { diff --git a/test/e2e/gatewayapi/httproute.go b/test/e2e/gatewayapi/httproute.go index 69d0584d..63844e02 100644 --- a/test/e2e/gatewayapi/httproute.go +++ b/test/e2e/gatewayapi/httproute.go @@ -19,12 +19,15 @@ import ( "strings" "time" + "github.com/gavv/httpexpect/v2" "github.com/gruntwork-io/terratest/modules/retry" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/gateway-api/apis/v1alpha2" "github.com/apache/apisix-ingress-controller/api/v1alpha1" @@ -101,26 +104,6 @@ spec: kind: GatewayProxy name: apisix-proxy-config ` - - var ResourceApplied = func(resourType, resourceName, resourceRaw string, observedGeneration int) { - Expect(s.CreateResourceFromString(resourceRaw)). - NotTo(HaveOccurred(), fmt.Sprintf("creating %s", resourType)) - - Eventually(func() string { - hryaml, err := s.GetResourceYaml(resourType, resourceName) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("getting %s yaml", resourType)) - return hryaml - }, "8s", "2s"). - Should( - SatisfyAll( - ContainSubstring(`status: "True"`), - ContainSubstring(fmt.Sprintf("observedGeneration: %d", observedGeneration)), - ), - fmt.Sprintf("checking %s condition status", resourType), - ) - time.Sleep(1 * time.Second) - } - var beforeEachHTTP = func() { By("create GatewayProxy") gatewayProxy := fmt.Sprintf(gatewayProxyYaml, s.Deployer.GetAdminEndpoint(), s.AdminKey()) @@ -132,24 +115,30 @@ spec: gatewayClassName := fmt.Sprintf("apisix-%d", time.Now().Unix()) err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, gatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(5 * time.Second) By("check GatewayClass condition") - gcyaml, err := s.GetResourceYaml("GatewayClass", gatewayClassName) - Expect(err).NotTo(HaveOccurred(), "getting GatewayClass yaml") - Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking GatewayClass condition status") - Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), "checking GatewayClass condition message") + framework.GatewayClassMustHaveCondition(s.GinkgoT, s.K8sClient, 8*time.Second, + types.NamespacedName{Namespace: s.Namespace(), Name: gatewayClassName}, + metav1.Condition{ + Type: string(gatewayv1.GatewayClassConditionStatusAccepted), + Status: metav1.ConditionTrue, + Message: "the gatewayclass has been accepted by the apisix-ingress-controller", + }, + ) By("create Gateway") err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGateway, gatewayClassName), s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(5 * time.Second) By("check Gateway condition") - gwyaml, err := s.GetResourceYaml("Gateway", "apisix") - Expect(err).NotTo(HaveOccurred(), "getting Gateway yaml") - Expect(gwyaml).To(ContainSubstring(`status: "True"`), "checking Gateway condition status") - Expect(gwyaml).To(ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controller"), "checking Gateway condition message") + framework.GatewayMustHaveCondition(s.GinkgoT, s.K8sClient, 8*time.Second, + types.NamespacedName{Namespace: s.Namespace(), Name: "apisix"}, + metav1.Condition{ + Type: string(gatewayv1.GatewayConditionAccepted), + Status: metav1.ConditionTrue, + Message: "the gateway has been accepted by the apisix-ingress-controller", + }, + ) } var beforeEachHTTPS = func() { @@ -161,28 +150,35 @@ spec: secretName := _secretName createSecret(s, secretName) + By("create GatewayClass") gatewayClassName := fmt.Sprintf("apisix-%d", time.Now().Unix()) err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, gatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(5 * time.Second) By("check GatewayClass condition") - gcyaml, err := s.GetResourceYaml("GatewayClass", gatewayClassName) - Expect(err).NotTo(HaveOccurred(), "getting GatewayClass yaml") - Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking GatewayClass condition status") - Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), "checking GatewayClass condition message") + framework.GatewayClassMustHaveCondition(s.GinkgoT, s.K8sClient, 8*time.Second, + types.NamespacedName{Namespace: s.Namespace(), Name: gatewayClassName}, + metav1.Condition{ + Type: string(gatewayv1.GatewayClassConditionStatusAccepted), + Status: metav1.ConditionTrue, + Message: "the gatewayclass has been accepted by the apisix-ingress-controller", + }, + ) By("create Gateway") err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGatewayHTTPS, gatewayClassName), s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(5 * time.Second) By("check Gateway condition") - gwyaml, err := s.GetResourceYaml("Gateway", "apisix") - Expect(err).NotTo(HaveOccurred(), "getting Gateway yaml") - Expect(gwyaml).To(ContainSubstring(`status: "True"`), "checking Gateway condition status") - Expect(gwyaml).To(ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controller"), "checking Gateway condition message") + framework.GatewayMustHaveCondition(s.GinkgoT, s.K8sClient, 8*time.Second, + types.NamespacedName{Namespace: s.Namespace(), Name: "apisix"}, + metav1.Condition{ + Type: string(gatewayv1.GatewayConditionAccepted), + Status: metav1.ConditionTrue, + Message: "the gateway has been accepted by the apisix-ingress-controller", + }, + ) } Context("HTTPRoute with HTTPS Gateway", func() { var exactRouteByGet = ` @@ -207,26 +203,24 @@ spec: BeforeEach(beforeEachHTTPS) - It("Create/Updtea/Delete HTTPRoute", func() { + It("Create/Update/Delete HTTPRoute", func() { + request := func() int { + return s.NewAPISIXHttpsClient("api6.com"). + GET("/get"). + WithHost("api6.com"). + Expect(). + Raw().StatusCode + } + By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, exactRouteByGet, func(_ context.Context) (ok bool, err error) { + return request() == http.StatusOK, nil + }) - By("access dataplane to check the HTTPRoute") - s.NewAPISIXHttpsClient("api6.com"). - GET("/get"). - WithHost("api6.com"). - Expect(). - Status(200) By("delete HTTPRoute") err := s.DeleteResourceFromString(exactRouteByGet) Expect(err).NotTo(HaveOccurred(), "deleting HTTPRoute") - time.Sleep(5 * time.Second) - - s.NewAPISIXHttpsClient("api6.com"). - GET("/get"). - WithHost("api6.com"). - Expect(). - Status(404) + Eventually(request).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound)) }) }) @@ -315,12 +309,16 @@ spec: additionalGatewayClassName = fmt.Sprintf("apisix-%d", time.Now().Unix()) err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, additionalGatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating additional GatewayClass") - time.Sleep(5 * time.Second) + By("Check additional GatewayClass condition") - gcyaml, err := s.GetResourceYaml("GatewayClass", additionalGatewayClassName) - Expect(err).NotTo(HaveOccurred(), "getting additional GatewayClass yaml") - Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking additional GatewayClass condition status") - Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), "checking additional GatewayClass condition message") + framework.GatewayClassMustHaveCondition(s.GinkgoT, s.K8sClient, 8*time.Second, + types.NamespacedName{Namespace: s.Namespace(), Name: additionalGatewayClassName}, + metav1.Condition{ + Type: string(gatewayv1.GatewayClassConditionStatusAccepted), + Status: metav1.ConditionTrue, + Message: "the gatewayclass has been accepted by the apisix-ingress-controller", + }, + ) additionalGatewayProxy := fmt.Sprintf(additionalGatewayProxyYaml, s.Deployer.GetAdminEndpoint(resources.DataplaneService), resources.AdminAPIKey) err = s.CreateResourceFromStringWithNamespace(additionalGatewayProxy, additionalNamespace) @@ -336,31 +334,27 @@ spec: }) It("HTTPRoute should be accessible through both gateways", func() { + request := func(client *httpexpect.Expect, host string) int { + return client.GET("/get").WithHost(host).Expect().Raw().StatusCode + } + By("Create HTTPRoute referencing both gateways") multiGatewayRoute := fmt.Sprintf(multiGatewayHTTPRoute, s.Namespace(), additionalNamespace) - ResourceApplied("HTTPRoute", "multi-gateway-route", multiGatewayRoute, 1) + s.ApplyHTTPRoute(types.NamespacedName{Name: "multi-gateway-route"}, multiGatewayRoute) By("Access through default gateway") - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(http.StatusOK) + Eventually(request).WithArguments(s.NewAPISIXClient(), "httpbin.example"). + WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) By("Access through additional gateway") client, err := s.NewAPISIXClientForGateway(additionalGatewayGroupID) Expect(err).NotTo(HaveOccurred(), "creating client for additional gateway") - - client. - GET("/get"). - WithHost("httpbin-additional.example"). - Expect(). - Status(http.StatusOK) + Eventually(request).WithArguments(client, "httpbin-additional.example"). + WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) By("Delete Additional Gateway") err = s.DeleteResourceFromStringWithNamespace(fmt.Sprintf(additionalGateway, additionalGatewayClassName), additionalNamespace) Expect(err).NotTo(HaveOccurred(), "deleting additional Gateway") - time.Sleep(5 * time.Second) By("HTTPRoute should still be accessible through default gateway") s.NewAPISIXClient(). @@ -372,12 +366,8 @@ spec: By("HTTPRoute should not be accessible through additional gateway") client, err = s.NewAPISIXClientForGateway(additionalGatewayGroupID) Expect(err).NotTo(HaveOccurred(), "creating client for additional gateway") - - client. - GET("/get"). - WithHost("httpbin-additional.example"). - Expect(). - Status(http.StatusNotFound) + Eventually(request).WithArguments(client, "httpbin-additional.example"). + WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound)) }) }) @@ -493,70 +483,52 @@ spec: It("Create/Update/Delete HTTPRoute", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, exactRouteByGet) + request := func(host string) int { + if host == "" { + return s.NewAPISIXClient().GET("/get").Expect().Raw().StatusCode + } + return s.NewAPISIXClient().GET("/get").WithHost(host).Expect().Raw().StatusCode + } By("access dataplane to check the HTTPRoute") + Eventually(request).WithArguments("httpbin.example").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) s.NewAPISIXClient(). GET("/get"). Expect(). Status(404) - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(200) - By("delete HTTPRoute") err := s.DeleteResourceFromString(exactRouteByGet) Expect(err).NotTo(HaveOccurred(), "deleting HTTPRoute") - time.Sleep(5 * time.Second) - - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(404) + Eventually(request).WithArguments("httpbin.example").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound)) }) It("Delete Gateway after apply HTTPRoute", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, exactRouteByGet) - By("access daataplane to check the HTTPRoute") - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(200) + request := func() int { + return s.NewAPISIXClient().GET("/get").WithHost("httpbin.example").Expect().Raw().StatusCode + } + By("access dataplane to check the HTTPRoute") + Eventually(request).WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) By("delete Gateway") err := s.DeleteResource("Gateway", "apisix") Expect(err).NotTo(HaveOccurred(), "deleting Gateway") - time.Sleep(5 * time.Second) - - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(404) + Eventually(request).WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound)) }) - It("Proxy External Service", func() { - By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", httprouteWithExternalName, 1) - - By("checking the external service response") - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.external"). - Expect(). - Status(200) + PIt("Proxy External Service", func() { + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, httprouteWithExternalName, func(_ context.Context) (done bool, err error) { + return s.NewAPISIXClient().GET("/get").WithHost("httpbin.external").Expect().Raw().StatusCode == http.StatusOK, nil + }) }) It("Match Port", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", invalidBackendPort, 1) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, invalidBackendPort) serviceResources, err := s.DefaultDataplaneResource().Service().List(context.Background()) Expect(err).NotTo(HaveOccurred(), "listing services") @@ -569,23 +541,23 @@ spec: }) It("Delete HTTPRoute during restart", func() { - By("create HTTPRoute httpbin") - ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) - - By("create HTTPRoute httpbin2") - ResourceApplied("HTTPRoute", "httpbin2", exactRouteByGet2, 1) - - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(200) + By("create HTTPRoute httpbin and httpbin2") + var ( + httpbinNN = types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"} + httpbin2NN = types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin2"} + ) + for nn, spec := range map[types.NamespacedName]string{ + httpbinNN: exactRouteByGet, + httpbin2NN: exactRouteByGet2, + } { + s.ApplyHTTPRoute(nn, spec) + } - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin2.example"). - Expect(). - Status(200) + request := func(host string) int { + return s.NewAPISIXClient().GET("/get").WithHost(host).Expect().Raw().StatusCode + } + Eventually(request).WithArguments("httpbin.example").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) + Eventually(request).WithArguments("httpbin2.example").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) s.Deployer.ScaleIngress(0) @@ -727,81 +699,57 @@ spec: It("HTTPRoute Exact Match", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) - - By("access daataplane to check the HTTPRoute") - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(200) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, exactRouteByGet) - s.NewAPISIXClient(). - GET("/get/xxx"). - WithHost("httpbin.example"). - Expect(). - Status(404) + request := func(uri string) int { + return s.NewAPISIXClient().GET(uri).WithHost("httpbin.example").Expect().Raw().StatusCode + } + By("access dataplane to check the HTTPRoute") + Eventually(request).WithArguments("/get").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) + Eventually(request).WithArguments("/get/xxx").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound)) }) It("HTTPRoute Prefix Match", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", prefixRouteByStatus, 1) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, prefixRouteByStatus) - By("access daataplane to check the HTTPRoute") - s.NewAPISIXClient(). - GET("/status/200"). - WithHost("httpbin.example"). - Expect(). - Status(200) - - s.NewAPISIXClient(). - GET("/status/201"). - WithHost("httpbin.example"). - Expect(). - Status(201) + request := func(uri string) int { + return s.NewAPISIXClient().GET(uri).WithHost("httpbin.example").Expect().Raw().StatusCode + } + By("access dataplane to check the HTTPRoute") + Eventually(request).WithArguments("/status/200").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) + Eventually(request).WithArguments("/status/201").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusCreated)) }) It("HTTPRoute Method Match", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", methodRouteGETAndDELETEByAnything, 1) - - By("access daataplane to check the HTTPRoute") - s.NewAPISIXClient(). - GET("/anything"). - WithHost("httpbin.example"). - Expect(). - Status(200) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, methodRouteGETAndDELETEByAnything) - s.NewAPISIXClient(). - DELETE("/anything"). - WithHost("httpbin.example"). - Expect(). - Status(200) - - s.NewAPISIXClient(). - POST("/anything"). - WithHost("httpbin.example"). - Expect(). - Status(404) + request := func(method string) int { + return s.NewAPISIXClient().Request(method, "/anything").WithHost("httpbin.example").Expect().Raw().StatusCode + } + By("access dataplane to check the HTTPRoute") + Eventually(request).WithArguments(http.MethodGet).WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) + Eventually(request).WithArguments(http.MethodDelete).WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) + Eventually(request).WithArguments(http.MethodPost).WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound)) }) It("HTTPRoute Vars Match", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", varsRoute, 1) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, varsRoute) + request := func(headers map[string]string) int { + cli := s.NewAPISIXClient().GET("/get").WithHost("httpbin.example") + for k, v := range headers { + cli = cli.WithHeader(k, v) + } + return cli.Expect().Raw().StatusCode + } By("access dataplane to check the HTTPRoute") - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(http.StatusNotFound) - - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - WithHeader("X-Route-Name", "httpbin"). - Expect(). - Status(http.StatusOK) + Eventually(request).WithArguments(map[string]string{}). + WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound)) + Eventually(request).WithArguments(map[string]string{"X-Route-Name": "httpbin"}). + WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) }) It("HTTPRoutePolicy in effect", func() { @@ -1326,9 +1274,17 @@ spec: It("HTTPRoute RequestHeaderModifier", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", reqHeaderModifyByHeaders, 1) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, reqHeaderModifyByHeaders, func(_ context.Context) (done bool, err error) { + return s.NewAPISIXClient(). + GET("/headers"). + WithHost("httpbin.example"). + WithHeader("X-Req-Add", "test"). + WithHeader("X-Req-Removed", "test"). + WithHeader("X-Req-Set", "test"). + Expect().Raw().StatusCode == http.StatusOK, nil + }) - By("access daataplane to check the HTTPRoute") + By("access dataplane to check the HTTPRoute") respExp := s.NewAPISIXClient(). GET("/headers"). WithHost("httpbin.example"). @@ -1347,9 +1303,14 @@ spec: It("HTTPRoute ResponseHeaderModifier", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", respHeaderModifyByHeaders, 1) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, respHeaderModifyByHeaders, func(_ context.Context) (done bool, err error) { + return s.NewAPISIXClient(). + GET("/headers"). + WithHost("httpbin.example"). + Expect().Raw().StatusCode == http.StatusOK, nil + }) - By("access daataplane to check the HTTPRoute") + By("access dataplane to check the HTTPRoute") respExp := s.NewAPISIXClient(). GET("/headers"). WithHost("httpbin.example"). @@ -1367,7 +1328,11 @@ spec: It("HTTPRoute RequestRedirect", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", httpsRedirectByHeaders, 1) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, httpsRedirectByHeaders, func(_ context.Context) (done bool, err error) { + return s.NewAPISIXClient().GET("/headers"). + WithHeader("Host", "httpbin.example"). + Expect().Raw().Header.Get("Location") == "https://httpbin.example:9443/headers", nil + }) s.NewAPISIXClient().GET("/headers"). WithHeader("Host", "httpbin.example"). @@ -1376,7 +1341,11 @@ spec: Header("Location").IsEqual("https://httpbin.example:9443/headers") By("update HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", hostnameRedirectByHeaders, 2) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, hostnameRedirectByHeaders, func(_ context.Context) (done bool, err error) { + return s.NewAPISIXClient().GET("/headers"). + WithHeader("Host", "httpbin.example"). + Expect().Raw().Header.Get("Location") == "http://httpbin.org/headers", nil + }) s.NewAPISIXClient().GET("/headers"). WithHeader("Host", "httpbin.example"). @@ -1385,8 +1354,8 @@ spec: Header("Location").IsEqual("http://httpbin.org/headers") }) - It("HTTPRoute RequestMirror", func() { - echoRoute := ` + PIt("HTTPRoute RequestMirror", func() { + const echoService = ` apiVersion: apps/v1 kind: Deployment metadata: @@ -1419,7 +1388,9 @@ spec: port: 80 protocol: TCP targetPort: 8080 ---- +` + + const echoRoute = ` apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: @@ -1444,14 +1415,22 @@ spec: - name: httpbin-service-e2e-test port: 80 ` - ResourceApplied("HTTPRoute", "httpbin", echoRoute, 1) + // apply echo server Deployment and Service + err := s.CreateResourceFromString(echoService) + Expect(err).NotTo(HaveOccurred(), "create echo service") + err = framework.WaitPodsAvailable(s.GinkgoT, s.KubectlOpts(), metav1.ListOptions{ + LabelSelector: "app=echo", + TimeoutSeconds: ptr.To(int64(10)), + }) + Expect(err).NotTo(HaveOccurred(), "wait for echo available") - time.Sleep(time.Second * 6) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, echoRoute, func(_ context.Context) (done bool, err error) { + return s.NewAPISIXClient().GET("/headers"). + WithHeader("Host", "httpbin.example"). + Expect().Raw().StatusCode == http.StatusOK, nil + }) - _ = s.NewAPISIXClient().GET("/headers"). - WithHeader("Host", "httpbin.example"). - Expect(). - Status(http.StatusOK) + time.Sleep(time.Second) echoLogs := s.GetDeploymentLogs("echo") Expect(echoLogs).To(ContainSubstring("GET /headers")) @@ -1459,7 +1438,18 @@ spec: It("HTTPRoute URLRewrite with ReplaceFullPath And Hostname", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", replaceFullPathAndHost, 1) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, replaceFullPathAndHost, + func(_ context.Context) (done bool, err error) { + return s.NewAPISIXClient().GET("/replace/201"). + WithHeader("Host", "httpbin.example"). + Expect().Raw().StatusCode == http.StatusOK, nil + }, + func(_ context.Context) (done bool, err error) { + return s.NewAPISIXClient().GET("/replace/500"). + WithHeader("Host", "httpbin.example"). + Expect().Raw().StatusCode == http.StatusOK, nil + }, + ) By("/replace/201 should be rewritten to /headers") s.NewAPISIXClient().GET("/replace/201"). @@ -1480,26 +1470,29 @@ spec: It("HTTPRoute URLRewrite with ReplacePrefixMatch", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", replacePrefixMatch, 1) - - By("/replace/201 should be rewritten to /status/201") - s.NewAPISIXClient().GET("/replace/201"). - WithHeader("Host", "httpbin.example"). - Expect(). - Status(http.StatusCreated) - - By("/replace/500 should be rewritten to /status/500") - s.NewAPISIXClient().GET("/replace/500"). - WithHeader("Host", "httpbin.example"). - Expect(). - Status(http.StatusInternalServerError) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, replacePrefixMatch, + func(_ context.Context) (done bool, err error) { + return s.NewAPISIXClient().GET("/replace/201"). + WithHeader("Host", "httpbin.example"). + Expect().Raw().StatusCode == http.StatusCreated, nil + }, + func(_ context.Context) (done bool, err error) { + return s.NewAPISIXClient().GET("/replace/500"). + WithHeader("Host", "httpbin.example"). + Expect().Raw().StatusCode == http.StatusInternalServerError, nil + }, + ) }) It("HTTPRoute ExtensionRef", func() { By("create HTTPRoute") err := s.CreateResourceFromString(echoPlugin) Expect(err).NotTo(HaveOccurred(), "creating PluginConfig") - ResourceApplied("HTTPRoute", "httpbin", extensionRefEchoPlugin, 1) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, extensionRefEchoPlugin, func(_ context.Context) (done bool, err error) { + return s.NewAPISIXClient().GET("/get"). + WithHeader("Host", "httpbin.example"). + Expect().Raw().StatusCode == http.StatusOK, nil + }) s.NewAPISIXClient().GET("/get"). WithHeader("Host", "httpbin.example"). @@ -1509,18 +1502,16 @@ spec: err = s.CreateResourceFromString(echoPluginUpdated) Expect(err).NotTo(HaveOccurred(), "updating PluginConfig") - time.Sleep(5 * time.Second) - - s.NewAPISIXClient().GET("/get"). - WithHeader("Host", "httpbin.example"). - Expect(). - Body(). - Contains("Updated") + Eventually(func() string { + return s.NewAPISIXClient().GET("/get"). + WithHeader("Host", "httpbin.example"). + Expect().Body().Raw() + }).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(ContainSubstring("Updated")) }) }) Context("HTTPRoute Multiple Backend", func() { - var sameWeiht = ` + var sameWeight = ` apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: @@ -1543,7 +1534,7 @@ spec: port: 80 weight: 50 ` - var oneWeiht = ` + var oneWeight = ` apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: @@ -1574,7 +1565,11 @@ spec: }) }) It("HTTPRoute Canary", func() { - ResourceApplied("HTTPRoute", "httpbin", sameWeiht, 1) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, sameWeight, func(_ context.Context) (done bool, err error) { + return s.NewAPISIXClient().GET("/get"). + WithHeader("Host", "httpbin.example"). + Expect().Raw().StatusCode == http.StatusOK, nil + }) var ( hitNginxCnt = 0 @@ -1595,7 +1590,21 @@ spec: } Expect(hitNginxCnt - hitHttpbinCnt).To(BeNumerically("~", 0, 2)) - ResourceApplied("HTTPRoute", "httpbin", oneWeiht, 2) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, oneWeight) + + var cnt int + for cnt < 5 { + body := s.NewAPISIXClient().GET("/get"). + WithHeader("Host", "httpbin.example"). + Expect(). + Status(http.StatusOK). + Body().Raw() + if strings.Contains(body, "Hello") { + cnt = 0 + } else { + cnt++ + } + } hitNginxCnt = 0 hitHttpbinCnt = 0 @@ -1660,7 +1669,12 @@ spec: It("Should sync HTTPRoute when GatewayProxy is updated", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) + s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, exactRouteByGet, func(_ context.Context) (done bool, err error) { + return s.NewAPISIXClient(). + GET("/get"). + WithHost("httpbin.example"). + Expect().Raw().StatusCode == http.StatusOK, nil + }) By("verify HTTPRoute works") s.NewAPISIXClient(). @@ -1691,14 +1705,14 @@ spec: updatedProxy := fmt.Sprintf(updatedGatewayProxy, s.Deployer.GetAdminEndpoint(resources.DataplaneService), resources.AdminAPIKey) err = s.CreateResourceFromString(updatedProxy) Expect(err).NotTo(HaveOccurred(), "updating GatewayProxy") - time.Sleep(5 * time.Second) By("verify HTTPRoute works for additional gateway group") - client. - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(200) + Eventually(func() int { + return client. + GET("/get"). + WithHost("httpbin.example"). + Expect().Raw().StatusCode + }).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) }) }) diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index 2121bd89..94826180 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -395,3 +395,11 @@ func (s *Scaffold) GetGatewayHTTPSEndpoint(identifier string) (string, error) { return resources.HttpsTunnel.Endpoint(), nil } + +func (s *Scaffold) CurrentGatewayGroupID() string { + return s.gatewaygroupid +} + +func (s *Scaffold) KubectlOpts() *k8s.KubectlOptions { + return s.kubectlOptions +}