diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/TestAssertions.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/TestAssertions.java index 6d679a8918..f8a7849dc4 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/TestAssertions.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/TestAssertions.java @@ -23,9 +23,6 @@ import org.assertj.core.api.Assertions; import org.awaitility.Awaitility; -import reactor.netty.http.client.HttpClient; -import reactor.util.retry.Retry; -import reactor.util.retry.RetryBackoffSpec; import org.springframework.boot.test.system.CapturedOutput; import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; @@ -34,11 +31,12 @@ import org.springframework.core.ParameterizedTypeReference; import org.springframework.core.ResolvableType; import org.springframework.http.HttpMethod; -import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.web.reactive.function.client.WebClient; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; +import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.builder; +import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.retrySpec; /** * @author wind57 @@ -116,12 +114,4 @@ static void invokeAndAssert(Util util, Set namespaces, int port, String } - private static WebClient.Builder builder() { - return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create())); - } - - private static RetryBackoffSpec retrySpec() { - return Retry.fixedDelay(15, Duration.ofSeconds(1)).filter(Objects::nonNull); - } - } diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/discovery/TestAssertions.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/discovery/TestAssertions.java index 64ca0ec8bb..278e4fa059 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/discovery/TestAssertions.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/discovery/TestAssertions.java @@ -22,10 +22,6 @@ import java.util.Map; import java.util.Objects; -import reactor.netty.http.client.HttpClient; -import reactor.util.retry.Retry; -import reactor.util.retry.RetryBackoffSpec; - import org.springframework.boot.test.json.BasicJsonTester; import org.springframework.boot.test.system.CapturedOutput; import org.springframework.cloud.client.ServiceInstance; @@ -33,12 +29,13 @@ import org.springframework.cloud.client.discovery.ReactiveDiscoveryClient; import org.springframework.cloud.kubernetes.commons.discovery.DefaultKubernetesServiceInstance; import org.springframework.http.HttpMethod; -import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.web.reactive.function.client.WebClient; import static java.util.AbstractMap.SimpleEntry; import static java.util.Map.Entry; import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.builder; +import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.retrySpec; import static org.testcontainers.shaded.org.awaitility.Awaitility.await; /** @@ -321,12 +318,4 @@ private static void waitForLogStatement(CapturedOutput output, String message) { .until(() -> output.getOut().contains(message)); } - private static WebClient.Builder builder() { - return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create())); - } - - private static RetryBackoffSpec retrySpec() { - return Retry.fixedDelay(15, Duration.ofSeconds(1)).filter(Objects::nonNull); - } - } diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-istio/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/istio/Fabric8IstioIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-istio/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/istio/Fabric8IstioIT.java index ad66be0974..d812d36471 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-istio/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/istio/Fabric8IstioIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-istio/src/test/java/org/springframework/cloud/kubernetes/fabric8/client/istio/Fabric8IstioIT.java @@ -17,9 +17,7 @@ package org.springframework.cloud.kubernetes.fabric8.client.istio; import java.io.InputStream; -import java.time.Duration; import java.util.List; -import java.util.Objects; import io.fabric8.kubernetes.api.model.Service; import io.fabric8.kubernetes.api.model.apps.Deployment; @@ -31,18 +29,17 @@ import org.junit.jupiter.api.Test; import org.testcontainers.containers.Container; import org.testcontainers.k3s.K3sContainer; -import reactor.netty.http.client.HttpClient; -import reactor.util.retry.Retry; -import reactor.util.retry.RetryBackoffSpec; import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; import org.springframework.cloud.kubernetes.integration.tests.commons.Images; import org.springframework.cloud.kubernetes.integration.tests.commons.Phase; import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Util; import org.springframework.http.HttpMethod; -import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.web.reactive.function.client.WebClient; +import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.builder; +import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.retrySpec; + /** * @author wind57 */ @@ -133,14 +130,6 @@ private static void appManifests(Phase phase) { } - private WebClient.Builder builder() { - return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create())); - } - - private RetryBackoffSpec retrySpec() { - return Retry.fixedDelay(15, Duration.ofSeconds(1)).filter(Objects::nonNull); - } - private static String istioctlPodName() { try { return K3S diff --git a/spring-cloud-kubernetes-test-support/pom.xml b/spring-cloud-kubernetes-test-support/pom.xml index dfd948ce63..9d9fcd18ab 100644 --- a/spring-cloud-kubernetes-test-support/pom.xml +++ b/spring-cloud-kubernetes-test-support/pom.xml @@ -22,6 +22,10 @@ org.springframework.boot spring-boot-starter-test + + org.springframework.boot + spring-boot-starter-webflux + io.kubernetes client-java-extended diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java index c6221e855f..7a4b0e7999 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java @@ -26,6 +26,7 @@ import java.time.Duration; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -38,10 +39,15 @@ import org.junit.jupiter.api.Assertions; import org.testcontainers.containers.Container; import org.testcontainers.k3s.K3sContainer; +import reactor.netty.http.client.HttpClient; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; import org.springframework.core.io.ClassPathResource; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.util.StreamUtils; import org.springframework.util.StringUtils; +import org.springframework.web.reactive.function.client.WebClient; import static org.awaitility.Awaitility.await; import static org.springframework.cloud.kubernetes.integration.tests.commons.Constants.KUBERNETES_VERSION_FILE; @@ -264,6 +270,14 @@ public static void waitForLogStatement(String message, K3sContainer k3sContainer } + public static WebClient.Builder builder() { + return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create())); + } + + public static RetryBackoffSpec retrySpec() { + return Retry.fixedDelay(15, Duration.ofSeconds(1)).filter(Objects::nonNull); + } + private static void loadImageFromPath(String tarName, K3sContainer container) { await().atMost(Duration.ofMinutes(2)).pollInterval(Duration.ofSeconds(1)).until(() -> { Container.ExecResult result = container.execInContainer("ctr", "i", "import", TMP_IMAGES + "/" + tarName); diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java index 05c275dbaa..aa877a7fc3 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/FixedPortsK3sContainer.java @@ -48,7 +48,7 @@ final class FixedPortsK3sContainer extends K3sContainer { * Command to use when starting rancher. Without "server" option, traefik is not * installed */ - private static final String RANCHER_COMMAND = "server"; + private static final String RANCHER_COMMAND = "server --disable=metric-server"; static final K3sContainer CONTAINER = new FixedPortsK3sContainer(DockerImageName.parse(RANCHER_VERSION)) .configureFixedPorts() diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/fabric8_client/Util.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/fabric8_client/Util.java index 8d98a078e0..0a75f60dfc 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/fabric8_client/Util.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/fabric8_client/Util.java @@ -20,10 +20,9 @@ import java.time.Duration; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.Set; import java.util.concurrent.TimeUnit; +import io.fabric8.kubernetes.api.model.APIService; import io.fabric8.kubernetes.api.model.ConfigMap; import io.fabric8.kubernetes.api.model.NamespaceBuilder; import io.fabric8.kubernetes.api.model.Pod; @@ -32,17 +31,13 @@ import io.fabric8.kubernetes.api.model.Service; import io.fabric8.kubernetes.api.model.ServiceAccount; import io.fabric8.kubernetes.api.model.apps.Deployment; -import io.fabric8.kubernetes.api.model.apps.DeploymentList; import io.fabric8.kubernetes.api.model.networking.v1.Ingress; import io.fabric8.kubernetes.api.model.networking.v1.IngressLoadBalancerIngress; -import io.fabric8.kubernetes.api.model.rbac.ClusterRole; import io.fabric8.kubernetes.api.model.rbac.Role; import io.fabric8.kubernetes.api.model.rbac.RoleBinding; import io.fabric8.kubernetes.client.Config; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientBuilder; -import io.fabric8.kubernetes.client.dsl.base.PatchContext; -import io.fabric8.kubernetes.client.dsl.base.PatchType; import io.fabric8.kubernetes.client.utils.Serialization; import jakarta.annotation.Nullable; import org.apache.commons.logging.Log; @@ -53,7 +48,6 @@ import org.springframework.cloud.kubernetes.integration.tests.commons.Phase; import static org.awaitility.Awaitility.await; -import static org.junit.jupiter.api.Assertions.fail; import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.loadImage; import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.pomVersion; import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.pullImage; @@ -75,6 +69,10 @@ public Util(K3sContainer container) { .build(); } + public KubernetesClient client() { + return client; + } + /** * This is the preferred method to use when creating a deployment alongside with a * service. It creates the given resources as-well as waits for them to be created. @@ -210,6 +208,21 @@ public void createNamespace(String name) { public void deleteNamespace(String name) { try { + + // sometimes we get errors like : + + // "message": "Discovery failed for some groups, + // 1 failing: unable to retrieve the complete list of server APIs: + // metrics.k8s.io/v1beta1: stale GroupVersion discovery: metrics.k8s.io/v1beta1" + + // but even when it works OK, the finalizers are slowing down the deletion + List apiServices = client.apiServices().list().getItems(); + apiServices.stream() + .map(apiService -> apiService.getMetadata().getName()) + .filter(apiServiceName -> apiServiceName.contains("metrics.k8s.io")) + .findFirst() + .ifPresent(apiServiceName -> client.apiServices().withName(apiServiceName).delete()); + client.namespaces() .resource(new NamespaceBuilder().withNewMetadata().withName(name).and().build()) .delete(); @@ -228,40 +241,6 @@ public void deleteNamespace(String name) { } - public void setUpClusterWide(String serviceAccountNamespace, Set namespaces) { - InputStream clusterRoleBindingAsStream = inputStream("cluster/cluster-role.yaml"); - InputStream serviceAccountAsStream = inputStream("cluster/service-account.yaml"); - InputStream roleBindingAsStream = inputStream("cluster/role-binding.yaml"); - - ClusterRole clusterRole = client.rbac().clusterRoles().load(clusterRoleBindingAsStream).item(); - if (client.rbac().clusterRoles().withName(clusterRole.getMetadata().getName()).get() == null) { - client.rbac().clusterRoles().resource(clusterRole).create(); - } - - ServiceAccount serviceAccountFromStream = client.serviceAccounts().load(serviceAccountAsStream).item(); - serviceAccountFromStream.getMetadata().setNamespace(serviceAccountNamespace); - if (client.serviceAccounts() - .inNamespace(serviceAccountNamespace) - .withName(serviceAccountFromStream.getMetadata().getName()) - .get() == null) { - client.serviceAccounts().inNamespace(serviceAccountNamespace).resource(serviceAccountFromStream).create(); - } - - RoleBinding roleBindingFromStream = client.rbac().roleBindings().load(roleBindingAsStream).item(); - namespaces.forEach(namespace -> { - roleBindingFromStream.getMetadata().setNamespace(namespace); - - if (client.rbac() - .roleBindings() - .inNamespace(namespace) - .withName(roleBindingFromStream.getMetadata().getName()) - .get() == null) { - client.rbac().roleBindings().inNamespace(namespace).resource(roleBindingFromStream).create(); - } - }); - - } - public void createAndWait(String namespace, @Nullable ConfigMap configMap, @Nullable Secret secret) { if (configMap != null) { client.configMaps().resource(configMap).create(); @@ -469,59 +448,6 @@ public void waitForIngress(String namespace, Ingress ingress) { } - public void patchWithReplace(String imageName, String deploymentName, String namespace, String patchBody, - Map labels) { - String body = patchBody.replace("image_name_here", imageName); - - client.apps() - .deployments() - .inNamespace(namespace) - .withName(deploymentName) - .patch(PatchContext.of(PatchType.JSON_MERGE), body); - - waitForDeploymentAfterPatch(deploymentName, namespace, labels); - } - - private void waitForDeploymentAfterPatch(String deploymentName, String namespace, Map labels) { - try { - await().pollDelay(Duration.ofSeconds(4)) - .pollInterval(Duration.ofSeconds(3)) - .atMost(60, TimeUnit.SECONDS) - .until(() -> isDeploymentReadyAfterPatch(deploymentName, namespace, labels)); - } - catch (Exception e) { - throw new RuntimeException(e); - } - - } - - private boolean isDeploymentReadyAfterPatch(String deploymentName, String namespace, Map labels) { - - DeploymentList deployments = client.apps().deployments().inNamespace(namespace).list(); - - if (deployments.getItems().isEmpty()) { - fail("No deployment with name " + deploymentName); - } - - Deployment deployment = deployments.getItems() - .stream() - .filter(x -> x.getMetadata().getName().equals(deploymentName)) - .findFirst() - .orElseThrow(); - // if no replicas are defined, it means only 1 is needed - int replicas = Optional.ofNullable(deployment.getSpec().getReplicas()).orElse(1); - - int numberOfPods = client.pods().inNamespace(namespace).withLabels(labels).list().getItems().size(); - - if (numberOfPods != replicas) { - LOG.info("number of pods not yet stabilized"); - return false; - } - - return replicas == Optional.ofNullable(deployment.getStatus().getReadyReplicas()).orElse(0); - - } - private void innerSetup(String namespace, InputStream serviceAccountAsStream, InputStream roleBindingAsStream, InputStream roleAsStream) { ServiceAccount serviceAccountFromStream = client.serviceAccounts() @@ -574,8 +500,4 @@ private String secretName(Secret secret) { return secret.getMetadata().getName(); } - public KubernetesClient client() { - return client; - } - }