Add timeout settings for potentially slow tasks

kokuwaio · Oct 23, 2024 · 54d8d9c · 54d8d9c
1 parent a058dfe
commit 54d8d9c
Show file tree

Hide file tree

Showing 15 changed files with 188 additions and 139 deletions.
diff --git a/docs/goal/image.md b/docs/goal/image.md
@@ -2,12 +2,14 @@
 
 Import images into k3s containerd.
 
-| Name | User Property | Description | Default |
-| -----| ------------- | ----------- | ------- |
-| `ctrImages` | `k3s.ctrImages` | Download given images via `ctr image pull` inside k3s container. | [] |
-| `tarFiles` | `k3s.tarFiles` | Import given tar files via `ctr image import` inside k3s container. | [] |
-| `dockerImages` | `k3s.dockerImages` | Copy given images from docker deamon via `ctr image import` inside k3s container. | [] |
-| `dockerPullAlways` | `k3s.dockerPullAlways` | Always pull docker images or only if not present. | false |
-| `pullTimeout` | `k3s.pullTimeout` |  Timout for `ctr image pull` or `docker pull` in seconds. | 1200 |
-| `skipImage` | `k3s.skipImage` | Skip image handling. | false |
-| `debug` | `k3s.debug` | Stream logs of docker and kubectl. | false |
+| Name | User Property          | Description | Default |
+| -----|------------------------| ----------- |---------|
+| `ctrImages` | `k3s.ctrImages`        | Download given images via `ctr image pull` inside k3s container. | []      |
+| `tarFiles` | `k3s.tarFiles`         | Import given tar files via `ctr image import` inside k3s container. | []      |
+| `dockerImages` | `k3s.dockerImages`     | Copy given images from docker deamon via `ctr image import` inside k3s container. | []      |
+| `dockerPullAlways` | `k3s.dockerPullAlways` | Always pull docker images or only if not present. | false   |
+| `pullTimeout` | `k3s.pullTimeout`      |  Timout for `ctr image pull` or `docker pull` in seconds. | 1200    |
+| `copyToContainerTimeout` | `k3s.copyToContainerTimeout` | Timout for saving image to tar in seconds. | 60      |
+| `saveImageTimeout` | `k3s.saveImageTimeout` | Timout for saving image to tar in seconds. | 60      | 
+| `skipImage` | `k3s.skipImage`        | Skip image handling. | false   |
+| `debug` | `k3s.debug`            | Stream logs of docker and kubectl. | false   |
diff --git a/src/main/java/io/kokuwa/maven/k3s/mojo/ApplyMojo.java b/src/main/java/io/kokuwa/maven/k3s/mojo/ApplyMojo.java
@@ -82,25 +82,26 @@ public void execute() throws MojoExecutionException {
 
 		// verify container and copy manifests
 
-		if (getDocker().getContainer().isEmpty()) {
+		if (getDocker().getContainer(getDefaultTaskTimeout()).isEmpty()) {
 			throw new MojoExecutionException("No k3s container found");
 		}
-		getDocker().copyToContainer(manifests, toLinuxPath(path));
+		getDocker().copyToContainer(manifests, toLinuxPath(path), getDefaultTaskTimeout());
 
 		// wait for service account, see https://github.com/kubernetes/kubernetes/issues/66689
 
 		var serviceAccount = new String[] { "kubectl", "get", "sa", "default", "--ignore-not-found", "--output=name" };
-		if (getDocker().exec(serviceAccount).isEmpty()) {
+		if (getDocker().exec(getDefaultTaskTimeout(), serviceAccount).isEmpty()) {
 			log.info("");
 			log.info("No service account found, waiting for sa ...");
-			Await.await(log, "k3s service account ready").until(() -> !getDocker().exec(serviceAccount).isEmpty());
+			Await.await(log, "k3s service account ready")
+					.until(() -> !getDocker().exec(getDefaultTaskTimeout(), serviceAccount).isEmpty());
 			log.info("Service account found, continue ...");
 			log.info("");
 		}
 
 		// wait for node getting ready
 
-		getDocker().exec("kubectl", "wait", "--for=condition=Ready", "node", "k3s");
+		getDocker().exec(getDefaultTaskTimeout(), "kubectl", "wait", "--for=condition=Ready", "node", "k3s");
 
 		// execute command
 
@@ -180,7 +181,7 @@ private Stream<Entry<String, Callable<Boolean>>> waitFor(Entry<String, List<Stri
 
 			var kind = entry.getKey();
 			var resources = getDocker()
-					.exec("kubectl", "get", kind,
+					.exec(getDefaultTaskTimeout(), "kubectl", "get", kind,
 							"--all-namespaces",
 							"--no-headers",
 							"--output=custom-columns=:.metadata.namespace,:.metadata.name")
@@ -210,7 +211,8 @@ private Stream<Entry<String, Callable<Boolean>>> waitFor(Entry<String, List<Stri
 						log.info("{} {} ... ready", kind, representation);
 						return true;
 					} catch (MojoExecutionException e) {
-						getDocker().exec("kubectl", "get", "--output=yaml", "--namespace=" + namespace, kind, name);
+						getDocker().exec(getDefaultTaskTimeout(), "kubectl", "get", "--output=yaml",
+								"--namespace=" + namespace, kind, name);
 						return false;
 					}
 				});

diff --git a/src/main/java/io/kokuwa/maven/k3s/mojo/ImageMojo.java b/src/main/java/io/kokuwa/maven/k3s/mojo/ImageMojo.java
@@ -72,6 +72,22 @@ public class ImageMojo extends K3sMojo {
 	@Parameter(property = "k3s.pullTimeout", defaultValue = "1200")
 	private Duration pullTimeout;
 
+	/**
+	 * Timout for copying resources to the container in seconds.
+	 *
+	 * @since 1.5.0
+	 */
+	@Parameter(property = "k3s.copyToContainerTimeout", defaultValue = "60")
+	private Duration copyToContainerTimeout;
+
+	/**
+	 * Timout for saving image to tar in seconds.
+	 *
+	 * @since 1.5.0
+	 */
+	@Parameter(property = "k3s.saveImageTimeout", defaultValue = "60")
+	private Duration saveImageTimeout;
+
 	/**
 	 * Skip starting of k3s container.
 	 *
@@ -91,7 +107,7 @@ public void execute() throws MojoExecutionException {
 
 		// verify container
 
-		if (getDocker().getContainer().isEmpty()) {
+		if (getDocker().getContainer(getDefaultTaskTimeout()).isEmpty()) {
 			throw new MojoExecutionException("No k3s container found");
 		}
 
@@ -152,12 +168,13 @@ private boolean tar(Map<String, Map<String, ?>> existingImages, Path tarFile) {
 			var destination = "/tmp/" + tarFile.getFileName() + "_" + System.nanoTime();
 			var outputPattern = Pattern.compile("^unpacking (?<image>.*) \\(sha256:[0-9a-f]{64}\\).*$");
 
-			getDocker().copyToContainer(tarFile, destination);
+			getDocker().copyToContainer(tarFile, destination, copyToContainerTimeout);
 			for (var output : getDocker().exec(pullTimeout, "ctr", "image", "import", destination.toString())) {
 				var matcher = outputPattern.matcher(output);
 				if (matcher.matches()) {
-					getDocker().exec("ctr", "image", "label", matcher.group("image"), labelPath + "=" + tarFile);
-					getDocker().exec("ctr", "image", "label", matcher.group("image"),
+					getDocker().exec(getDefaultTaskTimeout(), "ctr", "image", "label", matcher.group("image"),
+							labelPath + "=" + tarFile);
+					getDocker().exec(getDefaultTaskTimeout(), "ctr", "image", "label", matcher.group("image"),
 							labelChecksum + "=" + newChecksum);
 				} else {
 					log.warn("Tar {} import output cannot be parsed: {}", tarFile, output);
@@ -192,7 +209,7 @@ private boolean docker(Map<String, Map<String, ?>> existingImages, String image)
 
 		// pull image
 
-		var digest = getDocker().getImage(image).map(ContainerImage::getDigest).orElse(null);
+		var digest = getDocker().getImage(image, getDefaultTaskTimeout()).map(ContainerImage::getDigest).orElse(null);
 		if (dockerPullAlways || digest == null) {
 			if (digest != null) {
 				log.debug("Image {} found in docker, pull always ...", image);
@@ -205,7 +222,7 @@ private boolean docker(Map<String, Map<String, ?>> existingImages, String image)
 				log.error("Failed to pull docker image {}", image, e);
 				return false;
 			}
-			digest = getDocker().getImage(image).map(ContainerImage::getDigest).orElse(null);
+			digest = getDocker().getImage(image, getDefaultTaskTimeout()).map(ContainerImage::getDigest).orElse(null);
 		} else {
 			log.debug("Image {} found in docker", image);
 		}
@@ -230,10 +247,10 @@ private boolean docker(Map<String, Map<String, ?>> existingImages, String image)
 		var source = Path.of(System.getProperty("java.io.tmpdir")).resolve(filename);
 		var destination = "/tmp/" + filename;
 		try {
-			getDocker().saveImage(image, source);
-			getDocker().copyToContainer(source, destination);
+			getDocker().saveImage(image, source, saveImageTimeout);
+			getDocker().copyToContainer(source, destination, copyToContainerTimeout);
 			getDocker().exec(pullTimeout, "ctr", "image", "import", destination.toString());
-			getDocker().exec("ctr", "image", "label", normalizedImage, label + "=" + digest);
+			getDocker().exec(getDefaultTaskTimeout(), "ctr", "image", "label", normalizedImage, label + "=" + digest);
 		} catch (MojoExecutionException e) {
 			log.error("Failed to import tar {}", source, e);
 			return false;
@@ -249,7 +266,7 @@ private boolean docker(Map<String, Map<String, ?>> existingImages, String image)
 	 * @return Image name with labels.
 	 */
 	private Map<String, Map<String, ?>> getCtrImages() throws MojoExecutionException {
-		return getDocker().exec("ctr", "image", "list").stream().map(String::strip)
+		return getDocker().exec(getDefaultTaskTimeout(), "ctr", "image", "list").stream().map(String::strip)
 				.filter(row -> !row.startsWith("REF") && !row.startsWith("sha256:"))
 				.map(row -> row.split("(\\s)+"))
 				.filter(parts -> {
@@ -288,6 +305,14 @@ public void setPullTimeout(int pullTimeout) {
 		this.pullTimeout = Duration.ofSeconds(pullTimeout);
 	}
 
+	public void setCopyToContainerTimeout(int copyToContainerTimeout) {
+		this.copyToContainerTimeout = Duration.ofSeconds(copyToContainerTimeout);
+	}
+
+	public void setSaveImageTimeout(int saveImageTimeout) {
+		this.saveImageTimeout = Duration.ofSeconds(saveImageTimeout);
+	}
+
 	public void setSkipImage(boolean skipImage) {
 		this.skipImage = skipImage;
 	}

diff --git a/src/main/java/io/kokuwa/maven/k3s/mojo/K3sMojo.java b/src/main/java/io/kokuwa/maven/k3s/mojo/K3sMojo.java
@@ -2,6 +2,7 @@
 
 import java.io.File;
 import java.nio.file.Path;
+import java.time.Duration;
 
 import io.kokuwa.maven.k3s.util.Docker;
 import io.kokuwa.maven.k3s.util.Marker;
@@ -27,6 +28,14 @@ public abstract class K3sMojo extends AbstractMojo {
 	@Parameter(property = "k3s.debug", defaultValue = "false")
 	private boolean debug;
 
+	/**
+	 * Default timout for plugin tasks in seconds.
+	 *
+	 * @since 1.5.0
+	 */
+	@Parameter(property = "k3s.defaultTaskTimeout", defaultValue = "30")
+	private Duration defaultTaskTimeout;
+
 	/**
 	 * Skip plugin.
 	 *
@@ -60,6 +69,10 @@ public Marker getMarker() {
 		return marker;
 	}
 
+	public Duration getDefaultTaskTimeout() {
+		return defaultTaskTimeout;
+	}
+
 	@Deprecated
 	@Override
 	public Log getLog() {
@@ -85,6 +98,10 @@ public void setDebug(boolean debug) {
 		this.debug = debug;
 	}
 
+	public void setDefaultTaskTimeout(int defaultTaskTimeout) {
+		this.defaultTaskTimeout = Duration.ofSeconds(defaultTaskTimeout);
+	}
+
 	public void setSkip(boolean skip) {
 		this.skip = skip;
 	}

diff --git a/src/main/java/io/kokuwa/maven/k3s/mojo/RemoveMojo.java b/src/main/java/io/kokuwa/maven/k3s/mojo/RemoveMojo.java
@@ -36,9 +36,9 @@ public void execute() throws MojoExecutionException {
 			return;
 		}
 
-		getDocker().removeContainer();
+		getDocker().removeContainer(getDefaultTaskTimeout());
 		if (includeCache) {
-			getDocker().removeVolume();
+			getDocker().removeVolume(getDefaultTaskTimeout());
 			log.info("Deleted cache volume.");
 		}
 	}

diff --git a/src/main/java/io/kokuwa/maven/k3s/mojo/RestartMojo.java b/src/main/java/io/kokuwa/maven/k3s/mojo/RestartMojo.java
@@ -81,7 +81,7 @@ public void execute() throws MojoExecutionException {
 
 		// verify container
 
-		if (getDocker().getContainer().isEmpty()) {
+		if (getDocker().getContainer(getDefaultTaskTimeout()).isEmpty()) {
 			throw new MojoExecutionException("No k3s container found");
 		}
 
@@ -118,14 +118,17 @@ private Callable<Boolean> restart(String resoure) {
 
 		return () -> {
 			try {
-				getDocker().exec("kubectl", "rollout", "restart", kind, name, "--namespace=" + namespace);
+				getDocker().exec(timeout, "kubectl", "rollout", "restart", kind, name, "--namespace=" + namespace);
 				log.info("{} {}/{} restarted", kind, namespace, name);
-				getDocker().exec("kubectl", "rollout", "status", kind, name, "--namespace=" + namespace,
+				// Docker exec timeout should be greater than "inner" K3s timeout
+				getDocker().exec(timeout.plusSeconds(10), "kubectl", "rollout", "status", kind, name,
+						"--namespace=" + namespace,
 						"--timeout=" + timeout.getSeconds() + "s");
 				log.info("{} {}/{} restart finished", kind, namespace, name);
 				return true;
 			} catch (MojoExecutionException e) {
-				getDocker().exec("kubectl", "get", "--output=yaml", "--namespace=" + namespace, kind, name);
+				getDocker().exec(getDefaultTaskTimeout(), "kubectl", "get", "--output=yaml", "--namespace=" + namespace,
+						kind, name);
 				return false;
 			}
 		};

diff --git a/src/main/java/io/kokuwa/maven/k3s/mojo/RunMojo.java b/src/main/java/io/kokuwa/maven/k3s/mojo/RunMojo.java
@@ -243,14 +243,14 @@ public void execute() throws MojoExecutionException {
 
 		var create = true;
 		var restart = false;
-		var container = getDocker().getContainer().orElse(null);
+		var container = getDocker().getContainer(getDefaultTaskTimeout()).orElse(null);
 		if (container != null) {
 			if (failIfExists) {
 				throw new MojoExecutionException("Container with id '" + container.id
 						+ "' found. Please remove that container or set 'k3s.failIfExists' to false.");
 			} else if (replaceIfExists) {
 				log.info("Container with id '{}' found, replacing", container.id);
-				getDocker().removeContainer();
+				getDocker().removeContainer(getDefaultTaskTimeout());
 			} else if (!container.isRunning()) {
 				log.warn("Container with id '{}' found in stopped state, restart container", container.id);
 				create = false;
@@ -268,20 +268,21 @@ public void execute() throws MojoExecutionException {
 			if (create) {
 				createAndStartK3sContainer();
 			} else if (restart) {
-				getDocker().startContainer();
+				getDocker().startContainer(getDefaultTaskTimeout());
 			}
 
 			// wait for k3s api to be ready
 
 			var await = Await.await(log, "k3s api available").timeout(nodeTimeout);
-			getDocker().waitForLog(await, output -> output.stream().anyMatch(l -> l.contains("k3s is up and running")));
+			getDocker().waitForLog(await, output -> output.stream().anyMatch(l -> l.contains("k3s is up and running")),
+					getDefaultTaskTimeout());
 
 			// write file that k3s started
 
 			getMarker().writeStarted();
 		}
 
-		getDocker().copyFromContainer("/etc/rancher/k3s/k3s.yaml", kubeconfig);
+		getDocker().copyFromContainer("/etc/rancher/k3s/k3s.yaml", kubeconfig, getDefaultTaskTimeout());
 		log.info("k3s ready: KUBECONFIG={} kubectl get all --all-namespaces", kubeconfig);
 	}
 
@@ -339,8 +340,8 @@ private void createAndStartK3sContainer() throws MojoExecutionException {
 		}
 		var ports = new ArrayList<>(portBindings);
 		ports.add(portKubeApi + ":" + portKubeApi);
-		getDocker().createContainer(image, ports, command, registries);
-		getDocker().createVolume();
+		getDocker().createContainer(image, ports, command, registries, getDefaultTaskTimeout());
+		getDocker().createVolume(getDefaultTaskTimeout());
 	}
 
 	// setter