Suspend and delete job, rather than kill pod

OctopusDeploy · Dec 15, 2023 · e111b91 · e111b91
1 parent daee3a3
commit e111b91
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 6 deletions.
diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesJobService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesJobService.cs
@@ -16,6 +16,7 @@ public interface IKubernetesJobService
         Task CreateJob(V1Job job, CancellationToken cancellationToken);
         Task Delete(ScriptTicket scriptTicket, CancellationToken cancellationToken);
         Task Watch(ScriptTicket scriptTicket, Func<V1Job, bool> onChange, Action<Exception> onError, CancellationToken cancellationToken);
+        Task SuspendJob(ScriptTicket scriptTicket, CancellationToken cancellationToken);
     }
 
     public class KubernetesJobService : KubernetesService, IKubernetesJobService
@@ -40,6 +41,18 @@ public KubernetesJobService(IKubernetesClientConfigProvider configProvider)
             }
         }
 
+        public async Task SuspendJob(ScriptTicket scriptTicket, CancellationToken cancellationToken)
+        {
+            var jobName = BuildJobName(scriptTicket);
+
+            var patchSpec = new V1JobSpec
+            {
+                Suspend = true
+            };
+            var patchYaml = KubernetesYaml.Serialize(patchSpec);
+            await Client.PatchNamespacedJobAsync(new V1Patch(patchYaml, V1Patch.PatchType.MergePatch), jobName, KubernetesConfig.Namespace, cancellationToken: cancellationToken);
+        }
+
         public async Task Watch(ScriptTicket scriptTicket, Func<V1Job, bool> onChange, Action<Exception> onError, CancellationToken cancellationToken)
         {
             var jobName = BuildJobName(scriptTicket);
@@ -65,7 +78,7 @@ public async Task Watch(ScriptTicket scriptTicket, Func<V1Job, bool> onChange, A
             }
         }
 
-        public string BuildJobName(ScriptTicket scriptTicket) => $"octopus-{scriptTicket.TaskId}".ToLowerInvariant();
+        public string BuildJobName(ScriptTicket scriptTicket) => $"octopus-job-{scriptTicket.TaskId}".ToLowerInvariant();
 
         public async Task CreateJob(V1Job job, CancellationToken cancellationToken) => await Client.CreateNamespacedJobAsync(job, KubernetesConfig.Namespace, cancellationToken: cancellationToken);
 

diff --git a/source/Octopus.Tentacle/Kubernetes/Scripts/RunningKubernetesJob.cs b/source/Octopus.Tentacle/Kubernetes/Scripts/RunningKubernetesJob.cs
@@ -87,15 +87,16 @@ public async Task Execute()
                 //we use a using to make sure this callback is deregistered
                 using var cancellationTokenRegistration = scriptCancellationToken.Register(() =>
                 {
-                    writer.WriteVerbose($"Cancelling Kubernetes Job '{jobName}'.");
-
-                    //we spawn the pod cancellation on a background thread (as this callback runs synchronously)
+                    //we spawn the job cancellation on a background thread (as this callback runs synchronously)
                     Task.Run(async () =>
                     {
                         try
                         {
-                            //to cancel the job we just kill the underlying pod(s)
-                            await podService.DeletePodsForJob(jobName, CancellationToken.None);
+                            writer.WriteVerbose($"Cancelling Kubernetes Job '{jobName}'.");
+                            //first we suspend the job, which terminates the underlying pods
+                            await jobService.SuspendJob(scriptTicket, CancellationToken.None);
+                            //then we delete the job (because we no longer need it)
+                            await jobService.Delete(scriptTicket, CancellationToken.None);
                             writer.WriteVerbose($"Cancelled Kubernetes Job '{jobName}'.");
                         }
                         catch (Exception e)