Delete error'd server

ENH: The VMs which are created but are in error state will be deleted.

actions/hv.create.test.server.yaml

@@ -31,4 +31,9 @@ parameters:
     description: Test all avaliable flavors for the hypervisor, otherwise test random flavor
     default: false
     required: true
+  delete_on_failure:
+    type: boolean
+    description: Delete servers that errored during creation
+    default: true
+    required: true
 runner_type: python-script

actions/hv.post.reboot.yaml

@@ -6,7 +6,7 @@ name: hv.post.reboot
 parameters:
   hypervisor_hostname:
     type: string
-    description: Hostname of hypervisor to run action against 
+    description: Hostname of hypervisor to run action against
     required: true
   lib_entry_point:
     type: string

lib/openstack_api/openstack_server.py

@@ -3,6 +3,8 @@
 from openstack.connection import Connection
 from openstack.compute.v2.image import Image
 from openstack.compute.v2.server import Server
+from openstack.exceptions import ResourceTimeout
+from openstack.exceptions import ResourceFailure
 
 
 def snapshot_and_migrate_server(
@@ -67,6 +69,7 @@ def build_server(
     image_name: str,
     network_name: str,
     hypervisor_hostname: Optional[str] = None,
+    delete_on_failure: Optional[bool] = False,
 ) -> Server:
     """
     Builds a server, with option to specify a hypervisor
@@ -100,10 +103,14 @@ def build_server(
             "openstack_api_version": "2.74",
         }
     )
-
-    conn.compute.wait_for_server(
-        server, status="ACTIVE", failures=None, interval=5, wait=300
-    )
+    try:
+        conn.compute.wait_for_server(
+            server, status="ACTIVE", failures=None, interval=5, wait=300
+        )
+    except (ResourceTimeout, ResourceFailure) as e:
+        if delete_on_failure:
+            conn.compute.delete_server(server, force=True)
+        raise e
     return server
 
 

lib/workflows/hv_create_test_server.py

@@ -6,7 +6,10 @@
 
 
 def create_test_server(
-    conn: Connection, hypervisor_name: str, test_all_flavors: bool
+    conn: Connection,
+    hypervisor_name: str,
+    test_all_flavors: bool,
+    delete_on_failure: bool,
 ) -> None:
     """
     Create a test server on a hypervisor, option to test all possible flavors avaliable to the hypervisor
@@ -32,5 +35,6 @@ def create_test_server(
             "ubuntu-focal-20.04-nogui",
             "Internal",
             hypervisor_name,
+            delete_on_failure,
         )
         delete_server(conn, server.id)

tests/lib/openstack_api/test_openstack_server.py

@@ -176,6 +176,7 @@ def test_build_server():
         "test-image",
         "test-network",
         "hvxyz.nubes.rl.ac.uk",
+        False,
     )
 
     mock_conn.compute.find_flavor.assert_called_once_with("test-flavor")

tests/lib/workflows/test_hv_create_test_server.py

@@ -23,7 +23,7 @@ def test_create_single_test_server(
     mock_server = MagicMock()
     mock_build_server.return_value = mock_server
 
-    create_test_server(mock_conn, "hvxyz.nubes.rl.ac.uk", False)
+    create_test_server(mock_conn, "hvxyz.nubes.rl.ac.uk", False, False)
 
     mock_get_available_flavors.assert_called_once_with(
         mock_conn, "hvxyz.nubes.rl.ac.uk"
@@ -35,6 +35,7 @@ def test_create_single_test_server(
         "ubuntu-focal-20.04-nogui",
         "Internal",
         "hvxyz.nubes.rl.ac.uk",
+        False,
     )
     mock_delete_server.assert_called_once_with(mock_conn, mock_server.id)
 
@@ -55,7 +56,7 @@ def test_create_test_server_all_flavors(
     mock_server = MagicMock()
     mock_build_server.return_value = mock_server
 
-    create_test_server(mock_conn, "hvxyz.nubes.rl.ac.uk", True)
+    create_test_server(mock_conn, "hvxyz.nubes.rl.ac.uk", True, True)
 
     mock_get_available_flavors.assert_called_once_with(
         mock_conn, "hvxyz.nubes.rl.ac.uk"
@@ -69,5 +70,6 @@ def test_create_test_server_all_flavors(
             "ubuntu-focal-20.04-nogui",
             "Internal",
             "hvxyz.nubes.rl.ac.uk",
+            True,
         )
         mock_delete_server.assert_any_call(mock_conn, mock_server.id)