Delete error'd server

actions/hv.create.test.server.yaml

@@ -31,4 +31,9 @@ parameters:
     description: Test all avaliable flavors for the hypervisor, otherwise test random flavor
     default: false
     required: true
+  delete_on_failure:
+    type: boolean
+    description: Delete servers that errored during creation
+    default: true
+    required: true
 runner_type: python-script

actions/hv.post.reboot.yaml

@@ -6,7 +6,7 @@ name: hv.post.reboot
 parameters:
   hypervisor_hostname:
     type: string
-    description: Hostname of hypervisor to run action against 
+    description: Hostname of hypervisor to run action against
     required: true
   lib_entry_point:
     type: string

lib/openstack_api/openstack_server.py

@@ -3,6 +3,8 @@
 from openstack.connection import Connection
 from openstack.compute.v2.image import Image
 from openstack.compute.v2.server import Server
+from openstack.exceptions import ResourceTimeout
+from openstack.exceptions import ResourceFailure
 
 
 def snapshot_and_migrate_server(
@@ -67,6 +69,7 @@ def build_server(
     image_name: str,
     network_name: str,
     hypervisor_hostname: Optional[str] = None,
+    delete_on_failure: Optional[bool] = False,
 ) -> Server:
     """
     Builds a server, with option to specify a hypervisor
@@ -100,10 +103,14 @@ def build_server(
             "openstack_api_version": "2.74",
         }
     )
-
-    conn.compute.wait_for_server(
-        server, status="ACTIVE", failures=None, interval=5, wait=300
-    )
+    try:
+        conn.compute.wait_for_server(
+            server, status="ACTIVE", failures=None, interval=5, wait=300
+        )
+    except (ResourceTimeout, ResourceFailure) as e:
+        if delete_on_failure:
+            conn.compute.delete_server(server, force=True)
+        raise e
     return server
 
 

lib/workflows/hv_create_test_server.py

@@ -6,7 +6,10 @@
 
 
 def create_test_server(
-    conn: Connection, hypervisor_name: str, test_all_flavors: bool
+    conn: Connection,
+    hypervisor_name: str,
+    test_all_flavors: bool,
+    delete_on_failure: bool,
 ) -> None:
     """
     Create a test server on a hypervisor, option to test all possible flavors avaliable to the hypervisor
@@ -32,5 +35,6 @@ def create_test_server(
             "ubuntu-focal-20.04-nogui",
             "Internal",
             hypervisor_name,
+            delete_on_failure,
         )
         delete_server(conn, server.id)

tests/lib/openstack_api/test_openstack_server.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 from unittest.mock import MagicMock, patch
+from openstack.exceptions import ResourceFailure
 import pytest
 from openstack_api.openstack_server import (
     build_server,
@@ -176,6 +177,7 @@ def test_build_server():
         "test-image",
         "test-network",
         "hvxyz.nubes.rl.ac.uk",
+        False,
     )
 
     mock_conn.compute.find_flavor.assert_called_once_with("test-flavor")
@@ -199,6 +201,98 @@ def test_build_server():
     assert res == mock_server
 
 
+def test_build_server_delete_on_failure():
+    """
+    Test build server with delete on failure set to True
+    """
+    mock_conn = MagicMock()
+
+    mock_conn.compute.find_flavor.return_value = MagicMock()
+    mock_conn.image.find_image.return_value = MagicMock()
+    mock_conn.network.find_network.return_value = MagicMock()
+
+    mock_server = MagicMock()
+    mock_conn.compute.create_server.return_value = mock_server
+
+    mock_conn.compute.wait_for_server.side_effect = ResourceFailure("Failure")
+
+    with pytest.raises(ResourceFailure):
+        build_server(
+            mock_conn,
+            "test-server",
+            "test-flavor",
+            "test-image",
+            "test-network",
+            "hvxyz.nubes.rl.ac.uk",
+            delete_on_failure=True,
+        )
+
+    mock_conn.compute.find_flavor.assert_called_once_with("test-flavor")
+    mock_conn.image.find_image.assert_called_once_with("test-image")
+    mock_conn.network.find_network.assert_called_once_with("test-network")
+
+    mock_conn.compute.create_server.assert_called_once_with(
+        **{
+            "name": "test-server",
+            "imageRef": mock_conn.image.find_image.return_value.id,
+            "flavorRef": mock_conn.compute.find_flavor.return_value.id,
+            "networks": [{"uuid": mock_conn.network.find_network.return_value.id}],
+            "host": "hvxyz.nubes.rl.ac.uk",
+            "openstack_api_version": "2.74",
+        }
+    )
+    mock_conn.compute.wait_for_server.assert_called_once_with(
+        mock_server, status="ACTIVE", failures=None, interval=5, wait=300
+    )
+    mock_conn.compute.delete_server.assert_called_once_with(mock_server, force=True)
+
+
+def test_build_server_delete_on_failure_false():
+    """
+    Test build server with delete on failure set to False
+    """
+    mock_conn = MagicMock()
+
+    mock_conn.compute.find_flavor.return_value = MagicMock()
+    mock_conn.image.find_image.return_value = MagicMock()
+    mock_conn.network.find_network.return_value = MagicMock()
+
+    mock_server = MagicMock()
+    mock_conn.compute.create_server.return_value = mock_server
+
+    mock_conn.compute.wait_for_server.side_effect = ResourceFailure("Failure")
+
+    with pytest.raises(ResourceFailure):
+        build_server(
+            mock_conn,
+            "test-server",
+            "test-flavor",
+            "test-image",
+            "test-network",
+            "hvxyz.nubes.rl.ac.uk",
+            delete_on_failure=False,
+        )
+
+    mock_conn.compute.find_flavor.assert_called_once_with("test-flavor")
+    mock_conn.image.find_image.assert_called_once_with("test-image")
+    mock_conn.network.find_network.assert_called_once_with("test-network")
+
+    mock_conn.compute.create_server.assert_called_once_with(
+        **{
+            "name": "test-server",
+            "imageRef": mock_conn.image.find_image.return_value.id,
+            "flavorRef": mock_conn.compute.find_flavor.return_value.id,
+            "networks": [{"uuid": mock_conn.network.find_network.return_value.id}],
+            "host": "hvxyz.nubes.rl.ac.uk",
+            "openstack_api_version": "2.74",
+        }
+    )
+    mock_conn.compute.wait_for_server.assert_called_once_with(
+        mock_server, status="ACTIVE", failures=None, interval=5, wait=300
+    )
+    mock_conn.compute.delete_server.assert_not_called()
+
+
 def test_delete_server():
     """
     Test deleting server

tests/lib/workflows/test_hv_create_test_server.py

@@ -1,14 +1,20 @@
 from unittest.mock import MagicMock, patch
+import pytest
 
 from workflows.hv_create_test_server import create_test_server
 
 
+@pytest.mark.parametrize("delete_error_server", [True, False])
 @patch("workflows.hv_create_test_server.delete_server")
 @patch("workflows.hv_create_test_server.build_server")
 @patch("workflows.hv_create_test_server.random")
 @patch("workflows.hv_create_test_server.get_available_flavors")
 def test_create_single_test_server(
-    mock_get_available_flavors, mock_random, mock_build_server, mock_delete_server
+    mock_get_available_flavors,
+    mock_random,
+    mock_build_server,
+    mock_delete_server,
+    delete_error_server,
 ):
     """
     Test build single server on a hypervisor
@@ -23,7 +29,7 @@ def test_create_single_test_server(
     mock_server = MagicMock()
     mock_build_server.return_value = mock_server
 
-    create_test_server(mock_conn, "hvxyz.nubes.rl.ac.uk", False)
+    create_test_server(mock_conn, "hvxyz.nubes.rl.ac.uk", False, delete_error_server)
 
     mock_get_available_flavors.assert_called_once_with(
         mock_conn, "hvxyz.nubes.rl.ac.uk"
@@ -35,6 +41,7 @@ def test_create_single_test_server(
         "ubuntu-focal-20.04-nogui",
         "Internal",
         "hvxyz.nubes.rl.ac.uk",
+        delete_error_server,
     )
     mock_delete_server.assert_called_once_with(mock_conn, mock_server.id)
 
@@ -55,7 +62,7 @@ def test_create_test_server_all_flavors(
     mock_server = MagicMock()
     mock_build_server.return_value = mock_server
 
-    create_test_server(mock_conn, "hvxyz.nubes.rl.ac.uk", True)
+    create_test_server(mock_conn, "hvxyz.nubes.rl.ac.uk", True, True)
 
     mock_get_available_flavors.assert_called_once_with(
         mock_conn, "hvxyz.nubes.rl.ac.uk"
@@ -69,5 +76,6 @@ def test_create_test_server_all_flavors(
             "ubuntu-focal-20.04-nogui",
             "Internal",
             "hvxyz.nubes.rl.ac.uk",
+            True,
         )
         mock_delete_server.assert_any_call(mock_conn, mock_server.id)